Add support for Yarn workspaces

bin/node2nix.js

@@ -33,7 +33,8 @@ var switches = [
     ['--no-bypass-cache', 'Specifies that package builds do not need to bypass the content addressable cache (required for NPM 5.x)'],
     ['--no-copy-node-env', 'Do not create a copy of the Nix expression that builds NPM packages'],
     ['--use-fetchgit-private', 'Use fetchGitPrivate instead of fetchgit in the generated Nix expressions'],
-    ['--strip-optional-dependencies', 'Strips the optional dependencies from the regular dependencies in the NPM registry']
+    ['--strip-optional-dependencies', 'Strips the optional dependencies from the regular dependencies in the NPM registry'],
+    ['--yarn-workspace FILE', 'Use a Yarn workspace package.json to find project interdependencies'],
 ];
 
 var parser = new optparse.OptionParser(switches);
@@ -59,6 +60,7 @@ var noCopyNodeEnv = false;
 var bypassCache = true;
 var useFetchGitPrivate = false;
 var stripOptionalDependencies = false;
+var yarnWorkspaceJSON;
 var executable;
 
 /* Define process rules for option parameters */
@@ -185,6 +187,10 @@ parser.on('strip-optional-dependencies', function(arg, value) {
     stripOptionalDependencies = true;
 });
 
+parser.on('yarn-workspace', function(arg, value) {
+    yarnWorkspaceJSON = value;
+});
+
 /* Define process rules for non-option parameters */
 
 parser.on(1, function(opt) {
@@ -247,7 +253,7 @@ if(version) {
 }
 
 /* Perform the NPM to Nix conversion */
-node2nix.npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, registryAuthToken, noCopyNodeEnv, bypassCache, useFetchGitPrivate, stripOptionalDependencies, function(err) {
+node2nix.npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, registryAuthToken, noCopyNodeEnv, bypassCache, useFetchGitPrivate, stripOptionalDependencies, yarnWorkspaceJSON, function(err) {
     if(err) {
         process.stderr.write(err + "\n");
         process.exit(1);

lib/Package.js

@@ -1,6 +1,5 @@
 var path = require('path');
 var slasp = require('slasp');
-var semver = require('semver');
 var nijs = require('nijs');
 var Source = require('./sources/Source.js').Source;
 var inherit = require('nijs/lib/ast/util/inherit.js').inherit;
@@ -54,21 +53,14 @@ inherit(nijs.NixASTNode, Package);
  *     or null if no such package exists
  */
 Package.prototype.findMatchingProvidedDependencyByParent = function(name, versionSpec) {
-    if(this.parent === null) { // If there is no parent, then we can also not provide a dependency
-        return null;
+    var dependency = this.providedDependencies[name];
+    if(dependency !== undefined && dependency.source.versionSatisfies(versionSpec)) { // If we found a dependency with the same name, see if the version fits
+        return dependency
     } else {
-        var dependency = this.parent.providedDependencies[name];
-
-        if(dependency === undefined) {
-            return this.parent.findMatchingProvidedDependencyByParent(name, versionSpec); // If the parent does not provide the dependency, try the parent's parent
-        } else if(dependency === null) {
-             return null; // If we have encountered a bundled dependency with the same name, consider it a conflict (is not a perfect resolution, but does not result in an error)
+        if(this.parent == null) { // If there is no parent, then we can also not provide a dependency
+            return null;
         } else {
-            if(semver.satisfies(dependency.source.config.version, versionSpec, true)) { // If we found a dependency with the same name, see if the version fits
-                return dependency;
-            } else {
-                return null; // If there is a version mismatch, then a conflicting version has been encountered
-            }
+            return this.parent.findMatchingProvidedDependencyByParent(name, versionSpec); // If the parent does not provide the dependency, try the parent
         }
     }
 };

lib/expressions/OutputExpression.js

@@ -43,7 +43,7 @@ OutputExpression.prototype.toNixAST = function() {
         },
         body: new nijs.NixLet({
             value: {
-                sources: this.sourcesCache
+                sources: this.sourcesCache.toNixAST()
             }
         })
     });

lib/expressions/PackageExpression.js

@@ -86,6 +86,15 @@ PackageExpression.prototype.toNixAST = function() {
         })
     };
 
+    // Workspace dependencies must be provided
+    var sourcesAst = ast.body.value.sources;
+    for (var identifier in sourcesAst) {
+        var sourceAst = sourcesAst[identifier];
+        if (sourceAst.src instanceof nijs.NixExpression) {
+            ast.argSpec[sourceAst.name] = undefined;
+        }
+    }
+
     return ast;
 };
 

lib/node2nix.js

@@ -6,11 +6,13 @@ var fs = require('fs');
 var path = require('path');
 var slasp = require('slasp');
 var nijs = require('nijs');
+var glob = require('glob');
 
 var CollectionExpression = require('./expressions/CollectionExpression.js').CollectionExpression;
 var PackageExpression = require('./expressions/PackageExpression.js').PackageExpression;
 var CompositionExpression = require('./expressions/CompositionExpression.js').CompositionExpression;
 var DeploymentConfig = require('./DeploymentConfig.js').DeploymentConfig;
+var Package = require('./Package.js').Package;
 
 function copyNodeEnvExpr(nodeEnvNix, callback) {
     /* Compose a read stream that reads the build expression */
@@ -42,7 +44,7 @@ function copyNodeEnvExpr(nodeEnvNix, callback) {
  */
 exports.copyNodeEnvExpr = copyNodeEnvExpr;
 
-function npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, registryAuthToken, noCopyNodeEnv, bypassCache, useFetchGitPrivate, stripOptionalDependencies, callback) {
+function npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, registryAuthToken, noCopyNodeEnv, bypassCache, useFetchGitPrivate, stripOptionalDependencies, yarnWorkspaceJSON, callback) {
     var obj = JSON.parse(fs.readFileSync(inputJSON));
     var version = JSON.parse(fs.readFileSync(path.join(__dirname, "..", "package.json"))).version;
     var disclaimer = "# This file has been generated by node2nix " + version + ". Do not edit!\n\n";
@@ -80,13 +82,77 @@ function npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, su
                     // Display a warning if we expect a lock file to be used, but the user does not specify it
                     displayLockWarning = bypassCache && !lockJSON && fs.existsSync(path.join(path.dirname(inputJSON), path.basename(inputJSON, ".json")) + "-lock.json");
                 }
-
-                expr.resolveDependencies(callback);
+                callback();
             } else {
                 callback("The provided JSON file must consist of an object or an array");
             }
         },
 
+        /* Add Yarn workspace packages as virtual dependencies at the top level. */
+        function(callback) {
+            if(yarnWorkspaceJSON !== undefined && expr.package /* not for CollectionExpression */) {
+                var yarnWorkspace = JSON.parse(fs.readFileSync(yarnWorkspaceJSON));
+
+                // Workspace references are globs, resolve them first.
+                var subPackagesJSON = new Set();
+                slasp.fromEach(function(callback) {
+                    callback(null, yarnWorkspace.workspaces);
+                }, function(idx, callback) {
+                    glob(yarnWorkspace.workspaces[idx] + '/package.json', {
+                        cwd: path.dirname(yarnWorkspaceJSON),
+                        absolute: true,
+                        nosort: true,
+                        nounique: true,
+                    }, function(err, matches) {
+                        if(err) {
+                            callback(err);
+                        } else {
+                            matches.forEach(function(match) {
+                                var matchRel = path.relative(baseDir, match);
+                                if (matchRel !== 'package.json') { // ignore self
+                                    subPackagesJSON.add(matchRel);
+                                }
+                            });
+                            callback();
+                        }
+                    });
+                }, function(err) {
+                    if (err) {
+                        callback(err);
+                    } else {
+                        // Bundle workspace packages and resolve transitive dependencies
+                        const subPackages = {};
+                        subPackagesJSON.forEach(function(subPackageJSON) {
+                            var config = JSON.parse(fs.readFileSync(subPackageJSON));
+                            var subPackagePath = subPackageJSON.slice(0, -13); // strip `package.json`
+                            subPackages[config.name] = new Package(deploymentConfig, lock, expr.package, config.name, "workspace:" + subPackagePath, baseDir, true, expr.sourcesCache);
+                        });
+                        slasp.fromEach(function(callback) {
+                            callback(null, subPackages);
+                        }, function(subPackageName, callback) {
+                            var subPackage = subPackages[subPackageName];
+                            slasp.sequence([
+                                function(callback) {
+                                    subPackage.source.fetch(callback);
+                                },
+                                function(callback) {
+                                    expr.sourcesCache.addSource(subPackage.source);
+                                    expr.package.providedDependencies[subPackageName] = subPackage;
+                                    subPackage.resolveDependenciesAndSources(callback);
+                                },
+                            ], callback);
+                        }, callback);
+                    }
+                });
+            } else {
+                callback();
+            }
+        },
+
+        function(callback) {
+            expr.resolveDependencies(callback);
+        },
+
         /* Write the output Nix expression to the specified output file */
         function(callback) {
             fs.writeFile(outputNix, disclaimer + nijs.jsToNix(expr, true), callback);

lib/sources/Source.js

@@ -43,6 +43,7 @@ Source.constructSource = function(registryURL, registryAuthToken, baseDir, outpu
     var GitSource = require('./GitSource.js').GitSource;
     var HTTPSource = require('./HTTPSource.js').HTTPSource;
     var LocalSource = require('./LocalSource.js').LocalSource;
+    var WorkspaceSource = require('./WorkspaceSource.js').WorkspaceSource;
     var NPMRegistrySource = require('./NPMRegistrySource.js').NPMRegistrySource;
 
     var parsedVersionSpec = semver.validRange(versionSpec, true);
@@ -66,6 +67,8 @@ Source.constructSource = function(registryURL, registryAuthToken, baseDir, outpu
         return new GitSource(baseDir, dependencyName, "git://github.com/"+versionSpec);
     } else if(parsedUrl.protocol == "file:") { // If the version is a file URL, simply compose a Nix path
         return new LocalSource(baseDir, dependencyName, outputDir, parsedUrl.path);
+    } else if(parsedUrl.protocol == "workspace:") { // If the version is a workspace URL, the package is provided as a Nix expression
+        return new WorkspaceSource(baseDir, dependencyName, outputDir, versionSpec.slice(10));
     } else if(versionSpec.substr(0, 3) == "../" || versionSpec.substr(0, 2) == "~/" || versionSpec.substr(0, 2) == "./" || versionSpec.substr(0, 1) == "/") { // If the version is a path, simply compose a Nix path
         return new LocalSource(baseDir, dependencyName, outputDir, versionSpec);
     } else { // In all other cases, just try the registry. Sometimes invalid semver ranges are encountered or a tag has been provided (e.g. 'latest', 'unstable')
@@ -85,6 +88,16 @@ Source.prototype.fetch = function(callback) {
     callback("fetch() is not implemented, please use a prototype that inherits from Source");
 };
 
+/**
+ * Whether the version of this source satisfies the given version specifier.
+ *
+ * @method
+ * @param {String} versionSpec Version specifier to commpare
+ */
+Source.prototype.versionSatisfies = function(versionSpec) {
+    return semver.satisfies(this.config.version, versionSpec, true);
+};
+
 /**
  * Takes a dependency object from a lock file and converts it into a source object.
  *

lib/sources/WorkspaceSource.js

@@ -0,0 +1,45 @@
+var nijs = require('nijs');
+var Source = require('./Source.js').Source;
+var LocalSource = require('./LocalSource.js').LocalSource;
+var inherit = require('nijs/lib/ast/util/inherit.js').inherit;
+
+/**
+ * Constructs a new WorkspaceSource instance.
+ *
+ * @class WorkspaceSource
+ * @extends LocalSource
+ * @classdesc Represents a dependency source that is part of the same project
+ *
+ * @constructor
+ * @param {String} baseDir Directory in which the referrer's package.json configuration resides
+ * @param {String} dependencyName Name of the dependency
+ * @param {String} outputDir Directory in which the nix expression will be written
+ * @param {String} versionSpec Version specifier of the Node.js package to fetch
+ */
+function WorkspaceSource(baseDir, dependencyName, outputDir, versionSpec) {
+    LocalSource.call(this, baseDir, dependencyName, outputDir, versionSpec);
+}
+
+/* WorkspaceSource inherits from LocalSource */
+inherit(LocalSource, WorkspaceSource);
+
+/**
+ * @see Source#versionSatisfies
+ */
+WorkspaceSource.prototype.versionSatisfies = function() {
+    return true; // Ignore version for packages part of the same project
+};
+
+/**
+ * @see NixASTNode#toNixAST
+ */
+WorkspaceSource.prototype.toNixAST = function() {
+    /* Override LocalSource method, but do call super Source method. */
+    var ast = Source.prototype.toNixAST.call(this);
+
+    ast.src = new nijs.NixExpression(ast.name);
+
+    return ast;
+};
+
+exports.WorkspaceSource = WorkspaceSource;

package.json

@@ -23,7 +23,8 @@
     "base64-js": "1.3.x",
     "slasp": "0.0.4",
     "nijs": "0.0.25",
-    "spdx-license-ids": "3.0.x"
+    "spdx-license-ids": "3.0.x",
+    "glob": "7.x.x"
   },
   "devDependencies": {
     "jsdoc": "*"