feat: support sync private package from custom registry

config/index.js

@@ -244,6 +244,20 @@ var config = {
   syncDownloadOptions: {
     // formatRedirectUrl: function (url, location)
   },
+
+  // all syncModel cannot sync scope pacakge, you can use this model to sync scope package from any resgitry
+  syncScope: false,
+  syncScopeInterval: '12h',
+  // scope package sync config
+  /**
+ * sync scope package from assign registry
+ * @param {Array<scope>} scopes
+ * @param {String} scope.scope scope name
+ * @param {String} scope.sourceCnpmWeb source cnpm registry web url for get scope all packages name
+ * @param {String} scope.sourceCnpmRegistry source cnpm registry url for sync packages
+ */
+  syncScopeConfig: [],
+
   handleSyncRegistry: 'http://127.0.0.1:7001',
 
   // default badge subject

controllers/sync.js

@@ -2,6 +2,7 @@
 
 var debug = require('debug')('cnpmjs.org:controllers:sync');
 var Log = require('../services/module_log');
+var npmService = require('../services/npm');
 var SyncModuleWorker = require('./sync_module_worker');
 var config = require('../config');
 
@@ -52,6 +53,55 @@ exports.sync = function* () {
   };
 };
 
+exports.scopeSync = function* () {
+  var scope = this.params.scope;
+
+  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
+    return item.scope === scope
+  })
+
+  if (!scopeConfig) {
+    this.status = 404;
+    this.body = {
+      error: 'no_scope',
+      reason: 'only has syncScopeConfig config can use this feature'
+    };
+    return;
+  }
+
+  var scopeCnpmWeb = scopeConfig.sourceCnpmWeb
+  var scopeCnpmRegistry = scopeConfig.sourceCnpmRegistry
+  var packages = yield* npmService.getScopePackagesShort(scope, scopeCnpmWeb)
+
+  debug('scopeSync %s with query: %j', scope, this.query);
+
+  var packageSyncWorkers = []
+
+  for (let i = 0; i < packages.length; i++) {
+    packageSyncWorkers.push(function* () {
+      var name = packages[i]
+      var logId = yield* SyncModuleWorker.sync(name, 'admin', {
+        type: 'package',
+        publish: true,
+        noDep: true,
+        syncUpstreamFirst: false,
+        syncPrivatePackage: { [scope]: scopeCnpmRegistry }
+      })
+      return { name: name, logId: logId }
+    })
+  }
+
+  var logIds = yield packageSyncWorkers
+
+  debug('scopeSync %s got log id %j', scope, logIds);
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    logIds: logIds
+  };
+};
+
 exports.getSyncLog = function* (next) {
   var logId = Number(this.params.id || this.params[1]);
   var offset = Number(this.query.offset) || 0;

controllers/sync_module_worker.js

@@ -49,6 +49,8 @@ function SyncModuleWorker(options) {
   this.names = options.name;
   this.startName = this.names[0];
 
+  this.syncPrivatePackage = options.syncPrivatePackage;
+
   this.username = options.username;
   this.concurrency = options.concurrency || 1;
   this._publish = options.publish === true; // _publish_on_cnpm
@@ -313,14 +315,21 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     return setImmediate(this.finish.bind(this));
   }
 
-  if (config.syncModel === 'none') {
+  const defineRegistry = this.getPrivatePackageDefineRegistry(name)
+
+  if (!defineRegistry && config.syncModel === 'none') {
     this.log('[c#%d] [%s] syncModel is none, ignore',
-      concurrencyId, name);
+     concurrencyId, name);
     return this.finish();
   }
 
-  // try to sync from official replicate when source npm registry is not cnpmjs.org
-  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
+  // try to sync from official replicate when no defineRegistry or source npm registry is not cnpmjs.org
+  let registry
+  if (defineRegistry) {
+    registry = defineRegistry
+  } else {
+    registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
+  }
 
   yield this.syncByName(concurrencyId, name, registry);
 };
@@ -520,8 +529,9 @@ SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry
 
   this.log('----------------- Syncing %s -------------------', name);
 
+  const isNeedSyncPrivatePackage = this.getPrivatePackageDefineRegistry(name)
   // ignore private scoped package
-  if (common.isPrivateScopedPackage(name)) {
+  if (!isNeedSyncPrivatePackage && common.isPrivateScopedPackage(name)) {
     this.log('[c#%d] [%s] ignore sync private scoped %j package',
       concurrencyId, name, config.scopes);
     yield this._doneOne(concurrencyId, name, true);
@@ -687,6 +697,21 @@ SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry
   return versions;
 };
 
+SyncModuleWorker.prototype.getPrivatePackageDefineRegistry = function (name) {
+  if (typeof name !== 'string') return false
+  return this.syncPrivatePackage && this.syncPrivatePackage[name.split('/')[0]]
+}
+
+SyncModuleWorker.prototype.isLocalModule = function (mods) {
+  var res = common.isLocalModule(mods)
+  if (!this.syncPrivatePackage) return res
+  if (!mods[0] || !mods[0].package || !mods[0].package.name) return res
+
+  if (this.getPrivatePackageDefineRegistry(mods[0].package.name)) return false
+
+  return res
+}
+
 function* _listStarUsers(modName) {
   var users = yield packageService.listStarUserNames(modName);
   var userMap = {};
@@ -727,7 +752,7 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
   var mods = yield packageService.listModulesByName(name);
   this.log('  [%s] start unpublished %d versions from local cnpm registry',
     name, mods.length);
-  if (common.isLocalModule(mods)) {
+  if (this.isLocalModule(mods)) {
     // publish on cnpm, dont sync this version package
     this.log('  [%s] publish on local cnpm registry, don\'t sync', name);
     return [];
@@ -795,7 +820,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var existsNpmMaintainers = result[3];
   var existsModuleAbbreviateds = result[4];
 
-  if (common.isLocalModule(moduleRows)) {
+  if (this.isLocalModule(moduleRows)) {
     // publish on cnpm, dont sync this version package
     that.log('  [%s] publish on local cnpm registry, don\'t sync', name);
     return [];
@@ -1907,6 +1932,7 @@ SyncModuleWorker.sync = function* (name, username, options) {
     publish: options.publish,
     syncUpstreamFirst: options.syncUpstreamFirst,
     syncFromBackupFile: options.syncFromBackupFile,
+    syncPrivatePackage: options.syncPrivatePackage
   });
   worker.start();
   return result.id;

controllers/web/show_scope_sync.js

@@ -0,0 +1,23 @@
+'use strict';
+var config = require('../../config');
+var npmService = require('../../services/npm');
+
+module.exports = function* showScopeSync () {
+  var scope = this.params.scope;
+  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
+    return item.scope === scope
+  })
+
+  if (!scopeConfig) {
+    return this.redirect('/');
+  }
+
+  var packages = yield* npmService.getScopePackagesShort(scope, scopeConfig.sourceCnpmWeb)
+
+  yield this.render('scope_sync', {
+    packages: packages,
+    scope: scopeConfig.scope,
+    sourceCnpmRegistry: scopeConfig.sourceCnpmRegistry,
+    title: 'Sync Scope Packages',
+  });
+};

dispatch.js

@@ -7,20 +7,21 @@ var cfork = require('cfork');
 var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
 var syncPath = path.join(__dirname, 'sync');
+var scopeSyncPath = path.join(__dirname, 'sync/sync_scope');
 
 console.log('Starting cnpmjs.org ...\ncluster: %s\nadmins: %j\nscopes: %j\nsourceNpmRegistry: %s\nsyncModel: %s',
   config.enableCluster, config.admins, config.scopes, config.sourceNpmRegistry, config.syncModel);
 
 if (config.enableCluster) {
   forkWorker();
-  if (config.syncModel !== 'none') {
-    forkSyncer();
-  }
+  config.syncModel !== 'none' && forkSyncer();
+  // scync assign pravate scope package
+  config.syncScope && forkScopeSyncer();
 } else {
   require(workerPath);
-  if (config.syncModel !== 'none') {
-    require(syncPath);
-  }
+  config.syncModel !== 'none' && require(syncPath);
+  // scync assign pravate scope package
+  config.syncScope && require(scopeSyncPath);
 }
 
 function forkWorker() {
@@ -52,3 +53,15 @@ function forkSyncer() {
     setTimeout(forkSyncer, 1000);
   });
 }
+
+function forkScopeSyncer() {
+  var syncer = childProcess.fork(scopeSyncPath);
+  syncer.on('exit', function (code, signal) {
+    var err = new Error(util.format('syncer %s died (code: %s, signal: %s, stdout: %s, stderr: %s)',
+      syncer.pid, code, signal, syncer.stdout, syncer.stderr));
+    err.name = 'SyncerWorkerDiedError';
+    console.error('[%s] [master:%s] syncer exit: %s: %s',
+      Date(), process.pid, err.name, err.message);
+    setTimeout(forkScopeSyncer, 1000);
+  });
+}

routes/web.js

@@ -5,6 +5,7 @@ var searchPackage = require('../controllers/web/package/search');
 var searchRange = require('../controllers/web/package/search_range');
 var listPrivates = require('../controllers/web/package/list_privates');
 var showSync = require('../controllers/web/show_sync');
+var showScopeSync = require('../controllers/web/show_scope_sync');
 var showUser = require('../controllers/web/user/show');
 var sync = require('../controllers/sync');
 var showTotal = require('../controllers/total');
@@ -35,6 +36,9 @@ function routes(app) {
   app.put(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)$/, sync.sync);
   app.put('/sync/:name', sync.sync);
 
+  app.get('/scopeSync/:scope', showScopeSync);
+  app.put('/scopeSync/:scope', sync.scopeSync);
+
   app.get(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)\/log\/(\d+)$/, sync.getSyncLog);
   app.get('/sync/:name/log/:id', sync.getSyncLog);
 

services/npm.js

@@ -198,3 +198,13 @@ exports.getPopular = function* (top, timeout) {
     return r[0];
   });
 };
+
+exports.getScopePackagesShort = function* (scope, registry) {
+  var response = yield* request('/browse/keyword/' + scope, {
+    timeout: 3000,
+    registry: registry,
+    dataType: 'text'
+  });
+  var res = response.data.match(/class="package-name">(\S*)<\/a>/g)
+  return res ? res.map(a => a.match(/class="package-name">(\S*)<\/a>/)[1]).filter(name => name.indexOf(`${scope}/`) === 0) : []
+};

sync/sync_scope.js

@@ -0,0 +1,89 @@
+'use strict';
+
+var thunkify = require('thunkify-wrap');
+const co = require('co');
+const ms = require('humanize-ms');
+var config = require('../config');
+var npmService = require('../services/npm');
+var SyncModuleWorker = require('../controllers/sync_module_worker');
+var logger = require('../common/logger');
+
+
+let syncing = false;
+const syncFn = co.wrap(function*() {
+  if (syncing) { return; }
+  syncing = true;
+  logger.syncInfo('Start syncing scope modules...');
+  let data;
+  let error;
+  try {
+    data = yield sync();
+  } catch (err) {
+    error = err;
+    error.message += ' (sync package error)';
+    logger.syncError(error);
+  }
+
+  if (data) {
+    logger.syncInfo(data);
+  }
+  if (!config.debug) {
+    sendMailToAdmin(error, data, new Date());
+  }
+  syncing = false;
+});
+
+syncFn().catch(onerror);
+setInterval(() => syncFn().catch(onerror), ms(config.syncScopeInterval));
+
+function onerror(err) {
+  logger.error('====================== scope sync error ========================');
+  logger.error(err);
+}
+
+function* getOtherCnpmDefineScopePackages(scopes) {
+  var arr = []
+  for (var i = 0; i < scopes.length; i++) {
+    var packageList = yield* npmService.getScopePackagesShort(scopes[i].scope, scopes[i].sourceCnpmWeb)
+    arr = arr.concat(packageList)
+  }
+  return arr
+}
+
+function* sync() {
+  var scopeConfig = config.syncScopeConfig
+  if (!scopeConfig || scopeConfig.length === 0) {
+    process.exit(0);
+  }
+  var packages = yield* getOtherCnpmDefineScopePackages(scopeConfig);
+
+  if (!packages.length) {
+    return;
+  }
+  logger.syncInfo('Total %d scope packages to sync: %j', packages.length, packages);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    noDep: true,
+    syncUpstreamFirst: false,
+    publish: true,
+    concurrency: config.syncConcurrency,
+    syncPrivatePackage: scopeConfig.reduce((arr, cur) => {
+      arr[cur.scope] = cur.sourceCnpmRegistry
+      return arr
+    }, {})
+  });
+  worker.start();
+  var end = thunkify.event(worker);
+  yield end();
+
+  logger.syncInfo('scope packages sync done, successes %d, fails %d, updates %d',
+      worker.successes.length, worker.fails.length, worker.updates.length);
+
+  return {
+    successes: worker.successes,
+    fails: worker.fails,
+    updates: worker.updates,
+  };
+};