From e4dae73990886517187827c17d8f25f330a223d5 Mon Sep 17 00:00:00 2001 From: wangjiaxin2 Date: Tue, 8 Feb 2022 10:29:42 +0800 Subject: [PATCH] feat: support sync private package from custom registry --- config/index.js | 14 ++++ controllers/sync.js | 50 ++++++++++++ controllers/sync_module_worker.js | 40 ++++++++-- controllers/web/show_scope_sync.js | 23 ++++++ dispatch.js | 25 ++++-- routes/web.js | 4 + services/npm.js | 10 +++ sync/sync_scope.js | 89 +++++++++++++++++++++ view/web/scope_sync.html | 124 +++++++++++++++++++++++++++++ 9 files changed, 366 insertions(+), 13 deletions(-) create mode 100644 controllers/web/show_scope_sync.js create mode 100644 sync/sync_scope.js create mode 100644 view/web/scope_sync.html diff --git a/config/index.js b/config/index.js index 2426cfaec..9fd6a5eac 100644 --- a/config/index.js +++ b/config/index.js @@ -244,6 +244,20 @@ var config = { syncDownloadOptions: { // formatRedirectUrl: function (url, location) }, + + // all syncModel cannot sync scope pacakge, you can use this model to sync scope package from any resgitry + syncScope: false, + syncScopeInterval: '12h', + // scope package sync config + /** + * sync scope package from assign registry + * @param {Array} scopes + * @param {String} scope.scope scope name + * @param {String} scope.sourceCnpmWeb source cnpm registry web url for get scope all packages name + * @param {String} scope.sourceCnpmRegistry source cnpm registry url for sync packages + */ + syncScopeConfig: [], + handleSyncRegistry: 'http://127.0.0.1:7001', // default badge subject diff --git a/controllers/sync.js b/controllers/sync.js index 9bd23ba6a..1be11b67b 100644 --- a/controllers/sync.js +++ b/controllers/sync.js @@ -2,6 +2,7 @@ var debug = require('debug')('cnpmjs.org:controllers:sync'); var Log = require('../services/module_log'); +var npmService = require('../services/npm'); var SyncModuleWorker = require('./sync_module_worker'); var config = require('../config'); @@ -52,6 +53,55 @@ exports.sync = function* () { }; }; +exports.scopeSync = function* () { + var scope = this.params.scope; + + var scopeConfig = (config.syncScopeConfig || []).find(function (item) { + return item.scope === scope + }) + + if (!scopeConfig) { + this.status = 404; + this.body = { + error: 'no_scope', + reason: 'only has syncScopeConfig config can use this feature' + }; + return; + } + + var scopeCnpmWeb = scopeConfig.sourceCnpmWeb + var scopeCnpmRegistry = scopeConfig.sourceCnpmRegistry + var packages = yield* npmService.getScopePackagesShort(scope, scopeCnpmWeb) + + debug('scopeSync %s with query: %j', scope, this.query); + + var packageSyncWorkers = [] + + for (let i = 0; i < packages.length; i++) { + packageSyncWorkers.push(function* () { + var name = packages[i] + var logId = yield* SyncModuleWorker.sync(name, 'admin', { + type: 'package', + publish: true, + noDep: true, + syncUpstreamFirst: false, + syncPrivatePackage: { [scope]: scopeCnpmRegistry } + }) + return { name: name, logId: logId } + }) + } + + var logIds = yield packageSyncWorkers + + debug('scopeSync %s got log id %j', scope, logIds); + + this.status = 201; + this.body = { + ok: true, + logIds: logIds + }; +}; + exports.getSyncLog = function* (next) { var logId = Number(this.params.id || this.params[1]); var offset = Number(this.query.offset) || 0; diff --git a/controllers/sync_module_worker.js b/controllers/sync_module_worker.js index b85d377d1..0f7edf5d0 100644 --- a/controllers/sync_module_worker.js +++ b/controllers/sync_module_worker.js @@ -49,6 +49,8 @@ function SyncModuleWorker(options) { this.names = options.name; this.startName = this.names[0]; + this.syncPrivatePackage = options.syncPrivatePackage; + this.username = options.username; this.concurrency = options.concurrency || 1; this._publish = options.publish === true; // _publish_on_cnpm @@ -313,14 +315,21 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) { return setImmediate(this.finish.bind(this)); } - if (config.syncModel === 'none') { + const defineRegistry = this.getPrivatePackageDefineRegistry(name) + + if (!defineRegistry && config.syncModel === 'none') { this.log('[c#%d] [%s] syncModel is none, ignore', - concurrencyId, name); + concurrencyId, name); return this.finish(); } - // try to sync from official replicate when source npm registry is not cnpmjs.org - const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate; + // try to sync from official replicate when no defineRegistry or source npm registry is not cnpmjs.org + let registry + if (defineRegistry) { + registry = defineRegistry + } else { + registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate; + } yield this.syncByName(concurrencyId, name, registry); }; @@ -520,8 +529,9 @@ SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry this.log('----------------- Syncing %s -------------------', name); + const isNeedSyncPrivatePackage = this.getPrivatePackageDefineRegistry(name) // ignore private scoped package - if (common.isPrivateScopedPackage(name)) { + if (!isNeedSyncPrivatePackage && common.isPrivateScopedPackage(name)) { this.log('[c#%d] [%s] ignore sync private scoped %j package', concurrencyId, name, config.scopes); yield this._doneOne(concurrencyId, name, true); @@ -687,6 +697,21 @@ SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry return versions; }; +SyncModuleWorker.prototype.getPrivatePackageDefineRegistry = function (name) { + if (typeof name !== 'string') return false + return this.syncPrivatePackage && this.syncPrivatePackage[name.split('/')[0]] +} + +SyncModuleWorker.prototype.isLocalModule = function (mods) { + var res = common.isLocalModule(mods) + if (!this.syncPrivatePackage) return res + if (!mods[0] || !mods[0].package || !mods[0].package.name) return res + + if (this.getPrivatePackageDefineRegistry(mods[0].package.name)) return false + + return res +} + function* _listStarUsers(modName) { var users = yield packageService.listStarUserNames(modName); var userMap = {}; @@ -727,7 +752,7 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) { var mods = yield packageService.listModulesByName(name); this.log(' [%s] start unpublished %d versions from local cnpm registry', name, mods.length); - if (common.isLocalModule(mods)) { + if (this.isLocalModule(mods)) { // publish on cnpm, dont sync this version package this.log(' [%s] publish on local cnpm registry, don\'t sync', name); return []; @@ -795,7 +820,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) { var existsNpmMaintainers = result[3]; var existsModuleAbbreviateds = result[4]; - if (common.isLocalModule(moduleRows)) { + if (this.isLocalModule(moduleRows)) { // publish on cnpm, dont sync this version package that.log(' [%s] publish on local cnpm registry, don\'t sync', name); return []; @@ -1907,6 +1932,7 @@ SyncModuleWorker.sync = function* (name, username, options) { publish: options.publish, syncUpstreamFirst: options.syncUpstreamFirst, syncFromBackupFile: options.syncFromBackupFile, + syncPrivatePackage: options.syncPrivatePackage }); worker.start(); return result.id; diff --git a/controllers/web/show_scope_sync.js b/controllers/web/show_scope_sync.js new file mode 100644 index 000000000..aef23e541 --- /dev/null +++ b/controllers/web/show_scope_sync.js @@ -0,0 +1,23 @@ +'use strict'; +var config = require('../../config'); +var npmService = require('../../services/npm'); + +module.exports = function* showScopeSync () { + var scope = this.params.scope; + var scopeConfig = (config.syncScopeConfig || []).find(function (item) { + return item.scope === scope + }) + + if (!scopeConfig) { + return this.redirect('/'); + } + + var packages = yield* npmService.getScopePackagesShort(scope, scopeConfig.sourceCnpmWeb) + + yield this.render('scope_sync', { + packages: packages, + scope: scopeConfig.scope, + sourceCnpmRegistry: scopeConfig.sourceCnpmRegistry, + title: 'Sync Scope Packages', + }); +}; diff --git a/dispatch.js b/dispatch.js index 1a1a7a8e9..96d50c1d7 100644 --- a/dispatch.js +++ b/dispatch.js @@ -7,20 +7,21 @@ var cfork = require('cfork'); var config = require('./config'); var workerPath = path.join(__dirname, 'worker.js'); var syncPath = path.join(__dirname, 'sync'); +var scopeSyncPath = path.join(__dirname, 'sync/sync_scope'); console.log('Starting cnpmjs.org ...\ncluster: %s\nadmins: %j\nscopes: %j\nsourceNpmRegistry: %s\nsyncModel: %s', config.enableCluster, config.admins, config.scopes, config.sourceNpmRegistry, config.syncModel); if (config.enableCluster) { forkWorker(); - if (config.syncModel !== 'none') { - forkSyncer(); - } + config.syncModel !== 'none' && forkSyncer(); + // scync assign pravate scope package + config.syncScope && forkScopeSyncer(); } else { require(workerPath); - if (config.syncModel !== 'none') { - require(syncPath); - } + config.syncModel !== 'none' && require(syncPath); + // scync assign pravate scope package + config.syncScope && require(scopeSyncPath); } function forkWorker() { @@ -52,3 +53,15 @@ function forkSyncer() { setTimeout(forkSyncer, 1000); }); } + +function forkScopeSyncer() { + var syncer = childProcess.fork(scopeSyncPath); + syncer.on('exit', function (code, signal) { + var err = new Error(util.format('syncer %s died (code: %s, signal: %s, stdout: %s, stderr: %s)', + syncer.pid, code, signal, syncer.stdout, syncer.stderr)); + err.name = 'SyncerWorkerDiedError'; + console.error('[%s] [master:%s] syncer exit: %s: %s', + Date(), process.pid, err.name, err.message); + setTimeout(forkScopeSyncer, 1000); + }); +} diff --git a/routes/web.js b/routes/web.js index e86191a3b..fe8019091 100644 --- a/routes/web.js +++ b/routes/web.js @@ -5,6 +5,7 @@ var searchPackage = require('../controllers/web/package/search'); var searchRange = require('../controllers/web/package/search_range'); var listPrivates = require('../controllers/web/package/list_privates'); var showSync = require('../controllers/web/show_sync'); +var showScopeSync = require('../controllers/web/show_scope_sync'); var showUser = require('../controllers/web/user/show'); var sync = require('../controllers/sync'); var showTotal = require('../controllers/total'); @@ -35,6 +36,9 @@ function routes(app) { app.put(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)$/, sync.sync); app.put('/sync/:name', sync.sync); + app.get('/scopeSync/:scope', showScopeSync); + app.put('/scopeSync/:scope', sync.scopeSync); + app.get(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)\/log\/(\d+)$/, sync.getSyncLog); app.get('/sync/:name/log/:id', sync.getSyncLog); diff --git a/services/npm.js b/services/npm.js index 40aeded0f..9ae23d2e4 100644 --- a/services/npm.js +++ b/services/npm.js @@ -198,3 +198,13 @@ exports.getPopular = function* (top, timeout) { return r[0]; }); }; + +exports.getScopePackagesShort = function* (scope, registry) { + var response = yield* request('/browse/keyword/' + scope, { + timeout: 3000, + registry: registry, + dataType: 'text' + }); + var res = response.data.match(/class="package-name">(\S*)<\/a>/g) + return res ? res.map(a => a.match(/class="package-name">(\S*)<\/a>/)[1]).filter(name => name.indexOf(`${scope}/`) === 0) : [] +}; diff --git a/sync/sync_scope.js b/sync/sync_scope.js new file mode 100644 index 000000000..fd385185f --- /dev/null +++ b/sync/sync_scope.js @@ -0,0 +1,89 @@ +'use strict'; + +var thunkify = require('thunkify-wrap'); +const co = require('co'); +const ms = require('humanize-ms'); +var config = require('../config'); +var npmService = require('../services/npm'); +var SyncModuleWorker = require('../controllers/sync_module_worker'); +var logger = require('../common/logger'); + + +let syncing = false; +const syncFn = co.wrap(function*() { + if (syncing) { return; } + syncing = true; + logger.syncInfo('Start syncing scope modules...'); + let data; + let error; + try { + data = yield sync(); + } catch (err) { + error = err; + error.message += ' (sync package error)'; + logger.syncError(error); + } + + if (data) { + logger.syncInfo(data); + } + if (!config.debug) { + sendMailToAdmin(error, data, new Date()); + } + syncing = false; +}); + +syncFn().catch(onerror); +setInterval(() => syncFn().catch(onerror), ms(config.syncScopeInterval)); + +function onerror(err) { + logger.error('====================== scope sync error ========================'); + logger.error(err); +} + +function* getOtherCnpmDefineScopePackages(scopes) { + var arr = [] + for (var i = 0; i < scopes.length; i++) { + var packageList = yield* npmService.getScopePackagesShort(scopes[i].scope, scopes[i].sourceCnpmWeb) + arr = arr.concat(packageList) + } + return arr +} + +function* sync() { + var scopeConfig = config.syncScopeConfig + if (!scopeConfig || scopeConfig.length === 0) { + process.exit(0); + } + var packages = yield* getOtherCnpmDefineScopePackages(scopeConfig); + + if (!packages.length) { + return; + } + logger.syncInfo('Total %d scope packages to sync: %j', packages.length, packages); + + var worker = new SyncModuleWorker({ + username: 'admin', + name: packages, + noDep: true, + syncUpstreamFirst: false, + publish: true, + concurrency: config.syncConcurrency, + syncPrivatePackage: scopeConfig.reduce((arr, cur) => { + arr[cur.scope] = cur.sourceCnpmRegistry + return arr + }, {}) + }); + worker.start(); + var end = thunkify.event(worker); + yield end(); + + logger.syncInfo('scope packages sync done, successes %d, fails %d, updates %d', + worker.successes.length, worker.fails.length, worker.updates.length); + + return { + successes: worker.successes, + fails: worker.fails, + updates: worker.updates, + }; +}; diff --git a/view/web/scope_sync.html b/view/web/scope_sync.html new file mode 100644 index 000000000..10df4b783 --- /dev/null +++ b/view/web/scope_sync.html @@ -0,0 +1,124 @@ +
+

Scope Packages Sync

+
+
<%= scope %>
+
source registry ：<%= sourceCnpmRegistry %>
+
scope package number：<%= packages.length %>
+
Last Sync Time:
+ +
+
    + <% for(var i=0;i +
  • <%= packages[i] %>： wait sync
    • + <% } %> +
+

Log

+
+ <% for(var i=0;i +

Sync <%= packages[i] %>

+
+
Sync started, please wait patiently.
+
+ 

+    <% } %>
+  

+

+