This repository has been archived by the owner on Jun 2, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 746
feat: add changes stream syncer #970
Merged
Merged
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,20 +1,6 @@ | ||
/**! | ||
* cnpmjs.org - common/logger.js | ||
* | ||
* Copyright(c) cnpmjs.org and other contributors. | ||
* MIT Licensed | ||
* | ||
* Authors: | ||
* dead_horse <[email protected]> (http://deadhorse.me) | ||
* fengmk2 <[email protected]> (http://fengmk2.github.com) | ||
*/ | ||
|
||
'use strict'; | ||
|
||
/** | ||
* Module dependencies. | ||
*/ | ||
|
||
var debug = require('debug')('cnpmjs.org:logger'); | ||
var formater = require('error-formater'); | ||
var Logger = require('mini-logger'); | ||
var utility = require('utility'); | ||
|
@@ -50,6 +36,9 @@ logger.syncInfo = function () { | |
if (typeof args[0] === 'string') { | ||
args[0] = util.format('[%s][%s] ', utility.logDate(), process.pid) + args[0]; | ||
} | ||
if (debug.enabled) { | ||
debug.apply(debug, args); | ||
} | ||
logger.sync_info.apply(logger, args); | ||
}; | ||
|
||
|
@@ -58,5 +47,8 @@ logger.syncError =function () { | |
if (typeof args[0] === 'string') { | ||
args[0] = util.format('[%s][%s] ', utility.logDate(), process.pid) + args[0]; | ||
} | ||
if (debug.enabled) { | ||
debug.apply(debug, args); | ||
} | ||
logger.sync_error.apply(logger, arguments); | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +1,5 @@ | ||
/** | ||
* Copyright(c) cnpmjs.org and other contributors. | ||
* MIT Licensed | ||
* | ||
* Authors: | ||
* dead_horse <[email protected]> | ||
* fengmk2 <[email protected]> (http://fengmk2.com) | ||
*/ | ||
|
||
'use strict'; | ||
|
||
/** | ||
* Module dependencies. | ||
*/ | ||
|
||
var mkdirp = require('mkdirp'); | ||
var copy = require('copy-to'); | ||
var path = require('path'); | ||
|
@@ -26,6 +13,7 @@ var dataDir = path.join(process.env.HOME || root, '.cnpmjs.org'); | |
|
||
var config = { | ||
version: version, | ||
dataDir: dataDir, | ||
|
||
/** | ||
* Cluster mode | ||
|
@@ -209,6 +197,10 @@ var config = { | |
// sync devDependencies or not, default is false | ||
syncDevDependencies: false, | ||
|
||
// changes streaming sync | ||
syncChangesStream: false, | ||
handleSyncRegistry: 'http://127.0.0.1:7001', | ||
|
||
// badge subject on http://shields.io/ | ||
badgePrefixURL: 'https://img.shields.io/badge', | ||
badgeSubject: 'cnpm', | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +1,5 @@ | ||
/**! | ||
* Copyright(c) cnpmjs.org and other contributors. | ||
* MIT Licensed | ||
* | ||
* Authors: | ||
* fengmk2 <[email protected]> (http://fengmk2.com) | ||
* dead_horse <[email protected]> (http://deadhorse.me) | ||
*/ | ||
|
||
'use strict'; | ||
|
||
/** | ||
* Module dependencies. | ||
*/ | ||
|
||
var debug = require('debug')('cnpmjs.org:sync_module_worker'); | ||
var co = require('co'); | ||
var gather = require('co-gather'); | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +1,5 @@ | ||
/**! | ||
* Copyright(c) cnpm and other contributors. | ||
* MIT Licensed | ||
* | ||
* Authors: | ||
* fengmk2 <[email protected]> (http://fengmk2.com) | ||
* dead_horse <[email protected]> (http://deadhorse.me) | ||
*/ | ||
|
||
'use strict'; | ||
|
||
/** | ||
* Module dependencies. | ||
*/ | ||
|
||
const Total = require('../services/total'); | ||
const version = require('../package.json').version; | ||
const config = require('../config'); | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +1,5 @@ | ||
/**! | ||
* Copyright(c) cnpmjs.org and other contributors. | ||
* MIT Licensed | ||
* | ||
* Authors: | ||
* dead_horse <[email protected]> | ||
* fengmk2 <[email protected]> (http://fengmk2.com) | ||
*/ | ||
|
||
'use strict'; | ||
|
||
/** | ||
* Module dependencies. | ||
*/ | ||
|
||
var childProcess = require('child_process'); | ||
var path = require('path'); | ||
var util = require('util'); | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
'use strict'; | ||
|
||
const ChangesStream = require('changes-stream'); | ||
const path = require('path'); | ||
const fs = require('mz/fs'); | ||
const os = require('os'); | ||
const urllib = require('urllib'); | ||
const streamAwait = require('await-event') | ||
const logger = require('../common/logger'); | ||
const config = require('../config'); | ||
|
||
const db = 'https://replicate.npmjs.com'; | ||
const lastSeqFile = path.join(config.dataDir, '.cnpmjs.org.last_seq.txt'); | ||
|
||
module.exports = function* sync() { | ||
const since = yield getLastSequence(); | ||
logger.syncInfo('start changes stream, since: %s', since); | ||
const changes = new ChangesStream({ | ||
db, | ||
since, | ||
include_docs: false, | ||
}); | ||
changes.await = streamAwait; | ||
changes.on('data', change => { | ||
logger.syncInfo('Get change: %j', change); | ||
syncPackage(change); | ||
}); | ||
|
||
yield changes.await('error'); | ||
}; | ||
|
||
function syncPackage(change) { | ||
const url = `${config.handleSyncRegistry}/${change.id}/sync`; | ||
urllib.request(url, { | ||
method: 'PUT', | ||
dataType: 'json', | ||
timeout: 10000, | ||
}, (err, data, res) => { | ||
if (err) { | ||
logger.syncInfo('%s:%s PUT %s error: %s, retry after 5s', | ||
change.seq, change.id, url, err); | ||
logger.syncError(err); | ||
syncPackage(change); | ||
setTimeout(() => syncPackage(change), 5000); | ||
} else { | ||
saveLastSequence(change.seq); | ||
logger.syncInfo('%s:%s sync request sent, log: %s/log/%s', | ||
change.seq, change.id, url, data.logId); | ||
} | ||
}); | ||
} | ||
|
||
function* getLastSequence() { | ||
let lastSeq; | ||
if (yield fs.exists(lastSeqFile)) { | ||
lastSeq = yield fs.readFile(lastSeqFile, 'utf8'); | ||
lastSeq = Number(lastSeq); | ||
} | ||
if (!lastSeq) { | ||
lastSeq = 2614765; | ||
} | ||
// const r = yield urllib.request(db, { | ||
// dataType: 'json', | ||
// timeout: 15000, | ||
// }); | ||
// logger.syncInfo('get registry info: %j', r.data); | ||
// if (lastSeq < r.data.update_seq) { | ||
// lastSeq = r.data.update_seq; | ||
// } | ||
return lastSeq; | ||
} | ||
|
||
function saveLastSequence(seq) { | ||
fs.writeFile(lastSeqFile, String(seq), () => {}); | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
这个接口只同步这个模块,不会同步他的依赖吧? 10s 够么?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
只是调用一下创建同步任务,不关心是否真正同步成功。即使失败,原来的定时同步在10分钟后也会确保同步成功。
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
记起来了,查询
是否成功日志的接口是另外的