Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Lock redis instance when doing a specific task #42

Open
wants to merge 24 commits into
base: master
Choose a base branch
from
Open
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions index.js
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ module.exports.setup = require('./services/setup');
module.exports.put = db.put;
module.exports.get = db.get;
module.exports.del = db.del;
module.exports.applyLock = db.applyLock;
module.exports.batch = db.batch;
module.exports.putMeta = db.putMeta;
module.exports.patchMeta = db.patchMeta;
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -24,7 +24,8 @@
"knex": "^0.15.2",
"pg": "7.4.3",
"pg-query-stream": "^1.1.1",
"postgres-migrations": "^3.0.2"
"postgres-migrations": "^3.0.2",
"redlock": "^3.1.2"
},
"devDependencies": {
"coveralls": "^3.0.1",
7 changes: 6 additions & 1 deletion redis/index.js
Original file line number Diff line number Diff line change
@@ -4,7 +4,9 @@ const bluebird = require('bluebird'),
Redis = require('ioredis'),
{ REDIS_URL, REDIS_HASH } = require('../services/constants'),
{ isPublished, isUri, isUser } = require('clayutils'),
{ notFoundError, logGenericError } = require('../services/errors');
{ notFoundError, logGenericError } = require('../services/errors'),
lock = require('./lock');

var log = require('../services/log').setup({ file: __filename });

/**
@@ -26,6 +28,8 @@ function createClient(testRedisUrl) {
module.exports.client = bluebird.promisifyAll(new Redis(redisUrl));
module.exports.client.on('error', logGenericError(__filename));

lock.setup(module.exports.client);

resolve({ server: redisUrl });
});
}
@@ -108,6 +112,7 @@ function del(key) {
}

module.exports.client = null;
module.exports.applyLock = lock.applyLock;
module.exports.createClient = createClient;
module.exports.get = get;
module.exports.put = put;
115 changes: 115 additions & 0 deletions redis/lock.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
'use strict';

const Redlock = require('redlock'),
{ logGenericError } = require('../services/errors'),
emptyModule = {
lock: () => Promise.resolve(),
unlock: () => Promise.resolve()
},
CONFIG = {
// the expected clock drift; for more details
// see http://redis.io/topics/distlock
driftFactor: 0.01, // time in ms

// the max number of times Redlock will attempt
// to lock a resource before erroring
retryCount: 0,

// the time in ms between attempts
retryDelay: 200, // time in ms

// the max time in ms randomly added to retries
// to improve performance under high contention
// see https://www.awsarchitectureblog.com/2015/03/backoff.html
retryJitter: 200 // time in ms
},
ACTION_RETRY_TOTAL = 5;

let log = require('../services/log').setup({ file: __filename }),
ACTION_RETRY_COUNT = 0;

function lockRedisForAction(resourceId, ttl) {
log('trace', `Trying to lock redis for resource id ${resourceId}`, { resourceId, processId: process.pid });
return module.exports.redlock.lock(resourceId, ttl);
}

function unlockWhenReady(lock, resourceId, cb) {
return cb()
.then(result => module.exports.redlock.unlock(lock)
.then(() => {
log('trace', `Releasing lock for resource id ${resourceId}`, { resourceId, processId: process.pid });
return result;
}));
}

function getFromState(id) {
return module.exports.redis.getAsync(id);
}

function setState(action, state, expire) {
return module.exports.redis.setAsync(action, state)
james-owen marked this conversation as resolved.
Show resolved Hide resolved
.then(() => {
if (expire) return module.exports.redis.expire(action, expire);
});
}

function sleepAndRun(cb, ms = 1000) {
return new Promise(resolve => setTimeout(() => cb().then(resolve), ms));
}

function applyLock(action, cb) {
const resourceId = `${action}-lock`,
ACTIONS = {
ONGOING: 'ON-GOING',
RETRY: 'RETRY',
FINISHED: 'FINISHED'
},
RETRY_TIME = 1500, // ms
KEY_TTL = 10 * 60, // secs
LOCK_TTL = 5000; // ms

return getFromState(action).then(state => {
/**
* If its ONGOING, just re-run this function after a while
fxisco marked this conversation as resolved.
Show resolved Hide resolved
* to see if the state changed.
*/
if (state === ACTIONS.ONGOING) {
return sleepAndRun(() => applyLock(action, cb), RETRY_TIME);
}

if (!state || state === 'RETRY') return setState(action, ACTIONS.ONGOING)
.then(() => lockRedisForAction(resourceId, LOCK_TTL))
.then(lock => unlockWhenReady(lock, resourceId, cb))
.then(() => setState(action, ACTIONS.FINISHED, KEY_TTL))
.catch(() => {
ACTION_RETRY_COUNT++;

if (ACTION_RETRY_COUNT === ACTION_RETRY_TOTAL) {
pedro-rosario marked this conversation as resolved.
Show resolved Hide resolved
log('error', `Action "${action}" could not be executed`);
return setState(action, ACTIONS.FINISHED, KEY_TTL);
}

return setState(action, ACTIONS.RETRY)
.then(() => sleepAndRun(() => applyLock(action, cb), RETRY_TIME));
});
});
}

function setup(instance) {
pedro-rosario marked this conversation as resolved.
Show resolved Hide resolved
if (!instance) return emptyModule;

const redlock = new Redlock([instance], CONFIG);

redlock.on('clientError', logGenericError(__filename));

module.exports.redis = instance;
james-owen marked this conversation as resolved.
Show resolved Hide resolved
module.exports.redlock = redlock;

return redlock;
}

module.exports.redis = {};
module.exports.redlock;

module.exports.setup = setup;
module.exports.applyLock = applyLock;
1 change: 1 addition & 0 deletions services/db.js
Original file line number Diff line number Diff line change
@@ -85,6 +85,7 @@ module.exports.put = put;
module.exports.get = get;
module.exports.del = del;
module.exports.batch = batch;
module.exports.applyLock = redis.applyLock;
module.exports.raw = postgres.raw;
module.exports.putMeta = postgres.putMeta;
module.exports.getMeta = postgres.getMeta;