From f027afb2dcf4593ccc2681ff276a1604ff14f4a8 Mon Sep 17 00:00:00 2001 From: Ajith Date: Tue, 14 Jan 2020 08:43:13 +0530 Subject: [PATCH 01/10] Added support for migration of app-list modules --- lib/migration.js | 16 +++++++--------- test/test.js | 17 ++++++----------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/lib/migration.js b/lib/migration.js index 3eafcef..7d49bc8 100644 --- a/lib/migration.js +++ b/lib/migration.js @@ -30,6 +30,7 @@ function migrate(mOptions, mCb) { mCb = mOptions; mOptions = {}; } + basePath = (mOptions && mOptions.basePath) || basePath; var migrationCb = function (err, oldDbVersion, migratedVersions) { endTime = new Date().getTime(); // eslint-disable-next-line no-console @@ -93,8 +94,9 @@ function migrate(mOptions, mCb) { }); } else { var SystemConfig = loopback.findModel('SystemConfig'); + var versionKey = 'dbVersion.' + basePath.replace(/\./g, '_').replace(/\//g, '_') SystemConfig.create({ - 'key': 'dbVersion', + 'key': versionKey, 'value': lastVersion }, opts, function (err, data) { /* istanbul ignore if */ @@ -473,6 +475,8 @@ function clearTable(table, cb) { function getListOfMigrationPaths(options, cb) { + basePath = (options && options.basePath) || basePath; + var versionKey = 'dbVersion.' + basePath.replace(/\./g, '_').replace(/\//g, '_'); var pathErrors = []; if (!(fs.existsSync(basePath) && fs.lstatSync(basePath).isDirectory())) { return cb(new Error('Path ' + basePath + ' does not exist. Not migrating.'), null); @@ -480,7 +484,7 @@ function getListOfMigrationPaths(options, cb) { var SystemConfig = loopback.findModel('SystemConfig'); SystemConfig.findOne({ where: { - key: 'dbVersion' + key: versionKey } }, opts, function (err, dbVersionInstance) { var fromVersion; @@ -531,13 +535,7 @@ function getListOfMigrationPaths(options, cb) { migrationDirs: retList, dbVersionInstance: dbVersionInstance }); - } else if (retList.length === 0) { - return cb(new Error('No (new) directories matched for migration. Nothing migrated.'), { - migratedVersions: migDirs, - migrationDirs: retList, - dbVersionInstance: dbVersionInstance - }); - } + } cb(null, { migratedVersions: migDirs, migrationDirs: retList, diff --git a/test/test.js b/test/test.js index 451a639..344be3b 100644 --- a/test/test.js +++ b/test/test.js @@ -118,9 +118,7 @@ describe(chalk.blue('oe-migration tests'), function (done) { // This function deletes all records in the MasterJobExecutorTestData table function clearTestData(cb) { var TAG = 'clearTestData:'; - SystemConfig.remove({ - key: 'dbVersion' - }, opts, function findCb(err, res) { + SystemConfig.remove({ }, opts, function findCb(err, res) { if (err) { console.error(TAG, 'Could not remove dbVersion record from SystemConfig ' + JSON.stringify(err)); cb(err); @@ -240,10 +238,9 @@ describe(chalk.blue('oe-migration tests'), function (done) { console.log(chalk.yellow('[' + new Date().toISOString() + '] : ', 'Starting ' + TAG)); var options = {}; migration.migrate(options, function (err, oldDbVersion, data) { - expect(err).not.to.be.null; + expect(err).to.be.null; expect(oldDbVersion).to.equal('2.5.0'); expect(data.migratedVersions).to.be.null; - expect(err.message).to.equal('No (new) directories matched for migration. Nothing migrated.'); done(); }); }); @@ -314,10 +311,9 @@ describe(chalk.blue('oe-migration tests'), function (done) { console.log(chalk.yellow('[' + new Date().toISOString() + '] : ', 'Starting ' + TAG)); var options = {}; migration.migrate(options, function (err, oldDbVersion, data) { - expect(err).not.to.be.null; + expect(err).to.be.null; expect(data.migratedVersions).to.be.null; expect(oldDbVersion).to.equal('2.5.0'); - expect(err.message).to.equal('No (new) directories matched for migration. Nothing migrated.'); done(); }); }); @@ -620,8 +616,7 @@ describe(chalk.blue('oe-migration tests'), function (done) { console.log(chalk.yellow('[' + new Date().toISOString() + '] : ', 'Starting ' + TAG)); migration.setBasePath(null); // Setting it back to default migration.migrate(function (err, oldDbVersion, data) { - expect(err).not.to.be.null; - expect(err.message).to.equal('No (new) directories matched for migration. Nothing migrated.'); + expect(err).to.be.null; expect(oldDbVersion).to.equal('13.0.0'); expect(data.migratedVersions).to.be.null; done(); @@ -714,7 +709,7 @@ describe(chalk.blue('oe-migration tests'), function (done) { clearMigrationTestTables(function () { deleteMigrationLog(function () { migration.migrate(options, function (err, oldDbVersion, data) { - expect(oldDbVersion).to.equal('15.0.0'); + expect(oldDbVersion).to.be.null; expect(data.migratedVersions).to.eql(['20.0.0']); done(); }); @@ -1087,7 +1082,7 @@ describe(chalk.blue('oe-migration tests'), function (done) { copyRecursiveSync(path.join(process.cwd(), 'test', 'db1', '24.0.0'), path.join(basePath, '24.0.0')); migration.migrate(options, function (err, oldDbVersion, data) { - expect(oldDbVersion).to.equal('21.0.0'); + expect(oldDbVersion).to.equal('15.0.0'); expect(data.migratedVersions).to.eql(['24.0.0']); done(); }); From 77983ba1c3ced5a86899498f69bcc39e5065017e Mon Sep 17 00:00:00 2001 From: Ajith Date: Tue, 14 Jan 2020 10:57:42 +0530 Subject: [PATCH 02/10] Changed SystemConfig key format --- README.md | 106 ++++++++++++++++++++++++++++++++++++++++------- lib/migration.js | 9 ++-- test/test.js | 4 +- 3 files changed, 100 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 0f1ec5b..335dac9 100644 --- a/README.md +++ b/README.md @@ -32,25 +32,28 @@ This migration module has the following features: 1. App list module - This module is built as an [app-list module](#bookmark0) which can be added to the application on a need basis. -2. Allows loading of data from [JSON files present in project sub-folder](#bookmark2) to database +2. Allows loading of data from [JSON files present in a project sub-folder](#bookmark2) or any other folder, to database -3. Allows [versioned data migration](#bookmark2), with data-folder name as the DB version. (de-linked from app package version). +3. Allows loading of data from [JSON files present in the project's app-list modules](#bookmark2b), to database -4. Re-running of migration script without adding any new data causes no harm +4. Allows [versioned data migration](#bookmark2), with data-folder name as the DB version. (de-linked from app package version). -5. Migration can be [triggered via a standalone command](#bookmark1) at the command prompt (node server/migratejs) +5. Re-running of migration script without adding any new data causes no harm -6. Allows sequential DDL/DML changes: +6. Migration can be [triggered via a standalone command](#bookmark1) at the command prompt (node server/migratejs) + +7. Allows sequential DDL/DML changes: - [Allows writing version-specific meta-data](#bookmark2) (model definitions) to specify structural changes to tables - Auto [populate default data](#bookmark2a) in existing records for a new column -7. Has an [option to download the existing data](#bookmark3) in the DB (arbitrary list of tables, or all tables) as a zip file -8. Allows migration of data over http as a [zip file upload](#bookmark4) -9. Allows [download of migration logs](#bookmark5) over http (logs persisted in MigrationLogs table) -10. Has option to [skip oeCloud Validations](#bookmark6) during migration -11. Has option to [use updateAttributes instead of upsert](#bookmark6) during migration -12. Has option to [clear the data in one or more tables](#bookmark6) before commencing data migration. -13. Has option to [rollback](#bookmark6a) to a previous db version +8. Has an [option to download the existing data](#bookmark3) in the DB (arbitrary list of tables, or all tables) as a zip file +9. Allows migration of data over http as a [zip file upload](#bookmark4) +10. Allows [download of migration logs](#bookmark5) over http (logs persisted in MigrationLogs table) +11. Has option to [skip oeCloud Validations](#bookmark6) during migration +12. Has option to [use updateAttributes instead of upsert](#bookmark6) during migration +13. Has option to [clear the data in one or more tables](#bookmark6) before commencing data migration. +14. Has option to [rollback](#bookmark6a) to a previous db version +15. Allows [execution of custom JS files](#Running Custom JS files) pre- and/or post- migration. @@ -113,7 +116,8 @@ The code snippets below show how steps 1 and 2 can be done: ## Usage This module can be used for the following purposes: -- Migration from Command-Line +- Migration of application seed data from Command-Line +- Migration of app-list module's seed data from Command-Line - Downloading zip file of DB data - Uploading zip file for migration @@ -249,6 +253,77 @@ An example of MigrationLogs is shown below: The `server/migrate.js` can be executed repeatedly with/without additional data in the `/db` folder. + + + +### Migration of app-list module seed data + +*oe-migration* supports the migration of *app-list* modules by leveraging the `options.basePath` parameter of the `migrate()` function. + +For this to work, + +1. Each *app-list* module should have its own seed-data in a folder named `db` at the module's root. +2. Create and run a new js file, say, *migrate-all.js* in your `/server/` folder with the following contents: + + ```javascript + + var app = require('oe-cloud'); + var async = require('async'); + var m = require('oe-migration'); + var path = require('path'); + var fs = require('fs'); + + app.boot(__dirname, function (err) { + if (err) { console.log(err); process.exit(1); } + + var appListBasePaths = getAppListBasePaths(); + + async.eachOfSeries(appListBasePaths, function (basePath, key, cb) { + m.migrate({ basePath: basePath }, function (err, oldDbVersion, migratedVersions) { + cb(err); + }); + }, function (err) { + if (err) { + console.log(err); + process.exit(1); + } else { + console.log('migration done'); + process.exit(0); + } + }); + }); + + + function getAppListBasePaths() { + var appListBasePaths = []; + var mdls = require('../server/app-list.json'); + mdls.forEach(function (o) { + var bPath; + if (o.path === './') { + bPath = path.resolve(process.cwd(), 'db'); + } else { + bPath = path.resolve(process.cwd(), 'node_modules', o.path, 'db'); + } + var isDir = false; + try { + isDir = fs.statSync(bPath).isDirectory(); + if (isDir === true) appListBasePaths.push(bPath); + } catch (e) {} + }); + return appListBasePaths; + } + ``` + A ready-made file with the above content is available in the [oe-app](http://evgit/oecloud.io/oe-demo-app) sample project at https://evgit/oecloud.io/oe-app/blob/master/server/migrate-all.js + You can copy this file to your `/server/` folder instead of creating a new file from scratch. + This file creation is a one-time activity, and the file itself can be part of your application. + +**Notes:** + +1. The above script, when run, does the migration of data in the `db` folders from each *app-list* module, in the order in which the module is specified in the application's `app-list.json` file. +2. The above script also migrates the main application's `db` folder, again, as per the order of the `./` module specified in `app-list.json` +3. The application developer is free to modify the sample migrate-all.js file to meet the applicatio's needs. For example, the developer could choose not to migrate data from one or more app-list modules, change the location from where data is migrated, etc., + + ### Running Custom JS files @@ -434,13 +509,16 @@ The `options` object can have the following properties, illustrated with an exam ```js { + basePath: '/data/db', // Optional String. Specifies the absolute path to a folder to be used as the "db" folder. + // Default value is the "db" folder at the root of the oeCloud application. + force: true, // Optional. If true, repeats migration from all db versions present in "db" folder, // ignoring the last version that was migrated (last version being in SystemConfig table) fromVersion: '2.0.0', // Optional. When used with force: true, sets the starting db version for // forced migration. Default: '0.0.0' - toVersion: '4.0.0' // Optional. When used with force: true, sets the ending db version for + toVersion: '4.0.0', // Optional. When used with force: true, sets the ending db version for // forced migration. Default: last available version in "db" folder verbose: true // Optional. Prints additional info to the console during migration, for diff --git a/lib/migration.js b/lib/migration.js index 7d49bc8..b8d6f89 100644 --- a/lib/migration.js +++ b/lib/migration.js @@ -10,6 +10,7 @@ var opts = { var options; var MigrationLog; var basePath = path.resolve(process.cwd(), typeof global.it !== 'function' ? '' : 'test', 'db'); +var moduleName = './'; var validationFunctions = {}; var startTime; var endTime; @@ -31,6 +32,7 @@ function migrate(mOptions, mCb) { mOptions = {}; } basePath = (mOptions && mOptions.basePath) || basePath; + moduleName = (mOptions && mOptions.moduleName) || './'; var migrationCb = function (err, oldDbVersion, migratedVersions) { endTime = new Date().getTime(); // eslint-disable-next-line no-console @@ -94,7 +96,7 @@ function migrate(mOptions, mCb) { }); } else { var SystemConfig = loopback.findModel('SystemConfig'); - var versionKey = 'dbVersion.' + basePath.replace(/\./g, '_').replace(/\//g, '_') + var versionKey = 'dbVersion.' + moduleName; SystemConfig.create({ 'key': versionKey, 'value': lastVersion @@ -476,7 +478,8 @@ function clearTable(table, cb) { function getListOfMigrationPaths(options, cb) { basePath = (options && options.basePath) || basePath; - var versionKey = 'dbVersion.' + basePath.replace(/\./g, '_').replace(/\//g, '_'); + moduleName = (options && options.moduleName) || './'; + var versionKey = 'dbVersion.' + moduleName; var pathErrors = []; if (!(fs.existsSync(basePath) && fs.lstatSync(basePath).isDirectory())) { return cb(new Error('Path ' + basePath + ' does not exist. Not migrating.'), null); @@ -535,7 +538,7 @@ function getListOfMigrationPaths(options, cb) { migrationDirs: retList, dbVersionInstance: dbVersionInstance }); - } + } cb(null, { migratedVersions: migDirs, migrationDirs: retList, diff --git a/test/test.js b/test/test.js index 344be3b..f56ef8d 100644 --- a/test/test.js +++ b/test/test.js @@ -709,7 +709,7 @@ describe(chalk.blue('oe-migration tests'), function (done) { clearMigrationTestTables(function () { deleteMigrationLog(function () { migration.migrate(options, function (err, oldDbVersion, data) { - expect(oldDbVersion).to.be.null; + expect(oldDbVersion).to.equal('15.0.0'); expect(data.migratedVersions).to.eql(['20.0.0']); done(); }); @@ -1082,7 +1082,7 @@ describe(chalk.blue('oe-migration tests'), function (done) { copyRecursiveSync(path.join(process.cwd(), 'test', 'db1', '24.0.0'), path.join(basePath, '24.0.0')); migration.migrate(options, function (err, oldDbVersion, data) { - expect(oldDbVersion).to.equal('15.0.0'); + expect(oldDbVersion).to.equal('21.0.0'); expect(data.migratedVersions).to.eql(['24.0.0']); done(); }); From 71688ed17ebf9fed66e33b47200f61a678173eea Mon Sep 17 00:00:00 2001 From: Ajith Date: Tue, 14 Jan 2020 11:31:51 +0530 Subject: [PATCH 03/10] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 335dac9..4c79f83 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,7 @@ For this to work, return appListBasePaths; } ``` - A ready-made file with the above content is available in the [oe-app](http://evgit/oecloud.io/oe-demo-app) sample project at https://evgit/oecloud.io/oe-app/blob/master/server/migrate-all.js + A ready-made file with the above content is available in the [oe-demo-app](http://evgit/oecloud.io/oe-demo-app) sample project at https://evgit/oecloud.io/oe-demo-app/blob/master/server/migrate-all.js You can copy this file to your `/server/` folder instead of creating a new file from scratch. This file creation is a one-time activity, and the file itself can be part of your application. @@ -512,6 +512,10 @@ The `options` object can have the following properties, illustrated with an exam basePath: '/data/db', // Optional String. Specifies the absolute path to a folder to be used as the "db" folder. // Default value is the "db" folder at the root of the oeCloud application. + moduleName: 'my-app', // Optional String. Specifies the name of the module/app for which this migration is done. + // Default value is "./", the same string that is specified for "current application" in app-list.json. + // This is used by oe-migration as a key to maintain the latest DB migration version in th SystemConfig table. + force: true, // Optional. If true, repeats migration from all db versions present in "db" folder, // ignoring the last version that was migrated (last version being in SystemConfig table) From e2c7f3904a942c7fb33cf033c10fb8d70a1d5a41 Mon Sep 17 00:00:00 2001 From: Ajith Date: Tue, 14 Jan 2020 11:34:31 +0530 Subject: [PATCH 04/10] Update README.md --- README.md | 56 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 4c79f83..f844dc6 100644 --- a/README.md +++ b/README.md @@ -272,45 +272,47 @@ For this to work, var m = require('oe-migration'); var path = require('path'); var fs = require('fs'); - + app.boot(__dirname, function (err) { - if (err) { console.log(err); process.exit(1); } - - var appListBasePaths = getAppListBasePaths(); - - async.eachOfSeries(appListBasePaths, function (basePath, key, cb) { - m.migrate({ basePath: basePath }, function (err, oldDbVersion, migratedVersions) { - cb(err); + if (err) { console.log(err); process.exit(1); } + + var appList = getAppList(); + + async.eachOfSeries(appList, function (mdl, key, cb) { + m.migrate({ moduleName: mdl.moduleName, basePath: mdl.basePath }, function (err, oldDbVersion, migratedVersions) { + cb(err); }); - }, function (err) { + }, function (err) { if (err) { - console.log(err); - process.exit(1); + console.log(err); + process.exit(1); } else { - console.log('migration done'); - process.exit(0); + console.log('migration done'); + process.exit(0); } + }); }); - }); - - - function getAppListBasePaths() { - var appListBasePaths = []; - var mdls = require('../server/app-list.json'); - mdls.forEach(function (o) { + + + function getAppList() { + var appList = []; + var mdls = require('../server/app-list.json'); + mdls.forEach(function (o) { var bPath; if (o.path === './') { - bPath = path.resolve(process.cwd(), 'db'); + bPath = path.resolve(process.cwd(), 'db'); } else { - bPath = path.resolve(process.cwd(), 'node_modules', o.path, 'db'); + bPath = path.resolve(process.cwd(), 'node_modules', o.path, 'db'); } var isDir = false; try { - isDir = fs.statSync(bPath).isDirectory(); - if (isDir === true) appListBasePaths.push(bPath); - } catch (e) {} - }); - return appListBasePaths; + isDir = fs.statSync(bPath).isDirectory(); + if (isDir === true) appList.push({ moduleName: o.path, basePath: bPath}); + } catch (e) { + if (e) console.log('Ignoring module', o.path, ' : No db folder'); + } + }); + return appList; } ``` A ready-made file with the above content is available in the [oe-demo-app](http://evgit/oecloud.io/oe-demo-app) sample project at https://evgit/oecloud.io/oe-demo-app/blob/master/server/migrate-all.js From 3faccadfe63250980fd98304aa85b136ad4ef01c Mon Sep 17 00:00:00 2001 From: Ajith Date: Thu, 16 Jan 2020 11:47:31 +0530 Subject: [PATCH 05/10] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f844dc6..40a98a2 100644 --- a/README.md +++ b/README.md @@ -323,8 +323,8 @@ For this to work, 1. The above script, when run, does the migration of data in the `db` folders from each *app-list* module, in the order in which the module is specified in the application's `app-list.json` file. 2. The above script also migrates the main application's `db` folder, again, as per the order of the `./` module specified in `app-list.json` -3. The application developer is free to modify the sample migrate-all.js file to meet the applicatio's needs. For example, the developer could choose not to migrate data from one or more app-list modules, change the location from where data is migrated, etc., - +3. The application developer is free to modify the sample migrate-all.js file to meet the application's needs. For example, the developer could choose not to migrate data from one or more app-list modules, change the location from where data is migrated, etc., +4. While modifying the *migrate-all.js* script, keep in mind the `moduleName` option of `migrate()` function. This is used by *oe-migration* as a key to maintain the latest DB migration version in the *SystemConfig* table. This should be set as the name of the module whose data is being migrated. If the module is the main application itself, `moduleName` can be omitted or set to the default value, `./` ### Running Custom JS files @@ -516,7 +516,7 @@ The `options` object can have the following properties, illustrated with an exam moduleName: 'my-app', // Optional String. Specifies the name of the module/app for which this migration is done. // Default value is "./", the same string that is specified for "current application" in app-list.json. - // This is used by oe-migration as a key to maintain the latest DB migration version in th SystemConfig table. + // This is used by oe-migration as a key to maintain the latest DB migration version in the SystemConfig table. force: true, // Optional. If true, repeats migration from all db versions present in "db" folder, // ignoring the last version that was migrated (last version being in SystemConfig table) From 107dbf8929e3706c29d8e00a5bd6135bf797fe11 Mon Sep 17 00:00:00 2001 From: Ajith Date: Fri, 17 Jan 2020 11:02:28 +0530 Subject: [PATCH 06/10] Added support for running JS files as part of migration --- README.md | 91 ++++++++++++---------- lib/migration.js | 66 ++++++++++------ test/db1/25.0.1/default/migration-test1.js | 4 + test/db1/25.0.1/default/migration-test2.js | 4 + test/db1/25.0.1/meta.json | 20 +++++ test/test.js | 20 ++++- 6 files changed, 142 insertions(+), 63 deletions(-) create mode 100644 test/db1/25.0.1/default/migration-test1.js create mode 100644 test/db1/25.0.1/default/migration-test2.js create mode 100644 test/db1/25.0.1/meta.json diff --git a/README.md b/README.md index 40a98a2..5cee189 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ ## Need Applications often need to load seed data into their databases before they can go live. -This data could be inserts, updates (whole record replacement) or updateAttributes (replacement of one or more field data). +This data could be inserts, updates (whole record replacement) or updateAttributes (replacement of one or more field data). Data may also need to be processed and loaded into the DB by application-specific JS files. They may also need to load incremental amounts of data from time to time after go-live, incorporating any structural changes to the tables. @@ -36,24 +36,26 @@ This migration module has the following features: 3. Allows loading of data from [JSON files present in the project's app-list modules](#bookmark2b), to database -4. Allows [versioned data migration](#bookmark2), with data-folder name as the DB version. (de-linked from app package version). +4. Allows execution of JS files for custom processing and migration to database -5. Re-running of migration script without adding any new data causes no harm +5. Allows [versioned data migration](#bookmark2), with data-folder name as the DB version. (de-linked from app package version). -6. Migration can be [triggered via a standalone command](#bookmark1) at the command prompt (node server/migratejs) +6. Re-running of migration script without adding any new data causes no harm -7. Allows sequential DDL/DML changes: +7. Migration can be [triggered via a standalone command](#bookmark1) at the command prompt (node server/migratejs) + +8. Allows sequential DDL/DML changes: - [Allows writing version-specific meta-data](#bookmark2) (model definitions) to specify structural changes to tables - Auto [populate default data](#bookmark2a) in existing records for a new column -8. Has an [option to download the existing data](#bookmark3) in the DB (arbitrary list of tables, or all tables) as a zip file -9. Allows migration of data over http as a [zip file upload](#bookmark4) -10. Allows [download of migration logs](#bookmark5) over http (logs persisted in MigrationLogs table) -11. Has option to [skip oeCloud Validations](#bookmark6) during migration -12. Has option to [use updateAttributes instead of upsert](#bookmark6) during migration -13. Has option to [clear the data in one or more tables](#bookmark6) before commencing data migration. -14. Has option to [rollback](#bookmark6a) to a previous db version -15. Allows [execution of custom JS files](#Running Custom JS files) pre- and/or post- migration. +9. Has an [option to download the existing data](#bookmark3) in the DB (arbitrary list of tables, or all tables) as a zip file +10. Allows migration of data over http as a [zip file upload](#bookmark4) +11. Allows [download of migration logs](#bookmark5) over http (logs persisted in MigrationLogs table) +12. Has option to [skip oeCloud Validations](#bookmark6) during migration +13. Has option to [use updateAttributes instead of upsert](#bookmark6) during migration +14. Has option to [clear the data in one or more tables](#bookmark6) before commencing data migration. +15. Has option to [rollback](#bookmark6a) to a previous db version +16. Allows [execution of custom JS files](#Running Custom JS files) pre- and/or post- migration. @@ -140,10 +142,12 @@ Once the above changes are done to the application, the migration can be done as | |-default- | | |-Customer.json | | |-Account.json + | | |-custom-script-1.js | | | |-tenant1- | | |-Customer.json | | |-Account.json + | | |-custom-script-2.js | | | |-meta.json | | @@ -158,6 +162,7 @@ Once the above changes are done to the application, the migration can be done as | | | |-tenant1- | | |-Account.json + | | |-custom-script-3.js | | | |-meta.json | | @@ -171,6 +176,7 @@ Once the above changes are done to the application, the migration can be done as | |-tenant1- | |-Account.json + | |-custom-script-4.js | |-meta.json @@ -199,11 +205,11 @@ then the records that existed in the table prior to the current db version migra }); }); ``` - A ready-made file with the above content is available in the [oe-app](http://evgit/oecloud.io/oe-app) sample project at https://evgit/oecloud.io/oe-app/blob/master/server/migrate.js + A ready-made file with the above content is available in the [oe-demo-app](http://evgit/oecloud.io/oe-demo-app) sample project at https://evgit/oecloud.io/oe-demo-app/blob/master/server/migrate.js You can copy this file to your `/server/` folder instead of creating a new file from scratch. This file creation is a one-time activity, and the file itself can be part of your application. - **Note:** This file does not pass the `options` parameter to the `migrate()` function. However, it is possible to configure some aspects of migration if you pass the appropriate `options` object. See [**migrate() function**](#migrate function) under the [**Configuration**](#Configuration) section below, for details. + **Note:** This sample file does not pass the `options` parameter to the `migrate()` function. However, it is possible to configure some aspects of migration if you pass the appropriate `options` object. See [**migrate() function**](#migrate function) under the [**Configuration**](#Configuration) section below, for details. 3. From a command prompt at the root of your application, run the following: @@ -329,29 +335,36 @@ For this to work, ### Running Custom JS files -In cases where migration needs additional complex logic to be executed, you can wrap the `migrate` call in custom javascript module callback. +*oe-migration* supports running arbitrary JS files in addition to loading data from json files. These files are to be placed and configured similar to how this is done for json files, in `meta.json`. See [Configuration](#Configuration) for details. +JS files are run by *oe-migration* in the order it appears in `meta.json`. + +The JS files that should be run as part of migration need to use the following standard: + +1. The JS file/script needs to export a single function +2. The exported function needs to have the following 2 arguments - + + a) opts - This Object would contain the context as defined in `meta.json` + b) cb - This is a callback function that needs to be called from within the script to signal the end of processing in the script. +3. The callback function may be called with an error object. This will halt the migration. +4. Failure to call `cb()` would cause the migration process to wait indefinitely. + +An example JS file is shown below: ```javascript - var app = require('oe-cloud'); - var preMigrate = require('some/path/pre-migrate.js'); - var postMigrate = require('some/path/post-migrate.js'); - app.boot(__dirname, function (err) { - if (err) { console.log(err); process.exit(1); } - var m = require('oe-migration'); - preMigrate('arguments', function(err, data) { - if(err) process.exit(1); - m.migrate(function(err, oldDbVersion, migratedVersions) { - if(err) process.exit(1); - postMigrate('arguments', function(err, data){ - if(err) process.exit(1); else process.exit(0); - } - }); - } - }); +module.exports = function(opts, cb) { + console.log(opts); // contains the ctx as specified in meta.json + // do processing + // more processing + + cb(err); // err should be undefined or null if all is well. + // Otherwise migration is halted. + +} + ``` -You must ensure the rerunnability of your custom javascript code for a rerunnable migration. + @@ -432,7 +445,7 @@ For e.g., one project could have the following files for each of the db versions /db/2.0.0/meta.json ``` -Each of these `meta.json` files define the contexts and data file details for its corresponding DB version migration. In addition, the `meta.json` can also optionally configure - +Each of these `meta.json` files define the contexts and data file/JS script details for its corresponding DB version migration. In addition, the `meta.json` can also optionally configure - - whether all tables or specified tables are cleared or not before migration - whether oeCloud validations are skipped for migration or not - whether the json data is to be used to do an updateAttributes instead of an upsert or not @@ -465,23 +478,23 @@ The structure of the `meta.json` file along with these configuration parameters, "files": [ { - "skipValidation": true, // Optional property, value is boolean. Default: false + "skipValidation": true, // Optional property, value is boolean. Default: false. Ignored if file is JS script // If true, skips oeCloud validation during migration of this file - "updateAttributes": true, // Optional property, value is boolean. Default: false + "updateAttributes": true, // Optional property, value is boolean. Default: false. Ignored if file is JS script // If true, does an updateAttributes instead of an upsert for this file. // If this is set to true, the json data needs to have an 'id' field for each // record. Alternatively, the 'key' property (see below) needs to be specified. - "key": "field2", // Optional property. Value is a string fieldname. Used in conjunction with + "key": "field2", // Optional property. Ignored if file is JS script. Value is a string fieldname. Used in conjunction with // "updateAttributes" (see above) Specifies a unique field other than 'id' to be // used as PK for performing updateAttributes using data in this file. - "model": "Customer", // Mandatory property. The model name to use for this file's migration + "model": "Customer", // Mandatory property if file is json. The model name to use for this json file's migration. Ignored if file is JS script "enabled": true, // Optional property. If false, skips migration from this file. Default: true - "file": "default/customer.json", // Mandatory property. Relative path under "db" folder to the json file's location + "file": "default/customer.js(on)", // Mandatory property. Relative path under "db" folder to the json/JS file's location "ctxId": "/default" // Mandatory property. Should match one of the keys under "contexts" }, diff --git a/lib/migration.js b/lib/migration.js index b8d6f89..1f55afe 100644 --- a/lib/migration.js +++ b/lib/migration.js @@ -303,30 +303,32 @@ function migrateFromPath(migrationPath, cb) { } function proceedWithMigration() { - var model = loopback.findModel(value.model); - if (!model) { - msg = value.model + ' model not found in application'; - mLog = {'logType': 'FATAL', 'model': value.model, 'dbVersion': dbVersion, 'filePath': filePath, - 'migrationDate': new Date(), 'tenant': value.ctxId, 'log': {'message': msg}}; - err = new Error(msg); - err.mLog = mLog; - return asyncCb(err); - } - if (value.skipValidation === true) { - // eslint-disable-next-line no-console - console.log('Skipping validations for model ' + value.model + ' (tenant ' + value.ctxId + ')'); - /* istanbul ignore else */ - if (!validationFunctions[value.model]) { - validationFunctions[value.model] = model.prototype.isValid; - model.prototype.isValid = function (done) { - done(true); - }; + if (filePath.endsWith('json')) { + var model = loopback.findModel(value.model); + if (!model) { + msg = value.model + ' model not found in application'; + mLog = {'logType': 'FATAL', 'model': value.model, 'dbVersion': dbVersion, 'filePath': filePath, + 'migrationDate': new Date(), 'tenant': value.ctxId, 'log': {'message': msg}}; + err = new Error(msg); + err.mLog = mLog; + return asyncCb(err); + } + if (value.skipValidation === true) { + // eslint-disable-next-line no-console + console.log('Skipping validations for model ' + value.model + ' (tenant ' + value.ctxId + ')'); + /* istanbul ignore else */ + if (!validationFunctions[value.model]) { + validationFunctions[value.model] = model.prototype.isValid; + model.prototype.isValid = function (done) { + done(true); + }; + } + } else if (validationFunctions[value.model]) { + model.prototype.isValid = validationFunctions[value.model]; + delete validationFunctions[value.model]; + // eslint-disable-next-line no-console + console.log('Restoring validation for model ' + value.model + ' (tenant ' + value.ctxId + ')'); } - } else if (validationFunctions[value.model]) { - model.prototype.isValid = validationFunctions[value.model]; - delete validationFunctions[value.model]; - // eslint-disable-next-line no-console - console.log('Restoring validation for model ' + value.model + ' (tenant ' + value.ctxId + ')'); } var dataList; try { @@ -343,6 +345,24 @@ function migrateFromPath(migrationPath, cb) { if (Object.prototype.toString.call(dataList) === '[object Object]') { dataList = [dataList]; } + if (Object.prototype.toString.call(dataList) === '[object Function]') { + dataList(opts, function (err) { + if (err) { + // eslint-disable-next-line no-console + console.error(err.message || err); + logMigration({'logType': 'ERROR', 'model': null, 'dbVersion': dbVersion, 'filePath': filePath, + 'migrationDate': new Date(), 'tenant': value.ctxId, 'log': {'message': err.message}}); + return asyncCb(err); + } + // eslint-disable-next-line no-console + console.log('Executed script ' + filePath + ' for tenant ' + value.ctxId); + + logMigration({'logType': 'INFO', 'model': null, 'dbVersion': dbVersion, 'filePath': filePath, + 'migrationDate': new Date(), 'tenant': value.ctxId, 'log': {'message': {filePath: filePath}}}); + return asyncCb(); + }); + return; + } var success = 0; var failed = 0; async.eachSeries(dataList, function (data, asyncCb2) { diff --git a/test/db1/25.0.1/default/migration-test1.js b/test/db1/25.0.1/default/migration-test1.js new file mode 100644 index 0000000..a491dd3 --- /dev/null +++ b/test/db1/25.0.1/default/migration-test1.js @@ -0,0 +1,4 @@ +module.exports = function(opts, cb) { + console.log(opts); + cb(); +} diff --git a/test/db1/25.0.1/default/migration-test2.js b/test/db1/25.0.1/default/migration-test2.js new file mode 100644 index 0000000..a491dd3 --- /dev/null +++ b/test/db1/25.0.1/default/migration-test2.js @@ -0,0 +1,4 @@ +module.exports = function(opts, cb) { + console.log(opts); + cb(); +} diff --git a/test/db1/25.0.1/meta.json b/test/db1/25.0.1/meta.json new file mode 100644 index 0000000..8839792 --- /dev/null +++ b/test/db1/25.0.1/meta.json @@ -0,0 +1,20 @@ +{ + "contexts": { + "/default": { + "id": 0, + "tenantId": "/default", + "remoteUser": "defaultuser" + } + }, + "files": [{ + "enabled": true, + "file": "default/migration-test1.js", + "ctxId": "/default" + }, + { + "enabled": true, + "file": "default/migration-test2.js", + "ctxId": "/default" + } + ] +} \ No newline at end of file diff --git a/test/test.js b/test/test.js index f56ef8d..a89bc5e 100644 --- a/test/test.js +++ b/test/test.js @@ -1107,6 +1107,24 @@ describe(chalk.blue('oe-migration tests'), function (done) { }); }); + + it('should successfuly perform migration with js files', function (done) { + var TAG = '[should successfuly perform migration with js files]'; + this.timeout(10000); + console.log(chalk.yellow('[' + new Date().toISOString() + '] : ', 'Starting ' + TAG)); + var options = {verbose: true}; + + migration.setBasePath(path.resolve(process.cwd(), 'test', 'db')); + copyRecursiveSync(path.join(process.cwd(), 'test', 'db1', '25.0.1'), path.join(basePath, '25.0.1')); + + migration.migrate(options, function (err, oldDbVersion, data) { + expect(oldDbVersion).to.equal('25.0.0'); + expect(data.migratedVersions).to.eql(['25.0.1']); + done(); + }); + }); + + it('The new data should have the default values populated', function (done) { var TAG = '[The new data should have the default values populated]'; this.timeout(1000000); @@ -1145,7 +1163,7 @@ describe(chalk.blue('oe-migration tests'), function (done) { expect(err).to.be.defined; expect(err.message).to.contain('Model MigrationTest4 specified in'); expect(err.message).to.contain('does not exist'); - expect(oldDbVersion).to.equal('25.0.0'); + expect(oldDbVersion).to.equal('25.0.1'); expect(data.migratedVersions).to.be.null; done(); }); From 1904349305e19a457cb9e74716da84ea0b9d2573 Mon Sep 17 00:00:00 2001 From: Ajith Date: Fri, 31 Jan 2020 14:43:04 +0530 Subject: [PATCH 07/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5cee189..24ea075 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ This migration module has the following features: 3. Allows loading of data from [JSON files present in the project's app-list modules](#bookmark2b), to database -4. Allows execution of JS files for custom processing and migration to database +4. Allows [execution of JS files](#Running Custom JS files) for custom processing and migration to database 5. Allows [versioned data migration](#bookmark2), with data-folder name as the DB version. (de-linked from app package version). From 38946263cac387d09afe8e364c834db65a260cd7 Mon Sep 17 00:00:00 2001 From: Ajith Date: Fri, 31 Jan 2020 14:55:16 +0530 Subject: [PATCH 08/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 24ea075..f749e78 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,7 @@ The `server/migrate.js` can be executed repeatedly with/without additional data ### Migration of app-list module seed data -*oe-migration* supports the migration of *app-list* modules by leveraging the `options.basePath` parameter of the `migrate()` function. +*oe-migration* supports the migration of seed-data from *app-list* modules by leveraging the `options.basePath` parameter of the `migrate()` function. For this to work, From 22df05181a2355c276115591a8f0ef3f895580e4 Mon Sep 17 00:00:00 2001 From: vamsee Date: Mon, 2 Mar 2020 15:38:48 +0530 Subject: [PATCH 09/10] remove oracledb tar for oracle ci --- .gitlab-ci.yml | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41b8089..2e2d945 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -108,7 +108,7 @@ oracletest: # - export DB_NAME=${CI_JOB_ID}_pg # - time npm install git+http://evgit/oecloud.io/oe-connector-oracle.git#master --no-optional - time npm install --no-optional - - mv /oracledb node_modules/ + # - mv /oracledb node_modules/ - node test/oracle-utility.js # - export ORACLE_USERNAME=${group}"_"${project} # - export ORACLE_USERNAME=$(echo $ORACLE_USERNAME | tr '[:lower:]' '[:upper:]') diff --git a/package.json b/package.json index 93b1f74..6850263 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "oe-migration", - "version": "2.1.0", + "version": "2.2.0", "description": "This module is an app-list module for oe-Cloud based applications. It provides the ability to perform data migration and/or data seeding into the app db from json files", "engines": { "node": ">=6" From f0ee3f63e465b4bc82e2fd361b3d87477a6c57e2 Mon Sep 17 00:00:00 2001 From: vamsee Date: Mon, 3 Aug 2020 00:17:04 +0530 Subject: [PATCH 10/10] commit for version 2.3.0 --- package.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 6850263..8c9b0a0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "oe-migration", - "version": "2.2.0", + "version": "2.3.0", "description": "This module is an app-list module for oe-Cloud based applications. It provides the ability to perform data migration and/or data seeding into the app db from json files", "engines": { "node": ">=6" @@ -29,11 +29,11 @@ "grunt-mocha-istanbul": "5.0.2", "istanbul": "0.4.5", "mocha": "5.2.0", - "oe-cloud": "git+http://evgit/oecloud.io/oe-cloud.git#master", - "oe-connector-mongodb": "git+http://evgit/oecloud.io/oe-connector-mongodb.git#master", - "oe-connector-oracle": "git+http://evgit/oecloud.io/oe-connector-oracle.git#master", - "oe-connector-postgresql": "git+http://evgit/oecloud.io/oe-connector-postgresql.git#master", - "oe-multi-tenancy": "git+http://evgit/oecloud.io/oe-multi-tenancy.git#master", + "oe-cloud": "git+http://evgit/oecloud.io/oe-cloud.git#2.3.0", + "oe-connector-mongodb": "git+http://evgit/oecloud.io/oe-connector-mongodb.git#2.2.0", + "oe-connector-oracle": "git+http://evgit/oecloud.io/oe-connector-oracle.git#2.3.0", + "oe-connector-postgresql": "git+http://evgit/oecloud.io/oe-connector-postgresql.git#2.3.0", + "oe-multi-tenancy": "git+http://evgit/oecloud.io/oe-multi-tenancy.git#2.3.0", "superagent-defaults": "0.1.14", "supertest": "3.4.2" },