diff --git a/packages/contentstack-import-setup/.eslintignore b/packages/contentstack-import-setup/.eslintignore new file mode 100644 index 0000000000..72d230bac9 --- /dev/null +++ b/packages/contentstack-import-setup/.eslintignore @@ -0,0 +1,2 @@ +# Build files +./lib \ No newline at end of file diff --git a/packages/contentstack-import-setup/.eslintrc b/packages/contentstack-import-setup/.eslintrc new file mode 100644 index 0000000000..55a92e2b63 --- /dev/null +++ b/packages/contentstack-import-setup/.eslintrc @@ -0,0 +1,56 @@ +{ + "env": { + "node": true + }, + "parser": "@typescript-eslint/parser", + "parserOptions": { + "project": "tsconfig.json", + "sourceType": "module" + }, + "extends": [ + "oclif-typescript", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + "@typescript-eslint/no-unused-vars": [ + "error", + { + "args": "none" + } + ], + "@typescript-eslint/prefer-namespace-keyword": "error", + "@typescript-eslint/quotes": [ + "error", + "single", + { + "avoidEscape": true, + "allowTemplateLiterals": true + } + ], + "semi": "off", + "@typescript-eslint/type-annotation-spacing": "error", + "@typescript-eslint/no-redeclare": "off", + "eqeqeq": [ + "error", + "smart" + ], + "id-match": "error", + "no-eval": "error", + "no-var": "error", + "quotes": "off", + "indent": "off", + "camelcase": "off", + "comma-dangle": "off", + "arrow-parens": "off", + "operator-linebreak": "off", + "object-curly-spacing": "off", + "node/no-missing-import": "off", + "lines-between-class-members": "off", + "padding-line-between-statements": "off", + "@typescript-eslint/ban-ts-ignore": "off", + "unicorn/no-abusive-eslint-disable": "off", + "@typescript-eslint/no-explicit-any": "off", + "unicorn/consistent-function-scoping": "off", + "@typescript-eslint/no-use-before-define": "off" + } +} \ No newline at end of file diff --git a/packages/contentstack-import-setup/.gitignore b/packages/contentstack-import-setup/.gitignore new file mode 100644 index 0000000000..91669fe532 --- /dev/null +++ b/packages/contentstack-import-setup/.gitignore @@ -0,0 +1,14 @@ +*-debug.log +*-error.log +/.nyc_output +/dist +/lib +/tmp +/yarn.lock +node_modules +.DS_Store +coverage +.vscode/ +/lib + +.env diff --git a/packages/contentstack-import-setup/.mocharc.json b/packages/contentstack-import-setup/.mocharc.json new file mode 100644 index 0000000000..18fcb173fa --- /dev/null +++ b/packages/contentstack-import-setup/.mocharc.json @@ -0,0 +1,8 @@ +{ + "require": ["test/helpers/init.js", "ts-node/register", "source-map-support/register"], + "watch-extensions": [ + "ts" + ], + "recursive": true, + "timeout": 5000 +} \ No newline at end of file diff --git a/packages/contentstack-import-setup/.nycrc.json b/packages/contentstack-import-setup/.nycrc.json new file mode 100644 index 0000000000..ec0b32b29f --- /dev/null +++ b/packages/contentstack-import-setup/.nycrc.json @@ -0,0 +1,5 @@ +{ + "inlcude": [ + "lib/**/*.js" + ] +} \ No newline at end of file diff --git a/packages/contentstack-import-setup/.prettierrc b/packages/contentstack-import-setup/.prettierrc new file mode 100644 index 0000000000..ba93fc77d9 --- /dev/null +++ b/packages/contentstack-import-setup/.prettierrc @@ -0,0 +1,7 @@ +{ + "semi": true, + "trailingComma": "all", + "singleQuote": true, + "printWidth": 120, + "tabWidth": 2 +} \ No newline at end of file diff --git a/packages/contentstack-import-setup/LICENSE b/packages/contentstack-import-setup/LICENSE new file mode 100644 index 0000000000..ffb4ad010b --- /dev/null +++ b/packages/contentstack-import-setup/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Contentstack + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/contentstack-import-setup/bin/dev.cmd b/packages/contentstack-import-setup/bin/dev.cmd new file mode 100644 index 0000000000..077b57ae75 --- /dev/null +++ b/packages/contentstack-import-setup/bin/dev.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\dev" %* \ No newline at end of file diff --git a/packages/contentstack-import-setup/bin/dev.js b/packages/contentstack-import-setup/bin/dev.js new file mode 100644 index 0000000000..b2e3af8096 --- /dev/null +++ b/packages/contentstack-import-setup/bin/dev.js @@ -0,0 +1,6 @@ +#!/usr/bin/env node_modules/.bin/ts-node +// eslint-disable-next-line node/shebang, unicorn/prefer-top-level-await +(async () => { + const oclif = await import('@oclif/core'); + await oclif.execute({ development: true, dir: __dirname }); +})(); diff --git a/packages/contentstack-import-setup/bin/run.cmd b/packages/contentstack-import-setup/bin/run.cmd new file mode 100644 index 0000000000..968fc30758 --- /dev/null +++ b/packages/contentstack-import-setup/bin/run.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\run" %* diff --git a/packages/contentstack-import-setup/bin/run.js b/packages/contentstack-import-setup/bin/run.js new file mode 100755 index 0000000000..8baf302391 --- /dev/null +++ b/packages/contentstack-import-setup/bin/run.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node + +// eslint-disable-next-line unicorn/prefer-top-level-await +(async () => { + const oclif = await import('@oclif/core'); + await oclif.execute({ development: false, dir: __dirname }); +})(); diff --git a/packages/contentstack-import-setup/example_config/auth_config.json b/packages/contentstack-import-setup/example_config/auth_config.json new file mode 100644 index 0000000000..e154879235 --- /dev/null +++ b/packages/contentstack-import-setup/example_config/auth_config.json @@ -0,0 +1,18 @@ +{ + "master_locale": { + "name": "English - United States", + "code": "en-us" + }, + "data": "file path", + "target_stack": "bltXXXXXXXXXX", + "branchName": "example1", + "moduleName": "content-types", + "concurrency": 1, + "importConcurrency": 5, + "fetchConcurrency": 5, + "writeConcurrency": 5, + "securedAssets": false, + "developerHubBaseUrl": "", + "createBackupDir": "./temp", + "cliLogsPath": "./tmp" +} \ No newline at end of file diff --git a/packages/contentstack-import-setup/example_config/management_config.json b/packages/contentstack-import-setup/example_config/management_config.json new file mode 100644 index 0000000000..b43edc20e9 --- /dev/null +++ b/packages/contentstack-import-setup/example_config/management_config.json @@ -0,0 +1,16 @@ +{ + "master_locale": { + "name": "English - United States", + "code": "en-us" + }, + "data": "file path", + "branchName": "example1", + "moduleName": "content-types", + "concurrency": 1, + "importConcurrency": 5, + "fetchConcurrency": 5, + "writeConcurrency": 5, + "securedAssets": false, + "developerHubBaseUrl": "", + "cliLogsPath": "./tmp" +} \ No newline at end of file diff --git a/packages/contentstack-import-setup/messages/index.json b/packages/contentstack-import-setup/messages/index.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/packages/contentstack-import-setup/messages/index.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/packages/contentstack-import-setup/package.json b/packages/contentstack-import-setup/package.json new file mode 100644 index 0000000000..3280576606 --- /dev/null +++ b/packages/contentstack-import-setup/package.json @@ -0,0 +1,97 @@ +{ + "name": "@contentstack/cli-cm-import-setup", + "description": "Contentstack CLI plugin to setup the mappers and configurations for the import command", + "version": "1.0.0", + "author": "Contentstack", + "bugs": "https://github.com/contentstack/cli/issues", + "dependencies": { + "@contentstack/cli-command": "~1.3.0", + "@contentstack/cli-utilities": "~1.7.1", + "@contentstack/management": "~1.17.0", + "@oclif/core": "^3.26.5", + "big-json": "^3.2.0", + "bluebird": "^3.7.2", + "chalk": "^4.1.2", + "debug": "^4.1.0", + "fs-extra": "^11.1.1", + "lodash": "^4.17.20", + "marked": "^4.0.17", + "merge": "^2.1.1", + "mkdirp": "^1.0.4", + "promise-limit": "^2.7.0", + "tslib": "^2.4.1", + "uuid": "^9.0.1", + "winston": "^3.7.2" + }, + "devDependencies": { + "@oclif/test": "^2.5.6", + "@types/big-json": "^3.2.0", + "@types/bluebird": "^3.5.38", + "@types/chai": "^4.2.18", + "@types/fs-extra": "^11.0.1", + "@types/mkdirp": "^1.0.2", + "@types/mocha": "^8.2.2", + "@types/node": "^14.14.32", + "@types/sinon": "^10.0.2", + "@types/tar": "^6.1.3", + "@types/uuid": "^9.0.7", + "@typescript-eslint/eslint-plugin": "^5.48.2", + "chai": "^4.2.0", + "eslint": "^8.18.0", + "eslint-config-oclif": "^4.0.0", + "globby": "^10.0.2", + "mocha": "^10.0.0", + "nyc": "^15.1.0", + "oclif": "^3.8.1", + "rimraf": "^2.7.1", + "sinon": "^11.1.1", + "tmp": "^0.2.2", + "ts-node": "^10.9.1", + "typescript": "^4.9.3" + }, + "scripts": { + "build": "npm run clean && npm run compile", + "clean": "rm -rf ./lib ./node_modules tsconfig.build.tsbuildinfo", + "compile": "tsc -b tsconfig.json", + "postpack": "rm -f oclif.manifest.json", + "prepack": "pnpm compile && oclif manifest && oclif readme", + "version": "oclif readme && git add README.md", + "test:report": "tsc -p test && nyc --reporter=lcov --extension .ts mocha --forbid-only \"test/**/*.test.ts\"", + "pretest": "tsc -p test", + "test": "nyc --extension .ts mocha --forbid-only \"test/**/*.test.ts\"", + "posttest": "npm run lint", + "lint": "eslint src/**/*.ts", + "format": "eslint src/**/*.ts --fix", + "test:integration": "mocha --forbid-only \"test/run.test.js\" --integration-test --timeout 60000", + "test:unit": "mocha --forbid-only \"test/unit/*.test.ts\"" + }, + "engines": { + "node": ">=14.0.0" + }, + "files": [ + "/bin", + "/lib", + "/messages", + "/npm-shrinkwrap.json", + "/oclif.manifest.json" + ], + "homepage": "https://github.com/contentstack/cli", + "keywords": [ + "contentstack", + "cli", + "plugin" + ], + "main": "./lib/commands/cm/stacks/import-setup.js", + "license": "MIT", + "oclif": { + "commands": "./lib/commands", + "bin": "csdx", + "repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-import-setup/<%- commandPath %>" + }, + "csdxConfig": { + "shortCommandName": { + "cm:stacks:import": "IMPRTSTP" + } + }, + "repository": "https://github.com/contentstack/cli" +} diff --git a/packages/contentstack-import-setup/src/commands/cm/stacks/import-setup.ts b/packages/contentstack-import-setup/src/commands/cm/stacks/import-setup.ts new file mode 100644 index 0000000000..cd1b01f65e --- /dev/null +++ b/packages/contentstack-import-setup/src/commands/cm/stacks/import-setup.ts @@ -0,0 +1,69 @@ +import path from 'node:path'; +import { Command } from '@contentstack/cli-command'; +import { + messageHandler, + printFlagDeprecation, + managementSDKClient, + flags, + FlagInput, + ContentstackClient, + pathValidator, +} from '@contentstack/cli-utilities'; + +import { ImportConfig } from '../../../types'; +import { setupImportConfig, formatError, log } from '../../../utils'; +import { ImportSetup } from 'src/import'; + +export default class ImportSetupCommand extends Command { + static description = messageHandler.parse('Import content from a stack'); + + static examples: string[] = [ + `csdx cm:stacks:import-setup --stack-api-key --data-dir --modules `, + ]; + + static flags: FlagInput = { + 'stack-api-key': flags.string({ + char: 'k', + description: 'API key of the target stack', + }), + 'data-dir': flags.string({ + char: 'd', + description: 'path and location where data is stored', + }), + alias: flags.string({ + char: 'a', + description: 'alias of the management token', + }), + modules: flags.string({ + required: false, + description: '[optional] specific module name', + }), + }; + + static aliases: string[] = ['cm:import']; + + static usage: string = 'cm:stacks:import [-k ] [-d ] [-a ] [--modules ]'; + + async run(): Promise { + try { + const { flags } = await this.parse(ImportSetupCommand); + let importSetupConfig = await setupImportConfig(flags); + const importSetup = new ImportSetup(importSetupConfig); + await importSetup.start(); + log( + importSetupConfig, + importSetupConfig.stackName + ? `Successfully generated mapper files for the stack named ${importSetupConfig.stackName} with the API key ${importSetupConfig.apiKey}.` + : `Mapper files have been generated for the stack ${importSetupConfig.apiKey} successfully!`, + 'success', + ); + log( + importSetupConfig, + `The mapper files have been stored at '${pathValidator(path.join(importSetupConfig.backupDirPath, 'mapper'))}'`, + 'success', + ); + } catch (error) { + log({ data: '' } as ImportConfig, `Failed to generate mapper files - ${formatError(error)}`, 'error'); + } + } +} diff --git a/packages/contentstack-import-setup/src/config/index.ts b/packages/contentstack-import-setup/src/config/index.ts new file mode 100644 index 0000000000..4fc6eadf9a --- /dev/null +++ b/packages/contentstack-import-setup/src/config/index.ts @@ -0,0 +1,62 @@ +import { entries } from 'lodash'; +import { DefaultConfig } from '../types'; + +const config: DefaultConfig = { + // use below hosts for eu region + // host:'https://eu-api.contentstack.com/v3', + // use below hosts for azure-na region + // host:'https://azure-na-api.contentstack.com/v3', + // use below hosts for azure-eu region + // host:'https://azure-eu-api.contentstack.com/v3', + // use below hosts for gcp-na region + // host:'https://gcp-na-api.contentstack.com', + // pass locale, only to migrate entries from that locale + // not passing `locale` will migrate all the locales present + // locales: ['fr-fr'], + host: 'https://api.contentstack.io/v3', + extensionHost: 'https://app.contentstack.com', + modules: { + customRoles: { + dirName: 'custom-roles', + fileName: 'custom-roles.json', + dependencies: ['environments', 'entries'], + }, + environments: { + dirName: 'environments', + fileName: 'environments.json', + }, + extensions: { + dirName: 'extensions', + fileName: 'extensions.json', + }, + assets: { + dirName: 'assets', + fileName: 'assets.json', + }, + 'content-types': { + dirName: 'content_types', + fileName: 'content_types.json', + dependencies: ['extensions', 'marketplace_apps', 'taxonomies'], + }, + entries: { + dirName: 'entries', + fileName: 'entries.json', + dependencies: ['assets', 'environments', 'marketplace_apps', 'taxonomies'], + }, + 'global-fields': { + dirName: 'global_fields', + fileName: 'globalfields.json', + dependencies: ['marketplace_apps'], + }, + marketplace_apps: { + dirName: 'marketplace_apps', + fileName: 'marketplace_apps.json', + }, + taxonomies: { + dirName: 'taxonomies', + fileName: 'taxonomies.json', + }, + }, +}; + +export default config; diff --git a/packages/contentstack-import-setup/src/import/assets.ts b/packages/contentstack-import-setup/src/import/assets.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/contentstack-import-setup/src/import/base-class.ts b/packages/contentstack-import-setup/src/import/base-class.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/contentstack-import-setup/src/import/content-types.ts b/packages/contentstack-import-setup/src/import/content-types.ts new file mode 100644 index 0000000000..bea9774754 --- /dev/null +++ b/packages/contentstack-import-setup/src/import/content-types.ts @@ -0,0 +1,45 @@ +import * as chalk from 'chalk'; +import { log, fsUtil } from '../utils'; +import { join } from 'path'; +import { ImportConfig, ModuleClassParams } from '../types'; +import ExtensionImportSetup from './extensions'; + +export default class ContentTypesImportSetup { + private config: ImportConfig; + private contentTypeFilePath: string; + private stackAPIClient: ModuleClassParams['stackAPIClient']; + private dependencies: ModuleClassParams['dependencies']; + private contentTypeConfig: ImportConfig['modules']['content-types']; + + constructor({ config, stackAPIClient, dependencies }: ModuleClassParams) { + this.config = config; + this.stackAPIClient = stackAPIClient; + this.dependencies = dependencies; + this.contentTypeConfig = config.modules['content-types']; + } + + /** + * + */ + async start() { + try { + // in content type we need to create mappers for marketplace apps, extension, taxonomies + // we can call the specific import setup for each of these modules + // Call the specific import setup for each module + // todo + // await this.importMarketplaceApps(); + await new ExtensionImportSetup({ + config: this.config, + dependencies: this.dependencies, + stackAPIClient: this.stackAPIClient, + }).start(); + + // todo + // await this.importTaxonomies(); + + log(this.config, chalk.green(`Mapper file created`), 'success'); + } catch (error) { + log(this.config, chalk.red(`Error generating ${error.message}`), 'error'); + } + } +} diff --git a/packages/contentstack-import-setup/src/import/custom-roles.ts b/packages/contentstack-import-setup/src/import/custom-roles.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/contentstack-import-setup/src/import/entries.ts b/packages/contentstack-import-setup/src/import/entries.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/contentstack-import-setup/src/import/extensions.ts b/packages/contentstack-import-setup/src/import/extensions.ts new file mode 100644 index 0000000000..f1b43bccdc --- /dev/null +++ b/packages/contentstack-import-setup/src/import/extensions.ts @@ -0,0 +1,74 @@ +import * as chalk from 'chalk'; +import { log, fsUtil } from '../utils'; +import { join } from 'path'; +import { ImportConfig, ModuleClassParams } from '../types'; +import { isEmpty } from 'lodash'; + +export default class ExtensionImportSetup { + private config: ImportConfig; + private extensionsFilePath: string; + private extensionMapper: Record; + private stackAPIClient: ModuleClassParams['stackAPIClient']; + private dependencies: ModuleClassParams['dependencies']; + private extensionsConfig: ImportConfig['modules']['extensions']; + private mapperDirPath: string; + private extensionsFolderPath: string; + private extUidMapperPath: string; + + constructor({ config, stackAPIClient }: ModuleClassParams) { + this.config = config; + this.stackAPIClient = stackAPIClient; + this.extensionsFilePath = join(this.config.contentDir, 'extensions', 'extensions.json'); + this.extensionsConfig = config.modules.extensions; + this.extUidMapperPath = join(this.config.backupDir, 'mapper', 'extensions', 'uid-mapping.json'); + this.extensionMapper = {}; + } + + /** + * Start the extension import setup + * This method reads the extensions from the content folder and generates a mapper file + * @returns {Promise} + */ + async start() { + try { + const extensions: any = await fsUtil.readFile(this.extensionsFilePath); + if (!isEmpty(extensions)) { + // 2. Create mapper directory + const mapperFilePath = join(this.config.backupDirPath, 'mapper', 'extensions'); + fsUtil.makeDirectory(mapperFilePath); // Use fsUtil + + for (const extension of Object.values(extensions) as any) { + const targetExtension: any = await this.getExtension(extension.title); + if (!targetExtension) { + log(this.config, chalk.red(`Extension with title '${extension.title}' not found in the stack!`), 'error'); + continue; + } + this.extensionMapper[extension.uid] = targetExtension.uid; + } + + await fsUtil.writeFile(this.extUidMapperPath, this.extensionMapper); + + log(this.config, chalk.green(`Mapper file created at '${this.extUidMapperPath}'`), 'success'); + } else { + log(this.config, chalk.red('No extensions found in the content folder!'), 'error'); + } + } catch (error) { + log(this.config, chalk.red(`Error generating extension mapper: ${error.message}`), 'error'); + } + } + + async getExtension(extension: any) { + // Implement this method to get the extension from the stack + return new Promise(async (resolve, reject) => { + const { items: [extensionsInStack] = [] } = + (await this.stackAPIClient + .extension() + .query({ query: { title: extension.title } }) + .findOne() + .catch((error) => { + reject(true); + })) || {}; + resolve(extensionsInStack); + }); + } +} diff --git a/packages/contentstack-import-setup/src/import/global-fields.ts b/packages/contentstack-import-setup/src/import/global-fields.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/contentstack-import-setup/src/import/import-setup.ts b/packages/contentstack-import-setup/src/import/import-setup.ts new file mode 100644 index 0000000000..1a24948c5c --- /dev/null +++ b/packages/contentstack-import-setup/src/import/import-setup.ts @@ -0,0 +1,78 @@ +import { ImportConfig } from '../types'; +import { log } from '../utils'; + +export default class ImportSetup { + protected config: ImportConfig; + public mapperLogic: { [key: string]: string[] } = {}; + + constructor(config: ImportConfig) { + this.config = config; + } + + /** + * Generate mapper logic + * This method generates mapper logic based on the selected modules + * @returns {Promise>} + */ + protected async generateMapperLogic() { + function getAllDependencies(module: string, visited: Set = new Set()): string[] { + if (visited.has(module)) return []; + + visited.add(module); + let dependencies = this.config.modules[module]?.dependencies || []; + + for (const dependency of dependencies) { + dependencies = dependencies.concat(getAllDependencies(dependency, visited)); + } + + return dependencies; + } + + for (const module of this.config.selectedModules) { + const allDependencies = getAllDependencies(module); + this.mapperLogic[module] = Array.from(new Set(allDependencies)); + } + + log(this.config, 'Mapper logic generated:', 'info'); + } + + /** + * Run module imports based on the selected modules + * This method dynamically imports modules based on the selected modules + * and runs the start method of each module + * @returns {Promise} + */ + protected async runModuleImports() { + for (const moduleName in this.mapperLogic) { + try { + const modulePath = `./${moduleName}`; + const { default: ModuleClass } = await import(modulePath); + + const modulePayload = { + config: this.config, + dependencies: this.mapperLogic[moduleName], + }; + + const moduleInstance = new ModuleClass(modulePayload); + await moduleInstance.start(); + } catch (error) { + log(this.config, `Error importing '${moduleName}': ${error.message}`, 'error'); + } + } + } + + /** + * Start the import setup process + * This method generates mapper logic and runs module imports + * based on the selected modules + * @returns {Promise} + */ + async start() { + try { + await this.generateMapperLogic(); + await this.runModuleImports(); + } catch (error) { + log(this.config, `An error occurred during import setup: ${error.message}`, 'error'); + } + } +} diff --git a/packages/contentstack-import-setup/src/import/index.ts b/packages/contentstack-import-setup/src/import/index.ts new file mode 100644 index 0000000000..826826fc42 --- /dev/null +++ b/packages/contentstack-import-setup/src/import/index.ts @@ -0,0 +1 @@ +export { default as ImportSetup } from './import-setup'; diff --git a/packages/contentstack-import-setup/src/types/default-config.ts b/packages/contentstack-import-setup/src/types/default-config.ts new file mode 100644 index 0000000000..9dc9f174dc --- /dev/null +++ b/packages/contentstack-import-setup/src/types/default-config.ts @@ -0,0 +1,52 @@ +import { Modules } from '.'; + +export default interface DefaultConfig { + host: string; + modules: { + customRoles: { + dirName: string; + fileName: string; + dependencies: Modules[]; + }; + environments: { + dirName: string; + fileName: string; + dependencies: Modules[]; + }; + extensions: { + dirName: string; + fileName: string; + dependencies: Modules[]; + }; + assets: { + dirName: string; + fileName: string; + dependencies: Modules[]; + }; + 'content-types': { + dirName: string; + fileName: string; + dependencies: Modules[]; + }; + entries: { + dirName: string; + fileName: string; + dependencies: Modules[]; + }; + 'global-fields': { + dirName: string; + fileName: string; + dependencies: Modules[]; + }; + marketplace_apps: { + dirName: string; + fileName: string; + dependencies: Modules[]; + }; + taxonomies: { + dirName: string; + fileName: string; + dependencies?: Modules[]; + }; + }; +} diff --git a/packages/contentstack-import-setup/src/types/import-config.ts b/packages/contentstack-import-setup/src/types/import-config.ts new file mode 100644 index 0000000000..08af70544b --- /dev/null +++ b/packages/contentstack-import-setup/src/types/import-config.ts @@ -0,0 +1,53 @@ +import { Modules } from '.'; +import DefaultConfig from './default-config'; + +export interface ExternalConfig { + source_stack?: string; + data: string; + fetchConcurrency: number; + writeConcurrency: number; + email?: string; + password?: string; +} + +export default interface ImportConfig extends DefaultConfig, ExternalConfig { + cliLogsPath?: string; + contentDir: string; + data: string; + management_token?: string; + apiKey: string; + forceStopMarketplaceAppsPrompt: boolean; + auth_token?: string; + contentTypes?: string[]; + branches?: branch[]; + branchEnabled?: boolean; + branchDir?: string; + moduleName?: Modules; + master_locale: masterLocale; + headers?: { + api_key: string; + access_token?: string; + authtoken?: string; + 'X-User-Agent': string; + }; + access_token?: string; + isAuthenticated?: boolean; + target_stack?: string; + masterLocale: masterLocale; + backupDir: string; + backupConcurrency?: number; + authtoken?: string; + destinationStackName?: string; + org_uid?: string; + contentVersion: number; + stackName?: string; +} + +type branch = { + uid: string; + source: string; +}; + +type masterLocale = { + code: string; +}; diff --git a/packages/contentstack-import-setup/src/types/index.ts b/packages/contentstack-import-setup/src/types/index.ts new file mode 100644 index 0000000000..c73668c6ed --- /dev/null +++ b/packages/contentstack-import-setup/src/types/index.ts @@ -0,0 +1,102 @@ +import { ContentstackClient } from '@contentstack/cli-utilities'; +import ImportConfig from './default-config'; + +export type ModuleClassParams = { + stackAPIClient: ReturnType; + config: ImportConfig; + dependencies: Modules; +}; +// eslint-disable-next-line @typescript-eslint/no-redeclare +export interface AuthOptions { + contentstackClient: ContentstackClient; +} + +export interface ContentStackManagementClient { + contentstackClient: object; +} + +export interface PrintOptions { + color?: string; +} + +export interface InquirePayload { + type: string; + name: string; + message: string; + choices?: Array; + transformer?: Function; +} + +export interface User { + email: string; + authtoken: string; +} + +export type Modules = + | 'stack' + | 'assets' + | 'locales' + | 'environments' + | 'extensions' + | 'webhooks' + | 'global-fields' + | 'entries' + | 'content-types' + | 'custom-roles' + | 'workflows' + | 'labels' + | 'marketplace-apps' + | 'taxonomies'; + +export interface ExtensionsRecord { + title: string; + uid: string; +} + +export interface MarketplaceAppsConfig { + dirName: string; + fileName: string; + dependencies?: Modules[]; +} + +export interface EnvironmentConfig { + dirName: string; + fileName: string; + dependencies?: Modules[]; +} + +export interface LabelConfig { + dirName: string; + fileName: string; +} + +export interface WebhookConfig { + dirName: string; + fileName: string; +} + +export interface WorkflowConfig { + dirName: string; + fileName: string; + invalidKeys: string[]; +} + +export interface CustomRoleConfig { + dirName: string; + fileName: string; + customRolesLocalesFileName: string; +} + +export interface TaxonomiesConfig { + dirName: string; + fileName: string; + dependencies?: Modules[]; +} +export { default as DefaultConfig } from './default-config'; +export { default as ImportConfig } from './import-config'; + +export type ExtensionType = { + uid: string; + scope: Record; + title: string; +}; diff --git a/packages/contentstack-import-setup/src/utils/backup-handler.ts b/packages/contentstack-import-setup/src/utils/backup-handler.ts new file mode 100755 index 0000000000..8253ad7858 --- /dev/null +++ b/packages/contentstack-import-setup/src/utils/backup-handler.ts @@ -0,0 +1,72 @@ +import * as path from 'path'; +import { copy } from 'fs-extra'; +import { cliux, sanitizePath } from '@contentstack/cli-utilities'; + +import { fileHelper, trace } from './index'; +import { ImportConfig } from '../types'; + +export default async function backupHandler(importConfig: ImportConfig): Promise { + if (importConfig.hasOwnProperty('useBackedupDir')) { + return importConfig.useBackedupDir; + } + + let backupDirPath: string; + const subDir = isSubDirectory(importConfig); + + if (subDir) { + backupDirPath = path.resolve( + sanitizePath(importConfig.contentDir), + '..', + '_backup_' + Math.floor(Math.random() * 1000), + ); + if (importConfig.createBackupDir) { + cliux.print( + `Warning!!! Provided backup directory path is a sub directory of the content directory, Cannot copy to a sub directory. Hence new backup directory created - ${backupDirPath}`, + { + color: 'yellow', + }, + ); + } + } else { + // NOTE: If the backup folder's directory is provided, create it at that location; otherwise, the default path (working directory). + backupDirPath = path.join(process.cwd(), '_backup_' + Math.floor(Math.random() * 1000)); + if (importConfig.createBackupDir) { + if (fileHelper.fileExistsSync(importConfig.createBackupDir)) { + fileHelper.removeDirSync(importConfig.createBackupDir); + } + fileHelper.makeDirectory(importConfig.createBackupDir); + backupDirPath = importConfig.createBackupDir; + } + } + + if (backupDirPath) { + cliux.print('Copying content to the backup directory...'); + return new Promise((resolve, reject) => { + return copy(importConfig.contentDir, backupDirPath, (error: any) => { + if (error) { + trace(error, 'error', true); + return reject(error); + } + resolve(backupDirPath); + }); + }); + } +} + +/** + * Check whether provided backup directory path is sub directory or not + * @param importConfig + * @returns + */ +function isSubDirectory(importConfig: ImportConfig) { + const parent = importConfig.contentDir; + const child = importConfig.createBackupDir ? importConfig.createBackupDir : process.cwd(); + const relative = path.relative(parent, child); + + if (relative) { + return !relative.startsWith('..') && !path.isAbsolute(relative); + } + + // true if both parent and child have same path + return true; +} diff --git a/packages/contentstack-import-setup/src/utils/common-helper.ts b/packages/contentstack-import-setup/src/utils/common-helper.ts new file mode 100644 index 0000000000..fceafe2d1b --- /dev/null +++ b/packages/contentstack-import-setup/src/utils/common-helper.ts @@ -0,0 +1,21 @@ +export const formatError = (error: any) => { + try { + if (typeof error === 'string') { + error = JSON.parse(error); + } else { + error = JSON.parse(error.message); + } + } catch (e) {} + let message = error?.errorMessage || error?.error_message || error?.message || error; + if (error && error.errors && Object.keys(error.errors).length > 0) { + Object.keys(error.errors).forEach((e) => { + let entity = e; + if (e === 'authorization') entity = 'Management Token'; + if (e === 'api_key') entity = 'Stack API key'; + if (e === 'uid') entity = 'Content Type'; + if (e === 'access_token') entity = 'Delivery Token'; + message += ' ' + [entity, error.errors[e]].join(' '); + }); + } + return message; +}; diff --git a/packages/contentstack-import-setup/src/utils/file-helper.ts b/packages/contentstack-import-setup/src/utils/file-helper.ts new file mode 100644 index 0000000000..e5c613b90d --- /dev/null +++ b/packages/contentstack-import-setup/src/utils/file-helper.ts @@ -0,0 +1,138 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import mkdirp from 'mkdirp'; +import * as bigJSON from 'big-json'; +import { FsUtility, sanitizePath } from '@contentstack/cli-utilities'; + +export const readFileSync = function (filePath: string, parse: boolean = true): any { + let data; + filePath = path.resolve(sanitizePath(filePath)); + if (fs.existsSync(filePath)) { + try { + data = parse ? JSON.parse(fs.readFileSync(filePath, 'utf-8')) : data; + } catch (error) { + return data; + } + } + return data; +}; + +// by default file type is json +export const readFile = async (filePath: string, options = { type: 'json' }): Promise => { + return new Promise((resolve, reject) => { + filePath = path.resolve(sanitizePath(filePath)); + fs.readFile(filePath, 'utf-8', (error, data) => { + if (error) { + if (error.code === 'ENOENT') { + return resolve(''); + } + reject(error); + } else { + if (options.type !== 'json') { + return resolve(data); + } + resolve(JSON.parse(data)); + } + }); + }); +}; + +export const readLargeFile = function (filePath: string, opts?: any): Promise { + if (typeof filePath !== 'string') { + return; + } + filePath = path.resolve(sanitizePath(filePath)); + if (fs.existsSync(filePath)) { + return new Promise((resolve, reject) => { + const readStream = fs.createReadStream(filePath, { encoding: 'utf-8' }); + const parseStream = bigJSON.createParseStream(); + parseStream.on('data', function (data: any) { + if (opts?.type === 'array') { + return resolve(Object.values(data)); + } + resolve(data); + }); + parseStream.on('error', function (error: Error) { + console.log('error', error); + reject(error); + }); + readStream.pipe(parseStream as any); + }); + } +}; + +export const writeFileSync = function (filePath: string, data: any): void { + data = typeof data === 'object' ? JSON.stringify(data) : data || '{}'; + fs.writeFileSync(filePath, data); +}; + +export const writeFile = function (filePath: string, data: any): Promise { + return new Promise((resolve, reject) => { + data = typeof data === 'object' ? JSON.stringify(data) : data || '{}'; + fs.writeFile(filePath, data, (error) => { + if (error) { + return reject(error); + } + resolve('done'); + }); + }); +}; + +export const writeLargeFile = function (filePath: string, data: any): Promise { + if (typeof filePath !== 'string' || typeof data !== 'object') { + return; + } + filePath = path.resolve(sanitizePath(filePath)); + return new Promise((resolve, reject) => { + const stringifyStream = bigJSON.createStringifyStream({ + body: data, + }); + var writeStream = fs.createWriteStream(filePath, 'utf-8'); + stringifyStream.pipe(writeStream); + writeStream.on('finish', () => { + resolve(''); + }); + writeStream.on('error', (error) => { + reject(error); + }); + }); +}; + +export const makeDirectory = function (dir: string): void { + for (let key in arguments) { + const dirname = path.resolve(arguments[key]); + if (!fs.existsSync(dirname)) { + mkdirp.sync(dirname); + } + } +}; + +export const readdirSync = function (dirPath: string): any { + if (fs.existsSync(dirPath)) { + return fs.readdirSync(dirPath); + } else { + return []; + } +}; + +export const isFolderExist = async (folderPath: string): Promise => { + return new Promise((resolve, reject) => { + folderPath = path.resolve(sanitizePath(folderPath)); + fs.access(folderPath, (error) => { + if (error) { + return resolve(false); + } + resolve(true); + }); + }); +}; + +export const fileExistsSync = function (path: string) { + return fs.existsSync(path); +}; + +export const removeDirSync = function (path: string) { + fs.rmdirSync(path, { recursive: true }); +}; + +export const fsUtil = new FsUtility(); diff --git a/packages/contentstack-import-setup/src/utils/import-config-handler.ts b/packages/contentstack-import-setup/src/utils/import-config-handler.ts new file mode 100644 index 0000000000..f266634ba2 --- /dev/null +++ b/packages/contentstack-import-setup/src/utils/import-config-handler.ts @@ -0,0 +1,105 @@ +import merge from 'merge'; +import * as path from 'path'; +import { omit, filter, includes, isArray } from 'lodash'; +import { configHandler, isAuthenticated, cliux, sanitizePath } from '@contentstack/cli-utilities'; +import defaultConfig from '../config'; +import { readFile, fileExistsSync } from './file-helper'; +import { askContentDir, askAPIKey } from './interactive'; +import login from './login-handler'; +import { ImportConfig } from '../types'; + +const setupConfig = async (importCmdFlags: any): Promise => { + let config: ImportConfig = merge({}, defaultConfig); + // setup the config + if (importCmdFlags['config']) { + let externalConfig = await readFile(importCmdFlags['config']); + if (isArray(externalConfig['modules'])) { + config.modules.types = filter(config.modules.types, (module) => includes(externalConfig['modules'], module)); + externalConfig = omit(externalConfig, ['modules']); + } + config = merge.recursive(config, externalConfig); + } + + config.contentDir = importCmdFlags['data'] || importCmdFlags['data-dir'] || config.data || (await askContentDir()); + const pattern = /[*$%#<>{}!&?]/g; + if (pattern.test(config.contentDir)) { + cliux.print(`\nPlease add a directory path without any of the special characters: (*,&,{,},[,],$,%,<,>,?,!)`, { + color: 'yellow', + }); + config.contentDir = await askContentDir(); + } + config.contentDir = config.contentDir.replace(/['"]/g, ''); + config.contentDir = path.resolve(config.contentDir); + //Note to support the old key + config.data = config.contentDir; + if (fileExistsSync(path.join(config.contentDir, 'export-info.json'))) { + config.contentVersion = + ((await readFile(path.join(config.contentDir, 'export-info.json'))) || {}).contentVersion || 2; + } else { + config.contentVersion = 1; + } + + const managementTokenAlias = importCmdFlags['management-token-alias'] || importCmdFlags['alias']; + + if (managementTokenAlias) { + const { token, apiKey } = configHandler.get(`tokens.${managementTokenAlias}`) ?? {}; + config.management_token = token; + config.apiKey = apiKey; + if (!config.management_token) { + throw new Error(`No management token found on given alias ${managementTokenAlias}`); + } + } + + if (!config.management_token) { + if (!isAuthenticated()) { + if (config.email && config.password) { + await login(config); + } else { + throw new Error('Please login or provide an alias for the management token'); + } + } else { + config.apiKey = + importCmdFlags['stack-uid'] || importCmdFlags['stack-api-key'] || config.target_stack || (await askAPIKey()); + if (typeof config.apiKey !== 'string') { + throw new Error('Invalid API key received'); + } + } + } + + config.isAuthenticated = isAuthenticated(); + + //Note to support the old key + config.source_stack = config.apiKey; + + config.skipAudit = importCmdFlags['skip-audit']; + config.forceStopMarketplaceAppsPrompt = importCmdFlags.yes; + config.importWebhookStatus = importCmdFlags['import-webhook-status']; + config.skipPrivateAppRecreationIfExist = importCmdFlags['skip-app-recreation']; + + if (importCmdFlags['branch']) { + config.branchName = importCmdFlags['branch']; + config.branchDir = path.join(sanitizePath(config.contentDir), sanitizePath(config.branchName)); + } + if (importCmdFlags['module']) { + config.moduleName = importCmdFlags['module']; + config.singleModuleImport = true; + } + + if (importCmdFlags['backup-dir']) { + config.useBackedupDir = importCmdFlags['backup-dir']; + } + + // Note to support old modules + config.target_stack = config.apiKey; + + config.replaceExisting = importCmdFlags['replace-existing']; + config.skipExisting = importCmdFlags['skip-existing']; + + if (importCmdFlags['exclude-global-modules']) { + config['exclude-global-modules'] = importCmdFlags['exclude-global-modules']; + } + + return config; +}; + +export default setupConfig; diff --git a/packages/contentstack-import-setup/src/utils/index.ts b/packages/contentstack-import-setup/src/utils/index.ts new file mode 100644 index 0000000000..708fe2ce21 --- /dev/null +++ b/packages/contentstack-import-setup/src/utils/index.ts @@ -0,0 +1,8 @@ +export * as interactive from './interactive'; +export { default as setupImportConfig } from './import-config-handler'; +export * as fileHelper from './file-helper'; +export { fsUtil } from './file-helper'; +export { default as backupHandler } from './backup-handler'; +export { log, unlinkFileLogger } from './logger'; +export * from './log'; +export * from './common-helper'; diff --git a/packages/contentstack-import-setup/src/utils/interactive.ts b/packages/contentstack-import-setup/src/utils/interactive.ts new file mode 100644 index 0000000000..def816e066 --- /dev/null +++ b/packages/contentstack-import-setup/src/utils/interactive.ts @@ -0,0 +1,23 @@ +import { cliux, validatePath } from '@contentstack/cli-utilities'; +import * as path from 'path'; +import first from 'lodash/first'; +import split from 'lodash/split'; + +export const askContentDir = async (): Promise => { + let result = await cliux.inquire({ + type: 'input', + message: 'Enter the path for the content', + name: 'dir', + validate: validatePath, + }); + result = result.replace(/["']/g, ''); + return path.resolve(result); +}; + +export const askAPIKey = async (): Promise => { + return cliux.inquire({ + type: 'input', + message: 'Enter the stack api key', + name: 'apiKey', + }); +}; diff --git a/packages/contentstack-import-setup/src/utils/log.ts b/packages/contentstack-import-setup/src/utils/log.ts new file mode 100644 index 0000000000..aac6d63457 --- /dev/null +++ b/packages/contentstack-import-setup/src/utils/log.ts @@ -0,0 +1,38 @@ +import { join } from 'path'; +import { LogEntry } from 'winston/index'; +import { Logger, pathValidator, sanitizePath } from '@contentstack/cli-utilities'; +import { LogsType, MessageType } from '@contentstack/cli-utilities/lib/logger'; + +import { ImportConfig } from '../types'; + +let logger: Logger; + +export function isImportConfig(config: ImportConfig | MessageType): config is ImportConfig { + return (config as ImportConfig).data !== undefined && (config as ImportConfig)?.contentVersion !== undefined; +} + +export function log(entry: LogEntry): void; +export function log(error: MessageType, logType: LogsType): void; +export function log(error: MessageType, logType: 'error', hidden: boolean): void; +export function log(entryOrMessage: MessageType, logType?: LogsType, hidden?: boolean): Logger | void { + logger = initLogger(); + + if (logType === 'error') { + logger.log(entryOrMessage, logType, hidden); + } else { + logger.log(entryOrMessage, logType); + } +} + +export function initLogger(config?: ImportConfig | undefined) { + if (!logger) { + const basePath = pathValidator(join(sanitizePath(config?.cliLogsPath ?? process.cwd()), 'logs', 'import')); + logger = new Logger(Object.assign(config ?? {}, { basePath })); + } + + return logger; +} + +export { logger }; + +export const trace = log; diff --git a/packages/contentstack-import-setup/src/utils/logger.ts b/packages/contentstack-import-setup/src/utils/logger.ts new file mode 100644 index 0000000000..782e07c2c3 --- /dev/null +++ b/packages/contentstack-import-setup/src/utils/logger.ts @@ -0,0 +1,169 @@ +/*! + * Contentstack Export + * Copyright (c) 2024 Contentstack LLC + * MIT Licensed + */ + +import * as winston from 'winston'; +import * as path from 'path'; +import mkdirp from 'mkdirp'; +import { ImportConfig } from '../types'; +import { sanitizePath } from '@contentstack/cli-utilities'; + +const slice = Array.prototype.slice; + +const ansiRegexPattern = [ + '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)', + '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))', +].join('|'); + +function returnString(args: any[]) { + var returnStr = ''; + if (args && args.length) { + returnStr = args + .map(function (item) { + if (item && typeof item === 'object') { + try { + return JSON.stringify(item).replace(/authtoken\":\d"blt................/g, 'authtoken":"blt....'); + } catch (error) {} + return item; + } + return item; + }) + .join(' ') + .trim(); + } + returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), '').trim(); + return returnStr; +} +var myCustomLevels = { + levels: { + warn: 1, + info: 2, + debug: 3, + }, + colors: { + //colors aren't being used anywhere as of now, we're using chalk to add colors while logging + info: 'blue', + debug: 'green', + warn: 'yellow', + error: 'red', + }, +}; + +let logger: winston.Logger; +let errorLogger: winston.Logger; + +let successTransport; +let errorTransport; + +function init(_logPath: string) { + if (!logger || !errorLogger) { + const logsDir = path.resolve(sanitizePath(_logPath), 'logs', 'import'); + successTransport = { + filename: path.join(sanitizePath(logsDir), 'success.log'), + maxFiles: 20, + maxsize: 1000000, + tailable: true, + level: 'info', + }; + + errorTransport = { + filename: path.join(sanitizePath(logsDir), 'error.log'), + maxFiles: 20, + maxsize: 1000000, + tailable: true, + level: 'error', + }; + + logger = winston.createLogger({ + transports: [ + new winston.transports.File(successTransport), + new winston.transports.Console({ + format: winston.format.combine( + winston.format.simple(), + winston.format.colorize({ all: true, colors: { warn: 'yellow', info: 'white' } }), + ), + }), + ], + levels: myCustomLevels.levels, + }); + + errorLogger = winston.createLogger({ + transports: [ + new winston.transports.File(errorTransport), + new winston.transports.Console({ + level: 'error', + format: winston.format.combine( + winston.format.colorize({ all: true, colors: { error: 'red' } }), + winston.format.simple(), + ), + }), + ], + levels: { error: 0 }, + }); + } + + return { + log: function (message: any) { + let args = slice.call(arguments); + let logString = returnString(args); + if (logString) { + logger.log('info', logString); + } + }, + warn: function (message: any) { + let args = slice.call(arguments); + let logString = returnString(args); + if (logString) { + logger.log('warn', logString); + } + }, + error: function (message: any) { + let args = slice.call(arguments); + let logString = returnString(args); + if (logString) { + errorLogger.log('error', logString); + } + }, + debug: function () { + let args = slice.call(arguments); + let logString = returnString(args); + if (logString) { + logger.log('debug', logString); + } + }, + }; +} + +export const log = async (config: ImportConfig, message: any, type: string) => { + config.cliLogsPath = config.cliLogsPath || config.data || path.join(__dirname, 'logs'); + // ignoring the type argument, as we are not using it to create a logfile anymore + if (type !== 'error') { + // removed type argument from init method + if (type === 'warn') init(config.cliLogsPath).warn(message); //logged warning message in log file + else init(config.cliLogsPath).log(message); + } else { + init(config.cliLogsPath).error(message); + } +}; + +export const unlinkFileLogger = () => { + if (logger) { + const transports = logger.transports; + transports.forEach((transport: any) => { + if (transport.name === 'file') { + logger.remove(transport); + } + }); + } + + if (errorLogger) { + const transports = errorLogger.transports; + transports.forEach((transport: any) => { + if (transport.name === 'file') { + errorLogger.remove(transport); + } + }); + } +}; diff --git a/packages/contentstack-import-setup/src/utils/login-handler.ts b/packages/contentstack-import-setup/src/utils/login-handler.ts new file mode 100644 index 0000000000..996e7797a8 --- /dev/null +++ b/packages/contentstack-import-setup/src/utils/login-handler.ts @@ -0,0 +1,51 @@ +/* eslint-disable max-statements-per-line */ +/* eslint-disable no-console */ +/* eslint-disable no-empty */ +/*! + * Contentstack Import + * Copyright (c) 2024 Contentstack LLC + * MIT Licensed + */ + +import { log } from './logger'; +import { managementSDKClient, isAuthenticated } from '@contentstack/cli-utilities'; +import { ImportConfig } from '../types'; + +const login = async (config: ImportConfig): Promise => { + const client = await managementSDKClient(config); + if (config.email && config.password) { + const { user: { authtoken = null } = {} } = await client.login({ email: config.email, password: config.password }); + if (authtoken) { + config.headers = { + api_key: config.source_stack, + access_token: config.access_token, + authtoken: config.authtoken, + 'X-User-Agent': 'contentstack-export/v', + }; + log(config, 'Contentstack account authenticated successfully!', 'success'); + return config; + } else { + throw new Error('Invalid auth token received after login'); + } + } else if (config.management_token) { + return config; + } else if (isAuthenticated()) { + const stackAPIClient = client.stack({ + api_key: config.target_stack, + management_token: config.management_token, + }); + const stack = await stackAPIClient.fetch().catch((error: any) => { + let errorstack_key = error?.errors?.api_key; + if (errorstack_key) { + log(config, 'Stack Api key ' + errorstack_key[0] + 'Please enter valid Key', 'error'); + throw error; + } + log(config, error?.errorMessage, 'error'); + throw error; + }); + config.destinationStackName = stack.name; + return config; + } +}; + +export default login; diff --git a/packages/contentstack-import-setup/test/config.json b/packages/contentstack-import-setup/test/config.json new file mode 100644 index 0000000000..7e8958a42e --- /dev/null +++ b/packages/contentstack-import-setup/test/config.json @@ -0,0 +1,35 @@ +{ + "IS_TS": false, + "UNIT_EXECUTION_ORDER": [], + "INTEGRATION_EXECUTION_ORDER": [ + "management-token.test.js", + "assets.test.js", + "content-types.test.js", + "custom-roles.test.js", + "entries.test.js", + "environments.test.js", + "extensions.test.js", + "global-fields.test.js", + "locales.test.js", + "webhooks.test.js", + "workflows.test.js", + "auth-token.test.js", + "auth-token-modules/assets.test.js", + "auth-token-modules/content-types.test.js", + "auth-token-modules/custom-roles.test.js", + "auth-token-modules/entries.test.js", + "auth-token-modules/environments.test.js", + "auth-token-modules/extensions.test.js", + "auth-token-modules/global-fields.test.js", + "auth-token-modules/locales.test.js", + "auth-token-modules/webhooks.test.js", + "auth-token-modules/workflows.test.js" + ], + "ENABLE_PREREQUISITES": true, + "REGIONS": [ + "NA", + "EU", + "AZURE-NA", + "AZURE-EU" + ] +} \ No newline at end of file diff --git a/packages/contentstack-import-setup/test/mocha.opts b/packages/contentstack-import-setup/test/mocha.opts new file mode 100644 index 0000000000..c6d1cb290c --- /dev/null +++ b/packages/contentstack-import-setup/test/mocha.opts @@ -0,0 +1,3 @@ +--recursive +--reporter spec +--timeout 5000 diff --git a/packages/contentstack-import-setup/test/run.test.js b/packages/contentstack-import-setup/test/run.test.js new file mode 100644 index 0000000000..d32b19f656 --- /dev/null +++ b/packages/contentstack-import-setup/test/run.test.js @@ -0,0 +1,123 @@ +const { join } = require('path'); +const filter = require('lodash/filter'); +const forEach = require('lodash/forEach'); +const isEmpty = require('lodash/isEmpty'); +const isArray = require('lodash/isArray'); +const includes = require('lodash/includes'); +const { existsSync, readdirSync } = require('fs'); + +const { initEnvData, getLoginCredentials } = require('./integration/utils/helper') +const { INTEGRATION_EXECUTION_ORDER, IS_TS, ENABLE_PREREQUISITES } = require('./config.json'); + +// NOTE init env variables +require('dotenv-expand').expand(require('dotenv').config()); + +initEnvData(); // NOTE Prepare env data + +const args = process.argv.slice(2); +const testFileExtension = IS_TS ? '.ts' : '.js'; + +/** + * @method getFileName + * @param {string} file + * @returns {string} + */ +const getFileName = (file) => { + if (includes(file, '.test') && includes(file, testFileExtension)) return file; + else if (includes(file, '.test')) return `${file}${testFileExtension}`; + else if (!includes(file, '.test')) return `${file}.test${testFileExtension}`; + else return `${file}.test${testFileExtension}`; +}; + +/** + * @method includeInitFileIfExist + * @param {String} basePath + */ +const includeInitFileIfExist = (basePath, region) => { + const filePath = join(__dirname, basePath, `init.test${testFileExtension}`); + + try { + if (existsSync(filePath)) { + require(filePath)(region); + } + } catch (err) { } +}; + +/** + * @method includeCleanUpFileIfExist + * @param {String} basePath + */ +const includeCleanUpFileIfExist = async (basePath, region) => { + const filePath = join(__dirname, basePath, `clean-up.test${testFileExtension}`); + + try { + if (existsSync(filePath)) { + require(filePath)(region); + } + } catch (err) { } +} + +/** + * @method includeTestFiles + * @param {Array} files + * @param {string} basePath + */ +const includeTestFiles = async (files, basePath = 'integration') => { + let regions = getLoginCredentials(); + for (let region of Object.keys(regions)) { + if (ENABLE_PREREQUISITES) { + includeInitFileIfExist(basePath, regions[region]) // NOTE Run all the pre configurations + } + + files = filter(files, (name) => ( + !includes(`init.test${testFileExtension}`, name) && + !includes(`clean-up.test${testFileExtension}`, name) + )) // NOTE remove init, clean-up files + + forEach(files, (file) => { + const filename = getFileName(file); + const filePath = join(__dirname, basePath, filename); + try { + if (existsSync(filePath)) { + require(filePath)(regions[region]); + } else { + console.error(`File not found - ${filename}`); + } + } catch (err) { + console.err(err.message) + } + }); + + await includeCleanUpFileIfExist(basePath, regions[region]) // NOTE run all cleanup code/commands + } +}; + +/** + * @method run + * @param {Array | undefined | null | unknown} executionOrder + * @param {boolean} isIntegrationTest + */ +const run = ( + executionOrder, + isIntegrationTest = true +) => { + const testFolder = isIntegrationTest ? 'integration' : 'unit'; + + if (isArray(executionOrder) && !isEmpty(executionOrder)) { + includeTestFiles(executionOrder, testFolder); + } else { + const basePath = join(__dirname, testFolder); + const allIntegrationTestFiles = filter(readdirSync(basePath), (file) => + includes(file, `.test${testFileExtension}`) + ); + + includeTestFiles(allIntegrationTestFiles); + } +}; + +if (includes(args, '--integration-test')) { + run(INTEGRATION_EXECUTION_ORDER); +} else if (includes(args, '--unit-test')) { + // NOTE unit test case will be handled here + // run(UNIT_EXECUTION_ORDER, false); +} \ No newline at end of file diff --git a/packages/contentstack-import-setup/tsconfig.json b/packages/contentstack-import-setup/tsconfig.json new file mode 100644 index 0000000000..353cb22a4f --- /dev/null +++ b/packages/contentstack-import-setup/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "declaration": true, + "importHelpers": true, + "module": "commonjs", + "rootDir": "src", + "outDir": "lib", + "strict": false, + "target": "es2017", + "allowJs": true, + "skipLibCheck": true, + "sourceMap": false, + "esModuleInterop": true, + "noImplicitAny": true, + "lib": [ + "ES2019", + "es2020.promise" + ], + "strictPropertyInitialization": false, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "src/**/*", + "types/*" + ], + "exclude": [ + "node_modules", + "lib" + ] +} \ No newline at end of file diff --git a/packages/contentstack-import-setup/types/index.d.ts b/packages/contentstack-import-setup/types/index.d.ts new file mode 100644 index 0000000000..68a9e13535 --- /dev/null +++ b/packages/contentstack-import-setup/types/index.d.ts @@ -0,0 +1 @@ +declare module 'big-json';