merged changes

packages/contentstack-export/README.md

@@ -48,7 +48,7 @@ $ npm install -g @contentstack/cli-cm-export
 $ csdx COMMAND
 running command...
 $ csdx (--version)
-@contentstack/cli-cm-export/1.11.1 darwin-arm64 node-v20.8.0
+@contentstack/cli-cm-export/1.12.0 darwin-arm64 node-v20.8.0
 $ csdx --help [COMMAND]
 USAGE
   $ csdx COMMAND

packages/contentstack-export/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@contentstack/cli-cm-export",
   "description": "Contentstack CLI plugin to export content from stack",
-  "version": "1.11.1",
+  "version": "1.12.0",
   "author": "Contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {

packages/contentstack-export/src/utils/export-config-handler.ts

@@ -64,7 +64,7 @@ const setupConfig = async (exportCmdFlags: any): Promise<ExportConfig> => {
   config.source_stack = config.apiKey;
 
   config.forceStopMarketplaceAppsPrompt = exportCmdFlags.yes;
-  config.auth_token = configHandler.get('authtoken'); // TBD remove once dependent modules are updated
+  config.auth_token = configHandler.get('authtoken'); // TBD handle auth token in httpClient & sdk
   config.isAuthenticated = isAuthenticated();
 
   if (exportCmdFlags['branch']) {

packages/contentstack-import/README.md

@@ -47,7 +47,7 @@ $ npm install -g @contentstack/cli-cm-import
 $ csdx COMMAND
 running command...
 $ csdx (--version)
-@contentstack/cli-cm-import/1.15.0 darwin-arm64 node-v20.8.0
+@contentstack/cli-cm-import/1.16.0 darwin-arm64 node-v20.8.0
 $ csdx --help [COMMAND]
 USAGE
   $ csdx COMMAND

packages/contentstack-import/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@contentstack/cli-cm-import",
   "description": "Contentstack CLI plugin to import content into stack",
-  "version": "1.15.0",
+  "version": "1.16.0",
   "author": "Contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {

packages/contentstack-import/src/config/index.ts

@@ -154,7 +154,8 @@ const config: DefaultConfig = {
     personalization: {
       importData: true,
       dirName: 'personalization',
-      importOrder: ['projects', 'attributes', 'audiences', 'experiences'],
+      importOrder: ['projects', 'attributes', 'audiences', 'events', 'experiences'],
+      project_id: '',
       projects: {
         dirName: 'projects',
         fileName: 'projects.json',
@@ -174,6 +175,8 @@ const config: DefaultConfig = {
       experiences: {
         dirName: 'experiences',
         fileName: 'experiences.json',
+        thresholdTimer: 60000,
+        checkIntervalDuration: 10000,
       },
     },
     variantEntry: {

packages/contentstack-import/src/import/modules/personalization.ts

@@ -23,6 +23,7 @@ export default class ImportPersonalization {
           projects: Import.Project,
           attributes: Import.Attribute,
           audiences: Import.Audiences,
+          events: Import.Events,
           experiences: Import.Experiences
         };
 

packages/contentstack-import/src/types/default-config.ts

@@ -127,6 +127,7 @@ export default interface DefaultConfig {
       dirName: string;
       importData: boolean;
       importOrder: string[];
+      project_id?: string;
       projects: {
         dirName: string;
         fileName: string;
@@ -146,6 +147,8 @@ export default interface DefaultConfig {
       experiences: {
         dirName: string;
         fileName: string;
+        thresholdTimer: number;
+        checkIntervalDuration: number;
       };
     };
     variantEntry: {

packages/contentstack-import/src/utils/import-config-handler.ts

@@ -70,6 +70,7 @@ const setupConfig = async (importCmdFlags: any): Promise<ImportConfig> => {
   }
 
   config.isAuthenticated = isAuthenticated();
+  config.auth_token = configHandler.get('authtoken'); // TBD handle auth token in httpClient & sdk
 
   //Note to support the old key
   config.source_stack = config.apiKey;

packages/contentstack-variants/src/import/attribute.ts

@@ -6,6 +6,7 @@ import { APIConfig, AttributeStruct, ImportConfig } from '../types';
 
 export default class Attribute extends PersonalizationAdapter<ImportConfig> {
   private mapperDirPath: string;
+  private attrMapperDirPath: string;
   private attributesUidMapperPath: string;
   private attributesUidMapper: Record<string, unknown>;
   private personalizationConfig: ImportConfig['modules']['personalization'];
@@ -15,13 +16,14 @@ export default class Attribute extends PersonalizationAdapter<ImportConfig> {
     const conf: APIConfig = {
       config,
       baseURL: config.personalizationHost,
-      headers: { 'X-Project-Uid': config.project_id, authtoken: config.auth_token },
+      headers: { 'X-Project-Uid': config.modules.personalization.project_id, authtoken: config.auth_token },
     };
     super(Object.assign(config, conf));
     this.personalizationConfig = this.config.modules.personalization;
     this.attributeConfig = this.personalizationConfig.attributes;
     this.mapperDirPath = resolve(this.config.backupDir, 'mapper', this.personalizationConfig.dirName);
-    this.attributesUidMapperPath = resolve(this.mapperDirPath, this.attributeConfig.dirName, 'uid-mapping.json');
+    this.attrMapperDirPath = resolve(this.mapperDirPath, this.attributeConfig.dirName);
+    this.attributesUidMapperPath = resolve(this.attrMapperDirPath, 'uid-mapping.json');
     this.attributesUidMapper = {};
   }
 
@@ -31,7 +33,7 @@ export default class Attribute extends PersonalizationAdapter<ImportConfig> {
   async import() {
     log(this.config, this.$t(this.messages.IMPORT_MSG, { module: 'Attributes' }), 'info');
 
-    await fsUtil.makeDirectory(this.mapperDirPath);
+    await fsUtil.makeDirectory(this.attrMapperDirPath);
     const { dirName, fileName } = this.attributeConfig;
     const attributesPath = resolve(this.config.data, this.personalizationConfig.dirName, dirName, fileName);
 

packages/contentstack-variants/src/import/audiences.ts

@@ -6,6 +6,7 @@ import { PersonalizationAdapter, fsUtil, log, lookUpAttributes } from '../utils'
 
 export default class Audiences extends PersonalizationAdapter<ImportConfig> {
   private mapperDirPath: string;
+  private audienceMapperDirPath: string;
   private attributesMapperPath: string;
   private audiencesUidMapperPath: string;
   private audiencesUidMapper: Record<string, unknown>;
@@ -17,14 +18,15 @@ export default class Audiences extends PersonalizationAdapter<ImportConfig> {
     const conf: APIConfig = {
       config,
       baseURL: config.personalizationHost,
-      headers: { authtoken: config.auth_token, 'X-Project-Uid': config.project_id },
+      headers: { 'X-Project-Uid': config.modules.personalization.project_id, authtoken: config.auth_token },
     };
     super(Object.assign(config, conf));
     this.personalizationConfig = this.config.modules.personalization;
     this.audienceConfig = this.personalizationConfig.audiences;
     this.attributeConfig = this.personalizationConfig.attributes;
     this.mapperDirPath = resolve(this.config.backupDir, 'mapper', this.personalizationConfig.dirName);
-    this.audiencesUidMapperPath = resolve(this.mapperDirPath, this.audienceConfig.dirName, 'uid-mapping.json');
+    this.audienceMapperDirPath = resolve(this.mapperDirPath, this.audienceConfig.dirName);
+    this.audiencesUidMapperPath = resolve(this.audienceMapperDirPath, 'uid-mapping.json');
     this.attributesMapperPath = resolve(this.mapperDirPath, this.attributeConfig.dirName, 'uid-mapping.json');
     this.audiencesUidMapper = {};
   }
@@ -35,7 +37,7 @@ export default class Audiences extends PersonalizationAdapter<ImportConfig> {
   async import() {
     log(this.config, this.$t(this.messages.IMPORT_MSG, { module: 'Audiences' }), 'info');
 
-    await fsUtil.makeDirectory(this.mapperDirPath);
+    await fsUtil.makeDirectory(this.audienceMapperDirPath);
     const { dirName, fileName } = this.audienceConfig;
     const audiencesPath = resolve(this.config.data, this.personalizationConfig.dirName, dirName, fileName);
 

packages/contentstack-variants/src/import/events.ts

@@ -2,10 +2,11 @@ import { resolve } from 'path';
 import { existsSync } from 'fs';
 
 import { PersonalizationAdapter, fsUtil, log } from '../utils';
-import { APIConfig, AttributeStruct, ImportConfig } from '../types';
+import { APIConfig, EventStruct, ImportConfig } from '../types';
 
-export default class EventsImport extends PersonalizationAdapter<ImportConfig> {
+export default class Events extends PersonalizationAdapter<ImportConfig> {
   private mapperDirPath: string;
+  private eventMapperDirPath: string;
   private eventsUidMapperPath: string;
   private eventsUidMapper: Record<string, unknown>;
   private personalizationConfig: ImportConfig['modules']['personalization'];
@@ -15,18 +16,14 @@ export default class EventsImport extends PersonalizationAdapter<ImportConfig> {
     const conf: APIConfig = {
       config,
       baseURL: config.personalizationHost,
-      headers: { 'X-Project-Uid': config.project_id, authtoken: config.auth_token },
+      headers: { 'X-Project-Uid': config.modules.personalization.project_id, authtoken: config.auth_token },
     };
     super(Object.assign(config, conf));
     this.personalizationConfig = this.config.modules.personalization;
-    this.eventsConfig = this.personalizationConfig.attributes;
-    this.mapperDirPath = resolve(
-      this.config.backupDir,
-      'mapper',
-      this.personalizationConfig.dirName,
-      this.eventsConfig.dirName,
-    );
-    this.eventsUidMapperPath = resolve(this.mapperDirPath, 'uid-mapping.json');
+    this.eventsConfig = this.personalizationConfig.events;
+    this.mapperDirPath = resolve(this.config.backupDir, 'mapper', this.personalizationConfig.dirName);
+    this.eventMapperDirPath = resolve(this.mapperDirPath, this.eventsConfig.dirName);
+    this.eventsUidMapperPath = resolve(this.eventMapperDirPath, 'uid-mapping.json');
     this.eventsUidMapper = {};
   }
 
@@ -36,13 +33,13 @@ export default class EventsImport extends PersonalizationAdapter<ImportConfig> {
   async import() {
     log(this.config, this.$t(this.messages.IMPORT_MSG, { module: 'Events' }), 'info');
 
-    await fsUtil.makeDirectory(this.mapperDirPath);
+    await fsUtil.makeDirectory(this.eventMapperDirPath);
     const { dirName, fileName } = this.eventsConfig;
     const eventsPath = resolve(this.config.data, this.personalizationConfig.dirName, dirName, fileName);
 
     if (existsSync(eventsPath)) {
       try {
-        const events = fsUtil.readFile(eventsPath, true) as AttributeStruct[];
+        const events = fsUtil.readFile(eventsPath, true) as EventStruct[];
 
         for (const event of events) {
           const { key, description, uid } = event;

packages/contentstack-variants/src/import/experiences.ts

@@ -1,34 +1,56 @@
 import { resolve } from 'path';
 import { existsSync } from 'fs';
+import values from 'lodash/values';
+import cloneDeep from 'lodash/cloneDeep';
 
 import { APIConfig, ImportConfig, ExperienceStruct, CreateExperienceInput } from '../types';
 import { PersonalizationAdapter, fsUtil, log, lookUpAudiences, lookUpEvents } from '../utils';
 
 export default class Experiences extends PersonalizationAdapter<ImportConfig> {
   private mapperDirPath: string;
+  private cmsVariantPath: string;
+  private failedCmsExpPath: string;
+  private expMapperDirPath: string;
   private eventsMapperPath: string;
   private audiencesMapperPath: string;
+  private cmsVariantGroupPath: string;
+  private expThresholdTimer: number;
+  private maxValidateRetry: number;
   private experiencesUidMapperPath: string;
-  private experiencesUidMapper: Record<string, unknown>;
+  private expCheckIntervalDuration: number;
+  private cmsVariants: Record<string, unknown>;
+  private cmsVariantGroups: Record<string, unknown>;
+  private experiencesUidMapper: Record<string, string>;
+  private pendingVariantAndVariantGrpForExperience: string[];
   private personalizationConfig: ImportConfig['modules']['personalization'];
-  public audienceConfig: ImportConfig['modules']['personalization']['audiences'];
+  private audienceConfig: ImportConfig['modules']['personalization']['audiences'];
   private experienceConfig: ImportConfig['modules']['personalization']['experiences'];
 
   constructor(public readonly config: ImportConfig) {
     const conf: APIConfig = {
       config,
       baseURL: config.personalizationHost,
-      headers: { authtoken: config.auth_token, 'X-Project-Uid': config.project_id },
+      headers: { 'X-Project-Uid': config.modules.personalization.project_id, authtoken: config.auth_token },
     };
     super(Object.assign(config, conf));
     this.personalizationConfig = this.config.modules.personalization;
     this.experienceConfig = this.personalizationConfig.experiences;
     this.audienceConfig = this.personalizationConfig.audiences;
     this.mapperDirPath = resolve(this.config.backupDir, 'mapper', this.personalizationConfig.dirName);
-    this.experiencesUidMapperPath = resolve(this.mapperDirPath, this.experienceConfig.dirName, 'uid-mapping.json');
+    this.expMapperDirPath = resolve(this.mapperDirPath, this.experienceConfig.dirName);
+    this.experiencesUidMapperPath = resolve(this.expMapperDirPath, 'uid-mapping.json');
+    this.cmsVariantGroupPath = resolve(this.expMapperDirPath, 'cms-variant-groups.json');
+    this.cmsVariantPath = resolve(this.expMapperDirPath, 'cms-variants.json');
     this.audiencesMapperPath = resolve(this.mapperDirPath, this.audienceConfig.dirName, 'uid-mapping.json');
     this.eventsMapperPath = resolve(this.mapperDirPath, 'events', 'uid-mapping.json');
+    this.failedCmsExpPath = resolve(this.expMapperDirPath, 'failed-cms-experience.json');
     this.experiencesUidMapper = {};
+    this.cmsVariantGroups = {};
+    this.cmsVariants = {};
+    this.expThresholdTimer = this.experienceConfig?.thresholdTimer ?? 60000;
+    this.expCheckIntervalDuration = this.experienceConfig?.checkIntervalDuration ?? 10000;
+    this.maxValidateRetry = Math.round(this.expThresholdTimer / this.expCheckIntervalDuration);
+    this.pendingVariantAndVariantGrpForExperience = [];
   }
 
   /**
@@ -37,7 +59,7 @@ export default class Experiences extends PersonalizationAdapter<ImportConfig> {
   async import() {
     log(this.config, this.$t(this.messages.IMPORT_MSG, { module: 'Experiences' }), 'info');
 
-    await fsUtil.makeDirectory(this.mapperDirPath);
+    await fsUtil.makeDirectory(this.expMapperDirPath);
     const { dirName, fileName } = this.experienceConfig;
     const experiencePath = resolve(this.config.data, this.personalizationConfig.dirName, dirName, fileName);
 
@@ -50,17 +72,28 @@ export default class Experiences extends PersonalizationAdapter<ImportConfig> {
         for (const experience of experiences) {
           const { uid, ...restExperienceData } = experience;
           //check whether reference audience exists or not that referenced in variations having __type equal to AudienceBasedVariation & targeting
-          let experienceReqObj = lookUpAudiences(restExperienceData, audiencesUid);
+          let experienceReqObj: CreateExperienceInput = lookUpAudiences(restExperienceData, audiencesUid);
           //check whether events exists or not that referenced in metrics
           experienceReqObj = lookUpEvents(experienceReqObj, eventsUid);
 
           const expRes = await this.createExperience(experienceReqObj);
           //map old experience uid to new experience uid
           this.experiencesUidMapper[uid] = expRes?.uid ?? '';
         }
-
         fsUtil.writeFile(this.experiencesUidMapperPath, this.experiencesUidMapper);
         log(this.config, this.$t(this.messages.CREATE_SUCCESS, { module: 'Experiences' }), 'info');
+
+        log(this.config, this.messages.VALIDATE_VARIANT_AND_VARIANT_GRP, 'info');
+        this.pendingVariantAndVariantGrpForExperience = values(cloneDeep(this.experiencesUidMapper));
+        const jobRes = await this.validateVariantGroupAndVariantsCreated();
+        fsUtil.writeFile(this.cmsVariantPath, this.cmsVariants);
+        fsUtil.writeFile(this.cmsVariantGroupPath, this.cmsVariantGroups);
+        if (jobRes)
+          log(this.config, this.$t(this.messages.CREATE_SUCCESS, { module: 'Variant & Variant groups' }), 'info');
+
+        if (this.personalizationConfig.importData) {
+          //attach content types in experiences
+        }
       } catch (error: any) {
         if (error?.errorMessage || error?.message || error?.error_message) {
           log(this.config, this.$t(this.messages.CREATE_FAILURE, { module: 'Experiences' }), 'error');
@@ -70,4 +103,46 @@ export default class Experiences extends PersonalizationAdapter<ImportConfig> {
       }
     }
   }
+
+  /**
+   * function to validate if all variant groups and variants have been created using personalization background job
+   * store the variant groups data in mapper/personalization/experiences/cms-variant-groups.json and the variants data
+   * in mapper/personalization/experiences/cms-variants.json. If not, invoke validateVariantGroupAndVariantsCreated after some delay.
+   * @param retryCount Counter to track the number of times the function has been called
+   * @returns
+   */
+  async validateVariantGroupAndVariantsCreated(retryCount = 0): Promise<any> {
+    try {
+      const promises = this.pendingVariantAndVariantGrpForExperience.map(async (expUid) => {
+        const expRes = await this.getExperience(expUid);
+        if (expRes?._cms) {
+          this.cmsVariants[expUid] = expRes._cms?.variants ?? {};
+          this.cmsVariantGroups[expUid] = expRes._cms?.variantGroup ?? {};
+          return expUid; // Return the expUid for filtering later
+        }
+      });
+
+      await Promise.all(promises);
+      retryCount++;
+
+      if (this.pendingVariantAndVariantGrpForExperience?.length) {
+        if (retryCount !== this.maxValidateRetry) {
+          await this.delay(5000);
+          // Filter out the processed elements
+          this.pendingVariantAndVariantGrpForExperience = this.pendingVariantAndVariantGrpForExperience.filter(
+            (uid) => !this.cmsVariants[uid],
+          );
+          return this.validateVariantGroupAndVariantsCreated(retryCount);
+        } else {
+          log(this.config, this.messages.PERSONALIZATION_JOB_FAILURE, 'info');
+          fsUtil.writeFile(this.failedCmsExpPath, this.pendingVariantAndVariantGrpForExperience);
+          return false;
+        }
+      } else {
+        return true;
+      }
+    } catch (error) {
+      throw error;
+    }
+  }
 }

packages/contentstack-variants/src/import/index.ts

@@ -3,6 +3,7 @@ import Attribute from './attribute';
 import Audiences from './audiences';
 import Experiences from './experiences';
 import VariantEntries from './variant-entries';
+import Events from './events';
 
 // NOTE Acting as namespace to avoid the same class name conflicts in other modules
 export const Import = {
@@ -11,4 +12,5 @@ export const Import = {
   Audiences,
   Experiences,
   VariantEntries,
+  Events
 };

packages/contentstack-variants/src/import/project.ts

@@ -30,7 +30,7 @@ export default class Project extends PersonalizationAdapter<ImportConfig> {
         for (const project of projects) {
           const { name, description, connectedStackApiKey } = project;
           const projectRes = await this.createProject({ name, description, connectedStackApiKey });
-          this.config.project_id = projectRes?.uid;
+          this.config.modules.personalization.project_id = projectRes?.uid;
         }
 
         this.log(this.config, this.$t(this.messages.CREATE_SUCCESS, { module: 'Projects' }), 'info');