Merge pull request #21 from contentstack/feat/CS-44636

Feat: variant entry relational data handled [CS-44636]

packages/contentstack-import/src/import/module-importer.ts

@@ -24,8 +24,9 @@ class ModuleImporter {
 
   async start(): Promise<any> {
     if (!this.importConfig.management_token) {
-      const stackName: Record<string, unknown> = await this.stackAPIClient.fetch();
-      this.importConfig.stackName = stackName.name as string;
+      const stackDetails: Record<string, unknown> = await this.stackAPIClient.fetch();
+      this.importConfig.stackName = stackDetails.name as string;
+      this.importConfig.org_uid = stackDetails.org_uid as string;
     }
     if (this.importConfig.branchName) {
       await validateBranch(this.stackAPIClient, this.importConfig, this.importConfig.branchName);

packages/contentstack-import/src/import/modules/personalization.ts

@@ -1,4 +1,4 @@
-import { Import } from '@contentstack/cli-variants';
+import { Import, LogType } from '@contentstack/cli-variants';
 
 import { log } from '../../utils';
 import { ImportConfig, ModuleClassParams } from '../../types';
@@ -24,14 +24,14 @@ export default class ImportPersonalization {
           attributes: Import.Attribute,
           audiences: Import.Audiences,
           events: Import.Events,
-          experiences: Import.Experiences
+          experiences: Import.Experiences,
         };
 
         const order: (keyof typeof moduleMapper)[] = this.personalization.importOrder as (keyof typeof moduleMapper)[];
 
         for (const module of order) {
           const Module = moduleMapper[module];
-          await new Module(this.config).import();
+          await new Module(this.config, log as unknown as LogType).import();
         }
       }
     } catch (error) {

packages/contentstack-variants/src/import/project.ts

@@ -28,8 +28,8 @@ export default class Project extends PersonalizationAdapter<ImportConfig> {
         const projects = JSON.parse(readFileSync(projectPath, 'utf8')) as CreateProjectInput[];
 
         for (const project of projects) {
-          const { name, description, connectedStackApiKey } = project;
-          const projectRes = await this.createProject({ name, description, connectedStackApiKey });
+          const { name, description } = project;
+          const projectRes = await this.createProject({ name, description, connectedStackApiKey: this.config.apiKey });
           this.config.modules.personalization.project_id = projectRes?.uid;
         }
 

packages/contentstack-variants/src/import/variant-entries.ts

@@ -1,3 +1,4 @@
+import omit from 'lodash/omit';
 import chunk from 'lodash/chunk';
 import values from 'lodash/values';
 import entries from 'lodash/entries';
@@ -12,25 +13,39 @@ import {
   LogType,
   APIConfig,
   AdapterType,
+  AnyProperty,
   ImportConfig,
+  ContentTypeStruct,
   VariantEntryStruct,
+  ImportHelperMethodsConfig,
   EntryDataForVariantEntries,
 } from '../types';
 
 export default class VariantEntries extends VariantAdapter<VariantHttpClient<ImportConfig>> {
   public entriesDirPath: string;
+  public entriesMapperPath: string;
   public variantEntryBasePath!: string;
   public variantIdList!: { [key: string]: string };
   public personalizationconfig: ImportConfig['modules']['personalization'];
 
-  constructor(readonly config: ImportConfig, private readonly log: LogType = console.log) {
+  public taxonomies!: AnyProperty;
+  public assetUrlMapper!: AnyProperty;
+  public assetUidMapper!: AnyProperty;
+  public entriesUidMapper!: AnyProperty;
+  private installedExtensions!: AnyProperty[];
+
+  constructor(
+    readonly config: ImportConfig & { helpers?: ImportHelperMethodsConfig },
+    private readonly log: LogType = console.log,
+  ) {
     const conf: APIConfig & AdapterType<VariantHttpClient<ImportConfig>, APIConfig> = {
       config,
       httpClient: true,
       baseURL: config.host,
       Adapter: VariantHttpClient<ImportConfig>,
     };
-    super(Object.assign(config, conf));
+    super(Object.assign(omit(config, ['helpers']), conf));
+    this.entriesMapperPath = resolve(config.backupDir, 'mapper', 'entries');
     this.personalizationconfig = this.config.modules.personalization;
     this.entriesDirPath = resolve(config.backupDir, config.branchName || '', config.modules.entries.dirName);
   }
@@ -44,12 +59,12 @@ export default class VariantEntries extends VariantAdapter<VariantHttpClient<Imp
    * message indicating that no entries were found and return.
    */
   async import() {
-    const filePath = resolve(this.config.backupDir, 'mapper', 'entries', 'data-for-variant-entry.json');
+    const filePath = resolve(this.entriesMapperPath, 'data-for-variant-entry.json');
     const variantIdPath = join(
       this.config.backupDir,
       'mapper',
-      'personalization',
-      'experiences',
+      this.personalizationconfig.dirName,
+      this.personalizationconfig.experiences.dirName,
       'variants-uid-mapping.json',
     );
 
@@ -70,9 +85,24 @@ export default class VariantEntries extends VariantAdapter<VariantHttpClient<Imp
       return;
     }
 
+    const entriesUidMapperPath = join(this.entriesMapperPath, 'uid-mapping.json');
+    const assetUidMapperPath = resolve(this.config.backupDir, 'mapper', 'assets', 'uid-mapping.json');
+    const assetUrlMapperPath = resolve(this.config.backupDir, 'mapper', 'assets', 'url-mapping.json');
+    const taxonomiesPath = join(this.config.backupDir, 'mapper', 'taxonomies', 'terms', 'success.json');
+    const marketplaceAppMapperPath = join(this.config.backupDir, 'mapper', 'marketplace_apps', 'uid-mapping.json');
+
     // NOTE Read and store list of variant IDs
     this.variantIdList = JSON.parse(readFileSync(variantIdPath, 'utf8'));
 
+    // NOTE entry relational data lookup dependencies.
+    this.entriesUidMapper = JSON.parse(await readFileSync(entriesUidMapperPath, 'utf8'));
+    this.installedExtensions = (
+      JSON.parse(await readFileSync(marketplaceAppMapperPath, 'utf8')) || { extension_uid: {} }
+    ).extension_uid;
+    this.taxonomies = existsSync(taxonomiesPath) ? JSON.parse(readFileSync(taxonomiesPath, 'utf8')) : {};
+    this.assetUidMapper = JSON.parse(readFileSync(assetUidMapperPath, 'utf8')) || {};
+    this.assetUrlMapper = JSON.parse(readFileSync(assetUrlMapperPath, 'utf8')) || {};
+
     for (const entriesForVariant of entriesForVariants) {
       await this.importVariantEntries(entriesForVariant);
     }
@@ -86,15 +116,19 @@ export default class VariantEntries extends VariantAdapter<VariantHttpClient<Imp
   async importVariantEntries(entriesForVariant: EntryDataForVariantEntries) {
     const variantEntry = this.config.modules.variantEntry;
     const { content_type, locale, entry_uid } = entriesForVariant;
+    const ctConfig = this.config.modules['content-types'];
+    const contentType: ContentTypeStruct = JSON.parse(
+      readFileSync(resolve(this.config.backupDir, ctConfig.dirName, `${content_type}.json`), 'utf8'),
+    );
     const variantEntryBasePath = join(this.entriesDirPath, content_type, locale, variantEntry.dirName, entry_uid);
     const fs = new FsUtility({ basePath: variantEntryBasePath });
 
     for (const _ in fs.indexFileContent) {
       try {
         const variantEntries = (await fs.readChunkFiles.next()) as VariantEntryStruct;
         if (variantEntries) {
-          let apiContent = orderBy(values(variantEntries), '_version');
-          await this.handleCuncurrency(apiContent, entriesForVariant);
+          const apiContent = orderBy(values(variantEntries), '_version');
+          await this.handleCuncurrency(contentType, apiContent, entriesForVariant);
         }
       } catch (error) {
         this.log(this.config, error, 'error');
@@ -116,11 +150,15 @@ export default class VariantEntries extends VariantAdapter<VariantHttpClient<Imp
    * `Promise.allSettled`. The function also includes logic to handle variant IDs and delays between
    * batch processing.
    */
-  async handleCuncurrency(variantEntries: VariantEntryStruct[], entriesForVariant: EntryDataForVariantEntries) {
+  async handleCuncurrency(
+    contentType: ContentTypeStruct,
+    variantEntries: VariantEntryStruct[],
+    entriesForVariant: EntryDataForVariantEntries,
+  ) {
     let batchNo = 0;
-    const variantEntry = this.config.modules.variantEntry;
+    const variantEntryConfig = this.config.modules.variantEntry;
     const { content_type, locale, entry_uid } = entriesForVariant;
-    const batches = chunk(variantEntries, variantEntry.apiConcurrency | 5);
+    const batches = chunk(variantEntries, variantEntryConfig.apiConcurrency | 5);
 
     if (isEmpty(batches)) return;
 
@@ -133,7 +171,7 @@ export default class VariantEntries extends VariantAdapter<VariantHttpClient<Imp
         // NOTE Find new variant Id by old Id
         const variant_id = this.variantIdList[variantEntry.variant_id];
         // NOTE Replace all the relation data UID's
-        variantEntry = this.handleVariantEntryRelationalData(variantEntry);
+        variantEntry = this.handleVariantEntryRelationalData(contentType, variantEntry);
 
         if (variant_id) {
           const promise = this.variantInstance.createVariantEntry(variantEntry, {
@@ -152,6 +190,8 @@ export default class VariantEntries extends VariantAdapter<VariantHttpClient<Imp
       // NOTE Handle the API response here
       const resultSet = await Promise.allSettled(allPromise);
 
+      this.log(this.config, `Batch No. (${batchNo}/${batches.length}) of variant entry import is complete`, 'success');
+
       // NOTE publish all the entries
       this.publishVariantEntries(resultSet);
 
@@ -161,9 +201,71 @@ export default class VariantEntries extends VariantAdapter<VariantHttpClient<Imp
     }
   }
 
-  handleVariantEntryRelationalData(variantEntry: VariantEntryStruct) {
-    // FIXME: Handle relational data
-    console.log('Handle/Replace variant entry relational data');
+  /**
+   * The function `handleVariantEntryRelationalData` processes relational data for a variant entry
+   * based on the provided content type and configuration helpers.
+   * @param {ContentTypeStruct} contentType - The `contentType` parameter in the
+   * `handleVariantEntryRelationalData` function is of type `ContentTypeStruct`. It is used to define
+   * the structure of the content type being processed within the function. This parameter likely
+   * contains information about the schema and configuration of the content type.
+   * @param {VariantEntryStruct} variantEntry - The `variantEntry` parameter in the
+   * `handleVariantEntryRelationalData` function is a data structure that represents a variant entry.
+   * It is of type `VariantEntryStruct` and contains information related to a specific entry variant.
+   * This function is responsible for performing various operations on the `variantEntry`
+   * @returns The function `handleVariantEntryRelationalData` returns the `variantEntry` after
+   * performing various lookups and replacements on it based on the provided `contentType` and
+   * `config.helpers`.
+   */
+  handleVariantEntryRelationalData(
+    contentType: ContentTypeStruct,
+    variantEntry: VariantEntryStruct,
+  ): VariantEntryStruct {
+    if (this.config.helpers) {
+      const { lookUpTerms, lookupAssets, lookupExtension, lookupEntries, restoreJsonRteEntryRefs } =
+        this.config.helpers;
+
+      // FIXME Not sure why do we even need lookupExtension in entries [Ref taken from entries import]
+      // Feel free to remove this flow if it's not valid
+      // NOTE Find and replace extension's UID
+      if (lookupExtension) {
+        lookupExtension(this.config, contentType.schema, this.config.preserveStackVersion, this.installedExtensions);
+      }
+
+      // NOTE Find and replace RTE Ref UIDs
+      variantEntry = restoreJsonRteEntryRefs(variantEntry, variantEntry, contentType.schema, {
+        uidMapper: this.entriesUidMapper,
+        mappedAssetUids: this.assetUidMapper,
+        mappedAssetUrls: this.assetUrlMapper,
+      }) as VariantEntryStruct;
+
+      // NOTE Find and replace Entry Ref UIDs
+      variantEntry = lookupEntries(
+        {
+          entry: variantEntry,
+          content_type: contentType,
+        },
+        this.entriesUidMapper,
+        join(this.entriesMapperPath, contentType.uid, variantEntry.locale),
+      );
+
+      // NOTE: will remove term if term doesn't exists in taxonomy
+      // FIXME: Validate if taxonomy support available for variant entries,
+      // if not, feel free to remove this lookup flow.
+      lookUpTerms(contentType?.schema, variantEntry, this.taxonomies, this.config);
+
+      // NOTE Find and replace asset's UID
+      variantEntry = lookupAssets(
+        {
+          entry: variantEntry,
+          content_type: contentType,
+        },
+        this.assetUidMapper,
+        this.assetUrlMapper,
+        this.entriesDirPath,
+        this.installedExtensions,
+      );
+    }
+
     return variantEntry;
   }
 

packages/contentstack-variants/src/types/content-types.ts

@@ -0,0 +1,41 @@
+import { AnyProperty } from './utils';
+import { ImportConfig } from './import-config';
+
+export type ContentTypeStruct = {
+  uid: string;
+  title: string;
+  description: string;
+  schema: AnyProperty[];
+  options: AnyProperty;
+} & AnyProperty;
+
+export type ImportHelperMethodsConfig = {
+  lookupAssets: (
+    data: Record<string, any>,
+    mappedAssetUids: Record<string, any>,
+    mappedAssetUrls: Record<string, any>,
+    assetUidMapperPath: string,
+    installedExtensions: Record<string, any>[],
+  ) => any;
+  lookupExtension?: (config: ImportConfig, schema: any, preserveStackVersion: any, installedExtensions: any) => void;
+  restoreJsonRteEntryRefs: (
+    entry: Record<string, any>,
+    sourceStackEntry: any,
+    ctSchema: any,
+    { uidMapper, mappedAssetUids, mappedAssetUrls }: any,
+  ) => Record<string, any>;
+  lookupEntries: (
+    data: {
+      content_type: any;
+      entry: any;
+    },
+    mappedUids: Record<string, any>,
+    uidMapperPath: string,
+  ) => any;
+  lookUpTerms: (
+    ctSchema: Record<string, any>[],
+    entry: any,
+    taxonomiesAndTermData: Record<string, any>,
+    importConfig: ImportConfig,
+  ) => void;
+};

packages/contentstack-variants/src/types/import-config.ts

@@ -3,6 +3,12 @@ import { AnyProperty } from './utils';
 export interface ImportDefaultConfig extends AnyProperty {
   personalizationHost: string;
   modules: {
+    'content-types': {
+      dirName: string;
+      fileName: string;
+      validKeys: string[];
+      limit: number;
+    };
     entries: {
       dirName: string;
       fileName: string;

packages/contentstack-variants/src/types/index.ts

@@ -2,6 +2,7 @@ export * from './utils'
 export * from './export-config'
 export * from './import-config'
 export * from './variant-entry'
+export * from './content-types'
 export * from './adapter-helper'
 export * from './variant-api-adapter';
 export * from './personalization-api-adapter'