Devrel 1067 import assets and folders

src/commands/export.ts

@@ -5,8 +5,8 @@ import JSZip from "jszip";
 import packageFile from "../../package.json" assert { type: "json" };
 import { RegisterCommand } from "../types/yargs.js";
 import { serially } from "../utils/requests.js";
-import { assetFoldersExportEntity } from "./importExportEntities/entities/assetFolders.js";
-import { assetsExportEntity } from "./importExportEntities/entities/assets.js";
+import { assetFoldersEntity } from "./importExportEntities/entities/assetFolders.js";
+import { assetsEntity } from "./importExportEntities/entities/assets.js";
 import { collectionsEntity } from "./importExportEntities/entities/collections.js";
 import { contentItemsExportEntity } from "./importExportEntities/entities/contentItems.js";
 import { contentTypesExportEntity } from "./importExportEntities/entities/contentTypes.js";
@@ -59,8 +59,8 @@ const entityDefinitions: ReadonlyArray<EntityDefinition<any>> = [
   contentTypesExportEntity,
   contentItemsExportEntity,
   languageVariantsExportEntity,
-  assetFoldersExportEntity,
-  assetsExportEntity,
+  assetFoldersEntity,
+  assetsEntity,
 ];
 
 type ExportEntitiesParams = Readonly<{

src/commands/import.ts

@@ -4,6 +4,8 @@ import JSZip from "jszip";
 
 import { RegisterCommand } from "../types/yargs.js";
 import { serially } from "../utils/requests.js";
+import { assetFoldersEntity } from "./importExportEntities/entities/assetFolders.js";
+import { assetsEntity } from "./importExportEntities/entities/assets.js";
 import { collectionsEntity } from "./importExportEntities/entities/collections.js";
 import { languagesEntity } from "./importExportEntities/entities/languages.js";
 import { taxonomiesEntity } from "./importExportEntities/entities/taxonomies.js";
@@ -37,6 +39,8 @@ const entityDefinitions: ReadonlyArray<EntityDefinition<any>> = [
   collectionsEntity,
   languagesEntity,
   taxonomiesEntity,
+  assetFoldersEntity,
+  assetsEntity,
 ];
 
 type ImportEntitiesParams = Readonly<{
@@ -55,12 +59,7 @@ const importEntities = async (params: ImportEntitiesParams) => {
 
   console.log("Importing entities...");
 
-  let context: ImportContext = {
-    collectionIdsByOldIds: new Map(),
-    languageIdsByOldIds: new Map(),
-    taxonomyGroupIdsByOldIds: new Map(),
-    taxonomyTermIdsByOldIds: new Map(),
-  };
+  let context = createInitialContext();
 
   await serially(entityDefinitions.map(def => async () => {
     console.log(`Importing ${def.name}...`);
@@ -82,3 +81,12 @@ const importEntities = async (params: ImportEntitiesParams) => {
 
   console.log(`All entities were successfully imported into environment ${params.environmentId}.`);
 };
+
+const createInitialContext = (): ImportContext => ({
+  collectionIdsByOldIds: new Map(),
+  languageIdsByOldIds: new Map(),
+  taxonomyGroupIdsByOldIds: new Map(),
+  taxonomyTermIdsByOldIds: new Map(),
+  assetFolderIdsByOldIds: new Map(),
+  assetIdsByOldIds: new Map(),
+});

src/commands/importExportEntities/entities/assetFolders.ts

@@ -1,11 +1,43 @@
-import { AssetFolderContracts } from "@kontent-ai/management-sdk";
+import { AssetFolderContracts, AssetFolderModels } from "@kontent-ai/management-sdk";
 
+import { zip } from "../../../utils/array.js";
 import { EntityDefinition } from "../entityDefinition.js";
 
-export const assetFoldersExportEntity: EntityDefinition<ReadonlyArray<AssetFolderContracts.IAssetFolderContract>> = {
+export const assetFoldersEntity: EntityDefinition<ReadonlyArray<AssetFolderContracts.IAssetFolderContract>> = {
   name: "assetFolders",
   fetchEntities: client => client.listAssetFolders().toPromise().then(res => res.rawData.folders),
   serializeEntities: JSON.stringify,
-  importEntities: () => { throw new Error("Not supported yet.")},
-  deserializeEntities: () => { throw new Error("Not supported yet.")},
+  deserializeEntities: JSON.parse,
+  importEntities: async (client, fileFolders, context) => {
+    const projectFolders = await client
+      .modifyAssetFolders()
+      .withData(fileFolders.map(createPatchToAddFolder))
+      .toPromise()
+      .then(res => res.rawData.folders);
+
+    return {
+      ...context,
+      assetFolderIdsByOldIds: new Map(zip(fileFolders, projectFolders).flatMap(extractFolderIdEntries)),
+    };
+  },
 };
+
+const createPatchToAddFolder = (folder: AssetFolderContracts.IAssetFolderContract): AssetFolderModels.IModifyAssetFoldersData => ({
+    op: "addInto",
+    value: {
+      name: folder.name,
+      external_id: folder.external_id ?? folder.id,
+      folders: folder.folders.map(createSubFolder),
+    },
+  });
+
+const createSubFolder = (folder: AssetFolderContracts.IAssetFolderContract): AssetFolderModels.IAddOrModifyAssetFolderData => ({
+  name: folder.name,
+  folders: folder.folders.map(createSubFolder),
+  external_id: folder.external_id ?? folder.id,
+});
+
+const extractFolderIdEntries = ([fileFolder, projectFolder]: readonly [AssetFolderContracts.IAssetFolderContract, AssetFolderContracts.IAssetFolderContract]): ReadonlyArray<readonly [string, string]> => [
+  [fileFolder.id, projectFolder.id] as const,
+  ...zip(fileFolder.folders, projectFolder.folders).flatMap(extractFolderIdEntries),
+];

src/commands/importExportEntities/entities/assets.ts

@@ -1,25 +1,93 @@
-import { AssetContracts } from "@kontent-ai/management-sdk";
+import { AssetContracts, ManagementClient } from "@kontent-ai/management-sdk";
 import JSZip from "jszip";
 
 import { serially } from "../../../utils/requests.js";
-import { EntityDefinition } from "../entityDefinition.js";
+import { EntityDefinition, ImportContext } from "../entityDefinition.js";
 
-export const assetsExportEntity: EntityDefinition<ReadonlyArray<AssetContracts.IAssetModelContract>> = {
+const assetsBinariesFolderName = "assets";
+const createFileName = (asset: AssetContracts.IAssetModelContract) => `${asset.id}-${asset.file_name}`;
+
+type AssetWithElements = AssetContracts.IAssetModelContract & { readonly elements: ReadonlyArray<unknown> };
+
+export const assetsEntity: EntityDefinition<ReadonlyArray<AssetWithElements>> = {
   name: "assets",
-  fetchEntities: client => client.listAssets().toAllPromise().then(res => res.data.items.map(a => a._raw)),
+  fetchEntities: client => client.listAssets().toAllPromise().then(res => res.data.items.map(a => a._raw as AssetWithElements)),
   serializeEntities: JSON.stringify,
   addOtherFiles: async (assets, zip) => {
-    const assetsZip = zip.folder("assets");
+    const assetsZip = zip.folder(assetsBinariesFolderName);
     if (!assetsZip) {
       throw new Error("Cannot create a folder in zip.");
     }
     await serially(assets.map(a => () => saveAsset(assetsZip, a)))
   },
-  importEntities: () => { throw new Error("Not supported yet.")},
-  deserializeEntities: () => { throw new Error("Not supported yet.")},
+  deserializeEntities: JSON.parse,
+  importEntities: async (client, fileAssets, context, zip) => {
+    const fileAssetsWithElements = fileAssets.filter(a => !!a.elements.length);
+    if (fileAssetsWithElements.length) {
+      throw new Error(`It is not possible to import assets with elements at the moment. Assets that contain elements are: ${fileAssetsWithElements.map(a => a.id).join(", ")}.`);
+    }
+
+    const assetsZip = zip.folder(assetsBinariesFolderName);
+    if (!fileAssets.length) {
+      return;
+    }
+    if (!assetsZip) {
+      throw new Error(`It is not possible to import assets, because the folder with asset binaries ("${assetsBinariesFolderName}") is missing.`);
+    }
+
+    const assetIdEntries = await serially(fileAssets.map(createImportAssetFetcher(assetsZip, client, context)));
+
+    return {
+      ...context,
+      assetIdsByOldIds: new Map(assetIdEntries),
+    };
+  },
 };
 
 const saveAsset = async (zip: JSZip, asset: AssetContracts.IAssetModelContract) => {
   const file = await fetch(asset.url).then(res => res.blob()).then(res => res.arrayBuffer());
-  zip.file(asset.file_name, file);
+  zip.file(createFileName(asset), file);
 };
+
+const createImportAssetFetcher = (zip: JSZip, client: ManagementClient, context: ImportContext) =>
+  (fileAsset: AssetContracts.IAssetModelContract) => async (): Promise<readonly [string, string]> => {
+    const binary = await zip.file(createFileName(fileAsset))?.async("nodebuffer");
+
+    if (!binary) {
+      throw new Error(`Failed to load a binary file "${fileAsset.file_name}" for asset "${fileAsset.id}".`);
+    }
+    const folderId = fileAsset.folder?.id ? context.assetFolderIdsByOldIds.get(fileAsset.folder.id) : undefined;
+    const collectionId = fileAsset.collection?.reference?.id ? context.collectionIdsByOldIds.get(fileAsset.collection.reference.id) : undefined;
+
+    const fileRef = await client
+      .uploadBinaryFile()
+      .withData({
+        filename: fileAsset.file_name,
+        contentType: fileAsset.type,
+        binaryData: binary,
+      })
+      .toPromise()
+      .then(res => res.data);
+
+    const projectAsset = await client
+      .addAsset()
+      .withData(() => ({
+        title: fileAsset.title,
+        codename: fileAsset.codename,
+        ...folderId ? { folder: { id: folderId } } : undefined,
+        file_reference: fileRef,
+        ...collectionId ? { collection: { reference: { id: collectionId } } } : undefined,
+        external_id: fileAsset.external_id || fileAsset.codename,
+        descriptions: fileAsset.descriptions.map(d => {
+          const newLanguageId = context.languageIdsByOldIds.get(d.language.id ?? "");
+          if (!newLanguageId) {
+            throw new Error(`There is no language id for old language id "${d.language.id}". This should never happen.`);
+          }
+          return ({ description: d.description, language: { id: newLanguageId } });
+        })
+      }))
+      .toPromise()
+      .then(res => res.data);
+
+    return [fileAsset.id, projectAsset.id] as const;
+  };

src/commands/importExportEntities/entityDefinition.ts

@@ -15,6 +15,8 @@ export type ImportContext = Readonly<{
   languageIdsByOldIds: ReadonlyMap<string, string>;
   taxonomyGroupIdsByOldIds: IdsMap;
   taxonomyTermIdsByOldIds: IdsMap;
+  assetFolderIdsByOldIds: IdsMap;
+  assetIdsByOldIds: IdsMap;
 }>;
 
 type IdsMap = ReadonlyMap<string, string>;