feat: add use-server-side-upsert option

src/cli/record/import.ts

@@ -45,6 +45,12 @@ const builder = (args: yargs.Argv) =>
       describe: "The fields to be imported in comma-separated",
       type: "string",
       requiresArg: true,
+    })
+    .option("use-server-side-upsert", {
+      describe:
+        "Use server-side upsert. This option is under early development.",
+      type: "boolean",
+      default: false,
     });
 
 type Args = yargs.Arguments<
@@ -65,6 +71,7 @@ const handler = (args: Args) => {
     filePath: args["file-path"],
     updateKey: args["update-key"],
     fields: args.fields?.split(","),
+    useServerSideUpsert: args["use-server-side-upsert"],
     encoding: args.encoding,
     pfxFilePath: args["pfx-file-path"],
     pfxFilePassword: args["pfx-file-password"],

src/record/import/index.ts

@@ -4,13 +4,15 @@ import type { SupportedImportEncoding } from "../../utils/file";
 import { extractFileFormat, openFsStreamWithEncode } from "../../utils/file";
 import { addRecords } from "./usecases/add";
 import { upsertRecords } from "./usecases/upsert";
+import { upsertRecords as upsertRecordsServerSide } from "./usecases/upsertServerSide";
 import { createSchema } from "./schema";
 import { noop as defaultTransformer } from "./schema/transformers/noop";
 import { userSelected } from "./schema/transformers/userSelected";
 import { logger } from "../../utils/log";
 import { LocalRecordRepositoryFromStream } from "./repositories/localRecordRepositoryFromStream";
 import { RunError } from "../error";
 import { isMismatchEncoding } from "../../utils/encoding";
+import { emitExperimentalWarning } from "../../utils/stability";
 
 export type Options = {
   app: string;
@@ -19,6 +21,7 @@ export type Options = {
   updateKey?: string;
   encoding?: SupportedImportEncoding;
   fields?: string[];
+  useServerSideUpsert?: boolean;
 };
 
 export const run: (
@@ -32,6 +35,7 @@ export const run: (
       attachmentsDir,
       updateKey,
       fields,
+      useServerSideUpsert,
       ...restApiClientOptions
     } = argv;
 
@@ -60,17 +64,35 @@ export const run: (
 
     const skipMissingFields = !fields;
     if (updateKey) {
-      await upsertRecords(
-        apiClient,
-        app,
-        localRecordRepository,
-        schema,
-        updateKey,
-        {
-          attachmentsDir,
-          skipMissingFields,
-        },
-      );
+      if (useServerSideUpsert) {
+        emitExperimentalWarning(
+          "Use server-side upsert. This option is under early development.",
+          logger,
+        );
+        await upsertRecordsServerSide(
+          apiClient,
+          app,
+          localRecordRepository,
+          schema,
+          updateKey,
+          {
+            attachmentsDir,
+            skipMissingFields,
+          },
+        );
+      } else {
+        await upsertRecords(
+          apiClient,
+          app,
+          localRecordRepository,
+          schema,
+          updateKey,
+          {
+            attachmentsDir,
+            skipMissingFields,
+          },
+        );
+      }
     } else {
       await addRecords(apiClient, app, localRecordRepository, schema, {
         attachmentsDir,

src/record/import/usecases/upsert/updateKeyServerSide.ts

@@ -0,0 +1,161 @@
+import type {
+  KintoneFormFieldProperty,
+  KintoneRestAPIClient,
+} from "@kintone/rest-api-client";
+
+import type { FieldSchema, RecordSchema } from "../../types/schema";
+import type { LocalRecord } from "../../types/record";
+import type { LocalRecordRepository } from "../interface";
+import { withIndex } from "../../../../utils/iterator";
+
+type UpdateKeyField = {
+  code: string;
+  type: SupportedUpdateKeyFieldType["type"];
+};
+
+export class UpdateKey {
+  private readonly field: UpdateKeyField;
+  private readonly appCode: string;
+
+  constructor(field: UpdateKeyField, appCode: string) {
+    this.field = field;
+    this.appCode = appCode;
+  }
+
+  static build = async (
+    apiClient: KintoneRestAPIClient,
+    app: string,
+    updateKeyCode: string,
+    schema: RecordSchema,
+  ): Promise<UpdateKey> => {
+    const updateKeyField = findUpdateKeyInSchema(updateKeyCode, schema);
+    const appCode = (await apiClient.app.getApp({ id: app })).code;
+
+    return new UpdateKey(updateKeyField, appCode);
+  };
+
+  getUpdateKeyField = () => this.field;
+
+  validateUpdateKeyInRecords = (records: LocalRecordRepository) =>
+    validateUpdateKeyInRecords(this.field, this.appCode, records);
+
+  findUpdateKeyValueFromRecord = (record: LocalRecord): string => {
+    const fieldValue = record.data[this.field.code].value as string;
+    if (fieldValue.length === 0) {
+      return fieldValue;
+    }
+    if (this.field.type === "RECORD_NUMBER") {
+      return removeAppCode(fieldValue, this.appCode);
+    }
+    return fieldValue;
+  };
+}
+
+const findUpdateKeyInSchema = (
+  updateKey: string,
+  schema: RecordSchema,
+): UpdateKeyField => {
+  const updateKeySchema = schema.fields.find(
+    (fieldSchema) => fieldSchema.code === updateKey,
+  );
+
+  if (updateKeySchema === undefined) {
+    throw new Error("no such update key");
+  }
+
+  if (!isSupportedUpdateKeyFieldType(updateKeySchema)) {
+    throw new Error("unsupported field type for update key");
+  }
+
+  if (
+    updateKeySchema.type === "SINGLE_LINE_TEXT" ||
+    updateKeySchema.type === "NUMBER"
+  ) {
+    if (!updateKeySchema.unique) {
+      throw new Error("update key field should set to unique");
+    }
+  }
+
+  return { code: updateKey, type: updateKeySchema.type };
+};
+
+type SupportedUpdateKeyFieldType =
+  | KintoneFormFieldProperty.RecordNumber
+  | KintoneFormFieldProperty.SingleLineText
+  | KintoneFormFieldProperty.Number;
+
+const isSupportedUpdateKeyFieldType = (
+  fieldSchema: FieldSchema,
+): fieldSchema is SupportedUpdateKeyFieldType => {
+  const supportedUpdateKeyFieldTypes = [
+    "RECORD_NUMBER",
+    "SINGLE_LINE_TEXT",
+    "NUMBER",
+  ];
+  return supportedUpdateKeyFieldTypes.includes(fieldSchema.type);
+};
+
+const validateUpdateKeyInRecords = async (
+  updateKey: UpdateKeyField,
+  appCode: string,
+  recordRepository: LocalRecordRepository,
+) => {
+  let hasAppCodePrevious: boolean = false;
+  for await (const { data: record, index } of withIndex(
+    recordRepository.reader(),
+  )) {
+    if (!(updateKey.code in record.data)) {
+      throw new Error(
+        `The field specified as "Key to Bulk Update" (${updateKey.code}) does not exist on the input`,
+      );
+    }
+    const value = record.data[updateKey.code].value;
+    if (typeof value !== "string") {
+      throw new Error(
+        `The value of the "Key to Bulk Update" (${updateKey.code}) on the input is invalid`,
+      );
+    }
+
+    if (value.length === 0) {
+      return;
+    }
+
+    if (updateKey.type === "RECORD_NUMBER") {
+      const _hasAppCode = hasAppCode(value, appCode);
+      if (index !== 0 && _hasAppCode !== hasAppCodePrevious) {
+        throw new Error(
+          `The "Key to Bulk Update" should not be mixed with those with and without app code`,
+        );
+      }
+      hasAppCodePrevious = _hasAppCode;
+    }
+  }
+};
+
+const removeAppCode = (input: string, appCode: string) => {
+  if (hasAppCode(input, appCode)) {
+    return input.slice(appCode.length + 1);
+  }
+  return input;
+};
+
+export const hasAppCode = (input: string, appCode: string): boolean => {
+  const matchWithAppCode = isValidUpdateKeyWithAppCode(input, appCode);
+  const matchWithoutAppCode = isValidUpdateKeyWithoutAppCode(input);
+
+  if (!matchWithAppCode && !matchWithoutAppCode) {
+    throw new Error(`The "Key to Bulk Update" value is invalid (${input})`);
+  }
+
+  return matchWithAppCode && !matchWithoutAppCode;
+};
+
+const isValidUpdateKeyWithAppCode = (input: string, appCode: string) => {
+  return (
+    input.startsWith(appCode + "-") &&
+    isValidUpdateKeyWithoutAppCode(input.slice(appCode.length + 1))
+  );
+};
+
+const isValidUpdateKeyWithoutAppCode = (input: string) =>
+  input.match(/^[0-9]+$/) !== null;