Merge pull request #8994 from weseek/fix/import-service

fix: Import data
weseek · Aug 22, 2024 · 361a776 · 361a776
2 parents f211b96 + 071f147
commit 361a776
Show file tree

Hide file tree

Showing 57 changed files with 759 additions and 518 deletions.
diff --git a/apps/app/package.json b/apps/app/package.json
@@ -217,7 +217,8 @@
   "// comments for defDependencies": {
     "bootstrap": "v5.3.3 has a bug. refs: https://github.com/twbs/bootstrap/issues/39798",
     "@handsontable/react": "v3 requires handsontable >= 7.0.0.",
-    "handsontable": "v7.0.0 or above is no loger MIT lisence."
+    "handsontable": "v7.0.0 or above is no loger MIT lisence.",
+    "mongodb": "mongoose which is used requires [email protected]."
   },
   "devDependencies": {
     "@growi/core-styles": "link:../../packages/core-styles",
@@ -261,6 +262,7 @@
     "jest-localstorage-mock": "^2.4.14",
     "load-css-file": "^1.0.0",
     "material-icons": "^1.11.3",
+    "mongodb": "4.16.0",
     "mongodb-memory-server-core": "^9.1.1",
     "morgan": "^1.10.0",
     "null-loader": "^4.0.1",

diff --git a/apps/app/src/client/components/Admin/ExportArchiveDataPage.tsx b/apps/app/src/client/components/Admin/ExportArchiveDataPage.tsx
@@ -14,7 +14,7 @@ import SelectCollectionsModal from './ExportArchiveData/SelectCollectionsModal';
 
 
 const IGNORED_COLLECTION_NAMES = [
-  'sessions', 'rlflx', 'activities', 'yjs-writings', 'transferkeys',
+  'sessions', 'rlflx', 'yjs-writings', 'transferkeys',
 ];
 
 const ExportArchiveDataPage = (): JSX.Element => {

diff --git a/apps/app/src/client/components/Admin/G2GDataTransferExportForm.tsx b/apps/app/src/client/components/Admin/G2GDataTransferExportForm.tsx
@@ -5,8 +5,8 @@ import React, {
 import { useTranslation } from 'next-i18next';
 
 import GrowiArchiveImportOption from '~/models/admin/growi-archive-import-option';
-import ImportOptionForPages from '~/models/admin/import-option-for-pages';
-import ImportOptionForRevisions from '~/models/admin/import-option-for-revisions';
+import { ImportOptionForPages } from '~/models/admin/import-option-for-pages';
+import { ImportOptionForRevisions } from '~/models/admin/import-option-for-revisions';
 
 import ImportCollectionConfigurationModal from './ImportData/GrowiArchive/ImportCollectionConfigurationModal';
 import ImportCollectionItem, { DEFAULT_MODE, MODE_RESTRICTED_COLLECTION } from './ImportData/GrowiArchive/ImportCollectionItem';

diff --git a/apps/app/src/client/components/Admin/ImportData/GrowiArchive/ImportForm.jsx b/apps/app/src/client/components/Admin/ImportData/GrowiArchive/ImportForm.jsx
@@ -6,8 +6,8 @@ import PropTypes from 'prop-types';
 import { apiv3Post } from '~/client/util/apiv3-client';
 import { toastSuccess, toastError } from '~/client/util/toastr';
 import GrowiArchiveImportOption from '~/models/admin/growi-archive-import-option';
-import ImportOptionForPages from '~/models/admin/import-option-for-pages';
-import ImportOptionForRevisions from '~/models/admin/import-option-for-revisions';
+import { ImportOptionForPages } from '~/models/admin/import-option-for-pages';
+import { ImportOptionForRevisions } from '~/models/admin/import-option-for-revisions';
 import { useAdminSocket } from '~/stores/socket-io';
 
 
@@ -41,7 +41,7 @@ class ImportForm extends React.Component {
       isImporting: false,
       isImported: false,
       progressMap: [],
-      errorsMap: [],
+      errorsMap: {},
 
       selectedCollections: new Set(),
 
@@ -73,7 +73,7 @@ class ImportForm extends React.Component {
         : DEFAULT_MODE;
       // create GrowiArchiveImportOption instance
       const ImportOption = IMPORT_OPTION_CLASS_MAPPING[collectionName] || GrowiArchiveImportOption;
-      this.initialState.optionsMap[collectionName] = new ImportOption(initialMode);
+      this.initialState.optionsMap[collectionName] = new ImportOption(collectionName, initialMode);
     });
 
     this.state = this.initialState;
@@ -109,8 +109,10 @@ class ImportForm extends React.Component {
       const { progressMap, errorsMap } = this.state;
       progressMap[collectionName] = collectionProgress;
 
-      const errors = errorsMap[collectionName] || [];
-      errorsMap[collectionName] = errors.concat(appendedErrors);
+      if (appendedErrors != null) {
+        const errors = errorsMap[collectionName] || [];
+        errorsMap[collectionName] = errors.concat(appendedErrors);
+      }
 
       this.setState({
         isImporting: true,
@@ -303,7 +305,7 @@ class ImportForm extends React.Component {
       await apiv3Post('/import', {
         fileName,
         collections: Array.from(selectedCollections),
-        optionsMap,
+        options: Object.values(optionsMap),
       });
 
       if (onPostImport != null) {
@@ -378,7 +380,7 @@ class ImportForm extends React.Component {
       <div className="row">
         {collectionNames.map((collectionName) => {
           const collectionProgress = progressMap[collectionName];
-          const errors = errorsMap[collectionName];
+          const errorsCount = errorsMap[collectionName]?.length ?? 0;
           const isConfigButtonAvailable = Object.keys(IMPORT_OPTION_CLASS_MAPPING).includes(collectionName);
 
           return (
@@ -388,7 +390,7 @@ class ImportForm extends React.Component {
                 isImported={collectionProgress ? isImported : false}
                 insertedCount={collectionProgress ? collectionProgress.insertedCount : 0}
                 modifiedCount={collectionProgress ? collectionProgress.modifiedCount : 0}
-                errorsCount={errors ? errors.length : 0}
+                errorsCount={errorsCount}
                 collectionName={collectionName}
                 isSelected={selectedCollections.has(collectionName)}
                 option={optionsMap[collectionName]}

diff --git a/apps/app/src/migrations/20180926134048-make-email-unique.js b/apps/app/src/migrations/20180926134048-make-email-unique.js
@@ -1,8 +1,10 @@
 import mongoose from 'mongoose';
 
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import userModelFactory from '~/server/models/user';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
+
 const logger = loggerFactory('growi:migrate:make-email-unique');
 
 module.exports = {
@@ -11,7 +13,7 @@ module.exports = {
     logger.info('Start migration');
     mongoose.connect(getMongoUri(), mongoOptions);
 
-    const User = getModelSafely('User') || require('~/server/models/user')();
+    const User = userModelFactory();
 
     // get all users who has 'deleted@deleted' email
     const users = await User.find({ email: 'deleted@deleted' });

diff --git a/apps/app/src/migrations/20181019114028-abolish-page-group-relation.js b/apps/app/src/migrations/20181019114028-abolish-page-group-relation.js
@@ -1,9 +1,11 @@
 import mongoose from 'mongoose';
 
-import getPageModel from '~/server/models/page';
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import pageModelFactory from '~/server/models/page';
+import userGroupModelFactory from '~/server/models/user-group';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
+
 const logger = loggerFactory('growi:migrate:abolish-page-group-relation');
 
 async function isCollectionExists(db, collectionName) {
@@ -38,8 +40,8 @@ module.exports = {
       return;
     }
 
-    const Page = getModelSafely('Page') || getPageModel();
-    const UserGroup = getModelSafely('UserGroup') || require('~/server/models/user-group')();
+    const Page = pageModelFactory();
+    const UserGroup = userGroupModelFactory();
 
     // retrieve all documents from 'pagegrouprelations'
     const relations = await db.collection('pagegrouprelations').find().toArray();
@@ -75,8 +77,8 @@ module.exports = {
     logger.info('Rollback migration');
     mongoose.connect(getMongoUri(), mongoOptions);
 
-    const Page = getModelSafely('Page') || getPageModel();
-    const UserGroup = getModelSafely('UserGroup') || require('~/server/models/user-group')();
+    const Page = pageModelFactory();
+    const UserGroup = userGroupModelFactory();
 
     // retrieve all Page documents which granted by UserGroup
     const relatedPages = await Page.find({ grant: Page.GRANT_USER_GROUP });

diff --git a/apps/app/src/migrations/20190618104011-add-config-app-installed.js b/apps/app/src/migrations/20190618104011-add-config-app-installed.js
@@ -1,8 +1,8 @@
 import mongoose from 'mongoose';
 
-// eslint-disable-next-line import/no-named-as-default
 import { Config } from '~/server/models/config';
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import userModelFactory from '~/server/models/user';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
 const logger = loggerFactory('growi:migrate:add-config-app-installed');
@@ -21,7 +21,7 @@ module.exports = {
     logger.info('Apply migration');
     mongoose.connect(getMongoUri(), mongoOptions);
 
-    const User = getModelSafely('User') || require('~/server/models/user')();
+    const User = userModelFactory();
 
     // find 'app:siteUrl'
     const appInstalled = await Config.findOne({

diff --git a/apps/app/src/migrations/20200402160380-remove-deleteduser-from-relationgroup.js b/apps/app/src/migrations/20200402160380-remove-deleteduser-from-relationgroup.js
@@ -1,17 +1,19 @@
 import mongoose from 'mongoose';
 
+import userModelFactory from '~/server/models/user';
 import UserGroupRelation from '~/server/models/user-group-relation';
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
+
 const logger = loggerFactory('growi:migrate:remove-deleteduser-from-relationgroup');
 
 module.exports = {
   async up(db) {
     logger.info('Apply migration');
     mongoose.connect(getMongoUri(), mongoOptions);
 
-    const User = getModelSafely('User') || require('~/server/models/user')();
+    const User = userModelFactory();
 
     const deletedUsers = await User.find({ status: 4 }); // deleted user
     const requests = await UserGroupRelation.remove({ relatedUser: deletedUsers });

diff --git a/apps/app/src/migrations/20200620203632-normalize-locale-id.js b/apps/app/src/migrations/20200620203632-normalize-locale-id.js
@@ -1,8 +1,8 @@
 import mongoose from 'mongoose';
 
-// eslint-disable-next-line import/no-named-as-default
 import { Config } from '~/server/models/config';
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import userModelFactory from '~/server/models/user';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
 const logger = loggerFactory('growi:migrate:normalize-locale-id');
@@ -12,7 +12,7 @@ module.exports = {
     logger.info('Apply migration');
     mongoose.connect(getMongoUri(), mongoOptions);
 
-    const User = getModelSafely('User') || require('~/server/models/user')();
+    const User = userModelFactory();
 
     await Promise.all([
       // update en-US -> en_US

diff --git a/apps/app/src/migrations/20210906194521-slack-app-integration-set-default-value.js b/apps/app/src/migrations/20210906194521-slack-app-integration-set-default-value.js
@@ -1,17 +1,18 @@
 import mongoose from 'mongoose';
 
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import slackAppIntegrationFactory from '~/server/models/slack-app-integration';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
+
 const logger = loggerFactory('growi:migrate:slack-app-integration-set-default-value');
 
 module.exports = {
   async up(db) {
     logger.info('Apply migration');
     mongoose.connect(getMongoUri(), mongoOptions);
 
-    // Add columns + set all default commands if supportedCommandsForBroadcastUse column does not exist
-    const SlackAppIntegration = getModelSafely('SlackAppIntegration') || require('~/server/models/slack-app-integration')();
+    const SlackAppIntegration = slackAppIntegrationFactory();
 
     // Add togetter command if supportedCommandsForBroadcastUse already exists
     const slackAppIntegrations = await SlackAppIntegration.find();

diff --git a/apps/app/src/migrations/20210913153942-migrate-slack-app-integration-schema.js b/apps/app/src/migrations/20210913153942-migrate-slack-app-integration-schema.js
@@ -1,7 +1,8 @@
 import { defaultSupportedCommandsNameForBroadcastUse, defaultSupportedCommandsNameForSingleUse } from '@growi/slack';
 import mongoose from 'mongoose';
 
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import slackAppIntegrationFactory from '~/server/models/slack-app-integration';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
 
@@ -26,7 +27,7 @@ module.exports = {
       await mongoose.connect(getMongoUri(), mongoOptions);
     }
 
-    const SlackAppIntegration = getModelSafely('SlackAppIntegration') || require('~/server/models/slack-app-integration')();
+    const SlackAppIntegration = slackAppIntegrationFactory();
 
     const slackAppIntegrations = await SlackAppIntegration.find();
 
@@ -95,7 +96,7 @@ module.exports = {
     logger.info('Rollback migration');
     await mongoose.connect(getMongoUri(), mongoOptions);
 
-    const SlackAppIntegration = getModelSafely('SlackAppIntegration') || require('~/server/models/slack-app-integration')();
+    const SlackAppIntegration = slackAppIntegrationFactory();
 
     const slackAppIntegrations = await SlackAppIntegration.find();
 

diff --git a/apps/app/src/migrations/20211005120030-slack-app-integration-rename-keys.js b/apps/app/src/migrations/20211005120030-slack-app-integration-rename-keys.js
@@ -1,16 +1,16 @@
 import mongoose from 'mongoose';
 
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import slackAppIntegrationFactory from '~/server/models/slack-app-integration';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
-
 const logger = loggerFactory('growi:migrate:slack-app-integration-rename-keys');
 
 module.exports = {
   async up(db) {
     mongoose.connect(getMongoUri(), mongoOptions);
 
-    const SlackAppIntegration = getModelSafely('SlackAppIntegration') || require('~/server/models/slack-app-integration')();
+    const SlackAppIntegration = slackAppIntegrationFactory();
 
     const slackAppIntegrations = await SlackAppIntegration.find();
 
@@ -47,7 +47,7 @@ module.exports = {
   async down(db, next) {
     mongoose.connect(getMongoUri(), mongoOptions);
 
-    const SlackAppIntegration = getModelSafely('SlackAppIntegration') || require('~/server/models/slack-app-integration')();
+    const SlackAppIntegration = slackAppIntegrationFactory();
 
     const slackAppIntegrations = await SlackAppIntegration.find();
 

diff --git a/apps/app/src/migrations/20220411114257-set-sparse-option-to-slack-member-id.js b/apps/app/src/migrations/20220411114257-set-sparse-option-to-slack-member-id.js
@@ -1,6 +1,7 @@
 import mongoose from 'mongoose';
 
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import userModelFactory from '~/server/models/user';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
 const logger = loggerFactory('growi:migrate:set-sparse-option-to-slack-member-id');
@@ -13,7 +14,7 @@ module.exports = {
     logger.info('Apply migration');
     mongoose.connect(getMongoUri(), mongoOptions);
 
-    const User = getModelSafely('User') || require('~/server/models/user')();
+    const User = userModelFactory();
     await User.syncIndexes();
 
     logger.info('Migration has successfully applied');

diff --git a/apps/app/src/migrations/20220613064207-add-attachment-type-to-existing-attachments.js b/apps/app/src/migrations/20220613064207-add-attachment-type-to-existing-attachments.js
@@ -1,7 +1,7 @@
 import mongoose from 'mongoose';
 
 import { AttachmentType } from '~/server/interfaces/attachment';
-import { Attachment } from '~/server/models';
+import { Attachment } from '~/server/models/attachment';
 import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 

diff --git a/apps/app/src/migrations/20230731075753-add_installed_date_to_config.js b/apps/app/src/migrations/20230731075753-add_installed_date_to_config.js
@@ -1,9 +1,8 @@
-// eslint-disable-next-line import/no-named-as-default
 import { Config } from '~/server/models/config';
-import { getModelSafely, getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
+import userModelFactory from '~/server/models/user';
+import { getMongoUri, mongoOptions } from '~/server/util/mongoose-utils';
 import loggerFactory from '~/utils/logger';
 
-
 const logger = loggerFactory('growi:migrate:add-installed-date-to-config');
 
 const mongoose = require('mongoose');
@@ -12,7 +11,7 @@ module.exports = {
   async up() {
     logger.info('Apply migration');
     mongoose.connect(getMongoUri(), mongoOptions);
-    const User = getModelSafely('User') || require('~/server/models/user')();
+    const User = userModelFactory();
 
     const appInstalled = await Config.findOne({ key: 'app:installed' });
     if (appInstalled != null && appInstalled.createdAt == null) {

diff --git a/apps/app/src/models/admin/growi-archive-import-option.js b/apps/app/src/models/admin/growi-archive-import-option.js
@@ -1,6 +1,7 @@
 class GrowiArchiveImportOption {
 
-  constructor(mode, initProps = {}) {
+  constructor(collectionName, mode, initProps = {}) {
+    this.collectionName = collectionName;
     this.mode = mode;
 
     Object.entries(initProps).forEach(([key, value]) => {

diff --git a/apps/app/src/models/admin/import-option-for-pages.js b/apps/app/src/models/admin/import-option-for-pages.js