Implement ETL process for migrating Schemes & Concepts from RDM tables to Lingo models

arches_rdm/media/js/views/components/etl_modules/migrate-to-lingo.js

@@ -0,0 +1,70 @@
+define([
+    'knockout',
+    'knockout-mapping',
+    'jquery',
+    'uuid',
+    'arches',
+    'viewmodels/alert',
+    'viewmodels/alert-json',
+    'templates/views/components/etl_modules/migrate-to-lingo.htm',
+], function(ko, koMapping, $, uuid, arches, AlertViewModel, JsonErrorAlertViewModel, migrateRDMTemplate) {
+    const viewModel = function(params) {
+        const self = this;
+
+        this.loadDetails = params.load_details;
+        this.state = params.state;
+        this.loading = params.loading || ko.observable();
+        this.alert = params.alert;
+        this.moduleId = params.etlmoduleid;
+        this.loadId = params.loadId || uuid.generate();
+        this.formData = new window.FormData();
+        this.graphs = ko.observable();
+        this.selectedLoadEvent = params.selectedLoadEvent || ko.observable();
+        this.formatTime = params.formatTime;
+        this.timeDifference = params.timeDifference;
+        this.activeTab = params.activeTab || ko.observable();
+        this.editHistoryUrl = `${arches.urls.edit_history}?transactionid=${ko.unwrap(params.selectedLoadEvent)?.loadid}`;
+
+        self.runRDMMigration = function() {
+            self.loading(true);            
+            self.submit('start').then(data => {
+                params.activeTab("import");
+                self.formData.append('async', true);
+                self.submit('write').then(data => {
+                    console.log(data.results);
+                }).fail(function(err) {
+                    console.log(err);
+                    self.alert(
+                        new JsonErrorAlertViewModel(
+                            'ep-alert-red',
+                            err.responseJSON["data"],
+                            null,
+                            function(){}
+                        )
+                    )
+                }).always(() => {
+                    self.loading(false);
+                });
+            }).fail(error => console.log(error.responseJSON.data));
+        };
+
+        this.submit = function(action) {
+            self.formData.append('action', action);
+            self.formData.append('loadid', self.loadId);
+            self.formData.append('module', self.moduleId);
+            return $.ajax({
+                type: "POST",
+                url: arches.urls.etl_manager,
+                data: self.formData,
+                cache: false,
+                processData: false,
+                contentType: false,
+            });
+        };
+    };
+    ko.components.register('migrate-to-lingo', {
+        viewModel: viewModel,
+        template: migrateRDMTemplate,
+    });
+    return viewModel;
+}); 

arches_rdm/migrations/0002_rdm_to_lingo_etl_module.py

@@ -0,0 +1,69 @@
+# Generated by Django 4.2.8 on 2024-01-05 11:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('arches_rdm', '0001_initial'),
+        ('models', '10260_add_iiifmanifest_globalid'),
+    ]
+
+    add_migrate_to_lingo = """
+        INSERT INTO etl_modules (
+            etlmoduleid,
+            name,
+            description,
+            etl_type,
+            component,
+            componentname,
+            modulename,
+            classname,
+            config,
+            icon,
+            slug,
+            helpsortorder,
+            helptemplate,
+            reversible)
+        VALUES (
+            '11cad3ca-e155-44b1-9910-c50b3def47f6',
+            'Migrate to Lingo',
+            'Migrate schemes and concepts from the RDM to Arches Lingo',
+            'import',
+            'views/components/etl_modules/migrate-to-lingo',
+            'migrate-to-lingo',
+            'migrate_to_lingo.py',
+            'RDMMtoLingoMigrator',
+            '{"bgColor": "#ffa564", "circleColor": "#ffd2b1", "show": true}',
+            'fa fa-usb',
+            'migrate-to-lingo',
+            6,
+            'migrate-to-lingo-help',
+            true);
+        """
+    remove_migrate_to_lingo = """
+        DELETE FROM load_staging WHERE loadid IN (SELECT loadid FROM load_event WHERE etl_module_id = '11cad3ca-e155-44b1-9910-c50b3def47f6');
+        DELETE FROM load_errors WHERE loadid IN (SELECT loadid FROM load_event WHERE etl_module_id = '11cad3ca-e155-44b1-9910-c50b3def47f6');
+        DELETE FROM load_event WHERE etl_module_id = '11cad3ca-e155-44b1-9910-c50b3def47f6';
+        DELETE FROM etl_modules WHERE etlmoduleid = '11cad3ca-e155-44b1-9910-c50b3def47f6';
+    """
+
+    #TODO: Remove operation before shipping to users
+    show_etl_manager = """
+        UPDATE plugins SET config = '{"show": true}' WHERE pluginid = '7720e9fa-876c-4127-a77a-b099cd2a5d45';
+    """
+    hide_etl_manager = """
+        UPDATE plugins SET config = '{"show": false}' WHERE pluginid = '7720e9fa-876c-4127-a77a-b099cd2a5d45';
+    """
+
+    operations = [
+        migrations.RunSQL(
+            add_migrate_to_lingo,
+            remove_migrate_to_lingo,
+        ),
+        migrations.RunSQL(
+            show_etl_manager,
+            hide_etl_manager,
+        ),
+    ]