diff --git a/migrator/setup.cfg b/migrator/setup.cfg index 7199fcf2..961dab0d 100644 --- a/migrator/setup.cfg +++ b/migrator/setup.cfg @@ -19,7 +19,7 @@ python_requires = >=3.9 zip_safe = False install_requires = idutils>=1.2.1 - invenio-rdm-migrator>=2.0.0 + invenio-rdm-migrator>=3.0.0 nameparser>=1.1.1 kafka-python>=2.0.2 gssapi>=1.8.2 diff --git a/migrator/tests/actions/conftest.py b/migrator/tests/actions/conftest.py new file mode 100644 index 00000000..132c74e3 --- /dev/null +++ b/migrator/tests/actions/conftest.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2023 CERN. +# +# ZenodoRDM is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Migrator tests configuration.""" + +import pytest +from invenio_rdm_migrator.extract import Extract, Tx +from invenio_rdm_migrator.state import STATE, StateDB +from invenio_rdm_migrator.streams.models.files import FilesBucket +from invenio_rdm_migrator.streams.models.pids import PersistentIdentifier +from invenio_rdm_migrator.streams.models.records import ( + RDMDraftMetadata, + RDMParentMetadata, + RDMVersionState, +) +from invenio_rdm_migrator.streams.models.users import ( + LoginInformation, + SessionActivity, + User, +) +from invenio_rdm_migrator.streams.records.state import ParentModelValidator +from sqlalchemy import create_engine + + +# FIXME: deduplicate code allowing invenio-rdm-migrator to define re-usable fixtures +@pytest.fixture(scope="function") +def state(tmp_dir): + """Yields a state. + + Do not call `save` on this fixture. The in memory database will be reset on each + function, therefore no information will be persisted from test to test. + """ + state_db = StateDB( + db_dir=tmp_dir.name, validators={"parents": ParentModelValidator} + ) + STATE.initialized_state(state_db) + + return STATE + + +@pytest.fixture(scope="function") +def secret_keys_state(state): + """Adds secret keys to global state.""" + state.VALUES.add( + "old_secret_key", + {"value": bytes("OLDKEY", "utf-8")}, + ) + state.VALUES.add( + "new_secret_key", + {"value": bytes("NEWKEY", "utf-8")}, + ) + return + + +@pytest.fixture(scope="function") +def test_extract_cls(): + """Extract class with customizable tx.""" + + class TestExtractor(Extract): + """Test extractor.""" + + tx = None + """Must be set before usage.qwa""" + + def run(self): + """Yield one element at a time.""" + yield Tx(id=self.tx["tx_id"], operations=self.tx["operations"]) + + return TestExtractor + + +@pytest.fixture(scope="session") +def db_uri(): + """Database connection string.""" + return "postgresql+psycopg://invenio:invenio@localhost:5432/invenio" + + +@pytest.fixture(scope="module") +def db_engine(db_uri): + """Setup database. + + Scope: module + + Normally, tests should use the function-scoped :py:data:`db` fixture + instead. This fixture takes care of creating the database/tables and + removing the tables once tests are done. + """ + tables = [ + FilesBucket, + LoginInformation, + PersistentIdentifier, + RDMDraftMetadata, + RDMParentMetadata, + RDMVersionState, + SessionActivity, + User, + ] + eng = create_engine(db_uri) + + # create tables + for model in tables: + model.__table__.create(bind=eng, checkfirst=True) + + yield eng + + # remove tables + for model in tables: + model.__table__.drop(eng) diff --git a/migrator/tests/actions/drafts/conftest.py b/migrator/tests/actions/drafts/conftest.py new file mode 100644 index 00000000..c87c81b8 --- /dev/null +++ b/migrator/tests/actions/drafts/conftest.py @@ -0,0 +1,1676 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2023 CERN. +# +# ZenodoRDM is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Migrator tests configuration.""" + +import pytest +from invenio_rdm_migrator.load.postgresql.transactions.operations import OperationType + + +@pytest.fixture() +def create_draft_tx(): + """Transaction data to create a draft. + + As it would be after the extraction step. + """ + return { + "tx_id": 533724568, + "operations": [ + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132090, + "pid_type": "recid", + "pid_value": "1217215", + "pid_provider": None, + "status": "K", + "object_type": None, + "object_uuid": None, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045929011, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829693272","1375829710016"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724568, + "lsn": 1375829710016, + "xmin": None, + }, + "op": OperationType.INSERT, + "ts_ms": 1688045929132, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132091, + "pid_type": "depid", + "pid_value": "1217215", + "pid_provider": None, + "status": "R", + "object_type": "rec", + "object_uuid": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045929011, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829693272","1375829710456"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724568, + "lsn": 1375829710456, + "xmin": None, + }, + "op": OperationType.INSERT, + "ts_ms": 1688045929132, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", + "default_location": 1, + "default_storage_class": "S", + "size": 0, + "quota_size": 50000000000, + "max_file_size": 50000000000, + "locked": False, + "deleted": False, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045929011, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829693272","1375829718624"]', + "schema": "public", + "table": "files_bucket", + "txId": 533724568, + "lsn": 1375829718624, + "xmin": None, + }, + "op": OperationType.INSERT, + "ts_ms": 1688045929132, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", + "json": '{"recid": 1217215,"$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json","_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"_deposit": {"id": "1217215","owners": [86261],"status": "draft","created_by": 86261},"conceptrecid": "1217214"}', + "version_id": 1, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045929011, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829693272","1375829740696"]', + "schema": "public", + "table": "records_metadata", + "txId": 533724568, + "lsn": 1375829740696, + "xmin": None, + }, + "op": OperationType.INSERT, + "ts_ms": 1688045929132, + "transaction": None, + }, + ], + } + + +@pytest.fixture() +def update_draft_tx(): + """Transaction data to update a draft. + + As it would be after the extraction step. + """ + return { + "tx_id": 533724569, + "operations": [ + { + "after": { + "issued_at": 1688045929214004, + "id": 10083674, + "remote_addr": "194.12.148.239", + "user_id": 86261, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045929339, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829782712","1375829782752"]', + "schema": "public", + "table": "transaction", + "txId": 533724586, + "lsn": 1375829782752, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045929643, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", + "json": '{"doi": "","recid": 1217215,"title": "test","$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json","license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"},"_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"_deposit": {,"owners": [86261],"status": "draft","created_by": 86261},"creators": [{"name": "me"}],"description": "
testing
","access_right": "open","conceptrecid": "1217214","resource_type": {"type": "publication", "subtype": "article"},"publication_date": "2023-06-29"}', + "version_id": 2, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045929339, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829782712","1375829785112"]', + "schema": "public", + "table": "records_metadata", + "txId": 533724586, + "lsn": 1375829785112, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045929643, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", + "json": '{"doi": "","_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"description": "testing
","license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"},"title": "test","conceptrecid": "1217214","recid": 1217215,"_deposit": {"status": "draft","owners": [86261],"id": "1217215","created_by": 86261},"publication_date": "2023-06-29","$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json","creators": [{"name": "me"}],"access_right": "open","resource_type": {"subtype": "article", "type": "publication"}}', + "version_id": 2, + "transaction_id": 10083674, + "end_transaction_id": None, + "operation_type": 1, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045929339, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829782712","1375829785848"]', + "schema": "public", + "table": "records_metadata_version", + "txId": 533724586, + "lsn": 1375829785848, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045929643, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", + "json": '{"conceptrecid": "1217214","$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json","_deposit": {"status": "draft","owners": [86261],"id": "1217215","created_by": 86261},"recid": 1217215,"_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"}}', + "version_id": 1, + "transaction_id": 10083673, + "end_transaction_id": 10083674, + "operation_type": 1, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045929339, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829782712","1375829796416"]', + "schema": "public", + "table": "records_metadata_version", + "txId": 533724586, + "lsn": 1375829796416, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045929643, + "transaction": None, + }, + ], + } + + +@pytest.fixture() +def draft_file_upload_tx(): + """Transaction data to publish a draft. + + As it would be after the extraction step. + """ + return { + "tx_id": 533724580, + "operations": [ + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", + "default_location": 1, + "default_storage_class": "S", + "size": 0, + "quota_size": 50000000000, + "max_file_size": 50000000000, + "locked": False, + "deleted": False, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045933202, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829808184","1375829808224"]', + "schema": "public", + "table": "files_bucket", + "txId": 533724612, + "lsn": 1375829808224, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045933693, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "bucket_id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", + "key": "IMG_3535.jpg", + "version_id": "f8200dc7-55b6-4785-abd0-f3d13b143c98", + "file_id": None, + "_mimetype": None, + "is_head": True, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045933202, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829808184","1375829808360"]', + "schema": "public", + "table": "files_object", + "txId": 533724612, + "lsn": 1375829808360, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045933693, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", + "uri": None, + "storage_class": None, + "size": 0, + "checksum": None, + "readable": False, + "writable": True, + "last_check_at": None, + "last_check": True, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045933202, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829808184","1375829826872"]', + "schema": "public", + "table": "files_files", + "txId": 533724612, + "lsn": 1375829826872, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045933693, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "bucket_id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", + "key": "IMG_3535.jpg", + "version_id": "f8200dc7-55b6-4785-abd0-f3d13b143c98", + "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", + "_mimetype": None, + "is_head": True, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045933202, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829808184","1375829842808"]', + "schema": "public", + "table": "files_object", + "txId": 533724612, + "lsn": 1375829842808, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045933693, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", + "uri": "root://eosmedia.cern.ch//eos/media/zenodo/test/data/e9/4b/243e-9c0c-44df-bd1f-6decc374cf78/data", + "storage_class": "S", + "size": 1562554, + "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", + "readable": True, + "writable": False, + "last_check_at": None, + "last_check": True, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045933202, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829808184","1375829859112"]', + "schema": "public", + "table": "files_files", + "txId": 533724612, + "lsn": 1375829859112, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045933693, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", + "default_location": 1, + "default_storage_class": "S", + "size": 1562554, + "quota_size": 50000000000, + "max_file_size": 50000000000, + "locked": False, + "deleted": False, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045933202, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829808184","1375829875488"]', + "schema": "public", + "table": "files_bucket", + "txId": 533724612, + "lsn": 1375829875488, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045933693, + "transaction": None, + }, + ], + } + + +@pytest.fixture() +def draft_publish_tx(): + """Transaction data to publish a draft. + + As it would be after the extraction step. + """ + return { + "tx_id": 533724590, + "operations": [ + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132090, + "pid_type": "recid", + "pid_value": "1217215", + "pid_provider": None, + "status": "K", + "object_type": "rec", + "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829894912"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724641, + "lsn": 1375829894912, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132090, + "pid_type": "recid", + "pid_value": "1217215", + "pid_provider": None, + "status": "R", + "object_type": "rec", + "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829902832"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724641, + "lsn": 1375829902832, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132092, + "pid_type": "doi", + "pid_value": "10.5072/zenodo.1217215", + "pid_provider": "datacite", + "status": "K", + "object_type": "rec", + "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829903320"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724641, + "lsn": 1375829903320, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132093, + "pid_type": "oai", + "pid_value": "oai:zenodo.org:1217215", + "pid_provider": "oai", + "status": "R", + "object_type": "rec", + "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829926648"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724641, + "lsn": 1375829926648, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132094, + "pid_type": "doi", + "pid_value": "10.5072/zenodo.1217214", + "pid_provider": "datacite", + "status": "K", + "object_type": "rec", + "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829939792"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724641, + "lsn": 1375829939792, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", + "uri": "root://eosmedia.cern.ch//eos/media/zenodo/test/data/e9/4b/243e-9c0c-44df-bd1f-6decc374cf78/data", + "storage_class": "S", + "size": 1562554, + "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", + "readable": True, + "writable": False, + "last_check_at": 1688045937960147, + "last_check": True, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829940424"]', + "schema": "public", + "table": "files_files", + "txId": 533724641, + "lsn": 1375829940424, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", + "default_location": 1, + "default_storage_class": "S", + "size": 1562554, + "quota_size": 50000000000, + "max_file_size": 50000000000, + "locked": True, + "deleted": False, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829948808"]', + "schema": "public", + "table": "files_bucket", + "txId": 533724641, + "lsn": 1375829948808, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", + "default_location": 1, + "default_storage_class": "S", + "size": 0, + "quota_size": 50000000000, + "max_file_size": None, + "locked": False, + "deleted": False, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829948944"]', + "schema": "public", + "table": "files_bucket", + "txId": 533724641, + "lsn": 1375829948944, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", + "default_location": 1, + "default_storage_class": "S", + "size": 0, + "quota_size": 50000000000, + "max_file_size": None, + "locked": False, + "deleted": False, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829956640"]', + "schema": "public", + "table": "files_bucket", + "txId": 533724641, + "lsn": 1375829956640, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "bucket_id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", + "key": "IMG_3535.jpg", + "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", + "file_id": None, + "_mimetype": None, + "is_head": True, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829956776"]', + "schema": "public", + "table": "files_object", + "txId": 533724641, + "lsn": 1375829956776, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", + "default_location": 1, + "default_storage_class": "S", + "size": 1562554, + "quota_size": 50000000000, + "max_file_size": None, + "locked": False, + "deleted": False, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829970576"]', + "schema": "public", + "table": "files_bucket", + "txId": 533724641, + "lsn": 1375829970576, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", + "uri": "root://eosmedia.cern.ch//eos/media/zenodo/test/data/e9/4b/243e-9c0c-44df-bd1f-6decc374cf78/data", + "storage_class": "S", + "size": 1562554, + "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", + "readable": True, + "writable": False, + "last_check_at": 1688045937960147, + "last_check": True, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829970712"]', + "schema": "public", + "table": "files_files", + "txId": 533724641, + "lsn": 1375829970712, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "bucket_id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", + "key": "IMG_3535.jpg", + "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", + "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", + "_mimetype": None, + "is_head": True, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829970976"]', + "schema": "public", + "table": "files_object", + "txId": 533724641, + "lsn": 1375829970976, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "issued_at": 1688045938016661, + "id": 10083675, + "remote_addr": "194.12.148.239", + "user_id": 86261, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829971608"]', + "schema": "public", + "table": "transaction", + "txId": 533724641, + "lsn": 1375829971608, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "1d34b5c0-70b3-474b-999f-82f537f035cd", + "json": '{"doi": "10.5072/zenodo.1217215","_oai": {"id": "oai:zenodo.org:1217215"},"recid": 1217215,"title": "test","_files": [{"key": "IMG_3535.jpg","size": 1562554,"type": "jpg","bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1","file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78","checksum": "md5:3cc016be06f2be46d3a438db23c40bf3","version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba"}],"owners": [86261],"$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json","license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"},"_buckets": {"record": "930d05e3-7088-4f58-ba60-cd85fb3138c1","deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"_deposit": {"id": "1217215","pid": {"type": "recid","value": "1217215","revision_id": 0},"owners": [86261],"status": "published","created_by": 86261},"creators": [{"name": "me"}],"conceptdoi": "10.5072/zenodo.1217214","description": "testing
","access_right": "open","conceptrecid": "1217214","resource_type": {"type": "publication", "subtype": "article"},"publication_date": "2023-06-29"}', + "version_id": 1, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829982232"]', + "schema": "public", + "table": "records_metadata", + "txId": 533724641, + "lsn": 1375829982232, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "type": "struct", + "fields": [ + { + "type": "struct", + "fields": [ + { + "type": "int64", + "optional": True, + "name": "io.debezium.time.MicroTimestamp", + "version": 1, + "field": "created", + }, + { + "type": "int64", + "optional": True, + "name": "io.debezium.time.MicroTimestamp", + "version": 1, + "field": "updated", + }, + { + "type": "string", + "optional": False, + "name": "io.debezium.data.Uuid", + "version": 1, + "field": "id", + }, + { + "type": "string", + "optional": True, + "name": "io.debezium.data.Json", + "version": 1, + "field": "json", + }, + {"type": "int32", "optional": True, "field": "version_id"}, + { + "type": "int64", + "optional": False, + "field": "transaction_id", + }, + { + "type": "int64", + "optional": True, + "field": "end_transaction_id", + }, + { + "type": "int16", + "optional": False, + "field": "operation_type", + }, + ], + "optional": True, + "name": "zenodo-migration.public.Value", + "field": "before", + }, + { + "type": "struct", + "fields": [ + { + "type": "int64", + "optional": True, + "name": "io.debezium.time.MicroTimestamp", + "version": 1, + "field": "created", + }, + { + "type": "int64", + "optional": True, + "name": "io.debezium.time.MicroTimestamp", + "version": 1, + "field": "updated", + }, + { + "type": "string", + "optional": False, + "name": "io.debezium.data.Uuid", + "version": 1, + "field": "id", + }, + { + "type": "string", + "optional": True, + "name": "io.debezium.data.Json", + "version": 1, + "field": "json", + }, + {"type": "int32", "optional": True, "field": "version_id"}, + { + "type": "int64", + "optional": False, + "field": "transaction_id", + }, + { + "type": "int64", + "optional": True, + "field": "end_transaction_id", + }, + { + "type": "int16", + "optional": False, + "field": "operation_type", + }, + ], + "optional": True, + "name": "zenodo-migration.public.Value", + "field": "after", + }, + { + "type": "struct", + "fields": [ + {"type": "string", "optional": False, "field": "version"}, + {"type": "string", "optional": False, "field": "connector"}, + {"type": "string", "optional": False, "field": "name"}, + {"type": "int64", "optional": False, "field": "ts_ms"}, + { + "type": "string", + "optional": True, + "name": "io.debezium.data.Enum", + "version": 1, + "parameters": { + "allowed": "True,last,False,incremental" + }, + "default": "False", + "field": "snapshot", + }, + {"type": "string", "optional": False, "field": "db"}, + {"type": "string", "optional": True, "field": "sequence"}, + {"type": "string", "optional": False, "field": "schema"}, + {"type": "string", "optional": False, "field": "table"}, + {"type": "int64", "optional": True, "field": "txId"}, + {"type": "int64", "optional": True, "field": "lsn"}, + {"type": "int64", "optional": True, "field": "xmin"}, + ], + "optional": False, + "name": "io.debezium.connector.postgresql.Source", + "field": "source", + }, + {"type": "string", "optional": False, "field": "op"}, + {"type": "int64", "optional": True, "field": "ts_ms"}, + { + "type": "struct", + "fields": [ + {"type": "string", "optional": False, "field": "id"}, + { + "type": "int64", + "optional": False, + "field": "total_order", + }, + { + "type": "int64", + "optional": False, + "field": "data_collection_order", + }, + ], + "optional": True, + "name": "event.block", + "version": 1, + "field": "transaction", + }, + ], + "optional": False, + "name": "zenodo-migration.public.Envelope", + "version": 1, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "1d34b5c0-70b3-474b-999f-82f537f035cd", + "json": '{"description": "testing
","_oai": {"id": "oai:zenodo.org:1217215"},"conceptrecid": "1217214","publication_date": "2023-06-29","$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json","owners": [86261],"doi": "10.5072/zenodo.1217215","_buckets": {"record": "930d05e3-7088-4f58-ba60-cd85fb3138c1","deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"},"conceptdoi": "10.5072/zenodo.1217214","title": "test","_files": [{"file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78","key": "IMG_3535.jpg","type": "jpg","checksum": "md5:3cc016be06f2be46d3a438db23c40bf3","bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1","version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba","size": 1562554}],"recid": 1217215,"_deposit": {"status": "published","owners": [86261],"pid": {"revision_id": 0,"type": "recid","value": "1217215"},"id": "1217215","created_by": 86261},"creators": [{"name": "me"}],"access_right": "open","resource_type": {"subtype": "article", "type": "publication"}}', + "version_id": 1, + "transaction_id": 10083675, + "end_transaction_id": None, + "operation_type": 1, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375829992944"]', + "schema": "public", + "table": "records_metadata_version", + "txId": 533724641, + "lsn": 1375829992944, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "record_id": "1d34b5c0-70b3-474b-999f-82f537f035cd", + "bucket_id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830001816"]', + "schema": "public", + "table": "records_buckets", + "txId": 533724641, + "lsn": 1375830001816, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id_community": "zenodo", + "id_record": "1d34b5c0-70b3-474b-999f-82f537f035cd", + "id_user": None, + "expires_at": None, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830011336"]', + "schema": "public", + "table": "communities_community_record", + "txId": 533724641, + "lsn": 1375830011336, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "1d34b5c0-70b3-474b-999f-82f537f035cd", + "json": '{"doi": "10.5072/zenodo.1217215","_oai": {"id": "oai:zenodo.org:1217215"},"recid": 1217215,"title": "test","_files": [{"key": "IMG_3535.jpg","size": 1562554,"type": "jpg","bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1","file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78","checksum": "md5:3cc016be06f2be46d3a438db23c40bf3","version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba"}],"owners": [86261],"$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json","license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"},"_buckets": {"record": "930d05e3-7088-4f58-ba60-cd85fb3138c1","deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"_deposit": {"id": "1217215","pid": {"type": "recid","value": "1217215","revision_id": 0},"owners": [86261],"status": "published","created_by": 86261},"creators": [{"name": "me"}],"conceptdoi": "10.5072/zenodo.1217214","description": "testing
","access_right": "open","conceptrecid": "1217214","resource_type": {"type": "publication", "subtype": "article"},"publication_date": "2023-06-29"}', + "version_id": 2, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830032456"]', + "schema": "public", + "table": "records_metadata", + "txId": 533724641, + "lsn": 1375830032456, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "1d34b5c0-70b3-474b-999f-82f537f035cd", + "json": '{"description": "testing
","_oai": {"id": "oai:zenodo.org:1217215"},"conceptrecid": "1217214","publication_date": "2023-06-29","$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json","owners": [86261],"doi": "10.5072/zenodo.1217215","_buckets": {"record": "930d05e3-7088-4f58-ba60-cd85fb3138c1","deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"},"conceptdoi": "10.5072/zenodo.1217214","title": "test","_files": [{"file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78","key": "IMG_3535.jpg","type": "jpg","checksum": "md5:3cc016be06f2be46d3a438db23c40bf3","bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1","version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba","size": 1562554}],"recid": 1217215,"_deposit": {"status": "published","owners": [86261],"pid": {"revision_id": 0,"type": "recid","value": "1217215"},"id": "1217215","created_by": 86261},"creators": [{"name": "me"}],"access_right": "open","resource_type": {"subtype": "article", "type": "publication"}}', + "version_id": 2, + "transaction_id": 10083675, + "end_transaction_id": None, + "operation_type": 1, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830033768"]', + "schema": "public", + "table": "records_metadata_version", + "txId": 533724641, + "lsn": 1375830033768, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132089, + "pid_type": "recid", + "pid_value": "1217214", + "pid_provider": None, + "status": "R", + "object_type": None, + "object_uuid": None, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830034952"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724641, + "lsn": 1375830034952, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132090, + "pid_type": "recid", + "pid_value": "1217215", + "pid_provider": None, + "status": "R", + "object_type": "rec", + "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830035400"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724641, + "lsn": 1375830035400, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "6519f1ac-1b3c-4691-bb1e-a507280102a4", + "pid_id": 12132090, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830035536"]', + "schema": "public", + "table": "pidstore_redirect", + "txId": 533724641, + "lsn": 1375830035536, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": 12132089, + "pid_type": "recid", + "pid_value": "1217214", + "pid_provider": None, + "status": "M", + "object_type": None, + "object_uuid": "6519f1ac-1b3c-4691-bb1e-a507280102a4", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830050256"]', + "schema": "public", + "table": "pidstore_pid", + "txId": 533724641, + "lsn": 1375830050256, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "before": { + "created": 0, + "updated": 0, + "parent_id": 12132090, + "child_id": 12132091, + "relation_type": 0, + "index": None, + }, + "after": None, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830064184"]', + "schema": "public", + "table": "pidrelations_pidrelation", + "txId": 533724641, + "lsn": 1375830064184, + "xmin": None, + }, + "op": "d", + "ts_ms": 1688045938757, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", + "json": '{"doi": "10.5072/zenodo.1217215","_oai": {"id": "oai:zenodo.org:1217215"},"recid": 1217215,"title": "test","owners": [86261],"$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json","license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"},"_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"_deposit": {"id": "1217215","pid": {"type": "recid","value": "1217215","revision_id": 0},"owners": [86261],"status": "published","created_by": 86261},"creators": [{"name": "me"}],"conceptdoi": "10.5072/zenodo.1217214","communities": ["zenodo"],"description": "testing
","access_right": "open","conceptrecid": "1217214","resource_type": {"type": "publication", "subtype": "article"},"publication_date": "2023-06-29"}', + "version_id": 3, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830064256"]', + "schema": "public", + "table": "records_metadata", + "txId": 533724641, + "lsn": 1375830064256, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938758, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", + "json": '{"description": "testing
","_oai": {"id": "oai:zenodo.org:1217215"},"conceptrecid": "1217214","communities": ["zenodo"],"publication_date": "2023-06-29","$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json","owners": [86261],"doi": "10.5072/zenodo.1217215","_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"conceptdoi": "10.5072/zenodo.1217214","license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"},"title": "test","recid": 1217215,"_deposit": {"status": "published","owners": [86261],"pid": {"revision_id": 0,"type": "recid","value": "1217215"},"id": "1217215","created_by": 86261},"creators": [{"name": "me"}],"access_right": "open","resource_type": {"subtype": "article", "type": "publication"}}', + "version_id": 3, + "transaction_id": 10083675, + "end_transaction_id": None, + "operation_type": 1, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830065280"]', + "schema": "public", + "table": "records_metadata_version", + "txId": 533724641, + "lsn": 1375830065280, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938758, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", + "json": '{"doi": "","_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"},"description": "testing
","license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"},"title": "test","conceptrecid": "1217214","recid": 1217215,"_deposit": {"status": "draft","owners": [86261],"id": "1217215","created_by": 86261},"publication_date": "2023-06-29","$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json","creators": [{"name": "me"}],"access_right": "open","resource_type": {"subtype": "article", "type": "publication"}}', + "version_id": 2, + "transaction_id": 10083674, + "end_transaction_id": 10083675, + "operation_type": 1, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830066440"]', + "schema": "public", + "table": "records_metadata_version", + "txId": 533724641, + "lsn": 1375830066440, + "xmin": None, + }, + "op": "u", + "ts_ms": 1688045938758, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "id": "3db002c8-3c17-4746-9806-8e6585e913bd", + "user_id": 86261, + "agent": { + "$schema": "https://zenodo.org/schemas/sipstore/agent-webclient-v1.0.0.json", + "ip_address": "194.12.148.239", + "email": "ppanero@cern.ch", + }, + "archivable": True, + "archived": False, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830067928"]', + "schema": "public", + "table": "sipstore_sip", + "txId": 533724641, + "lsn": 1375830067928, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938758, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "sip_id": "3db002c8-3c17-4746-9806-8e6585e913bd", + "filepath": "IMG_3535.jpg", + "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830083728"]', + "schema": "public", + "table": "sipstore_sipfile", + "txId": 533724641, + "lsn": 1375830083728, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938758, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "sip_id": "3db002c8-3c17-4746-9806-8e6585e913bd", + "type_id": 1, + "content": { + "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, + "owners": [86261], + "doi": "10.5072/zenodo.1217215", + "_buckets": { + "record": "930d05e3-7088-4f58-ba60-cd85fb3138c1", + "deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211", + }, + "description": "testing
", + "conceptdoi": "10.5072/zenodo.1217214", + "title": "test", + "_oai": {"id": "oai:zenodo.org:1217215"}, + "_files": [ + { + "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", + "bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1", + "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", + "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", + "key": "IMG_3535.jpg", + "type": "jpg", + "size": 1562554, + } + ], + "conceptrecid": "1217214", + "recid": 1217215, + "_deposit": { + "status": "published", + "owners": [86261], + "pid": { + "revision_id": 0, + "type": "recid", + "value": "1217215", + }, + "id": "1217215", + "created_by": 86261, + }, + "publication_date": "2023-06-29", + "$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json", + "creators": [{"name": "me"}], + "access_right": "open", + "resource_type": {"subtype": "article", "type": "publication"}, + }, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830098696"]', + "schema": "public", + "table": "sipstore_sipmetadata", + "txId": 533724641, + "lsn": 1375830098696, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938758, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "sip_id": "3db002c8-3c17-4746-9806-8e6585e913bd", + "pid_id": 12132090, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830114176"]', + "schema": "public", + "table": "sipstore_recordsip", + "txId": 533724641, + "lsn": 1375830114176, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938758, + "transaction": None, + }, + { + "after": { + "created": "2023-06-29T13:00:00", + "updated": "2023-06-29T13:00:00", + "sip_id": "3db002c8-3c17-4746-9806-8e6585e913bd", + "type_id": 3, + "content": { + "files": [ + { + "file_uuid": "e94b243e-9c0c-44df-bd1f-6decc374cf78", + "sipfilepath": "IMG_3535.jpg", + "filepath": "data/files/e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg", + "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", + "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/data/files/e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg", + "size": 1562554, + "filename": "e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg", + }, + { + "checksum": "md5:dfd970a277c490fa7dda545d4a53577a", + "metadata_id": 1, + "filepath": "data/metadata/record-json.json", + "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/data/metadata/record-json.json", + "size": 1063, + }, + { + "content": "e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg IMG_3535.jpg", + "checksum": "md5:7f54f2e75aec7ab5dae6e1f32e949302", + "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/data/filenames.txt", + "filepath": "data/filenames.txt", + "size": 62, + }, + { + "content": "Source-Organization: European Organization for Nuclear Research\\nOrganization-Address: CERN, CH-1211 Geneva 23, Switzerland\\nBagging-Date: 2023-06-29 13:38:58.399351\\nPayload-Oxum: 1563679.3\\nExternal-Identifier: 3db002c8-3c17-4746-9806-8e6585e913bd/SIPBagIt-v1.0.0\\nExternal-Description: BagIt archive of Zenodo record. Description of the payload structure and data interpretation available at https://doi.org/10.5281/zenodo.841781", + "checksum": "md5:bc83be72ab46a8349765d6bb713dfce8", + "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/bag-info.txt", + "filepath": "bag-info.txt", + "size": 427, + }, + { + "content": "3cc016be06f2be46d3a438db23c40bf3 data/files/e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg\\ndfd970a277c490fa7dda545d4a53577a data/metadata/record-json.json\\n7f54f2e75aec7ab5dae6e1f32e949302 data/filenames.txt", + "checksum": "md5:023ee30c7031619aaba5b3efbb590051", + "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/manifest-md5.txt", + "filepath": "manifest-md5.txt", + "size": 209, + }, + { + "content": "BagIt-Version: 0.97\\nTag-File-Character-Encoding: UTF-8", + "checksum": "md5:e2a83605006c61c8e9056cf9ba2b87ba", + "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/bagit.txt", + "filepath": "bagit.txt", + "size": 54, + }, + { + "content": "bc83be72ab46a8349765d6bb713dfce8 bag-info.txt\\n023ee30c7031619aaba5b3efbb590051 manifest-md5.txt\\ne2a83605006c61c8e9056cf9ba2b87ba bagit.txt", + "checksum": "md5:09b6eb5dbbe7fc5e71d58073cf958d5b", + "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/tagmanifest-md5.txt", + "filepath": "tagmanifest-md5.txt", + "size": 138, + }, + ], + "$schema": "https://zenodo.org/schemas/sipstore/bagit-v1.0.0.json", + }, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1688045938408, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1375829940368","1375830124952"]', + "schema": "public", + "table": "sipstore_sipmetadata", + "txId": 533724641, + "lsn": 1375830124952, + "xmin": None, + }, + "op": "c", + "ts_ms": 1688045938758, + "transaction": None, + }, + ], + } diff --git a/migrator/tests/streams/actions/test_draft_actions_stream.py b/migrator/tests/actions/drafts/test_draft_actions_stream.py similarity index 67% rename from migrator/tests/streams/actions/test_draft_actions_stream.py rename to migrator/tests/actions/drafts/test_draft_actions_stream.py index d5440ae2..6cfa051d 100644 --- a/migrator/tests/streams/actions/test_draft_actions_stream.py +++ b/migrator/tests/actions/drafts/test_draft_actions_stream.py @@ -9,9 +9,7 @@ from uuid import UUID -import pytest import sqlalchemy as sa -from invenio_rdm_migrator.extract import Extract, Tx from invenio_rdm_migrator.load.postgresql.transactions import PostgreSQLTx from invenio_rdm_migrator.streams import Stream from invenio_rdm_migrator.streams.models.files import FilesBucket @@ -25,54 +23,19 @@ from zenodo_rdm_migrator.transform.transactions import ZenodoTxTransform -@pytest.fixture() -def test_extract_cls(create_draft_tx): - class TestExtractor(Extract): - """Test extractor.""" - - def run(self): - """Yield one element at a time.""" - yield Tx( - id=create_draft_tx["tx_id"], operations=create_draft_tx["operations"] - ) - - return TestExtractor - - -DB_URI = "postgresql+psycopg://invenio:invenio@localhost:5432/invenio" - - -@pytest.fixture(scope="function") -def db_engine(): - tables = [ - PersistentIdentifier, - FilesBucket, - RDMDraftMetadata, - RDMParentMetadata, - RDMVersionState, - ] - eng = sa.create_engine(DB_URI) - - # create tables - for model in tables: - model.__table__.create(bind=eng, checkfirst=True) - - yield eng - - # remove tables - for model in tables: - model.__table__.drop(eng) - - -def test_draft_create_action_stream(state, test_extract_cls, db_engine): +def test_draft_create_action_stream( + state, test_extract_cls, create_draft_tx, db_uri, db_engine +): """Creates a DB on disk and initializes all the migrator related tables on it.""" # test + test_extract_cls.tx = create_draft_tx + stream = Stream( name="action", extract=test_extract_cls(), transform=ZenodoTxTransform(), - load=PostgreSQLTx(DB_URI), + load=PostgreSQLTx(db_uri), ) stream.run() diff --git a/migrator/tests/actions/test_drafts_actions.py b/migrator/tests/actions/drafts/test_drafts_actions.py similarity index 78% rename from migrator/tests/actions/test_drafts_actions.py rename to migrator/tests/actions/drafts/test_drafts_actions.py index e9752839..24508d8b 100644 --- a/migrator/tests/actions/test_drafts_actions.py +++ b/migrator/tests/actions/drafts/test_drafts_actions.py @@ -10,14 +10,16 @@ from invenio_rdm_migrator.extract import Tx from invenio_rdm_migrator.load.postgresql.transactions.operations import OperationType -from invenio_rdm_migrator.streams.actions import DraftCreateAction +from invenio_rdm_migrator.streams.actions import ( + DraftCreateAction as LoadDraftCreateAction, +) -from zenodo_rdm_migrator.actions import ZenodoDraftCreateAction +from zenodo_rdm_migrator.actions import DraftCreateAction -def test_matches_with_valid_data(): +def test_matches_with_valid_data(secret_keys_state): assert ( - ZenodoDraftCreateAction.matches_action( + DraftCreateAction.matches_action( Tx( id=1, operations=[ @@ -43,7 +45,7 @@ def test_matches_with_valid_data(): ) -def test_matches_with_invalid_data(): +def test_matches_with_invalid_data(secret_keys_state): missing_pid = [ {"op": OperationType.INSERT, "source": {"table": "files_bucket"}, "after": {}}, { @@ -68,13 +70,12 @@ def test_matches_with_invalid_data(): for invalid_ops in [missing_pid, missing_bucket, missing_draft]: assert ( - ZenodoDraftCreateAction.matches_action(Tx(id=1, operations=invalid_ops)) - is False + DraftCreateAction.matches_action(Tx(id=1, operations=invalid_ops)) is False ) -def test_transform_with_valid_data(create_draft_tx): - action = ZenodoDraftCreateAction( +def test_transform_with_valid_data(secret_keys_state, create_draft_tx): + action = DraftCreateAction( Tx(id=create_draft_tx["tx_id"], operations=create_draft_tx["operations"]) ) - assert isinstance(action.transform(), DraftCreateAction) + assert isinstance(action.transform(), LoadDraftCreateAction) diff --git a/migrator/tests/actions/users/conftest.py b/migrator/tests/actions/users/conftest.py new file mode 100644 index 00000000..c4ef40be --- /dev/null +++ b/migrator/tests/actions/users/conftest.py @@ -0,0 +1,452 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2023 CERN. +# +# ZenodoRDM is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Migrator user actions tests configuration.""" + +import pytest +from invenio_rdm_migrator.load.postgresql.transactions.operations import OperationType + + +@pytest.fixture() +def register_user_tx(): + """Transaction data to register a users. + + As it would be after the extraction step. + """ + return { + "tx_id": 533724568, + "operations": [ + { + "after": { + "user_id": 123456, + "username": "migration_test_again", + "displayname": "migration_test_again", + "full_name": "", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1690906446964, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1444875706232","1444875765576"]', + "schema": "public", + "table": "userprofiles_userprofile", + "txId": 559140492, + "lsn": 1444875765576, + "xmin": None, + }, + "op": OperationType.INSERT, + }, + { + "after": { + "id": 123456, + "email": "someaddr@domain.org", + "password": "zmkNzdnG1PXP5C3dmZqlJw==", + "active": True, + "confirmed_at": None, + "last_login_at": None, + "current_login_at": None, + "last_login_ip": None, + "current_login_ip": None, + "login_count": None, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1690906446964, + "snapshot": "False", + "db": "zenodo", + "sequence": '["1444875706232","1444875751384"]', + "schema": "public", + "table": "accounts_user", + "txId": 559140492, + "lsn": 1444875751384, + "xmin": None, + }, + "op": OperationType.INSERT, + }, + ], + } + + +@pytest.fixture() +def login_user_tx(): + """Transaction data for user login. + + Can be treated as a user update. + As it would be after the extraction step. + """ + return { + "tx_id": 533724568, + "operations": [ + { + "after": { + "id": 123456, + "email": "someaddr@domain.org", + "password": "zmkNzdnG1PXP5C3dmZqlJw==", + "active": True, + "confirmed_at": None, + "last_login_at": 1690906447550349, + "current_login_at": 1690906447550349, + "last_login_ip": None, + "current_login_ip": "192.0.238.78", + "login_count": 1, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1690906447556, + "snapshot": False, + "db": "zenodo", + "sequence": '["1444875783960","1444875783960"]', + "schema": "public", + "table": "accounts_user", + "txId": 559140493, + "lsn": 1444875783960, + "xmin": None, + }, + "op": OperationType.UPDATE, + "ts_ms": 1690906447715, + "transaction": { + "id": "559140493:1444875783960", + "total_order": 1, + "data_collection_order": 1, + }, + } + ], + } + + +@pytest.fixture() +def confirm_user_tx(): + """Transaction data for user confirmation. + + Can be treated as a user update. + As it would be after the extraction step. + """ + return { + "tx_id": 533724568, + "operations": [ + # contains one login + confirm + { + "after": { + "id": 123456, + "email": "someaddr@domain.org", + "password": "zmkNzdnG1PXP5C3dmZqlJw==", + "active": True, + "confirmed_at": None, + "last_login_at": 1690906447550349, + "current_login_at": 1690906459606401, + "last_login_ip": "192.0.238.78", + "current_login_ip": "192.0.238.78", + "login_count": 2, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1690906459615, + "snapshot": "false", + "db": "zenodo", + "sequence": '["1444875944072","1444875976096"]', + "schema": "public", + "table": "accounts_user", + "txId": 559140515, + "lsn": 1444875976096, + "xmin": None, + }, + "op": OperationType.UPDATE, + "ts_ms": 1690906459854, + "transaction": { + "id": "559140515:1444875976096", + "total_order": 1, + "data_collection_order": 1, + }, + }, + { + "after": { + "id": 123456, + "email": "someaddr@domain.org", + "password": "zmkNzdnG1PXP5C3dmZqlJw==", + "active": True, + "confirmed_at": 1690906459612306, + "last_login_at": 1690906447550349, + "current_login_at": 1690906459606401, + "last_login_ip": "192.0.238.78", + "current_login_ip": "192.0.238.78", + "login_count": 2, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1690906459615, + "snapshot": "false", + "db": "zenodo", + "sequence": '["1444875944072","1444875976392"]', + "schema": "public", + "table": "accounts_user", + "txId": 559140515, + "lsn": 1444875976392, + "xmin": None, + }, + "op": OperationType.UPDATE, + "ts_ms": 1690906459854, + "transaction": { + "id": "559140515:1444875976392", + "total_order": 2, + "data_collection_order": 2, + }, + }, + ], + } + + +@pytest.fixture() +def change_user_profile_tx(): + """Transaction data for user confirmation. + + Can be treated as a user update. + As it would be after the extraction step. + """ + return { + "tx_id": 533724568, + "operations": [ + { + "after": { + "user_id": 123456, + "username": "test_mig_ration", + "displayname": "test_mig_ration", + "full_name": "Some new full name", # change full_name + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1691051405529, + "snapshot": "false", + "db": "zenodo", + "sequence": '["1447963025504","1447963025544"]', + "schema": "public", + "table": "userprofiles_userprofile", + "txId": 559790797, + "lsn": 1447963025544, + "xmin": None, + }, + "op": OperationType.UPDATE, + "ts_ms": 1691051405990, + "transaction": { + "id": "559790797:1447963025544", + "total_order": 1, + "data_collection_order": 1, + }, + }, + { + "after": { + "user_id": 123456, + "username": "another_mig_username", # change username + "displayname": "another_mig_username", + "full_name": "Some new full name", + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1691051420594, + "snapshot": "false", + "db": "zenodo", + "sequence": '["1447969038848","1447969038888"]', + "schema": "public", + "table": "userprofiles_userprofile", + "txId": 559791611, + "lsn": 1447969038888, + "xmin": None, + }, + "op": OperationType.UPDATE, + "ts_ms": 1691051420690, + "transaction": { + "id": "559791611:1447969038888", + "total_order": 1, + "data_collection_order": 1, + }, + }, + ], + } + + +@pytest.fixture() +def change_user_email_tx(): + """Transaction data for user confirmation. + + Can be treated as a user update. + As it would be after the extraction step. + """ + return { + "tx_id": 533724568, + "operations": [ + { + "after": { + "id": 123456, + "email": "somenewaddr@domain.org", + "password": "zmkNzdnG1PXP5C3dmZqlJw==", + "active": True, + "confirmed_at": None, # changing email un-confirms the user + "last_login_at": 1691051258769502, + "current_login_at": 1691051269040966, + "last_login_ip": "192.0.238.78", + "current_login_ip": "192.0.238.78", + "login_count": 2, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1691051428956, + "snapshot": "false", + "db": "zenodo", + "sequence": '["1447969535200","1447969540936"]', + "schema": "public", + "table": "accounts_user", + "txId": 559792087, + "lsn": 1447969540936, + "xmin": None, + }, + "op": OperationType.UPDATE, + "ts_ms": 1691051429304, + "transaction": { + "id": "559792087:1447969540936", + "total_order": 1, + "data_collection_order": 1, + }, + } + ], + } + + +@pytest.fixture() +def user_deactivation_tx(): + """Transaction data for user confirmation. + + Can be treated as a user update. + As it would be after the extraction step. + """ + return { + "tx_id": 533724568, + "operations": [ + { + "after": { + "id": 123456, + "email": "ppanero27+testingmoremore@gmail.com", + "password": "zmkNzdnG1PXP5C3dmZqlJw==", + "active": False, + "confirmed_at": 1691051452717105, + "last_login_at": 1691051269040966, + "current_login_at": 1691051452710434, + "last_login_ip": "192.0.238.78", + "current_login_ip": "192.0.238.78", + "login_count": 3, + }, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1691051494292, + "snapshot": "false", + "db": "zenodo", + "sequence": '["1447988344328","1447988266728"]', + "schema": "public", + "table": "accounts_user", + "txId": 559802332, + "lsn": 1447988266728, + "xmin": None, + }, + "op": OperationType.UPDATE, + "ts_ms": 1691051494531, + "transaction": { + "id": "559802332:1447988266728", + "total_order": 1, + "data_collection_order": 1, + }, + }, + { + "before": { + "created": 0, + "updated": 0, + "sid_s": "bc51d8ea3ccc285c_64cb64fa", + "user_id": None, + "browser": None, + "browser_version": None, + "country": None, + "device": None, + "ip": None, + "os": None, + }, + "after": None, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1691051494292, + "snapshot": "false", + "db": "zenodo", + "sequence": '["1447988344328","1447988343624"]', + "schema": "public", + "table": "accounts_user_session_activity", + "txId": 559802332, + "lsn": 1447988343624, + "xmin": None, + }, + "op": OperationType.DELETE, + "ts_ms": 1691051494531, + "transaction": { + "id": "559802332:1447988343624", + "total_order": 2, + "data_collection_order": 1, + }, + }, + { + "before": { + "created": 0, + "updated": 0, + "sid_s": "754493997337aa0a_64cb65bc", + "user_id": None, + "browser": None, + "browser_version": None, + "country": None, + "device": None, + "ip": None, + "os": None, + }, + "after": None, + "source": { + "version": "2.3.0.Final", + "connector": "postgresql", + "name": "zenodo-migration", + "ts_ms": 1691051494292, + "snapshot": "false", + "db": "zenodo", + "sequence": '["1447988344328","1447988343816"]', + "schema": "public", + "table": "accounts_user_session_activity", + "txId": 559802332, + "lsn": 1447988343816, + "xmin": None, + }, + "op": OperationType.DELETE, + "ts_ms": 1691051494531, + "transaction": { + "id": "559802332:1447988343816", + "total_order": 4, + "data_collection_order": 3, + }, + }, + ], + } diff --git a/migrator/tests/actions/users/test_user_actions.py b/migrator/tests/actions/users/test_user_actions.py new file mode 100644 index 00000000..71bd9386 --- /dev/null +++ b/migrator/tests/actions/users/test_user_actions.py @@ -0,0 +1,316 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2023 CERN. +# +# ZenodoRDM is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Test user actions for RDM migration.""" + + +from invenio_rdm_migrator.extract import Tx +from invenio_rdm_migrator.load.postgresql.transactions.operations import OperationType +from invenio_rdm_migrator.streams.actions import ( + UserDeactivationAction as LoadUserDeactivationAction, +) +from invenio_rdm_migrator.streams.actions import UserEditAction as LoadUserEditAction +from invenio_rdm_migrator.streams.actions import ( + UserRegistrationAction as LoadUserRegistrationAction, +) + +from zenodo_rdm_migrator.actions import ( + UserDeactivationAction, + UserEditAction, + UserRegistrationAction, +) + +### +# USER REGISTRATION +### + + +def test_user_registration_matches_with_valid_data(): + assert ( + UserRegistrationAction.matches_action( + Tx( + id=1, + operations=[ + { + "op": OperationType.INSERT, + "source": {"table": "userprofiles_userprofile"}, + "after": {}, + }, + { + "op": OperationType.INSERT, + "source": {"table": "accounts_user"}, + "after": {}, + }, + ], + ) + ) + is True + ) + + +def test_user_registration_matches_with_invalid_data(): + missing_profile = [ + {"op": OperationType.INSERT, "source": {"table": "accounts_user"}, "after": {}}, + ] + + missing_account = [ + { + "op": OperationType.INSERT, + "source": {"table": "userprofiles_userprofile"}, + "after": {}, + } + ] + + for invalid_ops in [missing_profile, missing_account]: + assert ( + UserRegistrationAction.matches_action(Tx(id=1, operations=invalid_ops)) + is False + ) + + +def test_user_registration_transform_with_valid_data( + secret_keys_state, register_user_tx +): + action = UserRegistrationAction( + Tx(id=register_user_tx["tx_id"], operations=register_user_tx["operations"]) + ) + assert isinstance(action.transform(), LoadUserRegistrationAction) + + +### +# USER EDIT +### + + +def test_user_edit_matches_with_one_op(): + assert ( + UserEditAction.matches_action( + Tx( + id=1, + operations=[ + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user"}, + "after": {}, + }, + ], + ) + ) + is True + ) + + +def test_user_edit_matches_with_multiple_ops(): + assert ( + UserEditAction.matches_action( + Tx( + id=1, + operations=[ + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user"}, + "after": {}, + }, + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user"}, + "after": {}, + }, + ], + ) + ) + is True + ) + + +def test_user_edit_matches_with_invalid_data(): + not_update = [ + {"op": OperationType.INSERT, "source": {"table": "accounts_user"}, "after": {}}, + ] + + not_account = [ + {"op": OperationType.UPDATE, "source": {"table": "another"}, "after": {}}, + ] + + one_not_account = [ + {"op": OperationType.UPDATE, "source": {"table": "accounts_user"}, "after": {}}, + {"op": OperationType.UPDATE, "source": {"table": "another"}, "after": {}}, + ] + + for invalid_ops in [not_update, not_account, one_not_account]: + assert UserEditAction.matches_action(Tx(id=1, operations=invalid_ops)) is False + + +def test_user_edit_transform_with_valid_data( + secret_keys_state, login_user_tx, confirm_user_tx +): + for tx in [login_user_tx, confirm_user_tx]: + action = UserEditAction(Tx(id=tx["tx_id"], operations=tx["operations"])) + assert isinstance(action.transform(), LoadUserEditAction) + + +### +# USER DEACTIVATION +### +def test_user_deactivation_matches_with_one_user(): + assert ( + UserDeactivationAction.matches_action( + Tx( + id=1, + operations=[ + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user"}, + "after": {"active": False}, + }, + ], + ) + ) + is True + ) + + +def test_user_deactivation_matches_with_one_session(): + assert ( + UserDeactivationAction.matches_action( + Tx( + id=1, + operations=[ + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user"}, + "after": {"active": False}, + }, + { + "op": OperationType.DELETE, + "source": {"table": "accounts_user_session_activity"}, + "after": {}, + }, + ], + ) + ) + is True + ) + + +def test_user_deactivation_matches_with_multiple_session(): + assert ( + UserDeactivationAction.matches_action( + Tx( + id=1, + operations=[ + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user"}, + "after": {"active": False}, + }, + { + "op": OperationType.DELETE, + "source": {"table": "accounts_user_session_activity"}, + "after": {}, + }, + { + "op": OperationType.DELETE, + "source": {"table": "accounts_user_session_activity"}, + "after": {}, + }, + ], + ) + ) + is True + ) + + +def test_user_deactivation_matches_with_invalid_data(): + not_update = [ + { + "op": OperationType.INSERT, + "source": {"table": "accounts_user"}, + "after": {"active": False}, + }, + ] + + still_active = [ + { + "op": OperationType.INSERT, + "source": {"table": "accounts_user"}, + "after": {"active": True}, + }, + ] + + not_account = [ + {"op": OperationType.UPDATE, "source": {"table": "another"}, "after": {}}, + ] + + account_no_session = [ + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user"}, + "after": {"active": False}, + }, + {"op": OperationType.DELETE, "source": {"table": "another"}, "after": {}}, + ] + + session_no_account = [ + { + "op": OperationType.DELETE, + "source": {"table": "accounts_user_session_activity"}, + "after": {}, + }, + {"op": OperationType.UPDATE, "source": {"table": "another"}, "after": {}}, + ] + + session_not_update = [ + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user"}, + "after": {"active": False}, + }, + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user_session_activity"}, + "after": {}, + }, + ] + + extra_table = [ + { + "op": OperationType.UPDATE, + "source": {"table": "accounts_user"}, + "after": {"active": False}, + }, + { + "op": OperationType.DELETE, + "source": {"table": "accounts_user_session_activity"}, + "after": {}, + }, + {"op": OperationType.UPDATE, "source": {"table": "another"}, "after": {}}, + ] + + for invalid_ops in [ + not_update, + still_active, + not_account, + account_no_session, + session_no_account, + session_not_update, + extra_table, + ]: + assert ( + UserDeactivationAction.matches_action(Tx(id=1, operations=invalid_ops)) + is False + ) + + +def test_user_edit_transform_with_valid_data(secret_keys_state, user_deactivation_tx): + action = UserDeactivationAction( + Tx( + id=user_deactivation_tx["tx_id"], + operations=user_deactivation_tx["operations"], + ) + ) + assert isinstance(action.transform(), LoadUserDeactivationAction) diff --git a/migrator/tests/actions/users/test_user_actions_stream.py b/migrator/tests/actions/users/test_user_actions_stream.py new file mode 100644 index 00000000..9a5e98be --- /dev/null +++ b/migrator/tests/actions/users/test_user_actions_stream.py @@ -0,0 +1,270 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2023 CERN. +# +# ZenodoRDM is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Test user actions stream.""" + +import pytest +import sqlalchemy as sa +from invenio_rdm_migrator.load.postgresql.transactions import PostgreSQLTx +from invenio_rdm_migrator.streams import Stream +from invenio_rdm_migrator.streams.models.users import ( + LoginInformation, + SessionActivity, + User, +) +from sqlalchemy.orm import Session +from sqlalchemy.orm.exc import ObjectDeletedError + +from zenodo_rdm_migrator.transform.transactions import ZenodoTxTransform + + +@pytest.fixture(scope="function") +def db_sessions(db_engine): + sessions_data = [ + { + "created": 1691051452717105, + "updated": 1691051452717105, + "sid_s": "754493997337aa0a_64cb65bc", + "user_id": 123456, + "browser": None, + "browser_version": None, + "country": None, + "device": None, + "ip": None, + "os": None, + }, + { + "created": 1691051452717105, + "updated": 1691051452717105, + "sid_s": "bc51d8ea3ccc285c_64cb64fa", + "user_id": 123456, + "browser": None, + "browser_version": None, + "country": None, + "device": None, + "ip": None, + "os": None, + }, + ] + + history = [] + with Session(db_engine) as session: + for session_data in sessions_data: + obj = SessionActivity(**session_data) + history.append(obj) + session.add(obj) + session.commit() + + yield + + # cleanup + with Session(db_engine) as session: + for obj in history: + session.delete(obj) + + try: + session.commit() + except ObjectDeletedError: # might be deleted on user inactivation + pass + + +@pytest.fixture(scope="function") +def db_user(db_engine): + user_data = { + "id": 123456, + "created": 1640995200000000, + "updated": 1640995200000000, + "username": "test_user", + "displayname": "test_user", + "email": "someaddr@domain.org", + "password": "zmkNzdnG1PXP5C3dmZqlJw==", + "active": True, + "confirmed_at": None, + "version_id": 1, + "profile": {"full_name": "User test"}, + "preferences": { + "visibility": "restricted", + "email_visibility": "restricted", + }, + } + + log_info_data = { + "user_id": 123456, + "last_login_at": None, + "current_login_at": None, + "last_login_ip": None, + "current_login_ip": None, + "login_count": 0, + } + + history = [] + with Session(db_engine) as session: + user_obj = User(**user_data) + history.append(user_obj) + session.add(user_obj) + log_info_obj = LoginInformation(**log_info_data) + history.append(log_info_obj) + session.add(log_info_obj) + session.commit() + + yield + + # cleanup + with Session(db_engine) as session: + for obj in history: + session.delete(obj) + session.commit() + + +def test_user_register_action_stream( + secret_keys_state, db_uri, db_engine, test_extract_cls, register_user_tx +): + test_extract_cls.tx = register_user_tx + + stream = Stream( + name="action", + extract=test_extract_cls(), + transform=ZenodoTxTransform(), + load=PostgreSQLTx(db_uri), + ) + stream.run() + + with db_engine.connect() as conn: + # User + users = list(conn.execute(sa.select(User))) + assert len(users) == 1 + assert list(users)[0]._mapping["id"] == 123456 + + # Login information + loginfo = list(conn.execute(sa.select(LoginInformation))) + assert len(loginfo) == 1 + assert list(loginfo)[0]._mapping["user_id"] == 123456 + + # cleanup + # not ideal should be done more generically with a fixture + conn.execute(sa.delete(User).where(User.__table__.columns.id == 123456)) + conn.execute( + sa.delete(LoginInformation).where( + LoginInformation.__table__.columns.user_id == 123456 + ) + ) + conn.commit() + + +def test_user_login_action_stream( + secret_keys_state, db_user, db_uri, db_engine, test_extract_cls, login_user_tx +): + test_extract_cls.tx = login_user_tx + + stream = Stream( + name="action", + extract=test_extract_cls(), + transform=ZenodoTxTransform(), + load=PostgreSQLTx(db_uri), + ) + stream.run() + + with db_engine.connect() as conn: + # Login information + loginfo = list(conn.execute(sa.select(LoginInformation))) + assert len(loginfo) == 1 + loginfo = list(loginfo)[0]._mapping + assert loginfo["last_login_at"] == "1690906447550349" + assert loginfo["current_login_at"] == "1690906447550349" + assert loginfo["last_login_ip"] == None + assert loginfo["current_login_ip"] == "192.0.238.78" + assert loginfo["login_count"] == 1 + + +def test_confirm_user_action_stream( + secret_keys_state, db_user, db_uri, db_engine, test_extract_cls, confirm_user_tx +): + test_extract_cls.tx = confirm_user_tx + stream = Stream( + name="action", + extract=test_extract_cls(), + transform=ZenodoTxTransform(), + load=PostgreSQLTx(db_uri), + ) + stream.run() + + with db_engine.connect() as conn: + users = list(conn.execute(sa.select(User))) + assert len(users) == 1 + assert list(users)[0]._mapping["confirmed_at"] == "1690906459612306" + + +@pytest.mark.skip("UserProfileEditAction not implemented yet") +def test_change_user_profile_stream( + db_user, db_uri, db_engine, test_extract_cls, change_user_profile_tx +): + test_extract_cls.tx = change_user_profile_tx + stream = Stream( + name="action", + extract=test_extract_cls(), + transform=ZenodoTxTransform(), + load=PostgreSQLTx(db_uri), + ) + stream.run() + + with db_engine.connect() as conn: + users = list(conn.execute(sa.select(User))) + assert len(users) == 1 + user = list(users)[0]._mapping + assert user["username"] == "another_mig_username" + assert user["displayname"] == "another_mig_username" + assert user["full_name"] == "Some new full name" + + +def test_confirm_user_action_stream( + secret_keys_state, + db_user, + db_uri, + db_engine, + test_extract_cls, + change_user_email_tx, +): + test_extract_cls.tx = change_user_email_tx + stream = Stream( + name="action", + extract=test_extract_cls(), + transform=ZenodoTxTransform(), + load=PostgreSQLTx(db_uri), + ) + stream.run() + + with db_engine.connect() as conn: + users = list(conn.execute(sa.select(User))) + assert len(users) == 1 + assert list(users)[0]._mapping["email"] == "somenewaddr@domain.org" + + +def test_deactivate_user_action_stream( + secret_keys_state, + db_user, + db_sessions, + db_uri, + db_engine, + test_extract_cls, + user_deactivation_tx, +): + test_extract_cls.tx = user_deactivation_tx + stream = Stream( + name="action", + extract=test_extract_cls(), + transform=ZenodoTxTransform(), + load=PostgreSQLTx(db_uri), + ) + stream.run() + + with db_engine.connect() as conn: + users = list(conn.execute(sa.select(User))) + assert len(users) == 1 + assert list(users)[0]._mapping["active"] == False + + sessions = list(conn.execute(sa.select(SessionActivity))) + assert len(sessions) == 0 diff --git a/migrator/tests/conftest.py b/migrator/tests/conftest.py index d3952877..0143bdb0 100644 --- a/migrator/tests/conftest.py +++ b/migrator/tests/conftest.py @@ -10,7 +10,6 @@ import tempfile import pytest -from invenio_rdm_migrator.load.postgresql.transactions.operations import OperationType @pytest.fixture(scope="function") @@ -19,1968 +18,3 @@ def tmp_dir(): tmp_dir = tempfile.TemporaryDirectory() yield tmp_dir tmp_dir.cleanup() - - -@pytest.fixture() -def create_draft_tx(): - """Transaction data to create a draft. - - As it would be after the extraction step. - """ - return { - "tx_id": 533724568, - "operations": [ - { - "after": { - "created": 1688045928842023, - "updated": 1688045928842033, - "id": 12132090, - "pid_type": "recid", - "pid_value": "1217215", - "pid_provider": None, - "status": "K", - "object_type": None, - "object_uuid": None, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045929011, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829693272","1375829710016"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724568, - "lsn": 1375829710016, - "xmin": None, - }, - "op": OperationType.INSERT, - "ts_ms": 1688045929132, - "transaction": None, - }, - { - "after": { - "created": 1688045928845041, - "updated": 1688045928845050, - "id": 12132091, - "pid_type": "depid", - "pid_value": "1217215", - "pid_provider": None, - "status": "R", - "object_type": "rec", - "object_uuid": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045929011, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829693272","1375829710456"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724568, - "lsn": 1375829710456, - "xmin": None, - }, - "op": OperationType.INSERT, - "ts_ms": 1688045929132, - "transaction": None, - }, - { - "after": { - "created": 1688045928885348, - "updated": 1688045928885363, - "id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - "default_location": 1, - "default_storage_class": "S", - "size": 0, - "quota_size": 50000000000, - "max_file_size": 50000000000, - "locked": False, - "deleted": False, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045929011, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829693272","1375829718624"]', - "schema": "public", - "table": "files_bucket", - "txId": 533724568, - "lsn": 1375829718624, - "xmin": None, - }, - "op": OperationType.INSERT, - "ts_ms": 1688045929132, - "transaction": None, - }, - { - "after": { - "created": 1688045928906896, - "updated": 1688045928906905, - "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", - # it assumes the json field comes as a dict not a string - "json": { - "recid": 1217215, - "$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json", - "_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"}, - "_deposit": { - "id": "1217215", - "owners": [86261], - "status": "draft", - "created_by": 86261, - }, - "conceptrecid": "1217214", - }, - "version_id": 1, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045929011, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829693272","1375829740696"]', - "schema": "public", - "table": "records_metadata", - "txId": 533724568, - "lsn": 1375829740696, - "xmin": None, - }, - "op": OperationType.INSERT, - "ts_ms": 1688045929132, - "transaction": None, - }, - ], - } - - -@pytest.fixture() -def update_draft_tx(): - """Transaction data to update a draft. - - As it would be after the extraction step. - """ - return { - "tx_id": 533724569, - "operations": [ - { - "after": { - "issued_at": 1688045929214004, - "id": 10083674, - "remote_addr": "194.12.148.239", - "user_id": 86261, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045929339, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829782712","1375829782752"]', - "schema": "public", - "table": "transaction", - "txId": 533724586, - "lsn": 1375829782752, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045929643, - "transaction": None, - }, - { - "after": { - "created": 1688045928906896, - "updated": 1688045929216560, - "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", - "json": { - "doi": "", - "recid": 1217215, - "title": "test", - "$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json", - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"}, - "_deposit": { - "id": "1217215", - "owners": [86261], - "status": "draft", - "created_by": 86261, - }, - "creators": [{"name": "me"}], - "description": "testing
", - "access_right": "open", - "conceptrecid": "1217214", - "resource_type": {"type": "publication", "subtype": "article"}, - "publication_date": "2023-06-29", - }, - "version_id": 2, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045929339, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829782712","1375829785112"]', - "schema": "public", - "table": "records_metadata", - "txId": 533724586, - "lsn": 1375829785112, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045929643, - "transaction": None, - }, - { - "after": { - "created": 1688045928906896, - "updated": 1688045929216560, - "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", - "json": { - "doi": "", - "_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"}, - "description": "testing
", - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "title": "test", - "conceptrecid": "1217214", - "recid": 1217215, - "_deposit": { - "status": "draft", - "owners": [86261], - "id": "1217215", - "created_by": 86261, - }, - "publication_date": "2023-06-29", - "$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json", - "creators": [{"name": "me"}], - "access_right": "open", - "resource_type": {"subtype": "article", "type": "publication"}, - }, - "version_id": 2, - "transaction_id": 10083674, - "end_transaction_id": None, - "operation_type": 1, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045929339, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829782712","1375829785848"]', - "schema": "public", - "table": "records_metadata_version", - "txId": 533724586, - "lsn": 1375829785848, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045929643, - "transaction": None, - }, - { - "after": { - "created": 1688045928906896, - "updated": 1688045928906905, - "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", - "json": { - "conceptrecid": "1217214", - "$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json", - "_deposit": { - "status": "draft", - "owners": [86261], - "id": "1217215", - "created_by": 86261, - }, - "recid": 1217215, - "_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"}, - }, - "version_id": 1, - "transaction_id": 10083673, - "end_transaction_id": 10083674, - "operation_type": 1, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045929339, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829782712","1375829796416"]', - "schema": "public", - "table": "records_metadata_version", - "txId": 533724586, - "lsn": 1375829796416, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045929643, - "transaction": None, - }, - ], - } - - -@pytest.fixture() -def draft_file_upload_tx(): - """Transaction data to publish a draft. - - As it would be after the extraction step. - """ - return { - "tx_id": 533724580, - "operations": [ - { - "after": { - "created": 1688045928885348, - "updated": 1688045933154720, - "id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - "default_location": 1, - "default_storage_class": "S", - "size": 0, - "quota_size": 50000000000, - "max_file_size": 50000000000, - "locked": False, - "deleted": False, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045933202, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829808184","1375829808224"]', - "schema": "public", - "table": "files_bucket", - "txId": 533724612, - "lsn": 1375829808224, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045933693, - "transaction": None, - }, - { - "after": { - "created": 1688045933156507, - "updated": 1688045933156516, - "bucket_id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - "key": "IMG_3535.jpg", - "version_id": "f8200dc7-55b6-4785-abd0-f3d13b143c98", - "file_id": None, - "_mimetype": None, - "is_head": True, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045933202, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829808184","1375829808360"]', - "schema": "public", - "table": "files_object", - "txId": 533724612, - "lsn": 1375829808360, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045933693, - "transaction": None, - }, - { - "after": { - "created": 1688045933160898, - "updated": 1688045933160906, - "id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "uri": None, - "storage_class": None, - "size": 0, - "checksum": None, - "readable": False, - "writable": True, - "last_check_at": None, - "last_check": True, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045933202, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829808184","1375829826872"]', - "schema": "public", - "table": "files_files", - "txId": 533724612, - "lsn": 1375829826872, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045933693, - "transaction": None, - }, - { - "after": { - "created": 1688045933156507, - "updated": 1688045933162201, - "bucket_id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - "key": "IMG_3535.jpg", - "version_id": "f8200dc7-55b6-4785-abd0-f3d13b143c98", - "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "_mimetype": None, - "is_head": True, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045933202, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829808184","1375829842808"]', - "schema": "public", - "table": "files_object", - "txId": 533724612, - "lsn": 1375829842808, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045933693, - "transaction": None, - }, - { - "after": { - "created": 1688045933160898, - "updated": 1688045933198028, - "id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "uri": "root://eosmedia.cern.ch//eos/media/zenodo/test/data/e9/4b/243e-9c0c-44df-bd1f-6decc374cf78/data", - "storage_class": "S", - "size": 1562554, - "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", - "readable": True, - "writable": False, - "last_check_at": None, - "last_check": True, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045933202, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829808184","1375829859112"]', - "schema": "public", - "table": "files_files", - "txId": 533724612, - "lsn": 1375829859112, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045933693, - "transaction": None, - }, - { - "after": { - "created": 1688045928885348, - "updated": 1688045933199979, - "id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - "default_location": 1, - "default_storage_class": "S", - "size": 1562554, - "quota_size": 50000000000, - "max_file_size": 50000000000, - "locked": False, - "deleted": False, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045933202, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829808184","1375829875488"]', - "schema": "public", - "table": "files_bucket", - "txId": 533724612, - "lsn": 1375829875488, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045933693, - "transaction": None, - }, - ], - } - - -@pytest.fixture() -def draft_publish_tx(): - """Transaction data to publish a draft. - - As it would be after the extraction step. - """ - return { - "tx_id": 533724590, - "operations": [ - { - "after": { - "created": 1688045928842023, - "updated": 1688045937867955, - "id": 12132090, - "pid_type": "recid", - "pid_value": "1217215", - "pid_provider": None, - "status": "K", - "object_type": "rec", - "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829894912"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724641, - "lsn": 1375829894912, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045928842023, - "updated": 1688045937916467, - "id": 12132090, - "pid_type": "recid", - "pid_value": "1217215", - "pid_provider": None, - "status": "R", - "object_type": "rec", - "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829902832"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724641, - "lsn": 1375829902832, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045937920797, - "updated": 1688045937920810, - "id": 12132092, - "pid_type": "doi", - "pid_value": "10.5072/zenodo.1217215", - "pid_provider": "datacite", - "status": "K", - "object_type": "rec", - "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829903320"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724641, - "lsn": 1375829903320, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045937925806, - "updated": 1688045937925813, - "id": 12132093, - "pid_type": "oai", - "pid_value": "oai:zenodo.org:1217215", - "pid_provider": "oai", - "status": "R", - "object_type": "rec", - "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829926648"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724641, - "lsn": 1375829926648, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045937931777, - "updated": 1688045937931784, - "id": 12132094, - "pid_type": "doi", - "pid_value": "10.5072/zenodo.1217214", - "pid_provider": "datacite", - "status": "K", - "object_type": "rec", - "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829939792"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724641, - "lsn": 1375829939792, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045933160898, - "updated": 1688045937961853, - "id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "uri": "root://eosmedia.cern.ch//eos/media/zenodo/test/data/e9/4b/243e-9c0c-44df-bd1f-6decc374cf78/data", - "storage_class": "S", - "size": 1562554, - "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", - "readable": True, - "writable": False, - "last_check_at": 1688045937960147, - "last_check": True, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829940424"]', - "schema": "public", - "table": "files_files", - "txId": 533724641, - "lsn": 1375829940424, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045928885348, - "updated": 1688045937975547, - "id": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - "default_location": 1, - "default_storage_class": "S", - "size": 1562554, - "quota_size": 50000000000, - "max_file_size": 50000000000, - "locked": True, - "deleted": False, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829948808"]', - "schema": "public", - "table": "files_bucket", - "txId": 533724641, - "lsn": 1375829948808, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045937983333, - "updated": 1688045937983350, - "id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "default_location": 1, - "default_storage_class": "S", - "size": 0, - "quota_size": 50000000000, - "max_file_size": None, - "locked": False, - "deleted": False, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829948944"]', - "schema": "public", - "table": "files_bucket", - "txId": 533724641, - "lsn": 1375829948944, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045937983333, - "updated": 1688045937992194, - "id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "default_location": 1, - "default_storage_class": "S", - "size": 0, - "quota_size": 50000000000, - "max_file_size": None, - "locked": False, - "deleted": False, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829956640"]', - "schema": "public", - "table": "files_bucket", - "txId": 533724641, - "lsn": 1375829956640, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045937993595, - "updated": 1688045937993603, - "bucket_id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "key": "IMG_3535.jpg", - "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", - "file_id": None, - "_mimetype": None, - "is_head": True, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829956776"]', - "schema": "public", - "table": "files_object", - "txId": 533724641, - "lsn": 1375829956776, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045937983333, - "updated": 1688045937999825, - "id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "default_location": 1, - "default_storage_class": "S", - "size": 1562554, - "quota_size": 50000000000, - "max_file_size": None, - "locked": False, - "deleted": False, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829970576"]', - "schema": "public", - "table": "files_bucket", - "txId": 533724641, - "lsn": 1375829970576, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045933160898, - "updated": 1688045938000812, - "id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "uri": "root://eosmedia.cern.ch//eos/media/zenodo/test/data/e9/4b/243e-9c0c-44df-bd1f-6decc374cf78/data", - "storage_class": "S", - "size": 1562554, - "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", - "readable": True, - "writable": False, - "last_check_at": 1688045937960147, - "last_check": True, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829970712"]', - "schema": "public", - "table": "files_files", - "txId": 533724641, - "lsn": 1375829970712, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045937993595, - "updated": 1688045938001697, - "bucket_id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "key": "IMG_3535.jpg", - "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", - "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "_mimetype": None, - "is_head": True, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829970976"]', - "schema": "public", - "table": "files_object", - "txId": 533724641, - "lsn": 1375829970976, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "issued_at": 1688045938016661, - "id": 10083675, - "remote_addr": "194.12.148.239", - "user_id": 86261, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829971608"]', - "schema": "public", - "table": "transaction", - "txId": 533724641, - "lsn": 1375829971608, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045938019228, - "updated": 1688045938019248, - "id": "1d34b5c0-70b3-474b-999f-82f537f035cd", - "json": { - "doi": "10.5072/zenodo.1217215", - "_oai": {"id": "oai:zenodo.org:1217215"}, - "recid": 1217215, - "title": "test", - "_files": [ - { - "key": "IMG_3535.jpg", - "size": 1562554, - "type": "jpg", - "bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", - "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", - } - ], - "owners": [86261], - "$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json", - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "_buckets": { - "record": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - }, - "_deposit": { - "id": "1217215", - "pid": { - "type": "recid", - "value": "1217215", - "revision_id": 0, - }, - "owners": [86261], - "status": "published", - "created_by": 86261, - }, - "creators": [{"name": "me"}], - "conceptdoi": "10.5072/zenodo.1217214", - "description": "testing
", - "access_right": "open", - "conceptrecid": "1217214", - "resource_type": {"type": "publication", "subtype": "article"}, - "publication_date": "2023-06-29", - }, - "version_id": 1, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829982232"]', - "schema": "public", - "table": "records_metadata", - "txId": 533724641, - "lsn": 1375829982232, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "type": "struct", - "fields": [ - { - "type": "struct", - "fields": [ - { - "type": "int64", - "optional": True, - "name": "io.debezium.time.MicroTimestamp", - "version": 1, - "field": "created", - }, - { - "type": "int64", - "optional": True, - "name": "io.debezium.time.MicroTimestamp", - "version": 1, - "field": "updated", - }, - { - "type": "string", - "optional": False, - "name": "io.debezium.data.Uuid", - "version": 1, - "field": "id", - }, - { - "type": "string", - "optional": True, - "name": "io.debezium.data.Json", - "version": 1, - "field": "json", - }, - {"type": "int32", "optional": True, "field": "version_id"}, - { - "type": "int64", - "optional": False, - "field": "transaction_id", - }, - { - "type": "int64", - "optional": True, - "field": "end_transaction_id", - }, - { - "type": "int16", - "optional": False, - "field": "operation_type", - }, - ], - "optional": True, - "name": "zenodo-migration.public.Value", - "field": "before", - }, - { - "type": "struct", - "fields": [ - { - "type": "int64", - "optional": True, - "name": "io.debezium.time.MicroTimestamp", - "version": 1, - "field": "created", - }, - { - "type": "int64", - "optional": True, - "name": "io.debezium.time.MicroTimestamp", - "version": 1, - "field": "updated", - }, - { - "type": "string", - "optional": False, - "name": "io.debezium.data.Uuid", - "version": 1, - "field": "id", - }, - { - "type": "string", - "optional": True, - "name": "io.debezium.data.Json", - "version": 1, - "field": "json", - }, - {"type": "int32", "optional": True, "field": "version_id"}, - { - "type": "int64", - "optional": False, - "field": "transaction_id", - }, - { - "type": "int64", - "optional": True, - "field": "end_transaction_id", - }, - { - "type": "int16", - "optional": False, - "field": "operation_type", - }, - ], - "optional": True, - "name": "zenodo-migration.public.Value", - "field": "after", - }, - { - "type": "struct", - "fields": [ - {"type": "string", "optional": False, "field": "version"}, - {"type": "string", "optional": False, "field": "connector"}, - {"type": "string", "optional": False, "field": "name"}, - {"type": "int64", "optional": False, "field": "ts_ms"}, - { - "type": "string", - "optional": True, - "name": "io.debezium.data.Enum", - "version": 1, - "parameters": { - "allowed": "True,last,False,incremental" - }, - "default": "False", - "field": "snapshot", - }, - {"type": "string", "optional": False, "field": "db"}, - {"type": "string", "optional": True, "field": "sequence"}, - {"type": "string", "optional": False, "field": "schema"}, - {"type": "string", "optional": False, "field": "table"}, - {"type": "int64", "optional": True, "field": "txId"}, - {"type": "int64", "optional": True, "field": "lsn"}, - {"type": "int64", "optional": True, "field": "xmin"}, - ], - "optional": False, - "name": "io.debezium.connector.postgresql.Source", - "field": "source", - }, - {"type": "string", "optional": False, "field": "op"}, - {"type": "int64", "optional": True, "field": "ts_ms"}, - { - "type": "struct", - "fields": [ - {"type": "string", "optional": False, "field": "id"}, - { - "type": "int64", - "optional": False, - "field": "total_order", - }, - { - "type": "int64", - "optional": False, - "field": "data_collection_order", - }, - ], - "optional": True, - "name": "event.block", - "version": 1, - "field": "transaction", - }, - ], - "optional": False, - "name": "zenodo-migration.public.Envelope", - "version": 1, - }, - { - "after": { - "created": 1688045938019228, - "updated": 1688045938019248, - "id": "1d34b5c0-70b3-474b-999f-82f537f035cd", - "json": { - "description": "testing
", - "_oai": {"id": "oai:zenodo.org:1217215"}, - "conceptrecid": "1217214", - "publication_date": "2023-06-29", - "$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json", - "owners": [86261], - "doi": "10.5072/zenodo.1217215", - "_buckets": { - "record": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - }, - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "conceptdoi": "10.5072/zenodo.1217214", - "title": "test", - "_files": [ - { - "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "key": "IMG_3535.jpg", - "type": "jpg", - "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", - "bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", - "size": 1562554, - } - ], - "recid": 1217215, - "_deposit": { - "status": "published", - "owners": [86261], - "pid": { - "revision_id": 0, - "type": "recid", - "value": "1217215", - }, - "id": "1217215", - "created_by": 86261, - }, - "creators": [{"name": "me"}], - "access_right": "open", - "resource_type": {"subtype": "article", "type": "publication"}, - }, - "version_id": 1, - "transaction_id": 10083675, - "end_transaction_id": None, - "operation_type": 1, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375829992944"]', - "schema": "public", - "table": "records_metadata_version", - "txId": 533724641, - "lsn": 1375829992944, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "record_id": "1d34b5c0-70b3-474b-999f-82f537f035cd", - "bucket_id": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830001816"]', - "schema": "public", - "table": "records_buckets", - "txId": 533724641, - "lsn": 1375830001816, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045938115434, - "updated": 1688045938115442, - "id_community": "zenodo", - "id_record": "1d34b5c0-70b3-474b-999f-82f537f035cd", - "id_user": None, - "expires_at": None, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830011336"]', - "schema": "public", - "table": "communities_community_record", - "txId": 533724641, - "lsn": 1375830011336, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045938019228, - "updated": 1688045938209727, - "id": "1d34b5c0-70b3-474b-999f-82f537f035cd", - "json": { - "doi": "10.5072/zenodo.1217215", - "_oai": {"id": "oai:zenodo.org:1217215"}, - "recid": 1217215, - "title": "test", - "_files": [ - { - "key": "IMG_3535.jpg", - "size": 1562554, - "type": "jpg", - "bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", - "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", - } - ], - "owners": [86261], - "$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json", - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "_buckets": { - "record": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - }, - "_deposit": { - "id": "1217215", - "pid": { - "type": "recid", - "value": "1217215", - "revision_id": 0, - }, - "owners": [86261], - "status": "published", - "created_by": 86261, - }, - "creators": [{"name": "me"}], - "conceptdoi": "10.5072/zenodo.1217214", - "description": "testing
", - "access_right": "open", - "conceptrecid": "1217214", - "resource_type": {"type": "publication", "subtype": "article"}, - "publication_date": "2023-06-29", - }, - "version_id": 2, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830032456"]', - "schema": "public", - "table": "records_metadata", - "txId": 533724641, - "lsn": 1375830032456, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045938019228, - "updated": 1688045938209727, - "id": "1d34b5c0-70b3-474b-999f-82f537f035cd", - "json": { - "description": "testing
", - "_oai": {"id": "oai:zenodo.org:1217215"}, - "conceptrecid": "1217214", - "publication_date": "2023-06-29", - "$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json", - "owners": [86261], - "doi": "10.5072/zenodo.1217215", - "_buckets": { - "record": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - }, - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "conceptdoi": "10.5072/zenodo.1217214", - "title": "test", - "_files": [ - { - "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "key": "IMG_3535.jpg", - "type": "jpg", - "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", - "bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", - "size": 1562554, - } - ], - "recid": 1217215, - "_deposit": { - "status": "published", - "owners": [86261], - "pid": { - "revision_id": 0, - "type": "recid", - "value": "1217215", - }, - "id": "1217215", - "created_by": 86261, - }, - "creators": [{"name": "me"}], - "access_right": "open", - "resource_type": {"subtype": "article", "type": "publication"}, - }, - "version_id": 2, - "transaction_id": 10083675, - "end_transaction_id": None, - "operation_type": 1, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830033768"]', - "schema": "public", - "table": "records_metadata_version", - "txId": 533724641, - "lsn": 1375830033768, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045928834350, - "updated": 1688045938222339, - "id": 12132089, - "pid_type": "recid", - "pid_value": "1217214", - "pid_provider": None, - "status": "R", - "object_type": None, - "object_uuid": None, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830034952"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724641, - "lsn": 1375830034952, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045928842023, - "updated": 1688045938229213, - "id": 12132090, - "pid_type": "recid", - "pid_value": "1217215", - "pid_provider": None, - "status": "R", - "object_type": "rec", - "object_uuid": "1d34b5c0-70b3-474b-999f-82f537f035cd", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830035400"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724641, - "lsn": 1375830035400, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045938230563, - "updated": 1688045938230575, - "id": "6519f1ac-1b3c-4691-bb1e-a507280102a4", - "pid_id": 12132090, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830035536"]', - "schema": "public", - "table": "pidstore_redirect", - "txId": 533724641, - "lsn": 1375830035536, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045928834350, - "updated": 1688045938234262, - "id": 12132089, - "pid_type": "recid", - "pid_value": "1217214", - "pid_provider": None, - "status": "M", - "object_type": None, - "object_uuid": "6519f1ac-1b3c-4691-bb1e-a507280102a4", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830050256"]', - "schema": "public", - "table": "pidstore_pid", - "txId": 533724641, - "lsn": 1375830050256, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "before": { - "created": 0, - "updated": 0, - "parent_id": 12132090, - "child_id": 12132091, - "relation_type": 0, - "index": None, - }, - "after": None, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830064184"]', - "schema": "public", - "table": "pidrelations_pidrelation", - "txId": 533724641, - "lsn": 1375830064184, - "xmin": None, - }, - "op": "d", - "ts_ms": 1688045938757, - "transaction": None, - }, - { - "after": { - "created": 1688045928906896, - "updated": 1688045938258954, - "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", - "json": { - "doi": "10.5072/zenodo.1217215", - "_oai": {"id": "oai:zenodo.org:1217215"}, - "recid": 1217215, - "title": "test", - "owners": [86261], - "$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json", - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"}, - "_deposit": { - "id": "1217215", - "pid": { - "type": "recid", - "value": "1217215", - "revision_id": 0, - }, - "owners": [86261], - "status": "published", - "created_by": 86261, - }, - "creators": [{"name": "me"}], - "conceptdoi": "10.5072/zenodo.1217214", - "communities": ["zenodo"], - "description": "testing
", - "access_right": "open", - "conceptrecid": "1217214", - "resource_type": {"type": "publication", "subtype": "article"}, - "publication_date": "2023-06-29", - }, - "version_id": 3, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830064256"]', - "schema": "public", - "table": "records_metadata", - "txId": 533724641, - "lsn": 1375830064256, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938758, - "transaction": None, - }, - { - "after": { - "created": 1688045928906896, - "updated": 1688045938258954, - "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", - "json": { - "description": "testing
", - "_oai": {"id": "oai:zenodo.org:1217215"}, - "conceptrecid": "1217214", - "communities": ["zenodo"], - "publication_date": "2023-06-29", - "$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json", - "owners": [86261], - "doi": "10.5072/zenodo.1217215", - "_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"}, - "conceptdoi": "10.5072/zenodo.1217214", - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "title": "test", - "recid": 1217215, - "_deposit": { - "status": "published", - "owners": [86261], - "pid": { - "revision_id": 0, - "type": "recid", - "value": "1217215", - }, - "id": "1217215", - "created_by": 86261, - }, - "creators": [{"name": "me"}], - "access_right": "open", - "resource_type": {"subtype": "article", "type": "publication"}, - }, - "version_id": 3, - "transaction_id": 10083675, - "end_transaction_id": None, - "operation_type": 1, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830065280"]', - "schema": "public", - "table": "records_metadata_version", - "txId": 533724641, - "lsn": 1375830065280, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938758, - "transaction": None, - }, - { - "after": { - "created": 1688045928906896, - "updated": 1688045929216560, - "id": "b7547ab1-47d2-48e2-9867-ca597b4ebb41", - "json": { - "doi": "", - "_buckets": {"deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211"}, - "description": "testing
", - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "title": "test", - "conceptrecid": "1217214", - "recid": 1217215, - "_deposit": { - "status": "draft", - "owners": [86261], - "id": "1217215", - "created_by": 86261, - }, - "publication_date": "2023-06-29", - "$schema": "https://zenodo.org/schemas/deposits/records/record-v1.0.0.json", - "creators": [{"name": "me"}], - "access_right": "open", - "resource_type": {"subtype": "article", "type": "publication"}, - }, - "version_id": 2, - "transaction_id": 10083674, - "end_transaction_id": 10083675, - "operation_type": 1, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830066440"]', - "schema": "public", - "table": "records_metadata_version", - "txId": 533724641, - "lsn": 1375830066440, - "xmin": None, - }, - "op": "u", - "ts_ms": 1688045938758, - "transaction": None, - }, - { - "after": { - "created": 1688045938370975, - "updated": 1688045938370989, - "id": "3db002c8-3c17-4746-9806-8e6585e913bd", - "user_id": 86261, - "agent": { - "$schema": "https://zenodo.org/schemas/sipstore/agent-webclient-v1.0.0.json", - "ip_address": "194.12.148.239", - "email": "ppanero@cern.ch", - }, - "archivable": True, - "archived": False, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830067928"]', - "schema": "public", - "table": "sipstore_sip", - "txId": 533724641, - "lsn": 1375830067928, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938758, - "transaction": None, - }, - { - "after": { - "created": 1688045938377598, - "updated": 1688045938377607, - "sip_id": "3db002c8-3c17-4746-9806-8e6585e913bd", - "filepath": "IMG_3535.jpg", - "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830083728"]', - "schema": "public", - "table": "sipstore_sipfile", - "txId": 533724641, - "lsn": 1375830083728, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938758, - "transaction": None, - }, - { - "after": { - "created": 1688045938383940, - "updated": 1688045938383949, - "sip_id": "3db002c8-3c17-4746-9806-8e6585e913bd", - "type_id": 1, - "content": { - "license": {"$ref": "https://dx.zenodo.org/licenses/CC-BY-4.0"}, - "owners": [86261], - "doi": "10.5072/zenodo.1217215", - "_buckets": { - "record": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "deposit": "0e12b4b6-9cc7-46df-9a04-c11c478de211", - }, - "description": "testing
", - "conceptdoi": "10.5072/zenodo.1217214", - "title": "test", - "_oai": {"id": "oai:zenodo.org:1217215"}, - "_files": [ - { - "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", - "bucket": "930d05e3-7088-4f58-ba60-cd85fb3138c1", - "version_id": "5f17d253-70bd-42d7-9920-3b69258c44ba", - "file_id": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "key": "IMG_3535.jpg", - "type": "jpg", - "size": 1562554, - } - ], - "conceptrecid": "1217214", - "recid": 1217215, - "_deposit": { - "status": "published", - "owners": [86261], - "pid": { - "revision_id": 0, - "type": "recid", - "value": "1217215", - }, - "id": "1217215", - "created_by": 86261, - }, - "publication_date": "2023-06-29", - "$schema": "https://zenodo.org/schemas/records/record-v1.0.0.json", - "creators": [{"name": "me"}], - "access_right": "open", - "resource_type": {"subtype": "article", "type": "publication"}, - }, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830098696"]', - "schema": "public", - "table": "sipstore_sipmetadata", - "txId": 533724641, - "lsn": 1375830098696, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938758, - "transaction": None, - }, - { - "after": { - "created": 1688045938387970, - "updated": 1688045938387978, - "sip_id": "3db002c8-3c17-4746-9806-8e6585e913bd", - "pid_id": 12132090, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830114176"]', - "schema": "public", - "table": "sipstore_recordsip", - "txId": 533724641, - "lsn": 1375830114176, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938758, - "transaction": None, - }, - { - "after": { - "created": 1688045938406568, - "updated": 1688045938406576, - "sip_id": "3db002c8-3c17-4746-9806-8e6585e913bd", - "type_id": 3, - "content": { - "files": [ - { - "file_uuid": "e94b243e-9c0c-44df-bd1f-6decc374cf78", - "sipfilepath": "IMG_3535.jpg", - "filepath": "data/files/e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg", - "checksum": "md5:3cc016be06f2be46d3a438db23c40bf3", - "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/data/files/e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg", - "size": 1562554, - "filename": "e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg", - }, - { - "checksum": "md5:dfd970a277c490fa7dda545d4a53577a", - "metadata_id": 1, - "filepath": "data/metadata/record-json.json", - "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/data/metadata/record-json.json", - "size": 1063, - }, - { - "content": "e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg IMG_3535.jpg", - "checksum": "md5:7f54f2e75aec7ab5dae6e1f32e949302", - "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/data/filenames.txt", - "filepath": "data/filenames.txt", - "size": 62, - }, - { - "content": "Source-Organization: European Organization for Nuclear Research\\nOrganization-Address: CERN, CH-1211 Geneva 23, Switzerland\\nBagging-Date: 2023-06-29 13:38:58.399351\\nPayload-Oxum: 1563679.3\\nExternal-Identifier: 3db002c8-3c17-4746-9806-8e6585e913bd/SIPBagIt-v1.0.0\\nExternal-Description: BagIt archive of Zenodo record. Description of the payload structure and data interpretation available at https://doi.org/10.5281/zenodo.841781", - "checksum": "md5:bc83be72ab46a8349765d6bb713dfce8", - "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/bag-info.txt", - "filepath": "bag-info.txt", - "size": 427, - }, - { - "content": "3cc016be06f2be46d3a438db23c40bf3 data/files/e94b243e-9c0c-44df-bd1f-6decc374cf78-IMG_3535.jpg\\ndfd970a277c490fa7dda545d4a53577a data/metadata/record-json.json\\n7f54f2e75aec7ab5dae6e1f32e949302 data/filenames.txt", - "checksum": "md5:023ee30c7031619aaba5b3efbb590051", - "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/manifest-md5.txt", - "filepath": "manifest-md5.txt", - "size": 209, - }, - { - "content": "BagIt-Version: 0.97\\nTag-File-Character-Encoding: UTF-8", - "checksum": "md5:e2a83605006c61c8e9056cf9ba2b87ba", - "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/bagit.txt", - "filepath": "bagit.txt", - "size": 54, - }, - { - "content": "bc83be72ab46a8349765d6bb713dfce8 bag-info.txt\\n023ee30c7031619aaba5b3efbb590051 manifest-md5.txt\\ne2a83605006c61c8e9056cf9ba2b87ba bagit.txt", - "checksum": "md5:09b6eb5dbbe7fc5e71d58073cf958d5b", - "fullpath": "root://eospublic.cern.ch//eos/zenodo/test/archive/121/721/5/r/2023-06-29T13:38:58.370975+00:00/tagmanifest-md5.txt", - "filepath": "tagmanifest-md5.txt", - "size": 138, - }, - ], - "$schema": "https://zenodo.org/schemas/sipstore/bagit-v1.0.0.json", - }, - }, - "source": { - "version": "2.3.0.Final", - "connector": "postgresql", - "name": "zenodo-migration", - "ts_ms": 1688045938408, - "snapshot": "False", - "db": "zenodo", - "sequence": '["1375829940368","1375830124952"]', - "schema": "public", - "table": "sipstore_sipmetadata", - "txId": 533724641, - "lsn": 1375830124952, - "xmin": None, - }, - "op": "c", - "ts_ms": 1688045938758, - "transaction": None, - }, - ], - } diff --git a/migrator/tests/streams/actions/conftest.py b/migrator/tests/streams/actions/conftest.py deleted file mode 100644 index 93960a0f..00000000 --- a/migrator/tests/streams/actions/conftest.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2023 CERN. -# -# ZenodoRDM is free software; you can redistribute it and/or modify it -# under the terms of the MIT License; see LICENSE file for more details. - -"""Migrator tests configuration.""" - -import pytest -from invenio_rdm_migrator.state import STATE, StateDB, StateEntity -from invenio_rdm_migrator.streams.records.state import ParentModelValidator - - -# FIXME: deduplicate code allowing invenio-rdm-migrator to define re-usable fixtures -@pytest.fixture(scope="function") -def state(tmp_dir): - """Yields a state. - - Do not call `save` on this fixture. The in memory database will be reset on each - function, therefore no information will be persisted from test to test. - """ - state_db = StateDB( - db_dir=tmp_dir.name, validators={"parents": ParentModelValidator} - ) - STATE.initialized_state(state_db) - - return STATE diff --git a/migrator/zenodo_rdm_migrator/actions/__init__.py b/migrator/zenodo_rdm_migrator/actions/__init__.py index af56c3c0..2fa9c369 100644 --- a/migrator/zenodo_rdm_migrator/actions/__init__.py +++ b/migrator/zenodo_rdm_migrator/actions/__init__.py @@ -7,6 +7,12 @@ """Actions module.""" -from .drafts import ZenodoDraftCreateAction +from .drafts import DraftCreateAction +from .users import UserDeactivationAction, UserEditAction, UserRegistrationAction -__all__ = ("ZenodoDraftCreateAction",) +__all__ = ( + "DraftCreateAction", + "UserDeactivationAction", + "UserEditAction", + "UserRegistrationAction", +) diff --git a/migrator/zenodo_rdm_migrator/actions/drafts.py b/migrator/zenodo_rdm_migrator/actions/drafts.py index 45032015..ba366ce4 100644 --- a/migrator/zenodo_rdm_migrator/actions/drafts.py +++ b/migrator/zenodo_rdm_migrator/actions/drafts.py @@ -5,22 +5,24 @@ # Invenio-RDM-Migrator is free software; you can redistribute it and/or modify # it under the terms of the MIT License; see LICENSE file for more details. -"""Invenio RDM migration drafts row load module.""" +"""Invenio RDM migration drafts actions module.""" from invenio_rdm_migrator.actions import TransformAction from invenio_rdm_migrator.load.postgresql.transactions.operations import OperationType -from invenio_rdm_migrator.streams.actions import DraftCreateAction -from invenio_rdm_migrator.transform import IdentityTransform +from invenio_rdm_migrator.streams.actions import ( + DraftCreateAction as LoadDraftCreateAction, +) +from invenio_rdm_migrator.transform import IdentityTransform, JSONTransformMixin from ..transform.records import ZenodoRecordTransform -class ZenodoDraftCreateAction(TransformAction): +class DraftCreateAction(TransformAction, JSONTransformMixin): """Zenodo to RDM draft creation action.""" name = "create-zenodo-draft" - load_cls = DraftCreateAction + load_cls = LoadDraftCreateAction @classmethod def matches_action(cls, tx): # pragma: no cover @@ -46,7 +48,6 @@ def matches_action(cls, tx): # pragma: no cover def _transform_data(self): # pragma: no cover """Transforms the data and returns an instance of the mapped_cls.""" # draft files should be a separate transaction - for operation in self.tx.operations: table_name = operation["source"]["table"] @@ -56,6 +57,9 @@ def _transform_data(self): # pragma: no cover elif table_name == "files_bucket": bucket = IdentityTransform()._transform(operation["after"]) elif table_name == "records_metadata": + # need to json load the draft json + # the parent json is calculated on transform + self._load_json_fields(data=operation["after"], fields=["json"]) draft_and_parent = ZenodoRecordTransform()._transform( operation["after"] ) diff --git a/migrator/zenodo_rdm_migrator/actions/users.py b/migrator/zenodo_rdm_migrator/actions/users.py new file mode 100644 index 00000000..effba298 --- /dev/null +++ b/migrator/zenodo_rdm_migrator/actions/users.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2023 CERN. +# +# Invenio-RDM-Migrator is free software; you can redistribute it and/or modify +# it under the terms of the MIT License; see LICENSE file for more details. + +"""Invenio RDM migration users actions module.""" + + +from invenio_rdm_migrator.actions import TransformAction +from invenio_rdm_migrator.load.postgresql.transactions.operations import OperationType +from invenio_rdm_migrator.streams.actions import ( + UserDeactivationAction as LoadUserDeactivationAction, +) +from invenio_rdm_migrator.streams.actions import UserEditAction as LoadUserEditAction +from invenio_rdm_migrator.streams.actions import ( + UserRegistrationAction as LoadUserRegistrationAction, +) +from invenio_rdm_migrator.transform import IdentityTransform + +from ..transform.entries.users import ZenodoUserEntry + + +class UserRegistrationAction(TransformAction): + """Zenodo to RDM user registration action.""" + + name = "register-user" + load_cls = LoadUserRegistrationAction + + @classmethod + def matches_action(cls, tx): # pragma: no cover + """Checks if the data corresponds with that required by the action.""" + rules = { + "userprofiles_userprofile": OperationType.INSERT, + "accounts_user": OperationType.INSERT, + } + + for operation in tx.operations: + table_name = operation["source"]["table"] + + rule = rules.pop(table_name, None) + # no duplicated tables, can fail fast if rule is None + if not rule or not rule == operation["op"]: + return False + + return len(rules) == 0 + + def _transform_data(self): # pragma: no cover + """Transforms the data and returns an instance of the mapped_cls.""" + payload = {} + for operation in self.tx.operations: + payload = {**payload, **operation["after"]} + + # should be already in microseconds + ts = self.tx.operations[0]["source"]["ts_ms"] + payload["created"] = ts + payload["updated"] = ts + + user = ZenodoUserEntry().transform(payload) + login_info = user.pop("login_information") + + return dict(tx_id=self.tx.id, user=user, login_information=login_info) + + +class UserEditAction(TransformAction): + """Zenodo to RDM user edit action.""" + + name = "edit-user" + load_cls = LoadUserEditAction + + @classmethod + def matches_action(cls, tx): # pragma: no cover + """Checks if the data corresponds with that required by the action.""" + # all operations are updates on the user table + # in some cases (e.g. user confirmation there are two due to the login information) + for operation in tx.operations: + if ( + not "accounts_user" == operation["source"]["table"] + or not operation["op"] == OperationType.UPDATE + ): + return False + + return len(tx.operations) >= 1 # there was at least one matching op + + def _transform_data(self): # pragma: no cover + """Transforms the data and returns an instance of the mapped_cls.""" + # use only the latest updated information + payload = {**self.tx.operations[-1]["after"]} + + # should be already in microseconds + ts = self.tx.operations[0]["source"]["ts_ms"] + payload["created"] = ts + payload["updated"] = ts + + user = ZenodoUserEntry().transform(payload) + login_info = user.pop("login_information") + + return dict(tx_id=self.tx.id, user=user, login_information=login_info) + + +class UserDeactivationAction(TransformAction): + """Zenodo to RDM user deactivation action.""" + + name = "deactivate-user" + load_cls = LoadUserDeactivationAction + + @classmethod + def matches_action(cls, tx): # pragma: no cover + """Checks if the data corresponds with that required by the action.""" + # one update on the account_user and multiple session deletes + account_seen = False + for operation in tx.operations: + if "accounts_user" == operation["source"]["table"]: + update_not_active = ( + operation["after"]["active"] + or not operation["op"] == OperationType.UPDATE + ) + if update_not_active or account_seen: + return False + + account_seen = True + + elif "accounts_user_session_activity" == operation["source"]["table"]: + if not operation["op"] == OperationType.DELETE: + return False + else: + return False + + return True + + # TODO + def _transform_data(self): # pragma: no cover + """Transforms the data and returns an instance of the mapped_cls.""" + user = None + sessions = [] + ts = self.tx.operations[0]["source"]["ts_ms"] + for operation in self.tx.operations: + if "accounts_user" == operation["source"]["table"]: + operation["after"]["created"] = ts + operation["after"]["updated"] = ts + user = ZenodoUserEntry().transform(operation["after"]) + else: # can only be a session as per match function + # important: the data comes in "before" + sessions.append(IdentityTransform()._transform(operation["before"])) + + # cannot guarantee user will be the first of the previous loop + for session in sessions: + session["user_id"] = user["id"] + + login_info = user.pop("login_information") + return dict( + tx_id=self.tx.id, user=user, login_information=login_info, sessions=sessions + ) diff --git a/migrator/zenodo_rdm_migrator/transform/entries/users.py b/migrator/zenodo_rdm_migrator/transform/entries/users.py index 579dca64..95789f0d 100644 --- a/migrator/zenodo_rdm_migrator/transform/entries/users.py +++ b/migrator/zenodo_rdm_migrator/transform/entries/users.py @@ -77,5 +77,5 @@ def _login_information(self, entry): "current_login_at": entry.get("current_login_at"), "last_login_ip": entry.get("last_login_ip"), "current_login_ip": entry.get("current_login_ip"), - "login_count": entry.get("login_count"), + "login_count": entry.get("login_count") or 0, } diff --git a/migrator/zenodo_rdm_migrator/transform/transactions.py b/migrator/zenodo_rdm_migrator/transform/transactions.py index 23296625..4a6968fd 100644 --- a/migrator/zenodo_rdm_migrator/transform/transactions.py +++ b/migrator/zenodo_rdm_migrator/transform/transactions.py @@ -10,10 +10,20 @@ from invenio_rdm_migrator.transform import BaseTxTransform -from ..actions import ZenodoDraftCreateAction +from ..actions import ( + DraftCreateAction, + UserDeactivationAction, + UserEditAction, + UserRegistrationAction, +) class ZenodoTxTransform(BaseTxTransform): """Zenodo transaction transform.""" - actions = [ZenodoDraftCreateAction] + actions = [ + DraftCreateAction, + UserDeactivationAction, + UserEditAction, + UserRegistrationAction, + ]