From 94e9bd2099d99443041d776d19970b734470a8fa Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 12 Apr 2023 13:19:09 +0100 Subject: [PATCH 1/5] updating the action that is taken when wanting to re-parse a document to just archive everything. --- .../base/updated_document_actions.py | 33 ++----------------- .../tests/test_update_actions.py | 10 +----- 2 files changed, 3 insertions(+), 40 deletions(-) diff --git a/src/navigator_data_ingest/base/updated_document_actions.py b/src/navigator_data_ingest/base/updated_document_actions.py index 8fc86dec..496a8675 100644 --- a/src/navigator_data_ingest/base/updated_document_actions.py +++ b/src/navigator_data_ingest/base/updated_document_actions.py @@ -203,7 +203,7 @@ def parse( """ document_id, document_update = update _LOGGER.info( - "Updating document so as to parse during the next run.", + "Archiving document so as to re-download from source and parse during the next run.", extra={ "props": { "document_id": document_id, @@ -211,39 +211,10 @@ def parse( }, ) errors = [] - for prefix_path in [ - S3Path( - os.path.join( - "s3://", update_config.pipeline_bucket, update_config.parser_input - ) - ), - S3Path( - os.path.join( - "s3://", update_config.pipeline_bucket, update_config.embeddings_input - ) - ), - S3Path( - os.path.join( - "s3://", update_config.pipeline_bucket, update_config.indexer_input - ) - ), - ]: - # Might be translated and non-translated json objects - document_files = get_document_files( - prefix_path, document_id, suffix_filter="json" - ) - for document_file in document_files: - errors.append( - update_file_field( - document_path=document_file, - field=str(document_update.type.value), - new_value=document_update.csv_value, - existing_value=document_update.db_value, - ) - ) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") for prefix in [ + update_config.parser_input, update_config.embeddings_input, update_config.indexer_input, ]: diff --git a/src/navigator_data_ingest/tests/test_update_actions.py b/src/navigator_data_ingest/tests/test_update_actions.py index b9063e5a..1068d3d3 100644 --- a/src/navigator_data_ingest/tests/test_update_actions.py +++ b/src/navigator_data_ingest/tests/test_update_actions.py @@ -184,16 +184,8 @@ def test_parse( for s3_key in s3_document_keys ] - assert parser_input_doc.exists() + assert not parser_input_doc.exists() assert not embeddings_input_doc.exists() assert not embeddings_input_translated_doc.exists() assert not indexer_input_doc_json.exists() assert not indexer_input_doc_npy.exists() - - parser_input_doc_data = json.loads(parser_input_doc.read_text()) - assert ( - parser_input_doc_data[ - PipelineFieldMapping[UpdateTypes(update_to_source_url.type)] - ] - == update_to_source_url.csv_value - ) From 26ecda1445fa7ab3ab598312b69c9bde852e4568 Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 12 Apr 2023 13:25:18 +0100 Subject: [PATCH 2/5] Updating the test data instructions and the test data. --- HOW_TO_UPDATE_TESTS.md | 2 +- .../input/new_and_updated_documents.json | 96 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/HOW_TO_UPDATE_TESTS.md b/HOW_TO_UPDATE_TESTS.md index b15bbbc7..701c543c 100644 --- a/HOW_TO_UPDATE_TESTS.md +++ b/HOW_TO_UPDATE_TESTS.md @@ -24,7 +24,7 @@ Build the docker image locally Set up the test buckets - python setup_test_buckets ${document_bucket} ${pipeline_bucket} ${region} + python -m integration_tests.setup_test_buckets ${document_bucket} ${pipeline_bucket} ${region} Sync the test data to the s3 bucket diff --git a/integration_tests/data/pipeline_in/input/new_and_updated_documents.json b/integration_tests/data/pipeline_in/input/new_and_updated_documents.json index 59670d13..c2f28066 100644 --- a/integration_tests/data/pipeline_in/input/new_and_updated_documents.json +++ b/integration_tests/data/pipeline_in/input/new_and_updated_documents.json @@ -838,6 +838,102 @@ } ], "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" + }, + { + "publication_ts": "2013-01-01T00:00:00", + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "source_url": "http://existing.com", + "url": null, + "md5_sum": null, + "type": "EU Decision", + "source": "CCLW", + "import_id": "TESTCCLW.executive.3.3", + "category": "Law", + "frameworks": [], + "geography": "EUR", + "hazards": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ], + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" + }, + { + "publication_ts": "2013-01-01T00:00:00", + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "description", + "source_url": "http://existing.com", + "url": null, + "md5_sum": null, + "type": "EU Decision", + "source": "CCLW", + "import_id": "TESTCCLW.executive.4.4", + "category": "Law", + "frameworks": [], + "geography": "EUR", + "hazards": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ], + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" } ], "updated_documents": { From 37ae70b293544bf7db842b01c26dc9622e67ab4c Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 12 Apr 2023 13:48:22 +0100 Subject: [PATCH 3/5] Updating the tests and the instructions inline with the new parse functionality. --- HOW_TO_UPDATE_TESTS.md | 9 +- .../2023-03-29-17-29-45..json | 1 - .../2023-04-12-12-42-37..json | 32 ++++++ .../2023-03-29-17-29-47..json | 1 - .../2023-04-12-12-42-40..json | 1 + ...9-17-29-45.npy => 2023-04-12-12-42-40.npy} | Bin ...9-17-29-45.npy => 2023-04-12-12-42-39.npy} | Bin .../2023-03-29-17-29-45..json | 1 - .../2023-04-12-12-42-37..json | 32 ++++++ ...17-29-45..npy => 2023-04-12-12-42-37..npy} | Bin .../2023-03-29-17-29-47..json | 1 - .../2023-04-12-12-42-40..json | 1 + ...9-17-29-45.npy => 2023-04-12-12-42-40.npy} | Bin .../2023-04-12-12-42-37..json | 11 ++ .../2023-04-12-12-42-40..json | 1 + .../2023-04-12-12-40-04..json | 56 ++++++++++ .../2023-04-12-12-40-05..json | 1 + .../TESTCCLW.executive.3.3.json | 57 ++++++++++- .../TESTCCLW.executive.4.4.json | 57 ++++++++++- .../input/new_and_updated_documents.json | 96 ++++++++++++++++++ .../new_and_updated_documents.json_errors | 34 ++++--- .../TESTCCLW.executive.1332.1547.json | 60 +++++++++++ .../TESTCCLW.executive.1332.1548.json | 60 +++++++++++ .../TESTCCLW.executive.1332.1549.json | 77 ++++++++++++++ .../TESTCCLW.executive.1332.1550.json | 77 ++++++++++++++ .../TESTCCLW.executive.1332.1551.json | 77 ++++++++++++++ .../TESTCCLW.executive.1332.1552.json | 53 ++++++++++ .../TESTCCLW.executive.1332.1553.json | 56 ++++++++++ .../TESTCCLW.executive.1332.1554.json | 56 ++++++++++ .../TESTCCLW.executive.1332.1555.json | 44 ++++++++ .../TESTCCLW.executive.1332.1556.json | 57 +++++++++++ .../TESTCCLW.executive.1332.1557.json | 57 +++++++++++ .../TESTCCLW.executive.1332.1558.json | 57 +++++++++++ .../TESTCCLW.executive.1332.1559.json | 62 +++++++++++ .../TESTCCLW.executive.1332.1560.json | 62 +++++++++++ .../TESTCCLW.executive.1332.1562.json | 56 ++++++++++ .../TESTCCLW.executive.1332.1563.json | 56 ++++++++++ .../parser_input/TESTCCLW.executive.3.3.json | 56 ++++++++++ .../parser_input/TESTCCLW.executive.4.4.json | 56 ++++++++++ 39 files changed, 1447 insertions(+), 24 deletions(-) delete mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..json create mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json delete mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-03-29-17-29-47..json create mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/{2023-03-29-17-29-45.npy => 2023-04-12-12-42-40.npy} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/{2023-03-29-17-29-45.npy => 2023-04-12-12-42-39.npy} (100%) delete mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..json create mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/{2023-03-29-17-29-45..npy => 2023-04-12-12-42-37..npy} (100%) delete mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-03-29-17-29-47..json create mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/{2023-03-29-17-29-45.npy => 2023-04-12-12-42-40.npy} (100%) create mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json create mode 100644 integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json create mode 100644 integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.3.3/2023-04-12-12-40-04..json create mode 100644 integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.4.4/2023-04-12-12-40-05..json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1547.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1548.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1549.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1550.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1551.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1552.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1553.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1554.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1555.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1556.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1557.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1558.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1559.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1560.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1562.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1563.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.3.3.json create mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.4.4.json diff --git a/HOW_TO_UPDATE_TESTS.md b/HOW_TO_UPDATE_TESTS.md index 701c543c..0598f2c7 100644 --- a/HOW_TO_UPDATE_TESTS.md +++ b/HOW_TO_UPDATE_TESTS.md @@ -30,11 +30,16 @@ Sync the test data to the s3 bucket aws s3 sync integration_tests/data/pipeline_in s3://${pipeline_bucket} -Run the docker image +Run the docker image. If you are trying to figure out what the variables are look in the env var section of the following file: .github/workflows/integration-tests.yml. Also note that the prefixes used must match the subdirectory names of the data/pipeline_in directory. docker run -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -e API_HOST="" -e MACHINE_USER_EMAIL="" -e MACHINE_USER_PASSWORD="" navigator-data-ingest-test --pipeline-bucket ${PIPELINE_BUCKET} --document-bucket ${DOCUMENT_BUCKET} --input-file ${TEST_DATA_UPLOAD_PATH} --output-prefix ${OUTPUT_PREFIX} --embeddings-input-prefix ${EMBEDDINGS_INPUT_PREFIX} --indexer-input-prefix ${INDEXER_INPUT_PREFIX} -Assert that the output is correct and if so snyc the data locally to the pipeline_out directory +Example: + + docker run -e AWS_ACCESS_KEY_ID=XXXX -e AWS_SECRET_ACCESS_KEY=XXXX -e API_HOST="" -e MACHINE_USER_EMAIL="" -e MACHINE_USER_PASSWORD="" navigator-data-ingest-test --pipeline-bucket pipbucket123123123 --document-bucket docbucket123123123 --input-file input/new_and_updated_documents.json --output-prefix ingest_unit_test_parser_input --embeddings-input-prefix ingest_unit_test_embeddings_input --indexer-input-prefix ingest_unit_test_indexer_input + + +Assert that the output is correct and if so manually delete all the files in the pipeline_out directory and sync the data locally to the pipeline_out directory cd integration_tests/data/pipeline_out diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..json deleted file mode 100644 index 3daf5f83..00000000 --- a/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..json +++ /dev/null @@ -1 +0,0 @@ -{"document_name": "name", "document_description": "description", "document_id": "TESTCCLW.executive.3.3", "document_source_url": "http://new.com", "document_cdn_object": null, "document_content_type": "text/html", "document_md5_sum": null, "document_metadata": {}, "document_slug": "fake_slug", "languages": ["en"], "translated": false, "html_data": {"detected_title": "One Stop Shop Service", "detected_date": null, "has_valid_text": true, "text_blocks": [{"text": ["Why use a One Stop Shop"], "text_block_id": "b0", "language": "en", "type": "Text", "type_confidence": 1.0}]}, "pdf_data": null} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json new file mode 100644 index 00000000..3199a55f --- /dev/null +++ b/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json @@ -0,0 +1,32 @@ +{ + "document_name": "name", + "document_description": "description", + "document_id": "TESTCCLW.executive.3.3", + "document_source_url": "http://existing.com", + "document_cdn_object": null, + "document_content_type": "text/html", + "document_md5_sum": null, + "document_metadata": {}, + "document_slug": "fake_slug", + "languages": [ + "en" + ], + "translated": false, + "html_data": { + "detected_title": "One Stop Shop Service", + "detected_date": null, + "has_valid_text": true, + "text_blocks": [ + { + "text": [ + "Why use a One Stop Shop" + ], + "text_block_id": "b0", + "language": "en", + "type": "Text", + "type_confidence": 1.0 + } + ] + }, + "pdf_data": null +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-03-29-17-29-47..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-03-29-17-29-47..json deleted file mode 100644 index a829e058..00000000 --- a/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-03-29-17-29-47..json +++ /dev/null @@ -1 +0,0 @@ -{"document_name": "name", "document_description": "new description", "document_id": "TESTCCLW.executive.4.4", "document_source_url": "http://new.com", "document_cdn_object": null, "document_content_type": "text/html", "document_md5_sum": null, "document_metadata": {}, "document_slug": "fake_slug", "languages": ["en"], "translated": false, "html_data": {"detected_title": "One Stop Shop Service", "detected_date": null, "has_valid_text": true, "text_blocks": [{"text": ["Why use a One Stop Shop"], "text_block_id": "b0", "language": "en", "type": "Text", "type_confidence": 1.0}]}, "pdf_data": null} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json new file mode 100644 index 00000000..cbe861b6 --- /dev/null +++ b/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json @@ -0,0 +1 @@ +{"document_name": "name", "document_description": "new description", "document_id": "TESTCCLW.executive.4.4", "document_source_url": "http://existing.com", "document_cdn_object": null, "document_content_type": "text/html", "document_md5_sum": null, "document_metadata": {}, "document_slug": "fake_slug", "languages": ["en"], "translated": false, "html_data": {"detected_title": "One Stop Shop Service", "detected_date": null, "has_valid_text": true, "text_blocks": [{"text": ["Why use a One Stop Shop"], "text_block_id": "b0", "language": "en", "type": "Text", "type_confidence": 1.0}]}, "pdf_data": null} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/2023-03-29-17-29-45.npy b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/2023-04-12-12-42-40.npy similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/2023-03-29-17-29-45.npy rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/2023-04-12-12-42-40.npy diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/2023-03-29-17-29-45.npy b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/2023-04-12-12-42-39.npy similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/2023-03-29-17-29-45.npy rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/2023-04-12-12-42-39.npy diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..json deleted file mode 100644 index 3daf5f83..00000000 --- a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..json +++ /dev/null @@ -1 +0,0 @@ -{"document_name": "name", "document_description": "description", "document_id": "TESTCCLW.executive.3.3", "document_source_url": "http://new.com", "document_cdn_object": null, "document_content_type": "text/html", "document_md5_sum": null, "document_metadata": {}, "document_slug": "fake_slug", "languages": ["en"], "translated": false, "html_data": {"detected_title": "One Stop Shop Service", "detected_date": null, "has_valid_text": true, "text_blocks": [{"text": ["Why use a One Stop Shop"], "text_block_id": "b0", "language": "en", "type": "Text", "type_confidence": 1.0}]}, "pdf_data": null} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json new file mode 100644 index 00000000..3199a55f --- /dev/null +++ b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json @@ -0,0 +1,32 @@ +{ + "document_name": "name", + "document_description": "description", + "document_id": "TESTCCLW.executive.3.3", + "document_source_url": "http://existing.com", + "document_cdn_object": null, + "document_content_type": "text/html", + "document_md5_sum": null, + "document_metadata": {}, + "document_slug": "fake_slug", + "languages": [ + "en" + ], + "translated": false, + "html_data": { + "detected_title": "One Stop Shop Service", + "detected_date": null, + "has_valid_text": true, + "text_blocks": [ + { + "text": [ + "Why use a One Stop Shop" + ], + "text_block_id": "b0", + "language": "en", + "type": "Text", + "type_confidence": 1.0 + } + ] + }, + "pdf_data": null +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..npy b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..npy similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-03-29-17-29-45..npy rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..npy diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-03-29-17-29-47..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-03-29-17-29-47..json deleted file mode 100644 index a829e058..00000000 --- a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-03-29-17-29-47..json +++ /dev/null @@ -1 +0,0 @@ -{"document_name": "name", "document_description": "new description", "document_id": "TESTCCLW.executive.4.4", "document_source_url": "http://new.com", "document_cdn_object": null, "document_content_type": "text/html", "document_md5_sum": null, "document_metadata": {}, "document_slug": "fake_slug", "languages": ["en"], "translated": false, "html_data": {"detected_title": "One Stop Shop Service", "detected_date": null, "has_valid_text": true, "text_blocks": [{"text": ["Why use a One Stop Shop"], "text_block_id": "b0", "language": "en", "type": "Text", "type_confidence": 1.0}]}, "pdf_data": null} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json new file mode 100644 index 00000000..cbe861b6 --- /dev/null +++ b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json @@ -0,0 +1 @@ +{"document_name": "name", "document_description": "new description", "document_id": "TESTCCLW.executive.4.4", "document_source_url": "http://existing.com", "document_cdn_object": null, "document_content_type": "text/html", "document_md5_sum": null, "document_metadata": {}, "document_slug": "fake_slug", "languages": ["en"], "translated": false, "html_data": {"detected_title": "One Stop Shop Service", "detected_date": null, "has_valid_text": true, "text_blocks": [{"text": ["Why use a One Stop Shop"], "text_block_id": "b0", "language": "en", "type": "Text", "type_confidence": 1.0}]}, "pdf_data": null} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-03-29-17-29-45.npy b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40.npy similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-03-29-17-29-45.npy rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40.npy diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json new file mode 100644 index 00000000..70403c76 --- /dev/null +++ b/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json @@ -0,0 +1,11 @@ +{ + "document_name": "name", + "document_description": "description", + "document_id": "TESTCCLW.executive.3.3", + "document_source_url": "http://existing.com", + "document_cdn_object": null, + "document_content_type": "text/html", + "document_md5_sum": null, + "document_metadata": {}, + "document_slug": "fake_slug" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json new file mode 100644 index 00000000..e38493ba --- /dev/null +++ b/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json @@ -0,0 +1 @@ +{"document_name": "name", "document_description": "new description", "document_id": "TESTCCLW.executive.4.4", "document_source_url": "http://existing.com", "document_cdn_object": null, "document_content_type": "text/html", "document_md5_sum": null, "document_metadata": {}, "document_slug": "fake_slug"} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.3.3/2023-04-12-12-40-04..json b/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.3.3/2023-04-12-12-40-04..json new file mode 100644 index 00000000..6f51010c --- /dev/null +++ b/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.3.3/2023-04-12-12-40-04..json @@ -0,0 +1,56 @@ +{ + "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "document_description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "document_id": "TESTCCLW.executive.3.3", + "document_source_url": "http://existing.com", + "document_cdn_object": null, + "document_content_type": null, + "document_md5_sum": null, + "document_metadata": { + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "import_id": "TESTCCLW.executive.3.3", + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", + "publication_ts": "2013-01-01T00:00:00", + "source_url": "http://existing.com", + "type": "EU Decision", + "source": "CCLW", + "category": "Law", + "geography": "EUR", + "frameworks": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "hazards": [], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ] + }, + "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.4.4/2023-04-12-12-40-05..json b/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.4.4/2023-04-12-12-40-05..json new file mode 100644 index 00000000..2883e5c1 --- /dev/null +++ b/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.4.4/2023-04-12-12-40-05..json @@ -0,0 +1 @@ +{"document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", "document_description": "new description", "document_id": "TESTCCLW.executive.4.4", "document_source_url": "http://existing.com", "document_cdn_object": null, "document_content_type": null, "document_md5_sum": null, "document_metadata": {"name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", "description": "description", "import_id": "TESTCCLW.executive.4.4", "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", "publication_ts": "2013-01-01T00:00:00", "source_url": "http://existing.com", "type": "EU Decision", "source": "CCLW", "category": "Law", "geography": "EUR", "frameworks": [], "instruments": ["Capacity building|Governance", "Education, training and knowledge dissemination|Information"], "hazards": [], "keywords": ["Adaptation", "Institutions / Administrative Arrangements", "Research And Development", "Energy Supply", "Energy Demand", "REDD+ And LULUCF", "Transport"], "languages": ["English"], "sectors": ["Economy-wide", "Health", "Transport"], "topics": ["Adaptation", "Mitigation"], "events": [{"name": "Law passed", "description": "", "created_ts": "2013-11-20T00:00:00"}]}, "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017"} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/ingest_unit_test_parser_input/TESTCCLW.executive.3.3.json b/integration_tests/data/pipeline_out/ingest_unit_test_parser_input/TESTCCLW.executive.3.3.json index ffe60e54..6f51010c 100644 --- a/integration_tests/data/pipeline_out/ingest_unit_test_parser_input/TESTCCLW.executive.3.3.json +++ b/integration_tests/data/pipeline_out/ingest_unit_test_parser_input/TESTCCLW.executive.3.3.json @@ -1 +1,56 @@ -{"document_name": "name", "document_description": "description", "document_id": "TESTCCLW.executive.3.3", "document_source_url": "http://new.com", "document_cdn_object": null, "document_content_type": "text/html", "document_md5_sum": null, "document_metadata": {}, "document_slug": "fake_slug"} \ No newline at end of file +{ + "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "document_description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "document_id": "TESTCCLW.executive.3.3", + "document_source_url": "http://existing.com", + "document_cdn_object": null, + "document_content_type": null, + "document_md5_sum": null, + "document_metadata": { + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "import_id": "TESTCCLW.executive.3.3", + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", + "publication_ts": "2013-01-01T00:00:00", + "source_url": "http://existing.com", + "type": "EU Decision", + "source": "CCLW", + "category": "Law", + "geography": "EUR", + "frameworks": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "hazards": [], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ] + }, + "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/ingest_unit_test_parser_input/TESTCCLW.executive.4.4.json b/integration_tests/data/pipeline_out/ingest_unit_test_parser_input/TESTCCLW.executive.4.4.json index 9e280b63..77aaeb85 100644 --- a/integration_tests/data/pipeline_out/ingest_unit_test_parser_input/TESTCCLW.executive.4.4.json +++ b/integration_tests/data/pipeline_out/ingest_unit_test_parser_input/TESTCCLW.executive.4.4.json @@ -1 +1,56 @@ -{"document_name": "name", "document_description": "new description", "document_id": "TESTCCLW.executive.4.4", "document_source_url": "http://new.com", "document_cdn_object": null, "document_content_type": "text/html", "document_md5_sum": null, "document_metadata": {}, "document_slug": "fake_slug"} \ No newline at end of file +{ + "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "document_description": "description", + "document_id": "TESTCCLW.executive.4.4", + "document_source_url": "http://existing.com", + "document_cdn_object": null, + "document_content_type": null, + "document_md5_sum": null, + "document_metadata": { + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "description", + "import_id": "TESTCCLW.executive.4.4", + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", + "publication_ts": "2013-01-01T00:00:00", + "source_url": "http://existing.com", + "type": "EU Decision", + "source": "CCLW", + "category": "Law", + "geography": "EUR", + "frameworks": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "hazards": [], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ] + }, + "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/input/new_and_updated_documents.json b/integration_tests/data/pipeline_out/input/new_and_updated_documents.json index 59670d13..c2f28066 100644 --- a/integration_tests/data/pipeline_out/input/new_and_updated_documents.json +++ b/integration_tests/data/pipeline_out/input/new_and_updated_documents.json @@ -838,6 +838,102 @@ } ], "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" + }, + { + "publication_ts": "2013-01-01T00:00:00", + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "source_url": "http://existing.com", + "url": null, + "md5_sum": null, + "type": "EU Decision", + "source": "CCLW", + "import_id": "TESTCCLW.executive.3.3", + "category": "Law", + "frameworks": [], + "geography": "EUR", + "hazards": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ], + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" + }, + { + "publication_ts": "2013-01-01T00:00:00", + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "description", + "source_url": "http://existing.com", + "url": null, + "md5_sum": null, + "type": "EU Decision", + "source": "CCLW", + "import_id": "TESTCCLW.executive.4.4", + "category": "Law", + "frameworks": [], + "geography": "EUR", + "hazards": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ], + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" } ], "updated_documents": { diff --git a/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors b/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors index 2483a00c..d132e48f 100644 --- a/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors +++ b/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors @@ -1,18 +1,20 @@ [ - "ERROR ingesting 'TESTCCLW.executive.1332.1548': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1547': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1549': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1550': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1551': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1554': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1553': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1555': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1552': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1556': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1559': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1562': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1557': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1558': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1560': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1563': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n" + "ERROR ingesting 'TESTCCLW.executive.1332.1547': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1550': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1549': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1548': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1551': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1554': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1553': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1555': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1552': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1556': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1559': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1558': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1557': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1560': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1562': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1563': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.3.3': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.4.4': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n" ] \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1547.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1547.json new file mode 100644 index 00000000..f11049f6 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1547.json @@ -0,0 +1,60 @@ +{ + "document_name": "Presidential of the Republic of Indonesia Instruction Number 6 Year 2013 on Suspension of New Licenses and Improving Forest Governance of Primary Forest and Peatland", + "document_description": "The first iteration of this instruction was issued in 2011 in order to implement commitments under the agreements in the Letter of Intent signed with the Kingdom of Norway in May 2011. The Instruction is intended to facilitate Indonesia's participation in internationally financed REDD activities and places a moratorium on clearance of primary peatland and forests within moratorium areas.\u00a0The initial moratorium was extended by Presidential Instruction 6/2013.\u00a0In 2019, President Joko Widodo signed Presidential Instruction 5/2019, making the moratorium on the clearance of primary forest and peatlands in moratorium areas permanent.\u00a0", + "document_id": "TESTCCLW.executive.1332.1547", + "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaUVHIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--2edc173ca6e688a1d121c90f6282721aca9b0ca7/f", + "document_cdn_object": "IDN/2013/presidential-of-the-republic-of-indonesia-instruction-number-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland_3e465fc3e780f4933960e339bd0e70f6.pdf", + "document_content_type": "application/pdf", + "document_md5_sum": "3e465fc3e780f4933960e339bd0e70f6", + "document_metadata": { + "name": "Presidential of the Republic of Indonesia Instruction Number 6 Year 2013 on Suspension of New Licenses and Improving Forest Governance of Primary Forest and Peatland", + "description": "The first iteration of this instruction was issued in 2011 in order to implement commitments under the agreements in the Letter of Intent signed with the Kingdom of Norway in May 2011. The Instruction is intended to facilitate Indonesia's participation in internationally financed REDD activities and places a moratorium on clearance of primary peatland and forests within moratorium areas.\u00a0The initial moratorium was extended by Presidential Instruction 6/2013.\u00a0In 2019, President Joko Widodo signed Presidential Instruction 5/2019, making the moratorium on the clearance of primary forest and peatlands in moratorium areas permanent.\u00a0", + "import_id": "TESTCCLW.executive.1332.1547", + "slug": "indonesia_2013_presidential-instruction-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland", + "publication_ts": "2013-01-01T00:00:00", + "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaUVHIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--2edc173ca6e688a1d121c90f6282721aca9b0ca7/f", + "type": "Decree", + "source": "CCLW", + "category": "Policy", + "geography": "IDN", + "frameworks": [], + "instruments": [ + "Processes, plans and strategies|Governance" + ], + "hazards": [], + "keywords": [ + "REDD+ And LULUCF", + "Forests", + "Moratorium" + ], + "languages": [ + "Indonesian" + ], + "sectors": [ + "LULUCF", + "Residential and Commercial" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Presidential instruction issued", + "description": "", + "created_ts": "2011-12-25T00:00:00" + }, + { + "name": "Moratorium extended", + "description": "", + "created_ts": "2013-12-25T00:00:00" + }, + { + "name": "Moratorium extended", + "description": "", + "created_ts": "2019-08-07T00:00:00" + } + ] + }, + "document_slug": "indonesia_2013_presidential-instruction-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1548.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1548.json new file mode 100644 index 00000000..e6e37b6f --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1548.json @@ -0,0 +1,60 @@ +{ + "document_name": "Presidential of the Republic of Indonesia Instruction Number 6 Year 2013 on Suspension of New Licenses and Improving Forest Governance of Primary Forest and Peatland", + "document_description": "The first iteration of this instruction was issued in 2011 in order to implement commitments under the agreements in the Letter of Intent signed with the Kingdom of Norway in May 2011. The Instruction is intended to facilitate Indonesia's participation in internationally financed REDD activities and places a moratorium on clearance of primary peatland and forests within moratorium areas.\u00a0The initial moratorium was extended by Presidential Instruction 6/2013.\u00a0In 2019, President Joko Widodo signed Presidential Instruction 5/2019, making the moratorium on the clearance of primary forest and peatlands in moratorium areas permanent.\u00a0", + "document_id": "TESTCCLW.executive.1332.1548", + "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaUlHIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--8c435aa888628a4bbbef25307cab8c50b5be954a/f", + "document_cdn_object": "IDN/2013/presidential-of-the-republic-of-indonesia-instruction-number-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland_030ef732d9e7a72dc9f984daa66d219f.pdf", + "document_content_type": "application/pdf", + "document_md5_sum": "030ef732d9e7a72dc9f984daa66d219f", + "document_metadata": { + "name": "Presidential of the Republic of Indonesia Instruction Number 6 Year 2013 on Suspension of New Licenses and Improving Forest Governance of Primary Forest and Peatland", + "description": "The first iteration of this instruction was issued in 2011 in order to implement commitments under the agreements in the Letter of Intent signed with the Kingdom of Norway in May 2011. The Instruction is intended to facilitate Indonesia's participation in internationally financed REDD activities and places a moratorium on clearance of primary peatland and forests within moratorium areas.\u00a0The initial moratorium was extended by Presidential Instruction 6/2013.\u00a0In 2019, President Joko Widodo signed Presidential Instruction 5/2019, making the moratorium on the clearance of primary forest and peatlands in moratorium areas permanent.\u00a0", + "import_id": "TESTCCLW.executive.1332.1548", + "slug": "england_2013_presidential-of-the-republic-of-indonesia-instruction-number-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland", + "publication_ts": "2013-01-01T00:00:00", + "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaUlHIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--8c435aa888628a4bbbef25307cab8c50b5be954a/f", + "type": "Decree", + "source": "CCLW", + "category": "Policy", + "geography": "IDN", + "frameworks": [], + "instruments": [ + "Processes, plans and strategies|Governance" + ], + "hazards": [], + "keywords": [ + "REDD+ And LULUCF", + "Forests", + "Moratorium" + ], + "languages": [ + "English" + ], + "sectors": [ + "LULUCF", + "Residential and Commercial" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Presidential instruction issued", + "description": "", + "created_ts": "2011-12-25T00:00:00" + }, + { + "name": "Moratorium extended", + "description": "", + "created_ts": "2013-12-25T00:00:00" + }, + { + "name": "Moratorium extended", + "description": "", + "created_ts": "2019-08-07T00:00:00" + } + ] + }, + "document_slug": "england_2013_presidential-of-the-republic-of-indonesia-instruction-number-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1549.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1549.json new file mode 100644 index 00000000..3f95f9f0 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1549.json @@ -0,0 +1,77 @@ +{ + "document_name": "Act on Promotion of Global Warming Countermeasures", + "document_description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", + "document_id": "TESTCCLW.executive.1332.1549", + "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaGtJIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--f1aa609515dfd46437ed7ca9d2ab7a036564df28/f", + "document_cdn_object": "JPN/1998/act-on-promotion-of-global-warming-countermeasures_ef51c825593afef9b6db8892c0bb372f.pdf", + "document_content_type": "application/pdf", + "document_md5_sum": "ef51c825593afef9b6db8892c0bb372f", + "document_metadata": { + "name": "Act on Promotion of Global Warming Countermeasures", + "description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", + "import_id": "TESTCCLW.executive.1332.1549", + "slug": "japan_1998_act-on-promotion-of-global-warming-countermeasures", + "publication_ts": "1998-01-01T00:00:00", + "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaGtJIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--f1aa609515dfd46437ed7ca9d2ab7a036564df28/f", + "type": "Act", + "source": "CCLW", + "category": "Law", + "geography": "JPN", + "frameworks": [ + "Mitigation" + ], + "instruments": [ + "Capacity building|Governance", + "Institutional mandates|Governance", + "Processes, plans and strategies|Governance", + "Subnational and citizen participation|Governance", + "MRV|Governance" + ], + "hazards": [], + "keywords": [ + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "Transport" + ], + "languages": [ + "Japanese" + ], + "sectors": [ + "Energy", + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "1998-10-09T00:00:00" + }, + { + "name": "Entry into force", + "description": "", + "created_ts": "2005-02-16T00:00:00" + }, + { + "name": "Amendment", + "description": "", + "created_ts": "2014-05-30T00:00:00" + }, + { + "name": "Plan adopted", + "description": "", + "created_ts": "2016-05-01T00:00:00" + }, + { + "name": "climate neutral goal adopted", + "description": "", + "created_ts": "2021-06-04T00:00:00" + } + ] + }, + "document_slug": "japan_1998_act-on-promotion-of-global-warming-countermeasures" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1550.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1550.json new file mode 100644 index 00000000..fc1a3961 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1550.json @@ -0,0 +1,77 @@ +{ + "document_name": "Cabinet decisions for a law proposal to partially amend the law on the promotion of global warming countermeasures", + "document_description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", + "document_id": "TESTCCLW.executive.1332.1550", + "document_source_url": "https://www.env.go.jp/press/109218.html", + "document_cdn_object": null, + "document_content_type": "text/html", + "document_md5_sum": null, + "document_metadata": { + "name": "Cabinet decisions for a law proposal to partially amend the law on the promotion of global warming countermeasures", + "description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", + "import_id": "TESTCCLW.executive.1332.1550", + "slug": "japan_2020_cabinet-decisions-for-a-law-proposal-to-partially-amend-the-law-on-the-promotion-of-global-warming-countermeasures", + "publication_ts": "2020-01-01T00:00:00", + "source_url": "https://www.env.go.jp/press/109218.html", + "type": "Decision", + "source": "CCLW", + "category": "Law", + "geography": "JPN", + "frameworks": [ + "Mitigation" + ], + "instruments": [ + "Capacity building|Governance", + "Institutional mandates|Governance", + "Processes, plans and strategies|Governance", + "Subnational and citizen participation|Governance", + "MRV|Governance" + ], + "hazards": [], + "keywords": [ + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "Transport" + ], + "languages": [ + "Japanese" + ], + "sectors": [ + "Energy", + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "1998-10-09T00:00:00" + }, + { + "name": "Entry into force", + "description": "", + "created_ts": "2005-02-16T00:00:00" + }, + { + "name": "Amendment", + "description": "", + "created_ts": "2014-05-30T00:00:00" + }, + { + "name": "Plan adopted", + "description": "", + "created_ts": "2016-05-01T00:00:00" + }, + { + "name": "climate neutral goal adopted", + "description": "", + "created_ts": "2021-06-04T00:00:00" + } + ] + }, + "document_slug": "japan_2020_cabinet-decisions-for-a-law-proposal-to-partially-amend-the-law-on-the-promotion-of-global-warming-countermeasures" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1551.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1551.json new file mode 100644 index 00000000..38299fe2 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1551.json @@ -0,0 +1,77 @@ +{ + "document_name": "Revised Global Warming Countermeasures Promotion Law", + "document_description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", + "document_id": "TESTCCLW.executive.1332.1551", + "document_source_url": "https://www.env.go.jp/press/ontaihou/116348.pdf", + "document_cdn_object": "JPN/2021/revised-global-warming-countermeasures-promotion-law_5401db7b173822e3ba86e5230ae9fd90.pdf", + "document_content_type": "application/pdf", + "document_md5_sum": "5401db7b173822e3ba86e5230ae9fd90", + "document_metadata": { + "name": "Revised Global Warming Countermeasures Promotion Law", + "description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", + "import_id": "TESTCCLW.executive.1332.1551", + "slug": "japan_2021_revised-global-warming-countermeasures-promotion-law", + "publication_ts": "2021-01-01T00:00:00", + "source_url": "https://www.env.go.jp/press/ontaihou/116348.pdf", + "type": "Law", + "source": "CCLW", + "category": "Law", + "geography": "JPN", + "frameworks": [ + "Mitigation" + ], + "instruments": [ + "Capacity building|Governance", + "Institutional mandates|Governance", + "Processes, plans and strategies|Governance", + "Subnational and citizen participation|Governance", + "MRV|Governance" + ], + "hazards": [], + "keywords": [ + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "Transport" + ], + "languages": [ + "Japanese" + ], + "sectors": [ + "Energy", + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "1998-10-09T00:00:00" + }, + { + "name": "Entry into force", + "description": "", + "created_ts": "2005-02-16T00:00:00" + }, + { + "name": "Amendment", + "description": "", + "created_ts": "2014-05-30T00:00:00" + }, + { + "name": "Plan adopted", + "description": "", + "created_ts": "2016-05-01T00:00:00" + }, + { + "name": "climate neutral goal adopted", + "description": "", + "created_ts": "2021-06-04T00:00:00" + } + ] + }, + "document_slug": "japan_2021_revised-global-warming-countermeasures-promotion-law" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1552.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1552.json new file mode 100644 index 00000000..7f8f4709 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1552.json @@ -0,0 +1,53 @@ +{ + "document_name": "Basic Hydrogen Strategy", + "document_description": "This document sets Japan's vision on how to achieve a hydrogen-based society by 2050 and provides an action plan for its realisation. It specifically seeks to decarbonise the energy, industry and transportation sectors.The Strategic Roadmap for Hydrogen and Fuel Cells defines 1) new targets on the specification of basic technologies and the breakdown of costs, 2) necessary measures for achieving these goals; and 3) that Japan will convene a working group consisting of experts to review the status of implementation in each area stipulated by the roadmap.", + "document_id": "TESTCCLW.executive.1332.1552", + "document_source_url": "https://www.meti.go.jp/english/press/2017/pdf/1226_003b.pdf", + "document_cdn_object": null, + "document_content_type": null, + "document_md5_sum": null, + "document_metadata": { + "name": "Basic Hydrogen Strategy", + "description": "This document sets Japan's vision on how to achieve a hydrogen-based society by 2050 and provides an action plan for its realisation. It specifically seeks to decarbonise the energy, industry and transportation sectors.The Strategic Roadmap for Hydrogen and Fuel Cells defines 1) new targets on the specification of basic technologies and the breakdown of costs, 2) necessary measures for achieving these goals; and 3) that Japan will convene a working group consisting of experts to review the status of implementation in each area stipulated by the roadmap.", + "import_id": "TESTCCLW.executive.1332.1552", + "slug": "japan_2017_basic-hydrogen-strategy", + "publication_ts": "2017-01-01T00:00:00", + "source_url": "https://www.meti.go.jp/english/press/2017/pdf/1226_003b.pdf", + "type": "Strategy", + "source": "CCLW", + "category": "Policy", + "geography": "JPN", + "frameworks": [], + "instruments": [ + "Processes, plans and strategies|Governance" + ], + "hazards": [], + "keywords": [ + "Hydrogen" + ], + "languages": [ + "English" + ], + "sectors": [ + "Energy", + "Industry", + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "Released", + "description": "", + "created_ts": "2017-12-26T00:00:00" + }, + { + "name": "Amended", + "description": "", + "created_ts": "2019-05-12T00:00:00" + } + ] + }, + "document_slug": "japan_2017_basic-hydrogen-strategy" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1553.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1553.json new file mode 100644 index 00000000..2b767372 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1553.json @@ -0,0 +1,56 @@ +{ + "document_name": "Spanish Climate Change And Clean Energy Strategy Horizon 2007- 2012 -2020", + "document_description": "The Spanish Climate Change and Clean Energy Strategy (EECCEL) horizon 2007-2012-2020 is part of the Spanish Sustainable Development Strategy (EEDS). The EECCEL includes different measures that contribute to sustainable development within the scope of climate change and clean energy.\u00a0\u00a0This Strategy is based on the reference framework of the 'Spanish Strategy for the fulfilment of the objectives under the Kyoto Protocol', and it takes into account the measures and Programmes adopted by the Autonomous Communities.\u00a0\u00a0The strategy has two chapters. The first one defines actions to fight against climate change and the second one, actions to achieve cleaner energy. Each chapter includes a description of the present situation, the objectives to be reached, the suggested measures and a selection of indicators for the corresponding follow-up.\u00a0\u00a0The operational objectives are:\u00a0- To ensure the reduction of GHG emissions in Spain, giving special importance to measures related to the energy sector. According to the national inventory, in 2005, emissions from energy process represented about 78.87% of total national emissions.\u00a0- To contribute to sustainable development and the fulfilment of climate change commitments by strengthening the use of flexible project-based mechanisms.\u00a0- To promote additional reduction measures in sectors concerned with diffuse pollution.\u00a0- To apply the National Climate Change Adaptation Plan (NCCAP) so as to integrate adaptation measures and strategies in sectoral policies.\u00a0- To increase public awareness with respect to clean energy and climate change.\u00a0- To promote research, development and innovation in matters of climate change and clean energy.\u00a0- To guarantee energy supply security by means of cleaner energies, mainly from renewable sources, achieving other environmental benefits (for example, air quality) and limiting the growth rate of external energy dependence.\u00a0- To boost energy- and resource efficiency for companies and for end users.\u00a0\u00a0The government has adopted a Plan of Urgent Measures (PMU), which together with the 2008-2012 Energy Saving and Efficiency Action Plan aims to consolidate the trend change of GHG emissions in Spain initiated in 2006.", + "document_id": "TESTCCLW.executive.1332.1553", + "document_source_url": "https://www.lse.ac.uk/GranthamInstitute/wp-content/uploads/laws/1674%20English.pdf", + "document_cdn_object": "ESP/2007/spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020_2b072ba79a80c48468424f57a5937e93.pdf", + "document_content_type": "application/pdf", + "document_md5_sum": "2b072ba79a80c48468424f57a5937e93", + "document_metadata": { + "name": "Spanish Climate Change And Clean Energy Strategy Horizon 2007- 2012 -2020", + "description": "The Spanish Climate Change and Clean Energy Strategy (EECCEL) horizon 2007-2012-2020 is part of the Spanish Sustainable Development Strategy (EEDS). The EECCEL includes different measures that contribute to sustainable development within the scope of climate change and clean energy.\u00a0\u00a0This Strategy is based on the reference framework of the 'Spanish Strategy for the fulfilment of the objectives under the Kyoto Protocol', and it takes into account the measures and Programmes adopted by the Autonomous Communities.\u00a0\u00a0The strategy has two chapters. The first one defines actions to fight against climate change and the second one, actions to achieve cleaner energy. Each chapter includes a description of the present situation, the objectives to be reached, the suggested measures and a selection of indicators for the corresponding follow-up.\u00a0\u00a0The operational objectives are:\u00a0- To ensure the reduction of GHG emissions in Spain, giving special importance to measures related to the energy sector. According to the national inventory, in 2005, emissions from energy process represented about 78.87% of total national emissions.\u00a0- To contribute to sustainable development and the fulfilment of climate change commitments by strengthening the use of flexible project-based mechanisms.\u00a0- To promote additional reduction measures in sectors concerned with diffuse pollution.\u00a0- To apply the National Climate Change Adaptation Plan (NCCAP) so as to integrate adaptation measures and strategies in sectoral policies.\u00a0- To increase public awareness with respect to clean energy and climate change.\u00a0- To promote research, development and innovation in matters of climate change and clean energy.\u00a0- To guarantee energy supply security by means of cleaner energies, mainly from renewable sources, achieving other environmental benefits (for example, air quality) and limiting the growth rate of external energy dependence.\u00a0- To boost energy- and resource efficiency for companies and for end users.\u00a0\u00a0The government has adopted a Plan of Urgent Measures (PMU), which together with the 2008-2012 Energy Saving and Efficiency Action Plan aims to consolidate the trend change of GHG emissions in Spain initiated in 2006.", + "import_id": "TESTCCLW.executive.1332.1553", + "slug": "spain_2007_spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020", + "publication_ts": "2007-01-01T00:00:00", + "source_url": "https://www.lse.ac.uk/GranthamInstitute/wp-content/uploads/laws/1674%20English.pdf", + "type": "Strategy", + "source": "CCLW", + "category": "Policy", + "geography": "ESP", + "frameworks": [ + "Mitigation" + ], + "instruments": [ + "Capacity building|Governance", + "Processes, plans and strategies|Governance", + "Education, training and knowledge dissemination|Information" + ], + "hazards": [], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Energy" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2007-11-02T00:00:00" + } + ] + }, + "document_slug": "spain_2007_spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1554.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1554.json new file mode 100644 index 00000000..c90f9100 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1554.json @@ -0,0 +1,56 @@ +{ + "document_name": "Spanish Climate Change Strategy and Clean Energy Horizon 2007-202020", + "document_description": "The Spanish Climate Change and Clean Energy Strategy (EECCEL) horizon 2007-2012-2020 is part of the Spanish Sustainable Development Strategy (EEDS). The EECCEL includes different measures that contribute to sustainable development within the scope of climate change and clean energy.\u00a0\u00a0This Strategy is based on the reference framework of the 'Spanish Strategy for the fulfilment of the objectives under the Kyoto Protocol', and it takes into account the measures and Programmes adopted by the Autonomous Communities.\u00a0\u00a0The strategy has two chapters. The first one defines actions to fight against climate change and the second one, actions to achieve cleaner energy. Each chapter includes a description of the present situation, the objectives to be reached, the suggested measures and a selection of indicators for the corresponding follow-up.\u00a0\u00a0The operational objectives are:\u00a0- To ensure the reduction of GHG emissions in Spain, giving special importance to measures related to the energy sector. According to the national inventory, in 2005, emissions from energy process represented about 78.87% of total national emissions.\u00a0- To contribute to sustainable development and the fulfilment of climate change commitments by strengthening the use of flexible project-based mechanisms.\u00a0- To promote additional reduction measures in sectors concerned with diffuse pollution.\u00a0- To apply the National Climate Change Adaptation Plan (NCCAP) so as to integrate adaptation measures and strategies in sectoral policies.\u00a0- To increase public awareness with respect to clean energy and climate change.\u00a0- To promote research, development and innovation in matters of climate change and clean energy.\u00a0- To guarantee energy supply security by means of cleaner energies, mainly from renewable sources, achieving other environmental benefits (for example, air quality) and limiting the growth rate of external energy dependence.\u00a0- To boost energy- and resource efficiency for companies and for end users.\u00a0\u00a0The government has adopted a Plan of Urgent Measures (PMU), which together with the 2008-2012 Energy Saving and Efficiency Action Plan aims to consolidate the trend change of GHG emissions in Spain initiated in 2006.", + "document_id": "TESTCCLW.executive.1332.1554", + "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBclVLIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--0389b20592032a285dc31b2977adf54b040d8b98/1674%20Spanish.pdf", + "document_cdn_object": "ESP/2007/spanish-climate-change-strategy-and-clean-energy-horizon-2007-202020_7a8f5e1c8ed4b78d5e7a96eb09c2b561.pdf", + "document_content_type": "application/pdf", + "document_md5_sum": "7a8f5e1c8ed4b78d5e7a96eb09c2b561", + "document_metadata": { + "name": "Spanish Climate Change Strategy and Clean Energy Horizon 2007-202020", + "description": "The Spanish Climate Change and Clean Energy Strategy (EECCEL) horizon 2007-2012-2020 is part of the Spanish Sustainable Development Strategy (EEDS). The EECCEL includes different measures that contribute to sustainable development within the scope of climate change and clean energy.\u00a0\u00a0This Strategy is based on the reference framework of the 'Spanish Strategy for the fulfilment of the objectives under the Kyoto Protocol', and it takes into account the measures and Programmes adopted by the Autonomous Communities.\u00a0\u00a0The strategy has two chapters. The first one defines actions to fight against climate change and the second one, actions to achieve cleaner energy. Each chapter includes a description of the present situation, the objectives to be reached, the suggested measures and a selection of indicators for the corresponding follow-up.\u00a0\u00a0The operational objectives are:\u00a0- To ensure the reduction of GHG emissions in Spain, giving special importance to measures related to the energy sector. According to the national inventory, in 2005, emissions from energy process represented about 78.87% of total national emissions.\u00a0- To contribute to sustainable development and the fulfilment of climate change commitments by strengthening the use of flexible project-based mechanisms.\u00a0- To promote additional reduction measures in sectors concerned with diffuse pollution.\u00a0- To apply the National Climate Change Adaptation Plan (NCCAP) so as to integrate adaptation measures and strategies in sectoral policies.\u00a0- To increase public awareness with respect to clean energy and climate change.\u00a0- To promote research, development and innovation in matters of climate change and clean energy.\u00a0- To guarantee energy supply security by means of cleaner energies, mainly from renewable sources, achieving other environmental benefits (for example, air quality) and limiting the growth rate of external energy dependence.\u00a0- To boost energy- and resource efficiency for companies and for end users.\u00a0\u00a0The government has adopted a Plan of Urgent Measures (PMU), which together with the 2008-2012 Energy Saving and Efficiency Action Plan aims to consolidate the trend change of GHG emissions in Spain initiated in 2006.", + "import_id": "TESTCCLW.executive.1332.1554", + "slug": "spain_2007_spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020", + "publication_ts": "2007-01-01T00:00:00", + "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBclVLIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--0389b20592032a285dc31b2977adf54b040d8b98/1674%20Spanish.pdf", + "type": "Strategy", + "source": "CCLW", + "category": "Policy", + "geography": "ESP", + "frameworks": [ + "Mitigation" + ], + "instruments": [ + "Capacity building|Governance", + "Processes, plans and strategies|Governance", + "Education, training and knowledge dissemination|Information" + ], + "hazards": [], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "Transport" + ], + "languages": [ + "Spanish" + ], + "sectors": [ + "Energy" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2007-11-02T00:00:00" + } + ] + }, + "document_slug": "spain_2007_spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1555.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1555.json new file mode 100644 index 00000000..d12faa54 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1555.json @@ -0,0 +1,44 @@ +{ + "document_name": "Carbon Farming Initiative Amendment Bill 2014", + "document_description": "This bill establishes the Emissions Reduction Fund, and provides a transition for the Carbon Farming Initiative by amending the Carbon Credits (Carbon Farming Initiative) Act 2011 to provide for the Clean Energy Regulator to conduct auctions and enter into contracts to purchase emissions reductions; enable a broader range of emissions reduction projects to be approved; and amend the project eligibility criteria and processes for approving projects and crediting carbon credit units.\n\nThe Carbon Credits (Carbon Farming Initiative) Act 2011 set up a scheme for the issue of Australian Carbon Credit Units (ACCUs) in relation to eligible offsets projects. It was part of a package of three Acts to establish the Carbon Farming Initiative, including the Australian National Registry of Emissions Units Act 2011 and the Carbon Credits (Consequential Amendments) Act 2011. The package of Acts established the existence of ACCUs as personal property, which are generally transferable, and can be sold domestically or internationally subject to regulation.", + "document_id": "TESTCCLW.executive.1332.1555", + "document_source_url": null, + "document_cdn_object": null, + "document_content_type": null, + "document_md5_sum": null, + "document_metadata": { + "name": "Carbon Farming Initiative Amendment Bill 2014", + "description": "This bill establishes the Emissions Reduction Fund, and provides a transition for the Carbon Farming Initiative by amending the Carbon Credits (Carbon Farming Initiative) Act 2011 to provide for the Clean Energy Regulator to conduct auctions and enter into contracts to purchase emissions reductions; enable a broader range of emissions reduction projects to be approved; and amend the project eligibility criteria and processes for approving projects and crediting carbon credit units.\n\nThe Carbon Credits (Carbon Farming Initiative) Act 2011 set up a scheme for the issue of Australian Carbon Credit Units (ACCUs) in relation to eligible offsets projects. It was part of a package of three Acts to establish the Carbon Farming Initiative, including the Australian National Registry of Emissions Units Act 2011 and the Carbon Credits (Consequential Amendments) Act 2011. The package of Acts established the existence of ACCUs as personal property, which are generally transferable, and can be sold domestically or internationally subject to regulation.", + "import_id": "TESTCCLW.executive.1332.1555", + "slug": "australia_2014_carbon-farming-initiative-amendment-bill-2014", + "publication_ts": "2014-12-13T00:00:00", + "source_url": null, + "type": "Law", + "source": "CCLW", + "category": "Law", + "geography": "AUS", + "frameworks": [], + "instruments": [ + "Provision of climate funds|Direct Investment" + ], + "hazards": [], + "keywords": [ + "Carbon Pricing" + ], + "languages": [], + "sectors": [ + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2014-12-13T00:00:00" + } + ] + }, + "document_slug": "australia_2014_carbon-farming-initiative-amendment-bill-2014" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1556.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1556.json new file mode 100644 index 00000000..d9d01a6a --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1556.json @@ -0,0 +1,57 @@ +{ + "document_name": "Act On The Allocation And Trading Of Greenhouse-Gas Emission Permits", + "document_description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", + "document_id": "TESTCCLW.executive.1332.1556", + "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBck1JIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--c465bbadbc173d88ce267d8db9943fb62194160b/f", + "document_cdn_object": "KOR/2012/act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits_bb567c0dfbbbaa960a8b017adf399ee9.pdf", + "document_content_type": "application/pdf", + "document_md5_sum": "bb567c0dfbbbaa960a8b017adf399ee9", + "document_metadata": { + "name": "Act On The Allocation And Trading Of Greenhouse-Gas Emission Permits", + "description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", + "import_id": "TESTCCLW.executive.1332.1556", + "slug": "korea_2012_act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits", + "publication_ts": "2012-01-01T00:00:00", + "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBck1JIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--c465bbadbc173d88ce267d8db9943fb62194160b/f", + "type": "Act", + "source": "CCLW", + "category": "Law", + "geography": "KOR", + "frameworks": [], + "instruments": [ + "Processes, plans and strategies|Governance" + ], + "hazards": [], + "keywords": [ + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand" + ], + "languages": [ + "English" + ], + "sectors": [ + "Energy", + "Industry", + "Residential and Commercial", + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "entry into force of enforcement decree", + "description": "", + "created_ts": "2012-11-15T00:00:00" + }, + { + "name": "Amended", + "description": "", + "created_ts": "2020-06-01T00:00:00" + } + ] + }, + "document_slug": "korea_2012_act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1557.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1557.json new file mode 100644 index 00000000..ef328e3e --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1557.json @@ -0,0 +1,57 @@ +{ + "document_name": "Law on allocation and transactions of greenhouse gas emissions", + "document_description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", + "document_id": "TESTCCLW.executive.1332.1557", + "document_source_url": "https://www.law.go.kr/LSW/lsInfoP.do?efYd=20200601&lsiSeq=215913&ancYd=20200324&nwJoYnInfo=N&ancYnChk=0&ancNo=17104&chrClsCd=010202&efGubun=Y%230000#0000", + "document_cdn_object": null, + "document_content_type": "text/html", + "document_md5_sum": null, + "document_metadata": { + "name": "Law on allocation and transactions of greenhouse gas emissions", + "description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", + "import_id": "TESTCCLW.executive.1332.1557", + "slug": "korea_2012_act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits", + "publication_ts": "2020-01-01T00:00:00", + "source_url": "https://www.law.go.kr/LSW/lsInfoP.do?efYd=20200601&lsiSeq=215913&ancYd=20200324&nwJoYnInfo=N&ancYnChk=0&ancNo=17104&chrClsCd=010202&efGubun=Y%230000#0000", + "type": "Law", + "source": "CCLW", + "category": "Law", + "geography": "KOR", + "frameworks": [], + "instruments": [ + "Processes, plans and strategies|Governance" + ], + "hazards": [], + "keywords": [ + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand" + ], + "languages": [ + "Korean" + ], + "sectors": [ + "Energy", + "Industry", + "Residential and Commercial", + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "entry into force of enforcement decree", + "description": "", + "created_ts": "2012-11-15T00:00:00" + }, + { + "name": "Amended", + "description": "", + "created_ts": "2020-06-01T00:00:00" + } + ] + }, + "document_slug": "korea_2012_act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1558.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1558.json new file mode 100644 index 00000000..72198d78 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1558.json @@ -0,0 +1,57 @@ +{ + "document_name": "Emissions Trading System 3rd Planning period Allocation Plan", + "document_description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", + "document_id": "TESTCCLW.executive.1332.1558", + "document_source_url": "https://ors.gir.go.kr/home/board/read.do?menuId=2&boardMasterId=4&boardId=44", + "document_cdn_object": null, + "document_content_type": "text/html", + "document_md5_sum": null, + "document_metadata": { + "name": "Emissions Trading System 3rd Planning period Allocation Plan", + "description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", + "import_id": "TESTCCLW.executive.1332.1558", + "slug": "korea_2020_emissions-trading-system-3rd-planning-period-allocation-plan_1647_4582", + "publication_ts": "2020-01-01T00:00:00", + "source_url": "https://ors.gir.go.kr/home/board/read.do?menuId=2&boardMasterId=4&boardId=44", + "type": "Plan", + "source": "CCLW", + "category": "Law", + "geography": "KOR", + "frameworks": [], + "instruments": [ + "Processes, plans and strategies|Governance" + ], + "hazards": [], + "keywords": [ + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand" + ], + "languages": [ + "Korean" + ], + "sectors": [ + "Energy", + "Industry", + "Residential and Commercial", + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "entry into force of enforcement decree", + "description": "", + "created_ts": "2012-11-15T00:00:00" + }, + { + "name": "Amended", + "description": "", + "created_ts": "2020-06-01T00:00:00" + } + ] + }, + "document_slug": "korea_2020_emissions-trading-system-3rd-planning-period-allocation-plan_1647_4582" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1559.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1559.json new file mode 100644 index 00000000..a003d3df --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1559.json @@ -0,0 +1,62 @@ +{ + "document_name": "Law relating to the coordination of the federal sustainable development policy", + "document_description": "The latest version of the Act on Co-ordination of Federal Policy on Sustainable Development calls for the development of a long-term vision for Sustainable Development that would include goals for 2050 and be based on interdepartmental and vertical co-operation among between the federal level and federated entities through the specifically created Interdepartmental Commission on Sustainable Development.\u00a0A Federal Sustainable Development Plan is regularly elaborated and lays down specific co-ordination measures. So far, two FSDPs have been adopted, one for 2000-2004, the other for 2004-2008. The second FSDP has been extended due to the political context that prevented elaboration of a Plan for 2008-2012 and remains the current federal SD Plan. Three of the six main objectives of the FSDP are 'Managing natural resources more responsibly', 'Limiting climate change and increasing the use of clean energy', and 'Improving the transport system'. The measures proposed at the federal level in co-ordination with the regions include promoting alternative modes of transport, setting the right price for energy, promoting energy-conserving buildings and providing better public transport.", + "document_id": "TESTCCLW.executive.1332.1559", + "document_source_url": "https://www.ejustice.just.fgov.be/cgi_loi/change_lg.pl?language=fr&la=F&table_name=loi&cn=1997050535%20%20", + "document_cdn_object": null, + "document_content_type": "text/html", + "document_md5_sum": null, + "document_metadata": { + "name": "Law relating to the coordination of the federal sustainable development policy", + "description": "The latest version of the Act on Co-ordination of Federal Policy on Sustainable Development calls for the development of a long-term vision for Sustainable Development that would include goals for 2050 and be based on interdepartmental and vertical co-operation among between the federal level and federated entities through the specifically created Interdepartmental Commission on Sustainable Development.\u00a0A Federal Sustainable Development Plan is regularly elaborated and lays down specific co-ordination measures. So far, two FSDPs have been adopted, one for 2000-2004, the other for 2004-2008. The second FSDP has been extended due to the political context that prevented elaboration of a Plan for 2008-2012 and remains the current federal SD Plan. Three of the six main objectives of the FSDP are 'Managing natural resources more responsibly', 'Limiting climate change and increasing the use of clean energy', and 'Improving the transport system'. The measures proposed at the federal level in co-ordination with the regions include promoting alternative modes of transport, setting the right price for energy, promoting energy-conserving buildings and providing better public transport.", + "import_id": "TESTCCLW.executive.1332.1559", + "slug": "belgium_1997_law-relating-to-the-coordination-of-the-federal-sustainable-development-policy_1080_4597", + "publication_ts": "1997-01-01T00:00:00", + "source_url": "https://www.ejustice.just.fgov.be/cgi_loi/change_lg.pl?language=fr&la=F&table_name=loi&cn=1997050535%20%20", + "type": "Law", + "source": "CCLW", + "category": "Law", + "geography": "BEL", + "frameworks": [], + "instruments": [ + "Institutional mandates|Governance", + "Processes, plans and strategies|Governance", + "Subnational and citizen participation|Governance" + ], + "hazards": [], + "keywords": [ + "Energy Supply", + "Energy Demand", + "Transport" + ], + "languages": [ + "French" + ], + "sectors": [ + "Energy", + "Residential and Commercial", + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "1997-05-05T00:00:00" + }, + { + "name": "climate-relevant document adopted", + "description": "", + "created_ts": "2000-12-25T00:00:00" + }, + { + "name": "Last amended", + "description": "", + "created_ts": "2014-12-04T00:00:00" + } + ] + }, + "document_slug": "belgium_1997_law-relating-to-the-coordination-of-the-federal-sustainable-development-policy_1080_4597" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1560.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1560.json new file mode 100644 index 00000000..64795abb --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1560.json @@ -0,0 +1,62 @@ +{ + "document_name": "The Federal Sustainable Development Plan", + "document_description": "The latest version of the Act on Co-ordination of Federal Policy on Sustainable Development calls for the development of a long-term vision for Sustainable Development that would include goals for 2050 and be based on interdepartmental and vertical co-operation among between the federal level and federated entities through the specifically created Interdepartmental Commission on Sustainable Development.\u00a0A Federal Sustainable Development Plan is regularly elaborated and lays down specific co-ordination measures. So far, two FSDPs have been adopted, one for 2000-2004, the other for 2004-2008. The second FSDP has been extended due to the political context that prevented elaboration of a Plan for 2008-2012 and remains the current federal SD Plan. Three of the six main objectives of the FSDP are 'Managing natural resources more responsibly', 'Limiting climate change and increasing the use of clean energy', and 'Improving the transport system'. The measures proposed at the federal level in co-ordination with the regions include promoting alternative modes of transport, setting the right price for energy, promoting energy-conserving buildings and providing better public transport.", + "document_id": "TESTCCLW.executive.1332.1560", + "document_source_url": "https://www.developpementdurable.be/fr/politique-federale/strategie-federale/instruments/le-plan-federal-de-developpement-durable", + "document_cdn_object": null, + "document_content_type": "text/html", + "document_md5_sum": null, + "document_metadata": { + "name": "The Federal Sustainable Development Plan", + "description": "The latest version of the Act on Co-ordination of Federal Policy on Sustainable Development calls for the development of a long-term vision for Sustainable Development that would include goals for 2050 and be based on interdepartmental and vertical co-operation among between the federal level and federated entities through the specifically created Interdepartmental Commission on Sustainable Development.\u00a0A Federal Sustainable Development Plan is regularly elaborated and lays down specific co-ordination measures. So far, two FSDPs have been adopted, one for 2000-2004, the other for 2004-2008. The second FSDP has been extended due to the political context that prevented elaboration of a Plan for 2008-2012 and remains the current federal SD Plan. Three of the six main objectives of the FSDP are 'Managing natural resources more responsibly', 'Limiting climate change and increasing the use of clean energy', and 'Improving the transport system'. The measures proposed at the federal level in co-ordination with the regions include promoting alternative modes of transport, setting the right price for energy, promoting energy-conserving buildings and providing better public transport.", + "import_id": "TESTCCLW.executive.1332.1560", + "slug": "belgium_2000_the-federal-sustainable-development-plan_1080_4598", + "publication_ts": "2000-01-01T00:00:00", + "source_url": "https://www.developpementdurable.be/fr/politique-federale/strategie-federale/instruments/le-plan-federal-de-developpement-durable", + "type": "Plan", + "source": "CCLW", + "category": "Law", + "geography": "BEL", + "frameworks": [], + "instruments": [ + "Institutional mandates|Governance", + "Processes, plans and strategies|Governance", + "Subnational and citizen participation|Governance" + ], + "hazards": [], + "keywords": [ + "Energy Supply", + "Energy Demand", + "Transport" + ], + "languages": [ + "French" + ], + "sectors": [ + "Energy", + "Residential and Commercial", + "Transport" + ], + "topics": [ + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "1997-05-05T00:00:00" + }, + { + "name": "climate-relevant document adopted", + "description": "", + "created_ts": "2000-12-25T00:00:00" + }, + { + "name": "Last amended", + "description": "", + "created_ts": "2014-12-04T00:00:00" + } + ] + }, + "document_slug": "belgium_2000_the-federal-sustainable-development-plan_1080_4598" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1562.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1562.json new file mode 100644 index 00000000..cddcd5df --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1562.json @@ -0,0 +1,56 @@ +{ + "document_name": "Decision No 1386/2013/EU of the European Parliament and of the Council of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "document_description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "document_id": "TESTCCLW.executive.1332.1562", + "document_source_url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?qid=1516647740078&uri=CELEX:32013D1386", + "document_cdn_object": null, + "document_content_type": "text/html", + "document_md5_sum": null, + "document_metadata": { + "name": "Decision No 1386/2013/EU of the European Parliament and of the Council of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "import_id": "TESTCCLW.executive.1332.1562", + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3016", + "publication_ts": "2013-01-01T00:00:00", + "source_url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?qid=1516647740078&uri=CELEX:32013D1386", + "type": "EU Decision", + "source": "CCLW", + "category": "Law", + "geography": "EUR", + "frameworks": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "hazards": [], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ] + }, + "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3016" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1563.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1563.json new file mode 100644 index 00000000..dd482389 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1563.json @@ -0,0 +1,56 @@ +{ + "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "document_description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "document_id": "TESTCCLW.executive.1332.1563", + "document_source_url": "https://www.lse.ac.uk/GranthamInstitute/wp-content/uploads/2018/02/living-well-planet-EU-CELEX3A32013D13863AEN3ATXT.pdf", + "document_cdn_object": "EUR/2013/decision-no-1386-2013-eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_e2471666433a8ff0fbd46c9fd0db8a67.pdf", + "document_content_type": "application/pdf", + "document_md5_sum": "e2471666433a8ff0fbd46c9fd0db8a67", + "document_metadata": { + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "import_id": "TESTCCLW.executive.1332.1563", + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", + "publication_ts": "2013-01-01T00:00:00", + "source_url": "https://www.lse.ac.uk/GranthamInstitute/wp-content/uploads/2018/02/living-well-planet-EU-CELEX3A32013D13863AEN3ATXT.pdf", + "type": "EU Decision", + "source": "CCLW", + "category": "Law", + "geography": "EUR", + "frameworks": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "hazards": [], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ] + }, + "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.3.3.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.3.3.json new file mode 100644 index 00000000..6f51010c --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.3.3.json @@ -0,0 +1,56 @@ +{ + "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "document_description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "document_id": "TESTCCLW.executive.3.3", + "document_source_url": "http://existing.com", + "document_cdn_object": null, + "document_content_type": null, + "document_md5_sum": null, + "document_metadata": { + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", + "import_id": "TESTCCLW.executive.3.3", + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", + "publication_ts": "2013-01-01T00:00:00", + "source_url": "http://existing.com", + "type": "EU Decision", + "source": "CCLW", + "category": "Law", + "geography": "EUR", + "frameworks": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "hazards": [], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ] + }, + "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" +} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.4.4.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.4.4.json new file mode 100644 index 00000000..77aaeb85 --- /dev/null +++ b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.4.4.json @@ -0,0 +1,56 @@ +{ + "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "document_description": "description", + "document_id": "TESTCCLW.executive.4.4", + "document_source_url": "http://existing.com", + "document_cdn_object": null, + "document_content_type": null, + "document_md5_sum": null, + "document_metadata": { + "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", + "description": "description", + "import_id": "TESTCCLW.executive.4.4", + "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", + "publication_ts": "2013-01-01T00:00:00", + "source_url": "http://existing.com", + "type": "EU Decision", + "source": "CCLW", + "category": "Law", + "geography": "EUR", + "frameworks": [], + "instruments": [ + "Capacity building|Governance", + "Education, training and knowledge dissemination|Information" + ], + "hazards": [], + "keywords": [ + "Adaptation", + "Institutions / Administrative Arrangements", + "Research And Development", + "Energy Supply", + "Energy Demand", + "REDD+ And LULUCF", + "Transport" + ], + "languages": [ + "English" + ], + "sectors": [ + "Economy-wide", + "Health", + "Transport" + ], + "topics": [ + "Adaptation", + "Mitigation" + ], + "events": [ + { + "name": "Law passed", + "description": "", + "created_ts": "2013-11-20T00:00:00" + } + ] + }, + "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" +} \ No newline at end of file From 005a657b1ea2e2a1f107c00cc78ed7b9fd2033b0 Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 12 Apr 2023 13:51:05 +0100 Subject: [PATCH 4/5] Further updates to the instructions. --- HOW_TO_UPDATE_TESTS.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/HOW_TO_UPDATE_TESTS.md b/HOW_TO_UPDATE_TESTS.md index 0598f2c7..a2bc06a1 100644 --- a/HOW_TO_UPDATE_TESTS.md +++ b/HOW_TO_UPDATE_TESTS.md @@ -22,6 +22,11 @@ Build the docker image locally make build_test + +MAKE SURE YOU HAVE THE CORRECT AWS CREDENTIALS SET UP. + + export AWS_PROFILE=${PROFILE_NAME} + Set up the test buckets python -m integration_tests.setup_test_buckets ${document_bucket} ${pipeline_bucket} ${region} @@ -43,4 +48,8 @@ Assert that the output is correct and if so manually delete all the files in the cd integration_tests/data/pipeline_out - aws s3 sync s3://${pipeline_bucket}/ . \ No newline at end of file + aws s3 sync s3://${pipeline_bucket}/ . + +Remove the test buckets + + python -m integration_tests.remove_test_buckets ${document_bucket} ${pipeline_bucket} ${region} \ No newline at end of file From b19e704fc3bbc0907cdeeac69959d1b6abef8aef Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 12 Apr 2023 14:03:40 +0100 Subject: [PATCH 5/5] Correcting the expected output data. --- ...-42-37..json => 2023-04-12-13-01-01..json} | 0 ...-42-40..json => 2023-04-12-13-01-06..json} | 0 ...2-12-42-40.npy => 2023-04-12-13-01-05.npy} | Bin ...2-12-42-39.npy => 2023-04-12-13-01-05.npy} | Bin ...-42-37..json => 2023-04-12-13-01-01..json} | 0 ...12-42-37..npy => 2023-04-12-13-01-01..npy} | Bin ...2-12-42-40.npy => 2023-04-12-13-01-05.npy} | Bin ...-42-40..json => 2023-04-12-13-01-06..json} | 0 ...-42-37..json => 2023-04-12-13-01-01..json} | 0 ...-42-40..json => 2023-04-12-13-01-06..json} | 0 .../2023-04-12-12-40-04..json | 56 ------------- .../2023-04-12-12-40-05..json | 1 - .../new_and_updated_documents.json_errors | 36 ++++---- .../TESTCCLW.executive.1332.1547.json | 60 -------------- .../TESTCCLW.executive.1332.1548.json | 60 -------------- .../TESTCCLW.executive.1332.1549.json | 77 ------------------ .../TESTCCLW.executive.1332.1550.json | 77 ------------------ .../TESTCCLW.executive.1332.1551.json | 77 ------------------ .../TESTCCLW.executive.1332.1552.json | 53 ------------ .../TESTCCLW.executive.1332.1553.json | 56 ------------- .../TESTCCLW.executive.1332.1554.json | 56 ------------- .../TESTCCLW.executive.1332.1555.json | 44 ---------- .../TESTCCLW.executive.1332.1556.json | 57 ------------- .../TESTCCLW.executive.1332.1557.json | 57 ------------- .../TESTCCLW.executive.1332.1558.json | 57 ------------- .../TESTCCLW.executive.1332.1559.json | 62 -------------- .../TESTCCLW.executive.1332.1560.json | 62 -------------- .../TESTCCLW.executive.1332.1562.json | 56 ------------- .../TESTCCLW.executive.1332.1563.json | 56 ------------- .../parser_input/TESTCCLW.executive.3.3.json | 56 ------------- .../parser_input/TESTCCLW.executive.4.4.json | 56 ------------- 31 files changed, 18 insertions(+), 1154 deletions(-) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/{2023-04-12-12-42-37..json => 2023-04-12-13-01-01..json} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/{2023-04-12-12-42-40..json => 2023-04-12-13-01-06..json} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/{2023-04-12-12-42-40.npy => 2023-04-12-13-01-05.npy} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/{2023-04-12-12-42-39.npy => 2023-04-12-13-01-05.npy} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/{2023-04-12-12-42-37..json => 2023-04-12-13-01-01..json} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/{2023-04-12-12-42-37..npy => 2023-04-12-13-01-01..npy} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/{2023-04-12-12-42-40.npy => 2023-04-12-13-01-05.npy} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/{2023-04-12-12-42-40..json => 2023-04-12-13-01-06..json} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.3.3/{2023-04-12-12-42-37..json => 2023-04-12-13-01-01..json} (100%) rename integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.4.4/{2023-04-12-12-42-40..json => 2023-04-12-13-01-06..json} (100%) delete mode 100644 integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.3.3/2023-04-12-12-40-04..json delete mode 100644 integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.4.4/2023-04-12-12-40-05..json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1547.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1548.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1549.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1550.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1551.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1552.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1553.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1554.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1555.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1556.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1557.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1558.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1559.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1560.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1562.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1563.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.3.3.json delete mode 100644 integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.4.4.json diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-04-12-13-01-01..json similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3/2023-04-12-13-01-01..json diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-04-12-13-01-06..json similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4/2023-04-12-13-01-06..json diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/2023-04-12-12-42-40.npy b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/2023-04-12-13-01-05.npy similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/2023-04-12-12-42-40.npy rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1/2023-04-12-13-01-05.npy diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/2023-04-12-12-42-39.npy b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/2023-04-12-13-01-05.npy similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/2023-04-12-12-42-39.npy rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2/2023-04-12-13-01-05.npy diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-13-01-01..json similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-13-01-01..json diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..npy b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-13-01-01..npy similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..npy rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3/2023-04-12-13-01-01..npy diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40.npy b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-13-01-05.npy similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40.npy rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-13-01-05.npy diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-13-01-06..json similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4/2023-04-12-13-01-06..json diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.3.3/2023-04-12-13-01-01..json similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.3.3/2023-04-12-12-42-37..json rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.3.3/2023-04-12-13-01-01..json diff --git a/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json b/integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.4.4/2023-04-12-13-01-06..json similarity index 100% rename from integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.4.4/2023-04-12-12-42-40..json rename to integration_tests/data/pipeline_out/archive/ingest_unit_test_parser_input/TESTCCLW.executive.4.4/2023-04-12-13-01-06..json diff --git a/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.3.3/2023-04-12-12-40-04..json b/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.3.3/2023-04-12-12-40-04..json deleted file mode 100644 index 6f51010c..00000000 --- a/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.3.3/2023-04-12-12-40-04..json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "document_description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", - "document_id": "TESTCCLW.executive.3.3", - "document_source_url": "http://existing.com", - "document_cdn_object": null, - "document_content_type": null, - "document_md5_sum": null, - "document_metadata": { - "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", - "import_id": "TESTCCLW.executive.3.3", - "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", - "publication_ts": "2013-01-01T00:00:00", - "source_url": "http://existing.com", - "type": "EU Decision", - "source": "CCLW", - "category": "Law", - "geography": "EUR", - "frameworks": [], - "instruments": [ - "Capacity building|Governance", - "Education, training and knowledge dissemination|Information" - ], - "hazards": [], - "keywords": [ - "Adaptation", - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "REDD+ And LULUCF", - "Transport" - ], - "languages": [ - "English" - ], - "sectors": [ - "Economy-wide", - "Health", - "Transport" - ], - "topics": [ - "Adaptation", - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "2013-11-20T00:00:00" - } - ] - }, - "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.4.4/2023-04-12-12-40-05..json b/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.4.4/2023-04-12-12-40-05..json deleted file mode 100644 index 2883e5c1..00000000 --- a/integration_tests/data/pipeline_out/archive/parser_input/TESTCCLW.executive.4.4/2023-04-12-12-40-05..json +++ /dev/null @@ -1 +0,0 @@ -{"document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", "document_description": "new description", "document_id": "TESTCCLW.executive.4.4", "document_source_url": "http://existing.com", "document_cdn_object": null, "document_content_type": null, "document_md5_sum": null, "document_metadata": {"name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", "description": "description", "import_id": "TESTCCLW.executive.4.4", "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", "publication_ts": "2013-01-01T00:00:00", "source_url": "http://existing.com", "type": "EU Decision", "source": "CCLW", "category": "Law", "geography": "EUR", "frameworks": [], "instruments": ["Capacity building|Governance", "Education, training and knowledge dissemination|Information"], "hazards": [], "keywords": ["Adaptation", "Institutions / Administrative Arrangements", "Research And Development", "Energy Supply", "Energy Demand", "REDD+ And LULUCF", "Transport"], "languages": ["English"], "sectors": ["Economy-wide", "Health", "Transport"], "topics": ["Adaptation", "Mitigation"], "events": [{"name": "Law passed", "description": "", "created_ts": "2013-11-20T00:00:00"}]}, "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017"} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors b/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors index d132e48f..2b5e34f6 100644 --- a/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors +++ b/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors @@ -1,20 +1,20 @@ [ - "ERROR ingesting 'TESTCCLW.executive.1332.1547': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1550': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1549': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1548': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1551': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1554': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1553': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1555': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1552': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1556': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1559': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1558': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1557': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1560': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1562': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.1332.1563': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.3.3': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", - "ERROR ingesting 'TESTCCLW.executive.4.4': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n" + "ERROR ingesting 'TESTCCLW.executive.1332.1550': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1547': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1549': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1548': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1553': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1551': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1555': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1554': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1552': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1556': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1560': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1559': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1557': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1558': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1563': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.1332.1562': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.3.3': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n", + "ERROR ingesting 'TESTCCLW.executive.4.4': Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 382, in __call__\n result = fn(*args, **kwargs)\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 175, in update_document_details\n token = get_machine_user_token()\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/api_client.py\", line 49, in get_machine_user_token\n get_token_response = requests.post(f\"{api_host}/api/tokens\", data=login_data)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 573, in request\n prep = self.prepare_request(req)\n File \"/usr/local/lib/python3.9/site-packages/requests/sessions.py\", line 484, in prepare_request\n p.prepare(\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 368, in prepare\n self.prepare_url(url, params)\n File \"/usr/local/lib/python3.9/site-packages/requests/models.py\", line 439, in prepare_url\n raise MissingSchema(\nrequests.exceptions.MissingSchema: Invalid URL '/api/tokens': No scheme supplied. Perhaps you meant https:///api/tokens?\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/navigator-data-ingest/src/navigator_data_ingest/base/new_document_actions.py\", line 142, in _handle_document\n update_document_details(\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 289, in wrapped_f\n return self(f, *args, **kw)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 379, in __call__\n do = self.iter(retry_state=retry_state)\n File \"/usr/local/lib/python3.9/site-packages/tenacity/__init__.py\", line 326, in iter\n raise retry_exc from fut.exception()\ntenacity.RetryError: RetryError[]\n" ] \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1547.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1547.json deleted file mode 100644 index f11049f6..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1547.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "document_name": "Presidential of the Republic of Indonesia Instruction Number 6 Year 2013 on Suspension of New Licenses and Improving Forest Governance of Primary Forest and Peatland", - "document_description": "The first iteration of this instruction was issued in 2011 in order to implement commitments under the agreements in the Letter of Intent signed with the Kingdom of Norway in May 2011. The Instruction is intended to facilitate Indonesia's participation in internationally financed REDD activities and places a moratorium on clearance of primary peatland and forests within moratorium areas.\u00a0The initial moratorium was extended by Presidential Instruction 6/2013.\u00a0In 2019, President Joko Widodo signed Presidential Instruction 5/2019, making the moratorium on the clearance of primary forest and peatlands in moratorium areas permanent.\u00a0", - "document_id": "TESTCCLW.executive.1332.1547", - "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaUVHIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--2edc173ca6e688a1d121c90f6282721aca9b0ca7/f", - "document_cdn_object": "IDN/2013/presidential-of-the-republic-of-indonesia-instruction-number-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland_3e465fc3e780f4933960e339bd0e70f6.pdf", - "document_content_type": "application/pdf", - "document_md5_sum": "3e465fc3e780f4933960e339bd0e70f6", - "document_metadata": { - "name": "Presidential of the Republic of Indonesia Instruction Number 6 Year 2013 on Suspension of New Licenses and Improving Forest Governance of Primary Forest and Peatland", - "description": "The first iteration of this instruction was issued in 2011 in order to implement commitments under the agreements in the Letter of Intent signed with the Kingdom of Norway in May 2011. The Instruction is intended to facilitate Indonesia's participation in internationally financed REDD activities and places a moratorium on clearance of primary peatland and forests within moratorium areas.\u00a0The initial moratorium was extended by Presidential Instruction 6/2013.\u00a0In 2019, President Joko Widodo signed Presidential Instruction 5/2019, making the moratorium on the clearance of primary forest and peatlands in moratorium areas permanent.\u00a0", - "import_id": "TESTCCLW.executive.1332.1547", - "slug": "indonesia_2013_presidential-instruction-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland", - "publication_ts": "2013-01-01T00:00:00", - "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaUVHIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--2edc173ca6e688a1d121c90f6282721aca9b0ca7/f", - "type": "Decree", - "source": "CCLW", - "category": "Policy", - "geography": "IDN", - "frameworks": [], - "instruments": [ - "Processes, plans and strategies|Governance" - ], - "hazards": [], - "keywords": [ - "REDD+ And LULUCF", - "Forests", - "Moratorium" - ], - "languages": [ - "Indonesian" - ], - "sectors": [ - "LULUCF", - "Residential and Commercial" - ], - "topics": [ - "Adaptation", - "Mitigation" - ], - "events": [ - { - "name": "Presidential instruction issued", - "description": "", - "created_ts": "2011-12-25T00:00:00" - }, - { - "name": "Moratorium extended", - "description": "", - "created_ts": "2013-12-25T00:00:00" - }, - { - "name": "Moratorium extended", - "description": "", - "created_ts": "2019-08-07T00:00:00" - } - ] - }, - "document_slug": "indonesia_2013_presidential-instruction-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1548.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1548.json deleted file mode 100644 index e6e37b6f..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1548.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "document_name": "Presidential of the Republic of Indonesia Instruction Number 6 Year 2013 on Suspension of New Licenses and Improving Forest Governance of Primary Forest and Peatland", - "document_description": "The first iteration of this instruction was issued in 2011 in order to implement commitments under the agreements in the Letter of Intent signed with the Kingdom of Norway in May 2011. The Instruction is intended to facilitate Indonesia's participation in internationally financed REDD activities and places a moratorium on clearance of primary peatland and forests within moratorium areas.\u00a0The initial moratorium was extended by Presidential Instruction 6/2013.\u00a0In 2019, President Joko Widodo signed Presidential Instruction 5/2019, making the moratorium on the clearance of primary forest and peatlands in moratorium areas permanent.\u00a0", - "document_id": "TESTCCLW.executive.1332.1548", - "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaUlHIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--8c435aa888628a4bbbef25307cab8c50b5be954a/f", - "document_cdn_object": "IDN/2013/presidential-of-the-republic-of-indonesia-instruction-number-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland_030ef732d9e7a72dc9f984daa66d219f.pdf", - "document_content_type": "application/pdf", - "document_md5_sum": "030ef732d9e7a72dc9f984daa66d219f", - "document_metadata": { - "name": "Presidential of the Republic of Indonesia Instruction Number 6 Year 2013 on Suspension of New Licenses and Improving Forest Governance of Primary Forest and Peatland", - "description": "The first iteration of this instruction was issued in 2011 in order to implement commitments under the agreements in the Letter of Intent signed with the Kingdom of Norway in May 2011. The Instruction is intended to facilitate Indonesia's participation in internationally financed REDD activities and places a moratorium on clearance of primary peatland and forests within moratorium areas.\u00a0The initial moratorium was extended by Presidential Instruction 6/2013.\u00a0In 2019, President Joko Widodo signed Presidential Instruction 5/2019, making the moratorium on the clearance of primary forest and peatlands in moratorium areas permanent.\u00a0", - "import_id": "TESTCCLW.executive.1332.1548", - "slug": "england_2013_presidential-of-the-republic-of-indonesia-instruction-number-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland", - "publication_ts": "2013-01-01T00:00:00", - "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaUlHIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--8c435aa888628a4bbbef25307cab8c50b5be954a/f", - "type": "Decree", - "source": "CCLW", - "category": "Policy", - "geography": "IDN", - "frameworks": [], - "instruments": [ - "Processes, plans and strategies|Governance" - ], - "hazards": [], - "keywords": [ - "REDD+ And LULUCF", - "Forests", - "Moratorium" - ], - "languages": [ - "English" - ], - "sectors": [ - "LULUCF", - "Residential and Commercial" - ], - "topics": [ - "Adaptation", - "Mitigation" - ], - "events": [ - { - "name": "Presidential instruction issued", - "description": "", - "created_ts": "2011-12-25T00:00:00" - }, - { - "name": "Moratorium extended", - "description": "", - "created_ts": "2013-12-25T00:00:00" - }, - { - "name": "Moratorium extended", - "description": "", - "created_ts": "2019-08-07T00:00:00" - } - ] - }, - "document_slug": "england_2013_presidential-of-the-republic-of-indonesia-instruction-number-6-year-2013-on-suspension-of-new-licenses-and-improving-forest-governance-of-primary-forest-and-peatland" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1549.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1549.json deleted file mode 100644 index 3f95f9f0..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1549.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "document_name": "Act on Promotion of Global Warming Countermeasures", - "document_description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", - "document_id": "TESTCCLW.executive.1332.1549", - "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaGtJIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--f1aa609515dfd46437ed7ca9d2ab7a036564df28/f", - "document_cdn_object": "JPN/1998/act-on-promotion-of-global-warming-countermeasures_ef51c825593afef9b6db8892c0bb372f.pdf", - "document_content_type": "application/pdf", - "document_md5_sum": "ef51c825593afef9b6db8892c0bb372f", - "document_metadata": { - "name": "Act on Promotion of Global Warming Countermeasures", - "description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", - "import_id": "TESTCCLW.executive.1332.1549", - "slug": "japan_1998_act-on-promotion-of-global-warming-countermeasures", - "publication_ts": "1998-01-01T00:00:00", - "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBaGtJIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--f1aa609515dfd46437ed7ca9d2ab7a036564df28/f", - "type": "Act", - "source": "CCLW", - "category": "Law", - "geography": "JPN", - "frameworks": [ - "Mitigation" - ], - "instruments": [ - "Capacity building|Governance", - "Institutional mandates|Governance", - "Processes, plans and strategies|Governance", - "Subnational and citizen participation|Governance", - "MRV|Governance" - ], - "hazards": [], - "keywords": [ - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "Transport" - ], - "languages": [ - "Japanese" - ], - "sectors": [ - "Energy", - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "1998-10-09T00:00:00" - }, - { - "name": "Entry into force", - "description": "", - "created_ts": "2005-02-16T00:00:00" - }, - { - "name": "Amendment", - "description": "", - "created_ts": "2014-05-30T00:00:00" - }, - { - "name": "Plan adopted", - "description": "", - "created_ts": "2016-05-01T00:00:00" - }, - { - "name": "climate neutral goal adopted", - "description": "", - "created_ts": "2021-06-04T00:00:00" - } - ] - }, - "document_slug": "japan_1998_act-on-promotion-of-global-warming-countermeasures" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1550.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1550.json deleted file mode 100644 index fc1a3961..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1550.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "document_name": "Cabinet decisions for a law proposal to partially amend the law on the promotion of global warming countermeasures", - "document_description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", - "document_id": "TESTCCLW.executive.1332.1550", - "document_source_url": "https://www.env.go.jp/press/109218.html", - "document_cdn_object": null, - "document_content_type": "text/html", - "document_md5_sum": null, - "document_metadata": { - "name": "Cabinet decisions for a law proposal to partially amend the law on the promotion of global warming countermeasures", - "description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", - "import_id": "TESTCCLW.executive.1332.1550", - "slug": "japan_2020_cabinet-decisions-for-a-law-proposal-to-partially-amend-the-law-on-the-promotion-of-global-warming-countermeasures", - "publication_ts": "2020-01-01T00:00:00", - "source_url": "https://www.env.go.jp/press/109218.html", - "type": "Decision", - "source": "CCLW", - "category": "Law", - "geography": "JPN", - "frameworks": [ - "Mitigation" - ], - "instruments": [ - "Capacity building|Governance", - "Institutional mandates|Governance", - "Processes, plans and strategies|Governance", - "Subnational and citizen participation|Governance", - "MRV|Governance" - ], - "hazards": [], - "keywords": [ - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "Transport" - ], - "languages": [ - "Japanese" - ], - "sectors": [ - "Energy", - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "1998-10-09T00:00:00" - }, - { - "name": "Entry into force", - "description": "", - "created_ts": "2005-02-16T00:00:00" - }, - { - "name": "Amendment", - "description": "", - "created_ts": "2014-05-30T00:00:00" - }, - { - "name": "Plan adopted", - "description": "", - "created_ts": "2016-05-01T00:00:00" - }, - { - "name": "climate neutral goal adopted", - "description": "", - "created_ts": "2021-06-04T00:00:00" - } - ] - }, - "document_slug": "japan_2020_cabinet-decisions-for-a-law-proposal-to-partially-amend-the-law-on-the-promotion-of-global-warming-countermeasures" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1551.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1551.json deleted file mode 100644 index 38299fe2..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1551.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "document_name": "Revised Global Warming Countermeasures Promotion Law", - "document_description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", - "document_id": "TESTCCLW.executive.1332.1551", - "document_source_url": "https://www.env.go.jp/press/ontaihou/116348.pdf", - "document_cdn_object": "JPN/2021/revised-global-warming-countermeasures-promotion-law_5401db7b173822e3ba86e5230ae9fd90.pdf", - "document_content_type": "application/pdf", - "document_md5_sum": "5401db7b173822e3ba86e5230ae9fd90", - "document_metadata": { - "name": "Revised Global Warming Countermeasures Promotion Law", - "description": "This Law is one of the two key climate laws in Japan along with the Energy Conservation Law. The purpose of the Law is to reduce emissions of GHGs derived from anthropogenic activities. GHGs are carbon dioxide, methane, nitrous oxide, HFC, PFC and sulphur hexafluoride. The Council of Ministers for Global Environmental Conservation is established under the Law. The Council is chaired by the Prime Minister, and vice-chairmen are the Chief Cabinet Secretary, Minister of the Environment and Minister of Economy, Trade and Industry. Other members consist of all ministers other than vice-chairmen.\u00a0\u00a0Designated emitters, whose workplaces contain more than 1,500kL of oil equivalent of energy annually, are mandated to develop the Plan for Global Warming Countermeasure. While there is no reduction obligation under this law, annual emission of GHGs are reported to the Minister in charge. Emission reporting under this framework equals that of the reporting under the Energy Conservation Law.\u00a0\u00a0This Law stipulates that the State is responsible for implementing necessary measures to introduce Emission Trading Scheme (ETS) in Japan. It adds that examination and discussion of the design and the utilization of ETS starts upon the enactment of this Law.\u00a0\u00a0This Law also provides that the national and local governments are responsible for development and implementation of plans to reduce GHG emissions.\u00a0\u00a0The National Government adopted the Plan for Global Warming Countermeasures in May 2016, with the explicit aim to achieve the mid-term target set in Japan's INDC (2015) 26% GHG emissions reduction by 2030 (baseline 2013). In addition, the Plan also sets a long term goal of 80% GHG emissions reduction by 2050.\u00a0 Prefectural and municipal governments are also required to create local plans to reduce GHG emissions. The plans should include:\u00a0Duration of the planGoalsMeasures and actions intended for implementationPromotion of solar PV, wind and other renewable energiesMeasures and actions taken by business professionals and citizens to reduce GHG emissionPromotion of public transport use, conservation of green space and other GHG emission reduction measuresOn June 4th, 2021, the Diet approved the amending Act 54/2021 introducing a net zero target by 2050 into the law.", - "import_id": "TESTCCLW.executive.1332.1551", - "slug": "japan_2021_revised-global-warming-countermeasures-promotion-law", - "publication_ts": "2021-01-01T00:00:00", - "source_url": "https://www.env.go.jp/press/ontaihou/116348.pdf", - "type": "Law", - "source": "CCLW", - "category": "Law", - "geography": "JPN", - "frameworks": [ - "Mitigation" - ], - "instruments": [ - "Capacity building|Governance", - "Institutional mandates|Governance", - "Processes, plans and strategies|Governance", - "Subnational and citizen participation|Governance", - "MRV|Governance" - ], - "hazards": [], - "keywords": [ - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "Transport" - ], - "languages": [ - "Japanese" - ], - "sectors": [ - "Energy", - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "1998-10-09T00:00:00" - }, - { - "name": "Entry into force", - "description": "", - "created_ts": "2005-02-16T00:00:00" - }, - { - "name": "Amendment", - "description": "", - "created_ts": "2014-05-30T00:00:00" - }, - { - "name": "Plan adopted", - "description": "", - "created_ts": "2016-05-01T00:00:00" - }, - { - "name": "climate neutral goal adopted", - "description": "", - "created_ts": "2021-06-04T00:00:00" - } - ] - }, - "document_slug": "japan_2021_revised-global-warming-countermeasures-promotion-law" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1552.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1552.json deleted file mode 100644 index 7f8f4709..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1552.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "document_name": "Basic Hydrogen Strategy", - "document_description": "This document sets Japan's vision on how to achieve a hydrogen-based society by 2050 and provides an action plan for its realisation. It specifically seeks to decarbonise the energy, industry and transportation sectors.The Strategic Roadmap for Hydrogen and Fuel Cells defines 1) new targets on the specification of basic technologies and the breakdown of costs, 2) necessary measures for achieving these goals; and 3) that Japan will convene a working group consisting of experts to review the status of implementation in each area stipulated by the roadmap.", - "document_id": "TESTCCLW.executive.1332.1552", - "document_source_url": "https://www.meti.go.jp/english/press/2017/pdf/1226_003b.pdf", - "document_cdn_object": null, - "document_content_type": null, - "document_md5_sum": null, - "document_metadata": { - "name": "Basic Hydrogen Strategy", - "description": "This document sets Japan's vision on how to achieve a hydrogen-based society by 2050 and provides an action plan for its realisation. It specifically seeks to decarbonise the energy, industry and transportation sectors.The Strategic Roadmap for Hydrogen and Fuel Cells defines 1) new targets on the specification of basic technologies and the breakdown of costs, 2) necessary measures for achieving these goals; and 3) that Japan will convene a working group consisting of experts to review the status of implementation in each area stipulated by the roadmap.", - "import_id": "TESTCCLW.executive.1332.1552", - "slug": "japan_2017_basic-hydrogen-strategy", - "publication_ts": "2017-01-01T00:00:00", - "source_url": "https://www.meti.go.jp/english/press/2017/pdf/1226_003b.pdf", - "type": "Strategy", - "source": "CCLW", - "category": "Policy", - "geography": "JPN", - "frameworks": [], - "instruments": [ - "Processes, plans and strategies|Governance" - ], - "hazards": [], - "keywords": [ - "Hydrogen" - ], - "languages": [ - "English" - ], - "sectors": [ - "Energy", - "Industry", - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "Released", - "description": "", - "created_ts": "2017-12-26T00:00:00" - }, - { - "name": "Amended", - "description": "", - "created_ts": "2019-05-12T00:00:00" - } - ] - }, - "document_slug": "japan_2017_basic-hydrogen-strategy" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1553.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1553.json deleted file mode 100644 index 2b767372..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1553.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "document_name": "Spanish Climate Change And Clean Energy Strategy Horizon 2007- 2012 -2020", - "document_description": "The Spanish Climate Change and Clean Energy Strategy (EECCEL) horizon 2007-2012-2020 is part of the Spanish Sustainable Development Strategy (EEDS). The EECCEL includes different measures that contribute to sustainable development within the scope of climate change and clean energy.\u00a0\u00a0This Strategy is based on the reference framework of the 'Spanish Strategy for the fulfilment of the objectives under the Kyoto Protocol', and it takes into account the measures and Programmes adopted by the Autonomous Communities.\u00a0\u00a0The strategy has two chapters. The first one defines actions to fight against climate change and the second one, actions to achieve cleaner energy. Each chapter includes a description of the present situation, the objectives to be reached, the suggested measures and a selection of indicators for the corresponding follow-up.\u00a0\u00a0The operational objectives are:\u00a0- To ensure the reduction of GHG emissions in Spain, giving special importance to measures related to the energy sector. According to the national inventory, in 2005, emissions from energy process represented about 78.87% of total national emissions.\u00a0- To contribute to sustainable development and the fulfilment of climate change commitments by strengthening the use of flexible project-based mechanisms.\u00a0- To promote additional reduction measures in sectors concerned with diffuse pollution.\u00a0- To apply the National Climate Change Adaptation Plan (NCCAP) so as to integrate adaptation measures and strategies in sectoral policies.\u00a0- To increase public awareness with respect to clean energy and climate change.\u00a0- To promote research, development and innovation in matters of climate change and clean energy.\u00a0- To guarantee energy supply security by means of cleaner energies, mainly from renewable sources, achieving other environmental benefits (for example, air quality) and limiting the growth rate of external energy dependence.\u00a0- To boost energy- and resource efficiency for companies and for end users.\u00a0\u00a0The government has adopted a Plan of Urgent Measures (PMU), which together with the 2008-2012 Energy Saving and Efficiency Action Plan aims to consolidate the trend change of GHG emissions in Spain initiated in 2006.", - "document_id": "TESTCCLW.executive.1332.1553", - "document_source_url": "https://www.lse.ac.uk/GranthamInstitute/wp-content/uploads/laws/1674%20English.pdf", - "document_cdn_object": "ESP/2007/spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020_2b072ba79a80c48468424f57a5937e93.pdf", - "document_content_type": "application/pdf", - "document_md5_sum": "2b072ba79a80c48468424f57a5937e93", - "document_metadata": { - "name": "Spanish Climate Change And Clean Energy Strategy Horizon 2007- 2012 -2020", - "description": "The Spanish Climate Change and Clean Energy Strategy (EECCEL) horizon 2007-2012-2020 is part of the Spanish Sustainable Development Strategy (EEDS). The EECCEL includes different measures that contribute to sustainable development within the scope of climate change and clean energy.\u00a0\u00a0This Strategy is based on the reference framework of the 'Spanish Strategy for the fulfilment of the objectives under the Kyoto Protocol', and it takes into account the measures and Programmes adopted by the Autonomous Communities.\u00a0\u00a0The strategy has two chapters. The first one defines actions to fight against climate change and the second one, actions to achieve cleaner energy. Each chapter includes a description of the present situation, the objectives to be reached, the suggested measures and a selection of indicators for the corresponding follow-up.\u00a0\u00a0The operational objectives are:\u00a0- To ensure the reduction of GHG emissions in Spain, giving special importance to measures related to the energy sector. According to the national inventory, in 2005, emissions from energy process represented about 78.87% of total national emissions.\u00a0- To contribute to sustainable development and the fulfilment of climate change commitments by strengthening the use of flexible project-based mechanisms.\u00a0- To promote additional reduction measures in sectors concerned with diffuse pollution.\u00a0- To apply the National Climate Change Adaptation Plan (NCCAP) so as to integrate adaptation measures and strategies in sectoral policies.\u00a0- To increase public awareness with respect to clean energy and climate change.\u00a0- To promote research, development and innovation in matters of climate change and clean energy.\u00a0- To guarantee energy supply security by means of cleaner energies, mainly from renewable sources, achieving other environmental benefits (for example, air quality) and limiting the growth rate of external energy dependence.\u00a0- To boost energy- and resource efficiency for companies and for end users.\u00a0\u00a0The government has adopted a Plan of Urgent Measures (PMU), which together with the 2008-2012 Energy Saving and Efficiency Action Plan aims to consolidate the trend change of GHG emissions in Spain initiated in 2006.", - "import_id": "TESTCCLW.executive.1332.1553", - "slug": "spain_2007_spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020", - "publication_ts": "2007-01-01T00:00:00", - "source_url": "https://www.lse.ac.uk/GranthamInstitute/wp-content/uploads/laws/1674%20English.pdf", - "type": "Strategy", - "source": "CCLW", - "category": "Policy", - "geography": "ESP", - "frameworks": [ - "Mitigation" - ], - "instruments": [ - "Capacity building|Governance", - "Processes, plans and strategies|Governance", - "Education, training and knowledge dissemination|Information" - ], - "hazards": [], - "keywords": [ - "Adaptation", - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "Transport" - ], - "languages": [ - "English" - ], - "sectors": [ - "Energy" - ], - "topics": [ - "Adaptation", - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "2007-11-02T00:00:00" - } - ] - }, - "document_slug": "spain_2007_spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1554.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1554.json deleted file mode 100644 index c90f9100..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1554.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "document_name": "Spanish Climate Change Strategy and Clean Energy Horizon 2007-202020", - "document_description": "The Spanish Climate Change and Clean Energy Strategy (EECCEL) horizon 2007-2012-2020 is part of the Spanish Sustainable Development Strategy (EEDS). The EECCEL includes different measures that contribute to sustainable development within the scope of climate change and clean energy.\u00a0\u00a0This Strategy is based on the reference framework of the 'Spanish Strategy for the fulfilment of the objectives under the Kyoto Protocol', and it takes into account the measures and Programmes adopted by the Autonomous Communities.\u00a0\u00a0The strategy has two chapters. The first one defines actions to fight against climate change and the second one, actions to achieve cleaner energy. Each chapter includes a description of the present situation, the objectives to be reached, the suggested measures and a selection of indicators for the corresponding follow-up.\u00a0\u00a0The operational objectives are:\u00a0- To ensure the reduction of GHG emissions in Spain, giving special importance to measures related to the energy sector. According to the national inventory, in 2005, emissions from energy process represented about 78.87% of total national emissions.\u00a0- To contribute to sustainable development and the fulfilment of climate change commitments by strengthening the use of flexible project-based mechanisms.\u00a0- To promote additional reduction measures in sectors concerned with diffuse pollution.\u00a0- To apply the National Climate Change Adaptation Plan (NCCAP) so as to integrate adaptation measures and strategies in sectoral policies.\u00a0- To increase public awareness with respect to clean energy and climate change.\u00a0- To promote research, development and innovation in matters of climate change and clean energy.\u00a0- To guarantee energy supply security by means of cleaner energies, mainly from renewable sources, achieving other environmental benefits (for example, air quality) and limiting the growth rate of external energy dependence.\u00a0- To boost energy- and resource efficiency for companies and for end users.\u00a0\u00a0The government has adopted a Plan of Urgent Measures (PMU), which together with the 2008-2012 Energy Saving and Efficiency Action Plan aims to consolidate the trend change of GHG emissions in Spain initiated in 2006.", - "document_id": "TESTCCLW.executive.1332.1554", - "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBclVLIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--0389b20592032a285dc31b2977adf54b040d8b98/1674%20Spanish.pdf", - "document_cdn_object": "ESP/2007/spanish-climate-change-strategy-and-clean-energy-horizon-2007-202020_7a8f5e1c8ed4b78d5e7a96eb09c2b561.pdf", - "document_content_type": "application/pdf", - "document_md5_sum": "7a8f5e1c8ed4b78d5e7a96eb09c2b561", - "document_metadata": { - "name": "Spanish Climate Change Strategy and Clean Energy Horizon 2007-202020", - "description": "The Spanish Climate Change and Clean Energy Strategy (EECCEL) horizon 2007-2012-2020 is part of the Spanish Sustainable Development Strategy (EEDS). The EECCEL includes different measures that contribute to sustainable development within the scope of climate change and clean energy.\u00a0\u00a0This Strategy is based on the reference framework of the 'Spanish Strategy for the fulfilment of the objectives under the Kyoto Protocol', and it takes into account the measures and Programmes adopted by the Autonomous Communities.\u00a0\u00a0The strategy has two chapters. The first one defines actions to fight against climate change and the second one, actions to achieve cleaner energy. Each chapter includes a description of the present situation, the objectives to be reached, the suggested measures and a selection of indicators for the corresponding follow-up.\u00a0\u00a0The operational objectives are:\u00a0- To ensure the reduction of GHG emissions in Spain, giving special importance to measures related to the energy sector. According to the national inventory, in 2005, emissions from energy process represented about 78.87% of total national emissions.\u00a0- To contribute to sustainable development and the fulfilment of climate change commitments by strengthening the use of flexible project-based mechanisms.\u00a0- To promote additional reduction measures in sectors concerned with diffuse pollution.\u00a0- To apply the National Climate Change Adaptation Plan (NCCAP) so as to integrate adaptation measures and strategies in sectoral policies.\u00a0- To increase public awareness with respect to clean energy and climate change.\u00a0- To promote research, development and innovation in matters of climate change and clean energy.\u00a0- To guarantee energy supply security by means of cleaner energies, mainly from renewable sources, achieving other environmental benefits (for example, air quality) and limiting the growth rate of external energy dependence.\u00a0- To boost energy- and resource efficiency for companies and for end users.\u00a0\u00a0The government has adopted a Plan of Urgent Measures (PMU), which together with the 2008-2012 Energy Saving and Efficiency Action Plan aims to consolidate the trend change of GHG emissions in Spain initiated in 2006.", - "import_id": "TESTCCLW.executive.1332.1554", - "slug": "spain_2007_spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020", - "publication_ts": "2007-01-01T00:00:00", - "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBclVLIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--0389b20592032a285dc31b2977adf54b040d8b98/1674%20Spanish.pdf", - "type": "Strategy", - "source": "CCLW", - "category": "Policy", - "geography": "ESP", - "frameworks": [ - "Mitigation" - ], - "instruments": [ - "Capacity building|Governance", - "Processes, plans and strategies|Governance", - "Education, training and knowledge dissemination|Information" - ], - "hazards": [], - "keywords": [ - "Adaptation", - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "Transport" - ], - "languages": [ - "Spanish" - ], - "sectors": [ - "Energy" - ], - "topics": [ - "Adaptation", - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "2007-11-02T00:00:00" - } - ] - }, - "document_slug": "spain_2007_spanish-climate-change-and-clean-energy-strategy-horizon-2007-2012-2020" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1555.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1555.json deleted file mode 100644 index d12faa54..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1555.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "document_name": "Carbon Farming Initiative Amendment Bill 2014", - "document_description": "This bill establishes the Emissions Reduction Fund, and provides a transition for the Carbon Farming Initiative by amending the Carbon Credits (Carbon Farming Initiative) Act 2011 to provide for the Clean Energy Regulator to conduct auctions and enter into contracts to purchase emissions reductions; enable a broader range of emissions reduction projects to be approved; and amend the project eligibility criteria and processes for approving projects and crediting carbon credit units.\n\nThe Carbon Credits (Carbon Farming Initiative) Act 2011 set up a scheme for the issue of Australian Carbon Credit Units (ACCUs) in relation to eligible offsets projects. It was part of a package of three Acts to establish the Carbon Farming Initiative, including the Australian National Registry of Emissions Units Act 2011 and the Carbon Credits (Consequential Amendments) Act 2011. The package of Acts established the existence of ACCUs as personal property, which are generally transferable, and can be sold domestically or internationally subject to regulation.", - "document_id": "TESTCCLW.executive.1332.1555", - "document_source_url": null, - "document_cdn_object": null, - "document_content_type": null, - "document_md5_sum": null, - "document_metadata": { - "name": "Carbon Farming Initiative Amendment Bill 2014", - "description": "This bill establishes the Emissions Reduction Fund, and provides a transition for the Carbon Farming Initiative by amending the Carbon Credits (Carbon Farming Initiative) Act 2011 to provide for the Clean Energy Regulator to conduct auctions and enter into contracts to purchase emissions reductions; enable a broader range of emissions reduction projects to be approved; and amend the project eligibility criteria and processes for approving projects and crediting carbon credit units.\n\nThe Carbon Credits (Carbon Farming Initiative) Act 2011 set up a scheme for the issue of Australian Carbon Credit Units (ACCUs) in relation to eligible offsets projects. It was part of a package of three Acts to establish the Carbon Farming Initiative, including the Australian National Registry of Emissions Units Act 2011 and the Carbon Credits (Consequential Amendments) Act 2011. The package of Acts established the existence of ACCUs as personal property, which are generally transferable, and can be sold domestically or internationally subject to regulation.", - "import_id": "TESTCCLW.executive.1332.1555", - "slug": "australia_2014_carbon-farming-initiative-amendment-bill-2014", - "publication_ts": "2014-12-13T00:00:00", - "source_url": null, - "type": "Law", - "source": "CCLW", - "category": "Law", - "geography": "AUS", - "frameworks": [], - "instruments": [ - "Provision of climate funds|Direct Investment" - ], - "hazards": [], - "keywords": [ - "Carbon Pricing" - ], - "languages": [], - "sectors": [ - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "2014-12-13T00:00:00" - } - ] - }, - "document_slug": "australia_2014_carbon-farming-initiative-amendment-bill-2014" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1556.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1556.json deleted file mode 100644 index d9d01a6a..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1556.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "document_name": "Act On The Allocation And Trading Of Greenhouse-Gas Emission Permits", - "document_description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", - "document_id": "TESTCCLW.executive.1332.1556", - "document_source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBck1JIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--c465bbadbc173d88ce267d8db9943fb62194160b/f", - "document_cdn_object": "KOR/2012/act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits_bb567c0dfbbbaa960a8b017adf399ee9.pdf", - "document_content_type": "application/pdf", - "document_md5_sum": "bb567c0dfbbbaa960a8b017adf399ee9", - "document_metadata": { - "name": "Act On The Allocation And Trading Of Greenhouse-Gas Emission Permits", - "description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", - "import_id": "TESTCCLW.executive.1332.1556", - "slug": "korea_2012_act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits", - "publication_ts": "2012-01-01T00:00:00", - "source_url": "https://climate-laws.org/rails/active_storage/blobs/eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaHBBck1JIiwiZXhwIjpudWxsLCJwdXIiOiJibG9iX2lkIn19--c465bbadbc173d88ce267d8db9943fb62194160b/f", - "type": "Act", - "source": "CCLW", - "category": "Law", - "geography": "KOR", - "frameworks": [], - "instruments": [ - "Processes, plans and strategies|Governance" - ], - "hazards": [], - "keywords": [ - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand" - ], - "languages": [ - "English" - ], - "sectors": [ - "Energy", - "Industry", - "Residential and Commercial", - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "entry into force of enforcement decree", - "description": "", - "created_ts": "2012-11-15T00:00:00" - }, - { - "name": "Amended", - "description": "", - "created_ts": "2020-06-01T00:00:00" - } - ] - }, - "document_slug": "korea_2012_act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1557.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1557.json deleted file mode 100644 index ef328e3e..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1557.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "document_name": "Law on allocation and transactions of greenhouse gas emissions", - "document_description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", - "document_id": "TESTCCLW.executive.1332.1557", - "document_source_url": "https://www.law.go.kr/LSW/lsInfoP.do?efYd=20200601&lsiSeq=215913&ancYd=20200324&nwJoYnInfo=N&ancYnChk=0&ancNo=17104&chrClsCd=010202&efGubun=Y%230000#0000", - "document_cdn_object": null, - "document_content_type": "text/html", - "document_md5_sum": null, - "document_metadata": { - "name": "Law on allocation and transactions of greenhouse gas emissions", - "description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", - "import_id": "TESTCCLW.executive.1332.1557", - "slug": "korea_2012_act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits", - "publication_ts": "2020-01-01T00:00:00", - "source_url": "https://www.law.go.kr/LSW/lsInfoP.do?efYd=20200601&lsiSeq=215913&ancYd=20200324&nwJoYnInfo=N&ancYnChk=0&ancNo=17104&chrClsCd=010202&efGubun=Y%230000#0000", - "type": "Law", - "source": "CCLW", - "category": "Law", - "geography": "KOR", - "frameworks": [], - "instruments": [ - "Processes, plans and strategies|Governance" - ], - "hazards": [], - "keywords": [ - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand" - ], - "languages": [ - "Korean" - ], - "sectors": [ - "Energy", - "Industry", - "Residential and Commercial", - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "entry into force of enforcement decree", - "description": "", - "created_ts": "2012-11-15T00:00:00" - }, - { - "name": "Amended", - "description": "", - "created_ts": "2020-06-01T00:00:00" - } - ] - }, - "document_slug": "korea_2012_act-on-the-allocation-and-trading-of-greenhouse-gas-emission-permits" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1558.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1558.json deleted file mode 100644 index 72198d78..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1558.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "document_name": "Emissions Trading System 3rd Planning period Allocation Plan", - "document_description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", - "document_id": "TESTCCLW.executive.1332.1558", - "document_source_url": "https://ors.gir.go.kr/home/board/read.do?menuId=2&boardMasterId=4&boardId=44", - "document_cdn_object": null, - "document_content_type": "text/html", - "document_md5_sum": null, - "document_metadata": { - "name": "Emissions Trading System 3rd Planning period Allocation Plan", - "description": "The Act aims to achieve the national targets for reducing GHGs by introducing a system for trading GHG allowances through market mechanisms. The first phase of the trading scheme is due to start in 2015, covering companies that emit 125,000 metric tonnes or more of CO2 a year and factories, buildings and livestock farms that produce at least 25,000 tonnes of the gas annually.\u00a0\u00a0The basic plan for the emissions rights trading system shall be established every 5 years for a unit period of 10 years. An Emissions Rights Allocation Committee chaired by the Minister of Strategy and Finance will be established for deliberation and mediation of major issues regarding the emissions rights trading system. The competent authorities will allocate the total emissions rights for the unit period and for each year to relevant corporations. The emissions rights may be traded. Anyone who wants to trade their rights shall enter an account in the emissions rights register.\u00a0\u00a0The scheme determines that in the event that a corporation produces more GHGs than its allotted amount, the excess will be subject to a penalty of up to three times the average market price of the year, up to a limit of KRW100,000 (USD89.87) per one tonne of CO2.\u00a0\u00a0The Enforcement Decree outlines the rules and governance structure for the ETS, planned to begin on 1 January 2015. The ETS requires each company or organisation to set the goal of emissions reduction and fulfill the required reduction goal by utilising a market mechanism. All six Kyoto Protocol GHGs are included, and the scheme covers direct and indirect emissions from individual facilities producing over 25ktCO2e/yr, companies with multiple installations producing over 125ktCO2e/yr, and any other firm that voluntarily wishes to join the ETS.\u00a0\u00a0The Minister of Environment is responsible for controlling and operating the ETS. It operates the quota evaluation commission and the emissions certification committee, and encourages the participation of relevant ministries such as the Ministry of Industry, Trade and Energy, the Ministry of Agriculture, Food and Rural Affairs, and Ministry of Land, Infrastructure and Transport. The Minister of Strategy and Finance must set up the plan so that influential factors such as commodity price are taken into account.\u00a0\u00a0During the first phase of the ETS (2015-2017), liable entities will be allocated 100% of their emissions permits for free based on their average emissions. Therefore demand for units will only be generated by entities exceeding their predicted emission levels. This free allocation level will drop to 97% during the second phase (2018 to 2020) and below 90% in the third phase (2021-2025). By easing the cost burden of allowable emissions at an initial stage, it minimises the burden on industry; by expanding the range of paid quota in the mid- to long-term, it lays the foundation for cost-effective GHG reduction.\u00a0\u00a0Offsets are allowed for up to 10% of compliance obligations. International offsets can be used from Phase III, and shall be set within the range of less than 50% of the maximum offsets for the efficient reduction of domestic GHG. The specific criteria and procedures for the approval and certification of international offsets are yet to be established.\u00a0\u00a0The government agency in charge can receive applications from qualified organisations and may select the emissions trading system's exchange among them through the evaluation of the Committee on Green Growth. In order to stabilise the market at an initial stage, companies will be subject to quota assignment through Phases I and II. When necessary, the government agency in charge, through the quota committee, will take measures to stabilise the market: adding up to 25% of the allowance reserve, specifying the minimum and maximum of emissions rights to be held, restricting borrowing and carry-over, and restricting the limit of offset emissions right's offers.\u00a0\u00a0Financial support measures are allowed to industries whose competitiveness is negatively affected by the scheme. Financial and taxation incentives or subsidies can be provided for GHG reduction, technological development and distribution projects in relation to new and renewable energy.", - "import_id": "TESTCCLW.executive.1332.1558", - "slug": "korea_2020_emissions-trading-system-3rd-planning-period-allocation-plan_1647_4582", - "publication_ts": "2020-01-01T00:00:00", - "source_url": "https://ors.gir.go.kr/home/board/read.do?menuId=2&boardMasterId=4&boardId=44", - "type": "Plan", - "source": "CCLW", - "category": "Law", - "geography": "KOR", - "frameworks": [], - "instruments": [ - "Processes, plans and strategies|Governance" - ], - "hazards": [], - "keywords": [ - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand" - ], - "languages": [ - "Korean" - ], - "sectors": [ - "Energy", - "Industry", - "Residential and Commercial", - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "entry into force of enforcement decree", - "description": "", - "created_ts": "2012-11-15T00:00:00" - }, - { - "name": "Amended", - "description": "", - "created_ts": "2020-06-01T00:00:00" - } - ] - }, - "document_slug": "korea_2020_emissions-trading-system-3rd-planning-period-allocation-plan_1647_4582" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1559.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1559.json deleted file mode 100644 index a003d3df..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1559.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "document_name": "Law relating to the coordination of the federal sustainable development policy", - "document_description": "The latest version of the Act on Co-ordination of Federal Policy on Sustainable Development calls for the development of a long-term vision for Sustainable Development that would include goals for 2050 and be based on interdepartmental and vertical co-operation among between the federal level and federated entities through the specifically created Interdepartmental Commission on Sustainable Development.\u00a0A Federal Sustainable Development Plan is regularly elaborated and lays down specific co-ordination measures. So far, two FSDPs have been adopted, one for 2000-2004, the other for 2004-2008. The second FSDP has been extended due to the political context that prevented elaboration of a Plan for 2008-2012 and remains the current federal SD Plan. Three of the six main objectives of the FSDP are 'Managing natural resources more responsibly', 'Limiting climate change and increasing the use of clean energy', and 'Improving the transport system'. The measures proposed at the federal level in co-ordination with the regions include promoting alternative modes of transport, setting the right price for energy, promoting energy-conserving buildings and providing better public transport.", - "document_id": "TESTCCLW.executive.1332.1559", - "document_source_url": "https://www.ejustice.just.fgov.be/cgi_loi/change_lg.pl?language=fr&la=F&table_name=loi&cn=1997050535%20%20", - "document_cdn_object": null, - "document_content_type": "text/html", - "document_md5_sum": null, - "document_metadata": { - "name": "Law relating to the coordination of the federal sustainable development policy", - "description": "The latest version of the Act on Co-ordination of Federal Policy on Sustainable Development calls for the development of a long-term vision for Sustainable Development that would include goals for 2050 and be based on interdepartmental and vertical co-operation among between the federal level and federated entities through the specifically created Interdepartmental Commission on Sustainable Development.\u00a0A Federal Sustainable Development Plan is regularly elaborated and lays down specific co-ordination measures. So far, two FSDPs have been adopted, one for 2000-2004, the other for 2004-2008. The second FSDP has been extended due to the political context that prevented elaboration of a Plan for 2008-2012 and remains the current federal SD Plan. Three of the six main objectives of the FSDP are 'Managing natural resources more responsibly', 'Limiting climate change and increasing the use of clean energy', and 'Improving the transport system'. The measures proposed at the federal level in co-ordination with the regions include promoting alternative modes of transport, setting the right price for energy, promoting energy-conserving buildings and providing better public transport.", - "import_id": "TESTCCLW.executive.1332.1559", - "slug": "belgium_1997_law-relating-to-the-coordination-of-the-federal-sustainable-development-policy_1080_4597", - "publication_ts": "1997-01-01T00:00:00", - "source_url": "https://www.ejustice.just.fgov.be/cgi_loi/change_lg.pl?language=fr&la=F&table_name=loi&cn=1997050535%20%20", - "type": "Law", - "source": "CCLW", - "category": "Law", - "geography": "BEL", - "frameworks": [], - "instruments": [ - "Institutional mandates|Governance", - "Processes, plans and strategies|Governance", - "Subnational and citizen participation|Governance" - ], - "hazards": [], - "keywords": [ - "Energy Supply", - "Energy Demand", - "Transport" - ], - "languages": [ - "French" - ], - "sectors": [ - "Energy", - "Residential and Commercial", - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "1997-05-05T00:00:00" - }, - { - "name": "climate-relevant document adopted", - "description": "", - "created_ts": "2000-12-25T00:00:00" - }, - { - "name": "Last amended", - "description": "", - "created_ts": "2014-12-04T00:00:00" - } - ] - }, - "document_slug": "belgium_1997_law-relating-to-the-coordination-of-the-federal-sustainable-development-policy_1080_4597" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1560.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1560.json deleted file mode 100644 index 64795abb..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1560.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "document_name": "The Federal Sustainable Development Plan", - "document_description": "The latest version of the Act on Co-ordination of Federal Policy on Sustainable Development calls for the development of a long-term vision for Sustainable Development that would include goals for 2050 and be based on interdepartmental and vertical co-operation among between the federal level and federated entities through the specifically created Interdepartmental Commission on Sustainable Development.\u00a0A Federal Sustainable Development Plan is regularly elaborated and lays down specific co-ordination measures. So far, two FSDPs have been adopted, one for 2000-2004, the other for 2004-2008. The second FSDP has been extended due to the political context that prevented elaboration of a Plan for 2008-2012 and remains the current federal SD Plan. Three of the six main objectives of the FSDP are 'Managing natural resources more responsibly', 'Limiting climate change and increasing the use of clean energy', and 'Improving the transport system'. The measures proposed at the federal level in co-ordination with the regions include promoting alternative modes of transport, setting the right price for energy, promoting energy-conserving buildings and providing better public transport.", - "document_id": "TESTCCLW.executive.1332.1560", - "document_source_url": "https://www.developpementdurable.be/fr/politique-federale/strategie-federale/instruments/le-plan-federal-de-developpement-durable", - "document_cdn_object": null, - "document_content_type": "text/html", - "document_md5_sum": null, - "document_metadata": { - "name": "The Federal Sustainable Development Plan", - "description": "The latest version of the Act on Co-ordination of Federal Policy on Sustainable Development calls for the development of a long-term vision for Sustainable Development that would include goals for 2050 and be based on interdepartmental and vertical co-operation among between the federal level and federated entities through the specifically created Interdepartmental Commission on Sustainable Development.\u00a0A Federal Sustainable Development Plan is regularly elaborated and lays down specific co-ordination measures. So far, two FSDPs have been adopted, one for 2000-2004, the other for 2004-2008. The second FSDP has been extended due to the political context that prevented elaboration of a Plan for 2008-2012 and remains the current federal SD Plan. Three of the six main objectives of the FSDP are 'Managing natural resources more responsibly', 'Limiting climate change and increasing the use of clean energy', and 'Improving the transport system'. The measures proposed at the federal level in co-ordination with the regions include promoting alternative modes of transport, setting the right price for energy, promoting energy-conserving buildings and providing better public transport.", - "import_id": "TESTCCLW.executive.1332.1560", - "slug": "belgium_2000_the-federal-sustainable-development-plan_1080_4598", - "publication_ts": "2000-01-01T00:00:00", - "source_url": "https://www.developpementdurable.be/fr/politique-federale/strategie-federale/instruments/le-plan-federal-de-developpement-durable", - "type": "Plan", - "source": "CCLW", - "category": "Law", - "geography": "BEL", - "frameworks": [], - "instruments": [ - "Institutional mandates|Governance", - "Processes, plans and strategies|Governance", - "Subnational and citizen participation|Governance" - ], - "hazards": [], - "keywords": [ - "Energy Supply", - "Energy Demand", - "Transport" - ], - "languages": [ - "French" - ], - "sectors": [ - "Energy", - "Residential and Commercial", - "Transport" - ], - "topics": [ - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "1997-05-05T00:00:00" - }, - { - "name": "climate-relevant document adopted", - "description": "", - "created_ts": "2000-12-25T00:00:00" - }, - { - "name": "Last amended", - "description": "", - "created_ts": "2014-12-04T00:00:00" - } - ] - }, - "document_slug": "belgium_2000_the-federal-sustainable-development-plan_1080_4598" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1562.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1562.json deleted file mode 100644 index cddcd5df..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1562.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "document_name": "Decision No 1386/2013/EU of the European Parliament and of the Council of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "document_description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", - "document_id": "TESTCCLW.executive.1332.1562", - "document_source_url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?qid=1516647740078&uri=CELEX:32013D1386", - "document_cdn_object": null, - "document_content_type": "text/html", - "document_md5_sum": null, - "document_metadata": { - "name": "Decision No 1386/2013/EU of the European Parliament and of the Council of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", - "import_id": "TESTCCLW.executive.1332.1562", - "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3016", - "publication_ts": "2013-01-01T00:00:00", - "source_url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?qid=1516647740078&uri=CELEX:32013D1386", - "type": "EU Decision", - "source": "CCLW", - "category": "Law", - "geography": "EUR", - "frameworks": [], - "instruments": [ - "Capacity building|Governance", - "Education, training and knowledge dissemination|Information" - ], - "hazards": [], - "keywords": [ - "Adaptation", - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "REDD+ And LULUCF", - "Transport" - ], - "languages": [ - "English" - ], - "sectors": [ - "Economy-wide", - "Health", - "Transport" - ], - "topics": [ - "Adaptation", - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "2013-11-20T00:00:00" - } - ] - }, - "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3016" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1563.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1563.json deleted file mode 100644 index dd482389..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.1332.1563.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "document_description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", - "document_id": "TESTCCLW.executive.1332.1563", - "document_source_url": "https://www.lse.ac.uk/GranthamInstitute/wp-content/uploads/2018/02/living-well-planet-EU-CELEX3A32013D13863AEN3ATXT.pdf", - "document_cdn_object": "EUR/2013/decision-no-1386-2013-eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_e2471666433a8ff0fbd46c9fd0db8a67.pdf", - "document_content_type": "application/pdf", - "document_md5_sum": "e2471666433a8ff0fbd46c9fd0db8a67", - "document_metadata": { - "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", - "import_id": "TESTCCLW.executive.1332.1563", - "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", - "publication_ts": "2013-01-01T00:00:00", - "source_url": "https://www.lse.ac.uk/GranthamInstitute/wp-content/uploads/2018/02/living-well-planet-EU-CELEX3A32013D13863AEN3ATXT.pdf", - "type": "EU Decision", - "source": "CCLW", - "category": "Law", - "geography": "EUR", - "frameworks": [], - "instruments": [ - "Capacity building|Governance", - "Education, training and knowledge dissemination|Information" - ], - "hazards": [], - "keywords": [ - "Adaptation", - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "REDD+ And LULUCF", - "Transport" - ], - "languages": [ - "English" - ], - "sectors": [ - "Economy-wide", - "Health", - "Transport" - ], - "topics": [ - "Adaptation", - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "2013-11-20T00:00:00" - } - ] - }, - "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.3.3.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.3.3.json deleted file mode 100644 index 6f51010c..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.3.3.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "document_description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", - "document_id": "TESTCCLW.executive.3.3", - "document_source_url": "http://existing.com", - "document_cdn_object": null, - "document_content_type": null, - "document_md5_sum": null, - "document_metadata": { - "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "description": "The Decision no 1386/2013/EU sets up the General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet'. It adopts the '7th Environment Action programme' or \u20187th EAP'. The priority objectives of the 7th EAP are: (a) to protect, conserve and enhance the Union's natural capital; (b) to turn the Union into a resource-efficient, green and competitive low-carbon economy; (c) to safeguard the Union's citizens from environment-related pressures and risks to health and well-being; (d) to maximise the benefits of Union environment legislation by improving implementation; (e) to improve the knowledge and evidence base for Union environment policy; (f) to secure investment for environment and climate policy and address environmental externalities; (g) to improve environmental integration and policy coherence; (h) to enhance the sustainability of the Union's cities; (i) to increase the Union's effectiveness in addressing inter\u00ad national environmental and climate-related challenges.", - "import_id": "TESTCCLW.executive.3.3", - "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", - "publication_ts": "2013-01-01T00:00:00", - "source_url": "http://existing.com", - "type": "EU Decision", - "source": "CCLW", - "category": "Law", - "geography": "EUR", - "frameworks": [], - "instruments": [ - "Capacity building|Governance", - "Education, training and knowledge dissemination|Information" - ], - "hazards": [], - "keywords": [ - "Adaptation", - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "REDD+ And LULUCF", - "Transport" - ], - "languages": [ - "English" - ], - "sectors": [ - "Economy-wide", - "Health", - "Transport" - ], - "topics": [ - "Adaptation", - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "2013-11-20T00:00:00" - } - ] - }, - "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" -} \ No newline at end of file diff --git a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.4.4.json b/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.4.4.json deleted file mode 100644 index 77aaeb85..00000000 --- a/integration_tests/data/pipeline_out/parser_input/TESTCCLW.executive.4.4.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "document_name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "document_description": "description", - "document_id": "TESTCCLW.executive.4.4", - "document_source_url": "http://existing.com", - "document_cdn_object": null, - "document_content_type": null, - "document_md5_sum": null, - "document_metadata": { - "name": "DECISION No 1386/2013/EU OF THE EUROPEAN PARLIAMENT AND OF THE COUNCIL of 20 November 2013 on a General Union Environment Action Programme to 2020 \u2018Living well, within the limits of our planet\u2019", - "description": "description", - "import_id": "TESTCCLW.executive.4.4", - "slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017", - "publication_ts": "2013-01-01T00:00:00", - "source_url": "http://existing.com", - "type": "EU Decision", - "source": "CCLW", - "category": "Law", - "geography": "EUR", - "frameworks": [], - "instruments": [ - "Capacity building|Governance", - "Education, training and knowledge dissemination|Information" - ], - "hazards": [], - "keywords": [ - "Adaptation", - "Institutions / Administrative Arrangements", - "Research And Development", - "Energy Supply", - "Energy Demand", - "REDD+ And LULUCF", - "Transport" - ], - "languages": [ - "English" - ], - "sectors": [ - "Economy-wide", - "Health", - "Transport" - ], - "topics": [ - "Adaptation", - "Mitigation" - ], - "events": [ - { - "name": "Law passed", - "description": "", - "created_ts": "2013-11-20T00:00:00" - } - ] - }, - "document_slug": "european-union_2013_decision-no-13862013eu-of-the-european-parliament-and-of-the-council-of-20-november-2013-on-a-general-union-environment-action-programme-to-2020-living-well-within-the-limits-of-our-planet_8570_3017" -} \ No newline at end of file