Merge branch 'datacommonsorg:master' into master

README.md

@@ -167,7 +167,7 @@ you import modules and run tests, as below.
 
 Consider automating coding to satisfy some of these requirements.
 
-*   [Integrate pylint](http://pylint.pycqa.org/en/latest/user_guide/ide-integration.html)
+*   [Integrate pylint](https://pylint.readthedocs.io/en/stable/user_guide/installation/ide_integration/index.html#ide-integration)
     with your editor.
 *   Integrate yapf with
     [Visual Studio](https://code.visualstudio.com/docs/python/editing#_formatting),

import-automation/executor/app/executor/import_executor.py

@@ -36,8 +36,8 @@
 from app.service import import_service
 
 # Email address for status messages.
-_SUCCESS_EMAIL_ADDR = 'datacommons+release@google.com'
-_FAILURE_EMAIL_ADDR = 'datacommons-alerts+importautomation@google.com'
+_DEBUG_EMAIL_ADDR = 'datacommons-debug+imports@google.com'
+_ALERT_EMAIL_ADDR = 'datacommons-alerts+imports@google.com'
 
 _SEE_LOGS_MESSAGE = (
     'Please find logs in the Logs Explorer of the GCP project associated with'
@@ -300,7 +300,7 @@ def _import_one(
                 self.notifier.send(
                     subject=f'Import Automation Success - {import_name}',
                     body=msg,
-                    receiver_addresses=[_SUCCESS_EMAIL_ADDR],
+                    receiver_addresses=[_DEBUG_EMAIL_ADDR],
                 )
 
         except Exception as exc:
@@ -312,7 +312,7 @@ def _import_one(
                 self.notifier.send(
                     subject=f'Import Automation Failure - {import_name}',
                     body=msg,
-                    receiver_addresses=[_FAILURE_EMAIL_ADDR],
+                    receiver_addresses=[_ALERT_EMAIL_ADDR, _DEBUG_EMAIL_ADDR],
                 )
             raise exc
 

requirements.txt

@@ -29,3 +29,4 @@ xlrd==1.2.0
 zipp
 beautifulsoup4
 ratelimit
+xlsxwriter==3.2.0

requirements_all.txt

@@ -41,3 +41,4 @@ yapf
 zipp
 beautifulsoup4
 ratelimit
+xlsxwriter==3.2.0

scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/EurostatNUTS3_BirthDeathMigration.tmcf

@@ -1,49 +1,56 @@
-
 Node: E:EurostatNUTS3_BirthDeathMigration->E0
 typeOf: dcs:StatVarObservation
 variableMeasured: dcs:Count_Death
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->Count_Death
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E1
 typeOf: dcs:StatVarObservation
-variableMeasured: dcs:Count_BirthEvent_AsAFractionOf_Count_Person
+variableMeasured: dcs:Count_BirthEvent_AsAFractionOfCount_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
-value: C:EurostatNUTS3_BirthDeathMigration->Count_BirthEvent_AsAFractionOf_Count_Person
+value: C:EurostatNUTS3_BirthDeathMigration->Count_BirthEvent_AsAFractionOfCount_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E2
 typeOf: dcs:StatVarObservation
-variableMeasured: dcs:Count_Death_AsAFractionOf_Count_Person
+variableMeasured: dcs:Count_Death_AsAFractionOfCount_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
-value: C:EurostatNUTS3_BirthDeathMigration->Count_Death_AsAFractionOf_Count_Person
+value: C:EurostatNUTS3_BirthDeathMigration->Count_Death_AsAFractionOfCount_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E3
 typeOf: dcs:StatVarObservation
 variableMeasured: dcs:IncrementalCount_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->IncrementalCount_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E4
 typeOf: dcs:StatVarObservation
 variableMeasured: dcs:GrowthRate_Count_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->GrowthRate_Count_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E5
 typeOf: dcs:StatVarObservation
 variableMeasured: dcs:Count_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->Count_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E6
 typeOf: dcs:StatVarObservation
 variableMeasured: dcs:Count_BirthEvent
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->Count_BirthEvent
+measurementMethod: dcs:EurostatRegionalStatistics
+

scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/README.md

@@ -19,7 +19,8 @@ Data is available at the NUTS3 level and above.
 
 ### Download URL
 
-ZIP file is available for download from [here](https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?file=data/demo_r_gind3.tsv.gz).
+ZIP file is available for download from [here](https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/demo_r_gind3/?format=TSV&compressed=true).
+
 
 ### License
 
@@ -55,20 +56,29 @@ See parent README.
 #### Scripts
 
 - [import_data.py](import_data.py): import script.
-- [test_import.py](test_import.py): import testing script.
+- [import_data_test.py](import_data_test.py): import testing script.
 
 ### Import Procedure
 
+This script offers three modes of operation: download, process, or both download and process.
 To import data, run the following command:
 
 ```
-python3 import_data.py
+1. Download and Process (python3 import_data.py or no mode flag):
+
+2. Download Only (python3 import_data.py --mode=download):
+
+3. Process Only (python3 import_data.py --mode=process):
+
 ```
 
 ### Testing Procedure
 
+How to Create Sample Data: Extract a subset of rows from your source input file to generate sample input and output CSV files.
+
 To test import procedure, run the following command:
 
 ```
-python3 test_import.py
+python3 import_data_test.py
+
 ```