Enable overriding configs locally

.gitignore

@@ -13,4 +13,6 @@ dc_generated/
 */*/*/data/*
 lib/
 bin/
-pyvenv.cfg
+pyvenv.cfg
+# Ignore updates to the local configs json file.
+import-automation/executor/config_override.json

import-automation/executor/README.md

@@ -55,7 +55,7 @@ You can execute an import job from your local end by invoking the script below.
 
 Once the script runs to completion, the data directory's latest update is printed (along with the location on GCS) which can confirm whether the import actually produced new data. Note: it is a good idea to check the directory path printed to see if the expected import files are all there.
 
-Ensure this script is executed from the directory which contains `schedule_update_import.sh`, i.e. from `/data/import-automation/executor`.
+Ensure this script is executed from the directory which contains `schedule_update_import.sh`, i.e. from `/data/import-automation/executor`. Configs (`<repo_root>/import-automation/executor/app/configs.py`) are loaded from GCS. To override any configs locally, set them in the file `<repo_root>/import-automation/executor/config_override.json`. note that the config fields must belong to `<repo_root>/import-automation/executor/app/configs.py`, else the update will produce an Exception.
 
 ```
 Run `./schedule_update_import.sh -u <config_project_id> <path_to_import> <import_script_args>`

import-automation/executor/config_override.json

@@ -0,0 +1,5 @@
+{
+    "configs":{
+        "executor_type": "GKE"
+   }
+ }

import-automation/executor/schedule_update_import.py

@@ -29,6 +29,8 @@
 from app.service import github_api
 from google.cloud import storage
 
+_CONFIG_OVERRIDE_FILE = 'config_override.json'
+
 _FLAGS = flags.FLAGS
 
 flags.DEFINE_string('mode', '', 'Options: update or schedule.')
@@ -54,6 +56,17 @@
 logging.basicConfig(level=logging.INFO)
 
 
+def _override_configs(filename: str,
+                      config: configs.ExecutorConfig) -> configs.ExecutorConfig:
+    # Read configs from the local file.
+    d = json.load(open(filename))
+
+    # Update config with any fields and values provided in the local file.
+    # In case of any errors, the line below will raise an Exception which will
+    # report the problem which shoud be fixed in the local config json file.
+    return dataclasses.replace(config, **d["configs"])
+
+
 def _get_cloud_config() -> configs.ExecutorConfig:
     logging.info('Getting cloud config.')
     project_id = _FLAGS.config_project_id
@@ -197,8 +210,15 @@ def main(_):
     logging.info(f'Import script args: {args_list}')
     logging.info(f'Repo root directory: {repo_dir}')
 
-    # TODO: allow overriding/updating config params from a local config file as well.
+    # Loading configs from GCS and then using _CONFIG_OVERRIDE_FILE to
+    # override any fields provided in the file.
+    logging.info('Reading configs from GCS.')
     cfg = _get_cloud_config()
+
+    logging.info(
+        f'Updating any config fields from local file: {_CONFIG_OVERRIDE_FILE}.')
+    cfg = _override_configs(_CONFIG_OVERRIDE_FILE, cfg)
+
     if mode == 'update':
         logging.info("*************************************************")
         logging.info("***** Beginning Update. Can take a while. *******")