From d251fc88ff6a8826a357a0ef3ec2732df9de0100 Mon Sep 17 00:00:00 2001 From: Jehangir Amjad Date: Sun, 4 Feb 2024 07:20:40 -0800 Subject: [PATCH] removing the option to provide user script args as a flag --- import-automation/executor/README.md | 5 ++--- .../executor/schedule_update_import.py | 21 ------------------- .../executor/schedule_update_import.sh | 9 +++----- 3 files changed, 5 insertions(+), 30 deletions(-) diff --git a/import-automation/executor/README.md b/import-automation/executor/README.md index 83a554c0ab..eb2273b0f5 100644 --- a/import-automation/executor/README.md +++ b/import-automation/executor/README.md @@ -58,19 +58,18 @@ Once the script runs to completion, the data directory's latest update is printe Ensure this script is executed from the directory which contains `schedule_update_import.sh`, i.e. from `/data/import-automation/executor`. ``` -Run `./schedule_update_import.sh -u ` +Run `./schedule_update_import.sh -u ` ``` Run `./schedule_update_import.sh --help` for usage. `` is the GCP project id where the config file is stored, e.g. `datcom-import-automation`. `` is the path to the import (relative to the root directory of the `data` repo), with the name of the import provided with a colon, e.g. `scripts/us_usda/quickstats:UsdaAgSurvey`. -`import_script_args` is one string (enclosed by quotes) representing all the command line args to be provided to the import script, e.g. `"--start_year=2023 --another_flag=val"`. Example invocation: ``` -Run `./schedule_update_import.sh -u datcom-import-automation scripts/us_usda/quickstats:UsdaAgSurvey "--start_year=2023"` +Run `./schedule_update_import.sh -u datcom-import-automation scripts/us_usda/quickstats:UsdaAgSurvey` ``` ## Local Executor [should be deprecated soon] diff --git a/import-automation/executor/schedule_update_import.py b/import-automation/executor/schedule_update_import.py index 11e0c14e33..ad80743bf4 100644 --- a/import-automation/executor/schedule_update_import.py +++ b/import-automation/executor/schedule_update_import.py @@ -44,11 +44,6 @@ ':.' 'Example: scripts/us_usda/quickstats:UsdaAgSurvey') -flags.DEFINE_string( - 'import_script_args', '', - 'One string of command line args for the import script,' - 'e.g. "--flag1=value1 --flag2=value2"') - _FLAGS(sys.argv) logging.basicConfig(level=logging.INFO) @@ -117,7 +112,6 @@ def _print_fileupload_results(cfg: configs.ExecutorConfig, def update(cfg: configs.ExecutorConfig, absolute_import_path: str, - import_script_args: List[str] = [], local_repo_dir: str = "") -> import_executor.ExecutionResult: """Executes an update on the specified import. @@ -136,9 +130,6 @@ def update(cfg: configs.ExecutorConfig, : example: scripts/us_usda/quickstats:UsdaAgSurvey - import_script_args: a list of strings, each to be used as a command - line arg for the import script, - e.g. ['--flag1=value1', '--flag2=value2']. local_repo_dir: the full path to the GitHub repository on local. The path shoud be provided to the root directory of the repo, e.g. `/data`. @@ -146,10 +137,6 @@ def update(cfg: configs.ExecutorConfig, Returns: An import_executor.ExecutionResult object. """ - # Update the configs with user script args, if provided. - if import_script_args: - cfg.user_script_args = import_script_args - executor = import_executor.ImportExecutor( uploader=file_uploader.GCSFileUploader( project_id=cfg.gcs_project_id, @@ -181,12 +168,6 @@ def main(_): 'Flag: absolute_import_path is invalid. Path should be like:' 'scripts/us_usda/quickstats:UsdaAgSurvey') - # Converting string to list - args_list = import_script_args.split(' ') - if type(args_list) != type([]): - raise Exception( - 'Flag: import_script_args could not be parsed into a list.') - # Get the root repo directory (data). Assumption is that this script is being # called from a path within the data repo. cwd = os.getcwd() @@ -194,7 +175,6 @@ def main(_): logging.info(f'{mode} called with the following:') logging.info(f'Config Project ID: {_FLAGS.config_project_id}') logging.info(f'Import: {absolute_import_path}') - logging.info(f'Import script args: {args_list}') logging.info(f'Repo root directory: {repo_dir}') # TODO: allow overriding/updating config params from a local config file as well. @@ -206,7 +186,6 @@ def main(_): res = dataclasses.asdict( update(cfg, absolute_import_path, - import_script_args=args_list, local_repo_dir=repo_dir)) logging.info("*************************************************") logging.info("*********** Update Complete. ********************") diff --git a/import-automation/executor/schedule_update_import.sh b/import-automation/executor/schedule_update_import.sh index f599245b48..f88b983e1d 100755 --- a/import-automation/executor/schedule_update_import.sh +++ b/import-automation/executor/schedule_update_import.sh @@ -14,11 +14,9 @@ # limitations under the License. function help { - echo "#Usage: -us " + echo "#Usage: -us " echo "## is the GCP project ID where the config file is located." - echo "## Update an import specified by , e.g. scripts/us_usda/quickstats:UsdaAgSurvey" - echo "## Provide all args for the import script as one string, i.e. with quotes, e.g. \"--flag1=val1 --flag2=val2\"" - exit 1 + echo "## Update an import specified by , e.g. scripts/us_usda/quickstats:UsdaAgSurvey" exit 1 } if [[ $# -le 1 ]]; then @@ -41,12 +39,11 @@ done CONFIG_PROJECT_ID=$2 IMPORT_PATH=$3 -IMPORT_SCRIPT_ARGS=$4 python3 -m venv .env . .env/bin/activate pip3 install --disable-pip-version-check -r requirements.txt -python3 -m schedule_update_import --config_project_id=$CONFIG_PROJECT_ID --mode=$MODE --absolute_import_path=$IMPORT_PATH --import_script_args="$IMPORT_SCRIPT_ARGS" +python3 -m schedule_update_import --config_project_id=$CONFIG_PROJECT_ID --mode=$MODE --absolute_import_path=$IMPORT_PATH" deactivate \ No newline at end of file