Skip to content

Commit

Permalink
additional options for import-automation (datacommonsorg#943)
Browse files Browse the repository at this point in the history
* add options for user script args and env

* lint fix

* add cwd to user script launcher
  • Loading branch information
ajaits authored Dec 6, 2023
1 parent 580abfd commit 2b1f996
Show file tree
Hide file tree
Showing 3 changed files with 498 additions and 344 deletions.
14 changes: 9 additions & 5 deletions import-automation/executor/app/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Configurations for the executor.
"""Configurations for the executor.
The app endpoints accept a configs field that allows customization of all the
configurations. See main.py.
"""

import dataclasses
import os
from typing import List
import dataclasses

from google.cloud import logging

Expand All @@ -32,6 +31,7 @@ def _production():
@dataclasses.dataclass
class ExecutorConfig:
"""Configurations for the executor."""

# ID of the Google Cloud project that hosts the executor. The project
# needs to enable App Engine and Cloud Scheduler.
gcp_project_id: str = 'google.com:datcom-data'
Expand Down Expand Up @@ -102,9 +102,13 @@ class ExecutorConfig:
# ID of the location where Cloud Scheduler is hosted.
scheduler_location: str = 'us-central1'
# Maximum time a user script can run for in seconds.
user_script_timeout: float = 600
user_script_timeout: float = 3600
# Arguments for the user script
user_script_args: List[str] = ()
# Environment variables for the user script
user_script_env: dict = None
# Maximum time venv creation can take in seconds.
venv_create_timeout: float = 600
venv_create_timeout: float = 3600
# Maximum time downloading a file can take in seconds.
file_download_timeout: float = 600
# Maximum time downloading the repo can take in seconds.
Expand Down
Loading

0 comments on commit 2b1f996

Please sign in to comment.