From 69c78695b2c795d22c1ae962d4a444d28b00bb9e Mon Sep 17 00:00:00 2001 From: = Date: Mon, 30 Oct 2023 08:48:01 +0000 Subject: [PATCH] #20 refactoring - writeup refactoring in infrastructure deployment - add function in circumbot to run the bacalhau point job - adding types for the different jobs in order to execute the geospatial jobs. --- .aws/credentials.example | 4 ++++ .../infrastructure/infrastructure_stack.py | 2 +- bots/Discord/circombot.py | 3 ++- bots/Discord/cogs/__init__.py | 3 +++ bots/Dockerfile.discord | 2 -- bots/consumer/kafkaConsumer.py | 4 ++-- bots/utils/pipeline_construction_caller.py | 20 +------------------ job_scheduler/requirements.txt | 3 ++- job_scheduler/tests/test_job_scheduler.py | 6 ++---- job_scheduler/utils/types.py | 17 ++++++++++++++++ 10 files changed, 34 insertions(+), 30 deletions(-) create mode 100644 .aws/credentials.example create mode 100644 bots/Discord/cogs/__init__.py create mode 100644 job_scheduler/utils/types.py diff --git a/.aws/credentials.example b/.aws/credentials.example new file mode 100644 index 0000000..057dea0 --- /dev/null +++ b/.aws/credentials.example @@ -0,0 +1,4 @@ +[default] +sso_account_id= +region= +secret_key="" \ No newline at end of file diff --git a/aws_deployment/infrastructure/infrastructure_stack.py b/aws_deployment/infrastructure/infrastructure_stack.py index 2e92e9b..2648a70 100644 --- a/aws_deployment/infrastructure/infrastructure_stack.py +++ b/aws_deployment/infrastructure/infrastructure_stack.py @@ -11,7 +11,7 @@ from constructs import Construct """ -Credits to [aws-cdk examples](https://github.com/aws-samples/aws-cdk-examples) for the reference examples. +Credits to [aws-cdk examples](https://github.com/aws-samples/aws-cdk-examples) for the basic template. """ class InfrastructureStack(Stack): job_scheduler: _lambda.DockerImageFunction diff --git a/bots/Discord/circombot.py b/bots/Discord/circombot.py index 5f0203e..6ac3706 100644 --- a/bots/Discord/circombot.py +++ b/bots/Discord/circombot.py @@ -8,7 +8,7 @@ import logging import discord -from discord.ext import commands, tasks +from discord.ext import commands from discord.ext.commands import Bot, Context from storage import Database import aiosqlite @@ -16,6 +16,7 @@ from bots.consumer.kafkaConsumer import kafka_consume_list_jobs from bots.producer.kafkaProducer import kafka_producer_job from discord import app_commands + try: with open(f"{os.path.realpath(os.path.dirname(__file__))}/config.json") as file: config = json.load(file) diff --git a/bots/Discord/cogs/__init__.py b/bots/Discord/cogs/__init__.py new file mode 100644 index 0000000..ff95fbf --- /dev/null +++ b/bots/Discord/cogs/__init__.py @@ -0,0 +1,3 @@ +import sys +import os +sys.path.append(os.path.relpath(os.getcwd(), "/")) \ No newline at end of file diff --git a/bots/Dockerfile.discord b/bots/Dockerfile.discord index 2245448..08a68e5 100644 --- a/bots/Dockerfile.discord +++ b/bots/Dockerfile.discord @@ -13,9 +13,7 @@ ENV PATH=" ${PATH}:/root/.local/bin" ## copying the remaining folders for circumbot deployment. COPY . . - ## also .env separately from the root folder - COPY ../../.env ../../.env FROM build as run diff --git a/bots/consumer/kafkaConsumer.py b/bots/consumer/kafkaConsumer.py index ba811c5..fe8d4f9 100644 --- a/bots/consumer/kafkaConsumer.py +++ b/bots/consumer/kafkaConsumer.py @@ -11,7 +11,7 @@ import json import time import os -#from bots.utils.pipeline_construction_caller import execute_reconstruction_pipeline +from bots.utils.pipeline_construction_caller import createJobBacalauPoint from utils.model_helper import InputParametersPoint @@ -68,7 +68,7 @@ def kafka_consume_message_jobInput(topic: str = 'bacalhau_compute_job', username jobParameter.filename_shp = params[3] jobParameter.ipfs_image = params[4] jobParameter.username = params[5] - #createJobBacalauPoint(jobParameter) + createJobBacalauPoint(jobParameter) #return parameters diff --git a/bots/utils/pipeline_construction_caller.py b/bots/utils/pipeline_construction_caller.py index 6879822..5509958 100644 --- a/bots/utils/pipeline_construction_caller.py +++ b/bots/utils/pipeline_construction_caller.py @@ -81,24 +81,6 @@ def createJobBacalauPoint(parameter: InputParametersPoint) -> any: return job_json_details except SystemError as s: print(s) - - - -def execute_reconstruction_pipeline( - Xcoord="43.2946", - Ycoord="5.3695", - username="test", - ipfs_shp_file="bafkreicxd6u4avrcytevtvehaaimqbsqe5qerohji2nikcbfrh6ccb3lgu", - filename="pipeline_template.json", - algorithm_surface_reconstruction="0", #(poisson) - ): - """ - function to call the hosted pipeline construction container on ECS which will in turn run the intermediate series of computations on the bacalhau - """ - - pass - - def listJobs() -> JobResults: @@ -176,7 +158,7 @@ def vectorize_outputs(data:dict): # algorithm_file = paramsReconstruction.algo - +test = True diff --git a/job_scheduler/requirements.txt b/job_scheduler/requirements.txt index 6330057..56e824f 100644 --- a/job_scheduler/requirements.txt +++ b/job_scheduler/requirements.txt @@ -5,4 +5,5 @@ uvicorn python-dotenv httpx pytest -kafka \ No newline at end of file +kafka +fastapi-scheduler \ No newline at end of file diff --git a/job_scheduler/tests/test_job_scheduler.py b/job_scheduler/tests/test_job_scheduler.py index 8199d1a..4a0d251 100644 --- a/job_scheduler/tests/test_job_scheduler.py +++ b/job_scheduler/tests/test_job_scheduler.py @@ -23,7 +23,7 @@ class RequestCityGMLReconstruction(BaseModel): -def create_job_task(): +def create_job_task_surface_reconstruction(): parameters : Request = { "xcoord": '34', @@ -36,6 +36,4 @@ def create_job_task(): response = client.get("/jobs/requestJob/surface_reconstruction/ECS", params=parameters) assert response.status_code == 200 - -def get_values(): - pass \ No newline at end of file + \ No newline at end of file diff --git a/job_scheduler/utils/types.py b/job_scheduler/utils/types.py new file mode 100644 index 0000000..add9002 --- /dev/null +++ b/job_scheduler/utils/types.py @@ -0,0 +1,17 @@ +""" +here we store the various standards of data inputs/outputs that are being processed by the various jobs +""" +from typing import List + +class surface_reconstruction_pipeline: + coordinates: List[str] + laz_file: str + username:str + template_file: str + filename_pipeline: str + surface_reconstruction_algorithm: str + +class citygml_pipeline: + yaml_file_path: str + object_file_path: str + cityGML_output_path:str \ No newline at end of file