bot working on the localhost.

- resolving the module error
- refactoring the formating library
- producer and consumer broker scripts.
- discord bot scripts and docker deployments.

.github/workflows/codepipeline-deploy.yml

@@ -2,8 +2,6 @@ name: testing current infrastructure deployment checks
 on:
     push:
         branches:
-            - feature/* ## TODO: need to mention in the CONTRIBUTING.md  and readme file regarding the branch naming conventions.
-            - dev-adding-v0.1
             - main
         paths:
             - "aws_deployment/infrastructure/**"
@@ -20,7 +18,7 @@ jobs:
             - uses: youyo/aws-cdk-github-actions@v2 ## TODO: not used for now, will do once the ci_cd script is written and if there is need.
 
             - name: running_infrastructure_scripts
-              run: sh  infra_deployment_checks.sh
+              run: python ../../aws_deployment/app.py
             ## if the execution of the script gets completed successfully, then echo the message 
             - name: checks for successful stack upgrade.
               run: |

.gitignore

@@ -5,7 +5,7 @@ lidathdpip_env/
 .pytest_cache/
 web3storage/venv.web3storage
 **/.env
-**/.config.json
+**/config.json
 
 **/node_modules
 

bots/Discord/Commands.py

@@ -13,38 +13,18 @@
 import time
 
 from consumer.kafkaConsumer import  kafka_consume_list_jobs
-from producer.kafkaProducer import  kafka_do_surface_reconstruction, kafka_producer_polygon
+from producer.kafkaProducer import  kafka_produce_pipeline_reconstruction, kafka_producer_polygon
 
 
 class UserCommands(commands.Cog, name= "lidarhd" ):
     def __init__(self,bot):
         self.bot = bot
 
-    # def __init__(self, dockerimageId, clientId: str, bot) -> None:
-    #     self.imageName = dockerimageId
-    #     self.client = clientId # corresponds to the user id that want to access the infrastructure.
-    #     self.bot = bot
-
     async def on_ready(self):
         print(f'added as the {self.user}!')
 
 
-    # @commands.hybrid_command(name="job_point", describes="creates user jobs on the given coordinates")
-    # @app_commands.describe(scope="inputs are X and Y coordinates for the area you want to find 3D scans")
-    # async def job_point(self, context:Context, Xcoord, YCoord,ipfs_shp_file, ipfs_template_file):
-    #     """
-    #     fetches the input from the user and transfers to the output. 
-    #     """
-    #     username = context.author.name
-    #     print(f'Message transferred to the bacalhau compute job: {Xcoord, YCoord, ipfs_shp_file, ipfs_template_file }')
-    #     kafka_producer_job(Xcoord=Xcoord, Ycoord=YCoord, username=username, ipfs_shp_file=ipfs_shp_file, ipfs_filename_template=ipfs_template_file)
-    #     ## check whether there is any message in the 
-    #     time.sleep(10)
-    #     username = context.author.name
-    #     print("waited for 10 sec(for test), checking if the consume message is generated")        
-    #     time.sleep(10)
-    #     kafka_consume_message_jobResult(topic='bacalhau_result_job', keyID=username)
-    #     print("message is generated")
+
     @commands.hybrid_command(name="get_status", describes="gets the output status of the compute job")
     @app_commands.describe(scope="defines the details about the current job status")
     async def get_status(self,context: Context, jobId: str):
@@ -68,7 +48,7 @@ async def get_jobIds(self, context:Context):
     async def do_surface_reconstruction_pipeline(self, context: Context, Xcoord, YCoord,ipfs_shp_file, ipfs_template_file,  in_file: str, algorithm_category: int ):
         print(f'Message transferred to the bacalhau surface reconstruction job: {in_file, algorithm_category}')
 
-        kafka_do_surface_reconstruction(username=context.author.name, pathInputlasFile=in_file, algorithmType= algorithm_category)
+        kafka_produce_pipeline_reconstruction(username=context.author.name, pathInputlasFile=in_file, Xcoord=Xcoord, YCoord=YCoord, ipfs_shp_file=ipfs_shp_file,ipfs_template_file=ipfs_template_file)
 
 
 

bots/Discord/circombot.py

@@ -1,15 +1,19 @@
 import asyncio
 import json
-import logging
 import os
 import platform
 import random
 import sys
-
+import logging
 import discord
 from discord.ext import commands, tasks
 from discord.ext.commands import Bot, Context
+
+sys.path.append('../../')
+from bots.Discord.loggingFormatter import LoggingFormatter
 from bots.consumer import kafkaConsumer
+from  bots.producer import kafkaProducer
+from  bots.Discord.Commands import setup
 
 try:
     with open(f"{os.path.realpath(os.path.dirname(__file__))}/config.json") as file:
@@ -18,6 +22,7 @@
     print(notFound)
 
 intents = discord.Intents.default()
+intents.message_content = True
 
 bot: Bot = Bot(
 command_prefix= "/",
@@ -26,38 +31,7 @@
 )
 
 ## taken from krypt0nn repo that renders the logs them in the prettify way:
-
-class LoggingFormatter(logging.Formatter):
-    # Colors
-    black = "\x1b[30m"
-    red = "\x1b[31m"
-    green = "\x1b[32m"
-    yellow = "\x1b[33m"
-    blue = "\x1b[34m"
-    gray = "\x1b[38m"
-    # Styles
-    reset = "\x1b[0m"
-    bold = "\x1b[1m"
-
-    COLORS = {
-        logging.DEBUG: gray + bold,
-        logging.INFO: blue + bold,
-        logging.WARNING: yellow + bold,
-        logging.ERROR: red,
-        logging.CRITICAL: red + bold,
-    }
-
-    def format(self, record):
-        log_color = self.COLORS[record.levelno]
-        format = "(black){asctime}(reset) (levelcolor){levelname:<8}(reset) (green){name}(reset) {message}"
-        format = format.replace("(black)", self.black + self.bold)
-        format = format.replace("(reset)", self.reset)
-        format = format.replace("(levelcolor)", log_color)
-        format = format.replace("(green)", self.green + self.bold)
-        formatter = logging.Formatter(format, "%Y-%m-%d %H:%M:%S", style="{")
-        return formatter.format(record)
-
-logger = logging.FileHandler("discord.log")
+logger = logging.getLogger("discord.log")
 logger.setLevel(logging.INFO)
 
 # Console handler
@@ -156,36 +130,6 @@ async def on_command_error(context: Context, error) -> None:
         )
         await context.send(embed=embed)
 
-        """
-        Same as above, just for the @checks.is_owner() check.
-        """
-        embed = discord.Embed(
-            description="You are not the owner of the bot!", color=0xE02B2B
-        )
-        await context.send(embed=embed)
-        if context.guild:
-            bot.logger.warning(
-                f"{context.author} (ID: {context.author.id}) tried to execute an owner only command in the guild {context.guild.name} (ID: {context.guild.id}), but the user is not an owner of the bot."
-            )
-        else:
-            bot.logger.warning(
-                f"{context.author} (ID: {context.author.id}) tried to execute an owner only command in the bot's DMs, but the user is not an owner of the bot."
-            )
-    elif isinstance(error, commands.MissingPermissions):
-        embed = discord.Embed(
-            description="You are missing the permission(s) `"
-            + ", ".join(error.missing_permissions)
-            + "` to execute this command!",
-            color=0xE02B2B,
-        )
-        await context.send(embed=embed)
-    elif isinstance(error, commands.BotMissingPermissions):
-        embed = discord.Embed(
-            description="I am missing the permission(s) `"
-            + ", ".join(error.missing_permissions)
-            + "` to fully perform this command!",
-            color=0xE02B2B,
-        )
         await context.send(embed=embed)
     elif isinstance(error, commands.MissingRequiredArgument):
         embed = discord.Embed(
@@ -199,14 +143,16 @@ async def on_command_error(context: Context, error) -> None:
         raise error
 
 
-async def load_cogs() -> None:
+async def load_cogs(bot) -> None:
     try:
         await bot.load_extension(f"Commands")
+
+        await setup(bot=bot)
         bot.logger.info("commands package loaded")
     except Exception as e:
         bot.logger.error(f"Failed to load extension from commands.py")
 
-asyncio.run(load_cogs())
+asyncio.run(load_cogs(bot=bot))
 bot.run(config["token"])
 
 

bots/Discord/loggingFormatter.py

@@ -0,0 +1,31 @@
+import logging
+
+class LoggingFormatter(logging.Formatter):
+    # Colors
+    black = "\x1b[30m"
+    red = "\x1b[31m"
+    green = "\x1b[32m"
+    yellow = "\x1b[33m"
+    blue = "\x1b[34m"
+    gray = "\x1b[38m"
+    # Styles
+    reset = "\x1b[0m"
+    bold = "\x1b[1m"
+
+    COLORS = {
+        logging.DEBUG: gray + bold,
+        logging.INFO: blue + bold,
+        logging.WARNING: yellow + bold,
+        logging.ERROR: red,
+        logging.CRITICAL: red + bold,
+    }
+
+    def format(self, record):
+        log_color = self.COLORS[record.levelno]
+        format = "(black){asctime}(reset) (levelcolor){levelname:<8}(reset) (green){name}(reset) {message}"
+        format = format.replace("(black)", self.black + self.bold)
+        format = format.replace("(reset)", self.reset)
+        format = format.replace("(levelcolor)", log_color)
+        format = format.replace("(green)", self.green + self.bold)
+        formatter = logging.Formatter(format, "%Y-%m-%d %H:%M:%S", style="{")
+        return formatter.format(record)

bots/Dockerfile.discord

@@ -0,0 +1,28 @@
+FROM --platform=linux/amd64 python:3.11.1  as build 
+
+RUN mkdir -p /usr/src/app/bots/
+COPY requirements.txt  /usr/src/app/bots/requirements.txt
+
+
+WORKDIR /usr/src/app/bots/
+
+
+RUN pip install  --upgrade pip && pip install  -r requirements.txt
+
+ENV PATH=" ${PATH}:/root/.local/bin" 
+
+## copying the remaining folders for circumbot deployment.
+COPY . .
+
+## also .env separately from the  root folder 
+
+COPY ../../.env ../../.env
+
+FROM build as run
+
+
+COPY --from=build /usr/src/app/bots/ /usr/src/app/bots/
+
+WORKDIR /usr/src/app/bots/Discord
+
+CMD [ "python3", "circombot.py" ]

bots/__init__.py

@@ -1,4 +1,8 @@
 """
 microservices and packages consisting of telegram and  discord bots are deployed here.
 
-"""
+"""
+
+import sys
+import os
+sys.path.append('.')

bots/consumer/__init__.py

@@ -1,3 +1,6 @@
 """
 functions for consuming the message generated from the given broker package to the discord result.
-"""
+"""
+
+import sys
+sys.path.append('..')

bots/consumer/kafkaConsumer.py

@@ -1,45 +1,39 @@
+
+"""
+this script consumes the job from the producer and invokes the pipeline of the job_scheduler
+
+"""
+
 import platform
 from kafka import  KafkaConsumer
 from dotenv import load_dotenv, dotenv_values
 import logging
 import json
 import time
 import os
-from utils.pipeline_construction_caller import  execute_reconstruction_pipeline
+#from bots.utils.pipeline_construction_caller import  execute_reconstruction_pipeline
 
 from utils.model_helper import InputParametersPoint
 
-"""
-this script consumes the job from the producer and invokes the pipeline 
-"""
 
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 log=logging.getLogger("discord")
 
 load_dotenv(dotenv_path='./../.env')
-config = {**dotenv_values(dotenv_path='../../.env')}
-
-# server_details = os.getenv("KAFKA_BROKER_URL")
-# sasl_plain_username=os.getenv("SASL_PLAIN_USERNAME")
-# sasl_plain_password=os.getenv("SASL_PLAIN_PASSWORD")
-
-
-
 
-#print(server_details)
 
 consumer: KafkaConsumer = KafkaConsumer(
           bootstrap_servers=['growing-krill-8066-eu1-kafka.upstash.io:9092' ],
   sasl_mechanism='SCRAM-SHA-256',
   security_protocol='SASL_SSL',
-  sasl_plain_username= 'Z3Jvd2luZy1rcmlsbC04MDY2JCQqQu2eRNWf9N05-qGiH_wR8ikh6aZJgkpqoLc',
-  sasl_plain_password='YmJjMTU1MGEtMjJmNS00OGY5LWE1YTYtYzQ3YTU4YjQ2YTVj',
+  sasl_plain_username= os.getenv("SASL_PLAIN_USERNAME"),
+  sasl_plain_password=os.getenv("SASL_PLAIN_PASSWORD"),
         auto_offset_reset='earliest',
         consumer_timeout_ms=1000
     )
 
-def kafka_consume_message_jobInput(topic: str = 'bacalhau_compute_job', username: str = "" ) -> list(str):
+def kafka_consume_message_jobInput(topic: str = 'bacalhau_compute_job', username: str = "" ):
     """
     This consumes the bot input command messages and parameters and then parses and runs the corresponding job on the pipeline
     

bots/producer/__init__.py

@@ -1 +1,4 @@
-"script in order to import the functions for generating messages from bot to the brokers"
+"script in order to import the functions for generating messages from bot to the brokers"
+
+import sys
+sys.path.append('..')