From 0cd33c23a0707cdc578ce2c574d24b136671969a Mon Sep 17 00:00:00 2001 From: Kolman-Freecss Date: Mon, 7 Oct 2024 16:52:58 +0200 Subject: [PATCH] Decouple behaviours to main script && Fix paths && update deps --- .github/workflows/test-devops.yml | 4 +- .gitignore | 6 +- requirements.txt | 3 +- .../main/requirements.txt => __init__.py} | 0 src/local/Dockerfile | 8 +- src/local/__init__.py | 0 src/local/docker-compose.yml | 23 +- src/local/main/__init__.py | 0 src/local/main/build.py | 89 ++++--- src/local/main/config.py | 14 +- src/local/main/init_jenkins.py | 230 +++++++++--------- src/local/main/main.py | 16 ++ src/local/main/services.py | 15 +- src/local/requirements.txt | 6 +- 14 files changed, 236 insertions(+), 178 deletions(-) rename src/{local/main/requirements.txt => __init__.py} (100%) create mode 100644 src/local/__init__.py create mode 100644 src/local/main/__init__.py create mode 100644 src/local/main/main.py diff --git a/.github/workflows/test-devops.yml b/.github/workflows/test-devops.yml index be66d46..3d4d213 100644 --- a/.github/workflows/test-devops.yml +++ b/.github/workflows/test-devops.yml @@ -35,7 +35,7 @@ jobs: run: sudo apt-get update && sudo apt-get install docker-compose -y - name: Run Docker Compose - working-directory: src/env.local + working-directory: src/local run: docker-compose up -d - name: Wait for services to be ready @@ -48,5 +48,5 @@ jobs: - name: Tear down Docker Compose if: always() - working-directory: src/env.local + working-directory: src/local run: docker-compose down \ No newline at end of file diff --git a/.gitignore b/.gitignore index fcff114..fd1fdb7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,10 @@ # Ignore .env/env.local.env files .env/env.local +.env/env.docker-local # Ignore RSA temp files -*_rsa* \ No newline at end of file +*_rsa* + +# Python temp files +*.pyc \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6d40d6d..91de92f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ python-jenkins~=1.8.2 setuptools~=68.2.0 requests~=2.32.3 -urllib3~=2.2.3 \ No newline at end of file +urllib3~=2.2.3 +python-dotenv~=1.0.1 \ No newline at end of file diff --git a/src/local/main/requirements.txt b/src/__init__.py similarity index 100% rename from src/local/main/requirements.txt rename to src/__init__.py diff --git a/src/local/Dockerfile b/src/local/Dockerfile index d0f3184..90b73f9 100644 --- a/src/local/Dockerfile +++ b/src/local/Dockerfile @@ -29,14 +29,14 @@ RUN /app/venv/bin/pip install --no-cache-dir -r /app/requirements.txt # ---------- JENKINS DEPS ---------- -COPY jenkins.yaml /var/jenkins_home/casc.yaml +COPY jenkins_plugins/plugins.yaml /var/jenkins_home/casc.yaml ENV CASC_JENKINS_CONFIG=/var/jenkins_home/casc.yaml # Copy the shell script that installs dependencies COPY jenkins_install_deps.sh /usr/local/bin/jenkins_install_deps.sh # Ensure the shell scripts are executable -RUN chmod +x /usr/env.local/bin/jenkins_install_deps.helpers +RUN chmod +x /usr/local/bin/jenkins_install_deps.sh # Set the virtual environment as the default for Python ENV PATH="/app/venv/bin:$PATH" @@ -44,7 +44,7 @@ ENV PATH="/app/venv/bin:$PATH" RUN echo "Before executing the shell script" # Execute the dependency installation script -RUN /usr/env.local/bin/jenkins_install_deps.helpers +RUN /usr/local/bin/jenkins_install_deps.sh # ---------------------------------------- @@ -53,6 +53,6 @@ RUN echo "After executing the shell script and before executing the Python scrip RUN ls -l /app # Execute the copied shell script -#RUN /app/build.helpers +#RUN /app/build.sh RUN echo "After executing the Python script" \ No newline at end of file diff --git a/src/local/__init__.py b/src/local/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/local/docker-compose.yml b/src/local/docker-compose.yml index 34ca830..54c8e72 100644 --- a/src/local/docker-compose.yml +++ b/src/local/docker-compose.yml @@ -18,7 +18,7 @@ services: - jenkins_home:/var/jenkins_home # Volume for Jenkins data - ./init-scripts:/var/jenkins_init_scripts # We mount the init scripts - ./jenkins_plugins:/var/jenkins_home/plugins # We mount the plugins - entrypoint: ["/bin/bash", "-c", "/usr/bin/tini -- /usr/env.local/bin/jenkins.helpers && dockerd > /var/log/dockerd.log 2>&1 && tail -f /dev/null"] + entrypoint: ["/bin/bash", "-c", "/usr/bin/tini -- /usr/local/bin/jenkins.sh && dockerd > /var/log/dockerd.log 2>&1 && tail -f /dev/null"] healthcheck: test: ["CMD-SHELL", "curl -sS http://localhost:8080/login || exit 1"] interval: 30s @@ -31,25 +31,18 @@ services: image: python:3.10 container_name: kf-jenkins-init volumes: - - ./main:/usr/src/app - - ./.data:/usr/src/app/data - - ./requirements.txt:/usr/src/app/requirements.txt - working_dir: /usr/src/app + - ./main:/app/src # Between all the src we have a needed requirements.txt for this implantation + - ./requirements.txt:/app/requirements.txt + - ../../.env/env.docker-local:/app/.env/env.docker-local + working_dir: /app environment: - - JENKINS_URL=http://kf-jenkins:8080 - - JENKINS_USER=admin - - JENKINS_PASS=admin - - GITHUB_CREDENTIALS_ID=github-credentials - - GITHUB_USER=Kolman_Freecss - # TODO: This is a temporary PAT, it should be replaced by a secret (in prod environment) or not synch with the repository - - PAT_JENKINS= - - EXECUTION_ENVIRONMENT=docker-env.local + - ENV=docker-local command: > - bash -c "pip install -r requirements.txt && python init_jenkins.py && python build.py" + bash -c "ls -la . && ls -la src && pip install -r requirements.txt && python src/main.py" depends_on: jenkins: condition: service_healthy volumes: jenkins_home: - driver: env.local + driver: local diff --git a/src/local/main/__init__.py b/src/local/main/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/local/main/build.py b/src/local/main/build.py index 774cc02..ac4b03a 100644 --- a/src/local/main/build.py +++ b/src/local/main/build.py @@ -1,54 +1,65 @@ import os -import src.local.main.config as config_module -import src.local.main.services as services -from src.local.main.services import CredentialsType +import config as config_module +import services as services +from services import CredentialsType -try: - print(f"JENKINS INFO -> Jenkins URL: {config_module.get(config_module.ConfigKeys.JENKINS_URL)}, Username: {config_module.get(config_module.ConfigKeys.JENKINS_USER)}, API Token: {config_module.get(config_module.ConfigKeys.JENKINS_PASS)}") +def fetch(): + try: + print(f"JENKINS INFO -> Jenkins URL: {config_module.get(config_module.ConfigKeys.JENKINS_URL)}, Username: {config_module.get(config_module.ConfigKeys.JENKINS_USER)}, API Token: {config_module.get(config_module.ConfigKeys.JENKINS_PASS)}") + + # Jenkins version + user = services.jenkins_service.get_whoami() + version = services.jenkins_service.get_version() + print('JENKINS INFO -> Hello %s from Jenkins %s' % (user['fullName'], version)) + + # Print all files in the current directory + print(os.listdir('.')) + + # Configured on docker-compose (check volumes) + if config_module.ENV == 'local' or config_module.ENV == 'docker-env': + path_job = '../.data/Jenkinsfile_spring.xml' + else: + path_job = './data/Jenkinsfile_spring.xml' - # Jenkins version - user = services.jenkins_service.get_whoami() - version = services.jenkins_service.get_version() - print('JENKINS INFO -> Hello %s from Jenkins %s' % (user['fullName'], version)) + # List all content dir + if config_module.ENV != 'local' or config_module.ENV != 'docker-env': + try: + content = os.listdir('../') + print(content) + except Exception as e: + print(e) - # Print all files in the current directory - print(os.listdir('.')) + # Read XML job + job_config = open(path_job).read() - # Configured on docker-compose (check volumes) - if config_module.ENV == 'env.local': - path_job = '../.data/Jenkinsfile_spring.xml' - else: - path_job = './data/Jenkinsfile_spring.xml' + # Create credentials + services.build_credentials(CredentialsType.USER) + + job_name = "spring-pipeline" - # List all content dir - if config_module.ENV != 'env.local': try: - content = os.listdir('../') - print(content) + if services.jenkins_service.job_exists(job_name): + services.jenkins_service.delete_job(job_name) + print(f'Job {job_name} already exists and was deleted') + else: + print(f'Job {job_name} does not exist') + services.jenkins_service.create_job(job_name, job_config) + print('Job created successfully') except Exception as e: print(e) - # Read XML job - job_config = open(path_job).read() - - # Create credentials - services.build_credentials(CredentialsType.USER) - - job_name = "spring-pipeline" + services.jenkins_service.build_job(job_name) + print('Job execute successfully') + except Exception as e: + print("Error: ", e) +def start(): + print("Build:: ...") try: - if services.jenkins_service.job_exists(job_name): - services.jenkins_service.delete_job(job_name) - print(f'Job {job_name} already exists and was deleted') - else: - print(f'Job {job_name} does not exist') - services.jenkins_service.create_job(job_name, job_config) - print('Job created successfully') + fetch() except Exception as e: - print(e) + print("Error: ", e) - services.jenkins_service.build_job(job_name) - print('Job execute successfully') -except Exception as e: - print("Error: ", e) +if config_module.ENV == 'local': + start() \ No newline at end of file diff --git a/src/local/main/config.py b/src/local/main/config.py index ba3c2a3..4f0cb3f 100644 --- a/src/local/main/config.py +++ b/src/local/main/config.py @@ -17,10 +17,22 @@ class ConfigKeys(str, Enum): ENV = os.getenv('ENV') # Provided by Github Actions or Environment Variables if ENV == 'prod': env_files_path = '' +elif ENV == 'docker-local': + project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../')) # .. This avoid to duplicate our .env in src folder "app/.env" + if project_root.endswith(('/')): + project_root = project_root[:-1] + env_files_path = os.path.join(project_root, '.env', f'env.{ENV}') + if ENV == '' or ENV is None: + ENV = 'local' else: project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../..')) + if project_root.endswith(('/')): + project_root = project_root[:-1] env_files_path = os.path.join(project_root, '.env', f'env.{ENV}') - ENV = 'local' + if ENV == '' or ENV is None: + ENV = 'local' + +print (f"ENV: {ENV}, PROJECT_ROOT: {project_root}, ENV_FILES_PATH: {env_files_path}") # Sensitive data not injected through pipelines config = dotenv_values(env_files_path) diff --git a/src/local/main/init_jenkins.py b/src/local/main/init_jenkins.py index 80c86f7..51166c8 100644 --- a/src/local/main/init_jenkins.py +++ b/src/local/main/init_jenkins.py @@ -1,111 +1,123 @@ import jenkins -import src.local.main.config as config_module -import src.local.main.services as services -from src.local.main.services import CredentialsType - -# ------------------------------- Main ------------------------------- - -print(f"JENKINS INFO -> Jenkins URL: {config_module.get(config_module.ConfigKeys.JENKINS_URL)}, Username: {config_module.get(config_module.ConfigKeys.JENKINS_USER)}, API Token: {config_module.get(config_module.ConfigKeys.JENKINS_PASS)}") - -# ----------- Generate SSH key pair ----------- -print('Getting SSH key...') -private_key = services.get_ssh() # Get the private key from the SSH key pair to connect Jenkins node via SSH to the agent (machine defined) -print('Building credentials SSH on Jenkins...') -services.build_credentials(CredentialsType.SSH) - -# Jenkins version -user = services.jenkins_service.get_whoami() -version = services.jenkins_service.get_version() -print('JENKINS INFO -> Hello %s from Jenkins %s' % (user['fullName'], version)) - -# Node details -node_name = 'docker-node' -node_description = 'Node configured with Docker' -remote_fs = '/var/jenkins_home' -labels = 'docker' -num_executors = 2 - -# Script to install Docker on the node -install_docker_script = '''#!/bin/bash -if ! [ -x "$(command -v docker)" ]; then - echo "Docker is not installed. Proceeding to install Docker..." - curl -fsSL https://get.docker.com -o get-docker.helpers - helpers get-docker.helpers - sudo usermod -aG docker jenkins # Add the Jenkins user to the Docker group - echo "Docker installed successfully." -else - echo "Docker is already installed." -fi -''' - -# Node configuration with the Docker installation script -# node_config_xml = f''' -# -# {node_name} -# {node_description} -# {remote_fs} -# {num_executors} -# EXCLUSIVE -# -# -# {install_docker_script} -# -# -# -# ''' -node_config_xml = f''' - - {node_name} - {node_description} - {remote_fs} - {num_executors} - EXCLUSIVE - - - -''' - -# Create the node in Jenkins -try: - # List all nodes - print(services.jenkins_service.get_nodes()) - if services.jenkins_service.node_exists(node_name): - print(f"Node '{node_name}' already exists. Deleting it...") - services.jenkins_service.delete_node(node_name) - else: - print(f"Node '{node_name}' does not exist.") - print(f"Creating node '{node_name}' with Docker installation...") - - ssh_port = 22 - ssh_user = config_module.get(config_module.ConfigKeys.JENKINS_USER) - ssh_credentials = config_module.get(config_module.ConfigKeys.AGENT_CREDENTIALS_SSH) - ssh_agent_host = 'host.docker.internal' # Is the host where Jenkins Docker is running - print(f"SSH data -> Port: {ssh_port}, User: {ssh_user}, Credentials: {ssh_credentials}, Host: {ssh_agent_host}") - - params = { - 'port': ssh_port, - 'username': ssh_user, - 'credentialsId': ssh_credentials, - 'host': ssh_agent_host - } - print("Creating node with parameters") - services.jenkins_service.create_node( - node_name, - nodeDescription=node_description, - remoteFS=remote_fs, - labels=labels, - exclusive=True, - launcher=jenkins.LAUNCHER_SSH, - launcher_params=params) - - print(f"Node '{node_name}' created successfully with Docker installation.") - - # TODO: Init SSH Client on agent host - start_jenkins_agent = './start_jenkins_agent.helpers' - print(f'Now it will start the Jenkins agent with the following command: {start_jenkins_agent}') - # subprocess.run(['bash', start_jenkins_agent], check=True) - # print('Jenkins agent started successfully.') - -except jenkins.JenkinsException as e: - print(f"Error creating node: {e}") \ No newline at end of file +import config as config_module +import services as services +from services import CredentialsType + + +def fetch(): + # ------------------------------- Main ------------------------------- + + print(f"JENKINS INFO -> Jenkins URL: {config_module.get(config_module.ConfigKeys.JENKINS_URL)}, Username: {config_module.get(config_module.ConfigKeys.JENKINS_USER)}, API Token: {config_module.get(config_module.ConfigKeys.JENKINS_PASS)}") + + # ----------- Generate SSH key pair ----------- + print('Getting SSH key...') + private_key = services.get_ssh() # Get the private key from the SSH key pair to connect Jenkins node via SSH to the agent (machine defined) + print('Building credentials SSH on Jenkins...') + services.build_credentials(CredentialsType.SSH) + + # Jenkins version + user = services.jenkins_service.get_whoami() + version = services.jenkins_service.get_version() + print('JENKINS INFO -> Hello %s from Jenkins %s' % (user['fullName'], version)) + + # Node details + node_name = 'docker-node' + node_description = 'Node configured with Docker' + remote_fs = '/var/jenkins_home' + labels = 'docker' + num_executors = 2 + + # Script to install Docker on the node + install_docker_script = '''#!/bin/bash + if ! [ -x "$(command -v docker)" ]; then + echo "Docker is not installed. Proceeding to install Docker..." + curl -fsSL https://get.docker.com -o get-docker.helpers + helpers get-docker.helpers + sudo usermod -aG docker jenkins # Add the Jenkins user to the Docker group + echo "Docker installed successfully." + else + echo "Docker is already installed." + fi + ''' + + # Node configuration with the Docker installation script + # node_config_xml = f''' + # + # {node_name} + # {node_description} + # {remote_fs} + # {num_executors} + # EXCLUSIVE + # + # + # {install_docker_script} + # + # + # + # ''' + node_config_xml = f''' + + {node_name} + {node_description} + {remote_fs} + {num_executors} + EXCLUSIVE + + + + ''' + + # Create the node in Jenkins + try: + # List all nodes + print(services.jenkins_service.get_nodes()) + if services.jenkins_service.node_exists(node_name): + print(f"Node '{node_name}' already exists. Deleting it...") + services.jenkins_service.delete_node(node_name) + else: + print(f"Node '{node_name}' does not exist.") + print(f"Creating node '{node_name}' with Docker installation...") + + ssh_port = 22 + ssh_user = config_module.get(config_module.ConfigKeys.JENKINS_USER) + ssh_credentials = config_module.get(config_module.ConfigKeys.AGENT_CREDENTIALS_SSH) + ssh_agent_host = 'host.docker.internal' # Is the host where Jenkins Docker is running + print(f"SSH data -> Port: {ssh_port}, User: {ssh_user}, Credentials: {ssh_credentials}, Host: {ssh_agent_host}") + + params = { + 'port': ssh_port, + 'username': ssh_user, + 'credentialsId': ssh_credentials, + 'host': ssh_agent_host + } + print("Creating node with parameters") + services.jenkins_service.create_node( + node_name, + nodeDescription=node_description, + remoteFS=remote_fs, + labels=labels, + exclusive=True, + launcher=jenkins.LAUNCHER_SSH, + launcher_params=params) + + print(f"Node '{node_name}' created successfully with Docker installation.") + + # TODO: Init SSH Client on agent host + start_jenkins_agent = './start_jenkins_agent.helpers' + print(f'Now it will start the Jenkins agent with the following command: {start_jenkins_agent}') + # subprocess.run(['bash', start_jenkins_agent], check=True) + # print('Jenkins agent started successfully.') + + except jenkins.JenkinsException as e: + print(f"Error creating node: {e}") + +def start(): + print("Init_jenkins:: ...") + try: + fetch() + except Exception as e: + print("Error: ", e) + +if config_module.ENV == 'local': + start() \ No newline at end of file diff --git a/src/local/main/main.py b/src/local/main/main.py new file mode 100644 index 0000000..f86c55d --- /dev/null +++ b/src/local/main/main.py @@ -0,0 +1,16 @@ +import build +import init_jenkins + +def main(): + try: + print('Starting Jenkins configuration...') + init_jenkins.start() + print('Building Jenkins... Done!') + + print('Building Jenkins Job...') + build.start() + print('Building Jenkins Job... Done!') + except Exception as e: + print("Error: ", e) + +main() \ No newline at end of file diff --git a/src/local/main/services.py b/src/local/main/services.py index 84a15d2..5a045bd 100644 --- a/src/local/main/services.py +++ b/src/local/main/services.py @@ -6,12 +6,17 @@ import jenkins import requests -import src.local.main.config as config_module +import config as config_module -# Connect to the Jenkins server -jenkins_service = jenkins.Jenkins(config_module.get(config_module.ConfigKeys.JENKINS_URL), - config_module.get(config_module.ConfigKeys.JENKINS_USER), - config_module.get(config_module.ConfigKeys.JENKINS_PASS)) +print(f"Connecting to Jenkins. Connection Data -> URL: {config_module.get(config_module.ConfigKeys.JENKINS_URL)}, User: {config_module.get(config_module.ConfigKeys.JENKINS_USER)}, API Token: {config_module.get(config_module.ConfigKeys.JENKINS_PASS)}") + +try: + # Connect to the Jenkins server + jenkins_service = jenkins.Jenkins(config_module.get(config_module.ConfigKeys.JENKINS_URL), + config_module.get(config_module.ConfigKeys.JENKINS_USER), + config_module.get(config_module.ConfigKeys.JENKINS_PASS)) +except Exception as e: + print(f"Error connecting to Jenkins: {e}") class CredentialsType(str, Enum): USER = 'USER' diff --git a/src/local/requirements.txt b/src/local/requirements.txt index cb4426b..91de92f 100644 --- a/src/local/requirements.txt +++ b/src/local/requirements.txt @@ -1 +1,5 @@ -python-jenkins~=1.8.2 \ No newline at end of file +python-jenkins~=1.8.2 +setuptools~=68.2.0 +requests~=2.32.3 +urllib3~=2.2.3 +python-dotenv~=1.0.1 \ No newline at end of file