dfcx cicd pipeline

examples/dfcx_agent_cicd/cicd_code/UAT/cloudbuild_deploy.yaml

@@ -0,0 +1,49 @@
+steps:
+
+  - id: SHAGCSCopy
+    name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    #dir: 'set your path till the readme doc in the git'
+    entrypoint: /bin/bash
+    args:
+      - '-c'
+      - |
+        chmod 777 UAT/gcssha.sh
+        UAT/gcssha.sh $COMMIT_SHA
+
+  - id: deployagent
+    name: 'python:3.10'
+    #dir: 'set your path till the readme doc in the git'
+    entrypoint: /bin/bash
+    args:
+    - -c
+    - |
+      pip3 install -r UAT/requirements.txt
+      python3 -m UAT.deploy $COMMIT_SHA
+      echo $?
+
+  - id: CheckExitCode
+    name: 'gcr.io/cloud-builders/gcloud'
+    #dir: 'set your path till the readme doc in the git'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        if [[ "$$BUILD_STATUS" -ne 0 ]]; then
+          echo "Stopping the build due to a previous failure."
+          exit 1
+        fi
+
+
+  - id: triggerproddeploy
+    name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    #dir: 'set your path till the readme doc in the git'
+    entrypoint: /bin/bash
+    args:
+      - '-c'
+      - |
+        chmod 777 UAT/trigger.sh 
+        UAT/trigger.sh $LOCATION $COMMIT_SHA
+
+
+options:
+  logging: CLOUD_LOGGING_ONLY

examples/dfcx_agent_cicd/cicd_code/UAT/deploy.py

@@ -0,0 +1,71 @@
+""" UAT Deployment functions"""
+
+import sys
+import json
+import logging
+
+from shared.deployment import Deployment
+
+
+#from .shared.deployments import Deployment
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="UAT: %(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+def main(data):
+    """
+    Deploys and tests a Dialogflow CX agent in a UAT environment.
+
+    This function performs the following steps:
+
+    1. Initializes a Deployment object with the provided data.
+    2. Imports the agent to the specified UAT webhook environment.
+    3. Validates test cases.
+    4. Collects flow IDs.
+    5. Deletes versions based on count.
+    6. Cuts a new version.
+    7. Deploys the new version.
+    8. Updates the datastore with UAT information.
+
+    Args:
+        data: A dictionary containing configuration data, including the 'uat_webhook_env' key.
+    """
+
+    dep=Deployment(data)
+    # call the steps sequentially
+    dep.import_agent(webhookenv=data["uat_webhook_env"])
+    dep.test_case_validation()
+    dep.collect_flow_id()
+    dep.version_count_delete()
+    dep.version_cut()
+    dep.deploy_versions()
+    dep.datastore_update("uat")
+
+
+
+if __name__=="__main__":
+    # read env variables
+    with open("config.json" , encoding='utf-8') as config_file:
+        config = json.load(config_file)
+    SHA_ID=sys.argv[1]
+    obj=f"UAT/{config['agent_name']}/{SHA_ID}"
+    sha_gs_loc=(
+        f"gs://{config['bucket']}/UAT/{config['agent_name']}/{SHA_ID}"
+    )
+    logging.info("Agent location: %s" ,sha_gs_loc)
+    #adding additional variables to dict
+    config["sha_agent_gcs_location"]=sha_gs_loc
+    config["target_project_id"] = config["uat_project"]
+    config["target_environment_name"]=config["uat_env_deploy"]
+    with open("agent_artifacts/metadata.json" , encoding='utf-8') as metadata_file:
+        metadata = json.load(metadata_file)
+
+    config["source_flow_names"]=metadata["source_flow_names"]
+    config["updated_commit_message"]=metadata["updated_commit_message"]
+
+    # To execute steps in order
+    main(config)

examples/dfcx_agent_cicd/cicd_code/UAT/gcssha.sh

@@ -0,0 +1,21 @@
+
+# Set your GCS bucket name and destination directory
+apt-get update && apt-get install -y jq
+export GCS_BUCKET=$(jq -r .bucket config.json)
+export agent_name=$(jq -r .agent_name config.json)
+export DESTINATION_DIR="UAT/${agent_name}/"
+echo $DESTINATION_DIR
+# Create a local directory
+mkdir -p $1
+
+# Copy your two files to the local directory
+cp agent_artifacts/$agent_name $1
+cp agent_artifacts/metadata.json $1
+
+# Upload the local directory to GCS
+gsutil -m cp -r $1 "gs://$GCS_BUCKET/$DESTINATION_DIR"
+
+# Clean up the local directory if needed
+rm -r $1
+
+echo "Files copied and uploaded to GCS."

examples/dfcx_agent_cicd/cicd_code/UAT/requirements.txt

@@ -0,0 +1,3 @@
+dfcx-scrapi
+google-cloud-storage
+pandas

examples/dfcx_agent_cicd/cicd_code/UAT/trigger.sh

@@ -0,0 +1,22 @@
+echo $1
+apt-get update && apt-get install -y jq
+export devops_project_id=$(jq -r .devops_project config.json)
+export prod_project_id=$(jq -r .prod_project config.json)
+
+#Use below command to trigger the build if manual invokation is used. Since there is no secret , no extra charges
+
+export build_info=$(gcloud builds triggers run prodbuild --project=$devops_project_id --substitutions=_COMMIT_SHA=$2 --region=$1 --format=json)
+echo "devops prod triggerdone"
+
+
+#getting the trigger id of the above trigger
+
+export prod_build_id=$(echo "$build_info" | jq -r '.metadata.build.id')
+echo "build id returned back is"
+echo $prod_build_id
+
+
+#Trigger the build in prod project which is used for approval
+gcloud builds triggers run prodapprovebuild --project=$devops_project_id --substitutions=_APP_BUILD_ID=$prod_build_id --region=$1
+
+echo "prod project approve build triggered"

examples/dfcx_agent_cicd/cicd_code/approveprod/cloudbuild_appr.yaml

@@ -0,0 +1,36 @@
+steps:
+  - name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    args:
+      - '-c'
+      - |
+        apt-get update && apt-get install -y jq
+        echo $BUILD_ID
+
+        export devopsprojecthere=$(jq -r .devops_project config.json)
+        export build_info=$(gcloud builds describe $BUILD_ID --region=us-central1 --format=json)
+        export approverhere=$(echo "$build_info" | jq -r '.approval.result.approverAccount')
+        export commenthere=$(echo "$build_info" | jq -r '.approval.result.comment')
+        export tokenhere=$(gcloud auth print-access-token)
+        
+        echo $approverhere
+        echo $tokenhere
+
+        chmod 777 approveprod/trigger.sh
+
+        sed -i "s/tokenhere/$tokenhere/g" approveprod/trigger.sh
+        sed -i "s/approverhere/$approverhere/g" approveprod/trigger.sh
+        sed -i "s/devopsprojecthere/$devopsprojecthere/g" approveprod/trigger.sh
+        sed -i "s/commenthere/$commenthere/g" approveprod/trigger.sh
+        sed -i "s/appbuildhere/$_APP_BUILD_ID/g" approveprod/trigger.sh
+        cat approveprod/trigger.sh
+        approveprod/trigger.sh
+        echo $?
+        echo "prod build approved from code"
+
+        echo "error exit code"
+        
+    id: triggerexportbuild
+    entrypoint: /bin/bash
+options:
+  logging: CLOUD_LOGGING_ONLY
+  dynamicSubstitutions: true

examples/dfcx_agent_cicd/cicd_code/approveprod/trigger.sh

@@ -0,0 +1,6 @@
+curl --request POST \
+    'https://cloudbuild.googleapis.com/v1/projects/devopsprojecthere/locations/us-central1/builds/appbuildhere:approve?access_token=tokenhere' \
+    --header 'Accept: application/json'\
+    --header 'Content-Type:application/json' --data \
+    '{"approvalResult":{"decision":"APPROVED","comment":"commenthere","approverAccount":"approverhere"}}' \
+    --compressed

examples/dfcx_agent_cicd/cicd_code/config.json

@@ -0,0 +1,16 @@
+{
+    "agent_name" : "carrental",
+    "dev_env_pull" : "ready to deploy",
+    "uat_env_deploy" : "ready to test",
+    "prod_env_deploy" :"deployed",
+    "devprodsyncenv" :"deployed",
+    "bucket": "dfcx_agent_cicd_export",
+    "dev_project": "yourprojectid",
+    "uat_project" : "yourprojectid",
+    "prod_project": "yourprojectid",
+    "devops_project": "yourprojectid",
+    "uat_webhook_env": "uat",
+    "prod_webhook_env": "prod",
+    "uat_engine_id" :"",
+    "prod_engine_id" :""
+}

examples/dfcx_agent_cicd/cicd_code/export/cloudbuild_export.yaml

@@ -0,0 +1,147 @@
+
+availableSecrets:
+  secretManager:
+  - versionName: # your version name
+    env: 'SSH_KEY'
+
+steps:
+# Access the id_github file fvikramvikrom Secret Manager, and setup SSH
+  - id: mountsshkey 
+    name: 'gcr.io/cloud-builders/git'
+    #dir: 'set your path till the readme doc in the git'
+    secretEnv: ['SSH_KEY']
+    entrypoint: /bin/bash
+    args:
+    - -c
+    - |
+      echo "$$SSH_KEY" >> /root/.ssh/id_rsa
+      chmod 400 /root/.ssh/id_rsa
+      cp known_hosts.github /root/.ssh/known_hosts
+    volumes:
+    - name: 'ssh'
+      path: /root/.ssh
+
+  # Clone the repository
+  - id: clonerepo
+    name: 'gcr.io/cloud-builders/git'
+    #dir: 'set your path till the readme doc in the git'
+    args:
+    - clone
+    - --recurse-submodules
+    - [email protected]:$REPO_FULL_NAME
+    volumes:
+    - name: 'ssh'
+      path: /root/.ssh
+
+  - id: limitbuildcheck
+    name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    entrypoint: /bin/bash
+    args:
+    - -c
+    - |
+      export parallelbuild=$(gcloud builds list --region=$LOCATION --filter="substitutions.TRIGGER_NAME=$TRIGGER_NAME AND status=WORKING" --format="value(status)" | wc -l)        
+      export approvebuild=$(gcloud builds list --region=$LOCATION --format="value(status)" --filter="substitutions.TRIGGER_NAME='prodbuild' AND status='PENDING'" | wc -l)
+      if [ $parallelbuild -gt 1 ]
+      then
+        echo "parallel build running. This may corrupt the exported files in GCS location"
+        exit 1
+      else
+        echo "Proceeding. No other parallel export build"  
+      fi
+      if [ $approvebuild -gt 0 ]
+      then
+        echo "some other build waiting for approval"
+        exit 1
+      else
+        echo "Proceeding. No builds waiting for approval"  
+      fi
+
+  - id: fetchuser
+    #dir: 'set your path till the readme doc in the git'
+    name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    entrypoint: /bin/bash
+    args:
+      - '-c'
+      - |
+        echo $BUILD_ID
+        export buildhere=$BUILD_ID
+        export trigid=$(gcloud builds describe $BUILD_ID --region=$LOCATION --format="value(buildTriggerId)")
+        sed -i "s/triggerhere/$trigid/g" export/trigger.sh
+        chmod 777 export/trigger.sh
+        export w1=$(export/trigger.sh)
+        export w2=$(echo $w1 | cut -d " " -f2)
+        export runnerid=$w2
+        export runnername=$(echo $runnerid | cut -d '@' -f 1)
+        echo $buildhere
+        echo $runnerid
+        echo $runnername
+        pwd
+        ls
+        echo $runnername > ./runnername.txt
+        echo $runnerid > ./runnerid.txt
+        echo "path of runner id"
+        pwd
+
+        
+  - id: Exportgcs
+    #dir: 'set your path till the readme doc in the git'
+    name: 'python:3.10'
+    entrypoint: /bin/bash
+    args:
+    - -c
+    - |
+      ls
+      pwd
+      pip3 install -r export/requirements.txt
+      export runnerid=$(cat runnerid.txt)
+      echo "runner id is "
+      echo $runnerid
+      python3 -m export.export ${_USERCOMMITMESSAGE} $runnerid
+
+  - id: downloadartifacts
+    #dir: 'set your path till the readme doc in the git'
+    name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    entrypoint: /bin/bash
+    args:
+    - -c
+    - |
+      apt-get update && apt-get install -y jq
+      export agent_name=$(jq -r .agent_name config.json)
+      export bucket_name=$(jq -r .bucket config.json)
+      echo $agent_name
+      echo $bucket_name
+      mkdir agenttemp
+      gsutil cp "gs://$bucket_name/exports/dev/$agent_name" agenttemp/$agent_name
+      gsutil cp "gs://$bucket_name/exports/dev/${agent_name}_metadata.json" agenttemp/metadata.json
+
+  - id: csrcheckin
+    #dir: 'set your path till the readme doc in the git'
+    name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    entrypoint: /bin/bash
+    args:
+    - -c
+    - |
+      export runnerid=$(cat runnerid.txt)
+      export runnername=$(cat runnername.txt)
+
+      export agent_artifacts_path = $(dirname $(dirname $TRIGGER_BUILD_CONFIG_PATH))
+      chmod 777 export/repopush.sh
+      export/repopush.sh $REPO_NAME $agent_artifacts_path
+      cd $REPO_NAME/$agent_artifacts_path
+      ls
+      cd agent_artifacts
+      ls
+      git config --global user.name $runnername
+      git config --global user.email $runnerid
+      git add .
+      git diff --name-only
+      git commit --allow-empty -m "commited by $runnerid with message ${_USERCOMMITMESSAGE}"
+      
+      git push -u origin main
+    volumes:
+    - name: 'ssh'
+      path: /root/.ssh
+
+options:
+  logging: CLOUD_LOGGING_ONLY
+  dynamicSubstitutions: true