.gitlab-ci.yml

variables:
  GIT_STRATEGY: clone
  CAR_NAME: dr_drift
  KITCAR_REPO_PATH: /builds/kitcar/
  INIT_INPUT: 1\nn   # Do not install CUDA
  DISPLAY: ":1.0"

default:
  image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci
  retry:
    max: 2
    when:
    - runner_system_failure
    - stuck_or_timeout_failure
    - unknown_failure
  before_script:
  # Run the container start script to enable the pulseaudio & xserver
  - bash init/docker_start_script.sh
  # Print out python version for debugging
  - python3 -V
  # Install packages and requirements
  - printf $INIT_INPUT | ./init/init.sh || true
  # Source bashrc
  - source init/bashrc || true
  - source /home/kitcar/kitcar-rosbag/devel/setup.bash --extend

stages:
- lint
- build
- test
- deploy

lint-code:
  stage: lint
  variables:
    GIT_LFS_SKIP_SMUDGE: '1'  #  Don't pull Git LFS
  script:
  # Check if all files are correctly formatted and comply to our formatting rules.
  - pre-commit run --all-files

lint-catkin:
  stage: lint
  only:
    changes:
    - "**/CMakeLists.txt"
    - "**/package.xml"
  script:
  - catkin_lint --resolve-env . --ignore unknown_package

######## BUILD STAGE ########

# Build simulation
build-code:
  stage: build
  needs: []
  variables:
    GIT_LFS_SKIP_SMUDGE: '1'  #  Don't pull Git LFS
  script:
  - cd simulation
  - catkin_make
  artifacts:
    paths:
    - simulation/devel
    - simulation/build
    expire_in: 2 day

######## TEST STAGE #########

# Run module test in utils packages
test-python-packages:
  stage: test
  needs: ["build-code"]
  variables:
    GIT_LFS_SKIP_SMUDGE: '1'  #  Don't pull Git LFS
  script:
  - coverage run -m simulation.utils.geometry.test
  - coverage run -a -m simulation.utils.ros_base.test.parameter_object_test

  - coverage run -a -m simulation.utils.road.sections.test

  - coverage run -a -m simulation.src.simulation_evaluation.src.speaker.speakers.test
  - coverage run -a -m simulation.src.simulation_evaluation.src.state_machine.test

    # Test the urdf package
  - coverage run -a -m simulation.utils.urdf.test.test_core

    # Collect coverage
  - coverage report
  - cp .coverage .coverage_py
  artifacts:
    paths:
    - .coverage_py
    expire_in: 2 days

test-machine-learning-package:
  image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci
  stage: test
  needs: ["build-code"]
  variables:
    GIT_LFS_SKIP_SMUDGE: '1'  #  Don't pull Git LFS
    CUDA_VISIBLE_DEVICES: ""   # Don't use CUDA in CI
  script:
  - coverage run -m simulation.utils.machine_learning.models.test
  - coverage run -a -m simulation.utils.machine_learning.cycle_gan.models.test
  - coverage run -a -m simulation.utils.machine_learning.data.test
    # Test if the camera publishes images with the cycle gan turned on
  - dvc pull simulation/utils/machine_learning/cycle_gan/checkpoints/dr_drift/latest_net_g_b_to_a.pth
  - git lfs pull --include=simulation/utils/machine_learning/cycle_gan/resources/mask.png
  - rostest simulation_brain_link sensor_camera.test apply_gan:=true use_wasserstein_gan:=false
  - rostest simulation_brain_link sensor_camera.test apply_gan:=true use_wasserstein_gan:=true

    # Collect coverage
  - mv /root/.ros/.coverage* .
  - coverage combine -a
  - coverage report
  - cp .coverage .coverage_ml
  artifacts:
    paths:
    - .coverage_ml
    expire_in: 2 days

test-ros-packages:
  stage: test
  needs: ["build-code"]
  script:
  - cd simulation
  - catkin_make run_tests -j1
  - catkin_test_results    # Ensure that pipeline fails when test fails
  - cd ..

    # Collect coverage
  - mv /root/.ros/.coverage* .
  - coverage combine
  - coverage report
  - cp .coverage .coverage_ros
  artifacts:
    paths:
    - .coverage_ros
    expire_in: 2 days

# Run tests to ensure that the evaluation pipeline is working
test-drive-evaluation:
  stage: test
  needs: ["build-code"]
  tags:
    # Always require to use the workstation to ensure enough computational power
  - workstation
  variables:
    PATH_FILES: $KITCAR_REPO_PATH/kitcar-gazebo-simulation/simulation/src/simulation_evaluation/param/drive/paths
  parallel: 3
  script:
    # The drive.test is used to evaluate how the car drives.
    # With automatic_drive:=true the path of the car is predetermined and thus the correctness
    # of the evaluation can be tested.
  - mkdir rosbags
  - python3 -m simulation.utils.drive_test.run --config simulation/utils/drive_test/test/ci.yaml
    --runner_index $CI_NODE_INDEX --total_runners $CI_NODE_TOTAL
  artifacts:
    when: always
    paths:
    - rosbags
    expire_in: 1 week

test-documentation:
  image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci
  stage: test
  needs: ["build-code"]
  script:
    # Get DVC files
  - dvc pull -R docs
    # Test if the onboarding documentation is correct
  - python3 docs/content/tutorials/road_examples/example.py
  - cd docs
  - make doctest
  - cd ..
    # Calculate coverage
  - docstr-coverage simulation/ -F 0
  - echo "TOTAL $(docstr-coverage simulation/ -p -F 0 | cut -d . -f 1)%"

######### Documentation ########

# Build the sphinx documentation
# If this is the master branch, the website deployer will also update doc.kitcar-team.de!
build-documentation:
  image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci
  stage: deploy
  needs: ["test-documentation", "build-code"]
  script:
    # Get DVC files
  - dvc pull -R docs
    # Build the documentation
  - cd docs
  - make html
  artifacts: # Extract artifacts to view the documentation that has been generated
    paths:
    - docs/_build/html/*
    expire_in: 2 day

collect-coverage:
  image: git.kitcar-team.de:4567/kitcar/kitcar-python-utils/ci/python:3.8.9-alpine
  stage: deploy
  needs:
  - build-code
  - test-python-packages
  - test-machine-learning-package
  - test-ros-packages
  before_script:
    # Install coverage pip package
  - pip install coverage
  script:
  - coverage combine .coverage_py .coverage_ml .coverage_ros
  - coverage xml
  - coverage html
  artifacts:
    paths:
    - htmlcov
    expire_in: 20 days
    reports:
      cobertura: coverage.xml

cml-cyclegan:
  stage: deploy
  image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci:cml
  needs: []
  rules:
  - if: '$CI_COMMIT_REF_NAME == "master"'
  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    when: never
  - when: manual
  allow_failure: true
  variables:
    INIT_INPUT: 1\ny   # Install CUDA
    GIT_LFS_SKIP_SMUDGE: '1'  # Don't pull Git LFS
  tags:
  - gpu
  script:
  - git lfs pull --include=simulation/utils/machine_learning/cycle_gan/resources/mask.png
  - dvc pull data/real_images/maschinen_halle_parking -j 8
  - dvc pull data/simulated_images/test_images -j 8
  - cd simulation/utils/machine_learning/cycle_gan
    # pull cache from server and checkout
  - dvc pull dvc.yaml -j 8
    # Test current model, make videos and commit to dvc
  - dvc repro test_dr_drift -s
  - dvc repro test_discriminators_dr_drift -s
  - dvc repro make_video_dr_drift -s
  - cd ../../../../
    # Create content for the report.md
  - echo "# Dr Drift" >> report.md
  - echo "## Test Generators" >> report.md
  - L="$(cml-publish simulation/utils/machine_learning/cycle_gan/results/dr_drift/stacked.mp4
    --gitlab-uploads)"
  - echo "![]($L)" >> report.md
  - echo "[Link to mp4]($L)" >> report.md
  - echo "<br>" >> report.md
  - echo "## Test Discriminators" >> report.md
  - losses=`cat simulation/utils/machine_learning/cycle_gan/results/dr_drift/discriminator_losses.txt`
  - A="$(cut -d',' -f1 <<<"$losses")"
  - B="$(cut -d',' -f2 <<<"$losses")"
  - echo "**AVG-Loss Discriminator A =** $A" >> report.md
  - echo "<br>" >> report.md
  - echo "**AVG-Loss Discriminator B =** $B" >> report.md
  - echo "<br>" >> report.md
  - echo "## Loss values" >> report.md
  - cml-publish simulation/utils/machine_learning/cycle_gan/checkpoints/dr_drift/loss.png --gitlab-uploads
    --md --title 'Loss values' >> report.md
    # Send report.md as comment
  - cml-send-comment report.md

# Build and deploy the CI docker image to the gitlab registry
# This is done everytime the master branch is updated
docker-images:
  stage: deploy
  image: docker:20.10.5
  needs: ["build-code"]
  rules:
  - if: '$CI_COMMIT_REF_NAME == "master"'
  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    when: never
  - when: manual
    allow_failure: true
  tags:
  - docker
  - server
  services:
  - docker:20.10.5-dind
  variables:
    # Ensure that lfs files are available in resulting images!
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ''
  before_script:
  - export IMAGE_URL=$CI_REGISTRY/kitcar/kitcar-gazebo-simulation
  - |
    if [ "$CI_COMMIT_REF_NAME" == "master" ]; then
      export CI_IMAGE_TAG=latest
      export CI_IMAGE_TAG_CML=cml
      export CI_IMAGE_TAG_ROS=latest
    else
      export CI_IMAGE_TAG=branch_${CI_COMMIT_REF_NAME}
      export CI_IMAGE_TAG_CML=branch_${CI_COMMIT_REF_NAME}_cml
      export CI_IMAGE_TAG_ROS=branch_${CI_COMMIT_REF_NAME}
    fi
  - apk add --no-cache docker-compose git
  - docker --version
  - docker info
  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    # Clone kitcar-rosbag
  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@git.kitcar-team.de/kitcar/kitcar-rosbag.git

  - cd docker
    # Build Docker images
  - docker-compose build
    # Push docker images
  - docker-compose push

dvc-repro-images:
  image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci
  needs: ["build-code"]
  rules:
  - if: '$CI_COMMIT_REF_NAME == "master"'
  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    when: never
  - when: manual
  allow_failure: true
  stage: deploy
  parallel:
    matrix:
    - DATA_FOLDER: [labeled_images, simulated_images]
  tags:
  - workstation
  script:
  - ./gitlab-ci/setup_cml_bot_git.sh

  # Run ssh-agent (inside the build environment)
  - eval $(ssh-agent -s)
  # Add the SSH key stored in CML_BOT_SSH_PRIVATE_KEY variable to the agent store
  - ssh-add <(echo "$CML_BOT_SSH_PRIVATE_KEY")

  - export NEW_BRANCH_NAME=ci/${CI_COMMIT_BRANCH}_${CI_JOB_ID}
  - echo ${NEW_BRANCH_NAME}

  - mv /home/kitcar/kitcar-rosbag $KITCAR_REPO_PATH
  - ls $KITCAR_REPO_PATH
  - source $KITCAR_REPO_PATH/kitcar-rosbag/devel/setup.bash --extend
    # Actual script!
  - dvc pull simulation/utils/machine_learning/cycle_gan/checkpoints/dr_drift/latest_net_g_b_to_a.pth
  - cd data/${DATA_FOLDER}
  - dvc pull dvc.yaml -j 8 || true
  - dvc repro -s
  - |
    # Check if lock file has changed
    if [ $(./../../gitlab-ci/has_file_changed.sh dvc.lock) -eq 1 ]; then
      git add dvc.lock

      # Push to DVC Remote
      dvc push -j 8

      # Commit and Push to GitLab
      COMMIT_MSG="data: Reproduce ${DATA_FOLDER}"
      ./../../gitlab-ci/commit_and_push_to_branch.sh "${COMMIT_MSG}" "${NEW_BRANCH_NAME}"

      # Create Merge Request
      ENDPOINT_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests"
      BODY="{ \
        \"id\": ${CI_PROJECT_ID}, \
        \"source_branch\": \"${NEW_BRANCH_NAME}\", \
        \"target_branch\": \"${CI_COMMIT_BRANCH}\",\
        \"remove_source_branch\": true, \
        \"title\": \"${COMMIT_MSG}\", \
        \"labels\": \"topic::data\", \
        \"assignee_id\": 600 \
      }";
      curl \
          --request POST "${ENDPOINT_URL}" \
          --header "PRIVATE-TOKEN: ${AUTO_MERGE_REQUEST}" \
          --header "Content-Type: application/json" \
          --data "${BODY}";
    fi
  artifacts:
    paths:
    - data/labeled_images/dvc.lock
    - data/simulated_images/dvc.lock