.gitlab-ci.yml

stages:
  - bootstrap
  - build
  - post-deploy
  - cleanup

variables:
  PREVIEW_CONFIG: "config/preview.yaml"
  LIVE_CONFIG: "config/live.yaml"
  ARTIFACT_RESOURCE: "public"
  LIVE_DOMAIN: "https://docs.datadoghq.com/"
  PREVIEW_DOMAIN: "https://docs-staging.datadoghq.com/"
  # datadog-ci
  DATADOG_SITE: 'datadoghq.com'
  DATADOG_SUBDOMAIN: 'dd-corpsite'
  FORCE_COLOR: '1'
  GIT_DEPTH: 50
  # gitlab checkout
  DOCS_REPO_NAME: documentation
  REPO_URL: "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/${DOCS_REPO_NAME}.git"

# ================== copy scripts =============== #
before_script:
  - "[ -d local/bin ] && find local/bin/ -type f -exec cp {} /usr/local/bin \\;"  # load scripts
  - "[ -d local/etc ] && find local/etc/ -type f -exec cp {} /etc \\;"  # load configs
  - "[ -f /usr/local/bin/helpers.sh ] && source /usr/local/bin/helpers.sh"  # source helpers so they are available in the container


cache: &cache
  key:
    files:
      - yarn.lock
  paths:
    - node_modules/

# ================== templates ================== #
.base_template: &base_template
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/corp-ci:david.jones_update-git
  tags:
    - "runner:main"
    - "size:2xlarge"
  except:
    - tags
  only:
    - branches

# ================== Install Node Dependencies ================== #
install_dependencies:
  <<: *base_template
  stage: bootstrap
  cache:
    <<: *cache
  variables:
    GIT_STRATEGY: none
  script:
    - git version
    - git clone --branch $CI_COMMIT_REF_NAME --depth 1 --filter=blob:none --no-checkout $REPO_URL sparse-docs
    - cd sparse-docs && git sparse-checkout init --cone && git sparse-checkout set local && git checkout $CI_COMMIT_REF_NAME && cd ..
    - mkdir -p ./local/bin/sh && cp --remove-destination ./sparse-docs/local/bin/sh/preinstall.sh ./local/bin/sh/preinstall.sh
    - cp --remove-destination ./sparse-docs/package.json ./package.json
    - cp --remove-destination ./sparse-docs/yarn.lock ./yarn.lock
    - yarn install --link-duplicates --frozen-lockfile --cache-folder .yarn
    - rm -rf ./sparse-docs # cleanup runner
  only:
    changes:
      - yarn.lock
  except:
    # skip on automated translation prs
    - /^guacbot\/translation-[0-9]+$/i

# ================== preview ================== #
# If the branch has a name of <slack-user>/<feature-name> then ci builds a preview site
build_preview:
  <<: *base_template
  stage: build
  cache:
    <<: *cache
    policy: pull
  environment: "preview"
  variables:
    URL: ${PREVIEW_DOMAIN}
    CONFIG: ${PREVIEW_CONFIG}
    MESSAGE: ":gift_heart: Your preview site is available!\nNow running tests..."
    CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config_preview.yaml"
    LOCAL: "False"
  script:
    - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
    - touch Makefile.config
    - make BRANCH=${CI_COMMIT_REF_NAME} examples
    - sync_integration_descriptions
    - ./local/bin/py/build/pull_rbac.py $(get_secret 'dd_api_key') $(get_secret 'dd-app-key')
    #- sync_integration_descriptions_cached
    - placehold_translations
    - update_preview_baseurl
    - yarn run build:preview
    # remove service_checks json as we don't need to s3 push that..
    - rm -rf data/service_checks
    - in-isolation deploy_preview
    # remove static images so we can skip from artifact passed in gitlab
    - remove_static_from_repo
    - notify_slack "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_REF_NAME}> is ready for preview. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}| checks running>." "#31b834"
  artifacts:
    paths:
      - ${ARTIFACT_RESOURCE}
      - ./integrations_data
      - .github/
      - .vale.ini
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

# build_preview_legacy:
#   <<: *base_template
#   stage: build
#   environment: "preview"
#   variables:
#     URL: ${PREVIEW_DOMAIN}
#     CONFIG: ${PREVIEW_CONFIG}
#     MESSAGE: ":gift_heart: Your preview site is available!\nNow running tests..."
#     CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config_preview.yaml"
#     LOCAL: "False"
#   script:
#     - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
#     - version_static_assets
#     - touch Makefile.config
#     - make examples
#     - sync_integration_descriptions
#     #- sync_integration_descriptions_cached
#     - placehold_translations
#     - update_preview_baseurl
#     - build_hugo_site_new
#     - in-isolation deploy_preview
#     - collect_static_assets
#     # remove service_checks json as we don't need to s3 push that..
#     - rm -rf data/service_checks
#     - push_site_to_s3
#     - notify_slack "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_REF_NAME}> is ready for preview. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}| checks running>." "#31b834"
#     #- remove_static_from_repo
#     #- create_artifact
#     #- create_artifact_untracked
#   artifacts:
#     paths:
#       - ${ARTIFACT_RESOURCE}
#       - ./integrations_data
#   only:
#     - /.+?\/[a-zA-Z0-9_-]+/

link_checks_preview:
  <<: *base_template
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "preview"
  variables:
    URL: ${PREVIEW_DOMAIN}
    DISALLOW_BOTS: "1"
  script:
    - node ./node_modules/link-checker/dd-link-checker.js --env preview --configFilePath ${CI_PROJECT_DIR}/local/etc/link-check-config.js
  after_script:
    - if [ -e ${CI_PROJECT_DIR}/broken-links.csv  ]; then aws s3 cp --content-type "text/csv; charset=utf-8" --acl "public-read" --cache-control "no-cache" ${CI_PROJECT_DIR}/broken-links.csv s3://origin-static-assets/documentation/brokenlinks/${CI_COMMIT_REF_NAME}/; fi
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

missing_tms_preview:
  image:
    name: 486234852809.dkr.ecr.us-east-1.amazonaws.com/vale:v4036929-ee9929a-2.9.1
    entrypoint: [ "" ]
  tags: ["runner:main", "size:2xlarge"]
  stage: post-deploy
  cache: {}
  environment: "preview"
  variables:
    GIT_STRATEGY: none
  dependencies:
    - build_preview
  script:
    # remove dirs we don't need to scan
    - rm -rf ./public/fr ./public/ja ./public/api/v1/ ./public/api/v2/
    - GOMAXPROCS=$(nproc --all) vale --glob='*.{html}' ./public
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

index_algolia_preview:
  image:
    name: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docsearch-scraper:v2978988-4692ed5-0.8
    entrypoint: [""] # force an empty entrypoint (see https://gitlab.com/gitlab-org/gitlab-runner/issues/2692)
  tags: [ "runner:main","size:2xlarge" ]
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "preview"
  script:
    - |-
        echo APPLICATION_ID=$(get_secret 'ci.documentation.algolia_docsearch_application_id') > .env
        echo API_KEY=$(get_secret 'ci.documentation.algolia_docsearch_api_key') >> .env
        export DISPLAY=:99
        Xvfb :99 -ac &
        echo "n" | docsearch run ./local/etc/algolia/docs_datadoghq_preview.json
  dependencies:
    - build_preview
  when: manual
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

replica_algolia_preview:
  <<: *base_template
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "preview"
  script:
    - ALGOLIA_APP_ID=$(get_secret 'algolia_docsearch_application_id') ALGOLIA_ADMIN_KEY=$(get_secret 'algolia_docsearch_api_key') yarn run algolia:config
  dependencies:
    - build_preview
  when: manual
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

sourcemaps_preview:
  <<: *base_template
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "preview"
  script:
    - yarn run build:webpack:preview -- --devtool source-map
    - DATADOG_API_KEY="$(get_secret 'dd_synthetic_api_key_prod')" ./node_modules/.bin/datadog-ci sourcemaps upload ./public/static --service docs --minified-path-prefix "https://docs-staging.datadoghq.com/${CI_COMMIT_REF_NAME}/static/" --release-version "${CI_COMMIT_SHORT_SHA}"
  dependencies:
    - build_preview
  allow_failure: true
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

# ================== live ================== #
build_live:
  <<: *base_template
  stage: build
  cache:
    <<: *cache
    policy: pull
  environment: "live"
  variables:
    CONFIG: ${LIVE_CONFIG}
    URL: ${LIVE_DOMAIN}
    UNTRACKED_EXTRAS: "data,content/en/agent/basic_agent_usage/heroku.md,content/en/agent/basic_agent_usage/ansible.md,content/en/agent/basic_agent_usage/chef.md,content/en/agent/basic_agent_usage/puppet.md,content/en/developers/integrations,content/en/agent/basic_agent_usage/saltstack.md,content/en/developers/amazon_cloudformation.md,content/en/integrations,content/en/logs/log_collection/android.md,content/en/logs/log_collection/ios.md,content/en/tracing/setup_overview/setup/android.md,content/en/tracing/setup_overview/setup/ruby.md,content/en/security_platform/default_rules,content/en/serverless/forwarder.md,content/en/serverless/datadog_lambda_library/python.md,content/en/serverless/datadog_lambda_library/nodejs.md,content/en/serverless/datadog_lambda_library/ruby.md,content/en/serverless/datadog_lambda_library/go.md,content/en/serverless/datadog_lambda_library/java.md,content/en/real_user_monitoring/android.md,content/en/real_user_monitoring/browser/_index.md,content/en/real_user_monitoring/ios.md,content/en/serverless/serverless_integrations/plugin.md,content/en/serverless/serverless_integrations/macro.md,content/en/serverless/serverless_integrations/cli.md,content/en/logs/log_collection/javascript.md"
    CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config.yaml"
    LOCAL: "False"
  script:
    - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
    - notify_slack "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_SHA:0:8}> sent to gitlab for production deployment. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}|details>" "#FFD700"
    - touch Makefile.config
    - make examples
    - sync_integration_descriptions
    - sync_integration_metrics
    - ./local/bin/py/build/pull_rbac.py $(get_secret 'dd_api_key') $(get_secret 'dd-app-key')
    - placehold_translations
    - yarn run build:live
    - in-isolation deploy_live
    - create_artifact
    - create_artifact_untracked
  artifacts:
    when: on_success
    paths:
      - ${ARTIFACT_RESOURCE}
      - ./integrations_data
      - content/en
    expire_in: 1 week
  only:
    - master


# build_live_legacy:
#   <<: *base_template
#   stage: build
#   environment: "live"
#   variables:
#     CONFIG: ${LIVE_CONFIG}
#     URL: ${LIVE_DOMAIN}
#     UNTRACKED_EXTRAS: "data,content/en/agent/basic_agent_usage/heroku.md,content/en/agent/basic_agent_usage/ansible.md,content/en/agent/basic_agent_usage/chef.md,content/en/agent/basic_agent_usage/puppet.md,content/en/developers/integrations,content/en/agent/basic_agent_usage/saltstack.md,content/en/developers/amazon_cloudformation.md,content/en/integrations,content/en/logs/log_collection/android.md,content/en/logs/log_collection/ios.md,content/en/tracing/setup_overview/setup/android.md,content/en/tracing/setup_overview/setup/ruby.md,content/en/security_monitoring/default_rules"
#     CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config.yaml"
#     LOCAL: "False"
#   script:
#     - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
#     - notify_slack "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_SHA:0:8}> sent to gitlab for production deployment. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}|details>" "#FFD700"
#     - version_static_assets
#     - touch Makefile.config
#     - make examples
#     - sync_integration_descriptions
#     - sync_integration_metrics
#     - placehold_translations
#     - build_hugo_site_new
#     - in-isolation deploy_live
#     - collect_static_assets
#     - push_site_to_s3
#     - create_artifact
#     - create_artifact_untracked
#   artifacts:
#     when: on_success
#     paths:
#       - ${ARTIFACT_RESOURCE}
#       - ./integrations_data
#       - content/en
#     expire_in: 1 week
#   only:
#     - master


index_internal_doc:
  stage: post-deploy
  cache: {}
  environment: "live"
  image: registry.ddbuild.io/datadoc/webapp:prod-released
  dependencies:
    - build_live
  only:
    - master
  except: [ tags, schedules ]
  tags: [ "runner:main", "size:2xlarge" ]
  script:
    - INDEXING_SERVICE=https://datadoc.ddbuild.io index-repository hugo https://docs.datadoghq.com/ "Datadog" "Public Documentation" --source_folder=content/en --tags="team:Documentation,source:hugo-website,visibility:public,github-repository:documentation"

test_live_link_checks:
  <<: *base_template
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "live"
  variables:
    URL: ${LIVE_DOMAIN}
  script:
    - node ./node_modules/link-checker/dd-link-checker.js --env production --configFilePath ${CI_PROJECT_DIR}/local/etc/link-check-config.js
  after_script:
    - if [ -e ${CI_PROJECT_DIR}/broken-links.csv  ]; then aws s3 cp --content-type "text/csv; charset=utf-8" --acl "public-read" --cache-control "no-cache" ${CI_PROJECT_DIR}/broken-links.csv s3://origin-static-assets/documentation/brokenlinks/${CI_COMMIT_REF_NAME}/; fi
  only:
    - master

index_algolia:
  image:
    name: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docsearch-scraper:v2978988-4692ed5-0.8
    entrypoint: [ "" ] # force an empty entrypoint (see https://gitlab.com/gitlab-org/gitlab-runner/issues/2692)
  tags: ["runner:main","size:2xlarge"]
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "live"
  script:
    - |-
      echo APPLICATION_ID=$(get_secret 'ci.documentation.algolia_docsearch_application_id') > .env
      echo API_KEY=$(get_secret 'ci.documentation.algolia_docsearch_api_key') >> .env
      export DISPLAY=:99
      Xvfb :99 -ac &
      echo "n" | docsearch run ./local/etc/algolia/docs_datadoghq.json
  dependencies:
    - build_live
  only:
    - schedules

replica_algolia:
  <<: *base_template
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "live"
  script:
    - ALGOLIA_APP_ID=$(get_secret 'algolia_docsearch_application_id') ALGOLIA_ADMIN_KEY=$(get_secret 'algolia_docsearch_api_key') yarn run algolia:config
  dependencies:
    - build_live
  only:
    - schedules

sourcemaps_live:
  <<: *base_template
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "live"
  script:
    - yarn run build:webpack -- --devtool source-map
    - DATADOG_API_KEY="$(get_secret 'dd_synthetic_api_key_prod')" ./node_modules/.bin/datadog-ci sourcemaps upload ./public/static --service docs --minified-path-prefix "https://docs.datadoghq.com/static/" --release-version "${CI_COMMIT_SHORT_SHA}"
  dependencies:
    - build_live
  allow_failure: true
  only:
    - master

evaluate_missing_metrics:
  <<: *base_template
  stage: post-deploy
  cache: {}
  environment: "live"
  script:
    - chmod +x ./local/bin/py/missing_metrics.py && ./local/bin/py/missing_metrics.py -k $(get_secret 'dd-demo-api-key') -p $(get_secret 'dd-demo-app-key') -a $(get_secret 'dd_api_key') -b $(get_secret 'dd-app-key')
  only:
    - master


manage_translations:on-schedule:
  <<: *base_template
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "live"
  variables:
    URL: ${PREVIEW_DOMAIN}
    CONFIG: ${PREVIEW_CONFIG}
  script:
    - if [[ $(date +%u) -lt 6 ]]; then manage_translations "${APIKEY}"; fi
    - chmod +x ./local/bin/py/translate_metrics.py && ./local/bin/py/translate_metrics.py -k "${APIKEY}" -a $(get_secret 'dd_api_key')
  only:
    - schedules



#push_translations:
#  <<: *base_template
#  stage: post-deploy
#  environment: "live"
#  script:
#    - push_changed_translations
#  only:
#    - master
#  except:
#    - triggers
#    - schedules

test_missing_tms_live:
  <<: *base_template
  stage: post-deploy
  cache: {}
  environment: "live"
  variables:
    GIT_STRATEGY: none
  dependencies:
    - build_live
  script:
    - check_missing_tms
  only:
    - master

test_pa11y:
  <<: *base_template
  stage: post-deploy
  cache:
    <<: *cache
    policy: pull
  environment: "live"
  dependencies:
    - build_live
  script:
    - node ./local/bin/js/pa11y.js --env live --dd_api_key $(get_secret 'dd_api_key')
  only:
    - schedules

#rollback_live:
#  <<: *base_template
#  stage: cleanup
#  environment: "live"
#  when: on_failure
#  script:
#    - rollback_env
#  only:
#    - master

tag_successful_live_pipeline:
  <<: *base_template
  stage: cleanup
  cache: {}
  environment: "live"
  when: on_success
  script:
    - tag_successful_pipeline
  only:
    - master

post_success_event_to_dd_live:
  <<: *base_template
  stage: cleanup
  cache: {}
  environment: "live"
  variables:
    GIT_STRATEGY: none
  when: on_success
  script:
    - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} succeeded" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "success"
    - post_dd_metric "documentation.pipeline.completed" "1" "" "success"
  only:
    - master

post_failure_event_to_dd_live:
  <<: *base_template
  stage: cleanup
  cache: {}
  environment: "live"
  variables:
    GIT_STRATEGY: none
  when: on_failure
  script:
    - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} failed" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "error"
    - post_dd_metric "documentation.pipeline.completed" "0" "" "failure"
  only:
    - master