From ebbef35dda92b79bb3599a4c068a78e5055d9822 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 9 Dec 2020 16:59:55 -0500 Subject: [PATCH 1/7] Support Harbor API versioning --- harbor/assets/configuration/spec.yaml | 8 ++++++++ harbor/datadog_checks/harbor/api.py | 10 ++++++++-- .../datadog_checks/harbor/data/conf.yaml.example | 8 ++++++++ harbor/datadog_checks/harbor/harbor.py | 3 ++- harbor/tests/test_unit.py | 14 ++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/harbor/assets/configuration/spec.yaml b/harbor/assets/configuration/spec.yaml index a8228e63ac9b5..3f8fde067b8e4 100644 --- a/harbor/assets/configuration/spec.yaml +++ b/harbor/assets/configuration/spec.yaml @@ -13,6 +13,14 @@ files: required: true value: type: string + - name: api_version + description: | + The version of Harbor's REST API with which to use. The default is no version (the format before version 2). + + NOTE: In a future release, the default value will become `v2.0`. If you use an older version of Harbor, + set the version to `null` or an empty string. + value: + type: string - template: instances/http overrides: username.description: | diff --git a/harbor/datadog_checks/harbor/api.py b/harbor/datadog_checks/harbor/api.py index 4cda660eb53ce..4a90d586a8a3d 100644 --- a/harbor/datadog_checks/harbor/api.py +++ b/harbor/datadog_checks/harbor/api.py @@ -17,8 +17,9 @@ class HarborAPI(object): - def __init__(self, harbor_url, http): + def __init__(self, harbor_url, http, api_version=''): self.base_url = harbor_url + self.api_version = api_version self.http = http self._fetch_and_set_harbor_version() @@ -92,4 +93,9 @@ def _make_post_request(self, url, **kwargs): return resp.json() def _resolve_url(self, url): - return url.format(base_url=self.base_url) + url = url.format(base_url=self.base_url) + if self.api_version: + api_path = '{}/api/'.format(url) + url = url.replace(api_path, api_path + self.api_version, 1) + + return url diff --git a/harbor/datadog_checks/harbor/data/conf.yaml.example b/harbor/datadog_checks/harbor/data/conf.yaml.example index 2dbd8d0461a95..ac0a0f7a6e303 100644 --- a/harbor/datadog_checks/harbor/data/conf.yaml.example +++ b/harbor/datadog_checks/harbor/data/conf.yaml.example @@ -50,6 +50,14 @@ instances: # - url: + ## @param api_version - string - optional + ## The version of Harbor's REST API with which to use. The default is no version (the format before version 2). + ## + ## NOTE: In a future release, the default value will become `v2.0`. If you use an older version of Harbor, + ## set the version to `null` or an empty string. + # + # api_version: + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/harbor/datadog_checks/harbor/harbor.py b/harbor/datadog_checks/harbor/harbor.py index bfa47f8483ea5..5472f510183eb 100644 --- a/harbor/datadog_checks/harbor/harbor.py +++ b/harbor/datadog_checks/harbor/harbor.py @@ -104,9 +104,10 @@ def _submit_read_only_status(self, api, base_tags): def check(self, instance): harbor_url = instance["url"] + api_version = instance.get('api_version', '') tags = instance.get("tags", []) try: - api = HarborAPI(harbor_url, self.http) + api = HarborAPI(harbor_url, self.http, api_version) self._check_health(api, tags) self._check_registries_health(api, tags) self._submit_project_metrics(api, tags) diff --git a/harbor/tests/test_unit.py b/harbor/tests/test_unit.py index 446a2a0f8dd6b..4269ed06ba1b4 100644 --- a/harbor/tests/test_unit.py +++ b/harbor/tests/test_unit.py @@ -6,6 +6,7 @@ from requests import HTTPError from datadog_checks.base import AgentCheck +from datadog_checks.harbor.api import HarborAPI from .common import HARBOR_COMPONENTS, HARBOR_VERSION, VERSION_1_5, VERSION_1_6, VERSION_1_8 from .conftest import MockResponse @@ -94,3 +95,16 @@ def test_api__make_post_request(harbor_api): harbor_api.http.post = MagicMock(return_value=MockResponse(None, 500)) with pytest.raises(HTTPError): harbor_api._make_post_request('{base_url}/api/path') + + +def test_api__resolve_url(mocker): + mocker.patch('datadog_checks.harbor.api.HarborAPI._fetch_and_set_harbor_version', side_effect=lambda: None) + + harbor_api = HarborAPI('url', None) + assert harbor_api._resolve_url('{base_url}/path/path') == 'url/path/path' + + harbor_api = HarborAPI('url', None) + assert harbor_api._resolve_url('{base_url}/api/path') == 'url/api/path' + + harbor_api = HarborAPI('url', None, api_version='v2.0') + assert harbor_api._resolve_url('{base_url}/api/v2.0/path') == 'url/api/v2.0/path' From 1f5feda4167280533001cd4f51df2182b7e0b272 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 9 Dec 2020 18:08:30 -0500 Subject: [PATCH 2/7] Apply suggestions from code review Co-authored-by: cswatt --- harbor/assets/configuration/spec.yaml | 2 +- harbor/datadog_checks/harbor/data/conf.yaml.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/harbor/assets/configuration/spec.yaml b/harbor/assets/configuration/spec.yaml index 3f8fde067b8e4..0d8fd15447817 100644 --- a/harbor/assets/configuration/spec.yaml +++ b/harbor/assets/configuration/spec.yaml @@ -15,7 +15,7 @@ files: type: string - name: api_version description: | - The version of Harbor's REST API with which to use. The default is no version (the format before version 2). + The version of Harbor's REST API to use. The default is no version (the format before version 2). NOTE: In a future release, the default value will become `v2.0`. If you use an older version of Harbor, set the version to `null` or an empty string. diff --git a/harbor/datadog_checks/harbor/data/conf.yaml.example b/harbor/datadog_checks/harbor/data/conf.yaml.example index ac0a0f7a6e303..64c0d0145bb1d 100644 --- a/harbor/datadog_checks/harbor/data/conf.yaml.example +++ b/harbor/datadog_checks/harbor/data/conf.yaml.example @@ -51,7 +51,7 @@ instances: - url: ## @param api_version - string - optional - ## The version of Harbor's REST API with which to use. The default is no version (the format before version 2). + ## The version of Harbor's REST API to use. The default is no version (the format before version 2). ## ## NOTE: In a future release, the default value will become `v2.0`. If you use an older version of Harbor, ## set the version to `null` or an empty string. From f2f5034cc25bcb346b7da06eb2864fb836d15d66 Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Tue, 22 Dec 2020 15:10:13 +0100 Subject: [PATCH 3/7] Add v2 env --- .../compose/harbor-2.0.5/config/core/app.conf | 6 + .../compose/harbor-2.0.5/config/core/env | 55 ++++++ .../tests/compose/harbor-2.0.5/config/db/env | 1 + .../harbor-2.0.5/config/jobservice/config.yml | 35 ++++ .../harbor-2.0.5/config/jobservice/env | 13 ++ .../harbor-2.0.5/config/nginx/nginx.conf | 169 ++++++++++++++++++ .../harbor-2.0.5/config/registry/config.yml | 33 ++++ .../harbor-2.0.5/config/registry/passwd | 1 + .../harbor-2.0.5/config/registry/root.crt | 0 .../config/registryctl/config.yml | 4 + .../harbor-2.0.5/config/registryctl/env | 2 + .../compose/harbor-2.0.5/docker-compose.yml | 155 ++++++++++++++++ harbor/tox.ini | 4 +- 13 files changed, 477 insertions(+), 1 deletion(-) create mode 100644 harbor/tests/compose/harbor-2.0.5/config/core/app.conf create mode 100644 harbor/tests/compose/harbor-2.0.5/config/core/env create mode 100644 harbor/tests/compose/harbor-2.0.5/config/db/env create mode 100644 harbor/tests/compose/harbor-2.0.5/config/jobservice/config.yml create mode 100644 harbor/tests/compose/harbor-2.0.5/config/jobservice/env create mode 100644 harbor/tests/compose/harbor-2.0.5/config/nginx/nginx.conf create mode 100644 harbor/tests/compose/harbor-2.0.5/config/registry/config.yml create mode 100644 harbor/tests/compose/harbor-2.0.5/config/registry/passwd create mode 100755 harbor/tests/compose/harbor-2.0.5/config/registry/root.crt create mode 100644 harbor/tests/compose/harbor-2.0.5/config/registryctl/config.yml create mode 100644 harbor/tests/compose/harbor-2.0.5/config/registryctl/env create mode 100644 harbor/tests/compose/harbor-2.0.5/docker-compose.yml diff --git a/harbor/tests/compose/harbor-2.0.5/config/core/app.conf b/harbor/tests/compose/harbor-2.0.5/config/core/app.conf new file mode 100644 index 0000000000000..065abfa4dbcbb --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/core/app.conf @@ -0,0 +1,6 @@ +appname = Harbor +runmode = dev +enablegzip = true + +[dev] +httpport = 8080 \ No newline at end of file diff --git a/harbor/tests/compose/harbor-2.0.5/config/core/env b/harbor/tests/compose/harbor-2.0.5/config/core/env new file mode 100644 index 0000000000000..2eb0e4460aef8 --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/core/env @@ -0,0 +1,55 @@ +CONFIG_PATH=/etc/core/app.conf +UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem +_REDIS_URL=redis:6379,100,,0,30 +SYNC_QUOTA=true +CHART_CACHE_DRIVER=redis +_REDIS_URL_REG=redis://redis:6379/1 + +PORT=8080 +LOG_LEVEL=info +EXT_ENDPOINT=https://localhost +DATABASE_TYPE=postgresql +POSTGRESQL_HOST=postgresql +POSTGRESQL_PORT=5432 +POSTGRESQL_USERNAME=postgres +POSTGRESQL_PASSWORD=root123 +POSTGRESQL_DATABASE=registry +POSTGRESQL_SSLMODE=disable +POSTGRESQL_MAX_IDLE_CONNS=50 +POSTGRESQL_MAX_OPEN_CONNS=1000 +REGISTRY_URL=http://registry:5000 +TOKEN_SERVICE_URL=http://core:8080/service/token +HARBOR_ADMIN_PASSWORD=Harbor12345 +MAX_JOB_WORKERS=10 +CORE_SECRET=tZElALNPCW6qX0MB +JOBSERVICE_SECRET=dECXi9z1STAfrg3o +WITH_NOTARY=False +WITH_CLAIR=False +WITH_TRIVY=False +CLAIR_DB_PASSWORD= +CLAIR_DB_HOST= +CLAIR_DB_PORT= +CLAIR_DB_USERNAME= +CLAIR_DB= +CLAIR_DB_SSLMODE= +CORE_URL=http://core:8080 +CORE_LOCAL_URL=http://127.0.0.1:8080 +JOBSERVICE_URL=http://jobservice:8080 +CLAIR_URL=http://clair:6060 +CLAIR_ADAPTER_URL=http://clair-adapter:8080 +TRIVY_ADAPTER_URL=http://trivy-adapter:8080 +NOTARY_URL=http://notary-server:4443 +REGISTRY_STORAGE_PROVIDER_NAME=filesystem +READ_ONLY=false +RELOAD_KEY= +CHART_REPOSITORY_URL=http://chartmuseum:9999 +REGISTRY_CONTROLLER_URL=http://registryctl:8080 +WITH_CHARTMUSEUM=False +REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user +REGISTRY_CREDENTIAL_PASSWORD=kAua2hAEDj8oZyP37Rbesg4mkBt2za0X +CSRF_KEY=v5S1yyAHGC7j115wF757m2II85cHGWm1 + +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY=core,redis,.local,trivy-adapter,registryctl,clair,notary-server,.internal,localhost,portal,postgresql,chartmuseum,log,127.0.0.1,notary-signer,clair-adapter,jobservice,db,nginx,registry + diff --git a/harbor/tests/compose/harbor-2.0.5/config/db/env b/harbor/tests/compose/harbor-2.0.5/config/db/env new file mode 100644 index 0000000000000..1f140a432a3ef --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/db/env @@ -0,0 +1 @@ +POSTGRES_PASSWORD=root123 \ No newline at end of file diff --git a/harbor/tests/compose/harbor-2.0.5/config/jobservice/config.yml b/harbor/tests/compose/harbor-2.0.5/config/jobservice/config.yml new file mode 100644 index 0000000000000..7cb08684a5473 --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/jobservice/config.yml @@ -0,0 +1,35 @@ +--- +#Protocol used to serve +protocol: "http" + +#Server listening port +port: 8080 + +#Worker pool +worker_pool: + #Worker concurrency + workers: 10 + backend: "redis" + #Additional config if use 'redis' backend + redis_pool: + #redis://[arbitrary_username:password@]ipaddress:port/database_index + redis_url: redis://redis:6379/2 + namespace: "harbor_job_service_namespace" + idle_timeout_second: 3600 +#Loggers for the running job +job_loggers: + - name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT" + level: "INFO" # INFO/DEBUG/WARNING/ERROR/FATAL + - name: "FILE" + level: "INFO" + settings: # Customized settings of logger + base_dir: "/var/log/jobs" + sweeper: + duration: 1 #days + settings: # Customized settings of sweeper + work_dir: "/var/log/jobs" + +#Loggers for the job service +loggers: + - name: "STD_OUTPUT" # Same with above + level: "INFO" \ No newline at end of file diff --git a/harbor/tests/compose/harbor-2.0.5/config/jobservice/env b/harbor/tests/compose/harbor-2.0.5/config/jobservice/env new file mode 100644 index 0000000000000..6f13a8512391a --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/jobservice/env @@ -0,0 +1,13 @@ +CORE_SECRET=tZElALNPCW6qX0MB +REGISTRY_URL=http://registry:5000 +JOBSERVICE_SECRET=dECXi9z1STAfrg3o +CORE_URL=http://core:8080 +REGISTRY_CONTROLLER_URL=http://registryctl:8080 +JOBSERVICE_WEBHOOK_JOB_MAX_RETRY=10 + + +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY=core,redis,.local,trivy-adapter,registryctl,clair,notary-server,.internal,localhost,portal,postgresql,chartmuseum,log,127.0.0.1,notary-signer,clair-adapter,jobservice,db,nginx,registry +REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user +REGISTRY_CREDENTIAL_PASSWORD=kAua2hAEDj8oZyP37Rbesg4mkBt2za0X \ No newline at end of file diff --git a/harbor/tests/compose/harbor-2.0.5/config/nginx/nginx.conf b/harbor/tests/compose/harbor-2.0.5/config/nginx/nginx.conf new file mode 100644 index 0000000000000..e357cd3539e6c --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/nginx/nginx.conf @@ -0,0 +1,169 @@ +worker_processes auto; +pid /tmp/nginx.pid; + +events { + worker_connections 1024; + use epoll; + multi_accept on; +} + +http { + client_body_temp_path /tmp/client_body_temp; + proxy_temp_path /tmp/proxy_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + tcp_nodelay on; + include /etc/nginx/conf.d/*.upstream.conf; + + # this is necessary for us to be able to disable request buffering in all cases + proxy_http_version 1.1; + + upstream core { + server core:8080; + } + + upstream portal { + server portal:8080; + } + + log_format timed_combined '$remote_addr - ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent" ' + '$request_time $upstream_response_time $pipe'; + + access_log /dev/stdout timed_combined; + + include /etc/nginx/conf.d/*.server.conf; + + server { + listen 8443 ssl; +# server_name harbordomain.com; + server_tokens off; + # SSL + ssl_certificate /etc/cert/server.crt; + ssl_certificate_key /etc/cert/server.key; + + # Recommendations from https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html + ssl_protocols TLSv1.2; + ssl_ciphers '!aNULL:kECDH+AESGCM:ECDH+AESGCM:RSA+AESGCM:kECDH+AES:ECDH+AES:RSA+AES:'; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + + # disable any limits to avoid HTTP 413 for large image uploads + client_max_body_size 0; + + # required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486) + chunked_transfer_encoding on; + + # Add extra headers + add_header Strict-Transport-Security "max-age=31536000; includeSubdomains; preload"; + add_header X-Frame-Options DENY; + add_header Content-Security-Policy "frame-ancestors 'none'"; + + # costumized location config file can place to /etc/nginx dir with prefix harbor.https. and suffix .conf + include /etc/nginx/conf.d/harbor.https.*.conf; + + location / { + proxy_pass http://portal/; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cookie_path / "/; HttpOnly; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /c/ { + proxy_pass http://core/c/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cookie_path / "/; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /api/ { + proxy_pass http://core/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cookie_path / "/; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /chartrepo/ { + proxy_pass http://core/chartrepo/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cookie_path / "/; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /v1/ { + return 404; + } + + location /v2/ { + proxy_pass http://core/v2/; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + proxy_request_buffering off; + proxy_send_timeout 900; + proxy_read_timeout 900; + } + + location /service/ { + proxy_pass http://core/service/; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cookie_path / "/; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /service/notifications { + return 404; + } + } + server { + listen 8080; + #server_name harbordomain.com; + return 308 https://$host:443$request_uri; + } +} \ No newline at end of file diff --git a/harbor/tests/compose/harbor-2.0.5/config/registry/config.yml b/harbor/tests/compose/harbor-2.0.5/config/registry/config.yml new file mode 100644 index 0000000000000..20d6a0b3c53a7 --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/registry/config.yml @@ -0,0 +1,33 @@ +version: 0.1 +log: + level: info + fields: + service: registry +storage: + cache: + layerinfo: redis + filesystem: + rootdirectory: /storage + maintenance: + uploadpurging: + enabled: false + delete: + enabled: true +redis: + addr: redis:6379 + password: + db: 1 +http: + addr: :5000 + secret: placeholder + debug: + addr: localhost:5001 +auth: + htpasswd: + realm: harbor-registry-basic-realm + path: /etc/registry/passwd +validation: + disabled: true +compatibility: + schema1: + enabled: true \ No newline at end of file diff --git a/harbor/tests/compose/harbor-2.0.5/config/registry/passwd b/harbor/tests/compose/harbor-2.0.5/config/registry/passwd new file mode 100644 index 0000000000000..73f9676f443a2 --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/registry/passwd @@ -0,0 +1 @@ +harbor_registry_user:$2y$05$VF4Cgpmp/E7IUgGZJJi6P.HOcpn6qFscAqy/3YnhH3dqhqHE3BNXW diff --git a/harbor/tests/compose/harbor-2.0.5/config/registry/root.crt b/harbor/tests/compose/harbor-2.0.5/config/registry/root.crt new file mode 100755 index 0000000000000..e69de29bb2d1d diff --git a/harbor/tests/compose/harbor-2.0.5/config/registryctl/config.yml b/harbor/tests/compose/harbor-2.0.5/config/registryctl/config.yml new file mode 100644 index 0000000000000..d0dcb7393dd12 --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/registryctl/config.yml @@ -0,0 +1,4 @@ +--- +protocol: "http" +port: 8080 +log_level: "INFO" \ No newline at end of file diff --git a/harbor/tests/compose/harbor-2.0.5/config/registryctl/env b/harbor/tests/compose/harbor-2.0.5/config/registryctl/env new file mode 100644 index 0000000000000..07233ad4efee2 --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/registryctl/env @@ -0,0 +1,2 @@ +CORE_SECRET=tZElALNPCW6qX0MB +JOBSERVICE_SECRET=dECXi9z1STAfrg3o diff --git a/harbor/tests/compose/harbor-2.0.5/docker-compose.yml b/harbor/tests/compose/harbor-2.0.5/docker-compose.yml new file mode 100644 index 0000000000000..c2092533ca5c0 --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/docker-compose.yml @@ -0,0 +1,155 @@ +version: '2.3' +services: + registry: + image: goharbor/registry-photon:v2.0.5 + container_name: registry + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + volumes: + - ./config/registry/:/etc/registry/:z + networks: + - harbor + dns_search: . + registryctl: + image: goharbor/harbor-registryctl:v2.0.5 + container_name: registryctl + env_file: + - ./config/registryctl/env + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + volumes: + - ./config/registry/:/etc/registry/:z + - type: bind + source: ./config/registryctl/config.yml + target: /etc/registryctl/config.yml + networks: + - harbor + dns_search: . + postgresql: + image: goharbor/harbor-db:v2.0.5 + container_name: harbor-db + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - DAC_OVERRIDE + - SETGID + - SETUID + networks: + harbor: + dns_search: . + env_file: + - ./config/db/env + core: + image: goharbor/harbor-core:v2.0.5 + container_name: harbor-core + env_file: + - ./config/core/env + restart: always + cap_drop: + - ALL + cap_add: + - SETGID + - SETUID + volumes: + - ./config/core/certificates/:/etc/core/certificates/:z + - type: bind + source: ./config/core/app.conf + target: /etc/core/app.conf + - type: bind + source: ../common/secretkey + target: /etc/core/key + networks: + harbor: + dns_search: . + depends_on: + - registry + - redis + - postgresql + portal: + image: goharbor/harbor-portal:v2.0.5 + container_name: harbor-portal + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + - NET_BIND_SERVICE + networks: + - harbor + dns_search: . + + jobservice: + image: goharbor/harbor-jobservice:v2.0.5 + container_name: harbor-jobservice + env_file: + - ./config/jobservice/env + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + volumes: + - type: bind + source: ./config/jobservice/config.yml + target: /etc/jobservice/config.yml + networks: + - harbor + dns_search: . + depends_on: + - core + redis: + image: goharbor/redis-photon:v2.0.5 + container_name: redis + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + networks: + harbor: + dns_search: . + proxy: + image: goharbor/nginx-photon:v2.0.5 + container_name: nginx + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + - NET_BIND_SERVICE + volumes: + - ./config/nginx:/etc/nginx:z + - ../common/cert/:/etc/cert:z + networks: + - harbor + dns_search: . + ports: + - 80:8080 + - 443:8443 + depends_on: + - registry + - core + - portal +networks: + harbor: + external: false \ No newline at end of file diff --git a/harbor/tox.ini b/harbor/tox.ini index d2c610d4e8452..a4353d74be824 100644 --- a/harbor/tox.ini +++ b/harbor/tox.ini @@ -3,7 +3,7 @@ minversion = 2.0 skip_missing_interpreters = true basepython = py38 envlist = - py{27,38}-{1.4,1.5,1.6,1.7,1.8,1.10} + py{27,38}-{1.4,1.5,1.6,1.7,1.8,1.10,2.0} [testenv] ensure_default_envdir = true @@ -32,3 +32,5 @@ setenv = 1.8: HARBOR_VERSION=1.8.0 1.10: HARBOR_VERSION=1.10.0 1.10: HARBOR_USE_SSL=1 + 2.0: HARBOR_VERSION=2.0.5 + 2.0: HARBOR_USE_SSL=1 From aac3b192057011456d533a3cc7d29e71eb85ef62 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Tue, 22 Dec 2020 12:08:04 -0500 Subject: [PATCH 4/7] update --- harbor/assets/configuration/spec.yaml | 8 ------- harbor/datadog_checks/harbor/api.py | 23 +++++++++++++++---- harbor/datadog_checks/harbor/common.py | 1 + .../harbor/data/conf.yaml.example | 8 ------- harbor/datadog_checks/harbor/harbor.py | 3 +-- harbor/tests/test_unit.py | 14 ----------- 6 files changed, 20 insertions(+), 37 deletions(-) diff --git a/harbor/assets/configuration/spec.yaml b/harbor/assets/configuration/spec.yaml index 0d8fd15447817..a8228e63ac9b5 100644 --- a/harbor/assets/configuration/spec.yaml +++ b/harbor/assets/configuration/spec.yaml @@ -13,14 +13,6 @@ files: required: true value: type: string - - name: api_version - description: | - The version of Harbor's REST API to use. The default is no version (the format before version 2). - - NOTE: In a future release, the default value will become `v2.0`. If you use an older version of Harbor, - set the version to `null` or an empty string. - value: - type: string - template: instances/http overrides: username.description: | diff --git a/harbor/datadog_checks/harbor/api.py b/harbor/datadog_checks/harbor/api.py index 4a90d586a8a3d..c33d3e15c364a 100644 --- a/harbor/datadog_checks/harbor/api.py +++ b/harbor/datadog_checks/harbor/api.py @@ -2,6 +2,7 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) from .common import ( + API_VERSION_URL, CHARTREPO_HEALTH_URL, HEALTH_URL, PING_URL, @@ -15,12 +16,14 @@ VOLUME_INFO_URL, ) +LEGACY_API_VERSION = 'v1.0' + class HarborAPI(object): - def __init__(self, harbor_url, http, api_version=''): + def __init__(self, harbor_url, http): self.base_url = harbor_url - self.api_version = api_version self.http = http + self.api_version = LEGACY_API_VERSION self._fetch_and_set_harbor_version() def chartrepo_health(self): @@ -62,6 +65,14 @@ def _fetch_and_set_harbor_version(self): self.harbor_version = [int(s) for s in version_str] self.with_chartrepo = systeminfo.get('with_chartmuseum', False) + # Only available in v2+ + try: + api_version_info = self._make_get_request(API_VERSION_URL) + except Exception: + pass + else: + self.api_version = api_version_info.get('version', LEGACY_API_VERSION) + def read_only_status(self): systeminfo = self._make_get_request(SYSTEM_INFO_URL) return systeminfo.get('read_only', None) @@ -94,8 +105,10 @@ def _make_post_request(self, url, **kwargs): def _resolve_url(self, url): url = url.format(base_url=self.base_url) - if self.api_version: - api_path = '{}/api/'.format(url) - url = url.replace(api_path, api_path + self.api_version, 1) + + if self.api_version != LEGACY_API_VERSION: + api_path = '{}/api/'.format(self.base_url) + if url.startswith(api_path): + url = url.replace(api_path, api_path + self.api_version, 1) return url diff --git a/harbor/datadog_checks/harbor/common.py b/harbor/datadog_checks/harbor/common.py index 9ec6652a89825..116d836155359 100644 --- a/harbor/datadog_checks/harbor/common.py +++ b/harbor/datadog_checks/harbor/common.py @@ -2,6 +2,7 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) SYSTEM_INFO_URL = "{base_url}/api/systeminfo/" +API_VERSION_URL = "{base_url}/api/version" LOGIN_URL = "{base_url}/c/login/" LOGIN_PRE_1_7_URL = "{base_url}/login/" HEALTH_URL = "{base_url}/api/health/" diff --git a/harbor/datadog_checks/harbor/data/conf.yaml.example b/harbor/datadog_checks/harbor/data/conf.yaml.example index 64c0d0145bb1d..2dbd8d0461a95 100644 --- a/harbor/datadog_checks/harbor/data/conf.yaml.example +++ b/harbor/datadog_checks/harbor/data/conf.yaml.example @@ -50,14 +50,6 @@ instances: # - url: - ## @param api_version - string - optional - ## The version of Harbor's REST API to use. The default is no version (the format before version 2). - ## - ## NOTE: In a future release, the default value will become `v2.0`. If you use an older version of Harbor, - ## set the version to `null` or an empty string. - # - # api_version: - ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/harbor/datadog_checks/harbor/harbor.py b/harbor/datadog_checks/harbor/harbor.py index 5472f510183eb..bfa47f8483ea5 100644 --- a/harbor/datadog_checks/harbor/harbor.py +++ b/harbor/datadog_checks/harbor/harbor.py @@ -104,10 +104,9 @@ def _submit_read_only_status(self, api, base_tags): def check(self, instance): harbor_url = instance["url"] - api_version = instance.get('api_version', '') tags = instance.get("tags", []) try: - api = HarborAPI(harbor_url, self.http, api_version) + api = HarborAPI(harbor_url, self.http) self._check_health(api, tags) self._check_registries_health(api, tags) self._submit_project_metrics(api, tags) diff --git a/harbor/tests/test_unit.py b/harbor/tests/test_unit.py index 4269ed06ba1b4..446a2a0f8dd6b 100644 --- a/harbor/tests/test_unit.py +++ b/harbor/tests/test_unit.py @@ -6,7 +6,6 @@ from requests import HTTPError from datadog_checks.base import AgentCheck -from datadog_checks.harbor.api import HarborAPI from .common import HARBOR_COMPONENTS, HARBOR_VERSION, VERSION_1_5, VERSION_1_6, VERSION_1_8 from .conftest import MockResponse @@ -95,16 +94,3 @@ def test_api__make_post_request(harbor_api): harbor_api.http.post = MagicMock(return_value=MockResponse(None, 500)) with pytest.raises(HTTPError): harbor_api._make_post_request('{base_url}/api/path') - - -def test_api__resolve_url(mocker): - mocker.patch('datadog_checks.harbor.api.HarborAPI._fetch_and_set_harbor_version', side_effect=lambda: None) - - harbor_api = HarborAPI('url', None) - assert harbor_api._resolve_url('{base_url}/path/path') == 'url/path/path' - - harbor_api = HarborAPI('url', None) - assert harbor_api._resolve_url('{base_url}/api/path') == 'url/api/path' - - harbor_api = HarborAPI('url', None, api_version='v2.0') - assert harbor_api._resolve_url('{base_url}/api/v2.0/path') == 'url/api/v2.0/path' From 4c764f11027db5f5417a5cb81e5d3cc80caeae98 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Tue, 22 Dec 2020 14:28:09 -0500 Subject: [PATCH 5/7] fix --- harbor/datadog_checks/harbor/api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/harbor/datadog_checks/harbor/api.py b/harbor/datadog_checks/harbor/api.py index c33d3e15c364a..effcc72fa566d 100644 --- a/harbor/datadog_checks/harbor/api.py +++ b/harbor/datadog_checks/harbor/api.py @@ -60,11 +60,6 @@ def volume_info(self): return self._make_get_request(VOLUME_INFO_URL) def _fetch_and_set_harbor_version(self): - systeminfo = self._make_get_request(SYSTEM_INFO_URL) - version_str = systeminfo['harbor_version'].split('-')[0].lstrip('v').split('.')[:3] - self.harbor_version = [int(s) for s in version_str] - self.with_chartrepo = systeminfo.get('with_chartmuseum', False) - # Only available in v2+ try: api_version_info = self._make_get_request(API_VERSION_URL) @@ -73,6 +68,11 @@ def _fetch_and_set_harbor_version(self): else: self.api_version = api_version_info.get('version', LEGACY_API_VERSION) + systeminfo = self._make_get_request(SYSTEM_INFO_URL) + version_str = systeminfo['harbor_version'].split('-')[0].lstrip('v').split('.')[:3] + self.harbor_version = [int(s) for s in version_str] + self.with_chartrepo = systeminfo.get('with_chartmuseum', False) + def read_only_status(self): systeminfo = self._make_get_request(SYSTEM_INFO_URL) return systeminfo.get('read_only', None) @@ -109,6 +109,6 @@ def _resolve_url(self, url): if self.api_version != LEGACY_API_VERSION: api_path = '{}/api/'.format(self.base_url) if url.startswith(api_path): - url = url.replace(api_path, api_path + self.api_version, 1) + url = url.replace(api_path, '{}{}/'.format(api_path, self.api_version), 1) return url From d910f1a3dad9ce913326e48415f8a6990cb7b712 Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Tue, 29 Dec 2020 11:16:27 +0100 Subject: [PATCH 6/7] Tests for v2 --- harbor/datadog_checks/harbor/harbor.py | 6 +-- .../harbor-2.0.5/config/chartserver/env | 38 +++++++++++++++++++ .../compose/harbor-2.0.5/config/core/env | 12 +++--- .../harbor-2.0.5/config/jobservice/env | 8 ++-- .../harbor-2.0.5/config/registry/passwd | 2 +- .../harbor-2.0.5/config/registryctl/env | 4 +- .../compose/harbor-2.0.5/docker-compose.yml | 24 ++++++++++++ harbor/tox.ini | 4 +- 8 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 harbor/tests/compose/harbor-2.0.5/config/chartserver/env diff --git a/harbor/datadog_checks/harbor/harbor.py b/harbor/datadog_checks/harbor/harbor.py index bfa47f8483ea5..6ff9ff2d0ba1e 100644 --- a/harbor/datadog_checks/harbor/harbor.py +++ b/harbor/datadog_checks/harbor/harbor.py @@ -30,7 +30,7 @@ def _check_health(self, api, base_tags): try: chartrepo_health = api.chartrepo_health()[HEALTHY] except HTTPError as e: - if e.response.status_code == 403: + if e.response.status_code in (401, 403): self.log.info( "Provided user in harbor integration config is not an admin user. Ignoring chartrepo health" ) @@ -52,7 +52,7 @@ def _check_registries_health(self, api, base_tags): registries = api.registries() self.log.debug("Found %d registries", len(registries)) except HTTPError as e: - if e.response.status_code == 403: + if e.response.status_code in (401, 403): # Forbidden, user is not admin self.log.info( "Provided user in harbor integration config is not an admin user. Ignoring registries health checks" @@ -84,7 +84,7 @@ def _submit_disk_metrics(self, api, base_tags): try: volume_info = api.volume_info() except HTTPError as e: - if e.response.status_code == 403: + if e.response.status_code in (401, 403): # Forbidden, user is not admin self.log.warning( "Provided user in harbor integration config is not an admin user. Ignoring volume metrics" diff --git a/harbor/tests/compose/harbor-2.0.5/config/chartserver/env b/harbor/tests/compose/harbor-2.0.5/config/chartserver/env new file mode 100644 index 0000000000000..1e36136d624ef --- /dev/null +++ b/harbor/tests/compose/harbor-2.0.5/config/chartserver/env @@ -0,0 +1,38 @@ +## Settings should be set +PORT=9999 + +# Only support redis now. If redis is setup, then enable cache +CACHE=redis +CACHE_REDIS_ADDR=redis:6379 +CACHE_REDIS_PASSWORD= +CACHE_REDIS_DB=3 + +# Credential for internal communication +BASIC_AUTH_USER=chart_controller +BASIC_AUTH_PASS=i2DsDOygHVG2pgO3 + +# Multiple tenants +# Must be set with 1 to support project namespace +DEPTH=1 + +# Backend storage driver: e.g. "local", "amazon", "google" etc. +STORAGE=local + +# Storage driver settings +STORAGE_LOCAL_ROOTDIR=/chart_storage + +## Settings with default values. Just put here for future changes +DEBUG=false +LOG_JSON=true +DISABLE_METRICS=false +DISABLE_API=false +DISABLE_STATEFILES=false +ALLOW_OVERWRITE=true +CHART_URL= +AUTH_ANONYMOUS_GET=false +CONTEXT_PATH= +INDEX_LIMIT=0 +MAX_STORAGE_OBJECTS=0 +MAX_UPLOAD_SIZE=20971520 +CHART_POST_FORM_FIELD_NAME=chart +PROV_POST_FORM_FIELD_NAME=prov diff --git a/harbor/tests/compose/harbor-2.0.5/config/core/env b/harbor/tests/compose/harbor-2.0.5/config/core/env index 2eb0e4460aef8..b3e6324aad26a 100644 --- a/harbor/tests/compose/harbor-2.0.5/config/core/env +++ b/harbor/tests/compose/harbor-2.0.5/config/core/env @@ -21,8 +21,8 @@ REGISTRY_URL=http://registry:5000 TOKEN_SERVICE_URL=http://core:8080/service/token HARBOR_ADMIN_PASSWORD=Harbor12345 MAX_JOB_WORKERS=10 -CORE_SECRET=tZElALNPCW6qX0MB -JOBSERVICE_SECRET=dECXi9z1STAfrg3o +CORE_SECRET=i2DsDOygHVG2pgO3 +JOBSERVICE_SECRET=OAyMVs6vzmKfYe9H WITH_NOTARY=False WITH_CLAIR=False WITH_TRIVY=False @@ -44,12 +44,12 @@ READ_ONLY=false RELOAD_KEY= CHART_REPOSITORY_URL=http://chartmuseum:9999 REGISTRY_CONTROLLER_URL=http://registryctl:8080 -WITH_CHARTMUSEUM=False +WITH_CHARTMUSEUM=True REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user -REGISTRY_CREDENTIAL_PASSWORD=kAua2hAEDj8oZyP37Rbesg4mkBt2za0X -CSRF_KEY=v5S1yyAHGC7j115wF757m2II85cHGWm1 +REGISTRY_CREDENTIAL_PASSWORD=jjLARmMfS3KeidxdfsVwyvwIo8d5N92A +CSRF_KEY=IYqWGHvLCLbe41QbowKldiy0B17TCX2R HTTP_PROXY= HTTPS_PROXY= -NO_PROXY=core,redis,.local,trivy-adapter,registryctl,clair,notary-server,.internal,localhost,portal,postgresql,chartmuseum,log,127.0.0.1,notary-signer,clair-adapter,jobservice,db,nginx,registry +NO_PROXY=portal,registryctl,core,registry,trivy-adapter,log,db,localhost,chartmuseum,notary-server,clair,redis,notary-signer,postgresql,nginx,.internal,jobservice,127.0.0.1,.local,clair-adapter diff --git a/harbor/tests/compose/harbor-2.0.5/config/jobservice/env b/harbor/tests/compose/harbor-2.0.5/config/jobservice/env index 6f13a8512391a..a3770f380a47a 100644 --- a/harbor/tests/compose/harbor-2.0.5/config/jobservice/env +++ b/harbor/tests/compose/harbor-2.0.5/config/jobservice/env @@ -1,6 +1,6 @@ -CORE_SECRET=tZElALNPCW6qX0MB +CORE_SECRET=i2DsDOygHVG2pgO3 REGISTRY_URL=http://registry:5000 -JOBSERVICE_SECRET=dECXi9z1STAfrg3o +JOBSERVICE_SECRET=OAyMVs6vzmKfYe9H CORE_URL=http://core:8080 REGISTRY_CONTROLLER_URL=http://registryctl:8080 JOBSERVICE_WEBHOOK_JOB_MAX_RETRY=10 @@ -8,6 +8,6 @@ JOBSERVICE_WEBHOOK_JOB_MAX_RETRY=10 HTTP_PROXY= HTTPS_PROXY= -NO_PROXY=core,redis,.local,trivy-adapter,registryctl,clair,notary-server,.internal,localhost,portal,postgresql,chartmuseum,log,127.0.0.1,notary-signer,clair-adapter,jobservice,db,nginx,registry +NO_PROXY=portal,registryctl,core,registry,trivy-adapter,log,db,localhost,chartmuseum,notary-server,clair,redis,notary-signer,postgresql,nginx,.internal,jobservice,127.0.0.1,.local,clair-adapter REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user -REGISTRY_CREDENTIAL_PASSWORD=kAua2hAEDj8oZyP37Rbesg4mkBt2za0X \ No newline at end of file +REGISTRY_CREDENTIAL_PASSWORD=jjLARmMfS3KeidxdfsVwyvwIo8d5N92A \ No newline at end of file diff --git a/harbor/tests/compose/harbor-2.0.5/config/registry/passwd b/harbor/tests/compose/harbor-2.0.5/config/registry/passwd index 73f9676f443a2..314eb22e77db0 100644 --- a/harbor/tests/compose/harbor-2.0.5/config/registry/passwd +++ b/harbor/tests/compose/harbor-2.0.5/config/registry/passwd @@ -1 +1 @@ -harbor_registry_user:$2y$05$VF4Cgpmp/E7IUgGZJJi6P.HOcpn6qFscAqy/3YnhH3dqhqHE3BNXW +harbor_registry_user:$2y$05$H.jWeiWuqQBLap3wXuBENedMSIQmS7dfEYg8dd5D7DaeKktesWx1C diff --git a/harbor/tests/compose/harbor-2.0.5/config/registryctl/env b/harbor/tests/compose/harbor-2.0.5/config/registryctl/env index 07233ad4efee2..e3aaf88662485 100644 --- a/harbor/tests/compose/harbor-2.0.5/config/registryctl/env +++ b/harbor/tests/compose/harbor-2.0.5/config/registryctl/env @@ -1,2 +1,2 @@ -CORE_SECRET=tZElALNPCW6qX0MB -JOBSERVICE_SECRET=dECXi9z1STAfrg3o +CORE_SECRET=i2DsDOygHVG2pgO3 +JOBSERVICE_SECRET=OAyMVs6vzmKfYe9H diff --git a/harbor/tests/compose/harbor-2.0.5/docker-compose.yml b/harbor/tests/compose/harbor-2.0.5/docker-compose.yml index c2092533ca5c0..a986bbc0f0405 100644 --- a/harbor/tests/compose/harbor-2.0.5/docker-compose.yml +++ b/harbor/tests/compose/harbor-2.0.5/docker-compose.yml @@ -72,6 +72,9 @@ services: target: /etc/core/key networks: harbor: + harbor-chartmuseum: + aliases: + - harbor-core dns_search: . depends_on: - registry @@ -125,6 +128,9 @@ services: - SETUID networks: harbor: + harbor-chartmuseum: + aliases: + - redis dns_search: . proxy: image: goharbor/nginx-photon:v2.0.5 @@ -150,6 +156,24 @@ services: - registry - core - portal + chartmuseum: + container_name: chartmuseum + image: goharbor/chartmuseum-photon:v2.0.5 + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - DAC_OVERRIDE + - SETGID + - SETUID + networks: + - harbor-chartmuseum + dns_search: . + env_file: + ./config/chartserver/env networks: harbor: + external: false + harbor-chartmuseum: external: false \ No newline at end of file diff --git a/harbor/tox.ini b/harbor/tox.ini index a4353d74be824..53cb545430285 100644 --- a/harbor/tox.ini +++ b/harbor/tox.ini @@ -3,7 +3,7 @@ minversion = 2.0 skip_missing_interpreters = true basepython = py38 envlist = - py{27,38}-{1.4,1.5,1.6,1.7,1.8,1.10,2.0} + py{27,38}-{1.6,1.7,1.8,1.10,2.0} [testenv] ensure_default_envdir = true @@ -11,7 +11,7 @@ envdir = py27: {toxworkdir}/py27 py38: {toxworkdir}/py38 description = - py{27,38}: e2e ready + py{27,38}-{1.6,1.10,2.0}: e2e ready dd_check_style = true usedevelop = true platform = linux|darwin|win32 From e8d2fd1b92e1361954dd396cada6c2640d45465d Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Tue, 29 Dec 2020 17:05:24 +0100 Subject: [PATCH 7/7] Address review --- harbor/tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/harbor/tox.ini b/harbor/tox.ini index 53cb545430285..e6b2635053701 100644 --- a/harbor/tox.ini +++ b/harbor/tox.ini @@ -25,8 +25,6 @@ commands = pip install -r requirements.in pytest -v {posargs} setenv = - 1.4: HARBOR_VERSION=1.4.0 - 1.5: HARBOR_VERSION=1.5.0 1.6: HARBOR_VERSION=1.6.0 1.7: HARBOR_VERSION=1.7.0 1.8: HARBOR_VERSION=1.8.0