From 9c046d6f42fc6ef447ee1b9cdb9bde3e47a87431 Mon Sep 17 00:00:00 2001 From: i-chvets <113444075+i-chvets@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:32:01 -0400 Subject: [PATCH 1/3] Fix for broken requirements in track/1.7 (#277) Fix for broken requirements in notebook operators on track/1.7 Details are in: #265 Summary of changes: - Pinned SDI to 0.3.6 that has a fix. - Regenerayed requirements* - Reset not needed updates - Updated requirments - Updated workflow - Added trust for admission-webhook --- .github/workflows/integrate.yaml | 2 + .../requirements-integration.in | 10 +- .../requirements-integration.txt | 55 +++++++--- charms/jupyter-controller/requirements.txt | 12 ++- charms/jupyter-ui/requirements-integration.in | 2 +- .../jupyter-ui/requirements-integration.txt | 100 +++--------------- charms/jupyter-ui/requirements.in | 10 +- charms/jupyter-ui/requirements.txt | 76 ++++++------- tests/test_charms.py | 2 +- 9 files changed, 116 insertions(+), 153 deletions(-) diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index e9f3711a..4743bb3f 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -47,6 +47,7 @@ jobs: provider: microk8s channel: 1.24/stable charmcraft-channel: latest/candidate + juju-channel: 2.9/stable # TODO: Unpin this when this bug is resolved: https://bugs.launchpad.net/juju/+bug/1992833. # In particular, these tests failed deploying the prometheus-k8s charm where it gets an error in # the "metrics-endpoint-relation-changed" hook. @@ -78,6 +79,7 @@ jobs: provider: microk8s channel: 1.24/stable charmcraft-channel: latest/candidate + juju-channel: 2.9/stable # TODO: Unpin this when this bug is resolved: https://bugs.launchpad.net/juju/+bug/1992833. # In particular, these tests failed deploying the prometheus-k8s charm where it gets an error in # the "metrics-endpoint-relation-changed" hook. diff --git a/charms/jupyter-controller/requirements-integration.in b/charms/jupyter-controller/requirements-integration.in index 43f9d638..3e615266 100644 --- a/charms/jupyter-controller/requirements-integration.in +++ b/charms/jupyter-controller/requirements-integration.in @@ -1,5 +1,9 @@ -juju<3.1 +httpx +# Pinning to <3.0 to ensure compatibility with the 2.9 controller version +juju<3.0 +lightkube +pytest pytest-operator +pyyaml +requests tenacity -websockets --r requirements.txt diff --git a/charms/jupyter-controller/requirements-integration.txt b/charms/jupyter-controller/requirements-integration.txt index 17f95ea2..f58d44a0 100644 --- a/charms/jupyter-controller/requirements-integration.txt +++ b/charms/jupyter-controller/requirements-integration.txt @@ -2,8 +2,10 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile requirements-integration.in +# pip-compile ./requirements-integration.in # +anyio==3.7.1 + # via httpcore asttokens==2.2.1 # via stack-data attrs==22.1.0 @@ -16,6 +18,8 @@ cachetools==5.2.0 # via google-auth certifi==2022.12.7 # via + # httpcore + # httpx # kubernetes # requests cffi==1.15.1 @@ -31,13 +35,26 @@ decorator==5.1.1 # ipdb # ipython exceptiongroup==1.0.4 - # via pytest + # via + # anyio + # pytest executing==1.2.0 # via stack-data google-auth==2.15.0 # via kubernetes +h11==0.14.0 + # via httpcore +httpcore==0.17.3 + # via httpx +httpx==0.24.1 + # via + # -r ./requirements-integration.in + # lightkube idna==3.4 - # via requests + # via + # anyio + # httpx + # requests iniconfig==1.1.1 # via pytest ipdb==0.13.11 @@ -48,14 +65,18 @@ jedi==0.18.2 # via ipython jinja2==3.1.2 # via pytest-operator -juju==3.0.4 +juju==2.9.44.0 # via - # -r requirements-integration.in + # -r ./requirements-integration.in # pytest-operator jujubundlelib==0.5.7 # via theblues kubernetes==25.3.0 # via juju +lightkube==0.14.0 + # via -r ./requirements-integration.in +lightkube-models==1.27.1.4 + # via lightkube macaroonbakery==1.3.1 # via # juju @@ -68,10 +89,6 @@ mypy-extensions==0.4.3 # via typing-inspect oauthlib==3.2.2 # via requests-oauthlib -oci-image==1.0.0 - # via -r requirements.txt -ops==1.5.4 - # via -r requirements.txt packaging==22.0 # via pytest paramiko==2.12.0 @@ -116,26 +133,28 @@ pyrfc3339==1.1 # macaroonbakery pytest==7.2.0 # via + # -r ./requirements-integration.in # pytest-asyncio # pytest-operator pytest-asyncio==0.20.3 # via pytest-operator pytest-operator==0.22.0 - # via -r requirements-integration.in + # via -r ./requirements-integration.in python-dateutil==2.8.2 # via kubernetes pytz==2022.6 # via pyrfc3339 pyyaml==6.0 # via - # -r requirements.txt + # -r ./requirements-integration.in # juju # jujubundlelib # kubernetes - # ops + # lightkube # pytest-operator requests==2.28.1 # via + # -r ./requirements-integration.in # kubernetes # macaroonbakery # requests-oauthlib @@ -146,16 +165,22 @@ rsa==4.9 # via google-auth six==1.16.0 # via + # asttokens # google-auth # kubernetes # macaroonbakery # paramiko # pymacaroons # python-dateutil +sniffio==1.3.0 + # via + # anyio + # httpcore + # httpx stack-data==0.6.2 # via ipython tenacity==8.1.0 - # via -r requirements-integration.in + # via -r ./requirements-integration.in theblues==0.5.2 # via juju tomli==2.0.1 @@ -181,9 +206,7 @@ wcwidth==0.2.5 websocket-client==1.4.2 # via kubernetes websockets==7.0 - # via - # -r requirements-integration.in - # juju + # via juju # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/charms/jupyter-controller/requirements.txt b/charms/jupyter-controller/requirements.txt index 91717458..ea57606e 100644 --- a/charms/jupyter-controller/requirements.txt +++ b/charms/jupyter-controller/requirements.txt @@ -2,11 +2,13 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile requirements.in +# pip-compile ./requirements.in # oci-image==1.0.0 - # via -r requirements.in -ops==1.5.4 - # via -r requirements.in -pyyaml==6.0 + # via -r ./requirements.in +ops==2.4.1 + # via -r ./requirements.in +pyyaml==6.0.1 + # via ops +websocket-client==1.6.1 # via ops diff --git a/charms/jupyter-ui/requirements-integration.in b/charms/jupyter-ui/requirements-integration.in index d637482b..f41cdc9b 100644 --- a/charms/jupyter-ui/requirements-integration.in +++ b/charms/jupyter-ui/requirements-integration.in @@ -1,7 +1,7 @@ aiohttp jinja2 juju<3.1 +lightkube pytest-operator requests tenacity --r requirements.txt diff --git a/charms/jupyter-ui/requirements-integration.txt b/charms/jupyter-ui/requirements-integration.txt index a30bbfac..4fbaa8bd 100644 --- a/charms/jupyter-ui/requirements-integration.txt +++ b/charms/jupyter-ui/requirements-integration.txt @@ -2,25 +2,21 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking requirements-integration.in +# pip-compile ./requirements-integration.in # aiohttp==3.8.3 - # via -r requirements-integration.in + # via -r ./requirements-integration.in aiosignal==1.3.1 # via aiohttp anyio==3.6.2 - # via - # -r requirements.txt - # httpcore + # via httpcore asttokens==2.2.1 # via stack-data async-timeout==4.0.2 # via aiohttp attrs==22.1.0 # via - # -r requirements.txt # aiohttp - # jsonschema # pytest backcall==0.2.0 # via ipython @@ -30,7 +26,6 @@ cachetools==5.2.0 # via google-auth certifi==2022.12.7 # via - # -r requirements.txt # httpcore # httpx # kubernetes @@ -40,11 +35,7 @@ cffi==1.15.1 # cryptography # pynacl chardet==3.0.4 - # via - # -r requirements.txt - # requests -charmed-kubeflow-chisme==0.0.5 - # via -r requirements.txt + # via requests charset-normalizer==2.1.1 # via aiohttp cryptography==39.0.0 @@ -53,10 +44,6 @@ decorator==5.1.1 # via # ipdb # ipython -deepdiff==6.2.2 - # via - # -r requirements.txt - # charmed-kubeflow-chisme exceptiongroup==1.1.0 # via pytest executing==1.2.0 @@ -68,20 +55,13 @@ frozenlist==1.3.3 google-auth==2.15.0 # via kubernetes h11==0.14.0 - # via - # -r requirements.txt - # httpcore + # via httpcore httpcore==0.16.3 - # via - # -r requirements.txt - # httpx + # via httpx httpx==0.23.2 - # via - # -r requirements.txt - # lightkube + # via lightkube idna==2.10 # via - # -r requirements.txt # anyio # requests # rfc3986 @@ -96,38 +76,26 @@ jedi==0.18.2 # via ipython jinja2==3.1.2 # via - # -r requirements-integration.in - # -r requirements.txt - # charmed-kubeflow-chisme + # -r ./requirements-integration.in # pytest-operator -jsonschema==3.2.0 - # via - # -r requirements.txt - # serialized-data-interface juju==3.0.4 # via - # -r requirements-integration.in + # -r ./requirements-integration.in # pytest-operator jujubundlelib==0.5.7 # via theblues kubernetes==17.17.0 # via juju lightkube==0.12.0 - # via - # -r requirements.txt - # charmed-kubeflow-chisme + # via -r ./requirements-integration.in lightkube-models==1.25.4.4 - # via - # -r requirements.txt - # lightkube + # via lightkube macaroonbakery==1.3.1 # via # juju # theblues markupsafe==2.1.1 - # via - # -r requirements.txt - # jinja2 + # via jinja2 matplotlib-inline==0.1.6 # via ipython multidict==6.0.4 @@ -138,17 +106,6 @@ mypy-extensions==0.4.3 # via typing-inspect oauthlib==3.2.2 # via requests-oauthlib -oci-image==1.0.0 - # via -r requirements.txt -ops==1.5.4 - # via - # -r requirements.txt - # charmed-kubeflow-chisme - # serialized-data-interface -ordered-set==4.1.0 - # via - # -r requirements.txt - # deepdiff packaging==22.0 # via pytest paramiko==2.12.0 @@ -191,10 +148,6 @@ pyrfc3339==1.1 # via # juju # macaroonbakery -pyrsistent==0.19.2 - # via - # -r requirements.txt - # jsonschema pytest==7.2.0 # via # pytest-asyncio @@ -202,54 +155,35 @@ pytest==7.2.0 pytest-asyncio==0.20.3 # via pytest-operator pytest-operator==0.22.0 - # via -r requirements-integration.in + # via -r ./requirements-integration.in python-dateutil==2.8.2 # via kubernetes pytz==2022.7 # via pyrfc3339 pyyaml==5.4 # via - # -r requirements.txt # juju # jujubundlelib # kubernetes # lightkube - # ops # pytest-operator - # serialized-data-interface requests==2.25.0 # via - # -r requirements-integration.in - # -r requirements.txt + # -r ./requirements-integration.in # kubernetes # macaroonbakery # requests-oauthlib - # serialized-data-interface # theblues requests-oauthlib==1.3.1 # via kubernetes rfc3986[idna2008]==1.5.0 - # via - # -r requirements.txt - # httpx + # via httpx rsa==4.9 # via google-auth -ruamel-yaml==0.17.21 - # via - # -r requirements.txt - # charmed-kubeflow-chisme -ruamel-yaml-clib==0.2.7 - # via - # -r requirements.txt - # ruamel-yaml -serialized-data-interface==0.3.5 - # via -r requirements.txt six==1.16.0 # via - # -r requirements.txt # asttokens # google-auth - # jsonschema # kubernetes # macaroonbakery # paramiko @@ -257,14 +191,13 @@ six==1.16.0 # python-dateutil sniffio==1.3.0 # via - # -r requirements.txt # anyio # httpcore # httpx stack-data==0.6.2 # via ipython tenacity==8.1.0 - # via -r requirements-integration.in + # via -r ./requirements-integration.in theblues==0.5.2 # via juju tomli==2.0.1 @@ -283,7 +216,6 @@ typing-inspect==0.8.0 # via juju urllib3==1.26.13 # via - # -r requirements.txt # kubernetes # requests wcwidth==0.2.5 diff --git a/charms/jupyter-ui/requirements.in b/charms/jupyter-ui/requirements.in index b04d6daa..b242e21f 100644 --- a/charms/jupyter-ui/requirements.in +++ b/charms/jupyter-ui/requirements.in @@ -1,7 +1,7 @@ -ops -oci-image charmed-kubeflow-chisme -lightkube-models==1.25.4.4 -pyyaml +lightkube +# pin to <2 due to track/2.31 depends on <2 +ops<2 +# jupyter-ui's current implementation is not compatible +# with SDI>0.3. serialized-data-interface<0.4 -# Unittest will fail without pinning diff --git a/charms/jupyter-ui/requirements.txt b/charms/jupyter-ui/requirements.txt index 57987cb8..839ef806 100644 --- a/charms/jupyter-ui/requirements.txt +++ b/charms/jupyter-ui/requirements.txt @@ -2,82 +2,82 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking requirements.in +# pip-compile ./requirements.in # -anyio==3.6.2 +anyio==3.7.1 # via httpcore -attrs==22.1.0 +attrs==23.1.0 # via jsonschema -certifi==2022.12.7 +certifi==2023.7.22 # via # httpcore # httpx # requests -chardet==3.0.4 +charmed-kubeflow-chisme==0.0.11 + # via -r ./requirements.in +charset-normalizer==3.2.0 # via requests -charmed-kubeflow-chisme==0.0.5 - # via -r requirements.in -deepdiff==6.2.2 +deepdiff==6.2.1 # via charmed-kubeflow-chisme +exceptiongroup==1.1.2 + # via anyio h11==0.14.0 # via httpcore -httpcore==0.16.3 +httpcore==0.17.3 # via httpx -httpx==0.23.2 +httpx==0.24.1 # via lightkube -idna==2.10 +idna==3.4 # via # anyio + # httpx # requests - # rfc3986 +importlib-resources==6.0.0 + # via jsonschema jinja2==3.1.2 # via charmed-kubeflow-chisme -jsonschema==3.2.0 +jsonschema==4.17.3 # via serialized-data-interface -lightkube==0.12.0 - # via charmed-kubeflow-chisme -lightkube-models==1.25.4.4 +lightkube==0.14.0 # via - # -r requirements.in - # lightkube -markupsafe==2.1.1 + # -r ./requirements.in + # charmed-kubeflow-chisme +lightkube-models==1.27.1.4 + # via lightkube +markupsafe==2.1.3 # via jinja2 -oci-image==1.0.0 - # via -r requirements.in -ops==1.5.4 +ops==1.5.5 # via - # -r requirements.in + # -r ./requirements.in # charmed-kubeflow-chisme # serialized-data-interface ordered-set==4.1.0 # via deepdiff -pyrsistent==0.19.2 +pkgutil-resolve-name==1.3.10 + # via jsonschema +pyrsistent==0.19.3 # via jsonschema -pyyaml==5.4 +pyyaml==6.0.1 # via - # -r requirements.in # lightkube # ops # serialized-data-interface -requests==2.25.0 +requests==2.31.0 # via serialized-data-interface -rfc3986[idna2008]==1.5.0 - # via httpx -ruamel-yaml==0.17.21 +ruamel-yaml==0.17.32 # via charmed-kubeflow-chisme ruamel-yaml-clib==0.2.7 # via ruamel-yaml -serialized-data-interface==0.3.5 - # via -r requirements.in -six==1.16.0 - # via jsonschema +serialized-data-interface==0.3.6 + # via -r ./requirements.in sniffio==1.3.0 # via # anyio # httpcore # httpx -urllib3==1.26.13 +tenacity==8.2.2 + # via charmed-kubeflow-chisme +urllib3==2.0.4 # via requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +zipp==3.16.2 + # via importlib-resources diff --git a/tests/test_charms.py b/tests/test_charms.py index 8e83381d..ad702e0d 100644 --- a/tests/test_charms.py +++ b/tests/test_charms.py @@ -106,7 +106,7 @@ async def test_build_and_deploy(ops_test, lightkube_client, dummy_resources_for_ # Deploy jupyter-controller, admission-webhook, kubeflow-profiles and kubeflow-dashboard await ops_test.model.deploy(controller_charm, resources={"oci-image": controller_image_path}) - await ops_test.model.deploy("admission-webhook", channel="latest/edge") + await ops_test.model.deploy("admission-webhook", channel="latest/edge", trust=True) await ops_test.model.deploy("kubeflow-profiles", channel="latest/edge", trust=True) await ops_test.model.deploy("kubeflow-dashboard", channel="latest/edge", trust=True) await ops_test.model.add_relation("kubeflow-profiles", "kubeflow-dashboard") From 50a66876d6bd40daab95b7d934381c3375b9a7d7 Mon Sep 17 00:00:00 2001 From: i-chvets <113444075+i-chvets@users.noreply.github.com> Date: Sat, 29 Jul 2023 08:44:14 -0400 Subject: [PATCH 2/3] feat: get images script (#278) Used by CVE scanning workflows and can be used to collect images URL for airgapped setup. Script is specific for track/1.7 branch to capture all images for that branch. Branch is handled in kubeflow-ci and bundle automation using bundle file. Summary of changes: - Re-designed get images script to retrieve all images for current branch. Branch management is done outside of this script. --- tools/get-images.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 tools/get-images.sh diff --git a/tools/get-images.sh b/tools/get-images.sh new file mode 100755 index 00000000..b5df780f --- /dev/null +++ b/tools/get-images.sh @@ -0,0 +1,11 @@ +!/bin/bash +# +# This script returns list of container images that are managed by this charm and/or its workload +# +# dynamic list +IMAGE_LIST=() +IMAGE_LIST+=($(find $REPO -type f -name metadata.yaml -exec yq '.resources | to_entries | .[] | .value | ."upstream-source"' {} \;)) +IMAGE_LIST+=($(yq '.spawnerFormDefaults | .image | .options | .[]' charms/jupyter-ui/src/spawner_ui_config.yaml)) +IMAGE_LIST+=($(yq '.spawnerFormDefaults | .imageGroupOne | .options | .[]' charms/jupyter-ui/src/spawner_ui_config.yaml)) +IMAGE_LIST+=($(yq '.spawnerFormDefaults | .imageGroupTwo | .options | .[]' charms/jupyter-ui/src/spawner_ui_config.yaml)) +printf "%s\n" "${IMAGE_LIST[@]}" From a07a87e80e2e32d51e813b99e2166158a10fe537 Mon Sep 17 00:00:00 2001 From: i-chvets <113444075+i-chvets@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:15:28 -0400 Subject: [PATCH 3/3] Kf 3878 feat get images script (#279) * feat: get images script Used by CVE scanning workflows and can be used to collect images URL for airgapped setup. Script is specific for track/1.7 branch to capture all images for that branch. Branch is handled in kubeflow-ci and bundle automation using bundle file. Summary of changes: - Re-designed get images script to retrieve all images for current branch. Branch management is done outside of this script. - Fix: typo --- tools/get-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/get-images.sh b/tools/get-images.sh index b5df780f..34e14585 100755 --- a/tools/get-images.sh +++ b/tools/get-images.sh @@ -1,4 +1,4 @@ -!/bin/bash +#!/bin/bash # # This script returns list of container images that are managed by this charm and/or its workload #