use docker compose v2

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ### Unreleased
 
+# v14.0.0-dev
+- before upgrading to version 14.0 or above, you _must_ run `ktd destroy` for all existing 
+local Kubetools projects.
+- this is a `dev` release because of a bug in `docker/buildx` - there is a workaround for this but we don't want to use workarounds in production (see TROUBLESHOOTING in README)
+- this **BREAKS COMPATIBILITY** with previous versions of `ktd` (see README)
+- uses `docker compose` in place of `docker-compose`, meaning we use Docker compose v2 on the backend
+
 # v13.14.0
 - Fix docker-compose conflict when kubetools commands are called without activating their venv
 - Add Python 3.12 to supported versions, albeit without Flake8 because of CPython bug

README.md

@@ -17,7 +17,7 @@ And you would like:
 
 Kubetools provides the tooling required to achieve this, by way of two CLI tools:
 
-+ **`ktd`**: generates _100% local_ development environments using Docker/docker-compose
++ **`ktd`**: generates _100% local_ development environments using docker compose
 + **`kubetools`**: deploys projects to Kubernetes, handling any changes/jobs as required
 
 Both of these use a single configuration file, `kubetools.yml`, for example a basic `django` app:
@@ -80,15 +80,17 @@ cronjobs:
 With this in your current directory, you can now:
 
 ```sh
-# Bring up a local development environment using docker-compose
+# Bring up a local development environment using docker compose
 ktd up
+```
 
+```sh
 # Deploy the project to a Kubernetes namespace
 kubetools deploy my-namespace
 ```
 
 ## Installing
-
+To install Kubetools run:
 ```sh
 pip install kubetools
 ```
@@ -137,6 +139,23 @@ minikube delete
 ...
 ```
 
+
+## Troubleshooting
+There is a bug in `buildx` which is present in Docker Engine v25.0 and up, which is yet to be patched and causes `insecure-registries` to be ignored - [issue here](https://github.com/docker/buildx/issues/2226).
+Versions of Docker Desktop which use versions of Docker Engine lower than 25.0 are unaffected.
+
+If you try and run `ktd up` with reference to an unsecure registry, e.g. `http://docker-registry.example.net` and are affected by this bug, you will get an error message that is
+similar to the following:
+```
+ERROR: failed to do request: Head "https://docker-registry.example.net/3.6_alpine3.13_v0.1-multi": dialing docker-registry.example.net:443 with direct connection: connecting to 1.1.1.1:443: dial tcp 1.1.1.1:443: connect: connection refused
+```
+
+There are a few of workarounds:
+  * Migrate to secure registries (over HTTPS)
+  * Downgrade to Docker Desktop v4.26.1 or below and prefix `http://` to each of your `insecure_registries`' URLs.
+  *  Alternatively, use the legacy builder by setting the environment variable `DOCKER_BUILDKIT=0` for `ktd` commands.
+
+
 ## Releasing (admins/maintainers only)
 * Update [CHANGELOG](CHANGELOG.md) to add new version and document it
 * In GitHub, create a new release

kubetools/dev/backends/docker_compose/config.py

@@ -248,9 +248,8 @@ def create_compose_config(kubetools_config):
     if dev_network:
         compose_config['networks'] = {
             'default': {
-                'external': {
-                    'name': 'dev',
-                },
+                'name': 'dev',
+                'external': True,
             },
         }
 

kubetools/dev/backends/docker_compose/docker_util.py

@@ -1,5 +1,3 @@
-import sys
-
 from functools import lru_cache
 
 import docker
@@ -138,8 +136,7 @@ def get_containers_status(
         if not env:
             env = compose_project.replace(docker_name, '')
 
-        # Where the name is compose-name_container_N, get container
-        name = container.name.split('_')[1]
+        name = _get_container_name_from_full_name(container.name)
 
         status = container.status == 'running'
         ports = []
@@ -194,6 +191,17 @@ def get_containers_status(
     return env_to_containers.get(kubetools_config['env'], {})
 
 
+def _get_container_name_from_full_name(full_name):
+    # if the container was made in v1, it will be composename_container_N
+    converted_name = full_name.replace('_', '-')
+
+    # we need to keep the middle part of the name
+    # for example if we have a container called `composename-container-1-N`
+    # we want to keep `container-1`
+    middle_names = converted_name.split('-')[1:-1]
+    return '-'.join(middle_name for middle_name in middle_names)
+
+
 def get_container_status(kubetools_config, name):
     containers = get_containers_status(kubetools_config, container_name=name)
     return containers.get(name)
@@ -204,8 +212,7 @@ def run_compose_process(kubetools_config, command_args, **kwargs):
     create_compose_config(kubetools_config)
 
     compose_command = [
-        # Use current interpreter to run the docker-compose module installed in the same venv
-        sys.executable, '-m', 'compose',
+        'docker', 'compose',
         # Force us to look at the current directory, not relative to the compose
         # filename (ie .kubetools/compose-name.yml).
         '--project-directory', '.',

setup.py

@@ -59,8 +59,6 @@ def get_readme_content():
             # To support CronJob api versions 'batch/v1beta1' & 'batch/v1'
             'kubernetes>=21.7.0,<25.0.0',
             'tabulate<1',
-            # compose v2 has broken container naming
-            'docker-compose<2',
         ),
         extras_require={
             'dev': (

tests/configs/complex_named_app/k8s_cronjobs.yml

@@ -0,0 +1,40 @@
+kind: CronJob
+metadata:
+  name: generic-cronjob
+  labels: {
+    kubetools/name: generic-cronjob,
+    kubetools/project_name: complex-named-app,
+    kubetools/role: cronjob
+  }
+  annotations: {
+    app.kubernetes.io/managed-by: kubetools,
+    description: 'Run: [''generic-command'']'
+  }
+spec:
+  schedule: "*/1 * * * *"
+  startingDeadlineSeconds: 10
+  concurrencyPolicy: "Allow"
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          name: generic-cronjob
+          labels: {
+            kubetools/name: generic-cronjob,
+            kubetools/project_name: complex-named-app,
+            kubetools/role: cronjob
+          }
+          annotations: {
+            app.kubernetes.io/managed-by: kubetools,
+            description: 'Run: [''generic-command'']'
+          }
+        spec:
+          containers:
+          - command: [generic-command]
+            containerContext: generic-context
+            env:
+            - {name: KUBE, value: 'true'}
+            image: generic-image
+            imagePullPolicy: 'Always'
+            name: generic-container
+          restartPolicy: OnFailure

tests/configs/complex_named_app/k8s_deployments.yml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations: {app.kubernetes.io/managed-by: kubetools}
+  labels: {kubetools/name: complex-named-app, kubetools/project_name: complex-named-app, kubetools/role: app}
+  name: complex-named-app
+spec:
+  replicas: 1
+  revisionHistoryLimit: 5
+  selector:
+    matchLabels: {kubetools/name: complex-named-app, kubetools/project_name: complex-named-app,
+      kubetools/role: app}
+  template:
+    metadata:
+      labels: {kubetools/name: complex-named-app, kubetools/project_name: complex-named-app, kubetools/role: app}
+    spec:
+      containers:
+      - command: [generic-command]
+        containerContext: generic-context
+        env:
+        - {name: KUBE, value: 'true'}
+        image: generic-image
+        imagePullPolicy: Always
+        livenessProbe:
+          httpGet: {path: /ping, port: 80}
+          timeoutSeconds: 5
+        name: complex-webserver
+        readinessProbe:
+          httpGet: {path: /ping, port: 80}
+          timeoutSeconds: 5

tests/configs/complex_named_app/k8s_jobs.yml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  annotations: {app.kubernetes.io/managed-by: kubetools, description: 'Run: [''generic-command'']'}
+  labels: {job-id: UUID, kubetools/project_name: complex-named-app,
+    kubetools/role: job}
+  name: UUID
+spec:
+  completions: 1
+  parallelism: 1
+  selector: {job-id: UUID, kubetools/project_name: complex-named-app,
+    kubetools/role: job}
+  template:
+    metadata:
+      labels: {job-id: UUID, kubetools/project_name: complex-named-app,
+        kubetools/role: job}
+    spec:
+      containers:
+      - chdir: /
+        command: [generic-command]
+        env:
+        - {name: KUBE, value: 'true'}
+        - {name: KUBE_JOB_ID, value: UUID}
+        image: generic-image
+        imagePullPolicy: Always
+        name: upgrade
+        resources:
+          requests:
+            memory: "1Gi"
+      restartPolicy: Never

tests/configs/complex_named_app/k8s_services.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations: {app.kubernetes.io/managed-by: kubetools}
+  labels: {kubetools/name: complex-named-app, kubetools/project_name: complex-named-app, kubetools/role: app}
+  name: complex-named-app
+spec:
+  ports:
+  - {port: 80, targetPort: 80}
+  selector: {kubetools/name: complex-named-app, kubetools/project_name: complex-named-app, kubetools/role: app}
+  type: NodePort

tests/configs/complex_named_app/kubetools.yml

@@ -0,0 +1,38 @@
+name: complex-named-app
+
+
+containerContexts:
+  generic-context:
+    image: generic-image
+    command: [generic-command]
+    ports:
+      - 80
+
+upgrades:
+  - name: Upgrade the database
+    containerContext: generic-context
+    command: [generic-command, generic-arg]
+    resources:
+      requests:
+        memory: "1Gi"
+
+
+deployments:
+  complex-named-app:
+    containers:
+      complex-webserver:
+        command: [uwsgi, --ini, /etc/uwsgi.conf]
+        containerContext: generic-context
+        probes:
+          timeoutSeconds: 5
+          httpGet:
+            path: /ping
+
+
+cronjobs:
+  generic-cronjob:
+    schedule: "*/1 * * * *"
+    concurrency_policy: "Allow"
+    containers:
+      generic-container:
+        containerContext: generic-context

tests/test_config_generation.py

@@ -13,6 +13,9 @@ class TestKubernetesConfigGeneration(TestCase):
     def test_basic_app_configs(self):
         _test_configs('basic_app')
 
+    def test_complex_named_app_configs(self):
+        _test_configs('complex_named_app')
+
     def test_dependencies_configs(self):
         _test_configs('dependencies')
 

tests/test_docker_util.py

@@ -0,0 +1,37 @@
+from string import ascii_lowercase
+from unittest import TestCase
+
+from kubetools.dev.backends.docker_compose.config import dockerise_label
+from kubetools.dev.backends.docker_compose.docker_util import _get_container_name_from_full_name
+
+
+def generate_long_names(number_of_separators, separator):
+    return separator.join([
+        ch for ch in ascii_lowercase[:number_of_separators+1]])
+
+
+class TestDockerComposeNameConversion(TestCase):
+    # we need to check this works for dashes and underscores
+    def test_dockerise_label_dashes(self):
+        for i in range(5):
+            long_name = generate_long_names(i, '-')
+            dockerised_name = dockerise_label(long_name)
+            self.assertEqual(dockerised_name, ascii_lowercase[:i+1])
+
+    def test_dockerise_label_underscores(self):
+        for i in range(5):
+            long_name = generate_long_names(i, '_')
+            dockerised_name = dockerise_label(long_name)
+            self.assertEqual(dockerised_name, ascii_lowercase[:i+1])
+
+    def test_container_name_from_full_name_dashes(self):
+        for i in range(5):
+            container_name = generate_long_names(i, '-')
+            full_name = '-'.join([ascii_lowercase[:i+1], container_name, '1'])
+            self.assertEqual(_get_container_name_from_full_name(full_name), container_name)
+
+    def test_container_name_from_full_name_underscores(self):
+        for i in range(5):
+            container_name = generate_long_names(i, '-')
+            full_name = '_'.join([ascii_lowercase[:i+1], container_name, '1'])
+            self.assertEqual(_get_container_name_from_full_name(full_name), container_name)