diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 837666a21..000000000 --- a/.drone.yml +++ /dev/null @@ -1,431 +0,0 @@ ---- -########################################### -##### k3d CLI/binary release pipeline ##### -########################################### - -kind: pipeline -type: docker -name: main - -platform: - os: linux - arch: amd64 - -steps: - - - name: lint - image: golang:1.17 - commands: - - make ci-setup - - make check-fmt lint - when: - event: - - push - - pull_request - - tag - - - name: test - image: docker:20.10 - volumes: - - name: dockersock - path: /var/run - commands: - - apk add git bash curl sudo jq make - - sleep 5 # give docker enough time to start - - make e2e - when: - event: - - push - - pull_request - - tag - - - name: build - image: golang:1.17 - environment: - GIT_TAG: "${DRONE_TAG}" - commands: - - make ci-setup - - make build-cross - depends_on: - - lint - - test - when: - event: - - push - - tag - - - name: pre-release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - files: - - _dist/* - checksum: - - sha256 - prerelease: true - depends_on: - - lint - - test - - build - when: - event: - - tag - ref: - include: - # include only pre-release tags - - "refs/tags/*rc*" - - "refs/tags/*beta*" - - "refs/tags/*alpha*" - - "refs/tags/*test*" - - "refs/tags/*dev*" - - - name: release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - files: - - _dist/* - checksum: - - sha256 - depends_on: - - lint - - test - - build - when: - event: - - tag - ref: - exclude: - # exclude pre-release tags - - "refs/tags/*rc*" - - "refs/tags/*beta*" - - "refs/tags/*alpha*" - - "refs/tags/*test*" - - "refs/tags/*dev*" - -services: - # Starting the docker service to be used by dind - - name: docker - image: docker:20.10-dind - privileged: true - volumes: - - name: dockersock - path: /var/run - -volumes: - - name: dockersock - temp: {} - - ---- -########################### -###### Docker Images ###### -########################### -# -# +++ Docker Images +++ -# Tagged using the auto_tag feature of the docker plugin -# See http://plugins.drone.io/drone-plugins/drone-docker/#autotag -# > if event type is `tag` -# > > 1.0.0 produces docker tags 1, 1.0, 1.0.0 -# > > 1.0.0-rc.1 produces docker tags 1.0.0-rc.1 -# > if event type is `push` and target branch == default branch (main) -# > > tag `latest` - - -################################ -##### Docker Images: amd64 ##### -################################ - -kind: pipeline -type: docker -name: linux_amd64 - -platform: - os: linux - arch: amd64 - -steps: - - - name: build_push_binary - environment: - DOCKER_BUILDKIT: "1" - image: plugins/docker - settings: - repo: rancher/k3d - auto_tag: true - auto_tag_suffix: linux-amd64 - dockerfile: Dockerfile - target: binary-only - context: . - username: - from_secret: docker_username - password: - from_secret: docker_password - build_args: - - GIT_TAG_OVERRIDE=${DRONE_TAG} - - - name: build_push_dind - image: plugins/docker - environment: - DOCKER_BUILDKIT: "1" - settings: - repo: rancher/k3d - auto_tag: true - auto_tag_suffix: dind-linux-amd64 - dockerfile: Dockerfile - target: dind - context: . - username: - from_secret: docker_username - password: - from_secret: docker_password - build_args: - - GIT_TAG_OVERRIDE=${DRONE_TAG} - - ARCH=amd64 - - - name: build_push_proxy - image: plugins/docker - settings: - repo: rancher/k3d-proxy - auto_tag: true - auto_tag_suffix: linux-amd64 - dockerfile: proxy/Dockerfile - context: proxy/ - username: - from_secret: docker_username - password: - from_secret: docker_password - - - name: build_push_tools - image: plugins/docker - settings: - repo: rancher/k3d-tools - auto_tag: true - auto_tag_suffix: linux-amd64 - dockerfile: tools/Dockerfile - context: tools/ - username: - from_secret: docker_username - password: - from_secret: docker_password - -trigger: - event: - - tag # see note at the start of the "Docker Images" section: creates SemVer tagged images using the `auto_tag` option of the docker plugin - - push # `auto_tag` option only creates the `latest` tag if target branch is default branch (i.e. `main`) - -depends_on: - - main - ---- - -################################ -##### Docker Images: arm ##### -################################ - -kind: pipeline -type: docker -name: linux_arm - -platform: - os: linux - arch: arm - -steps: - - - name: build_push_proxy - image: plugins/docker - settings: - repo: rancher/k3d-proxy - auto_tag: true - auto_tag_suffix: linux-arm - dockerfile: proxy/Dockerfile - context: proxy/ - username: - from_secret: docker_username - password: - from_secret: docker_password - build_args: - - ARCH=arm - - - name: build_push_tools - image: plugins/docker - settings: - repo: rancher/k3d-tools - auto_tag: true - auto_tag_suffix: linux-arm - dockerfile: tools/Dockerfile - context: tools/ - username: - from_secret: docker_username - password: - from_secret: docker_password - -trigger: - event: - - tag # see note at the start of the "Docker Images" section: creates SemVer tagged images using the `auto_tag` option of the docker plugin - - push # `auto_tag` option only creates the `latest` tag if target branch is default branch (i.e. `main`) - -depends_on: - - main - ---- - -################################ -##### Docker Images: arm64 ##### -################################ - -kind: pipeline -type: docker -name: linux_arm64 - -platform: - os: linux - arch: arm64 - -steps: - - - name: build_push_binary - environment: - DOCKER_BUILDKIT: "1" - image: plugins/docker - settings: - repo: rancher/k3d - auto_tag: true - auto_tag_suffix: linux-arm64 - dockerfile: Dockerfile - target: binary-only - context: . - username: - from_secret: docker_username - password: - from_secret: docker_password - build_args: - - GIT_TAG_OVERRIDE=${DRONE_TAG} - - - name: build_push_dind - image: plugins/docker - environment: - DOCKER_BUILDKIT: "1" - settings: - repo: rancher/k3d - auto_tag: true - auto_tag_suffix: dind-linux-arm64 - dockerfile: Dockerfile - target: dind - context: . - username: - from_secret: docker_username - password: - from_secret: docker_password - build_args: - - GIT_TAG_OVERRIDE=${DRONE_TAG} - - ARCH=arm64 - - - name: build_push_proxy - image: plugins/docker - settings: - repo: rancher/k3d-proxy - auto_tag: true - auto_tag_suffix: linux-arm64 - dockerfile: proxy/Dockerfile - context: proxy/ - username: - from_secret: docker_username - password: - from_secret: docker_password - build_args: - - ARCH=arm64 - - - name: build_push_tools - image: plugins/docker - settings: - repo: rancher/k3d-tools - auto_tag: true - auto_tag_suffix: linux-arm64 - dockerfile: tools/Dockerfile - context: tools/ - username: - from_secret: docker_username - password: - from_secret: docker_password - -trigger: - event: - - tag # see note at the start of the "Docker Images" section: creates SemVer tagged images using the `auto_tag` option of the docker plugin - - push # `auto_tag` option only creates the `latest` tag if target branch is default branch (i.e. `main`) - -depends_on: - - main - ---- - -############################## -###### Docker Manifests ###### -############################## -kind: pipeline -type: docker -name: manifests - -platform: - os: linux - arch: amd64 - -steps: - - name: push_manifest_binary - image: plugins/manifest - settings: - username: - from_secret: docker_username - password: - from_secret: docker_password - spec: manifest.tmpl - auto_tag: true - ignore_missing: true # expected, as we dropped arm due to missing base image for that arch - - - name: push_manifest_dind - image: plugins/manifest - settings: - username: - from_secret: docker_username - password: - from_secret: docker_password - spec: dind-manifest.tmpl - auto_tag: true - ignore_missing: true # expected, as we dropped arm due to missing base image for that arch - - - name: push_manifest_proxy - image: plugins/manifest - settings: - username: - from_secret: docker_username - password: - from_secret: docker_password - spec: proxy/manifest.tmpl - auto_tag: true - ignore_missing: false - - - name: push_manifest_tools - image: plugins/manifest - settings: - username: - from_secret: docker_username - password: - from_secret: docker_password - spec: tools/manifest.tmpl - auto_tag: true - ignore_missing: false - -trigger: - event: - - tag # see note at the start of the "Docker Images" section: creates SemVer tagged images using the `auto_tag` option of the manifest plugin - - push # `auto_tag` option only creates the `latest` tag if target branch is default branch (i.e. `main`) - -depends_on: - - main - - linux_amd64 - - linux_arm - - linux_arm64 - diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index adde81f28..607d1f0f9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,27 +1,187 @@ -name: Release +name: Test & Release -on: workflow_dispatch +on: + - push + - workflow_dispatch + +env: + IMAGE_REGISTRY: ghcr.io + IMAGE_BASE_REPO: k3d-io + IMAGE_PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 + GO_VERSION: "1.17.x" + DOCKER_VERSION: "20.10" jobs: - release: + test-suite: + name: Full Test Suite runs-on: ubuntu-20.04 steps: + # Setup - uses: actions/checkout@v2 - name: Setup Go environment uses: actions/setup-go@v2 with: - go-version: "1.17.x" + go-version: "${{ env.GO_VERSION }}" - name: Setup Docker uses: docker-practice/actions-setup-docker@master with: - docker_version: "20.10" + docker_version: "${{ env.DOCKER_VERSION }}" - name: Setup CI Tools run: make ci-setup - - name: lint + # Code Check + - name: Run Static Analysis run: make lint - - name: test + # Tests + - name: Run Go Tests run: make test - - name: e2e + - name: Run E2E Tests + timeout-minutes: 20 run: make e2e - - name: build + # Builds + - name: Test Platform Builds + run: make build-cross + - name: Test Helper Image Builds + run: make build-helper-images + + release: + name: Build & Release + # Only run on tags + runs-on: ubuntu-20.04 + steps: + # Setup + - uses: actions/checkout@v2 + - name: Setup Go environment + uses: actions/setup-go@v2 + with: + go-version: "${{ env.GO_VERSION }}" + - name: Setup CI Tools + run: make ci-setup + # Go Build + - name: Build k3d Binary run: make build-cross + # Container Image Setup + - name: Setup Docker + uses: docker-practice/actions-setup-docker@master + with: + docker_version: "${{ env.DOCKER_VERSION }}" + - name: Log in to the Container registry + uses: docker/login-action@v1 + with: + registry: ${{ env.IMAGE_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + # Gather Docker Metadata + - name: Docker Metadata k3d-binary + id: meta-k3d-binary + env: + IMAGE_ID: k3d + uses: docker/metadata-action@v3 + with: + images: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_BASE_REPO }}/${{ env.IMAGE_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + bake-target: docker-metadata-${{ env.IMAGE_ID }} + tags: | + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=pr + type=sha + - name: Docker Metadata k3d-dind + id: meta-k3d-dind + env: + IMAGE_ID: k3d + IMAGE_SUFFIX: "-dind" + uses: docker/metadata-action@v3 + with: + images: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_BASE_REPO }}/${{ env.IMAGE_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + bake-target: docker-metadata-${{ env.IMAGE_ID }}${{ env.IMAGE_SUFFIX }} + tags: | + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=pr + type=sha + flavor: | + suffix=${{ env.IMAGE_SUFFIX }} + - name: Docker Metadata k3d-proxy + id: meta-k3d-proxy + env: + IMAGE_ID: k3d-proxy + uses: docker/metadata-action@v3 + with: + images: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_BASE_REPO }}/${{ env.IMAGE_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + bake-target: docker-metadata-${{ env.IMAGE_ID }} + tags: | + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=pr + type=sha + - name: Docker Metadata k3d-tools + id: meta-k3d-tools + env: + IMAGE_ID: k3d-tools + uses: docker/metadata-action@v3 + with: + images: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_BASE_REPO }}/${{ env.IMAGE_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + bake-target: docker-metadata-${{ env.IMAGE_ID }} + tags: | + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=pr + type=sha + - name: Merge Metadata Bake Definitions + run: | + INPUT=(${{ steps.meta-k3d-binary.outputs.bake-file }} ${{ steps.meta-k3d-dind.outputs.bake-file }} ${{ steps.meta-k3d-proxy.outputs.bake-file }} ${{ steps.meta-k3d-tools.outputs.bake-file }}) + OUT_FILE=./bake-metadata.json + OUT_FILE_TMP=./bake-metadata-tmp.json + + cat << EOF > $OUT_FILE + { + "target": {} + } + EOF + + for file in "${INPUT[@]}"; do + cat $OUT_FILE > $OUT_FILE_TMP + jq -s '.[0] * .[1]' $OUT_FILE_TMP $file > $OUT_FILE + done + + rm "$OUT_FILE_TMP" + # Build and Push container images + - name: Build Images + uses: docker/bake-action@v1.7.0 + with: + files: | + ./docker-bake.hcl + ./bake-metadata.json + targets: release + push: false + - name: Wait for tests to succeed + uses: lewagon/wait-on-check-action@v1.1.1 + with: + ref: ${{ github.ref }} + check-name: "Full Test Suite" + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 20 + - name: Push Images + if: startsWith(github.ref, 'refs/tags/') + uses: docker/bake-action@v1.7.0 + with: + files: | + ./docker-bake.hcl + ./bake-metadata.json + targets: release + push: true diff --git a/Dockerfile b/Dockerfile index 00a10b9d8..79238b47a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,22 +16,23 @@ RUN make build -e GIT_TAG_OVERRIDE=${GIT_TAG_OVERRIDE} && bin/k3d version # -> used e.g. in our CI pipelines for testing # ####################################################### FROM docker:$DOCKER_VERSION-dind as dind -ARG OS=linux -ARG ARCH=amd64 +ARG OS +ARG ARCH + +ENV OS=${OS} +ENV ARCH=${ARCH} + +# Helper script to install some tooling +COPY scripts/install-tools.sh /scripts/install-tools.sh # install some basic packages needed for testing, etc. -RUN docker version; \ - echo ">>> building for ${OS}/${ARCH}" && \ - apk update && \ +RUN apk update && \ apk add bash curl sudo jq git make netcat-openbsd # install kubectl to interact with the k3d cluster -RUN curl -L https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/${OS}/${ARCH}/kubectl -o /usr/local/bin/kubectl && \ - chmod +x /usr/local/bin/kubectl - # install yq (yaml processor) from source, as the busybox yq had some issues -RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_${OS}_${ARCH} -o /usr/bin/yq &&\ - chmod +x /usr/bin/yq +RUN /scripts/install-tools.sh kubectl yq + COPY --from=builder /app/bin/k3d /bin/k3d ######################################### diff --git a/Makefile b/Makefile index ee26cd578..586653c06 100644 --- a/Makefile +++ b/Makefile @@ -30,11 +30,11 @@ endif K3D_IMAGE_TAG := $(GIT_TAG:v%=%) # get latest k3s version: grep the tag and replace + with - (difference between git and dockerhub tags) -K3S_TAG := $(shell curl --silent "https://update.k3s.io/v1-release/channels/stable" | egrep -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/') +K3S_TAG := $(shell curl --silent --retry 3 "https://update.k3s.io/v1-release/channels/stable" | egrep -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/') ifeq ($(K3S_TAG),) $(warning K3S_TAG undefined: couldn't get latest k3s image tag!) -$(warning Output of curl: $(shell curl --silent "https://update.k3s.io/v1-release/channels/stable")) +$(warning Output of curl: $(shell curl "https://update.k3s.io/v1-release/channels/stable")) $(error exiting) endif @@ -228,6 +228,4 @@ ci-setup: $(GO) get $(PKG_GOX) @echo "Installing kubectl..." - curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl - chmod +x ./kubectl - mv ./kubectl /usr/local/bin/kubectl + ./scripts/install-tools.sh kubectl diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 000000000..3aaa9407c --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,40 @@ +// release group +group "release" { + targets = ["binary", "dind", "proxy", "tools"] +} + +// filled by GitHub Actions +target "docker-metadata-k3d" {} +target "docker-metadata-k3d-dind" {} +target "docker-metadata-k3d-proxy" {} +target "docker-metadata-k3d-tools" {} + +// default options for creating a release +target "default-release-options" { + platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7"] +} + +target "binary" { + inherits = ["default-release-options", "docker-metadata-k3d"] + dockerfile = "Dockerfile" + context = "." + target = "binary-only" +} + +target "dind" { + inherits = ["docker-metadata-k3d-dind"] // dind does not inherit defaults, as dind base image is not available for armv7 + platforms = ["linux/amd64", "linux/arm64"] + dockerfile = "Dockerfile" + context = "." + target = "dind" +} + +target "proxy" { + inherits = ["default-release-options", "docker-metadata-k3d-proxy"] + context = "proxy/" +} + +target "tools" { + inherits = ["default-release-options", "docker-metadata-k3d-tools"] + context = "tools/" +} diff --git a/proxy/Dockerfile b/proxy/Dockerfile index 3ffee0a30..0bb753383 100644 --- a/proxy/Dockerfile +++ b/proxy/Dockerfile @@ -1,16 +1,17 @@ FROM nginx:1.19-alpine # TODO:_ consider switching to https://github.com/abtreece/confd to not maintain a custom fork anymore -ARG CONFD_REPO=iwilltry42/confd -ARG CONFD_VERSION=0.17.0-rc.0 -ARG OS=linux -ARG ARCH=amd64 -RUN echo "Building for '${OS}/${ARCH}'..." \ - && mkdir -p /etc/confd \ - && wget "https://github.com/${CONFD_REPO}/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-${OS}-${ARCH}" -O /usr/bin/confd \ - && chmod +x /usr/bin/confd + +ARG OS +ARG ARCH + +ENV OS=${OS} +ENV ARCH=${ARCH} +COPY install-confd.sh /scripts/install-confd.sh +RUN mkdir -p /etc/confd \ + && /scripts/install-confd.sh COPY templates /etc/confd/templates/ COPY conf.d /etc/confd/conf.d/ COPY nginx-proxy /usr/bin/ -ENTRYPOINT nginx-proxy \ No newline at end of file +ENTRYPOINT nginx-proxy diff --git a/proxy/install-confd.sh b/proxy/install-confd.sh new file mode 100755 index 000000000..1940b59bc --- /dev/null +++ b/proxy/install-confd.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +# initArch discovers the architecture for this system. +initArch() { + if [ -z $ARCH ]; then + ARCH=$(uname -m) + case $ARCH in + armv5*) ARCH="armv5";; + armv6*) ARCH="armv6";; + armv7*) ARCH="arm";; + aarch64) ARCH="arm64";; + x86) ARCH="386";; + x86_64) ARCH="amd64";; + i686) ARCH="386";; + i386) ARCH="386";; + esac + fi +} + +# initOS discovers the operating system for this system. +initOS() { + if [ -z $OS ]; then + OS=$(uname|tr '[:upper:]' '[:lower:]') + + case "$OS" in + # Minimalist GNU for Windows + mingw*) OS='windows';; + esac + fi +} + + +install_confd() { + echo "Installing confd for $OS/$ARCH..." + CONFD_REPO=iwilltry42/confd + CONFD_VERSION=0.17.0-rc.0 + curl -sSfL "https://github.com/${CONFD_REPO}/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-${OS}-${ARCH}" -o ./confd + chmod +x ./confd + mv ./confd /usr/local/bin/confd +} + + +# +# MAIN +# + +initOS +initArch +install_confd diff --git a/scripts/install-tools.sh b/scripts/install-tools.sh new file mode 100755 index 000000000..0e30252e4 --- /dev/null +++ b/scripts/install-tools.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +# initArch discovers the architecture for this system. +initArch() { + if [ -z $ARCH ]; then + ARCH=$(uname -m) + case $ARCH in + armv5*) ARCH="armv5";; + armv6*) ARCH="armv6";; + armv7*) ARCH="arm";; + aarch64) ARCH="arm64";; + x86) ARCH="386";; + x86_64) ARCH="amd64";; + i686) ARCH="386";; + i386) ARCH="386";; + esac + fi +} + +# initOS discovers the operating system for this system. +initOS() { + if [ -z $OS ]; then + OS=$(uname|tr '[:upper:]' '[:lower:]') + + case "$OS" in + # Minimalist GNU for Windows + mingw*) OS='windows';; + esac + fi +} + +install_kubectl() { + echo "Installing kubectl for $OS/$ARCH..." + curl -sSfL "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/${OS}/${ARCH}/kubectl" -o ./kubectl + chmod +x ./kubectl + mv ./kubectl /usr/local/bin/kubectl +} + +install_yq() { + echo "Installing yq for $OS/$ARCH..." + curl -sSfL https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_${OS}_${ARCH} -o ./yq + chmod +x ./yq + mv ./yq /usr/local/bin/yq +} + +install_golangci_lint() { + echo "Installing golangci-lint..." + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.45.0 +} + +install_confd() { + echo "Installing confd for $OS/$ARCH..." + CONFD_REPO=iwilltry42/confd + CONFD_VERSION=0.17.0-rc.0 + curl -sSfL "https://github.com/${CONFD_REPO}/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-${OS}-${ARCH}" -o ./confd + chmod +x ./confd + mv ./confd /usr/local/bin/confd +} + + +# +# MAIN +# + +initOS +initArch + +for pkg in "$@"; do + case "$pkg" in + kubectl) install_kubectl;; + yq) install_yq;; + golangci-lint) install_golangci_lint;; + confd) install_confd;; + *) printf "ERROR: Unknown Package '%s'" $pkg;; + esac +done