diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 000000000..9af9840e8 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,54 @@ +# (C) Copyright Red Hat 2022. +# SPDX-License-Identifier: Apache-2.0 +# +# Build and run unit tests the project for each cloud provider. +--- +name: build +on: [pull_request] +jobs: + build_job: + name: build + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + provider: + # Please keep this list in alphabetical order. + - aws + - azure + - ibmcloud + - libvirt + runner: + - ubuntu-latest + go_version: + - 1.18 + steps: + - name: Checkout the pull request code + uses: actions/checkout@v3 + - name: Setup Golang version ${{ matrix.go_version }} + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go_version }} + - name: Install build dependencies + if: ${{ matrix.provider == 'libvirt' }} + run: | + sudo apt-get update -y + sudo apt-get install -y libvirt-dev + - name: Build + run: | + make CLOUD_PROVIDER=${{ matrix.provider }} build + - name: Test + run: | + go install github.com/jstemmer/go-junit-report@v1.0.0 + export CI="true" + sudo -E env PATH="$PATH" make CLOUD_PROVIDER=${{ matrix.provider }} test | tee tests_report.txt + sudo chmod o+rw tests_report.txt + cat tests_report.txt | $(go env GOPATH)/bin/go-junit-report -set-exit-code > tests_report_junit.xml + shell: bash + - name: Upload tests report + uses: actions/upload-artifact@v3 + if: always() + with: + name: tests_report_junit-${{ matrix.provider }}_${{ matrix.runner }}_${{ matrix.go_version }} + path: ${{ github.workspace }}/tests_report_junit.xml + retention-days: 1 diff --git a/.github/workflows/commit-message-check.yaml b/.github/workflows/commit-message-check.yaml new file mode 100644 index 000000000..405fab01d --- /dev/null +++ b/.github/workflows/commit-message-check.yaml @@ -0,0 +1,93 @@ +name: Commit Message Check +on: + pull_request: + types: + - opened + - reopened + - synchronize + +env: + error_msg: |+ + See the document below for help on formatting commits for the project. + + https://github.com/confidential-containers/community/blob/main/CONTRIBUTING.md#patch-format + +jobs: + commit-message-check: + runs-on: ubuntu-latest + name: Commit Message Check + steps: + - name: Get PR Commits + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + id: "get-pr-commits" + uses: tim-actions/get-pr-commits@v1.2.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + # Filter out revert commits + # The format of a revert commit is as follows: + # + # Revert "" + # + filter_out_pattern: '^Revert "' + + - name: Commit Body Missing Check + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }} + uses: tim-actions/commit-body-check@v1.0.2 + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} + + - name: Check Subject Line Length + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }} + uses: tim-actions/commit-message-checker-with-regex@v0.3.1 + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} + pattern: '^.{0,75}(\n.*)*$' + error: "Subject too long (max 75)" + post_error: ${{ env.error_msg }} + + - name: Check Body Line Length + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }} + uses: tim-actions/commit-message-checker-with-regex@v0.3.1 + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} + # Notes: + # + # - The subject line is not enforced here (see other check), but has + # to be specified at the start of the regex as the action is passed + # the entire commit message. + # + # - Body lines *can* be longer than the maximum if they start + # with a non-alphabetic character. + # + # This allows stack traces, log files snippets, emails, long URLs, + # etc to be specified. Some of these naturally "work" as they start + # with numeric timestamps or addresses. Emails can but quoted using + # the normal ">" character, markdown bullets ("-", "*") are also + # useful for lists of URLs, but it is always possible to override + # the check by simply space indenting the content you need to add. + # + # - A SoB comment can be any length (as it is unreasonable to penalise + # people with long names/email addresses :) + pattern: '^.+(\n([a-zA-Z].{0,149}|[^a-zA-Z\n].*|Signed-off-by:.*|))+$' + error: "Body line too long (max 72)" + post_error: ${{ env.error_msg }} + + - name: Check Fixes + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }} + uses: tim-actions/commit-message-checker-with-regex@v0.3.1 + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} + pattern: '\s*Fixes\s*:?\s*(#\d+|github\.com\/cloud-api-adaptor\/[a-z-.]*#\d+)|^\s*release\s*:' + flags: "i" + error: 'No "Fixes" found' + post_error: ${{ env.error_msg }} + one_pass_all_pass: "true" + + - name: Check Subsystem + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }} + uses: tim-actions/commit-message-checker-with-regex@v0.3.1 + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} + pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:' + error: "Failed to find subsystem in subject" + post_error: ${{ env.error_msg }} diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml new file mode 100644 index 000000000..cd9d25640 --- /dev/null +++ b/.github/workflows/image.yaml @@ -0,0 +1,49 @@ +# (C) Copyright Confidential Containers Contributors 2022. +# SPDX-License-Identifier: Apache-2.0 +# +# Build and push container images for each cloud provider. +--- +name: image +on: + push: + branches: + - 'staging' +jobs: + build_push_job: + name: build and push + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + provider: + # Please keep this list in alphabetical order. + - aws + - azure + - ibmcloud + - libvirt + - vsphere + runner: + - ubuntu-latest + go_version: + - 1.18 + steps: + - name: Checkout the code + uses: actions/checkout@v3 + - name: Setup Golang version ${{ matrix.go_version }} + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go_version }} + - name: Install build dependencies + if: ${{ matrix.provider == 'libvirt' }} + run: | + sudo apt-get update -y + sudo apt-get install -y libvirt-dev + - name: Login to quay Container Registry + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + - name: Build and push image + run: | + make CLOUD_PROVIDER=${{ matrix.provider }} image diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 000000000..537ec6acc --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,56 @@ +# (C) Copyright Red Hat 2022. +# SPDX-License-Identifier: Apache-2.0 +# +# Run linting tools on the sources of the project. +--- +name: lint +on: [pull_request] +jobs: + golangci-lint_job: + name: golangci-lint + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + go_version: + - 1.18 + steps: + - name: Checkout the pull request code + uses: actions/checkout@v3 + - name: Setup Golang version ${{ matrix.go_version }} + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go_version }} + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.46.2 + + go_check: + strategy: + matrix: + provider: + # Please keep this list in alphabetical order. + - aws + - azure + - ibmcloud + - libvirt + - vsphere + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@main + - uses: actions/setup-go@main + with: + go-version: 1.18 + - name: Install build dependencies + if: ${{ matrix.provider == 'libvirt' }} + run: | + sudo apt-get update -y + sudo apt-get install -y libvirt-dev + - name: check + run: make CLOUD_PROVIDER=${{ matrix.provider }} check + - name: escapes + run: make CLOUD_PROVIDER=${{ matrix.provider }} escapes diff --git a/.github/workflows/webhook.yaml b/.github/workflows/webhook.yaml new file mode 100644 index 000000000..90e7d379a --- /dev/null +++ b/.github/workflows/webhook.yaml @@ -0,0 +1,35 @@ +# Copyright Confidential Containers Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Run end-to-end tests if any webhook source files changed. +--- +name: webhook +on: + pull_request: + paths: + - 'webhook/**' +jobs: + test-e2e: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: webhook + steps: + - name: Checkout the pull request code + uses: actions/checkout@v3 + - name: Setup Golang version 1.18 + uses: actions/setup-go@v3 + with: + go-version: 1.18 + - name: Install kind + run: | + go install sigs.k8s.io/kind@v0.16.0 + - name: Install bats + run: | + git clone https://github.com/bats-core/bats-core.git + cd bats-core + ./install.sh ~/.local + - name: Run end-to-end tests + run: | + make test-e2e diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..e3d0d7102 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +#Directories +install/runtime-payload/bin +pause_bundle +output +staticlib +pause +umoci +skopeo + +# Binaries +agent-protocol-forwarder +cloud-api-adaptor +kata-agent + +# IDEs +.vscode + +# Files +vcenter.auto.pkrvars.hcl +settings.auto.pkrvars.hcl +*.qcow2 + diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 000000000..5c38e9342 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,383 @@ +# This file contains the configuration options for golangci-lint tool. It is a +# modified version of a reference file obtained from +# https://github.com/golangci/golangci-lint/blob/4deb32f2c083b244f64993e8ff40c97b52bc513a/.golangci.reference.yml + +# Options for analysis running. +run: + # The default concurrency value is the number of available CPU. + #concurrency: 4 + + # Timeout for analysis, e.g. 30s, 5m. + # Default: 1m + timeout: 5m + + # Exit code when at least one issue was found. + # Default: 1 + #issues-exit-code: 2 + + # Include test files or not. + # Default: true + #tests: false + + # List of build tags, all linters use it. + # Default: []. + build-tags: + - aws + - ibmcloud + - libvirt + - azure + + # Which dirs to skip: issues from them won't be reported. + # Can use regexp here: `generated.*`, regexp is applied on full path. + # Default value is empty list, + # but default dirs are skipped independently of this option's value (see skip-dirs-use-default). + # "/" will be replaced by current OS file path separator to properly work on Windows. + #skip-dirs: + # - src/external_libs + # - autogenerated_by_my_lib + + # Enables skipping of directories: + # - vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + # Default: true + #skip-dirs-use-default: false + + # Which files to skip: they will be analyzed, but issues from them won't be reported. + # Default value is empty list, + # but there is no need to include all autogenerated files, + # we confidently recognize autogenerated files. + # If it's not please let us know. + # "/" will be replaced by current OS file path separator to properly work on Windows. + #skip-files: + # - ".*\\.my\\.go$" + # - lib/bad.go + + # If set we pass it to "go list -mod={option}". From "go help modules": + # If invoked with -mod=readonly, the go command is disallowed from the implicit + # automatic updating of go.mod described above. Instead, it fails when any changes + # to go.mod are needed. This setting is most useful to check that go.mod does + # not need updates, such as in a continuous integration and testing system. + # If invoked with -mod=vendor, the go command assumes that the vendor + # directory holds the correct copies of dependencies and ignores + # the dependency descriptions in go.mod. + # + # Allowed values: readonly|vendor|mod + # By default, it isn't set. + #modules-download-mode: readonly + + # Allow multiple parallel golangci-lint instances running. + # If false (default) - golangci-lint acquires file lock on start. + allow-parallel-runners: false + + # Define the Go version limit. + # Mainly related to generics support in go1.18. + # Default: use Go version from the go.mod file, fallback on the env var `GOVERSION`, fallback on 1.17 + #go: '1.18' + + +# output configuration options +output: + # Format: colored-line-number|line-number|json|tab|checkstyle|code-climate|junit-xml|github-actions + # + # Multiple can be specified by separating them by comma, output can be provided + # for each of them by separating format name and path by colon symbol. + # Output path can be either `stdout`, `stderr` or path to the file to write to. + # Example: "checkstyle:report.json,colored-line-number" + # + # Default: colored-line-number + #format: json + + # Print lines of code with issue. + # Default: true + #print-issued-lines: false + + # Print linter name in the end of issue text. + # Default: true + #print-linter-name: false + + # Make issues output unique by line. + # Default: true + #uniq-by-line: false + + # Add a prefix to the output file references. + # Default is no prefix. + #path-prefix: "" + + # Sort results by: filepath, line and column. + #sort-results: false + + +# All available settings of linters enabled. +linters-settings: + errcheck: + # Report about not checking of errors in type assertions: `a := b.(MyStruct)`. + # Such cases aren't reported by default. + # Default: false + #check-type-assertions: true + + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`. + # Such cases aren't reported by default. + # Default: false + #check-blank: true + + # DEPRECATED comma-separated list of pairs of the form pkg:regex + # + # the regex is used to ignore names within pkg. (default "fmt:.*"). + # see https://github.com/kisielk/errcheck#the-deprecated-method for details + #ignore: fmt:.*,io/ioutil:^Read.* + + # To disable the errcheck built-in exclude list. + # See `-excludeonly` option in https://github.com/kisielk/errcheck#excluding-functions for details. + # Default: false + #disable-default-exclusions: true + + # DEPRECATED use exclude-functions instead. + # + # Path to a file containing a list of functions to exclude from checking. + # See https://github.com/kisielk/errcheck#excluding-functions for details. + #exclude: /path/to/file.txt + + # List of functions to exclude from checking, where each entry is a single function to exclude. + # See https://github.com/kisielk/errcheck#excluding-functions for details. + #exclude-functions: + # - io/ioutil.ReadFile + # - io.Copy(*bytes.Buffer) + # - io.Copy(os.Stdout) + + gofmt: + # Simplify code: gofmt with `-s` option. + # Default: true + #simplify: false + + gosimple: + # Select the Go version to target. + # Default: 1.13 + go: "1.18" + # https://staticcheck.io/docs/options#checks + checks: [ "all" ] + + govet: + # Report about shadowed variables. + # Default: false + check-shadowing: true + + # Settings per analyzer. + #settings: + # Analyzer name, run `go tool vet help` to see all analyzers. + # printf: + # Run `go tool vet help printf` to see available settings for `printf` analyzer. + # funcs: + # - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + # - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + # - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + # - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + + # Disable all analyzers. + # Default: false + #disable-all: true + # Enable analyzers by name. + # Run `go tool vet help` to see all analyzers. + #enable: + # - asmdecl + + # Enable all analyzers. + # Default: false + #enable-all: true + # Disable analyzers by name. + # Run `go tool vet help` to see all analyzers. + #disable: + # - asmdecl + + staticcheck: + # Select the Go version to target. + # Default: 1.13 + go: "1.18" + # https://staticcheck.io/docs/options#checks + checks: [ "all" ] + + varcheck: + # Check usage of exported fields and variables. + # Default: false + #exported-fields: true + + # The custom section can be used to define linter plugins to be loaded at runtime. + # See README documentation for more info. + #custom: + # Each custom linter should have a unique name. + # example: + # The path to the plugin *.so. Can be absolute or local. + # Required for each custom linter. + #path: /path/to/example.so + # The description of the linter. + # Optional. + #description: This is an example usage of a plugin linter. + # Intended to point to the repo location of the linter. + # Optional. + #original-url: github.com/golangci/example-linter + + +linters: + # Disable all linters. + # Default: false + disable-all: true + # Enable specific linter + # https://golangci-lint.run/usage/linters/#enabled-by-default-linters + enable: + # Explicitly listing the linters that would be enabled by default. + - deadcode + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + # Additional linters. + - gofmt + + # Enable all available linters. + # Default: false + #enable-all: true + # Disable specific linter + # https://golangci-lint.run/usage/linters/#disabled-by-default-linters--e--enable + #disable: + # - asciicheck + + # Enable presets. + # https://golangci-lint.run/usage/linters + #presets: + # - bugs + # - comment + # - complexity + # - error + # - format + # - import + # - metalinter + # - module + # - performance + # - sql + # - style + # - test + # - unused + + # Run only fast linters from enabled linters set (first run won't be fast) + # Default: false + #fast: true + + +issues: + # List of regexps of issue texts to exclude. + # + # But independently of this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. + # To list all excluded by default patterns execute `golangci-lint run --help` + # + # Default: [] + exclude: + - 'declaration of "(err|ctx)" shadows declaration at' + + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + - path: pkg/adaptor/hypervisor/registry/aws.go + text: 'other declaration of newServer' + - path: pkg/adaptor/hypervisor/registry/ibmcloud.go + text: '`newServer` redeclared in this block' + - path: pkg/adaptor/hypervisor/registry/libvirt.go + text: '`newServer` redeclared in this block' + # Exclude some linters from running on tests files. + #- path: _test\.go + # linters: + # - gocyclo + # - errcheck + # - dupl + # - gosec + + # Exclude known linters from partially hard-vendored code, + # which is impossible to exclude via `nolint` comments. + #- path: internal/hmac/ + # text: "weak cryptographic primitive" + # linters: + # - gosec + + # Exclude some `staticcheck` messages. + #- linters: + # - staticcheck + # text: "SA9003:" + + # Exclude `lll` issues for long lines with `go:generate`. + #- linters: + # - lll + # source: "^//go:generate " + + # Independently of option `exclude` we use default exclude patterns, + # it can be disabled by this option. + # To list all excluded by default patterns execute `golangci-lint run --help`. + # Default: true. + #exclude-use-default: false + + # If set to true exclude and exclude-rules regular expressions become case-sensitive. + # Default: false + exclude-case-sensitive: false + + # The list of ids of default excludes to include or disable. + # Default: [] + #include: + # - EXC0002 # disable excluding of issues about comments from golint. + + # Maximum issues count per one linter. + # Set to 0 to disable. + # Default: 50 + max-issues-per-linter: 0 + + # Maximum count of issues with the same text. + # Set to 0 to disable. + # Default: 3 + max-same-issues: 0 + + # Show only new issues: if there are unstaged changes or untracked files, + # only those changes are analyzed, else only changes in HEAD~ are analyzed. + # It's a super-useful option for integration of golangci-lint into existing large codebase. + # It's not practical to fix all existing issues at the moment of integration: + # much better don't allow issues in new code. + # + # Default: false. + #new: true + + # Show only new issues created after git revision `REV`. + #new-from-rev: HEAD + + # Show only new issues created in git patch with set file path. + #new-from-patch: path/to/patch/file + + # Fix found issues (if it's supported by the linter). + #fix: true + + +severity: + # Set the default severity for issues. + # + # If severity rules are defined and the issues do not match or no severity is provided to the rule + # this will be the default severity applied. + # Severities should match the supported severity names of the selected out format. + # - Code climate: https://docs.codeclimate.com/docs/issues#issue-severity + # - Checkstyle: https://checkstyle.sourceforge.io/property_types.html#severity + # - GitHub: https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message + # + # Default value is an empty string. + #default-severity: error + + # If set to true `severity-rules` regular expressions become case-sensitive. + # Default: false + #case-sensitive: true + + # When a list of severity rules are provided, severity information will be added to lint issues. + # Severity rules have the same filtering capability as exclude rules + # except you are allowed to specify one matcher per severity rule. + # Only affects out formats that support setting severity information. + # + # Default: [] + #rules: + # - linters: + # - dupl + # severity: info diff --git a/DCO1.1.txt b/DCO1.1.txt new file mode 100644 index 000000000..1ce2a45b0 --- /dev/null +++ b/DCO1.1.txt @@ -0,0 +1,37 @@ +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..9b212ec4d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.18 AS builder +ARG CLOUD_PROVIDER +ENV CLOUD_PROVIDER=${CLOUD_PROVIDER} +COPY . cloud-api-adaptor +RUN git clone -b CCv0 https://github.com/kata-containers/kata-containers +WORKDIR cloud-api-adaptor +RUN if [ "$CLOUD_PROVIDER" = "libvirt" ] ; then apt-get update -y && apt-get install -y libvirt-dev; fi +RUN make + +FROM fedora:36 +ARG CLOUD_PROVIDER +ENV CLOUD_PROVIDER=${CLOUD_PROVIDER} +RUN if [ "$CLOUD_PROVIDER" = "libvirt" ] ; then dnf install -y libvirt-libs genisoimage /usr/bin/ssh && dnf clean all; fi +COPY --from=builder /go/cloud-api-adaptor/cloud-api-adaptor /usr/local/bin/cloud-api-adaptor-$CLOUD_PROVIDER +COPY --from=builder /go/cloud-api-adaptor/entrypoint.sh /usr/local/bin/entrypoint.sh +CMD ["entrypoint.sh"] diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..effb56c33 --- /dev/null +++ b/Makefile @@ -0,0 +1,90 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +.PHONY: all build check fmt vet clean image deploy delete +ifndef CLOUD_PROVIDER +$(error CLOUD_PROVIDER is not set) +endif + +GOFLAGS ?= -tags=$(CLOUD_PROVIDER) +BINARIES := cloud-api-adaptor agent-protocol-forwarder +SOURCEDIRS := ./cmd ./pkg +PACKAGES := $(shell go list $(addsuffix /...,$(SOURCEDIRS))) +SOURCES := $(shell find $(SOURCEDIRS) -name '*.go' -print) + +all: build +build: $(BINARIES) + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +$(BINARIES): $(SOURCES) +ifeq ($(CLOUD_PROVIDER),libvirt) + go build $(GOFLAGS) -o "$@" "cmd/$@/main.go" +else + CGO_ENABLED=0 go build $(GOFLAGS) -o "$@" "cmd/$@/main.go" +endif + +##@ Development + +.PHONY: escapes +escapes: ## golang memeory escapes check +ifeq ($(CLOUD_PROVIDER),libvirt) + go build $(GOFLAGS) -gcflags="-m -l" -o cloud-api-adaptor cmd/cloud-api-adaptor/main.go | grep "escapes to heap" || true + go build $(GOFLAGS) -gcflags="-m -l" -o agent-protocol-forwarder cmd/agent-protocol-forwarder/main.go | grep "escapes to heap" || true +else + CGO_ENABLED=0 go build -gcflags="-m -l" $(GOFLAGS) -o cloud-api-adaptor cmd/cloud-api-adaptor/main.go | grep "escapes to heap" || true + CGO_ENABLED=0 go build -gcflags="-m -l" $(GOFLAGS) -o agent-protocol-forwarder cmd/agent-protocol-forwarder/main.go | grep "escapes to heap" || true +endif + +.PHONY: test +test: ## Run tests. + # Note: sending stderr to stdout so that tools like go-junit-report can + # parse build errors. + go test -v $(GOFLAGS) -cover $(PACKAGES) 2>&1 + +.PHONY: check +check: fmt vet ## Run go vet and go vet against the code. + +.PHONY: fmt +fmt: ## Run go fmt against code. + find $(SOURCEDIRS) -name '*.go' -print0 | xargs -0 gofmt -l -s -w + +.PHONY: vet +vet: ## Run go vet against code. + go vet $(GOFLAGS) $(PACKAGES) + +.PHONY: clean +clean: ## Remove binaries. + rm -fr $(BINARIES) + +##@ Build + +.PHONY: image +image: ## Build and push docker image to $registry + hack/build.sh + +##@ Deployment + +.PHONY: deploy +deploy: ## Deploy cloud-api-adaptor using the operator, according to install/overlays/$(CLOUD_PROVIDER)/kustomization.yaml file. + kubectl apply -f install/yamls/deploy.yaml + kubectl apply -k install/overlays/$(CLOUD_PROVIDER) + +.PHONY: delete +delete: ## Delete cloud-api-adaptor using the operator, according to install/overlays/$(CLOUD_PROVIDER)/kustomization.yaml file. + kubectl delete -k install/overlays/$(CLOUD_PROVIDER) diff --git a/README.md b/README.md new file mode 100644 index 000000000..3a7dd9817 --- /dev/null +++ b/README.md @@ -0,0 +1,48 @@ +# Introduction + +This repository contains the implementation of Kata [remote hypervisor](https://github.com/kata-containers/kata-containers/tree/CCv0). +Kata remote hypervisor enables creation of Kata VMs on any environment without requiring baremetal servers or nested +virtualization support. + +## Goals + +* Accept requests from Kata shim to create/delete Kata VM instances without requiring nested virtualization support. +* Manage VM instances in the cloud to run pods using cloud (virtualization) provider APIs +* Forward communication between kata shim on a worker node VM and kata agent on a pod VM +* Provide a mechanism to establish a network tunnel between a worker and pod VMs to Kubernetes pod network + +## Architecture + +The high level architecture is described in the picture below +![Architecture](./docs/architecture.png) + +## Components + +* Cloud API adaptor ([cmd/cloud-api-adaptor](./cmd/cloud-api-adaptor)) - `cloud-api-adator` implements the remote hypervisor support. +* Agent protocol forwarder ([cmd/agent-protocol-forwarder](./cmd/agent-protocol-forwarder)) + +## Installation + +Please refer to the instructions mentioned in the following [doc](install/README.md). + +## Supported Providers + +* aws +* azure +* ibmcloud +* libvirt +* vsphere + +### Adding a new provider + +Please refer to the instructions mentioned in the following [doc](./docs/addnewprovider.md). + +## Contribution + +This project uses [the Apache 2.0 license](./LICENSE). Contribution to this project requires the [DCO 1.1](./DCO1.1.txt) process to be followed. + +## Collaborations + +* Slack: [#confidential-containers-peerpod](https://cloud-native.slack.com/archives/C04A2EJ70BX) in [CNCF](https://communityinviter.com/apps/cloud-native/cncf) +* Zoom meeting: https://zoom.us/j/94601737867?pwd=MEF5NkN5ZkRDcUtCV09SQllMWWtzUT09 + * 8:00 - 9:00 UTC on each `Wednesday` \ No newline at end of file diff --git a/aws/README.md b/aws/README.md new file mode 100644 index 000000000..8014dca08 --- /dev/null +++ b/aws/README.md @@ -0,0 +1,37 @@ +# Setup instructions +## Prerequisites + +- Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` for AWS cli access + +- Install packer by following the instructions in the following [link](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli) + - Install packer's Amazon plugin `packer plugins install github.com/hashicorp/amazon` + +## Image Build + +- Set environment variables +``` +export AWS_ACCOUNT_ID="REPLACE_ME" +export AWS_REGION="REPLACE_ME" +``` + +- Either make sure default VPC is enabled: `aws ec2 create-default-vpc --region ${AWS_REGION}` or +create create new VPC with public internet access and set also the following environment variables +``` +export VPC_ID="REPLACE_ME" +export SUBNET_ID="REPLACE_ME" +``` + +- Create a custom AMI based on Ubuntu 20.04 having kata-agent and other dependencies + - [setting up authenticated registry support](../docs/registries-authentication.md) +``` +cd image +CLOUD_PROVIDER=aws make image +``` + +- Note down your newly created AMI_ID + +## Running cloud-api-adaptor + +- Update [kustomization.yaml](../install/overlays/aws/kustomization.yaml) with your AMI_ID + +- Deploy Cloud API Adaptor by following the [install](../install/README.md) guide diff --git a/aws/image/.gitignore b/aws/image/.gitignore new file mode 100644 index 000000000..08211c67f --- /dev/null +++ b/aws/image/.gitignore @@ -0,0 +1,4 @@ +agent-protocol-forwarder +kata-agent +skopeo +umoci diff --git a/aws/image/Makefile b/aws/image/Makefile new file mode 100644 index 000000000..fb8f48ef6 --- /dev/null +++ b/aws/image/Makefile @@ -0,0 +1,26 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +include ../../podvm/Makefile.inc + + +.PHONY: image clean + +image: $(IMAGE_FILE) + +$(IMAGE_FILE): $(BINARIES) $(FILES) + mkdir -p toupload + packer build -var vpc_id=${VPC_ID} \ + -var subnet_id=${SUBNET_ID} \ + -var account_id=${AWS_ACCOUNT_ID} \ + -var region=${AWS_REGION} \ + -var instance_type=${INSTANCE_TYPE} \ + -var ami_name=${IMAGE_NAME} . + rm -fr toupload + +clean: + rm -f "$(IMAGE_FILE)" "$(UBUNTU_IMAGE_FILE)" $(BINARIES) + rm -fr "$(SKOPEO_SRC)" "$(UMOCI_SRC)" "$(PAUSE_SRC)" "$(FILES_DIR)/$(PAUSE_BUNDLE)" + +.PHONY: force +force: diff --git a/aws/image/aws-ubuntu.pkr.hcl b/aws/image/aws-ubuntu.pkr.hcl new file mode 100644 index 000000000..ca217ec3f --- /dev/null +++ b/aws/image/aws-ubuntu.pkr.hcl @@ -0,0 +1,64 @@ +packer { + required_plugins { + amazon = { + version = ">= 0.0.2" + source = "github.com/hashicorp/amazon" + } + } +} + +source "amazon-ebs" "ubuntu" { + ami_name = "${var.ami_name}" + instance_type = "${var.instance_type}" + region = "${var.region}" + vpc_id = "${var.vpc_id}" + subnet_id = "${var.subnet_id}" + source_ami_filter { + filters = { + name = "ubuntu/images/*ubuntu*focal*" + root-device-type = "ebs" + virtualization-type = "hvm" + architecture = "x86_64" + } + + most_recent = true + owners = ["${var.account_id}", "aws-marketplace", "amazon"] + } + ssh_username = "ubuntu" +} + +build { + name = "peer-pod-ubuntu" + sources = [ + "source.amazon-ebs.ubuntu" + ] + + provisioner "shell-local" { + command = "tar cf toupload/files.tar -C ../../podvm files" + } + + provisioner "file" { + source = "./toupload" + destination = "/tmp/" + } + + provisioner "shell" { + inline = [ + "cd /tmp && tar xf toupload/files.tar", + "rm toupload/files.tar" + ] + } + + provisioner "file" { + source = "copy-files.sh" + destination = "~/copy-files.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/copy-files.sh" + ] + } + +} diff --git a/aws/image/copy-files.sh b/aws/image/copy-files.sh new file mode 100755 index 000000000..d9f0d4c5e --- /dev/null +++ b/aws/image/copy-files.sh @@ -0,0 +1,17 @@ +sudo mkdir -p /etc/containers +sudo cp /tmp/files/etc/agent-config.toml /etc/agent-config.toml +sudo cp -a /tmp/files/etc/containers/* /etc/containers/ +sudo cp -a /tmp/files/etc/systemd/* /etc/systemd/ +if [ -e /tmp/files/etc/aa-offline_fs_kbc-resources.json ]; then + sudo cp /tmp/files/etc/aa-offline_fs_kbc-resources.json /etc/aa-offline_fs_kbc-resources.json +fi + +sudo mkdir -p /usr/local/bin +sudo cp -a /tmp/files/usr/* /usr/ + +sudo cp -a /tmp/files/pause_bundle / + +if [ -e /tmp/files/auth.json ]; then + sudo mkdir -p /root/.config/containers/ + sudo cp -a /tmp/files/auth.json /root/.config/containers/auth.json +fi diff --git a/aws/image/variables.pkr.hcl b/aws/image/variables.pkr.hcl new file mode 100644 index 000000000..08b4cb126 --- /dev/null +++ b/aws/image/variables.pkr.hcl @@ -0,0 +1,31 @@ +// variables.pkr.hcl + +// For those variables that you don't provide a default for, you must +// set them from the command line, a var-file, or the environment. + +variable "ami_name" { + type = string + default = "peer-pod-ami" +} + +variable "instance_type" { + type = string + default = "t3.small" +} + + +variable "region" { + type = string +} + +variable "vpc_id" { + type = string +} + +variable "subnet_id" { + type = string +} + +variable "account_id" { + type = string +} diff --git a/azure/README.md b/azure/README.md new file mode 100644 index 000000000..966ea3b4e --- /dev/null +++ b/azure/README.md @@ -0,0 +1,68 @@ +# Setup instructions + +- Install packer by using the [following](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli) instructions. + +- Create a Resource Group + +```bash +export RESOURCE_GROUP="REPLACE_ME" +export LOCATION="REPLACE_ME" + +az group create --name "${RESOURCE_GROUP}" --location "${LOCATION}" +``` + +- Create Service Principal to build image + +```bash +SUBSCRIPTION_ID=$(az account show --query id --output tsv) + +az ad sp create-for-rbac \ + --role Contributor \ + --scopes "/subscriptions/$SUBSCRIPTION_ID" \ + --query "{ client_id: appId, client_secret: password, tenant_id: tenant }" +``` + +- Set environment variables + +The env var `CLIENT_ID`, `CLIENT_SECRET`, `TENANT_ID` can be copied from the output of the last command: + +```bash +export CLIENT_ID="REPLACE_ME" +export CLIENT_SECRET="REPLACE_ME" +export TENANT_ID="REPLACE_ME" +``` + +- Create a custom Azure VM image based on Ubuntu 20.04 having kata-agent and other dependencies. + [setting up authenticated registry support](../docs/registries-authentication.md) +```bash +export VM_SIZE="REPLACE_ME" +cd image +CLOUD_PROVIDER=azure make build && cd - +``` + +The output image id will be used while running the cloud-api-adaptor, which get's uploaded to your Azure portal using Packer. + +- Export your Azure VM information and run k8s on it + +```bash +VM_NAME="REPLACE_ME" +PEER_POD_NAME="OUTPUT_FROM_ABOVE" +``` + +# Running cloud-api-adaptor + +- If using Calico CNI, [configure](https://projectcalico.docs.tigera.io/networking/vxlan-ipip#configure-vxlan-encapsulation-for-all-inter-workload-traffic) VXLAN encapsulation for all inter workload traffic. + +- Create Service Principal for the CAA + +```bash +az ad sp create-for-rbac \ + -n peer-pod-vm-creator \ + --role Contributor \ + --scopes "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP" \ + --query "{ clientid: appId, secret: password, tenantid: tenant }" +``` + +- Update [kustomization.yaml](../install/overlays/azure/kustomization.yaml) with the required values. + +- Deploy Cloud API Adaptor by following the [install](../install/README.md) guide. diff --git a/azure/image/.gitignore b/azure/image/.gitignore new file mode 100644 index 000000000..b474b1ef9 --- /dev/null +++ b/azure/image/.gitignore @@ -0,0 +1,4 @@ +skopeo +umoci + +files/usr/ diff --git a/azure/image/Makefile b/azure/image/Makefile new file mode 100644 index 000000000..4edd948f4 --- /dev/null +++ b/azure/image/Makefile @@ -0,0 +1,126 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# + +.PHONY: build clean + +UBUNTU_RELEASE = focal +SKOPEO_VERSION = 1.5.0 +UMOCI_VERSION = 0.4.7 + +IMAGE_PREFIX := podvm + +ARCH := $(subst x86_64,amd64,$(shell uname -m)) + +FILES_DIR := files +FILES = $(shell find "$(FILES_DIR)" -type f -o -type l) + +ifndef IMAGE_NAME +COMMIT := $(shell commit=$$(git describe --match '' --dirty --always) && \ + if [ -n "$$(git ls-files --other --exclude-per-directory .gitignore "$(FILES_DIR)")" ]; then \ + commit="$${commit%-dirty}-dirty"; \ + fi && \ + echo "$$commit") +ifndef COMMIT +$(error Failed to derive an image name. Explicitly define IMAGE_NAME) +endif +IMAGE_NAME := $(IMAGE_PREFIX)-$(COMMIT)-$(ARCH) +endif +IMAGE_FILE := $(IMAGE_NAME) + + + +AGENT_PROTOCOL_FORWARDER = $(FILES_DIR)/usr/local/bin/agent-protocol-forwarder +KATA_AGENT = $(FILES_DIR)/usr/local/bin/kata-agent +PAUSE = $(FILES_DIR)/$(PAUSE_BUNDLE)/rootfs/pause +SKOPEO = $(FILES_DIR)/usr/bin/skopeo +UMOCI = $(FILES_DIR)/usr/local/bin/umoci + +BINARIES = $(AGENT_PROTOCOL_FORWARDER) $(KATA_AGENT) $(PAUSE) + +ifdef USE_SKOPEO +BINARIES += $(SKOPEO) $(UMOCI) +endif + +AGENT_PROTOCOL_FORWARDER_SRC = ../.. + +KATA_AGENT_SRC = ../../../kata-containers/src/agent +KATA_AGENT_BUILD_TYPE = release + +SKOPEO_SRC = skopeo +SKOPEO_REPO = https://github.com/containers/skopeo + +UMOCI_SRC = umoci +UMOCI_REPO = https://github.com/opencontainers/umoci + +# Embed the pause container image +# https://github.com/arronwy/kata-containers/commit/75b9f3fa3caaae62f49b4733f65cbab0cc87dbee +PAUSE_SRC = pause +PAUSE_REPO = docker://k8s.gcr.io/pause +PAUSE_VERSION = 3.6 +PAUSE_BUNDLE = pause_bundle + +# Static libseccomp is necessary for kata-agent +# https://github.com/kata-containers/kata-containers/issues/5044#issuecomment-1239773921 +STATIC_LIB_BUILDER = ../../../kata-containers/ci/install_libseccomp.sh +STATIC_LIB_DIR = $(abspath staticlib) +STATIC_LIB = $(STATIC_LIB_DIR)/kata-libseccomp/lib/libseccomp.a + +build: $(IMAGE_FILE) + +$(IMAGE_FILE): $(BINARIES) $(FILES) + mkdir -p toupload + packer build -var client_id=${CLIENT_ID} \ + -var client_secret=${CLIENT_SECRET} \ + -var subscription_id=${SUBSCRIPTION_ID} \ + -var tenant_id=${TENANT_ID} \ + -var location=${LOCATION} \ + -var vm_size=${VM_SIZE} \ + -var resource_group=${RESOURCE_GROUP} \ + -var az_image_name=${IMAGE_NAME} . + rm -fr toupload + +$(AGENT_PROTOCOL_FORWARDER): force + cd "$(AGENT_PROTOCOL_FORWARDER_SRC)" && $(MAKE) agent-protocol-forwarder + install -D --compare "$(AGENT_PROTOCOL_FORWARDER_SRC)/agent-protocol-forwarder" "$@" + +$(KATA_AGENT): force $(STATIC_LIB) + cd "$(KATA_AGENT_SRC)/../libs" && $(MAKE) BUILD_TYPE=$(KATA_AGENT_BUILD_TYPE) LIBSECCOMP_LINK_TYPE=static LIBSECCOMP_LIB_PATH=$(dir $(STATIC_LIB)) + cd "$(KATA_AGENT_SRC)" && $(MAKE) BUILD_TYPE=$(KATA_AGENT_BUILD_TYPE) LIBSECCOMP_LINK_TYPE=static LIBSECCOMP_LIB_PATH=$(dir $(STATIC_LIB)) + install -D --compare "$(KATA_AGENT_SRC)/target/$(shell uname -m)-unknown-linux-$(if $(findstring s390x,$(shell uname -m)),gnu,musl)/$(KATA_AGENT_BUILD_TYPE)/$(@F)" "$@" + +$(STATIC_LIB): + $(STATIC_LIB_BUILDER) $(STATIC_LIB_DIR)/kata-libseccomp $(STATIC_LIB_DIR)/kata-gperf + +# Skoepo package packages are available in RHEL/CentOS 8 or later and Ubuntu 20.10 or later +$(SKOPEO_SRC): + git clone -b "v$(SKOPEO_VERSION)" "$(SKOPEO_REPO)" "$(SKOPEO_SRC)" + +$(SKOPEO_SRC)/bin/skopeo: $(SKOPEO_SRC) + cd "$(SKOPEO_SRC)" && make bin/skopeo + +$(SKOPEO): $(SKOPEO_SRC)/bin/skopeo + install -D --compare "$(SKOPEO_SRC)/bin/skopeo" "$@" + +# The umoci release page only publishes amd64 binaries. https://github.com/opencontainers/umoci/releases +$(UMOCI_SRC): + git clone -b "v$(UMOCI_VERSION)" "$(UMOCI_REPO)" "$(UMOCI_SRC)" + +$(UMOCI_SRC)/umoci: $(UMOCI_SRC) + cd "$(UMOCI_SRC)" && make + +$(UMOCI): $(UMOCI_SRC)/umoci + install -D --compare "$(UMOCI_SRC)/umoci" "$@" + +$(PAUSE_SRC): $(SKOPEO_SRC)/bin/skopeo + $(SKOPEO_SRC)/bin/skopeo --policy "$(FILES_DIR)/etc/containers/policy.json" copy "$(PAUSE_REPO):$(PAUSE_VERSION)" "oci:$(PAUSE_SRC):$(PAUSE_VERSION)" + +$(PAUSE): | $(PAUSE_SRC) $(UMOCI_SRC)/umoci + $(UMOCI_SRC)/umoci unpack --rootless --image "$(PAUSE_SRC):$(PAUSE_VERSION)" "${FILES_DIR}/$(PAUSE_BUNDLE)" + +clean: + rm -f "$(IMAGE_FILE)" "$(UBUNTU_IMAGE_FILE)" $(BINARIES) + rm -fr "$(SKOPEO_SRC)" "$(UMOCI_SRC)" "$(PAUSE_SRC)" "$(FILES_DIR)/$(PAUSE_BUNDLE)" + +.PHONY: force +force: diff --git a/azure/image/azure-ubuntu.pkr.hcl b/azure/image/azure-ubuntu.pkr.hcl new file mode 100644 index 000000000..96c58870f --- /dev/null +++ b/azure/image/azure-ubuntu.pkr.hcl @@ -0,0 +1,62 @@ +source "azure-arm" "ubuntu" { + client_id = "${var.client_id}" + client_secret = "${var.client_secret}" + subscription_id = "${var.subscription_id}" + tenant_id = "${var.tenant_id}" + + vm_size = "${var.vm_size}" + location = "${var.location}" + os_type = "Linux" + image_publisher = "Canonical" + image_offer = "0001-com-ubuntu-minimal-jammy" + image_sku = "minimal-22_04-lts" + managed_image_name = "${var.az_image_name}" + managed_image_resource_group_name = "${var.resource_group}" + +} + +build { + name = "peer-pod-ubuntu" + sources = [ + "source.azure-arm.ubuntu" + ] + + provisioner "shell-local" { + command = "tar cf toupload/files.tar files" + } + + provisioner "file" { + source = "./toupload" + destination = "/tmp/" + } + + provisioner "shell" { + inline = [ + "cd /tmp && tar xf toupload/files.tar", + "rm toupload/files.tar" + ] + } + + provisioner "file" { + source = "copy-files.sh" + destination = "~/copy-files.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/copy-files.sh" + ] + } + + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; {{ .Vars }} sudo -E sh '{{ .Path }}'" + inline = [ + "/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync" + ] + inline_shebang = "/bin/sh -x" + } + + +} diff --git a/azure/image/copy-files.sh b/azure/image/copy-files.sh new file mode 100755 index 000000000..09b595799 --- /dev/null +++ b/azure/image/copy-files.sh @@ -0,0 +1,18 @@ +sudo mkdir -p /etc/containers +sudo cp /tmp/files/etc/agent-config.toml /etc/agent-config.toml +sudo cp -a /tmp/files/etc/containers/* /etc/containers/ +sudo cp -a /tmp/files/etc/systemd/* /etc/systemd/ +if [ -e /tmp/files/etc/aa-offline_fs_kbc-resources.json ]; then + sudo cp /tmp/files/etc/aa-offline_fs_kbc-resources.json /etc/aa-offline_fs_kbc-resources.json +fi + + +sudo mkdir -p /usr/local/bin +sudo cp -a /tmp/files/usr/* /usr/ + +sudo cp -a /tmp/files/pause_bundle / + +if [ -e /tmp/files/auth.json ]; then + sudo mkdir -p /root/.config/containers/ + sudo cp -a /tmp/files/auth.json /root/.config/containers/auth.json +fi diff --git a/azure/image/files/etc/agent-config.toml b/azure/image/files/etc/agent-config.toml new file mode 100644 index 000000000..60eaaf4ea --- /dev/null +++ b/azure/image/files/etc/agent-config.toml @@ -0,0 +1,44 @@ +# This disables signature verification which now defaults to true. +# We should consider a better solution. See #331 for more info +enable_signature_verification=false + +# When using the agent-config.toml the KATA_AGENT_SERVER_ADDR env var seems to be ignored, so set it here +server_addr="unix:///run/kata-containers/agent.sock" + +# temp workaround for kata-containers/kata-containers#5590 +[endpoints] +allowed = [ +"AddARPNeighborsRequest", +"AddSwapRequest", +"CloseStdinRequest", +"CopyFileRequest", +"CreateContainerRequest", +"CreateSandboxRequest", +"DestroySandboxRequest", +"ExecProcessRequest", +"GetMetricsRequest", +"GetOOMEventRequest", +"GuestDetailsRequest", +"ListInterfacesRequest", +"ListRoutesRequest", +"MemHotplugByProbeRequest", +"OnlineCPUMemRequest", +"PauseContainerRequest", +"PullImageRequest", +"ReadStreamRequest", +"RemoveContainerRequest", +"ReseedRandomDevRequest", +"ResumeContainerRequest", +"SetGuestDateTimeRequest", +"SignalProcessRequest", +"StartContainerRequest", +"StartTracingRequest", +"StatsContainerRequest", +"StopTracingRequest", +"TtyWinResizeRequest", +"UpdateContainerRequest", +"UpdateInterfaceRequest", +"UpdateRoutesRequest", +"WaitProcessRequest", +"WriteStreamRequest" +] \ No newline at end of file diff --git a/azure/image/files/etc/containers/policy.json b/azure/image/files/etc/containers/policy.json new file mode 100644 index 000000000..e0b40fba5 --- /dev/null +++ b/azure/image/files/etc/containers/policy.json @@ -0,0 +1,3 @@ +{ + "default": [{"type": "insecureAcceptAnything"}] +} diff --git a/azure/image/files/etc/systemd/system/agent-protocol-forwarder.service b/azure/image/files/etc/systemd/system/agent-protocol-forwarder.service new file mode 100644 index 000000000..8c51a31ab --- /dev/null +++ b/azure/image/files/etc/systemd/system/agent-protocol-forwarder.service @@ -0,0 +1,12 @@ +[Unit] +Description=Agent Protocol Forwarder +After=cloud-init.target +Wants=cloud-init.target +DefaultDependencies=no + + +[Service] +ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -kata-agent-socket /run/kata-containers/agent.sock + +[Install] +WantedBy=multi-user.target diff --git a/azure/image/files/etc/systemd/system/kata-agent.service b/azure/image/files/etc/systemd/system/kata-agent.service new file mode 100644 index 000000000..dce71efd5 --- /dev/null +++ b/azure/image/files/etc/systemd/system/kata-agent.service @@ -0,0 +1,15 @@ +[Unit] +Description=Kata Agent +After=network.target + +[Service] +ExecStart=/usr/local/bin/kata-agent --config /etc/agent-config.toml +ExecStartPre=-umount /sys/fs/cgroup/misc +ExecStartPre=ip netns add podns +ExecStartPre=ip netns exec podns ip link set lo up +ExecStopPost=ip netns delete podns +# Now specified in the agent-config.toml Environment="KATA_AGENT_SERVER_ADDR=unix:///run/kata-containers/agent.sock" +SyslogIdentifier=kata-agent + +[Install] +WantedBy=multi-user.target diff --git a/azure/image/files/etc/systemd/system/multi-user.target.wants/agent-protocol-forwarder.service b/azure/image/files/etc/systemd/system/multi-user.target.wants/agent-protocol-forwarder.service new file mode 120000 index 000000000..9ff49ce02 --- /dev/null +++ b/azure/image/files/etc/systemd/system/multi-user.target.wants/agent-protocol-forwarder.service @@ -0,0 +1 @@ +../agent-protocol-forwarder.service \ No newline at end of file diff --git a/azure/image/files/etc/systemd/system/multi-user.target.wants/kata-agent.service b/azure/image/files/etc/systemd/system/multi-user.target.wants/kata-agent.service new file mode 120000 index 000000000..b498ebe4a --- /dev/null +++ b/azure/image/files/etc/systemd/system/multi-user.target.wants/kata-agent.service @@ -0,0 +1 @@ +../kata-agent.service \ No newline at end of file diff --git a/azure/image/files/etc/systemd/system/multi-user.target.wants/run-image.mount b/azure/image/files/etc/systemd/system/multi-user.target.wants/run-image.mount new file mode 120000 index 000000000..07254bd41 --- /dev/null +++ b/azure/image/files/etc/systemd/system/multi-user.target.wants/run-image.mount @@ -0,0 +1 @@ +../run-image.mount \ No newline at end of file diff --git "a/azure/image/files/etc/systemd/system/multi-user.target.wants/run-kata\\x2dcontainers.mount" "b/azure/image/files/etc/systemd/system/multi-user.target.wants/run-kata\\x2dcontainers.mount" new file mode 120000 index 000000000..b92a05dad --- /dev/null +++ "b/azure/image/files/etc/systemd/system/multi-user.target.wants/run-kata\\x2dcontainers.mount" @@ -0,0 +1 @@ +../run-kata\x2dcontainers.mount \ No newline at end of file diff --git a/azure/image/files/etc/systemd/system/run-image.mount b/azure/image/files/etc/systemd/system/run-image.mount new file mode 100644 index 000000000..f4afe119c --- /dev/null +++ b/azure/image/files/etc/systemd/system/run-image.mount @@ -0,0 +1,12 @@ +[Unit] +Description=Mount unit for /run/image +Before=kata-agent.service + +[Mount] +What=/image +Where=/run/image +Type=none +Options=bind + +[Install] +WantedBy=multi-user.target diff --git "a/azure/image/files/etc/systemd/system/run-kata\\x2dcontainers.mount" "b/azure/image/files/etc/systemd/system/run-kata\\x2dcontainers.mount" new file mode 100644 index 000000000..64d5e27f6 --- /dev/null +++ "b/azure/image/files/etc/systemd/system/run-kata\\x2dcontainers.mount" @@ -0,0 +1,12 @@ +[Unit] +Description=Mount unit for /run/kata-containers +Before=kata-agent.service + +[Mount] +What=/kata-containers +Where=/run/kata-containers +Type=none +Options=bind + +[Install] +WantedBy=multi-user.target diff --git a/azure/image/variables.pkr.hcl b/azure/image/variables.pkr.hcl new file mode 100644 index 000000000..96f2723da --- /dev/null +++ b/azure/image/variables.pkr.hcl @@ -0,0 +1,41 @@ +// variables.pkr.hcl + +// For those variables that you don't provide a default for, you must +// set them from the command line, a var-file, or the environment. + +variable "az_image_name" { + type = string + default = "peer-pod-ami" +} + +// instance type +variable "vm_size" { + type = string + default = "Standard_A2_v2" +} + +// region +variable "location" { + type = string + default = "eastus" +} + +variable "resource_group" { + type = string +} + +variable "client_id" { + type = string +} + +variable "client_secret" { + type = string +} + +variable "subscription_id" { + type = string +} + +variable "tenant_id" { + type = string +} diff --git a/cmd/agent-protocol-forwarder/main.go b/cmd/agent-protocol-forwarder/main.go new file mode 100644 index 000000000..043f2bf30 --- /dev/null +++ b/cmd/agent-protocol-forwarder/main.go @@ -0,0 +1,89 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/confidential-containers/cloud-api-adaptor/cmd" + daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder" + "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder/interceptor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" +) + +const programName = "agent-protocol-forwarder" + +type Config struct { + configPath string + listenAddr string + kataAgentSocketPath string + kataAgentNamespace string + HostInterface string + + daemonConfig daemon.Config +} + +func load(path string, obj interface{}) error { + + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("failed to open %s: %w", path, err) + } + + if err := json.NewDecoder(file).Decode(obj); err != nil { + return fmt.Errorf("failed to decode a Agent Protocol Forwarder config file file: %s: %w", path, err) + } + + return nil +} + +func (cfg *Config) Setup() (cmd.Starter, error) { + + cmd.Parse(programName, os.Args, func(flags *flag.FlagSet) { + flags.StringVar(&cfg.configPath, "config", daemon.DefaultConfigPath, "Path to a deamon config file") + flags.StringVar(&cfg.listenAddr, "listen", daemon.DefaultListenAddr, "Listen address") + flags.StringVar(&cfg.kataAgentSocketPath, "kata-agent-socket", daemon.DefaultKataAgentSocketPath, "Path to a kata agent socket") + flags.StringVar(&cfg.kataAgentNamespace, "kata-agent-namespace", daemon.DefaultKataAgentNamespace, "Path to the network namespace where kata agent runs") + flags.StringVar(&cfg.HostInterface, "host-interface", "", "network interface name that is used for network tunnel traffic") + }) + + for path, obj := range map[string]interface{}{ + cfg.configPath: &cfg.daemonConfig, + } { + if err := load(path, obj); err != nil { + return nil, err + } + } + + interceptor := interceptor.NewInterceptor(cfg.kataAgentSocketPath, cfg.kataAgentNamespace) + + podNode := podnetwork.NewPodNode(cfg.kataAgentNamespace, cfg.HostInterface, cfg.daemonConfig.PodNetwork) + + daemon := daemon.NewDaemon(&cfg.daemonConfig, cfg.listenAddr, interceptor, podNode) + + return cmd.NewStarter(daemon), nil +} + +var config cmd.Config = &Config{} + +func main() { + + starter, err := config.Setup() + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err) + cmd.Exit(1) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if err := starter.Start(ctx); err != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err) + cmd.Exit(1) + } +} diff --git a/cmd/cloud-api-adaptor/main.go b/cmd/cloud-api-adaptor/main.go new file mode 100644 index 000000000..5fb364c4f --- /dev/null +++ b/cmd/cloud-api-adaptor/main.go @@ -0,0 +1,232 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "flag" + "fmt" + "os" + + "github.com/confidential-containers/cloud-api-adaptor/cmd" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/aws" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/azure" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/ibmcloud" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/libvirt" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/registry" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/vsphere" + daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" +) + +const programName = "cloud-api-adaptor" + +type daemonConfig struct { + helperDaemonRoot string + httpTunnelTimeout string + TunnelType string + HostInterface string + VXLANPort int + VXLANMinID int +} + +const DefaultShimTimeout = "60s" + +var vspherecfg vsphere.Config +var ibmcfg ibmcloud.Config +var awscfg aws.Config +var azurecfg azure.Config +var libvirtcfg libvirt.Config +var hypcfg hypervisor.Config + +func (cfg *daemonConfig) Setup() (cmd.Starter, error) { + + if len(os.Args) < 2 { + fmt.Printf("%s aws|azure|ibmcloud|libvirt \n", os.Args[0]) + cmd.Exit(1) + } + + //TODO: transition to better CLI library + + switch os.Args[1] { + case "aws": + cmd.Parse("aws", os.Args[1:], func(flags *flag.FlagSet) { + flags.StringVar(&awscfg.AccessKeyId, "aws-access-key-id", "", "Access Key ID, defaults to `AWS_ACCESS_KEY_ID`") + flags.StringVar(&awscfg.SecretKey, "aws-secret-key", "", "Secret Key, defaults to `AWS_SECRET_ACCESS_KEY`") + flags.StringVar(&awscfg.Region, "aws-region", "", "Region") + flags.StringVar(&awscfg.LoginProfile, "aws-profile", "test", "AWS Login Profile") + flags.StringVar(&awscfg.LaunchTemplateName, "aws-lt-name", "kata", "AWS Launch Template Name") + flags.BoolVar(&awscfg.UseLaunchTemplate, "use-lt", false, "Use EC2 Launch Template for the Pod VMs") + flags.StringVar(&awscfg.ImageId, "imageid", "", "Pod VM ami id") + flags.StringVar(&awscfg.InstanceType, "instance-type", "t3.small", "Pod VM instance type") + flags.Var(&awscfg.SecurityGroupIds, "securitygroupids", "Security Group Ids to be used for the Pod VM, comma separated") + flags.StringVar(&awscfg.KeyName, "keyname", "", "SSH Keypair name to be used with the Pod VM") + flags.StringVar(&awscfg.SubnetId, "subnetid", "", "Subnet ID to be used for the Pod VMs") + flags.StringVar(&hypcfg.SocketPath, "socket", hypervisor.DefaultSocketPath, "Unix domain socket path of remote hypervisor service") + flags.StringVar(&hypcfg.PodsDir, "pods-dir", hypervisor.DefaultPodsDir, "base directory for pod directories") + flags.StringVar(&hypcfg.HypProvider, "provider", "aws", "Hypervisor provider") + flags.StringVar(&hypcfg.CriSocketPath, "cri-runtime-endpoint", "", "cri runtime uds endpoint") + flags.StringVar(&hypcfg.PauseImage, "pause-image", hypervisor.DefaultPauseImage, "pause image to be used for the pods") + flags.StringVar(&cfg.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider") + flags.StringVar(&cfg.HostInterface, "host-interface", "", "Host Interface") + flags.IntVar(&cfg.VXLANPort, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only") + flags.IntVar(&cfg.VXLANMinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only") + }) + defaultToEnv(&awscfg.AccessKeyId, "AWS_ACCESS_KEY_ID") + defaultToEnv(&awscfg.SecretKey, "AWS_SECRET_ACCESS_KEY") + + case "azure": + cmd.Parse("azure", os.Args[1:], func(flags *flag.FlagSet) { + flags.StringVar(&azurecfg.ClientId, "clientid", "", "Client Id, defaults to `AZURE_CLIENT_ID`") + flags.StringVar(&azurecfg.ClientSecret, "secret", "", "Client Secret, defaults to `AZURE_CLIENT_SECRET`") + flags.StringVar(&azurecfg.TenantId, "tenantid", "", "Tenant Id, defaults to `AZURE_TENANT_ID`") + flags.StringVar(&azurecfg.ResourceGroupName, "resourcegroup", "", "Resource Group") + flags.StringVar(&azurecfg.Zone, "zone", "", "Zone") + flags.StringVar(&azurecfg.Region, "region", "", "Region") + flags.StringVar(&azurecfg.SubnetId, "subnetid", "", "Network Subnet Id") + flags.StringVar(&azurecfg.SecurityGroupId, "securitygroupid", "", "Security Group Id") + flags.StringVar(&azurecfg.Size, "instance-size", "", "Instance size") + flags.StringVar(&azurecfg.ImageId, "imageid", "", "Image Id") + flags.StringVar(&azurecfg.SubscriptionId, "subscriptionid", "", "Subscription ID") + flags.StringVar(&azurecfg.SSHKeyPath, "ssh-key-path", "$HOME/.ssh/id_rsa.pub", "Path to SSH public key") + flags.StringVar(&hypcfg.SocketPath, "socket", hypervisor.DefaultSocketPath, "Unix domain socket path of remote hypervisor service") + flags.StringVar(&hypcfg.PodsDir, "pods-dir", hypervisor.DefaultPodsDir, "base directory for pod directories") + flags.StringVar(&hypcfg.HypProvider, "provider", "azure", "Hypervisor provider") + flags.StringVar(&hypcfg.CriSocketPath, "cri-runtime-endpoint", "", "cri runtime uds endpoint") + flags.StringVar(&hypcfg.PauseImage, "pause-image", hypervisor.DefaultPauseImage, "pause image to be used for the pods") + flags.StringVar(&cfg.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider") + flags.StringVar(&cfg.HostInterface, "host-interface", "", "Host Interface") + flags.IntVar(&cfg.VXLANPort, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only") + flags.IntVar(&cfg.VXLANMinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only") + }) + defaultToEnv(&azurecfg.ClientId, "AZURE_CLIENT_ID") + defaultToEnv(&azurecfg.ClientSecret, "AZURE_CLIENT_SECRET") + defaultToEnv(&azurecfg.TenantId, "AZURE_TENANT_ID") + + case "ibmcloud": + cmd.Parse("ibmcloud", os.Args[1:], func(flags *flag.FlagSet) { + flags.StringVar(&ibmcfg.ApiKey, "api-key", "", "IBM Cloud API key, defaults to `IBMCLOUD_API_KEY`") + flags.StringVar(&ibmcfg.IamServiceURL, "iam-service-url", "https://iam.cloud.ibm.com/identity/token", "IBM Cloud IAM Service URL") + flags.StringVar(&ibmcfg.VpcServiceURL, "vpc-service-url", "https://jp-tok.iaas.cloud.ibm.com/v1", "IBM Cloud VPC Service URL") + flags.StringVar(&ibmcfg.ResourceGroupID, "resource-group-id", "", "Resource Group ID") + flags.StringVar(&ibmcfg.ProfileName, "profile-name", "", "Profile name") + flags.StringVar(&ibmcfg.ZoneName, "zone-name", "", "Zone name") + flags.StringVar(&ibmcfg.ImageID, "image-id", "", "Image ID") + flags.StringVar(&ibmcfg.PrimarySubnetID, "primary-subnet-id", "", "Primary subnet ID") + flags.StringVar(&ibmcfg.PrimarySecurityGroupID, "primary-security-group-id", "", "Primary security group ID") + flags.StringVar(&ibmcfg.SecondarySubnetID, "secondary-subnet-id", "", "Secondary subnet ID") + flags.StringVar(&ibmcfg.SecondarySecurityGroupID, "secondary-security-group-id", "", "Secondary security group ID") + flags.StringVar(&ibmcfg.KeyID, "key-id", "", "SSH Key ID") + flags.StringVar(&ibmcfg.VpcID, "vpc-id", "", "VPC ID") + flags.StringVar(&hypcfg.SocketPath, "socket", hypervisor.DefaultSocketPath, "Unix domain socket path of remote hypervisor service") + flags.StringVar(&hypcfg.PodsDir, "pods-dir", hypervisor.DefaultPodsDir, "base directory for pod directories") + flags.StringVar(&hypcfg.HypProvider, "provider", "ibmcloud", "Hypervisor provider") + flags.StringVar(&hypcfg.CriSocketPath, "cri-runtime-endpoint", "", "cri runtime uds endpoint") + flags.StringVar(&hypcfg.PauseImage, "pause-image", hypervisor.DefaultPauseImage, "pause image to be used for the pods") + flags.StringVar(&cfg.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider") + flags.StringVar(&cfg.HostInterface, "host-interface", "", "Host Interface") + flags.IntVar(&cfg.VXLANPort, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only") + flags.IntVar(&cfg.VXLANMinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only") + }) + defaultToEnv(&ibmcfg.ApiKey, "IBMCLOUD_API_KEY") + + case "libvirt": + cmd.Parse("libvirt", os.Args[1:], func(flags *flag.FlagSet) { + flags.StringVar(&libvirtcfg.URI, "uri", "qemu:///system", "libvirt URI") + flags.StringVar(&libvirtcfg.PoolName, "pool-name", "default", "libvirt storage pool") + flags.StringVar(&libvirtcfg.NetworkName, "network-name", "default", "libvirt network pool") + flags.StringVar(&libvirtcfg.DataDir, "data-dir", "/var/lib/libvirt/images", "libvirt storage dir") + flags.StringVar(&hypcfg.SocketPath, "socket", hypervisor.DefaultSocketPath, "Unix domain socket path of remote hypervisor service") + flags.StringVar(&hypcfg.PodsDir, "pods-dir", hypervisor.DefaultPodsDir, "base directory for pod directories") + flags.StringVar(&hypcfg.HypProvider, "provider", "libvirt", "Hypervisor provider") + flags.StringVar(&hypcfg.CriSocketPath, "cri-runtime-endpoint", "", "cri runtime uds endpoint") + flags.StringVar(&hypcfg.PauseImage, "pause-image", hypervisor.DefaultPauseImage, "pause image to be used for the pods") + flags.StringVar(&cfg.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider") + flags.StringVar(&cfg.HostInterface, "host-interface", "", "Host Interface") + flags.IntVar(&cfg.VXLANPort, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only") + flags.IntVar(&cfg.VXLANMinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only") + }) + + case "vsphere": + cmd.Parse("vsphere", os.Args[1:], func(flags *flag.FlagSet) { + flags.StringVar(&vspherecfg.VcenterURL, "vcenter-url", "", "URL of vCenter instance to connect to") + flags.StringVar(&vspherecfg.UserName, "user-name", "", "Username, defaults to `GOVC_USERNAME`") + flags.StringVar(&vspherecfg.Password, "password", "", "Password, defaults to `GOVC_PASSWORD`") + flag.BoolVar(&vspherecfg.Insecure, "insecure", true, "Disable certificate verification") + + flags.StringVar(&vspherecfg.Template, "template", "podvm-template", "vCenter template to deploy") + // GOVC_DATACENTER + flags.StringVar(&vspherecfg.Datacenter, "data-center", "", "vCenter desination datacenter name") + // GOVC_CLUSTER + flags.StringVar(&vspherecfg.Vcluster, "vcluster-name", "", "vCenter desination cluster name for DRS placement") + // GOVC_DATASTORE + flags.StringVar(&vspherecfg.Datastore, "data-store", "", "vCenter datastore") + // GOVC_RESOURCE_POOL + flags.StringVar(&vspherecfg.Resourcepool, "resource-pool", "", "vCenter desination resource pool") + // GOVC_FOLDER + flags.StringVar(&vspherecfg.Deployfolder, "deploy-folder", "", "vCenter vm desintation folder relative to the vm inventory path (your-data-center/vm). \nExample '-deploy-folder peerods' will create or use the existing folder peerpods as the \ndeploy-folder in /datacenter/vm/peerpods") + + flags.StringVar(&hypcfg.SocketPath, "socket", hypervisor.DefaultSocketPath, "Unix domain socket path of remote hypervisor service") + flags.StringVar(&hypcfg.PodsDir, "pods-dir", hypervisor.DefaultPodsDir, "base directory for pod directories") + flags.StringVar(&hypcfg.HypProvider, "provider", "vsphere", "Hypervisor provider") + flags.StringVar(&hypcfg.CriSocketPath, "cri-runtime-endpoint", "", "cri runtime uds endpoint") + flags.StringVar(&hypcfg.PauseImage, "pause-image", hypervisor.DefaultPauseImage, "pause image to be used for the pods") + flags.StringVar(&cfg.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider") + flags.StringVar(&cfg.HostInterface, "host-interface", "", "Host Interface") + flags.IntVar(&cfg.VXLANPort, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only") + flags.IntVar(&cfg.VXLANMinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only") + }) + defaultToEnv(&vspherecfg.UserName, "GOVC_USERNAME") + defaultToEnv(&vspherecfg.Password, "GOVC_PASSWORD") + + default: + os.Exit(1) + } + + workerNode := podnetwork.NewWorkerNode(cfg.TunnelType, cfg.HostInterface, cfg.VXLANPort, cfg.VXLANMinID) + + var hypervisorServer hypervisor.Server + + if hypcfg.HypProvider == "ibmcloud" { + hypervisorServer = registry.NewServer(hypcfg, ibmcfg, workerNode, daemon.DefaultListenPort) + } else if hypcfg.HypProvider == "aws" { + hypervisorServer = registry.NewServer(hypcfg, awscfg, workerNode, daemon.DefaultListenPort) + } else if hypcfg.HypProvider == "libvirt" { + hypervisorServer = registry.NewServer(hypcfg, libvirtcfg, workerNode, daemon.DefaultListenPort) + } else if hypcfg.HypProvider == "azure" { + hypervisorServer = registry.NewServer(hypcfg, azurecfg, workerNode, daemon.DefaultListenPort) + } else if hypcfg.HypProvider == "vsphere" { + hypervisorServer = registry.NewServer(hypcfg, vspherecfg, workerNode, daemon.DefaultListenPort) + } + + return cmd.NewStarter(hypervisorServer), nil +} + +func defaultToEnv(field *string, env string) { + if *field == "" { + *field = os.Getenv(env) + } +} + +var config cmd.Config = &daemonConfig{} + +func main() { + + starter, err := config.Setup() + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err) + cmd.Exit(1) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if err := starter.Start(ctx); err != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err) + cmd.Exit(1) + } +} diff --git a/cmd/exit.go b/cmd/exit.go new file mode 100644 index 000000000..7418d47b7 --- /dev/null +++ b/cmd/exit.go @@ -0,0 +1,11 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "os" +) + +// Exit calls os.Exit by default. This variable can be replaced for testing +var Exit = os.Exit diff --git a/cmd/parse.go b/cmd/parse.go new file mode 100644 index 000000000..b73f03882 --- /dev/null +++ b/cmd/parse.go @@ -0,0 +1,47 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "errors" + "flag" +) + +func Parse(programName string, args []string, fn func(flags *flag.FlagSet)) { + + var versionFlag bool + + flags := flag.NewFlagSet(programName, flag.ContinueOnError) + flags.SetOutput(flag.CommandLine.Output()) + flags.BoolVar(&versionFlag, "version", false, "Show version information") + + fn(flags) + + switch programName { + case "ibmcloud": + if len(args) < 10 { + flags.PrintDefaults() + Exit(1) + } + case "aws": + if len(args) < 2 { + flags.PrintDefaults() + Exit(1) + } + + } + + if err := flags.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + Exit(0) + } else { + Exit(1) + } + } + + if versionFlag { + ShowVersion(programName) + Exit(0) + } +} diff --git a/cmd/parse_test.go b/cmd/parse_test.go new file mode 100644 index 000000000..f984b5aa4 --- /dev/null +++ b/cmd/parse_test.go @@ -0,0 +1,136 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 +package cmd + +import ( + "bytes" + "flag" + "testing" +) + +func capture(fn func()) (string, bool, int) { + + oldOutput := flag.CommandLine.Output() + defer flag.CommandLine.SetOutput(oldOutput) + + var buffer bytes.Buffer + flag.CommandLine.SetOutput(&buffer) + + exitCh := make(chan int, 1) + Exit = func(exitCode int) { + exitCh <- exitCode + close(exitCh) + } + + fn() + + var exited bool + var exitCode int + select { + case exitCode = <-exitCh: + exited = true + default: + } + + return buffer.String(), exited, exitCode +} + +func TestParse(t *testing.T) { + + args := []string{"command", "-option", "hello"} + + var opt string + output, exited, _ := capture(func() { + Parse(args[0], args, func(flags *flag.FlagSet) { + flags.StringVar(&opt, "option", "", "an option") + }) + }) + + if e, a := "hello", opt; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := "", output; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := false, exited; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } +} + +func TestParseHelp(t *testing.T) { + + args := []string{"command", "-help"} + + var opt string + + output, exited, exitCode := capture(func() { + Parse(args[0], args, func(flags *flag.FlagSet) { + flags.StringVar(&opt, "option", "", "an option") + }) + }) + + if e, a := "", opt; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := "Usage of command:\n -option string\n \tan option\n -version\n \tShow version information\n", output; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := true, exited; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + if e, a := 0, exitCode; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } +} + +func TestParseVersion(t *testing.T) { + + args := []string{"command", "-version"} + + var opt string + + output, exited, exitCode := capture(func() { + Parse(args[0], args, func(flags *flag.FlagSet) { + flags.StringVar(&opt, "option", "", "an option") + }) + }) + + if e, a := "", opt; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := "", output; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := true, exited; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + if e, a := 0, exitCode; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } +} + +func TestParseError(t *testing.T) { + + args := []string{"command", "-undef"} + + var opt string + + output, exited, exitCode := capture(func() { + Parse(args[0], args, func(flags *flag.FlagSet) { + flags.StringVar(&opt, "option", "", "an option") + }) + }) + + if e, a := "", opt; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := "flag provided but not defined: -undef\nUsage of command:\n -option string\n \tan option\n -version\n \tShow version information\n", output; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := true, exited; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + if e, a := 1, exitCode; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } +} diff --git a/cmd/starter.go b/cmd/starter.go new file mode 100644 index 000000000..4523d6951 --- /dev/null +++ b/cmd/starter.go @@ -0,0 +1,106 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + + "golang.org/x/sys/unix" +) + +type Starter interface { + Start(context.Context) error + List() []Service +} +type Service interface { + Start(context.Context) error +} + +type Config interface { + Setup() (Starter, error) +} + +type starter struct { + services []*starterService +} +type starterService struct { + Service + errorCh chan error +} + +func NewStarter(services ...Service) Starter { + var s starter + + for _, service := range services { + starterService := &starterService{ + Service: service, + errorCh: make(chan error), + } + s.services = append(s.services, starterService) + } + return &s +} + +func (s *starter) Start(ctx context.Context) error { + + ctx, cancel := context.WithCancel(ctx) + + sigCh := make(chan os.Signal, 2) + signal.Notify(sigCh, os.Interrupt, unix.SIGTERM) + + go func() { + defer close(sigCh) + + sig := <-sigCh + + log.Printf("Signal %s received. Shutting down", sig.String()) + + cancel() + + sig = <-sigCh + + log.Printf("Signal %s received again. Force exiting", sig.String()) + + Exit(1) + }() + + for _, svc := range s.services { + var service = svc + + go func() { + defer close(service.errorCh) + + if err := service.Start(ctx); err != nil { + cancel() + service.errorCh <- err + } + }() + } + + <-ctx.Done() + + if err := ctx.Err(); err != context.Canceled { + return fmt.Errorf("context unexpectedly canceled: %w", err) + } + + for _, starter := range s.services { + if err := <-starter.errorCh; err != nil { + return fmt.Errorf("error running a service %T: %w", starter.Service, err) + } + } + + return nil +} + +func (s *starter) List() []Service { + var list []Service + for _, service := range s.services { + list = append(list, service.Service) + } + return list +} diff --git a/cmd/starter_test.go b/cmd/starter_test.go new file mode 100644 index 000000000..a625893b0 --- /dev/null +++ b/cmd/starter_test.go @@ -0,0 +1,215 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "bytes" + "context" + "errors" + "log" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +type mockService struct { + stopCh chan struct{} + error error +} + +func (m *mockService) Start(ctx context.Context) error { + + if m.error != nil { + return m.error + } + + if m.stopCh != nil { + <-m.stopCh + log.Printf("Killed") + return nil + } + + <-ctx.Done() + + if err := ctx.Err(); err != context.Canceled { + return errors.New("service canceled") + } + log.Printf("Shutted down") + + return nil +} + +func run(errCh chan error, exitCh chan struct{}, fn func() error) string { + defer close(errCh) + + Exit = func(_ int) { + close(exitCh) + } + + old := log.Writer() + defer func() { + log.SetOutput(old) + }() + + var buffer bytes.Buffer + log.SetOutput(&buffer) + log.SetFlags(0) + + err := fn() + + if err != nil { + errCh <- err + } + + return buffer.String() +} + +func kill(t *testing.T) { + t.Helper() + + if err := unix.Kill(unix.Getpid(), unix.SIGINT); err != nil { + t.Fatalf("Expect no error, got %#v", err) + } +} + +func TestStarter(t *testing.T) { + + ctx, cancel := context.WithCancel(context.Background()) + + exitCh := make(chan struct{}) + starter := NewStarter(&mockService{}, &mockService{stopCh: exitCh}) + if starter == nil { + t.Fatalf("Expect non nil, got %v", starter) + } + + errCh := make(chan error, 1) + var output string + + go func() { + defer cancel() + + output = run(errCh, exitCh, func() error { + return starter.Start(ctx) + }) + }() + + time.Sleep(time.Second) + + kill(t) + + time.Sleep(time.Second) + + kill(t) + + <-ctx.Done() + + if err := <-errCh; err != nil { + t.Fatalf("Expect no error, got %#v", err) + } + msg := "Signal interrupt received. Shutting down\nShutted down\nSignal interrupt received again. Force exiting\nKilled\n" + if e, a := msg, output; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } +} + +func prefix(errStr string) string { + i := strings.Index(errStr, ":") + return errStr[:i] +} + +func TestStarterWithError(t *testing.T) { + + ctx, cancel := context.WithCancel(context.Background()) + + serviceError := errors.New("failed to start") + starter := NewStarter(&mockService{}, &mockService{error: serviceError}) + if starter == nil { + t.Fatalf("Expect non nil, got %v", starter) + } + + exitCh := make(chan struct{}) + errCh := make(chan error, 1) + var output string + + go func() { + defer cancel() + + output = run(errCh, exitCh, func() error { + return starter.Start(ctx) + }) + }() + + <-ctx.Done() + err := <-errCh + + if e, a := "error running a service *cmd.mockService", prefix(err.Error()); e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := serviceError, errors.Unwrap(err); e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + if e, a := "Shutted down\n", output; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } +} + +func TestStarterWithTimeout(t *testing.T) { + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + + starter := NewStarter(&mockService{}) + if starter == nil { + t.Fatalf("Expect non nil, got %v", starter) + } + + exitCh := make(chan struct{}) + errCh := make(chan error, 1) + var output string + + go func() { + defer cancel() + + output = run(errCh, exitCh, func() error { + return starter.Start(ctx) + }) + }() + + <-ctx.Done() + err := <-errCh + + if e, a := "context unexpectedly canceled", prefix(err.Error()); e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + if e, a := context.DeadlineExceeded, errors.Unwrap(err); e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + if e, a := "", output; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } +} + +func TestStarterList(t *testing.T) { + + for _, services := range [][]Service{ + {}, + {&mockService{}}, + {&mockService{}, &mockService{}}, + } { + starter := NewStarter(services...) + if starter == nil { + t.Fatalf("Expect non nil, got %v", starter) + } + list := starter.List() + if e, a := len(services), len(list); e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + for i := range services { + if e, a := services[i], list[i]; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + } + } +} diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 000000000..cd537334f --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,25 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "fmt" + "runtime" +) + +var ( + version = "unknown" + gitCommit = "" +) + +func ShowVersion(programName string) { + + fmt.Printf("%s version %s\n", programName, version) + + if gitCommit != "" { + fmt.Printf("commit: %s\n", gitCommit) + } + + fmt.Printf("go: %s\n", runtime.Version()) +} diff --git a/cmd/version_test.go b/cmd/version_test.go new file mode 100644 index 000000000..dc5cd713e --- /dev/null +++ b/cmd/version_test.go @@ -0,0 +1,66 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "bytes" + "fmt" + "io" + "os" + "runtime" + "testing" +) + +func captureStdout(t *testing.T, fn func()) string { + + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("Expect no error, got %v", err) + } + defer r.Close() + + old := os.Stdout + os.Stdout = w + + fn() + + os.Stdout = old + w.Close() + + var src io.Reader = r + var dst bytes.Buffer + + if _, err := io.Copy(&dst, src); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + return dst.String() +} + +func TestShowVersion(t *testing.T) { + + programName := "test" + + version = "1.2.3" + gitCommit = "12345" + + output := captureStdout(t, func() { + ShowVersion(programName) + }) + + if e, a := fmt.Sprintf("test version 1.2.3\ncommit: 12345\ngo: %s\n", runtime.Version()), output; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + + version = "1.2.3" + gitCommit = "" + + output = captureStdout(t, func() { + ShowVersion(programName) + }) + + if e, a := fmt.Sprintf("test version 1.2.3\ngo: %s\n", runtime.Version()), output; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } +} diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 000000000..4f87fd0c3 --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,58 @@ +## Prerequisites +- make +- golang 1.18.y +- g++ + +## Clone the source code +``` +export BUILD_DIR=$HOME/remote-hyp +mkdir -p $BUILD_DIR && cd $BUILD_DIR + +git clone -b CCv0 https://github.com/kata-containers/kata-containers.git +git clone https://github.com/confidential-containers/cloud-api-adaptor.git +cd cloud-api-adaptor +``` + +## Build the binary + +Set `CLOUD_PROVIDER` variable to either `aws|azure|ibmcloud|libvirt` depending on your requirement. + +``` +export CLOUD_PROVIDER=aws +make +``` + +Note that libvirt go library uses `cgo` and hence there is no static build. + +Consequently you'll need to run the binary on the same OS/version where you have +built it. +You'll also need to install the libvirt dev packages before running the build. + +Example, if you are using Ubuntu then run the following command +``` +sudo apt-get install -y libvirt-dev +``` + +## Build Kata runtime and agent + +Install the prerequisites as mentioned in the following [link](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#requirements-to-build-individual-components) + +Install `protoc` +``` +wget -c https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip +sudo unzip protoc-3.11.4-linux-x86_64.zip -d /usr/local +``` + +Build the runtime + +``` +cd $BUILD_DIR/kata-containers/src/runtime +make +``` + +Build the agent + +``` +cd $BUILD_DIR/kata-containers/src/agent +make +``` diff --git a/docs/addnewprovider.md b/docs/addnewprovider.md new file mode 100644 index 000000000..f10501fe8 --- /dev/null +++ b/docs/addnewprovider.md @@ -0,0 +1,62 @@ +# :memo: Adding support for a new provider + +### Step 1: Add provider specific options + +Provider specific options goes under `cmd/cloud-api-adaptor` +- [Options](https://github.com/confidential-containers/cloud-api-adaptor/blob/staging/cmd/cloud-api-adaptor/main.go#L48) +- [Parsing](https://github.com/confidential-containers/cloud-api-adaptor/blob/staging/cmd/parse.go#L21) +- [Calling the specific provider](https://github.com/confidential-containers/cloud-api-adaptor/blob/staging/cmd/cloud-api-adaptor/main.go#L103) + + +### Step 2: Add provider specific code + +- The code goes under `pkg/adaptor/hypervisor/` +- Use BUILD TAGs to build the provider (eg // +build libvirt). + +:information_source: Note that there will be separate binaries for each provider. + +#### Step 2.1: Add provider entry point + +Add provider entry point to the registry. The `registry.newServer` method is the entry point for the provider code. + +:information_source: [Example code](https://github.com/confidential-containers/cloud-api-adaptor/blob/staging/pkg/adaptor/hypervisor/registry/libvirt.go) + +#### Step 2.2: Implement NewServer method + +The `NewServer` method creates the service which is responsible for VM lifecycle operations. + +Each provider implements `NewServer` method. + +By convention this should be in the file `/server.go` + +:information_source:[Example code](https://github.com/confidential-containers/cloud-api-adaptor/blob/staging/pkg/adaptor/hypervisor/libvirt/server.go#L36) + +#### Step 2.3: Implement NewService method + +Each provider implements `newService` method. + +By convention this should be in the file `/service.go` + +:information_source:[Example code](https://github.com/confidential-containers/cloud-api-adaptor/blob/staging/pkg/adaptor/hypervisor/libvirt/service.go#L44) + +#### Step 2.4: Implement Kata specific methods + +Add required methods + - CreateVM + - StartVM + - StopVM + - Version + +These methods are required by Kata and a Kata hypervisor needs to implement these methods. + +#### Step 2.5: Implement Kata specific methods + +Add additional files to modularize the code. + +See existing providers - `aws|azure|ibmcloud|libvirt` + +#### Step 3: Update Continuous Integration (CI) workflows + +Each provider should be built and tested on CI. + +Update the `provider` list under the `matrix` property in [`.github/workflows/build.yaml`](../.github/workflows/build.yaml). diff --git a/docs/architecture.png b/docs/architecture.png new file mode 100644 index 000000000..130781297 Binary files /dev/null and b/docs/architecture.png differ diff --git a/docs/registries-authentication.md b/docs/registries-authentication.md new file mode 100644 index 000000000..61e7073db --- /dev/null +++ b/docs/registries-authentication.md @@ -0,0 +1,29 @@ +## Container Image Registries Authentication + +To authenticate with private container image registry you are required to provide +[registry authentication file](https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md) to your podvm in order +to allow the image to be pulled directly. + +Registry authentication file can be provided either statically or at runtime. + + +### Statically embed authentication file in podvm image + +- `cd ~/cloud-api-adaptor/podvm/files/etc` +- Base64 your [auth.json](https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md), this can be done by doing `cat auth.json | base64 -w 0` +- Export the base64 encoded file `export AUTHFILE=` +- Create and Add the base64 encoded auth file into the `aa-offline_fs_kbc-resources.json` like so: +``` +cat < pod2pod-zvsi-2: ICMP echo request, id 22, seq 1, length 64 + pod2pod-zvsi-2 > 10.244.0.8: ICMP echo reply, id 22, seq 1, length 64 + 10.244.0.8 > pod2pod-zvsi-2: ICMP echo request, id 22, seq 2, length 64 + pod2pod-zvsi-2 > 10.244.0.8: ICMP echo reply, id 22, seq 2, length 64 + 10.244.0.8 > pod2pod-zvsi-2: ICMP echo request, id 22, seq 3, length 64 + pod2pod-zvsi-2 > 10.244.0.8: ICMP echo reply, id 22, seq 3, length 64 + 10.244.0.8 > pod2pod-zvsi-2: ICMP echo request, id 22, seq 4, length 64 + pod2pod-zvsi-2 > 10.244.0.8: ICMP echo reply, id 22, seq 4, length 64 + 10.244.0.8 > pod2pod-zvsi-2: ICMP echo request, id 22, seq 5, length 64 + pod2pod-zvsi-2 > 10.244.0.8: ICMP echo reply, id 22, seq 5, length 64 +02:20:26.231408 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 10.244.0.8 tell pod2pod-zvsi-2, length 28 +02:20:26.231579 ARP, Ethernet (len 6), IPv4 (len 4), Reply 10.244.0.8 is-at 02:00:0a:33:5d:23 (oui Unknown), length 46 + 10.244.0.8 > pod2pod-zvsi-2: ICMP echo request, id 22, seq 6, length 64 + pod2pod-zvsi-2 > 10.244.0.8: ICMP echo reply, id 22, seq 6, length 64 + 10.244.0.8 > pod2pod-zvsi-2: ICMP echo request, id 22, seq 7, length 64 + pod2pod-zvsi-2 > 10.244.0.8: ICMP echo reply, id 22, seq 7, length 64 + 10.244.0.8 > pod2pod-zvsi-2: ICMP echo request, id 22, seq 8, length 64 + pod2pod-zvsi-2 > 10.244.0.8: ICMP echo reply, id 22, seq 8, length 64 + 10.244.0.8 > pod2pod-zvsi-2: ICMP echo request, id 22, seq 9, length 64 + pod2pod-zvsi-2 > 10.244.0.8: ICMP echo reply, id 22, seq 9, length 64 + 10.244.0.8.44062 > pod2pod-zvsi-2.bgp: Flags [P.], cksum 0x4cee (correct), seq 2628078977:2628078996, ack 4013706653, win 128, options [nop,nop,TS val 515704189 ecr 607843040], length 19: BGP + pod2pod-zvsi-2.bgp > 10.244.0.8.44062: Flags [.], cksum 0x1620 (incorrect -> 0x75e6), seq 1, ack 19, win 128, options [nop,nop,TS val 607899138 ecr 515704189], length 0 +``` + +## Log dropped packets + +- Set iptables to log dropped packstes: +```bash +iptables -N LOGGING +iptables -A INPUT -j LOGGING +iptables -A OUTPUT -j LOGGING +iptables -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables-Dropped: " --log-level 4 +iptables -A LOGGING -j DROP +``` + +- Check dropping log: + +``` +# tail -f /var/log/syslog |grep IPTables-Dropped +Aug 29 08:39:59 pod2pod-zvsi-2 kernel: [365338.445299] IPTables-Dropped: IN= OUT=lo SRC=172.20.0.1 DST=172.20.0.1 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=52374 DF PROTO=TCP SPT=10909 DPT=2040 WINDOW=65535 RES=0x00 SYN URGP=0 +``` diff --git a/docs/vminfo.md b/docs/vminfo.md new file mode 100644 index 000000000..77fa64343 --- /dev/null +++ b/docs/vminfo.md @@ -0,0 +1,28 @@ +# VM information query service + +`cloud-api-adaptor` provides a query service for VM ID information via an Unix domain socket (default: `/run/peerpod/hypervisor.sock`). + +The query service is provided using [TTRPC](https://github.com/containerd/ttrpc), so you can implement a client to send a request via the Unix domain socket to query VM ID information using TTRPC. + +The TTRPC protocol definition is as follows. + +``` +service PodVMInfo { + rpc GetInfo(GetInfoRequest) returns (GetInfoResponse) {} +} + +message GetInfoRequest { + string PodName = 1; + string PodNamespace = 2; + + bool Wait = 3; +} + +message GetInfoResponse { + string VMID = 1; +} +``` + +You need to specify the pod name and namespace name of a pod running in a peer pod VM in a `GetInfo` request. The query service responds with a VM ID. The actual meaning of the VM ID value depends on the type of cloud provider. In the case of IBM Cloud, a VM ID is an ID of the virtual server instance (VSI). + +When you need to update the protocol definition, edit [`proto/podvminfo/podvminfo.proto`](/proto/podvminfo/podvminfo.proto), and run [`hack/update-proto.sh`](/hack/update-proto.sh). diff --git a/docs/vxlan-network-topology.md b/docs/vxlan-network-topology.md new file mode 100644 index 000000000..9f2d6deac --- /dev/null +++ b/docs/vxlan-network-topology.md @@ -0,0 +1,29 @@ + +## Network topology when using flannel + +![image](https://user-images.githubusercontent.com/9287071/203976500-d9d03718-fe7d-4dcd-ac6d-d082ded8001f.png) + +* Worker Node VM IP: 192.168.122.63 +* Pod VM IP: 192.168.122.253 +* Pod IP: 10.244.1.7 + + + +## Network topology when using Openshift SDN + +![image](https://user-images.githubusercontent.com/9287071/203976916-5860fa60-83a8-4c7c-a4ae-c0dd66ac06dc.png) + +* Worker Node VM IP: 192.168.10.110 +* Pod VM IP: 192.168.10.17 +* Pod IP: 10.132.2.73 + + + +## Network topology when using OVN-Kubernetes + +![image](https://user-images.githubusercontent.com/9287071/203977210-2b111177-003f-46e2-a0ed-9fd955586a92.png) + + +* Worker Node VM IP: 192.168.10.163 +* Pod VM IP: 192.168.10.201 +* Pod IP: 10.132.2.46 diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 000000000..a7fe10767 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +CLOUD_PROVIDER=${1:-$CLOUD_PROVIDER} +CRI_RUNTIME_ENDPOINT=${CRI_RUNTIME_ENDPOINT:-/run/peerpod/cri-runtime.sock} +optionals+="" + +# Ensure you add a space before the closing quote (") when updating the optionals +# example: +# following is the correct method: optionals+="-option val " +# following is the incorrect method: optionals+="-option val" + +[[ -S ${CRI_RUNTIME_ENDPOINT} ]] && optionals+="-cri-runtime-endpoint ${CRI_RUNTIME_ENDPOINT} " +[[ "${PAUSE_IMAGE}" ]] && optionals+="-pause-image ${PAUSE_IMAGE} " +[[ "${VXLAN_PORT}" ]] && optionals+="-vxlan-port ${VXLAN_PORT} " + +test_vars() { + for i in "$@"; do + [ -z "${!i}" ] && echo "\$$i is NOT set" && EXT=1 + done + [[ -n $EXT ]] && exit 1 +} + +aws() { +test_vars AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + +[[ "${PODVM_LAUNCHTEMPLATE_NAME}" ]] && optionals+="-use-lt -aws-lt-name ${PODVM_LAUNCHTEMPLATE_NAME} " # has precedence if set +[[ "${AWS_SG_IDS}" ]] && optionals+="-securitygroupids ${AWS_SG_IDS} " # MUST if template is not used +[[ "${PODVM_AMI_ID}" ]] && optionals+="-imageid ${PODVM_AMI_ID} " # MUST if template is not used +[[ "${PODVM_INSTANCE_TYPE}" ]] && optionals+="-instance-type ${PODVM_INSTANCE_TYPE} " # default t3.small +[[ "${SSH_KP_NAME}" ]] && optionals+="-keyname ${SSH_KP_NAME} " # if not retrieved from IMDS +[[ "${AWS_SUBNET_ID}" ]] && optionals+="-subnetid ${AWS_SUBNET_ID} " # if not set retrieved from IMDS +[[ "${AWS_REGION}" ]] && optionals+="-aws-region ${AWS_REGION} " # if not set retrieved from IMDS + +set -x +exec cloud-api-adaptor-aws aws \ + -aws-region "${AWS_REGION}" \ + -pods-dir /run/peerpod/pods \ + ${optionals} \ + -socket /run/peerpod/hypervisor.sock +} + +azure() { +test_vars AZURE_CLIENT_ID AZURE_CLIENT_SECRET AZURE_TENANT_ID + +set -x +exec cloud-api-adaptor-azure azure \ + -subscriptionid "${AZURE_SUBSCRIPTION_ID}" \ + -region "${AZURE_REGION}" \ + -instance-size "${AZURE_INSTANCE_SIZE}" \ + -resourcegroup "${AZURE_RESOURCE_GROUP}" \ + -vxlan-port 8472 \ + -subnetid "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AZURE_RESOURCE_GROUP}/providers/Microsoft.Network/virtualNetworks/${AZURE_VM_NAME}VNET/subnets/${AZURE_VM_NAME}Subnet" \ + -securitygroupid "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AZURE_RESOURCE_GROUP}/providers/Microsoft.Network/networkSecurityGroups/${AZURE_VM_NAME}NSG" \ + -imageid "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AZURE_RESOURCE_GROUP}/providers/Microsoft.Compute/images/${AZURE_IMAGE}" \ + ${optionals} +} + +ibmcloud() { +test_vars IBMCLOUD_API_KEY + +set -x +exec cloud-api-adaptor-ibmcloud ibmcloud \ + -iam-service-url "${IBMCLOUD_IAM_ENDPOINT}" \ + -vpc-service-url "${IBMCLOUD_VPC_ENDPOINT}" \ + -resource-group-id "${IBMCLOUD_RESOURCE_GROUP_ID}" \ + -key-id "${IBMCLOUD_SSH_KEY_ID}" \ + -image-id "${IBMCLOUD_PODVM_IMAGE_ID}" \ + -profile-name "${IBMCLOUD_PODVM_INSTANCE_PROFILE_NAME}" \ + -zone-name "${IBMCLOUD_ZONE}" \ + -primary-subnet-id "${IBMCLOUD_VPC_SUBNET_ID}" \ + -primary-security-group-id "${IBMCLOUD_VPC_SG_ID}" \ + -vpc-id "${IBMCLOUD_VPC_ID}" \ + -pods-dir /run/peerpod/pods \ + ${optionals} \ + -socket /run/peerpod/hypervisor.sock +} + +libvirt() { +test_vars LIBVIRT_URI + +set -x +exec cloud-api-adaptor-libvirt libvirt \ + -uri "${LIBVIRT_URI}" \ + -data-dir /opt/data-dir \ + -pods-dir /run/peerpod/pods \ + -network-name "${LIBVIRT_NET:-default}" \ + -pool-name "${LIBVIRT_POOL:-default}" \ + ${optionals} \ + -socket /run/peerpod/hypervisor.sock +} + +vsphere() { +test_vars GOVC_USERNAME GOVC_PASSWORD GOVC_URL GOVC_DATACENTER GOVC_DATASTORE + +[[ "${GOVC_TEMPLATE}" ]] && optionals+="-template ${GOVC_TEMPLATE} " +[[ "${GOVC_VCLUSTER}" ]] && optionals+="-vcluster ${GOVC_VCLUSTER} " +[[ "${GOVC_RESOURCE_POOL}" ]] && optionals+="-resource-pool ${GOVC_RESOURCE_POOL} " +[[ "${GOVC_FOLDER}" ]] && optionals+="-deploy-folder ${GOVC_FOLDER} " + +set -x +exec cloud-api-adaptor-vsphere vsphere \ + -vcenter-url ${GOVC_URL} \ + -data-center ${GOVC_DATACENTER} \ + -data-store ${GOVC_DATASTORE} \ + ${optionals} \ + -socket /run/peerpod/hypervisor.sock +} + +help_msg() { + cat < google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..7b4e45445 --- /dev/null +++ b/go.sum @@ -0,0 +1,2571 @@ +4d63.com/gochecknoglobals v0.1.0/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo= +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= +bitbucket.org/creachadair/shell v0.0.6/go.mod h1:8Qqi/cYk7vPnsOePHroKXDJYmb5x7ENhtiFtfZq8K+M= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.60.0/go.mod h1:yw2G51M9IfRboUH61Us8GqCeF1PzPblB823Mn2q2eAU= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.5.0/go.mod h1:ZEwJccE3z93Z2HWvstpri00jOg7oO4UZDtKhwDwqF0w= +cloud.google.com/go/spanner v1.7.0/go.mod h1:sd3K2gZ9Fd0vMPLXzeCrF6fq4i63Q7aTLW/lBIfBkIk= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5/go.mod h1:v4VVB6oBMz/c9fRY6vZrwr5xKRWOH5NPDjQZlPk0Gbs= +contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= +github.com/Antonboom/errname v0.1.5/go.mod h1:DugbBstvPFQbv/5uLcRRzfrNqKE9tVdVCqWCLp6Cifo= +github.com/Antonboom/nilnil v0.1.0/go.mod h1:PhHLvRPSghY5Y7mX4TW+BHZQYo1A8flE5H20D3IPZBo= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v66.0.0+incompatible h1:bmmC38SlE8/E81nNADlgmVGurPWMHDX2YNXVQMrBpEE= +github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2 h1:lneMk5qtUMulXa/eVxjVd+/bDYMEDIqYpLzLa2/EsNI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v3 v3.0.1 h1:H3g2mkmu105ON0c/Gqx3Bm+bzoIijLom8LmV9Gjn7X0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v3 v3.0.1/go.mod h1:EAc3kjhZf9soch7yLID8PeKcE6VfKvQTllSBHYVdXd8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= +github.com/IBM/go-sdk-core/v4 v4.5.1/go.mod h1:lTUXbqIX6/aAbSCkP6q59+dyFsTwZAc0ewRS2vJWVbg= +github.com/IBM/go-sdk-core/v5 v5.2.0/go.mod h1:vyNdbFujJtdTj9HbihtvKwwS3k/GKSKpOx9ZIQ6MWDY= +github.com/IBM/go-sdk-core/v5 v5.6.3 h1:xj44tzfBnjGJXKthMeaIn9wcYH+z4dXTuhntsBp+NdQ= +github.com/IBM/go-sdk-core/v5 v5.6.3/go.mod h1:tt/B9rxLkRtglE7pvqLuYikgCXaZFL3btdruJaoUeek= +github.com/IBM/vpc-go-sdk v1.0.1 h1:D2cu4KRsM8Q8bLWz/uxp8m7nzUm33mcgDv1sD0w/E8M= +github.com/IBM/vpc-go-sdk v1.0.1/go.mod h1:bhd7r482lV30UJz46r2oRgYGawGEo+TuS41ZLIY65y0= +github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.0.3/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0= +github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= +github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= +github.com/ProtonMail/go-crypto v0.0.0-20220517143526-88bb52951d5b/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/ReneKroon/ttlcache/v2 v2.11.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= +github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= +github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/ashanbrown/forbidigo v1.2.0/go.mod h1:vVW7PEdqEFqapJe95xHkTfB1+XvZXBFg8t0sG2FIxmI= +github.com/ashanbrown/makezero v0.0.0-20210520155254-b6261585ddde/go.mod h1:oG9Dnez7/ESBqc4EdrdNlryeo7d0KcW1ftXHm7nU/UU= +github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aws/aws-sdk-go v1.36.30/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.44/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v1.15.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI= +github.com/aws/aws-sdk-go-v2 v1.16.5 h1:Ah9h1TZD9E2S1LzHpViBO3Jz9FPL5+rmflmb8hXirtI= +github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= +github.com/aws/aws-sdk-go-v2/config v1.15.11 h1:qfec8AtiCqVbwMcx51G1yO2PYVfWfhp2lWkDH65V9HA= +github.com/aws/aws-sdk-go-v2/config v1.15.11/go.mod h1:mD5tNFciV7YHNjPpFYqJ6KGpoSfY107oZULvTHIxtbI= +github.com/aws/aws-sdk-go-v2/credentials v1.12.6 h1:No1wZFW4bcM/uF6Tzzj6IbaeQJM+xxqXOYmoObm33ws= +github.com/aws/aws-sdk-go-v2/credentials v1.12.6/go.mod h1:mQgnRmBPF2S/M01W4T4Obp3ZaZB6o1s/R8cOUda9vtI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 h1:+NZzDh/RpcQTpo9xMFUgkseIam6PC+YJbdhbQp1NOXI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6/go.mod h1:ClLMcuQA/wcHPmOIfNzNI4Y1Q0oDbmEkbYhMFOzHDh8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.6/go.mod h1:SSPEdf9spsFgJyhjrXvawfpyzrXHBCUe+2eQ1CjC1Ak= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 h1:Zt7DDk5V7SyQULUUwIKzsROtVzp/kVvcz15uQx/Tkow= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.0/go.mod h1:viTrxhAuejD+LszDahzAE2x40YjYWhMqzHxv2ZiWaME= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 h1:eeXdGVtXEe+2Jc49+/vAzna3FAQnUD4AagAw8tzbmfc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 h1:L/l0WbIpIadRO7i44jZh1/XeXpNDX0sokFppb4ZnXUI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13/go.mod h1:hiM/y1XPp3DoEPhoVEYc/CZcS58dP6RKJRDFp99wdX0= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.31.0 h1:8Q6WYZvgBylP7kh7S5gQvGKpsjphcZPEuytwl/JxxoA= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.31.0/go.mod h1:Z8942YP2VgLQpgPCx06iXCrOt7mxxCe0dESCm9FFhgs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.0/go.mod h1:R31ot6BgESRCIoxwfKtIHzZMo/vsZn2un81g9BJ4nmo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 h1:0ZxYAZ1cn7Swi/US55VKciCE6RhRHIwCKIWaMLdT6pg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6/go.mod h1:DxAPjquoEHf3rUHh1b9+47RAaXB8/7cB6jkzCt/GOEI= +github.com/aws/aws-sdk-go-v2/service/kms v1.17.3/go.mod h1:EKkrWWXwWYf8x3Nrm6Oix3zZP9NRBHqxw5buFGVBHA0= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 h1:Gju1UO3E8ceuoYc/AHcdXLuTZ0WGE1PT2BYDwcYhJg8= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.9/go.mod h1:UqRD9bBt15P0ofRyDZX6CfsIqPpzeHOhZKWzgSuAzpo= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 h1:HLzjwQM9975FQWSF3uENDGHT1gFQm/q3QXu2BYIcI08= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBFtDzZGSUCv6qaNOyEGCJ1AY= +github.com/aws/smithy-go v1.11.1/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= +github.com/aws/smithy-go v1.11.3 h1:DQixirEFM9IaKxX1olZ3ke3nvxRS2xMDteKIDWxozW8= +github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/beeker1121/goque v1.0.3-0.20191103205551-d618510128af/go.mod h1:84CWnaDz4g1tEVnFLnuBigmGK15oPohy0RfvSN8d4eg= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbzot9mhmeI= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/blizzy78/varnamelen v0.3.0/go.mod h1:hbwRdBvoBqxk34XyQ6HA0UH3G0/1TKuv5AC4eaBT0Ec= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bombsimon/wsl/v3 v3.3.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= +github.com/breml/bidichk v0.1.1/go.mod h1:zbfeitpevDUGI7V91Uzzuwrn4Vls8MoBMrwtt78jmso= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/butuzov/ireturn v0.1.1/go.mod h1:Wh6Zl3IMtTpaIKbmwzqi6olnM9ptYQxxVacMsOEFPoc= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charithe/durationcheck v0.0.9/go.mod h1:SSbRIBVfMjCi/kEB6K65XEA83D6prSM8ap1UCpNKtgg= +github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af/go.mod h1:Qjyv4H3//PWVzTeCezG2b9IRn6myJxJSr4TD/xo6ojU= +github.com/checkpoint-restore/checkpointctl v0.0.0-20220321135231-33f4a66335f0/go.mod h1:67kWC1PXQLR3lM/mmNnu3Kzn7K4TSWZAGUuQP1JSngk= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/checkpoint-restore/go-criu/v5 v5.2.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= +github.com/container-orchestrated-devices/container-device-interface v0.4.0/go.mod h1:E1zcucIkq9P3eyNmY+68dBQsTcsXJh9cgRo2IVNScKQ= +github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= +github.com/containerd/cgroups v1.0.5-0.20220625035431-cf7417bca682/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= +github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= +github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= +github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= +github.com/containerd/containerd v1.6.6 h1:xJNPhbrmz8xAMDNoVjHy9YHtWwEQNS+CDkcIRh7t8Y0= +github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= +github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= +github.com/containerd/cri-containerd v1.19.0/go.mod h1:wxbGdReWGCalzGOEpifoHeYCK4xAgnj4o/4bVB+9voU= +github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= +github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= +github.com/containerd/go-cni v1.1.6/go.mod h1:BWtoWl5ghVymxu6MBjg79W9NZrCRyHIdUtk4cauMe34= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= +github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo= +github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= +github.com/containerd/stargz-snapshotter/estargz v0.9.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0= +github.com/containerd/stargz-snapshotter/estargz v0.10.1/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0= +github.com/containerd/stargz-snapshotter/estargz v0.11.4/go.mod h1:7vRJIcImfY8bpifnMjt+HTJoQxASq7T28MYbP15/Nf0= +github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= +github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI= +github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= +github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= +github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= +github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= +github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNGz0C1d3wVYlHE= +github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= +github.com/containers/buildah v1.27.0/go.mod h1:anH3ExvDXRNP9zLQCrOc1vWb5CrhqLF/aYFim4tslvA= +github.com/containers/common v0.49.1/go.mod h1:ueM5hT0itKqCQvVJDs+EtjornAQtrHYxQJzP2gxeGIg= +github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= +github.com/containers/image/v5 v5.22.0/go.mod h1:D8Ksv2RNB8qLJ7xe1P3rgJJOSQpahA6amv2Ax++/YO4= +github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= +github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g= +github.com/containers/ocicrypt v1.1.5/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc= +github.com/containers/podman/v4 v4.2.0 h1:mqQ0CtdSOTfsl6IEcO0UiA/Yi/9Yxoe/mSEC4h21CK8= +github.com/containers/podman/v4 v4.2.0/go.mod h1:sUxBZd/VXjXXsHuQURWaQaetb0Ugqd1C6y7uScgmf4o= +github.com/containers/psgo v1.7.2/go.mod h1:SLpqxsPOHtTqRygjutCPXmeU2PoEFzV3gzJplN4BMx0= +github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4= +github.com/containers/storage v1.38.0/go.mod h1:lBzt28gAk5ADZuRtwdndRJyqX22vnRaXmlF+7ktfMYc= +github.com/containers/storage v1.42.0/go.mod h1:JiUJwOgOo1dr2DdOUc1MRe2GCAXABYoYmOdPF8yvH78= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= +github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc/v3 v3.2.0/go.mod h1:rEJ/idjfUyfkBit1eI1fvyr+64/g9dcKpAm8MJMesvo= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= +github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/daixiang0/gci v0.2.9/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc= +github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/digitalocean/go-libvirt v0.0.0-20201209184759-e2a69bcd5bd1/go.mod h1:QS1XzqZLcDniNYrN7EZefq3wIyb/M2WmJbql4ZKoc1Q= +github.com/digitalocean/go-qemu v0.0.0-20210326154740-ac9e0b687001/go.mod h1:IetBE52JfFxK46p2n2Rqm+p5Gx1gpu2hRHsrbnPOWZQ= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/disiqueira/gotree/v3 v3.0.2/go.mod h1:ZuyjE4+mUQZlbpkI24AmruZKhg3VHEgPLDY8Qk+uUu8= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.16+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= +github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.16+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11/go.mod h1:a6bNUGTbQBsY6VRHTr4h/rkOXjl244DyRD0tx3fgq4Q= +github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/go-plugins-helpers v0.0.0-20211224144127-6eecb7beb651/go.mod h1:LFyLie6XcDbyKGeVK6bHe+9aJTYCxWLBg5IrJZOaXKA= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dougm/pretty v0.0.0-20171025230240-2ee9d7453c02/go.mod h1:7NQ3kWOx2cZOSjtcveTa5nqupVr2s6/83sG+rTlI7uA= +github.com/dtylman/scp v0.0.0-20181017070807-f3000a34aef4/go.mod h1:jN1ZaUPSNA8jm10nmaRLky84qV/iCeiHmcEf3EbP+dc= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/esimonov/ifshort v1.0.3/go.mod h1:yZqNJUrNn20K8Q9n2CrjTKYyVEmX209Hgu+M1LBpeZE= +github.com/ettle/strcase v0.1.1/go.mod h1:hzDLsPC7/lwKyBOywSHEP89nt2pDgdy+No1NBA9o9VY= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.5.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01/go.mod h1:ypD5nozFk9vcGw1ATYefw6jHe/jZP++Z15/+VTMcWhc= +github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52/go.mod h1:yIquW87NGRw1FU5p5lEkpnt/QxoH5uPAOUlOVkAUuMg= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:PjfxuH4FZdUyfMdtBio2lsRr1AKEaVPwelzuHuh8Lqc= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-docopt v0.0.0-20140912013429-f6dd2ebbb31e/go.mod h1:HyVoz1Mz5Co8TFO8EupIdlcpwShBmY98dkT2xeHkvEI= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= +github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsouza/go-dockerclient v1.7.7/go.mod h1:njNCXvoZj3sLPjf3yO0DPHf1mdLdCPDYPc14GskKA4Y= +github.com/fsouza/go-dockerclient v1.8.1/go.mod h1:zmA2ogSxRnXmbZcy0Aq7yhRoCdP/bDns/qghCK9SWtM= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= +github.com/fullstorydev/grpcurl v1.6.0/go.mod h1:ZQ+ayqbKMJNhzLmbpCiurTVlaK2M/3nqZCxaQ2Ze/sM= +github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E= +github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.7.1/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-critic/go-critic v0.6.1/go.mod h1:SdNCfU0yF3UBjtaZGw6586/WocupMOJuiqgom5DsQxM= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gorp/gorp/v3 v3.0.2/go.mod h1:BJ3q1ejpV8cVALtcXvXaXyTOlMmJhWDxTmncaR6rwBY= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ini/ini v1.28.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-ldap/ldap/v3 v3.1.10/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= +github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.2 h1:dxy7PGTqEh94zj2E3h1cUmQQWiM1+aeCROfAr02EmK8= +github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= +github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= +github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= +github.com/go-openapi/runtime v0.19.21/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.10/go.mod h1:qBBipho+3EoIqn6YDI+4RnQEtj6jT/IdKm+PAlXxSUc= +github.com/go-openapi/strfmt v0.20.1/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= +github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-openapi/strfmt v0.21.1 h1:G6s2t5V5kGCHLVbSdZ/6lI8Wm4OzoPFkc3/cjAsKQrM= +github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= +github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= +github.com/go-openapi/validate v0.22.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-rod/rod v0.107.3/go.mod h1:4SqYRUrcc4dSr9iT36YRZ4hdUAPg3A0O8RhxAMh0eCQ= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.1/go.mod h1:4oGA3EZXTVItV/ipGiOx7NWkY5veFfcsOJVS2YxltLw= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/attrs v0.1.0/go.mod h1:fmNpaWyHM0tRm8gCZWKx8yY9fvaNLo2PyzBNSrBZ5Hw= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.8.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= +github.com/gobuffalo/envy v1.9.0/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= +github.com/gobuffalo/fizz v1.10.0/go.mod h1:J2XGPO0AfJ1zKw7+2BA+6FEGAkyEsdCOLvN93WCT2WI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gobuffalo/flect v0.2.1/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/genny/v2 v2.0.5/go.mod h1:kRkJuAw9mdI37AiEYjV4Dl+TgkBDYf8HZVjLkqe5eBg= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/github_flavored_markdown v1.1.0/go.mod h1:TSpTKWcRTI0+v7W3x8dkSKMLJSUpuVitlptCkpeY8ic= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/helpers v0.6.0/go.mod h1:pncVrer7x/KRvnL5aJABLAuT/RhKRR9klL6dkUOhyv8= +github.com/gobuffalo/helpers v0.6.1/go.mod h1:wInbDi0vTJKZBviURTLRMFLE4+nF2uRuuL2fnlYo7w4= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/logger v1.0.3/go.mod h1:SoeejUwldiS7ZsyCBphOGURmWdwUFXs0J7TCjEhjKxM= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/nulls v0.2.0/go.mod h1:w4q8RoSCEt87Q0K0sRIZWYeIxkxog5mh3eN3C/n+dUc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= +github.com/gobuffalo/packd v1.0.0/go.mod h1:6VTc4htmJRFB7u1m/4LeMTWjFoYrUiBkU9Fdec9hrhI= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/packr/v2 v2.8.0/go.mod h1:PDk2k3vGevNE3SwVyVRgQCCXETC9SaONCNSXT1Q8M1g= +github.com/gobuffalo/plush/v4 v4.0.0/go.mod h1:ErFS3UxKqEb8fpFJT7lYErfN/Nw6vHGiDMTjxpk5bQ0= +github.com/gobuffalo/pop/v5 v5.3.1/go.mod h1:vcEDhh6cJ3WVENqJDFt/6z7zNb7lLnlN8vj3n5G9rYA= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gobuffalo/tags/v3 v3.0.2/go.mod h1:ZQeN6TCTiwAFnS0dNcbDtSgZDwNKSpqajvVtt6mlYpA= +github.com/gobuffalo/tags/v3 v3.1.0/go.mod h1:ZQeN6TCTiwAFnS0dNcbDtSgZDwNKSpqajvVtt6mlYpA= +github.com/gobuffalo/validate/v3 v3.0.0/go.mod h1:HFpjq+AIiA2RHoQnQVTFKF/ZpUPXwyw82LgyDPxQ9r0= +github.com/gobuffalo/validate/v3 v3.1.0/go.mod h1:HFpjq+AIiA2RHoQnQVTFKF/ZpUPXwyw82LgyDPxQ9r0= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.43.0/go.mod h1:VIFlUqidx5ggxDfQagdvd9E67UjMXtTHBkBQ7sHoC5Q= +github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.3.5/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/revgrep v0.0.0-20210930125155-c22e5001d4f2/go.mod h1:LK+zW4MpyytAWQRz0M4xnzEk50lSvqDQKfx304apFkY= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= +github.com/google/certificate-transparency-go v1.0.22-0.20181127102053-c25855a82c75/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= +github.com/google/certificate-transparency-go v1.1.1/go.mod h1:FDKqPvSXawb2ecErVRrD+nfy23RCzyl7eqVCEmlT1Zs= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= +github.com/google/go-containerregistry v0.10.0/go.mod h1:C7uwbB1QUAtvnknyd3ethxJRd4gtEjU/9WLXzckfI1Y= +github.com/google/go-intervals v0.0.2/go.mod h1:MkaR3LNRfeKLPmqgJYs4E66z5InYjmCjbbr4TQlcT6Y= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/trillian v1.3.11/go.mod h1:0tPraVHrSDkA3BO6vKX67zgLXs6SsOAbHEivX+9mPgw= +github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= +github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254/go.mod h1:M9mZEtGIsR1oDaZagNPNG9iq9n2HrhZ17dsXk73V3Lw= +github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= +github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gostaticanalysis/analysisutil v0.1.0/go.mod h1:dMhHRU9KTiDcuLGdy87/2gTR8WruwYZrKdRq9m1O6uw= +github.com/gostaticanalysis/analysisutil v0.4.1/go.mod h1:18U/DLpRgIUd459wGxVHE0fRgmo1UgHDcbw7F5idXu0= +github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc= +github.com/gostaticanalysis/comment v1.3.0/go.mod h1:xMicKDx7XRXYdVwY9f9wQpDJVnqWxw9wCauCMKp+IBI= +github.com/gostaticanalysis/comment v1.4.1/go.mod h1:ih6ZxzTHLdadaiSnF5WY3dxUoXfXAlTaRzuaNDlSado= +github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM= +github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5/go.mod h1:qZEedyP/sY1lTGV1uJ3VhWZ2mqag3IkWsDHVbplHXak= +github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW0HU0GPE3+5PWN4A= +github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= +github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.1.0 h1:QsGcniKx5/LuX2eYoeL+Np3UKYPNaN7YKpTh29h8rbw= +github.com/hashicorp/go-hclog v1.1.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= +github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/base62 v0.1.1/go.mod h1:EdWO6czbmthiwZ3/PUsDV+UD1D5IRU4ActiaWGwt0Yw= +github.com/hashicorp/go-secure-stdlib/mlock v0.1.1/go.mod h1:zq93CJChV6L9QTfGKtfBxKqD7BqqXx5O04A/ns2p5+I= +github.com/hashicorp/go-secure-stdlib/mlock v0.1.2/go.mod h1:zq93CJChV6L9QTfGKtfBxKqD7BqqXx5O04A/ns2p5+I= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.1/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/password v0.1.1/go.mod h1:9hH302QllNwu1o2TGYtSk8I8kTAN0ca1EHpwhm5Mmzo= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.1/go.mod h1:l8slYwnJA26yBz+ErHpp2IRCLr0vuOMGBORIz4rRiAs= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/vault/api v1.7.2/go.mod h1:xbfA+1AvxFseDzxxdWaL0uO99n1+tndus4GCrtouy0M= +github.com/hashicorp/vault/sdk v0.5.1/go.mod h1:DoGraE9kKGNcVgPmTuX357Fm6WAx1Okvde8Vp3dPDoU= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/honeycombio/beeline-go v1.1.1/go.mod h1:kN0cfUGBMfA87DyCYbiiLoSzWsnw3bluZvNEWtatHxk= +github.com/honeycombio/libhoney-go v1.15.2/go.mod h1:JzhRPYgoBCd0rZvudrqmej4Ntx0w7AT3wAJpf5+t1WA= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= +github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/insomniacslk/dhcp v0.0.0-20220119180841-3c283ff8b7dd/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E= +github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4= +github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.6.0/go.mod h1:yeseQo4xhQbgyJs2c87RAXOH2i624N0Fh1KSPJya7qo= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jgautheron/goconst v1.5.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= +github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= +github.com/jhump/protoreflect v1.6.1/go.mod h1:RZQ/lnuN+zqeRVpQigTwO6o0AJUkxbnSnpuG7toUTG4= +github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/josharian/txtarfs v0.0.0-20210218200122-0702f000015a/go.mod h1:izVPOvVRsHiKkeGCT6tYBNWyDVuzj9wAaBb5R9qamfw= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= +github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok= +github.com/jsimonetti/rtnetlink v0.0.0-20201110080708-d2c240429e6c/go.mod h1:huN4d1phzjhlOsNIjFsw2SVRbwIHj3fJDMEU2SDPTmg= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/julz/importas v0.0.0-20210419104244-841f0c0fe66d/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/karrick/godirwalk v1.15.3/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kata-containers/kata-containers v0.0.0-20220913141151-9b49a6ddc6fd h1:qU3kPJB1cBwOwmuYbUnksH+0mTgDsqunPF1kXaW+4r4= +github.com/kata-containers/kata-containers v0.0.0-20220913141151-9b49a6ddc6fd/go.mod h1:tXmBXaVnZkUP+T6WQrFvfZgZfp17OqxdaDtqMOGUBDM= +github.com/kata-containers/kata-containers/src/runtime v0.0.0-20220913141151-9b49a6ddc6fd h1:lfWVfG2l+KjhqYm/wcpPo+eN6KNPnzioncD58oONX3E= +github.com/kata-containers/kata-containers/src/runtime v0.0.0-20220913141151-9b49a6ddc6fd/go.mod h1:06RnSkAG8aNZF9GUJMH09b39U+mo0cJPJkLCH8RSXK0= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.14.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kulti/thelper v0.4.0/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U= +github.com/kunwardeep/paralleltest v1.0.3/go.mod h1:vLydzomDFpk7yu5UX02RmP0H8QfRPOV/oFhWN85Mjb4= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg= +github.com/labstack/echo/v4 v4.3.0/go.mod h1:PvmtTvhVqKDzDQy4d3bWzPjZLzom4iQbAZy2sgZ/qI8= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/ldez/gomoddirectives v0.2.2/go.mod h1:cpgBogWITnCfRq2qGoDkKMEVSaarhdBr6g8G04uz6d0= +github.com/ldez/tagliatelle v0.2.0/go.mod h1:8s6WJQwEYHbKZDsp/LjArytKOG8qaMrKQQ3mFukHs88= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/letsencrypt/boulder v0.0.0-20220331220046-b23ab962616e/go.mod h1:Bl3mfF2LHYepsU2XfzMceIglyByfPe1IFAXtO+p37Qk= +github.com/letsencrypt/challtestsrv v1.2.1/go.mod h1:Ur4e4FvELUXLGhkMztHOsPIsvGxD/kzSJninOrkM+zc= +github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJiIbETBPTl9ATXQag= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/luna-duclos/instrumentedsql v1.1.3/go.mod h1:9J1njvFds+zN7y85EDhN9XNQLANWwZt2ULeIC8yMNYs= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= +github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= +github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matoous/godox v0.0.0-20210227103229-6504466cf951/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc= +github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y= +github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= +github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o= +github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= +github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= +github.com/mdlayher/socket v0.2.0/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= +github.com/mdlayher/vsock v1.1.0/go.mod h1:nsVhPsVuBBwAKh6i6PzdNoke6/TNYTjkxoRKAp/+pXs= +github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= +github.com/mgechev/revive v1.1.2/go.mod h1:bnXsMr+ZTH09V5rssEI+jHAZ4z+ZdyhgO/zsy3EhK+0= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= +github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/vpnkit v0.5.0/go.mod h1:KyjUrL9cb6ZSNNAUwZfqRjhwwgJ3BJN+kXh0t43WTUQ= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mozilla/scribe v0.0.0-20180711195314-fb71baf557c1/go.mod h1:FIczTrinKo8VaLxe6PWTPEXRXDIHz2QAwiaBaP5/4a8= +github.com/mozilla/tls-observatory v0.0.0-20210609171429-7bc42856d2e5/go.mod h1:FUqVoUPHSEdDR0MnFM3Dh8AU0pZHLXUD127SAJGER/s= +github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo= +github.com/mwitkow/go-proto-validators v0.2.0/go.mod h1:ZfA1hW+UH/2ZHOWvQ3HnQaU0DtnpXu850MZiy+YUgcc= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= +github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nishanths/exhaustive v0.2.3/go.mod h1:bhIX678Nx8inLM9PbpvK1yv6oGtoP8BfaIeMzgBNKvc= +github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62PewwiQTlm/7Rj+cxVYqZvDIUc+JjZq6GHAC1fsObQ= +github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= +github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198/go.mod h1:j4h1pJW6ZcJTgMZWP3+7RlG3zTaP02aDZ/Qw0sppK7Q= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.0.3/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab h1:YQZXa3elcHgKXAa2GjVFC9M3JeP7ZPyFD1YByDx/dgQ= +github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/runtime-tools v0.0.0-20190417131837-cd1349b7c47e/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/runtime-tools v0.9.1-0.20220714195903-17b3287fafb7/go.mod h1:/tgP02fPXGHkU3/qKK1Y0Db4yqNyGm03vLq/mzHzcS4= +github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opencontainers/selinux v1.8.5/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= +github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/openshift/imagebuilder v1.2.4-0.20220711175835-4151e43600df/go.mod h1:TRYHe4CH9U6nkDjxjBNM5klrLbJBrRbpJE5SaRwUBsQ= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= +github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= +github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polyfloyd/go-errorlint v0.0.0-20210722154253-910bb7978349/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= +github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/pseudomuto/protoc-gen-doc v1.3.2/go.mod h1:y5+P6n3iGrbKG+9O04V5ld71in3v/bX88wUwgt+U8EA= +github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q= +github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= +github.com/quasilyte/go-ruleguard v0.3.1-0.20210203134552-1b5a410e1cc8/go.mod h1:KsAh3x0e7Fkpgs+Q9pNLS5XpFSvYCEVl5gP9Pp1xp30= +github.com/quasilyte/go-ruleguard v0.3.13/go.mod h1:Ul8wwdqR6kBVOCt2dipDBkE+T6vAV/iixkrKuRTN1oQ= +github.com/quasilyte/go-ruleguard/dsl v0.3.0/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/go-ruleguard/dsl v0.3.10/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/go-ruleguard/rules v0.0.0-20201231183845-9e62ed36efe1/go.mod h1:7JTjp89EGyU1d6XfBiXihJNG37wB2VRkd125Q1u7Plc= +github.com/quasilyte/go-ruleguard/rules v0.0.0-20210428214800-545e0d2e0bf7/go.mod h1:4cgAphtvu7Ftv7vOT2ZOYhC6CvBxZixcasr8qIOTA50= +github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= +github.com/rasky/go-xdr v0.0.0-20170217172119-4930550ba2e2/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451 h1:d1PiN4RxzIFXCJTvRkvSkKqwtRAl5ZV4lATKtQI0B7I= +github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rootless-containers/rootlesskit v1.0.1/go.mod h1:t2UAiYagxrJ+wmpFAUIZPcqsm4k2B7ve6g7lILKbloc= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryancurrah/gomodguard v1.2.3/go.mod h1:rYbA/4Tg5c54mV1sv4sQTP5WOPBcoLtnBZ7/TEhXAbg= +github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= +github.com/sanposhiho/wastedassign/v2 v2.0.6/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/secure-systems-lab/go-securesystemslib v0.3.1/go.mod h1:o8hhjkbNl2gOamKUA/eNW3xUrntHT9L4W89W1nfj43U= +github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs= +github.com/securego/gosec/v2 v2.9.1/go.mod h1:oDcDLcatOJxkCGaCaq8lua1jTnYf6Sou4wdiJ1n4iHc= +github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= +github.com/shirou/gopsutil/v3 v3.21.10/go.mod h1:t75NhzCZ/dYyPQjyQmrAYP6c8+LCdFANeBMdLPCNnew= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sigstore/sigstore v1.3.1-0.20220629021053-b95fc0d626c1/go.mod h1:y83NePRM98MJpbGgBgi54UZduhG0aD7lYngAVCx+i/E= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sivchari/tenv v1.4.7/go.mod h1:5nF+bITvkebQVanjU6IuMbvIot/7ReNsUV7I5NbprB0= +github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= +github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/sylabs/sif/v2 v2.7.1/go.mod h1:bBse2nEFd3yHkmq6KmAOFEWQg5LdFYiQUdVcgamxlc8= +github.com/sylvia7788/contextcheck v1.0.4/go.mod h1:vuPKJMQ7MQ91ZTqfdyreNKwZjyUg6KO+IebVyQDedZQ= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= +github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= +github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= +github.com/tetafro/godot v1.4.11/go.mod h1:LR3CJpxDVGlYOWn3ZZg1PgNZdTUvzsZWu8xaEohUpn8= +github.com/theupdateframework/go-tuf v0.3.0/go.mod h1:E5XP0wXitrFUHe4b8cUcAAdxBW4LbfnqF4WXXGLgWNo= +github.com/theupdateframework/go-tuf v0.3.1/go.mod h1:lhHZ3Vt2pdAh15h0Cc6gWdlI+Okn2ZznD3q/cNjd5jw= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tomarrell/wrapcheck/v2 v2.4.0/go.mod h1:68bQ/eJg55BROaRTbMjC7vuhL2OgfoG8bLp9ZyoBfyY= +github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= +github.com/tommy-muehle/go-mnd/v2 v2.4.0/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/u-root/uio v0.0.0-20210528114334-82958018845c/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.5.1/go.mod h1:oDzoM7pVwz6wHn5ogWgFUU1s4VJayeQS+aEZDqXIEJs= +github.com/uudashr/gocognit v1.0.5/go.mod h1:wgYz0mitoKOTysqxTDMOUXg+Jb5SvtihkfmugIZYpEA= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/VPSJnLYn+LmLk8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= +github.com/vbauerster/mpb/v7 v7.4.2/go.mod h1:UmOiIUI8aPqWXIps0ciik3RKMdzx7+ooQpq+fBcXwBA= +github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= +github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20220115184804-dd687eb2f2d4 h1:fB26rIBlWTVJyEB6ONHdoEvUbvwoudH0/cMEXHiD1RU= +github.com/vishvananda/netlink v1.1.1-0.20220115184804-dd687eb2f2d4/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmware/govmomi v0.29.0 h1:SHJQ7DUc4fltFZv16znJNGHR1/XhiDK5iKxm2OqwkuU= +github.com/vmware/govmomi v0.29.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/weppos/publicsuffix-go v0.15.1-0.20210807195340-dc689ff0bb59/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= +github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= +github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yeya24/promlinter v0.1.0/go.mod h1:rs5vtZzeBHqqMwXqFScncpCF6u06lezhZepno9AB1Oc= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= +github.com/ysmood/got v0.31.2/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= +github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= +github.com/zmap/zcrypto v0.0.0-20210811211718-6f9bc4aff20f/go.mod h1:y/9hjFEub4DtQxTHp/pqticBgdYeCwL97vojV3lsvHY= +github.com/zmap/zlint/v3 v3.3.1-0.20211019173530-cb17369b4628/go.mod h1:O+4OXRfNLKqOyDl4eKZ1SBlYudKGUBGRFcv+m1KLr28= +gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965/go.mod h1:TBB3sR7/jg4RCThC/cgT4fB8mAbbMO307TycfgeR59w= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd v0.0.0-20200513171258-e048e166ab9c/go.mod h1:xCI7ZzBfRuGgBXyXO6yfWfDmlWd35khcWpUa4L0xI/k= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.4.2/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.mongodb.org/mongo-driver v1.7.5 h1:ny3p0reEpgsR2cfA5cjgwFZg3Cv/ofFh/8jbhGtz9VI= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.mozilla.org/mozlog v0.0.0-20170222151521-4bb13139d403/go.mod h1:jHoPAGnDrCy6kaI2tAze5Prf0Nr0w/oNkROt2lw3n3o= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/contrib/propagators v0.19.0/go.mod h1:4QOdZClXISU5S43xZxk5tYaWcpb+lehqfKtE6PK6msE= +go.opentelemetry.io/otel v0.19.0/go.mod h1:j9bF567N9EfomkSidSfmMwIwIBuP37AMAIzVW85OxSg= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= +go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= +go.opentelemetry.io/otel/metric v0.19.0/go.mod h1:8f9fglJPRnXuskQmKpnad31lcLJ2VmNNqIsx/uIwBSc= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.19.0/go.mod h1:tI4yxwh8U21v7JD6R3BcA/2+RBoTKFexE/PJ/nSO7IA= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM= +go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.19.0/go.mod h1:4IXiNextNOpPnRlI4ryK69mn5iC84bjBWZQA5DXz/qg= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs= +go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +goji.io/v3 v3.0.0/go.mod h1:c02FFnNiVNCDo+DpR2IhBQpM9r5G1BG/MkHNTPUJ13U= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88 h1:Tgea0cVUD0ivh5ADBX4WwuI12DUd2to3nCYe2eayMIw= +golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190321232350-e250d351ecad/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190916130336-e45ffcd953cc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200308013534-11ec41452d41/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200624225443-88f3c62a19ff/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200630154851-b2d8b0336632/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200706234117-b22de6825cf7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201001104356-43ebab892c4c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201028025901-8cd080b735b3/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201114224030-61ea331ec02b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201118003311-bd56c0adb394/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210104081019-d8d6ddbec6ee/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= +golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.10.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.6/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= +k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= +k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/cri-api v0.23.1 h1:0DHL/hpTf4Fp+QkUXFefWcp1fhjXr9OlNdY9X99c+O8= +k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +libvirt.org/go/libvirt v1.8002.0 h1:X8gz2Sa1ek4S5FznpDpeRz6JpNb7NdkfzTii5GMIwDY= +libvirt.org/go/libvirt v1.8002.0/go.mod h1:1WiFE8EjZfq+FCVog+rvr1yatKbKZ9FaFMZgEqxEJqQ= +libvirt.org/go/libvirtxml v1.8002.0 h1:ES5bU3/G/dykJ4WIO5NJ3cTvvr5xSCaqwjYeRxkTX40= +libvirt.org/go/libvirtxml v1.8002.0/go.mod h1:7Oq2BLDstLr/XtoQD8Fr3mfDNrzlI3utYKySXF2xkng= +mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7/go.mod h1:hBpJkZE8H/sb+VRFvw2+rBpHNsTBcvSpk61hr8mzXZE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/build.sh b/hack/build.sh new file mode 100755 index 000000000..fcc6a7026 --- /dev/null +++ b/hack/build.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +script_dir=$(dirname "$(readlink -f "$0")") + +registry="${registry:-quay.io/confidential-containers/cloud-api-adaptor-${CLOUD_PROVIDER}}" + +supported_arches=( + "linux/amd64" +) + +function setup_env_for_arch() { + case "$1" in + "linux/amd64") + kernel_arch="x86_64" + ;; + (*) echo "$1 is not supported" && exit 1 + esac +} + +function build_caa_payload() { + pushd "${script_dir}/.." + + tag=$(date +%Y%m%d%H%M%s) + + for arch in ${supported_arches[@]}; do + setup_env_for_arch "${arch}" + + echo "Building cloud-api-adaptor ${CLOUD_PROVIDER} image for ${arch}" + docker buildx build \ + --build-arg ARCH="${kernel_arch}" \ + --build-arg CLOUD_PROVIDER="${CLOUD_PROVIDER}" \ + -f Dockerfile \ + -t "${registry}:${kernel_arch}-${tag}" \ + --platform="${arch}" \ + --load \ + . + docker push "${registry}:${kernel_arch}-${tag}" + done + + docker manifest create \ + ${registry}:${tag} \ + --amend ${registry}:x86_64-${tag} + + docker manifest create \ + ${registry}:latest \ + --amend ${registry}:x86_64-${tag} + + docker manifest push ${registry}:${tag} + docker manifest push --purge ${registry}:latest + + popd +} + +function main() { + build_caa_payload + # TODO: kustomize with the tag +} + +main "$@" diff --git a/hack/update-proto.sh b/hack/update-proto.sh new file mode 100755 index 000000000..0709f98ec --- /dev/null +++ b/hack/update-proto.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o errexit -o pipefail -o nounset + +basedir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd -P) + +cd "$basedir/proto" + +protoc --gogottrpc_out=. \ + --gogottrpc_opt=plugins=ttrpc+fieldpath,paths=source_relative \ + podvminfo/podvminfo.proto diff --git a/ibmcloud/README.md b/ibmcloud/README.md new file mode 100644 index 000000000..0b5ef6072 --- /dev/null +++ b/ibmcloud/README.md @@ -0,0 +1,605 @@ +# Setup procedure for IBM Cloud + +This guide describes how to set up a demo environment on IBM Cloud for peer pod VMs. + +This procedure has been confirmed using the following repositories. +* https://github.com/confidential-containers/cloud-api-adaptor/tree/staging +* https://github.com/kata-containers/kata-containers/tree/CCv0 + +The setup procedure includes the following sub tasks. + +* Create a Virtual Private Cloud (VPC) including security groups, subnet, and gateway +* Create a Kubernetes cluster on two virtual server instances (VSIs) +* Build a custom VM image for pod VMs +* Install cloud-api-adaptor on a worker node +* Run a demo + +## Prerequisites + +To automate preparation of VPC and VSIs, you need to install terraform and ansible on your `development machine`. Please follow the the official installation guides. + +* [Install Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) + +> **Tip:** If you are using Ubuntu linux, you can run follow commands simply: +> ```bash +> $ sudo apt-get update && sudo apt-get install -y gnupg software-properties-common curl +> $ curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - +> $ sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" +> $ sudo apt-get install terraform -y +> ``` + +* [Install Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) + +> **Tip:** If you are using Ubuntu linux, you can run follow commands simply: +> ```bash +> $ sudo apt-get install -y python3 +> $ sudo ln -s /usr/bin/python3 /usr/bin/python +> $ sudo add-apt-repository --yes --update ppa:ansible/ansible +> $ sudo apt-get install ansible -y +> ``` + +Optionally, you can install IBM Cloud CLI. + +* [Installing the stand-alone IBM Cloud CLI](https://cloud.ibm.com/docs/cli?topic=cli-install-ibmcloud-cli) + +> **Tips** +> - If you are using Ubuntu linux, you can run follow commands simply: +> ```bash +> $ curl -fsSL https://clis.cloud.ibm.com/install/linux | sh +> $ ibmcloud plugin install vpc-infrastructure cloud-object-storage +> ``` +> - You can use the [IBM Cloud Web UI](https://cloud.ibm.com/vpc-ext/overview) for most of the operations of IBM Cloud. And please make sure that you are selecting the correct region in the Web UI. +> + +* You need IBM Cloud API key. You can create your own API key at [https://cloud.ibm.com/iam/apikeys](https://cloud.ibm.com/iam/apikeys). + +## Create and test the demo environment on IBM Cloud for peer pod VMs - Basic Usage + +### End to end Terraform configuration + +You can create the demo environment for peer pod VMs on IBM Cloud Virtual Private Cloud (VPC) with the Terraform configuration located in [ibmcloud/terraform](./terraform). This Terraform configuration will: + +* Set up VPC infrastructure network resources, including the VPC, subnets and security groups +* Set up VPC infrastructure compute resources, including virtual server instances for the Kubernetes control plane and worker +* Optionally import a SSH public key to VPC Infrastructure +* Build and install software package dependencies on the Kubernetes control plane and worker instances +* Configure the Kubernetes control plane and worker instances +* Build the peer pod VM image on the Kubernetes worker instance +* Set up IBM Cloud Object Storage (COS) resources +* Push the built peer pod VM image to COS and import it from COS to VPC Infrastructure as a custom image +* Start the `cloud-api-adaptor` process on the Kubernetes worker instance +* Create an nginx pod that runs on a peer pod instance on the Kubernetes cluster +* Test the nginx peer pod + +The Terraform configuration supports building the demo environment on both the x86 (Intel) and s390x (IBM Z) architectures. + +To use the Terraform configuration, you need to create a file `terraform.tfvars` in the [configuration directory](./terraform) to specify parameters for the Terraform configuration. The `terraform.tfvars` file with all mandatory parameters looks like this: + +``` +ibmcloud_api_key = "" +ibmcloud_user_id = "" +cluster_name = "" +ssh_key_name = "" +podvm_image_name = "" +cos_bucket_name = "" +``` + +When all parameters are specified the `terraform.tfvars` will have the additional lines: + +``` +region_name = "" +zone_name = "" +ssh_pub_key = "" +cos_service_instance_name = "" +cos_bucket_region = "" +floating_ip_name = "" +image_name = "" +instance_profile_name = "" +primary_security_group_name = "" +primary_subnet_name = "" +public_gateway_name = "" +vpc_name = "" +``` + +#### Parameters + +> **Notes:** +> - `ibmcloud_api_key` is your IBM Cloud API key that you created at [https://cloud.ibm.com/iam/apikeys](https://cloud.ibm.com/iam/apikeys). +> - `region_name` (optional) is the IBM Cloud region Terraform will create the demo environment in. If not set it defaults to `jp-tok`. +> - `ibmcloud_user_id` is the IBM Cloud user ID who owns the API key specified using the `ibmcloud_api_key` parameter. You can look up the user ID by running +> ```bash +> $ ibmcloud login --apikey -r +> $ ibmcloud account users +> ``` +> If command `ibmcloud account users` displays multiple user IDs, choose the user ID whose state is `ACTIVE`. +> - `cluster_name` is a name of a Kubernetes cluster. This name is used for the prefix of the names of control plane and worker node virtual server instances. +> - `ssh_key_name` is the name of your SSH key registered in IBM Cloud or the name of a new SSH key if a public key is also provided using the optional `ssh_pub_key` parameter. You can add your SSH key at [https://cloud.ibm.com/vpc-ext/compute/sshKeys](https://cloud.ibm.com/vpc-ext/compute/sshKeys). For more information about SSH keys see [managing SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). The SSH key will be installed on the Kubernetes control plane and worker nodes and is used to access them from your `development machine`. +> - `podvm_image_name` is the name of the VPC infrastructure custom image for the peer pod VM that the Kubernetes worker will build. This name will have `-amd64` or `-s390x` appended to it to name the image that eventually gets imported to VPC infrastructure custom images, depending on if the `image_name` parameter for the instance the peer pod VM image is built on uses the amd64 or s390x CPU architecture +> - `cos_bucket_name` is the name of the COS bucket that will store the peer pod .vsi image. This bucket name must be unique across all IBM Cloud accounts. +> - `cos_bucket_region` (optional) is the name of the region that the COS bucket will be in, this can be regional (e.g. jp-tok) or cross-regional (e.g. eu). If not provided will be the region specified by `region_name`. +> - `zone_name` (optional) is the zone in the region Terraform will create the demo environment in. If not set it defaults to `jp-tok-2`. +> - `ssh_pub_key` (optional) is an variable for a SSH public key which has **not** been registered in IBM Cloud in the targeted region. Terraform will manage this key instead. You cannot register the same SSH public key in the same region twice under different SSHs key names. +> - `cos_service_instance_name` (optional) is the name of the COS service instance Terraform will create. If not set it defaults to `cos-image-instance`. +> - `floating_ip_name` (optional) is the name of the floating IP that is assigned to the Kubernetes worker. If not set it defaults to `tok-gateway-ip`. +> - `image_name` (optional) is a name of IBM Cloud infrastructure image. This name is used to create virtual server instances for the Kubernetes control plane and worker. For more information, about VPC custom images, see [IBM Cloud Importing and managing custom images](https://cloud.ibm.com/docs/vpc?topic=vpc-managing-images). If not set it defaults to `ibm-ubuntu-20-04-3-minimal-amd64-1`. +> - `resource_group_id` (optional) is the resource group ID in IBM Cloud, under which the peer pod will be created. If not set it defaults to your default resource group. +> - `instance_profile_name` (optional) is a name of IBM Cloud virtual server instance profile. This name is used to create virtual server instances for the Kubernetes control plane and worker. For more information, about virtual server instance profile, see [instance profiles](https://cloud.ibm.com/docs/vpc?topic=vpc-profiles). If not set it defaults to `bx2-2x8`, which uses the amd64 architecture, has 2 vCPUs and 8 GB memory. +> - `primary_security_group_name` (optional) is the name of the security group Terraform will create. If not set it defaults to `tok-primary-security-group`. +> - `primary_subnet_name` (optional) is the name of the subnet Terraform will create. If not set it defaults to `tok-primary-subnet`. +> - `public_gateway_name` (optional) is the name of the public gateway Terraform will create. If not set it defaults to `tok-gateway`. +> - `vpc_name` (optional) is the name of the VPC Terraform will create. If not set it defaults to `tok-vpc`. +> - `cloud_api_adaptor_repo` (optional) is the repository URL of Cloud API Adaptor. If not set it defaults to `https://github.com/confidential-containers/cloud-api-adaptor.git`. +> - `cloud_api_adaptor_branch` (optional) is the branch name of Cloud API Adaptor. If not set it defaults to `staging`. +> - `kata_containers_repo` (optional) is the repository URL of Kata Containers. If not set it defaults to `https://github.com/kata-containers/kata-containers.git`. +> - `kata_containers_branch` (optional) is the branch name of Kata Containers. If not set it defaults to `CCv0`. +> - `containerd_repo` (optional) is the repository URL of containerd. If not set it defaults to `https://github.com/confidential-containers/containerd.git`. +> - `containerd_branch` (optional) is the branch name of containerd. If not set it defaults to `CC-main`. + +> **Hint:** In order to create a cluster based on a different type of VSI image you can use the `instance_profile_name` and `image_name` parameters. E.g., to create a **s390x** architecture based cluster, include the following two lines in the `terraform.tfvars` file +> +> instance_profile_name = "bz2-2x8" +> image_name = "ibm-ubuntu-18-04-1-minimal-s390x-3" +> + +After writing you `terraform.tfvars` file you can create your VPC by executing the following commands on your `development machine`: +```bash +$ cd ibmcloud/terraform +$ terraform init +$ terraform plan +$ terraform apply +``` + +The following IBM Cloud resources will be created when running the end-to-end Terraform configuration. Please check the `main.tf` file in each subdirectory of [ibmcloud/terraform/](./terraform) for details regarding which resources each individual module creates. +* VPC +* Security groups +* Subnets +* Public gateway +* Floating IP for the public gateway +* Virtual server instances for the Kubernetes control plane and worker +* Floating IPs for the Kubernetes control plane and worker virtual server instances +* COS Instance with 1 COS bucket +* Custom image for the peer pod VM image +* Virtual server instance for the peer pod running the nginx workload +* SSH key, if you specified the optional `ssh_pub_key` variable + +On a cluster using `instance_profile_name = "bz2-2x8"` and `image_name = "ibm-ubuntu-18-04-1-minimal-s390x-3"` the end-to-end playbook takes approximately 50 minutes to complete. + +## Create and test the demo environment on IBM Cloud for peer pod VMs - advanced usage + +The individual modules this Terraform configuration calls can also be ran as stand-alone Terraform configurations. This is recommended for experienced users who want to try to set up the demo environment on pre-existing infrastructure. + +### Create a VPC + +First, you need to create a VPC. The Terraform configuration files are in [ibmcloud/terraform/common](./terraform/common/). + +To use the Terraform configuration, you need to create a file `terraform.tfvars` at in the same directory of the other files of the Terraform configuration to specify your IBM Cloud API Key. The `terraform.tfvars` looks like this. +``` +ibmcloud_api_key = "" +``` + +You can also customize the other parameters by specifying custom values in `terraform.tfvars`. The default values of such parameters are defined in [variables.tf](./terraform/common/variables.tf) + +Then, you can create your VPC by executing the following commands on your `development machine`. + +```bash +$ cd ibmcloud/terraform/common +$ terraform init +$ terraform plan +$ terraform apply +``` + +### Create a Kubernetes cluster + +Another Terraform configuration is available at [ibmcloud/terraform/cluster](./terraform/cluster) to create a Kubernetes cluster on the VPC you just created, or on a pre-existing VPC. This configuration is called as a Terraform module by the end-to-end configuration, but it can be ran as a stand-alone Terraform configuration. + +> **Tip:** You can create multiple clusters by using different cluster names. + +As usual, you need to create `terraform.tfvars` to specify parameter values. The `terraform.tfvars` looks like this. + +``` +ibmcloud_api_key = "" +ssh_key_name = "" +cluster_name = "" +primary_subnet_name = "" OR primary_subnet_id = "" +primary_security_group_name = "" OR primary_security_group_id = "" +vpc_name = "" OR vpc_id = "" +``` + +If you created your VPC, subnet and security group without using the `common` Terraform configuration you should provide the name or ID of your existing resources as the `primary_subnet_name/primary_subnet_id`, `primary_security_group_name/primary_security_group_id` and `vpc_name/vpc_id` parameters. If you created your VPC, subnet and security group using the `common` Terraform configuration you can find the IDs of your VPC, subnet and security group in the outputs of the `common` Terraform configuration. + +If you don't have your public key already configured in IBM Cloud you can add +``` +ssh_pub_key = "" +``` + +Additionally, you can customize source code repositories to be extracted under `/root` of each worker node. By default, the repository URLs and branch names listed below are used to fetch repositories. You can add variable definitions to `terraform.tfvars` to customize them. + +``` +cloud_api_adaptor_repo = "https://github.com/confidential-containers/cloud-api-adaptor.git" +cloud_api_adaptor_branch = "staging" +kata_containers_repo = "https://github.com/kata-containers/kata-containers.git" +kata_containers_branch = "CCv0" +containerd_repo = "https://github.com/confidential-containers/containerd.git" +containerd_branch = "CC-main" +``` + +> **Hint:** In order to create the cluster based on a different type of VSI image you can overwrite more parameters here e.g. to create a **s390x** based cluster add follow two lines to the `terraform.tfvars` file. +> +> instance_profile_name = "bz2-2x8" +> image_name = "ibm-ubuntu-18-04-1-minimal-s390x-3" +> +> **Notes:** +> - Some resources can be specified using their name or ID. For example, the subnet can be specified using the `primary_subnet_name` or `primary_subnet_id` variables. Where this option exists the `..._name` and `..._id` variables are mutually exclusive. +> - Resources that can be specified using either their name or ID must exist when the Terraform configuration is planned or applied. +> - Variables with the same name as variables in the end to end Terraform configuration are as described in that [configuration's parameters](#parameters). +> - If you want to create more than one cluster in the same VPC, you need to use a different `cluster_name` for each cluster. +> - Additional variables and their defaults are defined in the [variables.tf](./terraform/cluster/variables.tf) file for this Terraform configuration + +Then, execute the following commands to create a new Kubernetes cluster consisting of two Virtual server instances. One for a control plane node, and another one for a worker node. Please check [main.tf](terraform/cluster/main.tf) for the details. + +```bash +$ cd ibmcloud/terraform/cluster +$ terraform init +$ terraform plan +$ terraform apply +``` + +> **Tip:** You can check the status of provisioned Kubernetes node VM instances at [https://cloud.ibm.com/vpc-ext/compute/vs](https://cloud.ibm.com/vpc-ext/compute/vs). + +The SSH key installed on control-plane and worker nodes will be output at the end of the terraform configuration. + +```bash +Outputs: +ssh_key_name = +``` + +This Terraform configuration also triggers execution of two Ansible playbooks to set up Kubernetes and other prerequisite software in the two nodes. Please check [ansible/kube-playbook.yml](terraform/cluster/ansible/kube-playbook.yml) and [ansible/kata-playbook.yml](terraform/cluster/ansible/kata-playbook.yml) for the details. + +If ansible fails for some reason, you can rerun the Ansible playbooks as follows. +```bash +$ cd ansible +$ ansible-playbook -i inventory -u root ./kube-playbook.yml && ansible-playbook -i inventory -u root ./kata-playbook.yml +``` + +When ansible fails, Terraform does not execute the setup script for Kubernetes. In this case, you can manually run it as follows. + +```bash +$ ./scripts/setup.sh --bastion --control-plane --workers +``` + +> **Note:** You do not need to run this script manually, when everything goes well. +As there is only a single node. All of the rest look correct though! + +When two VSIs are successfully provisioned, a floating IP address is assigned to the worker node. You can use the floating IP address to access the worker node from the Internet, or to ssh into the worker node from your `development machine`: +```bash +$ ssh root@floating-ip-of-worker-node +``` + +### Build a pod VM image + +You need to build a pod VM image for peer pod VMs. A pod VM image contains the following components. + +* Kata agent +* Agent protocol forwarder +* skopeo +* umoci + +The build scripts are located in [ibmcloud/image](./image). The prerequisite software to build a pod VM image is already installed in the worker node by [the Ansible playbook](terraform/cluster/ansible/playbook.yml) for convenience. + +You need to configure Cloud Object Storage (COS) to upload your custom VM image. + +https://cloud.ibm.com/objectstorage/ + + +First, create a COS service instance if you have not create one. Then, create a COS bucket with the COS instance. The COS service instance and bucket names are necessary to upload a custom VM image. + +You can use the Terraform configuration located at [ibmcloud/terraform/cos](./terraform/cos) to use Terraform to create a COS service instance, COS bucket, and IAM Authorization Policy automatically. These resources are configured to store the peer pod VM images. Create a `terraform.tfvars` file in the configurations directory that includes these fields: + +``` +ibmcloud_api_key = "" +cos_bucket_name = "" +cos_service_instance_name = "" +cos_bucket_region = "" +``` + +> **Notes:** +> - Variables with the same name as variables in the end to end Terraform configuration are as described in that [configuration's parameters](#parameters). +> - Additional variables and their defaults are defined in the [variables.tf](./terraform/cos/variables.tf) file for this Terraform configuration. +> - The bucket can be regional or cross-regional see [Endpoints & Locations](https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-endpoints) + +Then run the Template via the following commands: +```bash +$ cd ibmcloud/terraform/cos +$ terraform init +$ terraform plan +$ terraform apply +``` + +You can use a Terraform configuration located at [ibmcloud/terraform/podvm-build](./terraform/podvm-build) to use Terraform and Ansible to build a pod VM image on the k8s worker node, upload it to a COS bucket and verify it. The architecture of the pod VM image built on the k8s worker node will be the same as that of the node. For example, a k8s worker node using an Intel **x86** VSI will build an Intel **x86** pod VM image and a k8s worker node using an IBM **s390x** VSI will build an IBM **s390x** pod VM image. + +> **Warning:** Building a pod VM image on a worker node using the Terraform configuration is not recommended for production, and we need to build a pod VM image somewhere secure to protect workloads running in a peer pod VM. + +Create the `terraform.tfvars` file in [the configuration directory](./terraform/podvm-build). The `terraform.tfvars` looks like this. +``` +ibmcloud_api_key = "" +ibmcloud_user_id = "" +cluster_name = "" +cos_service_instance_name = "" OR cos_service_instance_id = "" +cos_bucket_name = "" +cos_bucket_region = "" +``` + +If you created your COS resources without using with `cos` Terraform configuration you should provide the name or ID of your existing COS service instance as the `cos_service_instance_name/cos_service_instance_id` parameter and the name of your existing COS bucket as the `cos_bucket_name` parameter. If you created your COS service instance `cos` Terraform configuration you can find the ID of your COS service instance in the outputs of the `cos` Terraform configuration. + +> **Notes:** +> - The `cos_service_instance` resource can be specified using its name or ID. The `cos_service_instance_name` and `cos_service_instance_id` variables are mutually exclusive. +> - Variables with the same name as variables in the end to end Terraform configuration are as described in that [configuration's parameters](#parameters). +> - All COS and VPC infrastructure resources must exist before running this Terraform configuration. +> - Additional variables and their defaults are defined in the [variables.tf](./terraform/podvm-build/variables.tf) file for this Terraform configuration. +> - If you don't specify the optional `podvm_image_name` variable then the name of the custom image created will be based on [the latest commit hash](https://github.com/confidential-containers/cloud-api-adaptor/commits/staging) of the confidential containers cloud API adaptor staging branch. +> - If you specify `podvm_image_name` yourself you must add the `-amd64` or `-s390x` suffix to `podvm_image_name` depending on the CPU architecture of the instance you are building the peer pod VM image on. This is different to the behaviour of this parameter in the e2e Terraform configuration. +> - The `Operator` and `Console Admin` roles must be [assigned](https://cloud.ibm.com/docs/vpc?topic=vpc-vsi_is_connecting_console&interface=ui) to the user. The Terraform configuration will create the `Console Admin` role for the user `ibmcloud_user_id` is set to in the configuration `terraform.tfvars`. + +Execute the following commands on your `development machine` to build, upload and verify the pod VM image. + +```bash +$ cd ibmcloud/terraform/podvm-build +$ terraform init +$ terraform plan +$ terraform apply +``` + +> **Notes:** +> - If your worker node is **s390x** based, the suffix of the created QCOW2 file for the custom image will be `-s390x` otherwise it will be `-amd64`. +> - It typically takes about 15~20 minutes for task `Build peer pod VM image` +> - It typically takes about 7~10 minutes for task `Push peer pod VM image to Cloud Object Store and verify the image` +> - After all tasks finish, when you creating a server from the image it will only takes 1~5 minutes. +> - You can check the name and ID of the new image at [https://cloud.ibm.com/vpc-ext/compute/images](https://cloud.ibm.com/vpc-ext/compute/images). Alternatively, you can use the `ibmcloud` command to list your images as follows. +> ```bash +> $ ibmcloud is images --visibility=private +> ``` + +### Enabling Attestation agent and Authenticated Registry +**Prerequisites:** +- An ibmcloud worker node using the cloud-api-adaptor +- A [auth.json](https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md] file with your credentials) +- SSH'd into the worker node: `ssh root@floating-ip-of-worker-node` + +Once you have the prerequisites use [static registries authentication setup](../docs/registries-authentication.md#statically-embed-authentication-file-in-podvm-image) + +**Building the image:** +- `cd ~/cloud-api-adaptor/ibmcloud/image` +- Export these variables: +``` +export CLOUD_PROVIDER=ibmcloud +export IMAGE_NAME= +export IBMCLOUD_COS_REGION= +export IBMCLOUD_VPC_REGION=$IBMCLOUD_COS_REGION +export IBMCLOUD_VPC_NAME= +export IBMCLOUD_VPC_SUBNET_NAME= +export IBMCLOUD_COS_SERVICE_ENDPOINT="https://s3.${IBMCLOUD_COS_REGION}.cloud-object-storage.appdomain.cloud" +export IBMCLOUD_COS_SERVICE_INSTANCE= +export IBMCLOUD_COS_BUCKET= +export IBMCLOUD_API_KEY= +export IBMCLOUD_API_ENDPOINT="https://cloud.ibm.com" +export GOPATH=/root/go +``` +- run `AA_KBC="offline_fs_kbc" make push` +> **Note:** The image ID can be found at the end of the logs when the make steps have completed. This can then be substituted for the image id in the start cloud api adaptor steps below. + +## Install custom Kata shim + +The Ansible playbook automatically installs the custom Kata shim binary and its configuration file to worker node. If you want to rebuild the Kata shim, please follow the steps below. + +```bash +$ cd /root/kata-containers/src/runtime +$ make $PWD/containerd-shim-kata-v2 +$ install containerd-shim-kata-v2 /usr/local/bin/ +``` + +A minimum Kata shim configuration file at `/etc/kata-containers/configuration.toml` looks like this. + +``` +[runtime] +internetworking_model = "none" +disable_new_netns = true +disable_guest_seccomp = true +enable_pprof = true +enable_debug = true +[hypervisor.remote] +remote_hypervisor_socket = "/run/peerpod/hypervisor.sock" +[agent.kata] +``` + +## Install Cloud API adaptor + +The Ansible playbook automatically installs the Cloud API adaptor binary to worker node. If you want to rebuild it, please follow the steps below. + +```bash +$ cd /root/cloud-api-adaptor +$ CLOUD_PROVIDER=ibmcloud make +$ install cloud-api-adaptor /usr/local/bin/ +``` + +## Launch Cloud API adaptor + +A terraform configuration that will start the `cloud-api-adaptor` process on the Kubernetes worker node is available in [ibmcloud/terraform/start-cloud-api-adaptor](./terraform/start-cloud-api-adaptor). + +Create a `terraform.tfvars` file in the [configuration directory](./terraform/start-cloud-api-adaptor) for this Terraform configuration on your `development machine`. The `terraform.tfvars` file should look like this + +``` +ibmcloud_api_key = "" +cluster_name = "" +ssh_key_name = "" OR ssh_key_id = "" +podvm_image_name = "" OR podvm_image_id = "" +vpc_name = "" OR vpc_id = "" +primary_subnet_name = "" OR primary_subnet_id = "" +primary_security_group_name = "" OR primary_security_group_id = "" +``` + +If you created your infrastructure using the other Terraform configurations included in this repo then you can find the ID of your SSH key in the output values of the `cluster` Terraform configuration, the ID of your peer pod VM image in the output values of the `podvm-image` Terraform configuration and the IDs of your VPC, subnet and security group in the output values of the `common` Terraform configuration. If you created these resources without using the Terraform configurations then you should find the names or IDs of these resources in your existing IBM Cloud infrastructure and provide them to the respective parameters. + +> **Hints:** +> - The `instance_profile_name` optional variable sets the CPU architecture, number of vCPUs and memory of each peer pod virtual server instance. E.g., the `bz2-2x8` instance profile uses the s390x CPU architecture, has 2 vCPUs and 8 GiB of memory +> - If you created the cluster based on an s390x architecture VSI image you must set the `instance_profile_name` parameter to the name of an s390x-architecture instance profile. E.g., if your cluster uses the **s390x** CPU architecture add the following line to the `terraform.tfvars` file +> +> instance_profile_name = "bz2-2x8" +> +> `bz2-2x8` can be replaced with the name of a different s390x-architecture instance profile + +> **Notes:** +> - Some resources can be specified using their name or ID. Where this option exists the `..._name` and `..._id` variables are mutually exclusive. +> - Variables with the same name as variables in the end to end Terraform configuration are as described in that [configuration's parameters](#parameters). +> - Additional variables and their defaults are defined in the [variables.tf](./terraform/start-cloud-api-adaptor/variables.tf) file for this Terraform configuration. +> - To view [IBM Cloud Custom images for VPC](https://cloud.ibm.com/vpc-ext/compute/images) for your chosen region to view the name of the pod VM custom image that was built and uploaded, or run the command `ibmcloud is images --visibility=private`. + +Execute the following commands on your `development machine` to start the cloud api adaptor on your worker instance: + +```bash +$ cd ibmcloud/terraform/start-cloud-api-adaptor +$ terraform init +$ terraform plan +$ terraform apply +``` + +After `terraform apply` completes the `cloud-api-adaptor` process will run on the Kubernetes worker instance until it is deleted. See the subsection [delete the demo configuration and pod](./README.md#delete-the-demo-configuration-and-pod). + +## Demo + +### Deploy the nginx pod and sniff test nginx + +A Terraform configuration that will deploy an nginx pod to the Kubernetes cluster is available in [ibmcloud/terraform/run-nginx-demo](./terraform/run-nginx-demo). This configuration will also check the nginx peer pod virtual server instance has been successfully created. + +Create a `terraform.tfvars` file in the [configuration directory](./terraform/run-nginx-demo) for this Terraform configuration on your `development machine`. The `terraform.tfvars` file should look like this + +``` +ibmcloud_api_key = "" +cluster_name = "" +vpc_name = "" OR vpc_id = "" +podvm_image_name = "" OR podvm_image_id = "" +``` + +If you created your infrastructure using the other Terraform configurations included in this repo then you can find the ID of your peer pod VM image in the output values of the `podvm-image` Terraform configuration and the ID of your VPC the output values of the `common` Terraform configuration. If you created these resources without using the Terraform configurations then you should find the names or IDs of these resources in your existing IBM Cloud infrastructure and provide them to the respective parameters. + +> **Notes:** +> - The `vpc` resource can be specified using its name or ID. The `vpc_name` and `vpc_id` variables are mutually exclusive. +> - Variables with the same name as variables in the end to end Terraform configuration are as described in that [configuration's parameters](#parameters). +> - Additional variables and their defaults are defined in the [variables.tf](./terraform/run-nginx-demo/variables.tf) file for this Terraform configuration. + +Execute the following commands on your `development machine` to deploy the nginx demo workload: + +```bash +$ cd ibmcloud/terraform/run-nginx-demo +$ terraform init +$ terraform plan +$ terraform apply +``` + +Deploying the demo workload will create a new configMap, secret, nginx Pod and NodePort service on your Kubernetes cluster. It will also create a new virtual server instance for the peer pod in your IBM Cloud VPC. The `run-nginx-demo` Terraform configuration will also sniff test the deployed nginx server by accessing the HTTP port of the NodePort service, test that the CPU architecture of the Kubernetes worker matches that of the peer pod instance and test the volumes from configMap and secret be mounted correctly. + +> **Tip:** You can run the nginx sniff test manually if you log into the Kubernetes worker node using the floating IP that was assigned to it +> ```bash +> $ ssh root@floating-ip-of-worker-node +> ``` +> Then run the command +> ```bash +> $ curl http://localhost:30080 +> ``` +> You can also check the CPU architecture the pod VM instance is using by running the command +> ```bash +> $ kubectl exec nginx -- uname -a +> ``` +> While logged into to Kubernetes worker node. If you are using a `s390x` based image as the pod VM image, the output looks like this. +> ``` +> Linux nginx 5.4.0-109-generic #123-Ubuntu SMP [Date] s390x GNU/Linux +> ``` + +> **Note:** The cloud API adaptor establishes a network tunnel between the worker and pod VM, and the network traffic to/from the pod VM is transparently transferred via the tunnel. + +> **Tip:** You can also check the status of pod VM instance at [https://cloud.ibm.com/vpc-ext/compute/vs](https://cloud.ibm.com/vpc-ext/compute/vs). Alternatively, you can use the `ibmcloud` command to list your images as follows. +> ```bash +> $ ibmcloud is instances +> ``` + +> **Tip:** When the peer pod instance is created and it fails to start due to [capacity problems](https://cloud.ibm.com/docs/vpc?topic=vpc-instance-status-messages#cannot-start-capacity). +> Please stop `cloud-api-adaptor` on worker node, try to run peer pod instance on another zone: +> - Create a new subnet on the target zone by hand. +> - Start `cloud-api-adaptor` with new `vpc_zone` and `primary-subnet-id` on worker node. +> - Create nginx demo again. + +If you want to re-run the check, run: +```bash +$ terraform destroy +$ terraform plan +$ terraform apply +``` + +## Clean up + +If you want to clean up the IBM Cloud resources created in the above instructions, you can use the following steps: + +### Delete the demo environment end-to-end configuration + +To do a full clean up of the demo environment, from your development machine navigate to the `terraform/` repository directory for the end-to-end Terraform configuration with: + +```bash +$ cd ibmcloud/terraform +$ terraform destroy +``` + +This should delete all resources except the peer pod VM custom image, which you can delete by following the [Delete the peer pod VM image](#Delete-the-peer-pod-VM-image) instructions. + +### Delete the demo configuration and pod +From your development machine navigate to the `run-nginx-demo` repository directory and delete nginx pod on your Kubernetes cluster with: +```bash +$ cd ibmcloud/terraform/run-nginx-demo +$ terraform destroy +``` + +If the `cloud-api-adaptor` process was still running `terraform destroy` for this Terraform configuration should automatically delete the peer pod created VM instance too. If the `cloud-api-adaptor` process has stopped, then you can manually check for extra pod VSIs by running: +```bash +$ ibmcloud is instances +``` +to see if there are instances other than the control-plane and worker instances. + +If so these can be deleted with: +```bash +$ ibmcloud is instance-delete +``` + +### Stop the cloud API adaptor process +From your development machine navigate to the `start-cloud-api-adaptor` repository directory and stop the cloud API adaptor process on the Kubernetes worker node with: +```bash +$ cd ibmcloud/terraform/start-cloud-api-adaptor +$ terraform destroy +``` + +### Delete the peer pod VM image + +To check and then delete the custom peer pod VM image created run: +```bash +$ ibmcloud is images --visibility=private +$ ibmcloud is image-delete +``` + +### Delete the cluster + +From your development machine navigate to the `cloud-api-adaptor` repository directory and delete the VPC Kubernetes +cluster with: +```bash +$ cd ibmcloud/terraform/cluster +$ terraform destroy +``` + +### Delete the VPC + +From your development machine navigate to the `cloud-api-adaptor` repository directory and delete the VPC, security +groups, subnet and gateway with: +```bash +$ cd ibmcloud/terraform/common +$ terraform destroy +``` + +## Troubleshooting + +Please see the [Troubleshooting Guide](./TROUBLESHOOTING.md), if needed. diff --git a/ibmcloud/SECURE_EXECUTION.md b/ibmcloud/SECURE_EXECUTION.md new file mode 100644 index 000000000..b912ee668 --- /dev/null +++ b/ibmcloud/SECURE_EXECUTION.md @@ -0,0 +1,150 @@ +# Secure Execution Support for Peer Pods on IBM Cloud VPC + +IBM Cloud VPC already supports to create Hyper Protect Virtual Server on LinuxONE. And Confidential Computing is enabled by using the [IBM Secure Execution](https://www.ibm.com/docs/en/linux-on-systems?topic=execution-introduction) for Linux technology. For more information, please refer to [Confidential computing with LinuxONE](https://cloud.ibm.com/docs/vpc?topic=vpc-about-se) + +To support Secure Execution for Peer Pods on IBM Cloud VPC, we need build a Secure Execution enabled custom VM image for pod VMs, to replace the non-SE custom image. This document describes how to build a SE enabled custom image, based on the existing [image build script](./image/build.sh) for IBM Cloud VPC. + +> **Note**: In [the document](https://www.ibm.com/docs/en/linux-on-systems?topic=execution-secure-workload) describe the details about execution secure workload, you can go through the document to get a simplified view about how your workload is protected. + +## Set up a demo environment without Secure Execution on your development machine + +Follow the [README.md](./README.md) to setup a demo environment on IBM Cloud VPC. We need create a **s390x** architecture based cluster, so please include the following two lines in the `terraform.tfvars` file: +``` +instance_profile_name = "bz2-2x8" +image_name = "ibm-ubuntu-18-04-1-minimal-s390x-3" +``` + +## Prepare to build SE enabled image + +1. **Build genprotimg on worker node** + +`genprotimg` is used to build an encrypted boot record from a given kernel, initial RAM disk, parameters, and public host-key document. + +The worker node is using Ubuntu 18.04, so we need build genprotimg from the source on GitHub: +```bash +ssh root@ip-of-your-worker-node +git clone https://github.com/ibm-s390-linux/s390-tools.git +cd s390-tools/genprotimg/ +apt install -y libglib2.0-dev libssl-dev libcurl4-openssl-dev +make install +``` + +2. **Download host key document from Resource Link on your development machine** + +The host key must match the host system for which the image is prepared. You can download a host key document from Resource Link. + +As a registered user, access the search page: +``` +https://www.ibm.com/servers/resourcelink/hom03010.nsf/pages/HKDSearch?OpenDocument +``` + +If you have never signed in to Resource Link, you need to register before you can access the host key document search page. Please refer to document [Obtaining a host key document from Resource Link](https://www.ibm.com/docs/en/linux-on-systems?topic=execution-obtain-host-key-document#lxse_obtain_hkd) for details. + +Need input the **machine type** and **machine serial number**, which can be obtained from `/proc/sysinfo` of the s390x worker node. Check the values of **Type** and **Sequence Code**. Machine serial number would be the last 5 or 7 characters. Please refer to [this documentation](https://www.ibm.com/docs/en/linux-on-systems?topic=tasks-find-machine-serial) for details. + +When you obtain the host key document, please copy the downloaded host key document `HKD--.crt` to the worker node host keys directory, the directory can be any, if you use `/root/hostkeys/` as the host keys directory, you can use follow commands: +```bash +ssh root@ip-of-peer-pod-worker-node mkdir /root/hostkeys/ +scp HKD--.crt root@ip-of-peer-pod-worker-node:/root/hostkeys/ +``` + +> **Note** +> - You can download multiple different host keys and prepare only one SE enabled image for different host systems. +> +> eg. If you use `/root/hostkeys/` as the hosy keys directory and you want to deploy the SE enabled image on two host systems which the machine type are `8562` the machine serial number are `1234567` and `7654321` then the hosy keys directory tree looks like as follow: +> ```bash +> ssh root@ip-of-peer-pod-worker-node tree /root/hostkeys/ +> ... +> /root/hostkeys/ +> ├── HKD-8562-1234567.crt +> └── HKD-8562-7654321.crt +> +> 0 directories, 2 files +> ``` +> - Get the **machine type** and **machine serial number** from worker node only works when there is only one s390x arch host system in the target zone, if there are multiple s390x arch host systems in the target zone, please try to create z vsis on the target s390x arch host system one by one then get the **machine type** and **machine serial number** from the created z vsis on diferent host systems +> - Another way maybe you can contact the system admin to get the **machine type** and **machine serial number** for target host systems + +## Build one SE enabled custom image + +Please confirm the IBM Cloud API key, COS instance name and COS bucket name, VPC name, subnet name, region and zone when you set up the demo environment, set `SE_BOOT` as `1`, set `HOST_KEYS_DIR` with the host key documents directory (eg. `/root/hostkeys/`), set `IMAGE_NAME` as `se--s390x` (eg. `se-podvm-s390x`) then run follow commands: +``` +export IBMCLOUD_API_KEY= +export IBMCLOUD_API_ENDPOINT="https://cloud.ibm.com" +export IBMCLOUD_COS_REGION= +export IBMCLOUD_COS_SERVICE_INSTANCE= +export IBMCLOUD_COS_BUCKET= +export IBMCLOUD_COS_SERVICE_ENDPOINT="https://s3.${IBMCLOUD_COS_REGION}.cloud-object-storage.appdomain.cloud" +export IBMCLOUD_VPC_NAME= +export IBMCLOUD_VPC_SUBNET_NAME= +export IBMCLOUD_VPC_REGION= +export IBMCLOUD_VPC_ZONE= +export SE_BOOT=1 +export HOST_KEYS_DIR=/root/hostkeys +export IMAGE_NAME=se-podvm-s390x +export CLOUD_PROVIDER=ibmcloud + +cd /root/cloud-api-adaptor/ibmcloud/image +make push +make verify +``` +> **Note** +> - `make push` will call the [build script](./image/build.sh) to build SE enabled qcow2 image first and then call the [push script](./image/push.sh) to upload the built se-enabled qcow2 image to cos bucket and then create one se-enabled custom image. +- The [build script](./image/build.sh) is following the steps in [the document](https://www.ibm.com/docs/en/linux-on-systems?topic=execution-workload-owner-tasks) to prepare SE enabled qcow2 image +> - `make verify` will call the [verify script](./image/verify.sh) to create one SE enabled vsi and then delete it +> - The `IMAGE_NAME` is must end with `-s390x` when `SE_BOOT=1` + +### Troubleshooting +The build se-image will use `/dev/nbd0` and `/dev/nbd1`, please make sure they are avaliable to use +```bash +qemu-nbd --disconnect /dev/nbd0 +qemu-nbd --disconnect /dev/nbd1 +lsblk +``` +If the device can't be disconnected: +- When the `MOUNTPOINT` is not empty, please umount it +``` +umount +``` +- When there is one `crypt` partation, plesse close it +``` +lsblk +... +nbd1 43:32 0 100G 0 disk +├─nbd1p1 43:33 0 255M 0 part +└─nbd1p2 43:34 0 92.9G 0 part + └─LUKS-ad6e1db7-6833-4638-8eb1-08972554149b 253:0 0 92.9G 0 crypt + +cryptsetup close LUKS-ad6e1db7-6833-4638-8eb1-08972554149b +``` + +## Restart cloud-api-adaptor on worker node with new image ID + +When the se enabled custom image is created, get the image ID. + +Go the development machine to run terraform. We can use the terraform configuration [start-cloud-api-adaptor](./terraform/start-cloud-api-adaptor/) to start the `cloud-api-adaptor` process on the Kubernetes worker node. + +Create a `terraform.tfvars` file in the [configuration directory](./terraform/start-cloud-api-adaptor). The `terraform.tfvars` file should look like this: +``` +ibmcloud_api_key = "" +region_name = "" +cluster_name = "" +ssh_key_name = "" OR ssh_key_id = "" +vpc_name = "" OR vpc_id = "" +primary_subnet_name = "" OR primary_subnet_id = "" +primary_security_group_name = "" OR primary_security_group_id = "" +podvm_image_id = "" +instance_profile_name = "bz2e-2x8" +``` + +Please set `podvm_image_id` as the new custom image ID above. And set `instance_profile_name` as a valid confidential computing profile name(eg `bz2e-2x8`). + +Usually, we need destroy existing `cloud-api-adaptor` process first, and then start the new process with the new Pod VM image ID: +```bash +terraform init +terraform destroy +terraform apply +``` + +## Test + +We can follow guide [Deploy the nginx pod and sniff test nginx](./README.md#deploy-the-nginx-pod-and-sniff-test-nginx) to deploy a nginx pod. Verify the Peer Pod VM is created with the SE enabled custom image. diff --git a/ibmcloud/TROUBLESHOOTING.md b/ibmcloud/TROUBLESHOOTING.md new file mode 100644 index 000000000..0f20dc610 --- /dev/null +++ b/ibmcloud/TROUBLESHOOTING.md @@ -0,0 +1,213 @@ +# Troubleshooting Guide + +This will be updated over time, as issues arise and experience grows. + +## Cluster creation when switching use `ssh_pub_key` + +If you provide your SSH public key via the `ssh_pub_key` Terraform variable and, having already run `terraform apply`, you run `terraform apply` again then you may see the following error regarding your SSH key: +> Error: Error deleting SSH Key : SSH-key still in use + +This is caused by the IBM Cloud terraform provider forcing the replacement of the SSH key resource. As such Terraform may try to delete the existing SSH key while the Virtual Server Instances are using it. + +To resolve this issue: + +- If running the end-to-end Terraform configuration in [ibmcloud/terraform](./terraform), run `terraform state rm module.cluster.ibm_is_ssh_key.created_ssh_key[0]` to remove the SSH key resource from Terraform's internal state. +- If running the stand-alone cluster creation Terraform configuration in [ibmcloud/terraform/cluster](./terraform/cluster), run `terraform state rm ibm_is_ssh_key.created_ssh_key[0]` to remove the SSH key resource from Terraform's internal state. +- Delete the `ssh_pub_key` variable from the `terraform.tfvars` file +- Then run `terraform apply` again + +This will remove the SSH Key from the list of resources whose lifecycle is managed by Terraform. + +When you delete the cluster you will need to manually delete the SSH Key in your IBM Cloud VPC Infrastructure. + +## Issue with `podvm-build` playbook related to IBM Cloud + +This issue was observed by one engineer, and occurred during the `terraform plan` stage of the [podvm-build](https://github.com/confidential-containers/cloud-api-adaptor/tree/staging/ibmcloud/terraform/podvm-build) playbook, where the command failed with: - + +```text +│ Error: Iteration over null value +│ +│ on main.tf line 16, in locals: +│ 15: is_policies_and_roles = flatten([ +│ 16: for policy in data.ibm_iam_user_policy.user_policies.policies: [ +│ 17: for resource in policy.resources: policy.roles +│ 18: if resource.service == "is" && resource.resource_group_id == "" && resource.resource_instance_id == "" +│ 19: ] +│ 20: ]) +│ ├──────────────── +│ │ data.ibm_iam_user_policy.user_policies.policies is null +│ +│ A null value cannot be used as the collection in a 'for' expression. +``` + +This was reproduced as follows: - + +### Clone the repo + +`git clone -b staging git@github.com:confidential-containers/cloud-api-adaptor.git` + +### Switch to the podvm-build subdirectory + +`cd cloud-api-adaptor/ibmcloud/terraform/podvm-build` + +### Setup terraform.tfvars + +`echo 'ibmcloud_api_key=""' >> terraform.tfvars` + +`echo 'ibmcloud_user_id="david_hay@uk.ibm.com"' >> terraform.tfvars` + +`echo 'cluster_name="davehay-cluster"' >> terraform.tfvars` + +### Initialise Terraform + +`terraform init` + +```text +Initializing the backend... + +Initializing provider plugins... + +- Finding latest version of hashicorp/local... +- Finding latest version of hashicorp/null... +- Finding ibm-cloud/ibm versions matching "~> 1.34.0"... +- Installing hashicorp/local v2.2.3... +- Installed hashicorp/local v2.2.3 (signed by HashiCorp) +- Installing hashicorp/null v3.1.1... +- Installed hashicorp/null v3.1.1 (signed by HashiCorp) +- Installing ibm-cloud/ibm v1.34.0... +- Installed ibm-cloud/ibm v1.34.0 (self-signed, key ID AAD3B791C49CC253) + +Partner and community providers are signed by their developers. +If you'd like to know more about provider signing, you can read about it here: + + +Terraform has created a lock file .terraform.lock.hcl to record the provider +selections it made above. Include this file in your version control repository +so that Terraform can guarantee to make the same selections by default when +you run "terraform init" in the future. + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your working directory. If you forget, other +commands will detect it and remind you to do so if necessary. +``` + +### Create Terraform execution plan + +`terraform plan` + +```text +data.ibm_is_subnet.primary: Reading... +data.ibm_iam_user_policy.user_policies: Reading... +data.ibm_is_floating_ip.worker: Reading... +data.ibm_is_instance.worker: Reading... +data.ibm_iam_user_policy.user_policies: Read complete after 1s [id=david_hay@uk.ibm.com] +data.ibm_is_floating_ip.worker: Read complete after 3s [id=r022-3db669a0-606e-45e1-b493-084c78bb2714] +data.ibm_is_subnet.primary: Read complete after 4s [id=02f7-f79cfbb4-6872-4956-8b6a-68f09063833d] +data.ibm_is_instance.worker: Read complete after 4s [id=02f7_05f17f9c-a1a3-4b92-8f53-3a8c1cec08c6] +╷ +│ Error: Iteration over null value +│ +│ on main.tf line 16, in locals: +│ 15: is_policies_and_roles = flatten([ +│ 16: for policy in data.ibm_iam_user_policy.user_policies.policies: [ +│ 17: for resource in policy.resources: policy.roles +│ 18: if resource.service == "is" && resource.resource_group_id == "" && resource.resource_instance_id == "" +│ 19: ] +│ 20: ]) +│ ├──────────────── +│ │ data.ibm_iam_user_policy.user_policies.policies is null +│ +│ A null value cannot be used as the collection in a 'for' expression. + +For more information, please see the following Slack threads: - +``` + +This issue was observed with various releases of the [terraform-provider-ibm](https://github.com/IBM-Cloud/terraform-provider-ibm), including `1.34.0`, `1.41.1` and `1.42.0-beta0`. + +The issue was also observed with a very simple Terraform playbook: - + +```code +locals { + is_policies_and_roles = flatten([ + for policy in data.ibm_iam_user_policy.user_policies.policies: [ + ] + ]) +} + +data "ibm_iam_user_policy" "user_policies" { + ibm_id = var.ibmcloud_user_id +} +``` + +and highlighted the fact that Terraform was unable to retrieve the required User Policies via `data.ibm_iam_user_policy.user_policies.policies` in order to be able to loop across the returned list. + +Further investigation suggested that the issue only occurred when the IBM Cloud account had a specific User Policy, as per the following example: - + +```text +Policy ID: 7152fe85-36da-46b5-ab9b-1eac8d212cd5 +Roles: Reader +Resources: + Service Name containers-kubernetes + Region us-south + namespace default +``` + +If this policy was removed from the account, the plan ran to completion, and the engineer was able to then run `terraform apply`. + +If the policy was then added back into the account, the same `A null value cannot be used as the collection in a 'for' expression` exception occurred. + +After discussions with the community who maintain the [terraform-provider-ibm](https://github.com/IBM-Cloud/terraform-provider-ibm) project, an issue - [Data source of IAM policies failing to list the policies if the policy has service specific attributes #3801](https://github.com/IBM-Cloud/terraform-provider-ibm/issues/3801) - was raised, and is being actively worked at time of writing ( May 2022 ). + +In the meantime, if this problem occurs for other users, a check for policies containing service-specific attributes should be undertaken, as per the above example, where the User Policy is tagged with a Service Name of `container-kubernetes`. + +Whilst it's not 100% clear which service-specific attributes might cause this issue, one can check using the [IBM Cloud CLI tool](https://cloud.ibm.com/docs/cli?topic=cli-getting-started), using a command similar to the following example: - + +`ibmcloud iam user-policies david_hay@uk.ibm.com --output JSON | jq '.[] | select(.resources[].attributes[].value=="containers-kubernetes") | {ID:.id,Resources:.resources,Roles:.roles}'` + +which should return output similar to the following: - + +```json +{ + "ID": "8993f73a-06b1-4e59-81e2-2bd49ddbee3d", + "Resources": [ + { + "attributes": [ + { + "name": "region", + "operator": "stringEquals", + "value": "us-south" + }, + { + "name": "serviceName", + "operator": "stringEquals", + "value": "containers-kubernetes" + }, + { + "name": "accountId", + "operator": "stringEquals", + "value": "f5e2ac71094077500e0d4b1ef85fdaec" + } + ] + } + ], + "Roles": [ + { + "role_id": "crn:v1:bluemix:public:iam::::role:Viewer", + "display_name": "Viewer", + "description": "As a viewer, you can view service instances, but you can't modify them." + } + ] +} +``` + +Commands such as this allow the engineer to selectively export, and then delete the User Policy or Policies that may be blocking the `terraform plan` stage, until the `terraform-provider-ibm` team are able to resolve [Data source of IAM policies failing to list the policies if the policy has service specific attributes #3801](https://github.com/IBM-Cloud/terraform-provider-ibm/issues/3801) + +In terms of troubleshooting tips, setting the `TF_LOG` and `TF_LOG_PATH` variables proved useful, in terms of confirming that the `terraform plan` step did correctly return IBM Cloud User Policies etc. from the target account. + +For more information, please see [Environment Variables](https://www.terraform.io/cli/config/environment-variables). diff --git a/ibmcloud/demo/nginx.yaml b/ibmcloud/demo/nginx.yaml new file mode 100644 index 000000000..080c73a8d --- /dev/null +++ b/ibmcloud/demo/nginx.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx + labels: + app: nginx +spec: + runtimeClassName: kata + containers: + - name: nginx + image: nginx + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: secret-volume + mountPath: /etc/secret + volumes: + - name: config-volume + configMap: + name: nginx-config + - name: secret-volume + secret: + secretName: nginx-secret +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-config +data: + example.txt: | + Hello, world +--- +apiVersion: v1 +kind: Secret +metadata: + name: nginx-secret +data: + password: MTIzNDU2 + username: YWRtaW4= +type: Opaque +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-service +spec: + type: NodePort + ports: + - name: port80 + port: 80 + targetPort: 80 + nodePort: 30080 + protocol: TCP + selector: + app: nginx diff --git a/ibmcloud/demo/runtime-class.yaml b/ibmcloud/demo/runtime-class.yaml new file mode 100644 index 000000000..de9323935 --- /dev/null +++ b/ibmcloud/demo/runtime-class.yaml @@ -0,0 +1,7 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: kata +handler: kata +scheduling: + nodeSelector: diff --git a/ibmcloud/image/.gitignore b/ibmcloud/image/.gitignore new file mode 100644 index 000000000..7d355e938 --- /dev/null +++ b/ibmcloud/image/.gitignore @@ -0,0 +1,14 @@ +/.bluemix +/*.img +/*.qcow2 +/src_mnt +/dst_mnt +/skopeo +/umoci +/pause +/staticlib +/files/usr/local/bin/agent-protocol-forwarder +/files/usr/local/bin/kata-agent +/files/usr/bin/skopeo +/files/usr/local/bin/umoci +/files/pause_bundle diff --git a/ibmcloud/image/Makefile b/ibmcloud/image/Makefile new file mode 100644 index 000000000..580471a15 --- /dev/null +++ b/ibmcloud/image/Makefile @@ -0,0 +1,42 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +include ../../podvm/Makefile.inc + +.PHONY: build push verify ubuntu clean + +UBUNTU_RELEASE = focal +UBUNTU_IMAGE_URL := https://cloud-images.ubuntu.com/$(UBUNTU_RELEASE)/current/$(UBUNTU_RELEASE)-server-cloudimg-$(ARCH).img +UBUNTU_IMAGE_FILE := $(notdir $(UBUNTU_IMAGE_URL)) + +UBUNTU_PACKAGES = jq + +IMAGE_SUFFIX := .qcow2 + +build: $(IMAGE_FILE) + +push: $(IMAGE_FILE) + ./push.sh --name "$(IMAGE_NAME)" --path "$(IMAGE_FILE)" + +verify: push + ./verify.sh --image "$(IMAGE_NAME)" + +SUDO = +WORKDIR = . + +$(IMAGE_FILE): $(UBUNTU_IMAGE_FILE) $(BINARIES) $(FILES) + rm -f "$(IMAGE_FILE)" + $(SUDO) ./build.sh --root "$(FILES_DIR)" --packages "$(UBUNTU_PACKAGES)" --base "$(UBUNTU_IMAGE_FILE)" --output "$(IMAGE_FILE)" --workdir "$(WORKDIR)" + +ubuntu: $(UBUNTU_IMAGE_FILE) +$(UBUNTU_IMAGE_FILE): + curl -s -L -o "$@" "$(UBUNTU_IMAGE_URL)" + +clean: + rm -f "$(IMAGE_FILE)" "$(UBUNTU_IMAGE_FILE)" $(BINARIES) + rm -fr "$(SKOPEO_SRC)" "$(UMOCI_SRC)" "$(PAUSE_SRC)" "$(FILES_DIR)/$(PAUSE_BUNDLE)" + +.PHONY: force +force: diff --git a/ibmcloud/image/build.sh b/ibmcloud/image/build.sh new file mode 100755 index 000000000..fbcf668ef --- /dev/null +++ b/ibmcloud/image/build.sh @@ -0,0 +1,431 @@ +#!/bin/bash +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit -o pipefail -o nounset + +cd "$(dirname "${BASH_SOURCE[0]}")" + +function usage() { + echo "Usage: $0 --base --output --root [--packages ]" +} + +declare -a packages + +workdir=. + +while (( $# )); do + case "$1" in + --base) base_img_path=$2 ;; + --output) dst_img_path=$2 ;; + --root) files_dir=$2 ;; + --packages) IFS=', ' read -a packages <<< "$2" ;; + --workdir) workdir=$2 ;; + --help) usage; exit 0 ;; + *) usage 1>&2; exit 1;; + esac + shift 2 +done + +if [[ -z "${base_img_path-}" || -z "${dst_img_path-}" || -z "${files_dir-}" ]]; then + usage 1>&2 + exit 1 +fi + +SE_BOOT=${SE_BOOT:-0} + +if [ "${SE_BOOT}" = "1" ]; then + if [[ -z "${HOST_KEYS_DIR-}" ]]; then + echo "HOST_KEYS_DIR is missed" 1>&2 + echo "CLOUD_PROVIDER=ibmcloud SE_BOOT=1 HOST_KEYS_DIR= make build" + exit 1 + fi + umount ./rootkeys/ || true + rm -rf ./rootkeys/ +fi + +base_img_path=$(realpath "$base_img_path") +src_img_path="$workdir/src.qcow2" +tmp_img_path="$workdir/tmp.qcow2" + +src_nbd=/dev/nbd0 +tmp_nbd=/dev/nbd1 + +src_mnt=./src_mnt +dst_mnt=./dst_mnt + +disksize=100G + +if [[ -e "$dst_img_path" ]]; then + echo "Error: image file already exists: $dst_img_path" 1>&2 + exit 1 +fi + +function cleanup () { + msg=$1 + if [ "${SE_BOOT}" = "1" ]; then + for mnt in "$dst_mnt/boot-se" "$dst_mnt/etc/keys" "$dst_mnt/sys"; do + mountpoint -q "$mnt" && umount "$mnt" || true + [[ -d "$mnt" ]] && rmdir "$mnt" 2> /dev/null || true + done + mountpoint -q ./rootkeys/ && umount ./rootkeys/ || true + [[ -d ./rootkeys/ ]] && rm -rf ./rootkeys/ + fi + for mnt in "$src_mnt/run" "$src_mnt/dev/pts" "$src_mnt/dev" \ + "$src_mnt/proc" "$src_mnt/sys" "$src_mnt" \ + "$dst_mnt/dev" "$dst_mnt/proc" "$dst_mnt"; do + mountpoint -q "$mnt" && umount "$mnt" + [[ -d "$mnt" ]] && rmdir "$mnt" 2> /dev/null || true + done + sleep 1 + [ -n "${LUKS_NAME:-}" ] && (cryptsetup close $LUKS_NAME 2> /dev/null || true) + qemu-nbd --disconnect "$src_nbd" + qemu-nbd --disconnect "$tmp_nbd" + kpartx -dsv "$src_nbd" + kpartx -dsv "$tmp_nbd" + + rm -f "$src_img_path" "$tmp_img_path" + + [[ -n "$msg" ]] && printf "\n%s" "$msg" + + return 0 +} + +trap 'cleanup "image creation failed"' 0 + +modprobe nbd + +rm -f "$src_img_path" "$tmp_img_path" +echo "Cleanuping build env" +cleanup "" +if [ "${SE_BOOT}" = "1" ]; then + echo "Finding host key files" + host_keys="" + for i in $(ls ${HOST_KEYS_DIR}/*.crt); do + echo "found host key file: \"${i}\"" + host_keys+="-k ${i} " + done + [[ -z $host_keys ]] && echo "Didn't find hosy key files, please set HOST_KEYS_DIR correctly "&& exit 1 +fi + +printf "\nCopying partitions from the base image $src_img_path\n" + +qemu-img create -f qcow2 -b "$base_img_path" "$src_img_path" $disksize +qemu-img create -f qcow2 "$tmp_img_path" $disksize + +qemu-nbd --connect="$src_nbd" "$src_img_path" +qemu-nbd --connect="$tmp_nbd" "$tmp_img_path" + +declare -a parts + +# https://alioth-lists.debian.net/pipermail/parted-devel/2006-December/000573.html +i=1 +while IFS=':;' read -a part; do + + if (( $i == 1 )); then + if [[ "${part[0]}" != BYT ]]; then + echo "unrecognized parted output" 1>&2 + exit 1 + fi + elif (( $i == 2 )); then + + if [[ "${part[0]}" != "$src_nbd" ]]; then + echo "device path is not an nbd device" 1>&2 + exit 1 + fi + if [[ "${part[3]}" != 512 || "${part[4]}" != 512 ]]; then + echo "sector size is not 512 bytes" 1>&2 + exit 1 + fi + + disklabel=${part[5]} + case "$disklabel" in + msdos) + ;; + gpt) + sgdisk -e "$src_nbd" + ;; + *) + echo "unrecognized disk label: $disklabel" 1>&2 + exit 1 + ;; + esac + else + part_number=${part[0]} + part_offset=$(echo "${part[1]}" | sed -e 's/B$//') + part_type=${part[4]} + + if [[ "$part_number" != 1 ]]; then + parts+=("$part_number") + else + if [[ "$part_type" != ext4 ]]; then + echo "fs type of partition 1 is not ext4" 1>&2 + exit 1 + fi + target_offset=$(( $part_offset / 512 )) + parted -s "$src_nbd" resizepart 1 100% + fi + fi + + (( i++ )) +done < <(parted -s -m "$src_nbd" unit B print) + +sleep 1 +resize2fs -f "${src_nbd}p1" + +if [ "${SE_BOOT}" = "1" ]; then + echo "Creating boot-se and root partitions" + parted -a optimal $tmp_nbd mklabel gpt \ + mkpart boot-se ext4 1MiB 256MiB \ + mkpart root 256MiB "${disksize}" \ + set 1 boot on +else + case "$disklabel" in + gpt) + sgdisk "$src_nbd" -R "$tmp_nbd" + sgdisk -G "$tmp_nbd" + ;; + msdos) + sfdisk -d "$src_nbd" | sfdisk "$tmp_nbd" + ;; + esac +fi + +if [ "${SE_BOOT}" = "1" ]; then + echo "Waiting for the two nbd partitions to show up" + while true; do + sleep 1 + [ -e ${tmp_nbd}p2 ] && break + done + printf "\nFormatting boot-se partition\n" + mke2fs -t ext4 -L boot-se ${tmp_nbd}p1 + export boot_uuid=$(blkid ${tmp_nbd}p1 -s PARTUUID -o value) + printf "\nSetting up encrypted root partition\n" + mkdir rootkeys || true + mount -t tmpfs rootkeys ./rootkeys + dd if=/dev/random of=./rootkeys/rootkey.bin bs=1 count=64 &> /dev/null + echo YES | cryptsetup luksFormat --type luks2 ${tmp_nbd}p2 --key-file ./rootkeys/rootkey.bin + echo "Setting LUKS name for root partition" + LUKS_NAME="LUKS-$(blkid -s UUID -o value ${tmp_nbd}p2)" + export LUKS_NAME + echo "LUKS name is: $LUKS_NAME" + cryptsetup open ${tmp_nbd}p2 $LUKS_NAME --key-file ./rootkeys/rootkey.bin +else + for part_number in "${parts[@]}"; do + dd if="${src_nbd}p$part_number" of="${tmp_nbd}p$part_number" bs=$((1024*1024)) + done +fi + +printf "\nMounting the root partition\n" + +src_part="${src_nbd}p1" +dst_part="${tmp_nbd}p1" + +mkdir -p "$src_mnt" +mount "$src_part" "$src_mnt" + +mount -t sysfs sysfs "$src_mnt/sys" +mount -t proc proc "$src_mnt/proc" +mount --bind /dev "$src_mnt/dev" +mount --bind /dev/pts "$src_mnt/dev/pts" + +mount -t tmpfs tmpfs "$src_mnt/run" +mkdir -p "$src_mnt/run/systemd/resolve" +cp /run/systemd/resolve/resolv.conf "$src_mnt/run/systemd/resolve/resolv.conf" +cp /run/systemd/resolve/stub-resolv.conf "$src_mnt/run/systemd/resolve/stub-resolv.conf" + +if (( ${#packages[@]} )); then + printf "\nInstalling packages: ${packages[*]}\n" + chroot "$src_mnt" apt-get update + chroot "$src_mnt" apt-get install -y "${packages[@]}" +fi + +case "$(uname -m)" in + s390x) + ;; + *) + printf "\nUpdating initramfs\n" + echo -e "virtio_pci\nvirtio_blk" >> "$src_mnt/etc/initramfs-tools/modules" + chroot "$src_mnt" update-initramfs -u + ;; + +esac + +chroot "$src_mnt" apt-get remove unattended-upgrades -y +chroot "$src_mnt" apt-get autoremove +chroot "$src_mnt" apt-get clean +chroot "$src_mnt" bash -c 'rm -rf /var/lib/apt/lists/*' + +cp -a "$files_dir"/* "$src_mnt" + +mkdir -p "$src_mnt/var/lib/kubelet" + +umount "$src_mnt/run" +umount "$src_mnt/dev/pts" +umount "$src_mnt/dev" +umount "$src_mnt/proc" +umount "$src_mnt/sys" + +mkdir -p "$dst_mnt" +if [ "${SE_BOOT}" = "1" ]; then + dst_part="${tmp_nbd}p2" + label="root" + printf "\nFormatting root partition\n" + mkfs.ext4 -L "$label" /dev/mapper/$LUKS_NAME + mount /dev/mapper/$LUKS_NAME "$dst_mnt" + mkdir ${dst_mnt}/etc + mkdir ${dst_mnt}/boot-se + mount -o norecovery ${tmp_nbd}p1 ${dst_mnt}/boot-se +else + label=$(lsblk -n -o label "$src_part") + mkfs.ext4 -L "$label" "$dst_part" + mount "$dst_part" "$dst_mnt" +fi + +echo "Copying the root filesystem" +tar_opts=(--numeric-owner --preserve-permissions --acl --selinux --xattrs --xattrs-include='*' --sparse) +tar -cf - "${tar_opts[@]}" --sort=none -C "$src_mnt" . | tar -xf - "${tar_opts[@]}" --preserve-order -C "$dst_mnt" + +echo "The root filesystem is ready" +sleep 1 +umount "$src_mnt" +mount -t sysfs sysfs "$dst_mnt/sys" +mount -t proc proc "$dst_mnt/proc" +mount --bind /dev "$dst_mnt/dev" + +if [ "${SE_BOOT}" = "1" ]; then + printf "\nPreparing secure execution boot image\n" + echo "mounting tmpfs to /etc/keys" + mkdir -p ${dst_mnt}/etc/keys + mount -t tmpfs keys ${dst_mnt}/etc/keys + # ADD CONFIGURATION + echo "adding fstab" + cat < ${dst_mnt}/etc/fstab +#This file was auto-generated +/dev/mapper/$LUKS_NAME / ext4 defaults 1 1 +PARTUUID=$boot_uuid /boot-se ext4 norecovery 1 2 +END + echo "adding luks keyfile for fs" + cp ./rootkeys/rootkey.bin ${dst_mnt}/etc/keys/luks-$(blkid -s UUID -o value /dev/mapper/$LUKS_NAME).key + chmod 600 ${dst_mnt}/etc/keys/luks-$(blkid -s UUID -o value /dev/mapper/$LUKS_NAME).key + cat < ${dst_mnt}/etc/crypttab +#This file was auto-generated +$LUKS_NAME UUID=$(blkid -s UUID -o value ${dst_part}) /etc/keys/luks-$(blkid -s UUID -o value /dev/mapper/$LUKS_NAME).key luks,discard,initramfs +END + chmod 744 ${dst_mnt}/etc/crypttab + # Disable virtio_rng + cat < ${dst_mnt}/etc/modprobe.d/blacklist-virtio.conf +#don't trust rng from hypervisor +blacklist virtio_rng +END + # Favor loading of TRNG module for newer Z machines + echo 's390_trng' >> ${dst_mnt}/etc/modules + + # Prep files needed for mkinitrd (if running encrypted /) + echo "KEYFILE_PATTERN=\"/etc/keys/*.key\"" >> ${dst_mnt}/etc/cryptsetup-initramfs/conf-hook + echo "UMASK=0077" >> ${dst_mnt}/etc/initramfs-tools/initramfs.conf + cat < ${dst_mnt}/etc/zipl.conf +[defaultboot] +default=linux +target=/boot-se + +targetbase=/dev/vda +targettype=scsi +targetblocksize=512 +targetoffset=2048 + +[linux] +image = /boot-se/se.img +END + echo "updating initial ram disk" + chroot "$dst_mnt" update-initramfs -u || true + echo "!!! Bootloader install errors prior to this line are intentional !!!!!" 1>&2 + printf "\nGenerating an IBM Secure Execution image\n" + # Clean up kernel names and make sure they are where we expect them + KERNEL_FILE=$(readlink ${dst_mnt}/boot/vmlinuz) + echo "using kernel: ${KERNEL_FILE}" + INITRD_FILE=$(readlink ${dst_mnt}/boot/initrd.img) + echo "using initrd: ${INITRD_FILE}" + SE_PARMLINE="root=/dev/mapper/$LUKS_NAME console=ttysclp0 quiet panic=0 rd.shell=0 blacklist=virtio_rng swiotlb=262144" + echo "${SE_PARMLINE}" > ${dst_mnt}/boot/parmfile + /usr/bin/genprotimg \ + -i ${dst_mnt}/boot/${KERNEL_FILE} \ + -r ${dst_mnt}/boot/${INITRD_FILE} \ + -p ${dst_mnt}/boot/parmfile \ + --no-verify \ + ${host_keys} \ + -o "$dst_mnt"/boot-se/se.img + # exit and throw an error if no se image was created + [ ! -e $dst_mnt/boot-se/se.img ] && exit 1 + # if building the image succeeded wipe /boot + rm -rf ${dst_mnt}/boot/* + printf "\nRunning zipl to prepare boot partition\n" + chroot $dst_mnt zipl --targetbase $tmp_nbd \ + --targettype scsi \ + --targetblocksize 512 \ + --targetoffset 2048 \ + --target /boot-se \ + --image /boot-se/se.img + printf "\nClean luks keyfile\n" + umount ./rootkeys/ || true + rm -rf ./rootkeys/ + umount $dst_mnt/etc/keys + umount $dst_mnt/boot-se +else + case "$(uname -m)" in + s390x) + printf "\nExecuting zipl" + helper="$dst_mnt/lib/s390-tools/zipl_helper.nbd" + cat < "$helper" +echo "targetbase=$tmp_nbd" +echo "targettype=scsi" +echo "targetblocksize=512" +echo "targetoffset=$target_offset" +END + chmod 755 "$helper" + chroot "$dst_mnt" zipl -V + rm "$helper" + ;; + *) + printf "\nUpdating GRUB settings" + sed -i -r -e 's|^GRUB_CMDLINE_LINUX=|#\0"|' "$dst_mnt/etc/default/grub" + cat <> "$dst_mnt/etc/default/grub" + +GRUB_CMDLINE_LINUX="nomodeset nofb vga=normal console=ttyS0" +GRUB_DISABLE_LINUX_UUID=true +GRUB_DISABLE_OS_PROBER=true +GRUB_DEVICE="LABEL=cloudimg-rootfs" +END + chroot "$dst_mnt" update-grub + ;; + esac +fi + +umount "$dst_mnt/dev" +umount "$dst_mnt/proc" +umount "$dst_mnt/sys" +umount "$dst_mnt" + +if [ "${SE_BOOT}" = "1" ]; then + echo "Closing encrypted root partition" + cryptsetup close $LUKS_NAME +fi + +qemu-nbd --disconnect "$src_nbd" +qemu-nbd --disconnect "$tmp_nbd" + +sleep 1 + +printf "\nGenerating QCOW2 image file\n" + +qemu-img convert -O qcow2 -c "$tmp_img_path" "$dst_img_path" + +trap "" 0 +cleanup "" + +printf "\nCompleted image creation" +printf "\n$dst_img_path\n" + +exit 0 diff --git a/ibmcloud/image/delete.sh b/ibmcloud/image/delete.sh new file mode 100755 index 000000000..e07853ef3 --- /dev/null +++ b/ibmcloud/image/delete.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit -o pipefail -o nounset + +cd "$(dirname "${BASH_SOURCE[0]}")" + +export IBMCLOUD_HOME=$(pwd -P) +./login.sh + +cos_bucket="paidvpcimagebucket" + +for img in "$@"; do + img=${img%.qcow2} + ibmcloud is image-delete --force "$img" + ibmcloud cos object-delete --bucket "$cos_bucket" --key "$img.qcow2" --force +done diff --git a/ibmcloud/image/login.sh b/ibmcloud/image/login.sh new file mode 100755 index 000000000..2637e764e --- /dev/null +++ b/ibmcloud/image/login.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit -o pipefail -o nounset + +cd "$(dirname "${BASH_SOURCE[0]}")" + +api_key=${IBMCLOUD_API_KEY-} +api_endpoint=${IBMCLOUD_API_ENDPOINT-https://cloud.ibm.com} +region=${IBMCLOUD_VPC_REGION:-jp-tok} +export IBMCLOUD_VERSION_CHECK=false + +ibmcloud config --http-timeout 60 +ibmcloud config --color false + +if ! ibmcloud iam oauth-tokens &> /dev/null; then + if [[ -n "$api_key" ]]; then + opts=(--apikey "$api_key") + else + opts=(--sso) + fi + ibmcloud login -a "$api_endpoint" -r "$region" "${opts[@]}" +fi + +required_plugins=(vpc-infrastructure cloud-object-storage) +installed_plugins=($(ibmcloud plugin list --output json | jq -r '.[].Name')) + +for plugin in "${required_plugins[@]}"; do + if ! [[ -n "${installed_plugins-}" && " ${installed_plugins[*]} " =~ " $plugin " ]]; then + ibmcloud plugin install "$plugin" + fi +done diff --git a/ibmcloud/image/multipart_upload.sh b/ibmcloud/image/multipart_upload.sh new file mode 100755 index 000000000..d2f3d14b5 --- /dev/null +++ b/ibmcloud/image/multipart_upload.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Copyright Confidential Containers Contributors +# + +set -o errexit -o pipefail -o nounset + +function usage() { + echo "Usage: $0 --file --bucket [--size ]" +} + +# cleanup, removes the split files and the structure if they have been created by this script +function cleanup() { + rm -f "${original_file}_structure.json" + rm -f "$original_file".* +} +trap cleanup EXIT + +while (( "$#" )); do + case "$1" in + --file) original_file=$2 ;; + --bucket) bucket=$2 ;; + --size) split_size=$2 ;; + --help) usage; exit 0 ;; + *) usage 1>&2; exit 1;; + esac + shift 2 +done + +# Default to 100M part size +split_size=${split_size:-100M} + +# Requires file and bucket to be specified to continue +if [[ -z "${original_file-}" || -z "${bucket-}" ]]; then + usage 1>&2 + exit 1 +fi + +# Divide the original file up into smaller sections +split "$original_file" -b $split_size "$original_file." +# Create a multipart-upload, need the upload-id for part uploads +upload_id=$(ibmcloud cos multipart-upload-create --bucket "$bucket" --key "$original_file" --output JSON | jq -r '.UploadId') +if [[ -z "$upload_id" ]]; then + echo "Unable to start upload, check permissions" 1>&2 + exit 1 +else + echo "Uploading to "$upload_id"" +fi + +# The complete command requires a description of the parts to piece together into the final object +# This is constructed in JSON, the end result looks something like +# {"Parts": [ {"ETag": "tagabcdef", "PartNumber": 1 } ... ]} +structure="{\"Parts\":[]}" +i=1 +for part in "$original_file".*; +do + echo "Uploading part ${part} to bucket ${bucket}"; + etag=$(ibmcloud cos part-upload --bucket "$bucket" --key "$original_file" --upload-id "$upload_id" --part-number $i --body "$part" --output JSON | jq -r '.ETag') + structure=$(echo "$structure" | jq --arg etag "$etag" --argjson i $i '.Parts += [{"ETag": $etag, "PartNumber": $i}]') + ((i=i+1)) +done + +# Temporarily store the structure to file for the cli command +echo "$structure" > "${original_file}_structure.json" +ibmcloud cos multipart-upload-complete --bucket "$bucket" --key "$original_file" --upload-id "$upload_id" --multipart-upload "file://${original_file}_structure.json" diff --git a/ibmcloud/image/push.sh b/ibmcloud/image/push.sh new file mode 100755 index 000000000..2f90bd82b --- /dev/null +++ b/ibmcloud/image/push.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit -o pipefail -o nounset + +cd "$(dirname "${BASH_SOURCE[0]}")" + +function usage() { + echo "Usage: $0 --name --path " +} + +while (( "$#" )); do + case "$1" in + --name) image_name=$2 ;; + --path) image_path=$2 ;; + --help) usage; exit 0 ;; + *) usage 1>&2; exit 1;; + esac + shift 2 +done + +if [[ -z "${image_name-}" || -z "${image_path-}" ]]; then + usage 1>&2 + exit 1 +fi + +export IBMCLOUD_HOME=$(pwd -P) +./login.sh + +region=${IBMCLOUD_COS_REGION:-jp-tok} +cos_service_endpoint=${IBMCLOUD_COS_SERVICE_ENDPOINT:-https://s3.jp-tok.cloud-object-storage.appdomain.cloud} +cos_service_instance=$IBMCLOUD_COS_SERVICE_INSTANCE +cos_bucket=$IBMCLOUD_COS_BUCKET + +object_key="$(basename "$image_path")" + +cos_crn=$(ibmcloud resource service-instance --output json "$cos_service_instance" | jq -r '.[].guid') + +ibmcloud cos config auth --method iam +ibmcloud cos config endpoint-url --url "$cos_service_endpoint" +ibmcloud cos config crn --crn "$cos_crn" + +echo -e "\nChecking any old image with name \"$image_name\"\n" +timeout=300 +interval=15 +while image_id=$(ibmcloud is images --output=json --visibility=private | jq -r ".[] | select(.name == \"$image_name\") | .id") && [[ -n "$image_id" ]]; do + + image_json=$(ibmcloud is image --output json "$image_id" || true) + [[ -z "$image_json" ]] && break + + image_status=$(jq -r .status <<< "$image_json") + if [[ "$image_status" == available ]]; then + + image_sha256=$(jq -r .file.checksums.sha256 <<< "$image_json") + if echo "$image_sha256" "$image_path" | sha256sum --check; then + echo -e "\nImage \"$image_name\" already exists as $image_id with the same content. No need to push\n" + echo -e "$image_id\n" + exit 0 + fi + + echo -e "\nImage \"$image_name\" already exists with a different content. Delete the old image\n" + ibmcloud is image-delete --force "$image_id" || true + + while ibmcloud is image --output json "$image_id" | jq -r '"Image status: \"" + .status + "\""'; do + sleep $interval + (( timeout -= $interval )) + done + else + echo "Image \"$image_name\" already exists, and its status is \"$image_status\"" + if (( $timeout <= 0 )); then + echo "Error: old image $image_id exists, but does not become available in $timeout seconds" 1>&2 + exit 1 + fi + echo "Check the image status again in $interval seconds..." + + sleep "$interval" + (( timeout -= $interval )) + fi +done + +echo -e "\nUploading $image_path to cloud object stroage at "$cos_bucket" with key $object_key\n" +./multipart_upload.sh --bucket "$cos_bucket" --file "$image_path" + +ibmcloud cos object-head --bucket "$cos_bucket" --key "$object_key" + +if [[ ! "$region" =~ - ]]; then + location="${region}-geo" +else + location=$region +fi +image_ref="cos://$location/$cos_bucket/$object_key" + +arch=$(uname -m) +[ "${SE_BOOT:-0}" = "1" ] && os_name="hyper-protect-1-0-s390x" || os_name="ubuntu-20-04-${arch/x86_64/amd64}" + +echo -e "\nCreating image \"$image_name\" with $image_ref\n" +image_json=$(ibmcloud is image-create "$image_name" --os-name "$os_name" --file "$image_ref" --output json) +image_id=$(jq -r .id <<< "$image_json") + +echo -e "Waiting until image \"$image_name\" becomes available\n" +timeout=300 +interval=30 +while [[ "$(jq -r .status <<< "$image_json")" != available ]]; do + jq -r '"Image status: \"" + .status + "\" [" + .status_reasons[].message + "]"' <<< "$image_json" + if (( $timeout <= 0 )); then + echo "Error: image $image_id does not become available in $timeout seconds" 1>&2 + exit 1 + fi + sleep "$interval" + image_json=$(ibmcloud is image --output json "$image_id") + (( timeout -= $interval )) +done + +echo -e "\nImage \"$image_name\" is now available\n" +echo -e "$image_id\n" diff --git a/ibmcloud/image/verify.sh b/ibmcloud/image/verify.sh new file mode 100755 index 000000000..b2fd6e323 --- /dev/null +++ b/ibmcloud/image/verify.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit -o pipefail -o nounset + +cd "$(dirname "${BASH_SOURCE[0]}")" + +function usage() { + echo "Usage: $0 --image " +} + +while (( $# )); do + case "$1" in + --image) image=$2 ;; + --help) usage; exit 0 ;; + *) usage 1>&2; exit 1;; + esac + shift 2 +done + +if [[ -z "${image-}" ]]; then + usage 1>&2 + exit 1 +fi + +vpc=$IBMCLOUD_VPC_NAME +zone=$IBMCLOUD_VPC_ZONE +subnet=$IBMCLOUD_VPC_SUBNET_NAME +image=${image%.qcow2} + +case "$image" in + *-amd64) profile=bx2-2x8 ;; + *-s390x) profile=bz2-2x8 ;; + *) echo "$0: image for unknown architecture: $image" 1>&2; exit 1 ;; +esac + +[ "${SE_BOOT:-0}" = "1" ] && profile=bz2e-2x8 + +name=$(printf "imagetest-%.8s-%s" "$(uuidgen)" "$image") + +export IBMCLOUD_HOME=$(pwd -P) +./login.sh + +echo "Create an instance of $image with profle $profile" + +id=$(ibmcloud is instance-create "$name" "$vpc" "$zone" "$profile" "$subnet" --image "$image" --output json | jq -r .id) + +trap "ibmcloud is instance-delete -f '$id'" 0 + +echo "Wait for instance $id to become \"running\"" + +while ! ibmcloud is instance $id --output json | jq -e '.status == "running"' > /dev/null; do + sleep 5 +done + +if [[ -n "${SKIP_VERIFY_CONSOLE:-}" ]]; then + echo "SKIP_VERIFY_CONSOLE is set. Skipping console check..." + exit 0 +fi + +echo "Watch console log..." + +python3 - "$id" <<'END' +import os +import pty +import sys +from select import select +from time import time + +cmd = ["ibmcloud", "is", "instance-console", sys.argv[1]] + +pid, fd = pty.fork() +if pid == 0: + os.execlp(cmd[0], *cmd) + +deadline = time() + 600 +exit_status = 1 +count = 0 + +while True: + fds, _, _ = select([fd], [], [], 5) + if fd in fds: + try: + data = os.read(fd, 1024) + except OSError: + data = b"" + if not data: + break + os.write(1, data) + if data.find(b" login: ") >= 0: + count += 1 + if data.find(b"Reached target ") >= 0 and data.find(b"Cloud-init target") >= 0: + count += 1 + if count >= 3: + exit_status = 0 + break + elif time() < deadline: + os.write(fd, b"\n") # send "return" + else: + break + +os.write(fd, b"\035\n") # send ^[ +os.write(1, b"\n") +sys.exit(exit_status) +END diff --git a/ibmcloud/terraform/.gitignore b/ibmcloud/terraform/.gitignore new file mode 100644 index 000000000..47ffd63cb --- /dev/null +++ b/ibmcloud/terraform/.gitignore @@ -0,0 +1,11 @@ +.terraform* +terraform.tfvars +terraform.tfstate* +/cluster/ansible/inventory +/cluster/ansible/group_vars/all +/podvm-build/ansible/inventory +/podvm-build/ansible/group_vars/all +/start-cloud-api-adaptor/ansible/inventory +/start-cloud-api-adaptor/ansible/group_vars/all +/run-nginx-demo/ansible/inventory +/ansible diff --git a/ibmcloud/terraform/check-podvm-instance/main.tf b/ibmcloud/terraform/check-podvm-instance/main.tf new file mode 100644 index 000000000..55617a184 --- /dev/null +++ b/ibmcloud/terraform/check-podvm-instance/main.tf @@ -0,0 +1,25 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +locals { + podvm_image_id = var.podvm_image_name != null ? data.ibm_is_image.podvm_image[0].id : var.podvm_image_id + number_of_peer_pod_vms = length(local.peer_pod_vms) + peer_pod_vms = [ + for instance in var.virtual_server_instances.instances: instance.name + if instance.image == local.podvm_image_id + ] +} + +data "ibm_is_image" "podvm_image" { + count = var.podvm_image_name != null ? 1 : 0 + name = var.podvm_image_name +} + +resource "null_resource" "check" { + provisioner "local-exec" { + command = "if [ ${local.number_of_peer_pod_vms} -eq 1 ]; then echo 1 Virtual Server instance ${local.peer_pod_vms[0]} that uses the peer pod VM found. Test passed; else echo ${local.number_of_peer_pod_vms} Virtual Server instances that use the peer pod VM found. Test failed; exit 1; fi" + } +} + diff --git a/ibmcloud/terraform/check-podvm-instance/outputs.tf b/ibmcloud/terraform/check-podvm-instance/outputs.tf new file mode 100644 index 000000000..fccbd701d --- /dev/null +++ b/ibmcloud/terraform/check-podvm-instance/outputs.tf @@ -0,0 +1,3 @@ +output "peer_pod_instances" { + value = local.peer_pod_vms +} \ No newline at end of file diff --git a/ibmcloud/terraform/check-podvm-instance/provider.tf b/ibmcloud/terraform/check-podvm-instance/provider.tf new file mode 100644 index 000000000..4d5aed54e --- /dev/null +++ b/ibmcloud/terraform/check-podvm-instance/provider.tf @@ -0,0 +1,4 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = var.region_name +} \ No newline at end of file diff --git a/ibmcloud/terraform/check-podvm-instance/variables.tf b/ibmcloud/terraform/check-podvm-instance/variables.tf new file mode 100644 index 000000000..8cc463bf6 --- /dev/null +++ b/ibmcloud/terraform/check-podvm-instance/variables.tf @@ -0,0 +1,21 @@ +variable "ibmcloud_api_key" { + sensitive = true +} + +variable "region_name" { + default = "jp-tok" +} + +variable "podvm_image_name" { + description = "Name of the VPC Custom Image used for the peer pod VM. Must be provided if podvm_image_id is not provided" + default = null +} + +variable "podvm_image_id" { + description = "ID of the VPC Custom Image used for the peer pod VM. Must be provided if podvm_image_name is not provided" + default = null +} + +variable "virtual_server_instances" { + description = "List of Virtual Server instances currently running in the VPC. Obtained by calling data \"ibm_is_instances\" \"instances\"" +} diff --git a/ibmcloud/terraform/check-podvm-instance/version.tf b/ibmcloud/terraform/check-podvm-instance/version.tf new file mode 100644 index 000000000..a86a95e90 --- /dev/null +++ b/ibmcloud/terraform/check-podvm-instance/version.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = "~> 1.43.0" + } + } +} \ No newline at end of file diff --git a/ibmcloud/terraform/cluster/ansible/kata-playbook.yml b/ibmcloud/terraform/cluster/ansible/kata-playbook.yml new file mode 100644 index 000000000..0d8605e30 --- /dev/null +++ b/ibmcloud/terraform/cluster/ansible/kata-playbook.yml @@ -0,0 +1,258 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +- hosts: all + remote_user: root + tasks: + - name: Install deb packages + apt: + name: + - "linux-modules-extra-{{ ansible_kernel }}" + - build-essential + - jq + - qemu-utils + - libgpgme-dev + - libassuan-dev + - libdevmapper-dev + - libseccomp-dev + - pkg-config + - git + - kpartx + - protobuf-compiler + - musl-tools + - cryptsetup + + - name: Install deb packages for Ubuntu 20.04 or later + apt: + name: + - libbtrfs-dev + when: + - ansible_facts['distribution'] == "Ubuntu" + - ansible_facts['distribution_major_version'] | int >= 20 + + - name: Install deb packages for Ubuntu 18.04 or before + apt: + name: + - btrfs-tools + when: + - ansible_facts['distribution'] == "Ubuntu" + - ansible_facts['distribution_major_version'] | int < 20 + + - name: Configure kernel modules to load at boot + copy: + dest: "{{ item.path }}" + content: "{{ item.content }}" + with_items: + - path: /etc/modules-load.d/peerpod.conf + content: vrf + + - name: Load kernel modules + modprobe: + name: "{{ item }}" + with_items: + - vrf + + - name: Install Go + shell: | + set -o errexit -o pipefail + arch="{{ ansible_architecture }}" + # gover=$(curl -sL 'https://golang.org/VERSION?m=text') + gover=go1.19.3 + curl -sL "https://go.dev/dl/$gover.linux-${arch/x86_64/amd64}.tar.gz" | tar -xzf - -C /usr/local + + if ! grep -q '^PATH=/usr/local/go/bin:\$PATH$' /root/.bashrc; then + echo 'PATH=/usr/local/go/bin:$PATH' >> /root/.bashrc + fi + args: + executable: /bin/bash + creates: /usr/local/go/bin/go + + - name: Install containerd + shell: | + set -o errexit -o pipefail + rm -fr /tmp/containerd + git clone -b "{{ containerd_branch }}" "{{ containerd_repo }}" /tmp/containerd + (cd /tmp/containerd && make && make install) + rm -fr /tmp/containerd + environment: + PATH: /usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + args: + executable: /bin/bash + creates: /usr/local/bin/containerd + + - name: Ensure /etc/containerd directory exists + file: + path: /etc/containerd + state: directory + + - name: Copy containerd config file + copy: + dest: /etc/containerd/config.toml + content: | + version = 2 + root = "/var/lib/containerd" + state = "/run/containerd" + oom_score = -999 + + [grpc] + address = "/run/containerd/containerd.sock" + uid = 0 + gid = 0 + + [debug] + address = "/run/containerd/debug.sock" + uid = 0 + gid = 0 + level = "debug" + + [plugins] + [plugins."io.containerd.runtime.v1.linux"] + shim_debug = true + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] + runtime_type = "io.containerd.kata.v2" + cri_handler = "cc" + + - name: Copy systemd unit file for containerd + copy: + dest: /etc/systemd/system/containerd.service + content: | + [Unit] + Description=containerd container runtime + Documentation=https://containerd.io + After=network.target + + [Service] + ExecStartPre=-/sbin/modprobe overlay + ExecStart=/usr/local/bin/containerd --config /etc/containerd/config.toml --log-level debug + Delegate=yes + KillMode=process + + [Install] + WantedBy=multi-user.target + + - name: Start containerd service + systemd: + name: containerd.service + state: started + + - name: Ensure /etc/kata-containers directory exists + file: + path: /etc/kata-containers + state: directory + + - name: Install Rust + shell: | + set -o errexit -o pipefail + arch="{{ ansible_architecture }}" + + curl --proto '=https' --tlsv1.2 -sSf -o /tmp/rustup-init https://sh.rustup.rs + sh /tmp/rustup-init -y + rm /tmp/rustup-init + source /root/.cargo/env + + case "$arch" in + x86_64) rustup target add "$arch-unknown-linux-musl" ;; + esac + + if ! grep -q '^source "\$HOME/.cargo/env"$' /root/.bashrc; then + echo 'source "$HOME/.cargo/env"' >> /root/.bashrc + fi + args: + executable: /bin/bash + creates: /root/.cargo/bin/cargo + + - name: Install IBM Cloud CLI + shell: | + set -o errexit -o pipefail + curl -fsSL https://clis.cloud.ibm.com/install/linux | sh + args: + executable: /bin/bash + creates: /usr/local/bin/ibmcloud + + - name: Install IBM Cloud CLI plugins + shell: | + set -o pipefail + plugin="{{ item }}" + + installed_plugins=($(ibmcloud plugin list --output json | jq -r '.[].Name')) + (( $? > 0 )) && exit 2 + + [[ " ${installed_plugins[*]} " =~ " $plugin " ]] && exit 0 + + ibmcloud plugin install "$plugin" + (( $? > 0 )) && exit 2 + exit 1 + args: + executable: /bin/bash + register: result + changed_when: result.rc == 1 + failed_when: result.rc > 1 + with_items: + - vpc-infrastructure + - cloud-object-storage + + - name: Checkout the Kata containers repository + shell: | + set -o errexit -o pipefail + cd /root + git clone -b "{{ kata_containers_branch }}" "{{ kata_containers_repo }}" + args: + executable: /bin/bash + creates: /root/kata-containers + + - name: Checkout the cloud-api-adaptor repository + shell: | + set -o errexit -o pipefail + cd /root + git clone -b "{{ cloud_api_adaptor_branch }}" "{{ cloud_api_adaptor_repo }}" + args: + executable: /bin/bash + creates: /root/cloud-api-adaptor + + - name: Install the Kata shim + shell: | + set -o errexit + cd /root/kata-containers/src/runtime + PATH=/usr/local/go/bin:$PATH make $PWD/containerd-shim-kata-v2 + install containerd-shim-kata-v2 /usr/local/bin/ + args: + executable: /bin/bash + creates: /usr/local/bin/containerd-shim-kata-v2 + + - name: Copy configuration file for kata containers + copy: + dest: /etc/kata-containers/configuration.toml + content: | + [runtime] + internetworking_model = "none" + disable_new_netns = true + disable_guest_seccomp = true + enable_pprof = true + enable_debug = true + [hypervisor.remote] + remote_hypervisor_socket = "/run/peerpod/hypervisor.sock" + remote_hypervisor_timeout = 600 + disable_guest_selinux = true + [agent.kata] + [image] + service_offload = true + + - name: Install cloud-api-adaptor + shell: | + set -o errexit + cd /root/cloud-api-adaptor + PATH=/usr/local/go/bin:$PATH + go mod tidy + CLOUD_PROVIDER=ibmcloud make cloud-api-adaptor + install cloud-api-adaptor /usr/local/bin/ + args: + executable: /bin/bash + creates: /usr/local/bin/cloud-api-adaptor diff --git a/ibmcloud/terraform/cluster/ansible/kube-playbook.yml b/ibmcloud/terraform/cluster/ansible/kube-playbook.yml new file mode 100644 index 000000000..66b5b8353 --- /dev/null +++ b/ibmcloud/terraform/cluster/ansible/kube-playbook.yml @@ -0,0 +1,218 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +- hosts: all + remote_user: root + tasks: + - name: Add Kubernetes apt repository signing key + apt_key: + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + + - name: Add Kubernetes apt repository + apt_repository: + repo: deb https://apt.kubernetes.io/ kubernetes-xenial main + filename: kubernetes + + - name: Install deb packages + apt: + name: + - "linux-modules-extra-{{ ansible_kernel }}" + - build-essential + - jq + - libseccomp-dev + - pkg-config + - kubelet + - kubeadm + - kubectl + - kubernetes-cni + - cri-tools + - git + - kpartx + + - name: Install deb packages for Ubuntu 20.04 or later + apt: + name: + - libbtrfs-dev + when: + - ansible_facts['distribution'] == "Ubuntu" + - ansible_facts['distribution_major_version'] | int >= 20 + + - name: Install deb packages for Ubuntu 18.04 or before + apt: + name: + - btrfs-tools + when: + - ansible_facts['distribution'] == "Ubuntu" + - ansible_facts['distribution_major_version'] | int < 20 + + - name: Hold deb packages + dpkg_selections: + name: "{{ item }}" + selection: hold + with_items: + - kubelet + - kubeadm + - kubectl + - kubernetes-cni + - cri-tools + + - name: Add local IP address in /etc/hosts + shell: | + if ! grep -q '^[^#].*\b{{ ansible_hostname }}\b' /etc/hosts; then + echo '{{ ansible_default_ipv4.address }} {{ ansible_hostname }}' >> /etc/hosts || exit 2 + exit 1 + fi + register: result + changed_when: result.rc == 1 + failed_when: result.rc > 1 + + - name: Disable swap + shell: | + [ -z "$(swapon --show)" ] && exit 0 + swapoff --all && exit 1 + exit 2 + register: result + changed_when: result.rc == 1 + failed_when: result.rc > 1 + + - name: Disable swap in fstab + replace: + path: /etc/fstab + regexp: '^([^#\s]+\s+){2}swap\s' + replace: '# \1' + + - name: Configure kernel modules to load at boot + copy: + dest: "{{ item.path }}" + content: "{{ item.content }}" + with_items: + - path: /etc/modules-load.d/k8s.conf + content: br_netfilter + + - name: Load kernel modules + modprobe: + name: "{{ item }}" + with_items: + - br_netfilter + + - name: Set sysctl parameters for Kubernetes + sysctl: + name: "{{ item.name }}" + value: "{{ item.value }}" + sysctl_file: /etc/sysctl.d/k8s.conf + sysctl_set: yes + reload: yes + with_items: + - { name: net.ipv4.ip_forward, value: 1 } + - { name: net.bridge.bridge-nf-call-ip6tables, value: 1 } + - { name: net.bridge.bridge-nf-call-iptables, value: 1 } + + - name: Install runc but kube might install its own version + shell: | + set -o errexit -o pipefail + arch="{{ ansible_architecture }}" + runc_release_url="https://api.github.com/repos/opencontainers/runc/releases/latest" + runc_download_url=$(curl -sL "$runc_release_url" | jq --arg arch "${arch/x86_64/amd64}" -r '.assets[] | select(.name == "runc."+$arch) | .browser_download_url') + curl -sL -o /usr/local/bin/runc "$runc_download_url" + chmod 755 /usr/local/bin/runc + args: + executable: /bin/bash + creates: /usr/local/bin/runc + + - name: Install Go CAA go scripts + shell: | + set -o errexit -o pipefail + arch="{{ ansible_architecture }}" + #gover=$(curl -sL 'https://golang.org/VERSION?m=text') + gover=go1.18.5 + curl -sL "https://go.dev/dl/$gover.linux-${arch/x86_64/amd64}.tar.gz" | tar -xzf - -C /usr/local + + if ! grep -q '^PATH=/usr/local/go/bin:\$PATH$' /root/.bashrc; then + echo 'PATH=/usr/local/go/bin:$PATH' >> /root/.bashrc + fi + args: + executable: /bin/bash + creates: /usr/local/go/bin/go + + - name: Create GOPATH + ansible.builtin.lineinfile: + path: /root/.bashrc + regexp: '^export GOPATH=' + line: export GOPATH=/root/go + + - name: Install containerd + shell: | + set -o errexit -o pipefail + rm -fr /tmp/containerd + git clone -b "{{ containerd_branch }}" "{{ containerd_repo }}" /tmp/containerd + (cd /tmp/containerd && make && make install) + rm -fr /tmp/containerd + environment: + PATH: /usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + args: + executable: /bin/bash + creates: /usr/local/bin/containerd + + - name: Ensure /etc/containerd directory exists specific to config + file: + path: /etc/containerd + state: directory + + - name: Copy containerd config file + copy: + dest: /etc/containerd/config.toml + content: | + version = 2 + root = "/var/lib/containerd" + state = "/run/containerd" + oom_score = -999 + + [grpc] + address = "/run/containerd/containerd.sock" + uid = 0 + gid = 0 + + [debug] + address = "/run/containerd/debug.sock" + uid = 0 + gid = 0 + level = "debug" + + [plugins] + [plugins."io.containerd.runtime.v1.linux"] + shim_debug = true + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] + runtime_type = "io.containerd.kata.v2" + cri_handler = "cc" + + - name: Copy systemd unit file for containerd + copy: + dest: /etc/systemd/system/containerd.service + content: | + [Unit] + Description=containerd container runtime + Documentation=https://containerd.io + After=network.target + + [Service] + ExecStartPre=-/sbin/modprobe overlay + ExecStart=/usr/local/bin/containerd --config /etc/containerd/config.toml --log-level debug + Delegate=yes + KillMode=process + + [Install] + WantedBy=multi-user.target + + - name: Start containerd service + systemd: + name: containerd.service + state: started + diff --git a/ibmcloud/terraform/cluster/main.tf b/ibmcloud/terraform/cluster/main.tf new file mode 100644 index 000000000..6ded8f905 --- /dev/null +++ b/ibmcloud/terraform/cluster/main.tf @@ -0,0 +1,138 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +locals { + template_name = "${var.cluster_name}-k8s-node" + controlplane_name = "${var.cluster_name}-cp" + controlplane_floating_ip_name = "${local.controlplane_name}-ip" + worker_name = "${var.cluster_name}-worker" + worker_floating_ip_name = "${local.worker_name}-ip" + controlplane_ip = resource.ibm_is_instance.controlplane.primary_network_interface[0].primary_ipv4_address + worker_ip = resource.ibm_is_instance.worker.primary_network_interface[0].primary_ipv4_address + bastion_ip = resource.ibm_is_floating_ip.worker.address + vpc_id = var.vpc_name != null ? data.ibm_is_vpc.vpc[0].id : var.vpc_id + primary_security_group_id = var.primary_security_group_name != null ? data.ibm_is_security_group.primary[0].id : var.primary_security_group_id + primary_subnet_id = var.primary_subnet_name != null ? data.ibm_is_subnet.primary[0].id : var.primary_subnet_id +} + +resource "ibm_is_ssh_key" "created_ssh_key" { + # Create the ssh key only if the public key is set + count = var.ssh_pub_key == "" ? 0 : 1 + name = var.ssh_key_name + public_key = var.ssh_pub_key +} + +data "ibm_is_ssh_key" "ssh_key" { + # Wait if the key needs creating first + depends_on = [ibm_is_ssh_key.created_ssh_key] + name = var.ssh_key_name +} + +data "ibm_is_image" "k8s_node" { + name = var.image_name +} + +data "ibm_is_vpc" "vpc" { + count = var.vpc_name != null ? 1 : 0 + name = var.vpc_name +} + +data "ibm_is_subnet" "primary" { + count = var.primary_subnet_name != null ? 1 : 0 + name = var.primary_subnet_name +} + +data "ibm_is_security_group" "primary" { + count = var.primary_security_group_name != null ? 1 : 0 + name = var.primary_security_group_name +} + +resource "ibm_is_instance_template" "k8s_node" { + name = local.template_name + image = data.ibm_is_image.k8s_node.id + profile = var.instance_profile_name + vpc = local.vpc_id + zone = var.zone_name + keys = [data.ibm_is_ssh_key.ssh_key.id] + + primary_network_interface { + subnet = local.primary_subnet_id + security_groups = [local.primary_security_group_id] + } +} + +resource "ibm_is_instance" "controlplane" { + name = local.controlplane_name + instance_template = ibm_is_instance_template.k8s_node.id +} + +resource "ibm_is_floating_ip" "controlplane" { + name = local.controlplane_floating_ip_name + target = ibm_is_instance.controlplane.primary_network_interface[0].id +} + +resource "ibm_is_instance" "worker" { + name = local.worker_name + instance_template = ibm_is_instance_template.k8s_node.id +} + + +resource "ibm_is_floating_ip" "worker" { + name = local.worker_floating_ip_name + target = ibm_is_instance.worker.primary_network_interface[0].id +} + +resource "local_file" "inventory" { + filename = "${var.ansible_dir}/inventory" + content = < [--ssh-private-key ] host1 [host2...]" +} + +declare -a worker_nodes + +while (( $# )); do + case "$1" in + --bastion) bastion=$2 ;; + --ssh-private-key) ssh_private_key=$2 ;; + --help) usage; exit 0 ;; + *) break + esac + shift 2 +done + +hosts=("$@") + +if [[ "${#hosts[@]}" -eq 0 || -z "${bastion-}" ]]; then + usage 1>&2 + exit 1 +fi + +tmpdir=$(mktemp -d) +ssh_known_hosts="$tmpdir/known_hosts" + +function cleanup () { + rm -fr "$tmpdir" +} + +trap cleanup 0 + +opts=(-o StrictHostKeyChecking=accept-new -o "UserKnownHostsFile=$ssh_known_hosts") +if [[ -n "${ssh_private_key-}" ]]; then + opts+=( -o "IdentityFile=$ssh_private_key" ) +fi + +key_type=ed25519 +key="$tmpdir/id_$key_type" + +ssh "${opts[@]}" -o ConnectionAttempts=300 -o ConnectTimeout=1 -n -l root "$bastion" true + +ssh "${opts[@]}" -n -l root "$bastion" bash -c "true; [[ -e '/root/.ssh/id_$key_type' ]] || ssh-keygen -t '$key_type' -N '' -f '/root/.ssh/id_$key_type'" + +touch "$key" +chmod 600 "$key" +ssh "${opts[@]}" -n -l root "$bastion" cat "/root/.ssh/id_$key_type" > "$key" +ssh "${opts[@]}" -n -l root "$bastion" cat "/root/.ssh/id_$key_type.pub" > "$key.pub" + +port=22022 +ssh_ctl_sock="$tmpdir/ssh-ctl.sock" + +for host in "${hosts[@]}"; do + ( + function stop() { + ssh "${opts[@]}" -S "$ssh_ctl_sock" -O exit -l root "$bastion" || true + } + trap stop 0 + + ssh "${opts[@]}" -o ExitOnForwardFailure=yes -S "$ssh_ctl_sock" -L "$port:$host:22" -M -f -N -l root "$bastion" + [[ $(uname) = "Darwin" ]] && force_option="-f" + ssh-copy-id ${force_option:-} "${opts[@]}" -i "$key.pub" -p "$port" root@localhost + + ssh-keygen -R "[localhost]:$port" -f "$ssh_known_hosts" + ) +done diff --git a/ibmcloud/terraform/cluster/scripts/setup.sh b/ibmcloud/terraform/cluster/scripts/setup.sh new file mode 100755 index 000000000..044c15904 --- /dev/null +++ b/ibmcloud/terraform/cluster/scripts/setup.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit -o pipefail -o nounset + +cd "$(dirname "${BASH_SOURCE[0]}")" + +function usage() { + echo "Usage: $0 --bastion-node --control-plane-node --worker-nodes [--ssh-private-key ]" +} + +declare -a workers + +while (( $# )); do + case "$1" in + --control-plane) control_plane=$2 ;; + --workers) IFS=', ' read -a workers <<< "$2" ;; + --bastion) bastion=$2 ;; + --ssh-private-key) ssh_private_key=$2 ;; + --help) usage; exit 0 ;; + *) usage 1>&2; exit 1;; + esac + shift 2 +done + +if [[ -z "${control_plane-}" || "${#workers[@]}" -eq 0 || -z "${bastion-}" ]]; then + usage 1>&2 + exit 1 +fi + +tmpdir=$(mktemp -d) +ssh_ctl_sock="$tmpdir/ssh-ctl.sock" +ssh_known_hosts="$tmpdir/known_hosts" + +opts=(-l root -o StrictHostKeyChecking=accept-new -o "UserKnownHostsFile=$ssh_known_hosts" -o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p root@$bastion") + +ssh "${opts[@]}" "root@$control_plane" \ + bash -x -c "true +if ! [[ -e /var/lib/kubelet/kubeadm-flags.env ]]; then + kubeadm init --apiserver-advertise-address=$control_plane --pod-network-cidr=172.20.0.0/16 --cri-socket unix:///run/containerd/containerd.sock + mkdir -p /root/.kube + cp -f /etc/kubernetes/admin.conf /root/.kube/config + + # The default setting of Flannel conflicts with the IP range of the Tokyo zones + # https://cloud.ibm.com/docs/vpc?topic=vpc-configuring-address-prefixes&locale=en + + curl -sL -o /tmp/kube-flannel.yml https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml + sed -i 's|"10.244.0.0/16"|"172.20.0.0/16"|' /tmp/kube-flannel.yml + kubectl apply -f /tmp/kube-flannel.yml +fi +" + +for worker in "${workers[@]}"; do + ssh "${opts[@]}" "root@$worker" \ + bash -x -c "true +if ! [[ -e /var/lib/kubelet/kubeadm-flags.env ]]; then + ssh -o StrictHostKeyChecking=accept-new root@$control_plane tar -cf - -C /root .kube/config | tar xf - -C /root + \$(kubeadm token create --print-join-command) --cri-socket unix:///run/containerd/containerd.sock +fi +" +done diff --git a/ibmcloud/terraform/cluster/variables.tf b/ibmcloud/terraform/cluster/variables.tf new file mode 100644 index 000000000..52ebc4659 --- /dev/null +++ b/ibmcloud/terraform/cluster/variables.tf @@ -0,0 +1,97 @@ + +variable "ibmcloud_api_key" { + sensitive = true +} + +variable "cluster_name" {} +variable "ssh_key_name" {} + +variable "image_name" { + default = "ibm-ubuntu-20-04-3-minimal-amd64-1" +} + +variable "instance_profile_name" { + default = "bx2-2x8" +} + +variable "primary_subnet_id" { + description = "ID of the primary subnet. This or the primary subnet name must be provided" + default = null +} + +variable "primary_subnet_name" { + description = "Name of the primary subnet. This or the primary subnet ID must be provided" + default = null +} + +variable "primary_security_group_id" { + description = "ID of the primary security group. This or the primary security group name must be provided" + default = null +} + +variable "primary_security_group_name" { + description = "Name of the primary security group. This or the primary security group ID must be provided" + default = null +} + +variable "region_name" { + default = "jp-tok" +} + +variable "ssh_pub_key" { + default = "" +} + +variable "vpc_id" { + description = "ID of the VPC. This or the VPC name must be provided" + default = null +} + +variable "vpc_name" { + description = "Name of the VPC. This or the VPC ID must be provided" + default = null +} + +variable "zone_name" { + default = "jp-tok-2" +} + +variable "ansible_dir" { + description = "Subdirectory for Ansible playbook, inventory and vars files" + default = "./ansible" +} + +variable "scripts_dir" { + description = "Subdirectory for shell scripts" + default = "./scripts" +} + +variable "cloud_api_adaptor_repo" { + description = "Repository URL of Cloud API Adaptor" + default = "https://github.com/confidential-containers/cloud-api-adaptor.git" +} + +variable "cloud_api_adaptor_branch" { + description = "Branch name of Cloud API Adaptor" + default = "staging" +} + +variable "kata_containers_repo" { + description = "Repository URL of Kata Containers" + default = "https://github.com/kata-containers/kata-containers.git" +} + +variable "kata_containers_branch" { + description = "Branch name of Kata Containers" + default = "CCv0" +} + +variable "containerd_repo" { + description = "Repository URL of containerd" + default = "https://github.com/confidential-containers/containerd.git" +} + +variable "containerd_branch" { + description = "Branch name of containerd" + default = "CC-main" +} diff --git a/ibmcloud/terraform/cluster/versions.tf b/ibmcloud/terraform/cluster/versions.tf new file mode 100644 index 000000000..943f98c5e --- /dev/null +++ b/ibmcloud/terraform/cluster/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = "~> 1.43.0" + } + } +} diff --git a/ibmcloud/terraform/common/main.tf b/ibmcloud/terraform/common/main.tf new file mode 100644 index 000000000..e6085c958 --- /dev/null +++ b/ibmcloud/terraform/common/main.tf @@ -0,0 +1,69 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +resource "ibm_is_vpc" "vpc" { + name = var.vpc_name +} + +resource "ibm_is_public_gateway" "gateway" { + name = var.public_gateway_name + vpc = ibm_is_vpc.vpc.id + zone = var.zone_name + floating_ip = { + id = ibm_is_floating_ip.gateway.id + } +} + +resource "ibm_is_floating_ip" "gateway" { + name = var.floating_ip_name + zone = var.zone_name +} + +resource "ibm_is_subnet" "primary" { + name = var.primary_subnet_name + vpc = ibm_is_vpc.vpc.id + zone = var.zone_name + total_ipv4_address_count = 256 + public_gateway = ibm_is_public_gateway.gateway.id +} + +resource "ibm_is_security_group" "primary" { + name = var.primary_security_group_name + vpc = ibm_is_vpc.vpc.id +} + +resource "ibm_is_security_group_rule" "primary_outbound" { + group = ibm_is_security_group.primary.id + direction = "outbound" + remote = "0.0.0.0/0" +} + +resource "ibm_is_security_group_rule" "primary_inbound" { + group = ibm_is_security_group.primary.id + direction = "inbound" + remote = ibm_is_security_group.primary.id +} + +resource "ibm_is_security_group_rule" "primary_ssh" { + group = ibm_is_security_group.primary.id + direction = "inbound" + remote = "0.0.0.0/0" + + tcp { + port_min = 22 + port_max = 22 + } +} + +resource "ibm_is_security_group_rule" "primary_ping" { + group = ibm_is_security_group.primary.id + direction = "inbound" + remote = "0.0.0.0/0" + + icmp { + code = 0 + type = 8 + } +} diff --git a/ibmcloud/terraform/common/outputs.tf b/ibmcloud/terraform/common/outputs.tf new file mode 100644 index 000000000..5c034ec53 --- /dev/null +++ b/ibmcloud/terraform/common/outputs.tf @@ -0,0 +1,28 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +output "primary_security_group_id" { + value = ibm_is_security_group.primary.id +} + +output "primary_subnet_id" { + value = ibm_is_subnet.primary.id +} + +output "vpc_id" { + value = ibm_is_vpc.vpc.id +} + +output "ssh_security_group_rule_id" { + value = ibm_is_security_group_rule.primary_ssh.id +} + +output "inbound_security_group_rule_id" { + value = ibm_is_security_group_rule.primary_inbound.id +} + +output "outbound_security_group_rule_id" { + value = ibm_is_security_group_rule.primary_outbound.id +} \ No newline at end of file diff --git a/ibmcloud/terraform/common/provider.tf b/ibmcloud/terraform/common/provider.tf new file mode 100644 index 000000000..c7c4880aa --- /dev/null +++ b/ibmcloud/terraform/common/provider.tf @@ -0,0 +1,4 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = var.region_name +} diff --git a/ibmcloud/terraform/common/variables.tf b/ibmcloud/terraform/common/variables.tf new file mode 100644 index 000000000..de3841632 --- /dev/null +++ b/ibmcloud/terraform/common/variables.tf @@ -0,0 +1,32 @@ + +variable "ibmcloud_api_key" { + sensitive = true +} + +variable "floating_ip_name" { + default = "tok-gateway-ip" +} + +variable "primary_security_group_name" { + default = "tok-primary-security-group" +} + +variable "primary_subnet_name" { + default = "tok-primary-subnet" +} + +variable "public_gateway_name" { + default = "tok-gateway" +} + +variable "region_name" { + default = "jp-tok" +} + +variable "vpc_name" { + default = "tok-vpc" +} + +variable "zone_name" { + default = "jp-tok-2" +} \ No newline at end of file diff --git a/ibmcloud/terraform/common/versions.tf b/ibmcloud/terraform/common/versions.tf new file mode 100644 index 000000000..943f98c5e --- /dev/null +++ b/ibmcloud/terraform/common/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = "~> 1.43.0" + } + } +} diff --git a/ibmcloud/terraform/cos/main.tf b/ibmcloud/terraform/cos/main.tf new file mode 100644 index 000000000..505bbf273 --- /dev/null +++ b/ibmcloud/terraform/cos/main.tf @@ -0,0 +1,39 @@ +locals { + resource_group_id = var.resource_group_name != null ? data.ibm_resource_group.group[0].id : data.ibm_resource_group.default_group.id + cos_bucket_region = var.cos_bucket_region != "" ? var.cos_bucket_region : var.region_name +} + +data "ibm_resource_group" "group" { + count = var.resource_group_name != null ? 1 : 0 + name = var.resource_group_name +} + +data "ibm_resource_group" "default_group" { + is_default = "true" +} + +########## Create a COS instance +resource "ibm_resource_instance" "cos_instance" { + name = var.cos_service_instance_name + service = "cloud-object-storage" + plan = "standard" + location = "global" + resource_group_id = local.resource_group_id +} + +####### Create IAM Authorization Policy +resource "ibm_iam_authorization_policy" "policy" { + source_service_name = "is" + source_resource_type = "image" + target_service_name = "cloud-object-storage" + target_resource_instance_id = ibm_resource_instance.cos_instance.guid + roles = ["Reader"] +} + +######## Create COS bucket +resource "ibm_cos_bucket" "bucket" { + bucket_name = var.cos_bucket_name + resource_instance_id = ibm_resource_instance.cos_instance.id + region_location = local.cos_bucket_region + storage_class = "standard" +} diff --git a/ibmcloud/terraform/cos/output.tf b/ibmcloud/terraform/cos/output.tf new file mode 100644 index 000000000..8c86a440c --- /dev/null +++ b/ibmcloud/terraform/cos/output.tf @@ -0,0 +1,37 @@ +############################################################################## +# Outputs +############################################################################## +output "resource_group_id" { + description = "Resource Group ID" + value = local.resource_group_id +} + +output "s3_region" { + description = "S3 Region" + value = ibm_cos_bucket.bucket.region_location +} + +output "s3_endpoint_private" { + description = "S3 private endpoint" + value = ibm_cos_bucket.bucket.s3_endpoint_private +} + +output "s3_endpoint_public" { + description = "S3 public endpoint" + value = ibm_cos_bucket.bucket.s3_endpoint_public +} + +output "cos_bucket_name" { + description = "ID of the created COS bucket" + value = ibm_cos_bucket.bucket.bucket_name +} + +output "cos_instance_id" { + description = "ID of the created COS instance" + value = ibm_resource_instance.cos_instance.id +} + +output "cos_bucket_region" { + description = "Region of the created COS bucket" + value = ibm_cos_bucket.bucket.region_location +} diff --git a/ibmcloud/terraform/cos/provider.tf b/ibmcloud/terraform/cos/provider.tf new file mode 100644 index 000000000..f4b0bb207 --- /dev/null +++ b/ibmcloud/terraform/cos/provider.tf @@ -0,0 +1,4 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = var.region_name +} diff --git a/ibmcloud/terraform/cos/variables.tf b/ibmcloud/terraform/cos/variables.tf new file mode 100644 index 000000000..5b70c375c --- /dev/null +++ b/ibmcloud/terraform/cos/variables.tf @@ -0,0 +1,39 @@ +############################################################################## +# Input Variables +############################################################################## + +# Resource Group Variables +variable "resource_group_name" { + type = string + description = "The resource group ID where the environment will be created" + default = null +} + +variable "ibmcloud_api_key" { + description = "API key to login to IBM Cloud" + type = string + sensitive = true +} + +variable "region_name" { + description = "Name of the Region to deploy in to" + type = string + default = "jp-tok" +} + +variable "cos_bucket_name" { + description = "Name of the COS bucket to create" + type = string +} + +variable "cos_service_instance_name" { + description = "Name of the COS instance to create" + type = string + default = "cos-image-instance" +} + +variable "cos_bucket_region" { + description = "Name of the region in which to create the COS instance" + type = string + default = "" +} diff --git a/ibmcloud/terraform/cos/versions.tf b/ibmcloud/terraform/cos/versions.tf new file mode 100644 index 000000000..943f98c5e --- /dev/null +++ b/ibmcloud/terraform/cos/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = "~> 1.43.0" + } + } +} diff --git a/ibmcloud/terraform/main.tf b/ibmcloud/terraform/main.tf new file mode 100644 index 000000000..42708d3cf --- /dev/null +++ b/ibmcloud/terraform/main.tf @@ -0,0 +1,102 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +locals { + podvm_image_name_with_arch = length(regexall("s390x", var.image_name)) == 1 ? "${var.podvm_image_name}-s390x" : "${var.podvm_image_name}-amd64" +} + +module "common" { + source = "./common" + ibmcloud_api_key = var.ibmcloud_api_key + floating_ip_name = var.floating_ip_name + primary_security_group_name = var.primary_security_group_name + primary_subnet_name = var.primary_subnet_name + public_gateway_name = var.public_gateway_name + region_name = var.region_name + vpc_name = var.vpc_name + zone_name = var.zone_name +} + +module "cos" { + source = "./cos" + ibmcloud_api_key = var.ibmcloud_api_key + region_name = var.region_name + cos_bucket_name = var.cos_bucket_name + cos_service_instance_name = var.cos_service_instance_name + cos_bucket_region = var.cos_bucket_region +} + +module "cluster" { + source = "./cluster" + ibmcloud_api_key = var.ibmcloud_api_key + cluster_name = var.cluster_name + ssh_key_name = var.ssh_key_name + image_name = var.image_name + instance_profile_name = var.instance_profile_name + primary_subnet_id = module.common.primary_subnet_id + primary_security_group_id = module.common.primary_security_group_id + region_name = var.region_name + ssh_pub_key = var.ssh_pub_key + vpc_id = module.common.vpc_id + zone_name = var.zone_name + ansible_dir = "./cluster/ansible" + scripts_dir = "./cluster/scripts" + cloud_api_adaptor_repo = var.cloud_api_adaptor_repo + cloud_api_adaptor_branch = var.cloud_api_adaptor_branch + kata_containers_repo = var.kata_containers_repo + kata_containers_branch = var.kata_containers_branch + containerd_repo = var.containerd_repo + containerd_branch = var.containerd_branch +} + +module "podvm_build" { + source = "./podvm-build" + ibmcloud_api_key = var.ibmcloud_api_key + ibmcloud_user_id = var.ibmcloud_user_id + cos_bucket_name = module.cos.cos_bucket_name + cos_service_instance_id = module.cos.cos_instance_id + cos_bucket_region = module.cos.cos_bucket_region + primary_subnet_name = var.primary_subnet_name + region_name = var.region_name + use_ibmcloud_test = var.use_ibmcloud_test + vpc_name = var.vpc_name + worker_ip = module.cluster.worker_ip + bastion_ip = module.cluster.bastion_ip + podvm_image_name = local.podvm_image_name_with_arch + skip_verify_console = var.skip_verify_console + ansible_dir = "./podvm-build/ansible" +} + +module "start_cloud_api_adaptor" { + source = "./start-cloud-api-adaptor" + ibmcloud_api_key = var.ibmcloud_api_key + ssh_key_id = module.cluster.ssh_key_id + worker_ip = module.cluster.worker_ip + bastion_ip = module.cluster.bastion_ip + podvm_image_id = module.podvm_build.podvm_image_id + region_name = var.region_name + vpc_id = module.common.vpc_id + primary_subnet_id = module.common.primary_subnet_id + primary_security_group_id = module.common.primary_security_group_id + instance_profile_name = var.instance_profile_name + ssh_security_group_rule_id = module.common.ssh_security_group_rule_id + inbound_security_group_rule_id = module.common.inbound_security_group_rule_id + outbound_security_group_rule_id = module.common.outbound_security_group_rule_id + ansible_dir = "./start-cloud-api-adaptor/ansible" +} + +module "run_nginx_demo" { + source = "./run-nginx-demo" + ibmcloud_api_key = var.ibmcloud_api_key + worker_ip = module.cluster.worker_ip + bastion_ip = module.cluster.bastion_ip + region_name = var.region_name + podvm_image_id = module.start_cloud_api_adaptor.cloud_api_adaptor_podvm_image_id + vpc_id = module.common.vpc_id + ssh_security_group_rule_id = module.common.ssh_security_group_rule_id + inbound_security_group_rule_id = module.common.inbound_security_group_rule_id + outbound_security_group_rule_id = module.common.outbound_security_group_rule_id + ansible_dir = "./run-nginx-demo/ansible" +} diff --git a/ibmcloud/terraform/podvm-build/ansible/playbook.yml b/ibmcloud/terraform/podvm-build/ansible/playbook.yml new file mode 100644 index 000000000..2c886781e --- /dev/null +++ b/ibmcloud/terraform/podvm-build/ansible/playbook.yml @@ -0,0 +1,45 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# +--- + +- hosts: all + remote_user: root + environment: + CLOUD_PROVIDER: ibmcloud + IMAGE_NAME: "{{ ibmcloud_vpc_image_name }}" + tasks: + - name: Set COS service endpoint for IBM Cloud + set_fact: + ibmcloud_cos_service_endpoint: https://s3.{{ ibmcloud_cos_bucket_region }}.cloud-object-storage.appdomain.cloud + when: ibmcloud_api_endpoint == "https://cloud.ibm.com" + - name: Set COS service endpoint for IBM Cloud test + set_fact: + ibmcloud_cos_service_endpoint: https://s3.us-west.cloud-object-storage.test.appdomain.cloud + when: ibmcloud_api_endpoint == "https://test.cloud.ibm.com" + - name: Build peer pod VM image + shell: + cmd: | + . ~/.bashrc + make build + chdir: cloud-api-adaptor/ibmcloud/image + - name: Push peer pod VM image to Cloud Object Store and verify the image + environment: + IBMCLOUD_API_KEY: "{{ ibmcloud_api_key }}" + IBMCLOUD_API_ENDPOINT: "{{ ibmcloud_api_endpoint }}" + IBMCLOUD_COS_SERVICE_INSTANCE: "{{ ibmcloud_cos_service_instance }}" + IBMCLOUD_COS_BUCKET: "{{ ibmcloud_cos_bucket }}" + IBMCLOUD_COS_REGION: "{{ ibmcloud_cos_bucket_region }}" + IBMCLOUD_COS_SERVICE_ENDPOINT: "{{ ibmcloud_cos_service_endpoint }}" + IBMCLOUD_VPC_NAME: "{{ ibmcloud_vpc_name }}" + IBMCLOUD_VPC_SUBNET_NAME: "{{ ibmcloud_vpc_subnet_name }}" + IBMCLOUD_VPC_REGION: "{{ ibmcloud_region_name }}" + IBMCLOUD_VPC_ZONE: "{{ ibmcloud_vpc_zone }}" + SKIP_VERIFY_CONSOLE: "{{ ibmcloud_skip_verify_console }}" + shell: + cmd: | + . ~/.bashrc + make push + make verify + chdir: cloud-api-adaptor/ibmcloud/image diff --git a/ibmcloud/terraform/podvm-build/main.tf b/ibmcloud/terraform/podvm-build/main.tf new file mode 100644 index 000000000..d79606dad --- /dev/null +++ b/ibmcloud/terraform/podvm-build/main.tf @@ -0,0 +1,105 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +locals { + worker_name = var.cluster_name != null ? "${var.cluster_name}-worker" : null + worker_floating_ip_name = local.worker_name != null ? "${local.worker_name}-ip" : null + worker_ip = local.worker_name != null ? data.ibm_is_instance.worker[0].primary_network_interface[0].primary_ipv4_address : var.worker_ip + bastion_ip = local.worker_floating_ip_name != null ? data.ibm_is_floating_ip.worker[0].address : var.bastion_ip + cos_service_instance_name_or_id = var.cos_service_instance_name != null ? var.cos_service_instance_name : var.cos_service_instance_id + ibmcloud_api_endpoint = var.use_ibmcloud_test ? "https://test.cloud.ibm.com" : "https://cloud.ibm.com" + podvm_image_name = var.podvm_image_name != null ? var.podvm_image_name : "" + cos_bucket_region = var.cos_bucket_region != "" ? var.cos_bucket_region : var.region_name + skip_verify_console = var.skip_verify_console ? "true" : "" + + is_policies_and_roles = flatten([ + for policy in data.ibm_iam_user_policy.user_policies.policies: [ + for resource in policy.resources: policy.roles + if resource.service == "is" && resource.resource_group_id == "" && resource.resource_instance_id == "" + ] + ]) + has_console_administrator_role = anytrue([ + for role in local.is_policies_and_roles: true + if role == "Console Administrator" + ]) +} + +data "ibm_is_instance" "worker" { + count = local.worker_name != null ? 1 : 0 + name = local.worker_name +} + +data "ibm_is_floating_ip" "worker" { + count = local.worker_floating_ip_name != null ? 1 : 0 + name = local.worker_floating_ip_name +} + +resource "local_file" "inventory" { + filename = "${var.ansible_dir}/inventory" + content = < /var/log/cloud-api-adaptor.log 2>&1 & \ No newline at end of file diff --git a/ibmcloud/terraform/start-cloud-api-adaptor/ansible/destroy_playbook.yml b/ibmcloud/terraform/start-cloud-api-adaptor/ansible/destroy_playbook.yml new file mode 100644 index 000000000..2c1b7c32d --- /dev/null +++ b/ibmcloud/terraform/start-cloud-api-adaptor/ansible/destroy_playbook.yml @@ -0,0 +1,23 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# +--- + +- hosts: all + remote_user: root + tasks: + - name: Record the number of processes containing 'cloud-api-adaptor' + shell: + cmd: | + ps -ef | grep -v grep | grep cloud-api-adaptor | wc -l + register: process_count + - name: Stop the Cloud API Adaptor process + shell: + cmd: | + killall cloud-api-adaptor + when: process_count.stdout | int > 0 + - name: Delete the peer pod hypervisor socket if it exists + file: + path: /run/peerpod/hypervisor.sock + state: absent \ No newline at end of file diff --git a/ibmcloud/terraform/start-cloud-api-adaptor/main.tf b/ibmcloud/terraform/start-cloud-api-adaptor/main.tf new file mode 100644 index 000000000..5ea1cc9a9 --- /dev/null +++ b/ibmcloud/terraform/start-cloud-api-adaptor/main.tf @@ -0,0 +1,133 @@ +# +# (C) Copyright IBM Corp. 2022. +# SPDX-License-Identifier: Apache-2.0 +# + +locals { + worker_name = var.cluster_name != null ? "${var.cluster_name}-worker" : null + worker_floating_ip_name = local.worker_name != null ? "${local.worker_name}-ip" : null + podvm_image_id = var.podvm_image_name != null ? data.ibm_is_image.podvm_image[0].id : var.podvm_image_id + worker_ip = local.worker_name != null ? data.ibm_is_instance.worker[0].primary_network_interface[0].primary_ipv4_address : var.worker_ip + bastion_ip = local.worker_floating_ip_name != null ? data.ibm_is_floating_ip.worker[0].address : var.bastion_ip + zone_name = data.ibm_is_subnet.primary_subnet_by_id.zone + vpc_id = var.vpc_name != null ? data.ibm_is_vpc.vpc[0].id : var.vpc_id + ssh_key_id = var.ssh_key_name != null ? data.ibm_is_ssh_key.ssh_key[0].id : var.ssh_key_id + primary_subnet_id = var.primary_subnet_name != null ? data.ibm_is_subnet.primary_subnet_by_name[0].id : var.primary_subnet_id + primary_security_group_id = var.primary_security_group_name != null ? data.ibm_is_security_group.primary_security_group[0].id : var.primary_security_group_id + resource_group_id = var.resource_group_name != null ? data.ibm_resource_group.group[0].id : data.ibm_resource_group.default_group.id + ibmcloud_api_endpoint = var.use_ibmcloud_test ? "https://test.cloud.ibm.com" : "https://cloud.ibm.com" +} + +data "ibm_resource_group" "group" { + count = var.resource_group_name != null ? 1 : 0 + name = var.resource_group_name +} + +data "ibm_resource_group" "default_group" { + is_default = "true" +} + +data "ibm_is_instance" "worker" { + count = local.worker_name != null ? 1 : 0 + name = local.worker_name +} + +data "ibm_is_image" "podvm_image" { + count = var.podvm_image_name != null ? 1 : 0 + name = var.podvm_image_name +} + +data "ibm_is_vpc" "vpc" { + count = var.vpc_name != null ? 1 : 0 + name = var.vpc_name +} + +data "ibm_is_ssh_key" "ssh_key" { + count = var.ssh_key_name != null ? 1 : 0 + name = var.ssh_key_name +} + +data "ibm_is_subnet" "primary_subnet_by_name" { + count = var.primary_subnet_name != null ? 1 : 0 + name = var.primary_subnet_name +} + +data "ibm_is_subnet" "primary_subnet_by_id" { + identifier = local.primary_subnet_id +} + +data "ibm_is_security_group" "primary_security_group" { + count = var.primary_security_group_name != null ? 1 : 0 + name = var.primary_security_group_name +} + +data "ibm_is_floating_ip" "worker" { + count = local.worker_floating_ip_name != null ? 1 : 0 + name = local.worker_floating_ip_name +} + +resource "local_file" "inventory" { + filename = "${var.ansible_dir}/inventory" + content = < + kubectl label node $NODENAME node-role.kubernetes.io/worker= + ``` + +## Deploy webhook + + Please refer to the instructions available in the following [doc](../webhook/docs/INSTALL.md). + +## Deploy cloud-api-adaptor + +* set CLOUD_PROVIDER + ``` + export CLOUD_PROVIDER= + ``` + +* `make deploy` deploys operator, runtime and cloud-api-adaptor pod in the configured cluster + * validate kubectl is available in your `$PATH` and `$KUBECONFIG` is set + * configure install/overlays/$(CLOUD_PROVIDER)/kustomization.yaml with your own settings + * [setting up authenticated registry support](../docs/registries-authentication.md) + +* `make delete` deletes the daemonset from the configured cluster + +### Verify + +* Check POD status + + ``` + kubectl get pods -n confidential-containers-system + ``` + A successful install should show all the PODs with "Running" status under the `confidential-containers-system` + namespace. + + ``` + NAME READY STATUS RESTARTS AGE + cc-operator-controller-manager-dc4846d94-nfnr7 2/2 Running 0 20h + cc-operator-daemon-install-bdp89 1/1 Running 0 5s + cc-operator-pre-install-daemon-hclk9 1/1 Running 0 9s + cloud-api-adaptor-daemonset-aws-7c66d68484-zpnnw 1/1 Running 0 9s + ``` + +* Check `RuntimeClasses` + + ``` + kubectl get runtimeclass + ``` + A successful install should show `kata` related `RuntimeClasses` + ``` + NAME HANDLER AGE + kata kata 6m7s + kata-clh kata-clh 6m7s + kata-qemu kata-qemu 6m7s + ``` + +* View cloud-api-adaptor logs + + ``` + kubectl logs pod/cloud-api-adaptor-daemonset-aws-7c66d68484-zpnnw -n confidential-containers-system + ``` + +## Building custom cloud-api-adaptor image + +* Set CLOUD_PROVIDER + ``` + export CLOUD_PROVIDER= + ``` + +* Set container registry and image name + ``` + export registry=/ + ``` + +* Build the container image and push it to `$registry` + ``` + make image + ``` + +## Building custom runtime and pre-install images + + These instructions should help you build your own images for development and testing. + + Before proceeding ensure you can build the kata runtime and the agent successfully by + following the instructions mentioned in the following [link](../docs/DEVELOPMENT.md). + +### Building Runtime Payload Image + +* Set container registry and image name + ``` + export registry=/ + ``` + +* Build the container image and push it to `$registry` + ``` + cd runtime-payload + make binaries + make build + ``` + + +### Building Pre-Install Payload Image + +* Set container registry and image name + ``` + export registry=/ + ``` + +* Build the container image and push it to `$registry` + ``` + cd pre-install-payload + make build + ``` + diff --git a/install/overlays/aws/cri_runtime_endpoint.yaml b/install/overlays/aws/cri_runtime_endpoint.yaml new file mode 100644 index 000000000..13c3af9d9 --- /dev/null +++ b/install/overlays/aws/cri_runtime_endpoint.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cloud-api-adaptor-daemonset + namespace: confidential-containers-system + labels: + app: cloud-api-adaptor +spec: + template: + spec: + containers: + - name: cloud-api-adaptor-con + volumeMounts: + - mountPath: /run/peerpod/cri-runtime.sock # in-container CRI_RUNTIME_ENDPOINT default + name: cri-runtime-endpoint + volumes: + - name: cri-runtime-endpoint + hostPath: + path: /run/containerd/containerd.sock # SET! (crio's default: /var/run/crio/crio.sock) + type: Socket + +# to apply this uncomment the patchesStrategicMerge of this file in kustomization.yaml diff --git a/install/overlays/aws/kustomization.yaml b/install/overlays/aws/kustomization.yaml new file mode 100644 index 000000000..204d09a05 --- /dev/null +++ b/install/overlays/aws/kustomization.yaml @@ -0,0 +1,41 @@ +bases: +- ../../yamls +nameSuffix: -aws + +images: +- name: kustomize.this/cloud-api-adaptor/image:url + newName: quay.io/confidential-containers/cloud-api-adaptor-aws # change image if needed + newTag: latest + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- name: peer-pods-cm + namespace: confidential-containers-system + literals: + - CLOUD_PROVIDER="aws" + #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image + #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 + #- PODVM_LAUNCHTEMPLATE_NAME="" # Uncomment and set if you want to use launch template + # Comment out all the following variables if using launch template + - PODVM_AMI_ID="" #set + #- PODVM_INSTANCE_TYPE="t3.small" # caa defaults to t3.small + #- AWS_SG_IDS="" # comma separated, if not set all SGs will be retrieved from IMDS + #- AWS_REGION="" # if not set retrieved from IMDS + #- SSH_KP_NAME="" # if not set retrieved from IMDS + #- AWS_SUBNET_ID="" # if not set retrieved from IMDS + +secretGenerator: +- name: auth-json-secret + namespace: confidential-containers-system + files: + #- auth.json # set - path to auth.json pull credentials file +- name: peer-pods-secret + namespace: confidential-containers-system + literals: + - AWS_ACCESS_KEY_ID="" # set + - AWS_SECRET_ACCESS_KEY="" # set + +patchesStrategicMerge: + #- cri_runtime_endpoint.yaml # set (modify host's runtime cri socket path in the file, default is /run/containerd/containerd.sock) diff --git a/install/overlays/azure/cri_runtime_endpoint.yaml b/install/overlays/azure/cri_runtime_endpoint.yaml new file mode 100644 index 000000000..13c3af9d9 --- /dev/null +++ b/install/overlays/azure/cri_runtime_endpoint.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cloud-api-adaptor-daemonset + namespace: confidential-containers-system + labels: + app: cloud-api-adaptor +spec: + template: + spec: + containers: + - name: cloud-api-adaptor-con + volumeMounts: + - mountPath: /run/peerpod/cri-runtime.sock # in-container CRI_RUNTIME_ENDPOINT default + name: cri-runtime-endpoint + volumes: + - name: cri-runtime-endpoint + hostPath: + path: /run/containerd/containerd.sock # SET! (crio's default: /var/run/crio/crio.sock) + type: Socket + +# to apply this uncomment the patchesStrategicMerge of this file in kustomization.yaml diff --git a/install/overlays/azure/kustomization.yaml b/install/overlays/azure/kustomization.yaml new file mode 100644 index 000000000..f32155be9 --- /dev/null +++ b/install/overlays/azure/kustomization.yaml @@ -0,0 +1,44 @@ +bases: +- ../../yamls +nameSuffix: -azure + +images: +- name: kustomize.this/cloud-api-adaptor/image:url + newName: quay.io/confidential-containers/cloud-api-adaptor-azure # change image if needed + newTag: latest + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- name: peer-pods-cm + namespace: confidential-containers-system + literals: + - CLOUD_PROVIDER="azure" + - AZURE_SUBSCRIPTION_ID="" #set + - AZURE_REGION="eastus" #set + - AZURE_INSTANCE_SIZE="Standard_D8as_v5" #set + - AZURE_RESOURCE_GROUP="" #set + - AZURE_VM_NAME="" #set + - AZURE_IMAGE="" #set + #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image + #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 + +secretGenerator: +- name: auth-json-secret + namespace: confidential-containers-system + files: + #- auth.json # set - path to auth.json pull credentials file +- name: peer-pods-secret + namespace: confidential-containers-system + literals: + - AZURE_CLIENT_ID="" # set + - AZURE_CLIENT_SECRET="" # set + - AZURE_TENANT_ID="" #set +- name: ssh-key-secret + namespace: confidential-containers-system + files: # key generation example: ssh-keygen -f ./id_rsa -N "" && sudo cat id_rsa.pub >> /root/.ssh/authorized_keys + #- id_rsa # set - path to private key + +patchesStrategicMerge: + #- cri_runtime_endpoint.yaml # set (modify host's runtime cri socket path in the file, default is /run/containerd/containerd.sock) diff --git a/install/overlays/ibmcloud/cri_runtime_endpoint.yaml b/install/overlays/ibmcloud/cri_runtime_endpoint.yaml new file mode 100644 index 000000000..13c3af9d9 --- /dev/null +++ b/install/overlays/ibmcloud/cri_runtime_endpoint.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cloud-api-adaptor-daemonset + namespace: confidential-containers-system + labels: + app: cloud-api-adaptor +spec: + template: + spec: + containers: + - name: cloud-api-adaptor-con + volumeMounts: + - mountPath: /run/peerpod/cri-runtime.sock # in-container CRI_RUNTIME_ENDPOINT default + name: cri-runtime-endpoint + volumes: + - name: cri-runtime-endpoint + hostPath: + path: /run/containerd/containerd.sock # SET! (crio's default: /var/run/crio/crio.sock) + type: Socket + +# to apply this uncomment the patchesStrategicMerge of this file in kustomization.yaml diff --git a/install/overlays/ibmcloud/kustomization.yaml b/install/overlays/ibmcloud/kustomization.yaml new file mode 100644 index 000000000..129f0a9ca --- /dev/null +++ b/install/overlays/ibmcloud/kustomization.yaml @@ -0,0 +1,43 @@ +bases: +- ../../yamls +nameSuffix: -ibmcloud + +images: +- name: kustomize.this/cloud-api-adaptor/image:url + newName: quay.io/confidential-containers/cloud-api-adaptor-ibmcloud # change image if needed + newTag: latest + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- name: peer-pods-cm + namespace: confidential-containers-system + literals: + - CLOUD_PROVIDER="ibmcloud" + - IBMCLOUD_VPC_ENDPOINT="" #set + - IBMCLOUD_RESOURCE_GROUP_ID="" #set + - IBMCLOUD_SSH_KEY_ID="" #set + - IBMCLOUD_PODVM_IMAGE_ID="" #set + - IBMCLOUD_PODVM_INSTANCE_PROFILE_NAME="" #set + - IBMCLOUD_ZONE="" #set + - IBMCLOUD_VPC_SUBNET_ID="" #set + - IBMCLOUD_VPC_SG_ID="" #set + - IBMCLOUD_VPC_ID="" #set + #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image + #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 + +secretGenerator: +- name: auth-json-secret + namespace: confidential-containers-system + files: + #- auth.json # set - path to auth.json pull credentials file +- name: peer-pods-secret + namespace: confidential-containers-system + literals: + - IBMCLOUD_API_KEY="" # set + - IBMCLOUD_IAM_ENDPOINT="" #set + - IBMCLOUD_ZONE="" #set + +patchesStrategicMerge: + - cri_runtime_endpoint.yaml # set (modify host's runtime cri socket path in the file, default is /run/containerd/containerd.sock) diff --git a/install/overlays/libvirt/cri_runtime_endpoint.yaml b/install/overlays/libvirt/cri_runtime_endpoint.yaml new file mode 100644 index 000000000..13c3af9d9 --- /dev/null +++ b/install/overlays/libvirt/cri_runtime_endpoint.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cloud-api-adaptor-daemonset + namespace: confidential-containers-system + labels: + app: cloud-api-adaptor +spec: + template: + spec: + containers: + - name: cloud-api-adaptor-con + volumeMounts: + - mountPath: /run/peerpod/cri-runtime.sock # in-container CRI_RUNTIME_ENDPOINT default + name: cri-runtime-endpoint + volumes: + - name: cri-runtime-endpoint + hostPath: + path: /run/containerd/containerd.sock # SET! (crio's default: /var/run/crio/crio.sock) + type: Socket + +# to apply this uncomment the patchesStrategicMerge of this file in kustomization.yaml diff --git a/install/overlays/libvirt/kustomization.yaml b/install/overlays/libvirt/kustomization.yaml new file mode 100644 index 000000000..7ad9c0e5b --- /dev/null +++ b/install/overlays/libvirt/kustomization.yaml @@ -0,0 +1,38 @@ +bases: +- ../../yamls +nameSuffix: -libvirt + +images: +- name: kustomize.this/cloud-api-adaptor/image:url + newName: quay.io/confidential-containers/cloud-api-adaptor-libvirt # change image if needed + newTag: latest + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- name: peer-pods-cm + namespace: confidential-containers-system + literals: + - CLOUD_PROVIDER="libvirt" + - LIBVIRT_URI="qemu+ssh://root@192.168.122.1/system?no_verify=1" #set + - LIBVIRT_NET="default" # set + - LIBVIRT_POOL="default" # set + #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image + #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 + +secretGenerator: +- name: auth-json-secret + namespace: confidential-containers-system + files: + #- auth.json # set - path to auth.json pull credentials file +- name: ssh-key-secret + namespace: confidential-containers-system + files: # key generation example: ssh-keygen -f ./id_rsa -N "" && sudo cat id_rsa.pub >> /root/.ssh/authorized_keys + #- id_rsa # set - path to private key +- name: peer-pods-secret + namespace: confidential-containers-system + literals: + +patchesStrategicMerge: + #- cri_runtime_endpoint.yaml # set (modify host's runtime cri socket path in the file, default is /run/containerd/containerd.sock) diff --git a/install/overlays/vsphere/cri_runtime_endpoint.yaml b/install/overlays/vsphere/cri_runtime_endpoint.yaml new file mode 100644 index 000000000..13c3af9d9 --- /dev/null +++ b/install/overlays/vsphere/cri_runtime_endpoint.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cloud-api-adaptor-daemonset + namespace: confidential-containers-system + labels: + app: cloud-api-adaptor +spec: + template: + spec: + containers: + - name: cloud-api-adaptor-con + volumeMounts: + - mountPath: /run/peerpod/cri-runtime.sock # in-container CRI_RUNTIME_ENDPOINT default + name: cri-runtime-endpoint + volumes: + - name: cri-runtime-endpoint + hostPath: + path: /run/containerd/containerd.sock # SET! (crio's default: /var/run/crio/crio.sock) + type: Socket + +# to apply this uncomment the patchesStrategicMerge of this file in kustomization.yaml diff --git a/install/overlays/vsphere/kustomization.yaml b/install/overlays/vsphere/kustomization.yaml new file mode 100644 index 000000000..b6376061f --- /dev/null +++ b/install/overlays/vsphere/kustomization.yaml @@ -0,0 +1,34 @@ +bases: +- ../../yamls +nameSuffix: -vsphere + +images: +- name: kustomize.this/cloud-api-adaptor/image:url + newName: quay.io/confidential-containers/cloud-api-adaptor-vsphere # change image if needed + newTag: latest + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- name: peer-pods-cm + namespace: confidential-containers-system + literals: + - CLOUD_PROVIDER="vsphere" + - GOVC_URL="" # set + #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image + #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 + +secretGenerator: +- name: auth-json-secret + namespace: confidential-containers-system + files: + #- auth.json # set - path to auth.json pull credentials file +- name: peer-pods-secret + namespace: confidential-containers-system + literals: + - GOVC_USERNAME="" # set + - GOVC_PASSWORD="" # set + +patchesStrategicMerge: + #- cri_runtime_endpoint.yaml # set (modify host's runtime cri socket path in the file, default is /run/containerd/containerd.sock) diff --git a/install/pre-install-payload/Dockerfile b/install/pre-install-payload/Dockerfile new file mode 100644 index 000000000..5ad502c63 --- /dev/null +++ b/install/pre-install-payload/Dockerfile @@ -0,0 +1,18 @@ +ARG IMAGE +FROM ${IMAGE:-docker.io/library/centos}:7 + +ARG ARCH +ARG SYSTEMD_ARTIFACTS=./config/remote-hyp.service +ARG CAA_ARTIFACTS=./scripts +ARG DESTINATION=/opt/confidential-containers-pre-install-artifacts + +COPY ${SYSTEMD_ARTIFACTS} ${DESTINATION}/etc/systemd/system/ +COPY ${CAA_ARTIFACTS}/* ${DESTINATION}/scripts/ + +RUN \ +echo "[kubernetes]" >> /etc/yum.repos.d/kubernetes.repo && \ +echo "name=Kubernetes" >> /etc/yum.repos.d/kubernetes.repo && \ +echo "baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$(uname -m)" >> /etc/yum.repos.d/kubernetes.repo && \ +echo "gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg" >> /etc/yum.repos.d/kubernetes.repo && \ +yum -y install kubectl && \ +yum clean all diff --git a/install/pre-install-payload/Makefile b/install/pre-install-payload/Makefile new file mode 100644 index 000000000..d583aca9a --- /dev/null +++ b/install/pre-install-payload/Makefile @@ -0,0 +1,2 @@ +build: + hack/build.sh diff --git a/install/pre-install-payload/config/remote-hyp.service b/install/pre-install-payload/config/remote-hyp.service new file mode 100644 index 000000000..9594e7c98 --- /dev/null +++ b/install/pre-install-payload/config/remote-hyp.service @@ -0,0 +1,12 @@ +[Unit] +Description=Run cloud-api-adaptor after network becomes reachable + +[Service] +Type=simple +RemainAfterExit=yes +EnvironmentFile=-/run/hyp.env +ExecStart=/opt/confidential-containers/scripts/run-hyp.sh +TimeoutStartSec=0 + +[Install] +WantedBy=default.target diff --git a/install/pre-install-payload/hack/build.sh b/install/pre-install-payload/hack/build.sh new file mode 100755 index 000000000..c896bdf16 --- /dev/null +++ b/install/pre-install-payload/hack/build.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +script_dir=$(dirname "$(readlink -f "$0")") + +registry="${registry:-quay.io/confidential-containers/peer-pods-pre-install-payload}" + +supported_arches=( + "linux/amd64" +) + +function setup_env_for_arch() { + case "$1" in + "linux/amd64") + image="docker.io/library/centos" + kernel_arch="x86_64" + ;; + (*) echo "$1 is not supported" && exit 1 + esac + +} + +function build_preinstall_payload() { + pushd "${script_dir}/.." + + tag=$(date +%Y%m%d%H%M%s) + + for arch in ${supported_arches[@]}; do + setup_env_for_arch "${arch}" + + echo "Building payload image for ${arch}" + docker buildx build \ + --build-arg ARCH="${kernel_arch}" \ + -f Dockerfile \ + -t "${registry}:${kernel_arch}-${tag}" \ + --platform="${arch}" \ + --load \ + . + docker push "${registry}:${kernel_arch}-${tag}" + done + + docker manifest create \ + ${registry}:${tag} \ + --amend ${registry}:x86_64-${tag} + + docker manifest create \ + ${registry}:latest \ + --amend ${registry}:x86_64-${tag} + + docker manifest push ${registry}:${tag} + docker manifest push --purge ${registry}:latest + + popd +} + +function main() { + build_preinstall_payload +} + +main "$@" diff --git a/install/pre-install-payload/scripts/cloud-api-adaptor-deploy.sh b/install/pre-install-payload/scripts/cloud-api-adaptor-deploy.sh new file mode 100755 index 000000000..6662cc0a0 --- /dev/null +++ b/install/pre-install-payload/scripts/cloud-api-adaptor-deploy.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +CONFIGMAP_NAME=hyp-env-cm +NAMESPACE=confidential-containers-system + +die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function install_artifacts() { + echo "Copying cloud-api-adaptor artifacts onto host" + + local artifacts_dir="/opt/confidential-containers-pre-install-artifacts" + + cp -a ${artifacts_dir}/etc/systemd/system/* /etc/systemd/system/ + cp -a ${artifacts_dir}/scripts /opt/confidential-containers/ +} + +function uninstall_artifacts() { + echo "Removing cloud-api-adaptor artifacts from host" + + rm -f /etc/systemd/system/remote-hyp.service + rm -fr /opt/confidential-containers/scripts +} + + +function copy_provider_config() { + echo "Copying cloud provider config to /run/hyp.env" + + kubectl get configmap $CONFIGMAP_NAME -n $NAMESPACE -o "jsonpath={ .data['hyp\.env']}" > /run/hyp.env +} + +function remove_provider_config() { + echo "Removing cloud provider config" + cp /dev/null /run/hyp.env +} + +label_node() { + case "${1}" in + install) + kubectl label node "${NODE_NAME}" cc-preinstall/done=true + ;; + uninstall) + kubectl label node "${NODE_NAME}" cc-postuninstall/done=true + ;; + *) + ;; + esac +} + +function print_help() { + echo "Help: ${0} [install/uninstall]" +} + +function main() { + # script requires that user is root + local euid=$(id -u) + if [ ${euid} -ne 0 ]; then + die "This script must be run as root" + fi + + local action=${1:-} + if [ -z "${action}" ]; then + print_help && die "" + fi + + + case "${action}" in + install) + install_artifacts + copy_provider_config + systemctl daemon-reload + systemctl start remote-hyp.service + ;; + uninstall) + systemctl stop remote-hyp.service + remove_provider_config + uninstall_artifacts + systemctl daemon-reload + ;; + *) + print_help + ;; + esac + + label_node "${action}" + + + # It is assumed this script will be called as a daemonset. As a result, do + # not return, otherwise the daemon will restart and reexecute the script. + sleep infinity +} + +main "$@" diff --git a/install/pre-install-payload/scripts/post-uninstall.sh b/install/pre-install-payload/scripts/post-uninstall.sh new file mode 100755 index 000000000..6a7f99c92 --- /dev/null +++ b/install/pre-install-payload/scripts/post-uninstall.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +/opt/confidential-containers-pre-install-artifacts/scripts/cloud-api-adaptor-deploy.sh uninstall diff --git a/install/pre-install-payload/scripts/pre-install.sh b/install/pre-install-payload/scripts/pre-install.sh new file mode 100755 index 000000000..49e7d68d5 --- /dev/null +++ b/install/pre-install-payload/scripts/pre-install.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +/opt/confidential-containers-pre-install-artifacts/scripts/cloud-api-adaptor-deploy.sh install diff --git a/install/pre-install-payload/scripts/run-hyp.sh b/install/pre-install-payload/scripts/run-hyp.sh new file mode 100755 index 000000000..3c30181ab --- /dev/null +++ b/install/pre-install-payload/scripts/run-hyp.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +AWS_EXEC="/opt/confidential-containers/bin/run-aws.sh" +IBMCLOUD_EXEC="/opt/confidential-containers/bin/run-ibmcloud.sh" +LIBVIRT_EXEC="/opt/confidential-containers/bin/run-libvirt.sh" + +: "${CAA_PROVIDER:=aws}" + +function wait_for_executable() { + + while ! test -f "$1"; do + sleep 10 + echo "Still waiting for $1" + done +} + +function main() { + + case "$CAA_PROVIDER" in + aws) + wait_for_executable $AWS_EXEC + $AWS_EXEC + ;; + ibmcloud) + wait_for_executable $IBMCLOUD_EXEC + $IBMCLOUD_EXEC + ;; + libvirt) + wait_for_executable $LIBVIRT_EXEC + $LIBVIRT_EXEC + ;; + *) + echo "No supported provider found" + exit 1 + ;; + + esac +} + +main + diff --git a/install/runtime-payload/Dockerfile b/install/runtime-payload/Dockerfile new file mode 100644 index 000000000..bb3cacf7f --- /dev/null +++ b/install/runtime-payload/Dockerfile @@ -0,0 +1,21 @@ +ARG IMAGE +FROM ${IMAGE:-quay.io/confidential-containers/runtime-payload:kata-containers-b11b6e3756a15e602a5bff1d4d47da8ecf23c593} + +ARG ARCH +ARG BINARIES=./bin +ARG SCRIPTS=./scripts +ARG CONFIG=./config + +#Copy containerd-shim to /opt/confidential-containers/bin +ADD ${BINARIES}/containerd-shim-kata-v2 /opt/kata-artifacts/opt/confidential-containers/bin/ + +#Copy configuration file +ADD ${CONFIG}/configuration-remote.toml /opt/kata-artifacts/opt/confidential-containers/share/defaults/kata-containers + +#Adjust the default config link +RUN cd /opt/kata-artifacts/opt/confidential-containers/share/defaults/kata-containers && \ + rm -f configuration.toml && \ + ln -sf configuration-remote.toml configuration.toml + +#Copy kata-deploy.sh +ADD ${SCRIPTS}/kata-deploy.sh /opt/kata-artifacts/scripts/ diff --git a/install/runtime-payload/Makefile b/install/runtime-payload/Makefile new file mode 100644 index 000000000..9cbc6e289 --- /dev/null +++ b/install/runtime-payload/Makefile @@ -0,0 +1,5 @@ +binaries: + hack/binaries.sh + +build: + hack/build.sh diff --git a/install/runtime-payload/config/configuration-remote.toml b/install/runtime-payload/config/configuration-remote.toml new file mode 100644 index 000000000..93229588a --- /dev/null +++ b/install/runtime-payload/config/configuration-remote.toml @@ -0,0 +1,582 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +# XXX: WARNING: this file is auto-generated. +# XXX: +# XXX: Source file: "config/configuration-qemu.toml.in" +# XXX: Project: +# XXX: Name: Kata Containers +# XXX: Type: kata + + +[hypervisor.remote] +remote_hypervisor_socket = "/run/peerpod/hypervisor.sock" +remote_hypervisor_timeout = 600 + + +# Enable confidential guest support. +# Toggling that setting may trigger different hardware features, ranging +# from memory encryption to both memory and CPU-state encryption and integrity. +# The Kata Containers runtime dynamically detects the available feature set and +# aims at enabling the largest possible one. +# Default false +# confidential_guest = true + +# Enable running QEMU VMM as a non-root user. +# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as +# a non-root random user. See documentation for the limitations of this mode. +# rootless = true + +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = [] + +# List of valid annotations values for the hypervisor +# Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/usr/bin/qemu-system-x86_64"] +valid_hypervisor_paths = ["/opt/kata/bin/qemu-system-x86_64"] + +# Optional space-separated list of options to pass to the guest kernel. +# For example, use `kernel_params = "vsyscall=emulate"` if you are having +# trouble running pre-2.15 glibc. +# +# WARNING: - any parameter specified here will take priority over the default +# parameter value of the same name used to start the virtual machine. +# Do not set values here unless you understand the impact of doing so as you +# may stop the virtual machine from booting. +# To see the list of default parameters, enable hypervisor debug, create a +# container and look for 'default-kernel-parameters' log entries. +kernel_params = "agent.log=debug" + +# Path to the firmware. +# If you want that qemu uses the default firmware leave this option empty +firmware = "" + +# Machine accelerators +# comma-separated list of machine accelerators to pass to the hypervisor. +# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` +machine_accelerators="" + +# CPU features +# comma-separated list of cpu features to pass to the cpu +# For example, `cpu_features = "pmu=off,vmx=off" +cpu_features="pmu=off" + +# Default number of vCPUs per SB/VM: +# unspecified or 0 --> will be set to 1 +# < 0 --> will be set to the actual number of physical cores +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores +default_vcpus = 1 + +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8. +default_maxvcpus = 0 + +# Bridges can be used to hot plug devices. +# Limitations: +# * Currently only pci bridges are supported +# * Until 30 devices per bridge can be hot plugged. +# * Until 5 PCI bridges can be cold plugged per VM. +# This limitation could be a bug in qemu or in the kernel +# Default number of bridges per SB/VM: +# unspecified or 0 --> will be set to 1 +# > 1 <= 5 --> will be set to the specified number +# > 5 --> will be set to 5 +default_bridges = 1 + +# Default memory size in MiB for SB/VM. +# If unspecified then it will be set 2048 MiB. +default_memory = 2048 +# +# Default memory slots per SB/VM. +# If unspecified then it will be set 10. +# This is will determine the times that memory will be hotadded to sandbox/VM. +#memory_slots = 10 + +# The size in MiB will be plused to max memory of hypervisor. +# It is the memory address space for the NVDIMM devie. +# If set block storage driver (block_device_driver) to "nvdimm", +# should set memory_offset to the size of block device. +# Default 0 +#memory_offset = 0 + +# Specifies virtio-mem will be enabled or not. +# Please note that this option should be used with the command +# "echo 1 > /proc/sys/vm/overcommit_memory". +# Default false +#enable_virtio_mem = true + +# Disable block device from being used for a container's rootfs. +# In case of a storage driver like devicemapper where a container's +# root file system is backed by a block device, the block device is passed +# directly to the hypervisor for performance reasons. +# This flag prevents the block device from being passed to the hypervisor, +# 9pfs is used instead to pass the rootfs. +disable_block_device_use = false + +# Shared file system type: +# - virtio-fs (default) +# - virtio-9p +shared_fs = "virtio-fs" + +# Path to vhost-user-fs daemon. +virtio_fs_daemon = "/opt/kata/libexec/kata-qemu/virtiofsd" + +# List of valid annotations values for the virtiofs daemon +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/usr/libexec/kata-qemu/virtiofsd"] +valid_virtio_fs_daemon_paths = ["/opt/kata/libexec/kata-qemu/virtiofsd"] + +# Default size of DAX cache in MiB +virtio_fs_cache_size = 0 + +# Extra args for virtiofsd daemon +# +# Format example: +# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +# +# see `virtiofsd -h` for possible options. +virtio_fs_extra_args = ["--thread-pool-size=1"] + +# Cache mode: +# +# - none +# Metadata, data, and pathname lookup are not cached in guest. They are +# always fetched from host and any changes are immediately pushed to host. +# +# - auto +# Metadata and pathname lookup cache expires after a configured amount of +# time (default is 1 second). Data is cached while the file is open (close +# to open consistency). +# +# - always +# Metadata, data, and pathname lookup are cached in guest and never expire. +virtio_fs_cache = "auto" + +# Block storage driver to be used for the hypervisor in case the container +# rootfs is backed by a block device. This is virtio-scsi, virtio-blk +# or nvdimm. +block_device_driver = "virtio-scsi" + +# Specifies cache-related options will be set to block devices or not. +# Default false +#block_device_cache_set = true + +# Specifies cache-related options for block devices. +# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. +# Default false +#block_device_cache_direct = true + +# Specifies cache-related options for block devices. +# Denotes whether flush requests for the device are ignored. +# Default false +#block_device_cache_noflush = true + +# Enable iothreads (data-plane) to be used. This causes IO to be +# handled in a separate IO thread. This is currently only implemented +# for SCSI. +# +enable_iothreads = false + +# Enable pre allocation of VM RAM, default false +# Enabling this will result in lower container density +# as all of the memory will be allocated and locked +# This is useful when you want to reserve all the memory +# upfront or in the cases where you want memory latencies +# to be very predictable +# Default false +#enable_mem_prealloc = true + +# Enable huge pages for VM RAM, default false +# Enabling this will result in the VM memory +# being allocated using huge pages. +# This is useful when you want to use vhost-user network +# stacks within the container. This will automatically +# result in memory pre allocation +#enable_hugepages = true + +# Enable vhost-user storage device, default false +# Enabling this will result in some Linux reserved block type +# major range 240-254 being chosen to represent vhost-user devices. +enable_vhost_user_store = false + +# The base directory specifically used for vhost-user devices. +# Its sub-path "block" is used for block devices; "block/sockets" is +# where we expect vhost-user sockets to live; "block/devices" is where +# simulated block device nodes for vhost-user devices to live. +vhost_user_store_path = "/var/run/kata-containers/vhost-user" + +# Enable vIOMMU, default false +# Enabling this will result in the VM having a vIOMMU device +# This will also add the following options to the kernel's +# command line: intel_iommu=on,iommu=pt +#enable_iommu = true + +# Enable IOMMU_PLATFORM, default false +# Enabling this will result in the VM device having iommu_platform=on set +#enable_iommu_platform = true + +# List of valid annotations values for the vhost user store path +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/var/run/kata-containers/vhost-user"] +valid_vhost_user_store_paths = ["/var/run/kata-containers/vhost-user"] + +# Enable file based guest memory support. The default is an empty string which +# will disable this feature. In the case of virtio-fs, this is enabled +# automatically and '/dev/shm' is used as the backing folder. +# This option will be ignored if VM templating is enabled. +#file_mem_backend = "" + +# List of valid annotations values for the file_mem_backend annotation +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: [""] +valid_file_mem_backends = [""] + +# Enable swap of vm memory. Default false. +# The behaviour is undefined if mem_prealloc is also set to true +#enable_swap = true + +# -pflash can add image file to VM. The arguments of it should be in format +# of ["/path/to/flash0.img", "/path/to/flash1.img"] +pflashes = [] + +# This option changes the default hypervisor and kernel parameters +# to enable debug output where available. +# +# Default false +enable_debug = true + +# Disable the customizations done in the runtime when it detects +# that it is running on top a VMM. This will result in the runtime +# behaving as it would when running on bare metal. +# +#disable_nesting_checks = true + +# This is the msize used for 9p shares. It is the number of bytes +# used for 9p packet payload. +#msize_9p = 8192 + +# If false and nvdimm is supported, use nvdimm device to plug guest image. +# Otherwise virtio-block device is used. +# Default is false +#disable_image_nvdimm = true + +# VFIO devices are hotplugged on a bridge by default. +# Enable hotplugging on root bus. This may be required for devices with +# a large PCI bar, as this is a current limitation with hotplugging on +# a bridge. +# Default false +#hotplug_vfio_on_root_bus = true + +# Before hot plugging a PCIe device, you need to add a pcie_root_port device. +# Use this parameter when using some large PCI bar devices, such as Nvidia GPU +# The value means the number of pcie_root_port +# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35" +# Default 0 +#pcie_root_port = 2 + +# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off +# security (vhost-net runs ring0) for network I/O performance. +#disable_vhost_net = true + +# +# Default entropy source. +# The path to a host source of entropy (including a real hardware RNG) +# /dev/urandom and /dev/random are two main options. +# Be aware that /dev/random is a blocking source of entropy. If the host +# runs out of entropy, the VMs boot time will increase leading to get startup +# timeouts. +# The source of entropy /dev/urandom is non-blocking and provides a +# generally acceptable source of entropy. It should work well for pretty much +# all practical purposes. +#entropy_source= "/dev/urandom" + +# List of valid annotations values for entropy_source +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/dev/urandom","/dev/random",""] +valid_entropy_sources = ["/dev/urandom","/dev/random",""] + +# Path to OCI hook binaries in the *guest rootfs*. +# This does not affect host-side hooks which must instead be added to +# the OCI spec passed to the runtime. +# +# You can create a rootfs with hooks by customizing the osbuilder scripts: +# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder +# +# Hooks must be stored in a subdirectory of guest_hook_path according to their +# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}". +# The agent will scan these directories for executable files and add them, in +# lexicographical order, to the lifecycle of the guest container. +# Hooks are executed in the runtime namespace of the guest. See the official documentation: +# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks +# Warnings will be logged if any error is encountered while scanning for hooks, +# but it will not abort container execution. +#guest_hook_path = "/usr/share/oci/hooks" +# +# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic. +# Default 0-sized value means unlimited rate. +#rx_rate_limiter_max_rate = 0 +# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block) +# to discipline traffic. +# Default 0-sized value means unlimited rate. +#tx_rate_limiter_max_rate = 0 + +# Set where to save the guest memory dump file. +# If set, when GUEST_PANICKED event occurred, +# guest memeory will be dumped to host filesystem under guest_memory_dump_path, +# This directory will be created automatically if it does not exist. +# +# The dumped file(also called vmcore) can be processed with crash or gdb. +# +# WARNING: +# Dump guest’s memory can take very long depending on the amount of guest memory +# and use much disk space. +#guest_memory_dump_path="/var/crash/kata" + +# If enable paging. +# Basically, if you want to use "gdb" rather than "crash", +# or need the guest-virtual addresses in the ELF vmcore, +# then you should enable paging. +# +# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details +#guest_memory_dump_paging=false + +# Enable swap in the guest. Default false. +# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device +# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness") +# is bigger than 0. +# The size of the swap device should be +# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes. +# If swap_in_bytes is not set, the size should be memory_limit_in_bytes. +# If swap_in_bytes and memory_limit_in_bytes is not set, the size should +# be default_memory. +#enable_guest_swap = true + +[factory] +# VM templating support. Once enabled, new VMs are created from template +# using vm cloning. They will share the same initial kernel, initramfs and +# agent memory by mapping it readonly. It helps speeding up new container +# creation and saves a lot of memory if there are many kata containers running +# on the same host. +# +# When disabled, new VMs are created from scratch. +# +# Note: Requires "initrd=" to be set ("image=" is not supported). +# +# Default false +#enable_template = true + +# Specifies the path of template. +# +# Default "/run/vc/vm/template" +#template_path = "/run/vc/vm/template" + +# The number of caches of VMCache: +# unspecified or == 0 --> VMCache is disabled +# > 0 --> will be set to the specified number +# +# VMCache is a function that creates VMs as caches before using it. +# It helps speed up new container creation. +# The function consists of a server and some clients communicating +# through Unix socket. The protocol is gRPC in protocols/cache/cache.proto. +# The VMCache server will create some VMs and cache them by factory cache. +# It will convert the VM to gRPC format and transport it when gets +# requestion from clients. +# Factory grpccache is the VMCache client. It will request gRPC format +# VM and convert it back to a VM. If VMCache function is enabled, +# kata-runtime will request VM from factory grpccache when it creates +# a new sandbox. +# +# Default 0 +#vm_cache_number = 0 + +# Specify the address of the Unix socket that is used by VMCache. +# +# Default /var/run/kata-containers/cache.sock +#vm_cache_endpoint = "/var/run/kata-containers/cache.sock" + +[agent.kata] +# If enabled, make the agent display debug-level messages. +# (default: disabled) +enable_debug = true + +# Enable agent tracing. +# +# If enabled, the agent will generate OpenTelemetry trace spans. +# +# Notes: +# +# - If the runtime also has tracing enabled, the agent spans will be +# associated with the appropriate runtime parent span. +# - If enabled, the runtime will wait for the container to shutdown, +# increasing the container shutdown time slightly. +# +# (default: disabled) +#enable_tracing = true + +# Comma separated list of kernel modules and their parameters. +# These modules will be loaded in the guest kernel using modprobe(8). +# The following example can be used to load two kernel modules with parameters +# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] +# The first word is considered as the module name and the rest as its parameters. +# Container will not be started when: +# * A kernel module is specified and the modprobe command is not installed in the guest +# or it fails loading the module. +# * The module is not available in the guest or it doesn't met the guest kernel +# requirements, like architecture and version. +# +kernel_modules=[] + +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +debug_console_enabled = true + +# Agent connection dialing timeout value in seconds +# (default: 30) +#dial_timeout = 30 + +[netmon] +# If enabled, the network monitoring process gets started when the +# sandbox is created. This allows for the detection of some additional +# network being added to the existing network namespace, after the +# sandbox has been created. +# (default: disabled) +#enable_netmon = true + +# Specify the path to the netmon binary. +path = "/opt/kata/libexec/kata-containers/kata-netmon" + +# If enabled, netmon messages will be sent to the system log +# (default: disabled) +enable_debug = true + +[runtime] +# If enabled, the runtime will log additional debug messages to the +# system log +# (default: disabled) +enable_debug = true +# +# Internetworking model +# Determines how the VM should be connected to the +# the container network interface +# Options: +# +# - macvtap +# Used when the Container network interface can be bridged using +# macvtap. +# +# - none +# Used when customize network. Only creates a tap device. No veth pair. +# +# - tcfilter +# Uses tc filter rules to redirect traffic from the network interface +# provided by plugin to a tap interface connected to the VM. +# +internetworking_model="none" + +# disable guest seccomp +# Determines whether container seccomp profiles are passed to the virtual +# machine and applied by the kata agent. If set to true, seccomp is not applied +# within the guest +# (default: true) +disable_guest_seccomp=true + +# If enabled, the runtime will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# (default: disabled) +#enable_tracing = true + +# Set the full url to the Jaeger HTTP Thrift collector. +# The default if not set will be "http://localhost:14268/api/traces" +#jaeger_endpoint = "" + +# Sets the username to be used if basic auth is required for Jaeger. +#jaeger_user = "" + +# Sets the password to be used if basic auth is required for Jaeger. +#jaeger_password = "" + +# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. +# This option may have some potential impacts to your host. It should only be used when you know what you're doing. +# `disable_new_netns` conflicts with `enable_netmon` +# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only +# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge +# (like OVS) directly. +# If you are using docker, `disable_new_netns` only works with `docker run --net=none` +# (default: false) +disable_new_netns = true + +# if enabled, the runtime will add all the kata processes inside one dedicated cgroup. +# The container cgroups in the host are not created, just one single cgroup per sandbox. +# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. +# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. +# The sandbox cgroup is constrained if there is no container type annotation. +# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType +sandbox_cgroup_only=false + +# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +sandbox_bind_mounts=[] + +# Enabled experimental feature list, format: ["a", "b"]. +# Experimental features are features not stable enough for production, +# they may break compatibility, and are prepared for a big version bump. +# Supported experimental features: +# (default: []) +experimental=[] + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# enable_pprof = true + +# WARNING: All the options in the following section have not been implemented yet. +# This section was added as a placeholder. DO NOT USE IT! +[image] +# Container image service. +# +# Offload the CRI image management service to the Kata agent. +# (default: false) +service_offload = true + +# Container image decryption keys provisioning. +# Applies only if service_offload is true. +# Keys can be provisioned locally (e.g. through a special command or +# a local file) or remotely (usually after the guest is remotely attested). +# The provision setting is a complete URL that lets the Kata agent decide +# which method to use in order to fetch the keys. +# +# Keys can be stored in a local file, in a measured and attested initrd: +#provision=data:///local/key/file +# +# Keys could be fetched through a special command or binary from the +# initrd (guest) image, e.g. a firmware call: +#provision=file:///path/to/bin/fetcher/in/guest +# +# Keys can be remotely provisioned. The Kata agent fetches them from e.g. +# a HTTPS URL: +#provision=https://my-key-broker.foo/tenant/ diff --git a/install/runtime-payload/hack/binaries.sh b/install/runtime-payload/hack/binaries.sh new file mode 100755 index 000000000..1d228ea3d --- /dev/null +++ b/install/runtime-payload/hack/binaries.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +script_dir=$(dirname "$(readlink -f "$0")") +bin_dir=${script_dir}/../bin +kata_dir=${script_dir}/../../../../kata-containers + +function build_kata_runtime() { + pushd "${kata_dir}/src/runtime" + make + cp containerd-shim-kata-v2 ${bin_dir}/ + popd +} +function main() { + mkdir -p ${bin_dir} + build_kata_runtime +} + +main "$@" diff --git a/install/runtime-payload/hack/build.sh b/install/runtime-payload/hack/build.sh new file mode 100755 index 000000000..cac5b05e7 --- /dev/null +++ b/install/runtime-payload/hack/build.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +script_dir=$(dirname "$(readlink -f "$0")") + +registry="${registry:-quay.io/confidential-containers/peer-pods-runtime-payload}" + +supported_arches=( + "linux/amd64" +) + +function setup_env_for_arch() { + case "$1" in + "linux/amd64") + image="docker.io/library/centos" + kernel_arch="x86_64" + ;; + (*) echo "$1 is not supported" && exit 1 + esac + +} + +function build_runtime_payload() { + pushd "${script_dir}/.." + + tag=$(date +%Y%m%d%H%M%s) + + for arch in ${supported_arches[@]}; do + setup_env_for_arch "${arch}" + + echo "Building payload image for ${arch}" + docker buildx build \ + --build-arg ARCH="${kernel_arch}" \ + -f Dockerfile \ + -t "${registry}:${kernel_arch}-${tag}" \ + --platform="${arch}" \ + --load \ + . + docker push "${registry}:${kernel_arch}-${tag}" + done + + docker manifest create \ + ${registry}:${tag} \ + --amend ${registry}:x86_64-${tag} + + docker manifest create \ + ${registry}:latest \ + --amend ${registry}:x86_64-${tag} + + docker manifest push ${registry}:${tag} + docker manifest push --purge ${registry}:latest + + popd +} + +function main() { + build_runtime_payload +} + +main "$@" diff --git a/install/runtime-payload/scripts/kata-deploy.sh b/install/runtime-payload/scripts/kata-deploy.sh new file mode 100755 index 000000000..d1c71ee22 --- /dev/null +++ b/install/runtime-payload/scripts/kata-deploy.sh @@ -0,0 +1,353 @@ +#!/usr/bin/env bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o pipefail +set -o nounset + +crio_drop_in_conf_dir="/etc/crio/crio.conf.d/" +crio_drop_in_conf_file="${crio_drop_in_conf_dir}/99-kata-deploy" +containerd_conf_file="/etc/containerd/config.toml" +containerd_conf_file_backup="${containerd_conf_file}.bak" + +shims=( + "remote" + "qemu" + "qemu-tdx" + "qemu-sev" + "clh" + "clh-tdx" +) + +default_shim="remote" + +# If we fail for any reason a message will be displayed +die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function print_usage() { + echo "Usage: $0 [install/cleanup/reset]" +} + +function get_container_runtime() { + + local runtime=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}') + if [ "$?" -ne 0 ]; then + die "invalid node name" + fi + if echo "$runtime" | grep -qE 'containerd.*-k3s'; then + if systemctl is-active --quiet rke2-agent; then + echo "rke2-agent" + elif systemctl is-active --quiet rke2-server; then + echo "rke2-server" + elif systemctl is-active --quiet k3s-agent; then + echo "k3s-agent" + else + echo "k3s" + fi + else + echo "$runtime" | awk -F '[:]' '{print $1}' + fi +} + +function install_artifacts() { + echo "copying kata artifacts onto host" + cp -a /opt/kata-artifacts/opt/confidential-containers/* /opt/confidential-containers/ + chmod +x /opt/confidential-containers/bin/* +} + +function configure_cri_runtime() { + configure_different_shims_base + + case $1 in + crio) + configure_crio + ;; + containerd | k3s | k3s-agent | rke2-agent | rke2-server) + configure_containerd + ;; + esac + systemctl daemon-reload + systemctl restart "$1" +} + +function configure_different_shims_base() { + # Currently containerd has an assumption on the location of the shimv2 implementation + # This forces kata-deploy to create files in a well-defined location that's part of + # the PATH, pointing to the containerd-shim-kata-v2 binary in /opt/confidential-contaienrs/bin + # Issues: + # https://github.com/containerd/containerd/issues/3073 + # https://github.com/containerd/containerd/issues/5006 + + mkdir -p /usr/local/bin + + for shim in "${shims[@]}"; do + local shim_binary="containerd-shim-kata-${shim}-v2" + local shim_file="/usr/local/bin/${shim_binary}" + local shim_backup="/usr/local/bin/${shim_binary}.bak" + + if [ -f "${shim_file}" ]; then + echo "warning: ${shim_binary} already exists" >&2 + if [ ! -f "${shim_backup}" ]; then + mv "${shim_file}" "${shim_backup}" + else + rm "${shim_file}" + fi + fi + + cat << EOF | tee "$shim_file" +#!/usr/bin/env bash +KATA_CONF_FILE=/opt/confidential-containers/share/defaults/kata-containers/configuration-${shim}.toml /opt/confidential-containers/bin/containerd-shim-kata-v2 "\$@" +EOF + chmod +x "$shim_file" + + if [ "${shim}" == "${default_shim}" ]; then + echo "Creating the default shim-v2 binary" + ln -sf "${shim_file}" /usr/local/bin/containerd-shim-kata-v2 + fi + done +} + +function cleanup_different_shims_base() { + for shim in "${shims[@]}"; do + local shim_binary="containerd-shim-kata-${shim}-v2" + local shim_file="/usr/local/bin/${shim_binary}" + local shim_backup="/usr/local/bin/${shim_binary}.bak" + + rm "${shim_file}" || true + + if [ -f "${shim_backup}" ]; then + mv "$shim_backup" "$shim_file" + fi + done + + rm /usr/local/bin/containerd-shim-kata-v2 +} + +function configure_crio_runtime() { + local runtime="kata" + local configuration="configuration" + if [ -n "${1-}" ]; then + runtime+="-$1" + configuration+="-$1" + fi + + local kata_path="/usr/local/bin/containerd-shim-${runtime}-v2" + local kata_conf="crio.runtime.runtimes.${runtime}" + local kata_config_path="/opt/confidential-containers/share/defaults/kata-containers/$configuration.toml" + + cat <" $containerd_conf_file; then + pluginid=\"io.containerd.grpc.v1.cri\" + fi + local runtime_table="plugins.${pluginid}.containerd.runtimes.$runtime" + local runtime_type="io.containerd.$runtime.v2" + local cri_handler_value="" + if echo "${runtime_type}" | grep -q -v -e "kata-remote\.v2" -e "kata\.v2"; then + cri_handler_value="cc" + fi + local options_table="$runtime_table.options" + local config_path="/opt/confidential-containers/share/defaults/kata-containers/$configuration.toml" + if grep -q "\[$runtime_table\]" $containerd_conf_file; then + echo "Configuration exists for $runtime_table, overwriting" + sed -i "/\[$runtime_table\]/,+1s#runtime_type.*#runtime_type = \"${runtime_type}\"#" $containerd_conf_file + else + cat < "$containerd_conf_file" + fi + # CRI-O isn't consistent with the naming -- let's use crio to match the service file + elif [ "$runtime" == "cri-o" ]; then + runtime="crio" + fi + + action=${1:-} + if [ -z "$action" ]; then + print_usage + die "invalid arguments" + fi + + # only install / remove / update if we are dealing with containerd + if [[ "$runtime" =~ ^(containerd|k3s|k3s-agent|rke2-agent|rke2-server|crio)$ ]]; then + + case "$action" in + install) + install_artifacts + configure_cri_runtime "$runtime" + kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=true + ;; + cleanup) + cleanup_cri_runtime "$runtime" + kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup + remove_artifacts + ;; + reset) + reset_runtime $runtime + ;; + *) + echo invalid arguments + print_usage + ;; + esac + fi + + #It is assumed this script will be called as a daemonset. As a result, do + # not return, otherwise the daemon will restart and rexecute the script + sleep infinity +} + +main "$@" diff --git a/install/yamls/caa-pod.yaml b/install/yamls/caa-pod.yaml new file mode 100644 index 000000000..4c129e83a --- /dev/null +++ b/install/yamls/caa-pod.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: cloud-api-adaptor + name: cloud-api-adaptor-daemonset + namespace: confidential-containers-system +spec: + selector: + matchLabels: + app: cloud-api-adaptor + template: + metadata: + labels: + app: cloud-api-adaptor + spec: + containers: + - command: + - /usr/local/bin/entrypoint.sh + envFrom: + - secretRef: + name: peer-pods-secret + - configMapRef: + name: peer-pods-cm + image: kustomize.this/cloud-api-adaptor/image:url + imagePullPolicy: Always + name: cloud-api-adaptor-con + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + volumeMounts: + - name: auth-json + mountPath: "/root/containers/" # hardcoded + readOnly: true + - mountPath: /root/.ssh/ + name: ssh + readOnly: true + - mountPath: /run/peerpod + name: pods-dir + - mountPath: /run/netns + mountPropagation: HostToContainer + name: netns + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/worker: "" + volumes: + - name: auth-json + secret: + secretName: auth-json-secret + optional: true # failing? + - name: ssh + secret: + defaultMode: 384 + optional: true + secretName: ssh-key-secret + - hostPath: + path: /run/peerpod + name: pods-dir + - hostPath: + path: /run/netns + name: netns diff --git a/install/yamls/ccruntime-peer-pods.yaml b/install/yamls/ccruntime-peer-pods.yaml new file mode 100644 index 000000000..0c6871319 --- /dev/null +++ b/install/yamls/ccruntime-peer-pods.yaml @@ -0,0 +1,129 @@ +apiVersion: confidentialcontainers.org/v1beta1 +kind: CcRuntime +metadata: + name: ccruntime-sample + namespace: confidential-containers-system +spec: + # Add fields here + runtimeName: kata + ccNodeSelector: + matchLabels: + node-role.kubernetes.io/worker: "" + config: + installType: bundle + payloadImage: quay.io/confidential-containers/peer-pods-runtime-payload:2022091512521663246365 + installDoneLabel: + katacontainers.io/kata-runtime: "true" + uninstallDoneLabel: + katacontainers.io/kata-runtime: "cleanup" + installerVolumeMounts: + - mountPath: /etc/crio/ + name: crio-conf + - mountPath: /etc/containerd/ + name: containerd-conf + - mountPath: /opt/confidential-containers/ + name: kata-artifacts + - mountPath: /var/run/dbus + name: dbus + - mountPath: /run/systemd + name: systemd + - mountPath: /usr/local/bin/ + name: local-bin + installerVolumes: + - hostPath: + path: /etc/crio/ + type: "" + name: crio-conf + - hostPath: + path: /etc/containerd/ + type: "" + name: containerd-conf + - hostPath: + path: /opt/confidential-containers/ + type: DirectoryOrCreate + name: kata-artifacts + - hostPath: + path: /var/run/dbus + type: "" + name: dbus + - hostPath: + path: /run/systemd + type: "" + name: systemd + - hostPath: + path: /usr/local/bin/ + type: "" + name: local-bin + installCmd: ["/opt/kata-artifacts/scripts/kata-deploy.sh", "install"] + uninstallCmd: ["/opt/kata-artifacts/scripts/kata-deploy.sh", "cleanup"] + cleanupCmd: ["/opt/kata-artifacts/scripts/kata-deploy.sh", "reset"] + postUninstall: + image: quay.io/confidential-containers/peer-pods-pre-install-payload:2022081008261660119970 + volumeMounts: + - mountPath: /opt/confidential-containers/ + name: confidential-containers-artifacts + - mountPath: /etc/systemd/system/ + name: etc-systemd-system + - mountPath: /var/run/dbus + name: dbus + - mountPath: /run/systemd + name: systemd + volumes: + - hostPath: + path: /opt/confidential-containers/ + type: DirectoryOrCreate + name: confidential-containers-artifacts + - hostPath: + path: /etc/systemd/system/ + type: "" + name: etc-systemd-system + - hostPath: + path: /var/run/dbus + type: "" + name: dbus + - hostPath: + path: /run/systemd + type: "" + name: systemd + preInstall: + image: quay.io/confidential-containers/peer-pods-pre-install-payload:2022081008261660119970 + volumeMounts: + - mountPath: /opt/confidential-containers/ + name: confidential-containers-artifacts + - mountPath: /etc/systemd/system/ + name: etc-systemd-system + - mountPath: /var/run/dbus + name: dbus + - mountPath: /run/systemd + name: systemd + - mountPath: /run/hyp.env + name: hyp-env + volumes: + - hostPath: + path: /opt/confidential-containers/ + type: DirectoryOrCreate + name: confidential-containers-artifacts + - hostPath: + path: /etc/systemd/system/ + type: "" + name: etc-systemd-system + - hostPath: + path: /var/run/dbus + type: "" + name: dbus + - hostPath: + path: /run/systemd + type: "" + name: systemd + - hostPath: + path: /run/hyp.env + type: FileOrCreate + name: hyp-env + environmentVariables: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: "CONFIGURE_CC" + value: "yes" diff --git a/install/yamls/deploy.yaml b/install/yamls/deploy.yaml new file mode 100644 index 000000000..fcb61b514 --- /dev/null +++ b/install/yamls/deploy.yaml @@ -0,0 +1,5854 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: confidential-containers-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cc-operator-controller-manager + namespace: confidential-containers-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cc-operator-leader-election-role + namespace: confidential-containers-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: cc-operator-manager-role +rules: +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - confidentialcontainers.org + resources: + - ccruntimes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - confidentialcontainers.org + resources: + - ccruntimes/finalizers + verbs: + - update +- apiGroups: + - confidentialcontainers.org + resources: + - ccruntimes/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - watch + - update +- apiGroups: + - node.k8s.io + resources: + - runtimeclasses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cc-operator-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cc-operator-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cc-operator-leader-election-rolebinding + namespace: confidential-containers-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cc-operator-leader-election-role +subjects: +- kind: ServiceAccount + name: cc-operator-controller-manager + namespace: confidential-containers-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cc-operator-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cc-operator-manager-role +subjects: +- kind: ServiceAccount + name: cc-operator-controller-manager + namespace: confidential-containers-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cc-operator-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cc-operator-proxy-role +subjects: +- kind: ServiceAccount + name: cc-operator-controller-manager + namespace: confidential-containers-system +--- +apiVersion: v1 +data: + controller_manager_config.yaml: | + apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 + kind: ControllerManagerConfig + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: 127.0.0.1:8080 + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: 69bf4d38.confidentialcontainers.org +kind: ConfigMap +metadata: + name: cc-operator-manager-config + namespace: confidential-containers-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: cc-operator-controller-manager-metrics-service + namespace: confidential-containers-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + control-plane: controller-manager + name: cc-operator-controller-manager + namespace: confidential-containers-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + labels: + control-plane: controller-manager + spec: + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + - --leader-elect + command: + - /manager + image: quay.io/confidential-containers/operator:latest + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 100m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + securityContext: + runAsNonRoot: true + serviceAccountName: cc-operator-controller-manager + terminationGracePeriodSeconds: 10 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null + name: ccruntimes.confidentialcontainers.org +spec: + group: confidentialcontainers.org + names: + kind: CcRuntime + listKind: CcRuntimeList + plural: ccruntimes + shortNames: + - ccr + singular: ccruntime + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CcRuntime is the Schema for the ccruntimes API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CcRuntimeSpec defines the desired state of CcRuntime + properties: + ccNodeSelector: + description: CcNodeSelector is used to select the worker nodes to + deploy the runtime if not specified, all worker nodes are selected + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + config: + description: CcInstallConfig is a placeholder struct + properties: + ImagePullSecret: + description: This specifies the registry secret to pull of the + container images + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + cleanupCmd: + description: This specifies the command for cleanup on the nodes + items: + type: string + type: array + environmentVariables: + description: This specifies the environment variables required + by the daemon set + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + guestInitrdImage: + description: This specifies the location of the container image + containing the guest initrd If both bundleImage and guestInitrdImage + are specified, then guestInitrdImage content will override the + equivalent one in payloadImage + type: string + guestKernelImage: + description: This specifies the location of the container image + containing the guest kernel If both bundleImage and guestKernelImage + are specified, then guestKernelImage content will override the + equivalent one in payloadImage + type: string + imagePullPolicy: + description: PullPolicy describes a policy for if/when to pull + a container image + type: string + installCmd: + description: This specifies the command for installation of the + runtime on the nodes + items: + type: string + type: array + installDoneLabel: + additionalProperties: + type: string + description: This specifies the label that the install daemonset + adds to nodes when the installation is done + type: object + installType: + description: This indicates whether to use native OS packaging + (rpm/deb) or Container image Default is bundle (container image) + enum: + - bundle + - osnative + type: string + installerVolumeMounts: + description: This specifies volume mounts required for the installer + pods + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other way + around. When not set, MountPropagationNone is used. This + field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + installerVolumes: + description: This specifies volumes required for the installer + pods + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'The partition in the volume that you want + to mount. If omitted, the default is to mount by volume + name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property + empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the ReadOnly + property in VolumeMounts to "true". If omitted, the + default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource + in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, Read + Write.' + type: string + diskName: + description: The Name of the data disk in the blob storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob + disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults + to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure + Storage Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the host + that shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection of + Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, rather + than the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to key + ring for User, default is /etc/ceph/user.secret More + info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to the + authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'Optional: User is the rados user name, + default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached + and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object containing + parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeID: + description: 'volume id used to identify the volume + in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value + between 0000 and 0777 or a decimal value between 0 + and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults + to 0644. Directories within the path are not affected + by this setting. This might be in conflict with other + options that affect the file mode, like fsGroup, and + the result can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced ConfigMap will be + projected into the volume as a file whose name is + the key and content is the value. If specified, the + listed keys will be projected into the specified paths, + and unlisted keys will not be present. If a key is + specified which is not present in the ConfigMap, the + volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to set + permissions on this file. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its keys + must be defined + type: boolean + type: object + csi: + description: CSI (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: Driver is the name of the CSI driver that + handles this volume. Consult with your admin for the + correct name as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", "xfs", + "ntfs". If not provided, the empty value is passed + to the associated CSI driver which will determine + the default filesystem to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference to + the secret object containing sensitive information + to pass to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the + secret object contains more than one secret, all secret + references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration for + the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific + properties that are passed to the CSI driver. Consult + your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in + conflict with other options that affect the file mode, + like fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the + pod: only annotations, labels, name and namespace + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used to set + permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must not + be absolute or contain the ''..'' path. Must + be utf-8 encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should back + this directory. The default is "" which means to use + the node''s default medium. Must be an empty string + (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'Total amount of local storage required + for this EmptyDir volume. The size limit is also applicable + for memory medium. The maximum usage on memory medium + EmptyDir would be the minimum value between the SizeLimit + specified here and the sum of memory limits of all + containers in a pod. The default is nil which means + that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "Ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is + tied to the pod that defines it - it will be created before + the pod starts, and deleted when the pod is removed. \n + Use this if: a) the volume is only needed while the pod + runs, b) features of normal volumes like restoring from + snapshot or capacity tracking are needed, c) the storage + driver is specified through a storage class, and d) the + storage driver supports dynamic volume provisioning through + \ a PersistentVolumeClaim (see EphemeralVolumeSource + for more information on the connection between this + volume type and PersistentVolumeClaim). \n Use PersistentVolumeClaim + or one of the vendor-specific APIs for volumes that persist + for longer than the lifecycle of an individual pod. \n + Use CSI for light-weight local ephemeral volumes if the + CSI driver is meant to be used that way - see the documentation + of the driver for more information. \n A pod can use both + types of ephemeral volumes and persistent volumes at the + same time. \n This is a beta feature and only available + when the GenericEphemeralVolume feature gate is enabled." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC + to provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the + PVC will be deleted together with the pod. The name + of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` + array entry. Pod validation will reject the pod if + the concatenated name is not valid for a PVC (for + example, too long). \n An existing PVC with that name + that is not owned by the pod will *not* be used for + the pod to avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the unrelated + PVC is removed. If such a pre-created PVC is meant + to be used by the pod, the PVC has to updated with + an owner reference to the pod once the pod exists. + Normally this should not be necessary, but it may + be useful when manually reconstructing a broken cluster. + \n This field is read-only and no changes will be + made by Kubernetes to the PVC after it has been created. + \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations + that will be copied into the PVC when creating + it. No other fields are allowed and will be rejected + during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the + PVC that gets created from this template. The + same fields as in a PersistentVolumeClaim are + also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired + access modes the volume should have. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify + either: * An existing VolumeSnapshot object + (snapshot.storage.k8s.io/VolumeSnapshot) * + An existing PVC (PersistentVolumeClaim) If + the provisioner or an external controller + can support the specified data source, it + will create a new volume based on the contents + of the specified data source. If the AnyVolumeDataSource + feature gate is enabled, this field will always + have the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup + is not specified, the specified Kind must + be in the core API group. For any other + third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which + to populate the volume with data, if a non-empty + volume is desired. This may be any local object + from a non-empty API group (non core object) + or a PersistentVolumeClaim object. When this + field is specified, volume binding will only + succeed if the type of the specified object + matches some installed volume populator or + dynamic provisioner. This field will replace + the functionality of the DataSource field + and as such if both fields are non-empty, + they must have the same value. For backwards + compatibility, both fields (DataSource and + DataSourceRef) will be set to the same value + automatically if one of them is empty and + the other is non-empty. There are two important + differences between DataSource and DataSourceRef: + * While DataSource only allows two specific + types of objects, DataSourceRef allows any + non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed + values (dropping them), DataSourceRef preserves + all values, and generates an error if a disallowed + value is specified. (Alpha) Using this field + requires the AnyVolumeDataSource feature gate + to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup + is not specified, the specified Kind must + be in the core API group. For any other + third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum + resources the volume should have. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. + If Requests is omitted for a container, + it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of + volume is required by the claim. Value of + Filesystem is implied when not included in + claim spec. + type: string + volumeName: + description: VolumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: FC represents a Fibre Channel resource that + is attached to a kubelet's host machine and then exposed + to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. TODO: how do we prevent errors in the + filesystem from compromising the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs + and lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: Driver is the name of the driver to use + for this volume. + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". The default filesystem depends on FlexVolume + script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to the + secret object containing sensitive information to + pass to the plugin scripts. This may be empty if no + secret object is specified. If the secret object contains + more than one secret, all secrets are passed to the + plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: Name of the dataset stored as metadata + -> name on the dataset for Flocker should be considered + as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then + exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'The partition in the volume that you want + to mount. If omitted, the default is to mount by volume + name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property + empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in GCE. + Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an + InitContainer that clones the repo using git, then mount + the EmptyDir into the Pod''s container.' + properties: + directory: + description: Target directory name. Must not contain + or start with '..'. If '.' is supplied, the volume + directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount on + the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name that + details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs + volume to be mounted with read-only permissions. Defaults + to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file or + directory on the host machine that is directly exposed + to the container. This is generally used for system agents + or other privileged things that are allowed to see the + host machine. Most containers will NOT need this. More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host + directory mounts and who can/can not mount host directories + as read/write.' + properties: + path: + description: 'Path of the directory on the host. If + the path is a symlink, it will follow the link to + the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource that + is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, new + iSCSI interface : will + be created for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI + transport. Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal is + either an IP or ip_addr:port if the port is other + than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator + authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + targetPortal: + description: iSCSI Target Portal. The Portal is either + an IP or ip_addr:port if the port is other than default + (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and unique + within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host that + shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address of + the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + pdID: + description: ID that identifies Photon Controller persistent + disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type to + mount Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, configmaps, + and downward API + properties: + defaultMode: + description: Mode bits used to set permissions on created + files by default. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. Directories within the + path are not affected by this setting. This might + be in conflict with other options that affect the + file mode, like fsGroup, and the result can be other + mode bits set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + configMap: + description: information about the configMap data + to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. Paths + must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used + to set permissions on this file. Must + be an octal value between 0000 and + 0777 or a decimal value between 0 + and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the + file to map the key to. May not be + an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + downwardAPI: + description: information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used + to set permissions on this file, must + be an octal value between 0000 and + 0777 or a decimal value between 0 + and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret data + to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the Secret, the volume setup will error + unless it is marked optional. Paths must + be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used + to set permissions on this file. Must + be an octal value between 0000 and + 0777 or a decimal value between 0 + and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the + file to map the key to. May not be + an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + type: object + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience defaults + to the identifier of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The kubelet + will start trying to rotate the token if + the token is older than 80 percent of its + time to live or if the token is older than + 24 hours.Defaults to 1 hour and must be + at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative to + the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default is + no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults + to false. + type: boolean + registry: + description: Registry represents a single or multiple + Quobyte Registry services specified as a string as + host:port pair (multiple entries are separated with + commas) which acts as the central registry for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned Quobyte + volumes, value is set by the plugin + type: string + user: + description: User to map volume access to Defaults to + serivceaccount user + type: string + volume: + description: Volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'The rados user name. Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: The host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection Domain + for the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret for + ScaleIO user and other sensitive information. If this + is not provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + sslEnabled: + description: Flag to enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a volume + should be ThickProvisioned or ThinProvisioned. Default + is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated with + the protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created in + the ScaleIO system that is associated with this volume + source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value + between 0000 and 0777 or a decimal value between 0 + and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults + to 0644. Directories within the path are not affected + by this setting. This might be in conflict with other + options that affect the file mode, like fsGroup, and + the result can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced Secret will be projected + into the volume as a file whose name is the key and + content is the value. If specified, the listed keys + will be projected into the specified paths, and unlisted + keys will not be present. If a key is specified which + is not present in the Secret, the volume setup will + error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start + with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to set + permissions on this file. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or its keys + must be defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace + to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use for + obtaining the StorageOS API credentials. If not specified, + default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeName: + description: VolumeName is the human-readable name of + the StorageOS volume. Volume names are only unique + within a namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope of + the volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows + the Kubernetes name scoping to be mirrored within + StorageOS for tighter integration. Set VolumeName + to any name to override the default behaviour. Set + to "default" if you are not using namespaces within + StorageOS. Namespaces that do not pre-exist within + StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) + profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: Path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + osNativeRepo: + description: This specifies the repo location to be used when + using rpm/deb packages Some examples add-apt-repository 'deb + [arch=amd64] https://repo.confidential-containers.org/apt/ubuntu’ add-apt-repository + ppa:confidential-containers/cc-bundle dnf install -y https://repo.confidential-containers.org/yum/centos/cc-bundle-repo.rpm + type: string + payloadImage: + description: This specifies the location of the container image + with all artifacts (Cc runtime binaries, initrd, kernel, config + etc) when using "bundle" installType + type: string + postUninstall: + description: This specifies the configuration for the post-uninstall + daemonset + properties: + cmd: + description: This specifies the command executes before UnInstallCmd + items: + type: string + type: array + environmentVariables: + description: This specifies the env variables for the post-uninstall + daemon set + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + image: + description: This specifies the pull spec for the postuninstall + daemonset image + type: string + volumeMounts: + description: This specifies the volumeMounts for the post-uninstall + daemon set + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which the + volume should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and the + other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's + root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves + similarly to SubPath but environment variable references + $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr and SubPath + are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + description: This specifies the volumes for the post-uninstall + daemon set + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS + Disk resource that is attached to a kubelet''s host + machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'The partition in the volume that you + want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the + volume partition for /dev/sda is "0" (or you can + leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the + ReadOnly property in VolumeMounts to "true". If + omitted, the default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource + in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk + mount on the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, + Read Write.' + type: string + diskName: + description: The Name of the data disk in the blob + storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob + disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed + data disk (only in managed availability set). + defaults to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure + Storage Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the + host that shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to + key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to + the authentication secret for User, default is + empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + user: + description: 'Optional: User is the rados user name, + default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached + and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be + a filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: + https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object + containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + volumeID: + description: 'volume id used to identify the volume + in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This + might be in conflict with other options that affect + the file mode, like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair + in the Data field of the referenced ConfigMap + will be projected into the volume as a file whose + name is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the ConfigMap, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to + set permissions on this file. Must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both + octal and decimal values, JSON requires + decimal values for mode bits. If not specified, + the volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + csi: + description: CSI (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: Driver is the name of the CSI driver + that handles this volume. Consult with your admin + for the correct name as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", + "xfs", "ntfs". If not provided, the empty value + is passed to the associated CSI driver which will + determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference + to the secret object containing sensitive information + to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no + secret is required. If the secret object contains + more than one secret, all secret references are + passed. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific + properties that are passed to the CSI driver. + Consult your driver's documentation for supported + values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about + the pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a Optional: mode bits + used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories + within the path are not affected by this setting. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing the + pod field + properties: + fieldRef: + description: 'Required: Selects a field of + the pod: only annotations, labels, name + and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used to + set permissions on this file, must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both + octal and decimal values, JSON requires + decimal values for mode bits. If not specified, + the volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' path. + Must be utf-8 encoded. The first item of + the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should + back this directory. The default is "" which means + to use the node''s default medium. Must be an + empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'Total amount of local storage required + for this EmptyDir volume. The size limit is also + applicable for memory medium. The maximum usage + on memory medium EmptyDir would be the minimum + value between the SizeLimit specified here and + the sum of memory limits of all containers in + a pod. The default is nil which means that the + limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "Ephemeral represents a volume that is + handled by a cluster storage driver. The volume's + lifecycle is tied to the pod that defines it - it + will be created before the pod starts, and deleted + when the pod is removed. \n Use this if: a) the volume + is only needed while the pod runs, b) features of + normal volumes like restoring from snapshot or capacity + \ tracking are needed, c) the storage driver is + specified through a storage class, and d) the storage + driver supports dynamic volume provisioning through + \ a PersistentVolumeClaim (see EphemeralVolumeSource + for more information on the connection between + this volume type and PersistentVolumeClaim). \n + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the + lifecycle of an individual pod. \n Use CSI for light-weight + local ephemeral volumes if the CSI driver is meant + to be used that way - see the documentation of the + driver for more information. \n A pod can use both + types of ephemeral volumes and persistent volumes + at the same time. \n This is a beta feature and only + available when the GenericEphemeralVolume feature + gate is enabled." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone + PVC to provision the volume. The pod in which + this EphemeralVolumeSource is embedded will be + the owner of the PVC, i.e. the PVC will be deleted + together with the pod. The name of the PVC will + be `-` where `` + is the name from the `PodSpec.Volumes` array entry. + Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too + long). \n An existing PVC with that name that + is not owned by the pod will *not* be used for + the pod to avoid using an unrelated volume by + mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created + PVC is meant to be used by the pod, the PVC has + to updated with an owner reference to the pod + once the pod exists. Normally this should not + be necessary, but it may be useful when manually + reconstructing a broken cluster. \n This field + is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. \n Required, + must not be nil." + properties: + metadata: + description: May contain labels and annotations + that will be copied into the PVC when creating + it. No other fields are allowed and will be + rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into + the PVC that gets created from this template. + The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired + access modes the volume should have. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to + specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller + can support the specified data source, + it will create a new volume based on the + contents of the specified data source. + If the AnyVolumeDataSource feature gate + is enabled, this field will always have + the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for + the resource being referenced. If + APIGroup is not specified, the specified + Kind must be in the core API group. + For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from + which to populate the volume with data, + if a non-empty volume is desired. This + may be any local object from a non-empty + API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, + volume binding will only succeed if the + type of the specified object matches some + installed volume populator or dynamic + provisioner. This field will replace the + functionality of the DataSource field + and as such if both fields are non-empty, + they must have the same value. For backwards + compatibility, both fields (DataSource + and DataSourceRef) will be set to the + same value automatically if one of them + is empty and the other is non-empty. There + are two important differences between + DataSource and DataSourceRef: * While + DataSource only allows two specific types + of objects, DataSourceRef allows any + non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed + values (dropping them), DataSourceRef preserves + all values, and generates an error if + a disallowed value is specified. (Alpha) + Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for + the resource being referenced. If + APIGroup is not specified, the specified + Kind must be in the core API group. + For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum + resources the volume should have. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the + minimum amount of compute resources + required. If Requests is omitted for + a container, it defaults to Limits + if that is explicitly specified, otherwise + to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type + of volume is required by the claim. Value + of Filesystem is implied when not included + in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: FC represents a Fibre Channel resource + that is attached to a kubelet's host machine and then + exposed to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be + a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. TODO: how + do we prevent errors in the filesystem from compromising + the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names + (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs + and lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume + resource that is provisioned/attached using an exec + based plugin. + properties: + driver: + description: Driver is the name of the driver to + use for this volume. + type: string + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". The default + filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if + any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to + the secret object containing sensitive information + to pass to the plugin scripts. This may be empty + if no secret object is specified. If the secret + object contains more than one secret, all secrets + are passed to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: Name of the dataset stored as metadata + -> name on the dataset for Flocker should be considered + as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique + identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'The partition in the volume that you + want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the + volume partition for /dev/sda is "0" (or you can + leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in + GCE. Used to identify the disk in GCE. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at + a particular revision. DEPRECATED: GitRepo is deprecated. + To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo + using git, then mount the EmptyDir into the Pod''s + container.' + properties: + directory: + description: Target directory name. Must not contain + or start with '..'. If '.' is supplied, the volume + directory will be the git repository. Otherwise, + if specified, the volume will contain the git + repository in the subdirectory with the given + name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name + that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs + volume to be mounted with read-only permissions. + Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file + or directory on the host machine that is directly + exposed to the container. This is generally used for + system agents or other privileged things that are + allowed to see the host machine. Most containers will + NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use + host directory mounts and who can/can not mount host + directories as read/write.' + properties: + path: + description: 'Path of the directory on the host. + If the path is a symlink, it will follow the link + to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults + to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource + that is attached to a kubelet''s host machine and + then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP + authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP + authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, + new iSCSI interface : + will be created for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI + transport. Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal + is either an IP or ip_addr:port if the port is + other than default (typically TCP ports 860 and + 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator + authentication + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + targetPortal: + description: iSCSI Target Portal. The Portal is + either an IP or ip_addr:port if the port is other + than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host + that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address + of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same + namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in + VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + pdID: + description: ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type + to mount Must be a filesystem type supported by + the host operating system. Ex. "ext4", "xfs". + Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: Mode bits used to set permissions on + created files by default. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. Directories within the path are not affected + by this setting. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected + along with other supported volume types + properties: + configMap: + description: information about the configMap + data to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the + volume as a file whose name is the key + and content is the value. If specified, + the listed keys will be projected into + the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the ConfigMap, + the volume setup will error unless it + is marked optional. Paths must be relative + and may not contain the '..' path or + start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits + used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts + both octal and decimal values, + JSON requires decimal values for + mode bits. If not specified, the + volume defaultMode will be used. + This might be in conflict with + other options that affect the + file mode, like fsGroup, and the + result can be other mode bits + set.' + format: int32 + type: integer + path: + description: The relative path of + the file to map the key to. May + not be an absolute path. May not + contain the path element '..'. + May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + downwardAPI: + description: information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name and namespace are + supported.' + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits + used to set permissions on this + file, must be an octal value between + 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts + both octal and decimal values, + JSON requires decimal values for + mode bits. If not specified, the + volume defaultMode will be used. + This might be in conflict with + other options that affect the + file mode, like fsGroup, and the + result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file + to be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource + of the container: only resources + limits and requests (limits.cpu, + limits.memory, requests.cpu and + requests.memory) are currently + supported.' + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret + data to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and + content is the value. If specified, + the listed keys will be projected into + the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the Secret, + the volume setup will error unless it + is marked optional. Paths must be relative + and may not contain the '..' path or + start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits + used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts + both octal and decimal values, + JSON requires decimal values for + mode bits. If not specified, the + volume defaultMode will be used. + This might be in conflict with + other options that affect the + file mode, like fsGroup, and the + result can be other mode bits + set.' + format: int32 + type: integer + path: + description: The relative path of + the file to map the key to. May + not be an absolute path. May not + contain the path element '..'. + May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + type: object + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended + audience of the token. A recipient of + a token must identify itself with an + identifier specified in the audience + of the token, and otherwise should reject + the token. The audience defaults to + the identifier of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the + requested duration of validity of the + service account token. As the token + approaches expiration, the kubelet volume + plugin will proactively rotate the service + account token. The kubelet will start + trying to rotate the token if the token + is older than 80 percent of its time + to live or if the token is older than + 24 hours.Defaults to 1 hour and must + be at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative + to the mount point of the file to project + the token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the + host that shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default + is no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte + volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: Registry represents a single or multiple + Quobyte Registry services specified as a string + as host:port pair (multiple entries are separated + with commas) which acts as the central registry + for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned + Quobyte volumes, value is set by the plugin + type: string + user: + description: User to map volume access to Defaults + to serivceaccount user + type: string + volume: + description: Volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + image: + description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for + RBDUser. Default is /etc/ceph/keyring. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + user: + description: 'The rados user name. Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Default is + "xfs". + type: string + gateway: + description: The host address of the ScaleIO API + Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection + Domain for the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret + for ScaleIO user and other sensitive information. + If this is not provided, Login operation will + fail. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + sslEnabled: + description: Flag to enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a + volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created + in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should + populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This + might be in conflict with other options that affect + the file mode, like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair + in the Data field of the referenced Secret will + be projected into the volume as a file whose name + is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the Secret, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to + set permissions on this file. Must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both + octal and decimal values, JSON requires + decimal values for mode bits. If not specified, + the volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or its keys + must be defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace + to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use + for obtaining the StorageOS API credentials. If + not specified, default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + volumeName: + description: VolumeName is the human-readable name + of the StorageOS volume. Volume names are only + unique within a namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope + of the volume within StorageOS. If no namespace + is specified then the Pod's namespace will be + used. This allows the Kubernetes name scoping + to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default + behaviour. Set to "default" if you are not using + namespaces within StorageOS. Namespaces that do + not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume + attached and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) + profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: Path that identifies vSphere volume + vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + preInstall: + description: This specifies the configuration for the pre-install + daemonset + properties: + cmd: + description: This specifies the command executes before InstallCmd + items: + type: string + type: array + environmentVariables: + description: This specifies the env variables for the pre-install + daemon set + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + image: + description: This specifies the image for the pre-install + scripts + type: string + volumeMounts: + description: This specifies the volumeMounts for the pre-install + daemon set + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which the + volume should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and the + other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's + root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves + similarly to SubPath but environment variable references + $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr and SubPath + are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + description: This specifies the volumes for the pre-install + daemon set + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS + Disk resource that is attached to a kubelet''s host + machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'The partition in the volume that you + want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the + volume partition for /dev/sda is "0" (or you can + leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the + ReadOnly property in VolumeMounts to "true". If + omitted, the default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource + in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk + mount on the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, + Read Write.' + type: string + diskName: + description: The Name of the data disk in the blob + storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob + disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed + data disk (only in managed availability set). + defaults to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure + Storage Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the + host that shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to + key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to + the authentication secret for User, default is + empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + user: + description: 'Optional: User is the rados user name, + default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached + and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be + a filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: + https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object + containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + volumeID: + description: 'volume id used to identify the volume + in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This + might be in conflict with other options that affect + the file mode, like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair + in the Data field of the referenced ConfigMap + will be projected into the volume as a file whose + name is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the ConfigMap, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to + set permissions on this file. Must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both + octal and decimal values, JSON requires + decimal values for mode bits. If not specified, + the volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + csi: + description: CSI (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: Driver is the name of the CSI driver + that handles this volume. Consult with your admin + for the correct name as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", + "xfs", "ntfs". If not provided, the empty value + is passed to the associated CSI driver which will + determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference + to the secret object containing sensitive information + to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no + secret is required. If the secret object contains + more than one secret, all secret references are + passed. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific + properties that are passed to the CSI driver. + Consult your driver's documentation for supported + values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about + the pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a Optional: mode bits + used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories + within the path are not affected by this setting. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing the + pod field + properties: + fieldRef: + description: 'Required: Selects a field of + the pod: only annotations, labels, name + and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used to + set permissions on this file, must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both + octal and decimal values, JSON requires + decimal values for mode bits. If not specified, + the volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' path. + Must be utf-8 encoded. The first item of + the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should + back this directory. The default is "" which means + to use the node''s default medium. Must be an + empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'Total amount of local storage required + for this EmptyDir volume. The size limit is also + applicable for memory medium. The maximum usage + on memory medium EmptyDir would be the minimum + value between the SizeLimit specified here and + the sum of memory limits of all containers in + a pod. The default is nil which means that the + limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "Ephemeral represents a volume that is + handled by a cluster storage driver. The volume's + lifecycle is tied to the pod that defines it - it + will be created before the pod starts, and deleted + when the pod is removed. \n Use this if: a) the volume + is only needed while the pod runs, b) features of + normal volumes like restoring from snapshot or capacity + \ tracking are needed, c) the storage driver is + specified through a storage class, and d) the storage + driver supports dynamic volume provisioning through + \ a PersistentVolumeClaim (see EphemeralVolumeSource + for more information on the connection between + this volume type and PersistentVolumeClaim). \n + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the + lifecycle of an individual pod. \n Use CSI for light-weight + local ephemeral volumes if the CSI driver is meant + to be used that way - see the documentation of the + driver for more information. \n A pod can use both + types of ephemeral volumes and persistent volumes + at the same time. \n This is a beta feature and only + available when the GenericEphemeralVolume feature + gate is enabled." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone + PVC to provision the volume. The pod in which + this EphemeralVolumeSource is embedded will be + the owner of the PVC, i.e. the PVC will be deleted + together with the pod. The name of the PVC will + be `-` where `` + is the name from the `PodSpec.Volumes` array entry. + Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too + long). \n An existing PVC with that name that + is not owned by the pod will *not* be used for + the pod to avoid using an unrelated volume by + mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created + PVC is meant to be used by the pod, the PVC has + to updated with an owner reference to the pod + once the pod exists. Normally this should not + be necessary, but it may be useful when manually + reconstructing a broken cluster. \n This field + is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. \n Required, + must not be nil." + properties: + metadata: + description: May contain labels and annotations + that will be copied into the PVC when creating + it. No other fields are allowed and will be + rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into + the PVC that gets created from this template. + The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired + access modes the volume should have. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to + specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller + can support the specified data source, + it will create a new volume based on the + contents of the specified data source. + If the AnyVolumeDataSource feature gate + is enabled, this field will always have + the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for + the resource being referenced. If + APIGroup is not specified, the specified + Kind must be in the core API group. + For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from + which to populate the volume with data, + if a non-empty volume is desired. This + may be any local object from a non-empty + API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, + volume binding will only succeed if the + type of the specified object matches some + installed volume populator or dynamic + provisioner. This field will replace the + functionality of the DataSource field + and as such if both fields are non-empty, + they must have the same value. For backwards + compatibility, both fields (DataSource + and DataSourceRef) will be set to the + same value automatically if one of them + is empty and the other is non-empty. There + are two important differences between + DataSource and DataSourceRef: * While + DataSource only allows two specific types + of objects, DataSourceRef allows any + non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed + values (dropping them), DataSourceRef preserves + all values, and generates an error if + a disallowed value is specified. (Alpha) + Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for + the resource being referenced. If + APIGroup is not specified, the specified + Kind must be in the core API group. + For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum + resources the volume should have. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the + minimum amount of compute resources + required. If Requests is omitted for + a container, it defaults to Limits + if that is explicitly specified, otherwise + to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type + of volume is required by the claim. Value + of Filesystem is implied when not included + in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: FC represents a Fibre Channel resource + that is attached to a kubelet's host machine and then + exposed to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be + a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. TODO: how + do we prevent errors in the filesystem from compromising + the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names + (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs + and lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume + resource that is provisioned/attached using an exec + based plugin. + properties: + driver: + description: Driver is the name of the driver to + use for this volume. + type: string + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". The default + filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if + any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to + the secret object containing sensitive information + to pass to the plugin scripts. This may be empty + if no secret object is specified. If the secret + object contains more than one secret, all secrets + are passed to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: Name of the dataset stored as metadata + -> name on the dataset for Flocker should be considered + as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique + identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'The partition in the volume that you + want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the + volume partition for /dev/sda is "0" (or you can + leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in + GCE. Used to identify the disk in GCE. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at + a particular revision. DEPRECATED: GitRepo is deprecated. + To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo + using git, then mount the EmptyDir into the Pod''s + container.' + properties: + directory: + description: Target directory name. Must not contain + or start with '..'. If '.' is supplied, the volume + directory will be the git repository. Otherwise, + if specified, the volume will contain the git + repository in the subdirectory with the given + name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name + that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs + volume to be mounted with read-only permissions. + Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file + or directory on the host machine that is directly + exposed to the container. This is generally used for + system agents or other privileged things that are + allowed to see the host machine. Most containers will + NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use + host directory mounts and who can/can not mount host + directories as read/write.' + properties: + path: + description: 'Path of the directory on the host. + If the path is a symlink, it will follow the link + to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults + to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource + that is attached to a kubelet''s host machine and + then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP + authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP + authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, + new iSCSI interface : + will be created for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI + transport. Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal + is either an IP or ip_addr:port if the port is + other than default (typically TCP ports 860 and + 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator + authentication + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + targetPortal: + description: iSCSI Target Portal. The Portal is + either an IP or ip_addr:port if the port is other + than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host + that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address + of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same + namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in + VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + pdID: + description: ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type + to mount Must be a filesystem type supported by + the host operating system. Ex. "ext4", "xfs". + Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: Mode bits used to set permissions on + created files by default. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. Directories within the path are not affected + by this setting. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected + along with other supported volume types + properties: + configMap: + description: information about the configMap + data to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the + volume as a file whose name is the key + and content is the value. If specified, + the listed keys will be projected into + the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the ConfigMap, + the volume setup will error unless it + is marked optional. Paths must be relative + and may not contain the '..' path or + start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits + used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts + both octal and decimal values, + JSON requires decimal values for + mode bits. If not specified, the + volume defaultMode will be used. + This might be in conflict with + other options that affect the + file mode, like fsGroup, and the + result can be other mode bits + set.' + format: int32 + type: integer + path: + description: The relative path of + the file to map the key to. May + not be an absolute path. May not + contain the path element '..'. + May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + downwardAPI: + description: information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name and namespace are + supported.' + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits + used to set permissions on this + file, must be an octal value between + 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts + both octal and decimal values, + JSON requires decimal values for + mode bits. If not specified, the + volume defaultMode will be used. + This might be in conflict with + other options that affect the + file mode, like fsGroup, and the + result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file + to be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource + of the container: only resources + limits and requests (limits.cpu, + limits.memory, requests.cpu and + requests.memory) are currently + supported.' + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret + data to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and + content is the value. If specified, + the listed keys will be projected into + the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the Secret, + the volume setup will error unless it + is marked optional. Paths must be relative + and may not contain the '..' path or + start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits + used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts + both octal and decimal values, + JSON requires decimal values for + mode bits. If not specified, the + volume defaultMode will be used. + This might be in conflict with + other options that affect the + file mode, like fsGroup, and the + result can be other mode bits + set.' + format: int32 + type: integer + path: + description: The relative path of + the file to map the key to. May + not be an absolute path. May not + contain the path element '..'. + May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + type: object + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended + audience of the token. A recipient of + a token must identify itself with an + identifier specified in the audience + of the token, and otherwise should reject + the token. The audience defaults to + the identifier of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the + requested duration of validity of the + service account token. As the token + approaches expiration, the kubelet volume + plugin will proactively rotate the service + account token. The kubelet will start + trying to rotate the token if the token + is older than 80 percent of its time + to live or if the token is older than + 24 hours.Defaults to 1 hour and must + be at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative + to the mount point of the file to project + the token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the + host that shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default + is no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte + volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: Registry represents a single or multiple + Quobyte Registry services specified as a string + as host:port pair (multiple entries are separated + with commas) which acts as the central registry + for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned + Quobyte volumes, value is set by the plugin + type: string + user: + description: User to map volume access to Defaults + to serivceaccount user + type: string + volume: + description: Volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + image: + description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for + RBDUser. Default is /etc/ceph/keyring. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + user: + description: 'The rados user name. Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Default is + "xfs". + type: string + gateway: + description: The host address of the ScaleIO API + Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection + Domain for the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret + for ScaleIO user and other sensitive information. + If this is not provided, Login operation will + fail. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + sslEnabled: + description: Flag to enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a + volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created + in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should + populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This + might be in conflict with other options that affect + the file mode, like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair + in the Data field of the referenced Secret will + be projected into the volume as a file whose name + is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the Secret, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to + set permissions on this file. Must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both + octal and decimal values, JSON requires + decimal values for mode bits. If not specified, + the volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or its keys + must be defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace + to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use + for obtaining the StorageOS API credentials. If + not specified, default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + volumeName: + description: VolumeName is the human-readable name + of the StorageOS volume. Volume names are only + unique within a namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope + of the volume within StorageOS. If no namespace + is specified then the Pod's namespace will be + used. This allows the Kubernetes name scoping + to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default + behaviour. Set to "default" if you are not using + namespaces within StorageOS. Namespaces that do + not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume + attached and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) + profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: Path that identifies vSphere volume + vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + runtimeImage: + description: This specifies the location of the container image + containing the Cc runtime binaries If both payloadImage and + runtimeImage are specified, then runtimeImage content will override + the equivalent one in payloadImage + type: string + uninstallCmd: + description: This specifies the command for uninstallation of + the runtime on the nodes + items: + type: string + type: array + uninstallDoneLabel: + additionalProperties: + type: string + description: This specifies the label that the uninstall daemonset + adds to nodes when the uninstallation is done + type: object + required: + - installType + - payloadImage + type: object + runtimeName: + enum: + - kata + type: string + required: + - config + - runtimeName + type: object + status: + description: CcRuntimeStatus defines the observed state of CcRuntime + properties: + installationStatus: + description: InstallationStatus reflects the status of the ongoing + runtime installation + properties: + completed: + description: Completed reflects the status of nodes that have + completed the installation + properties: + completedNodesCount: + description: CompletedNodesCount reflects the number of nodes + that have completed install operation + type: integer + completedNodesList: + description: CompletedNodesList reflects the list of nodes + that have completed install operation + items: + type: string + type: array + type: object + failed: + description: Failed reflects the status of nodes that have failed + installation + properties: + failedNodesCount: + description: FailedNodesCount reflects the number of nodes + that have failed installation + type: integer + failedNodesList: + description: FailedNodesList reflects the list of nodes that + have failed installation + items: + description: FailedNodeStatus holds the name and the error + message of the failed node + properties: + error: + description: Error message of the failed node reported + by the installation daemon + type: string + name: + description: Name of the failed node + type: string + required: + - error + - name + type: object + type: array + type: object + inProgress: + description: InProgress reflects the status of nodes that are + in the process of installation + properties: + binariesInstallNodesList: + items: + type: string + type: array + inProgressNodesCount: + description: InProgressNodesCount reflects the number of nodes + that are in the process of installation + type: integer + type: object + type: object + runtimeClass: + description: RuntimeClass is the name of the runtime class as used + in container runtime configuration + type: string + runtimeName: + description: Cc Runtime Name + enum: + - kata + type: string + totalNodesCount: + description: TotalNodesCounts is the total number of worker nodes + targeted by this CR + type: integer + unInstallationStatus: + description: UnInstallationStatus reflects the status of the ongoing + runtime uninstallation + properties: + completed: + description: Completed reflects the status of nodes that have + completed the uninstallation operation + properties: + completedNodesCount: + description: CompletedNodesCount reflects the number of nodes + that have completed install operation + type: integer + completedNodesList: + description: CompletedNodesList reflects the list of nodes + that have completed install operation + items: + type: string + type: array + type: object + failed: + description: Failed reflects the status of nodes that have failed + uninstallation + properties: + failedNodesCount: + description: FailedNodesCount reflects the number of nodes + that have failed installation + type: integer + failedNodesList: + description: FailedNodesList reflects the list of nodes that + have failed installation + items: + description: FailedNodeStatus holds the name and the error + message of the failed node + properties: + error: + description: Error message of the failed node reported + by the installation daemon + type: string + name: + description: Name of the failed node + type: string + required: + - error + - name + type: object + type: array + type: object + inProgress: + description: InProgress reflects the status of nodes that are + in the process of uninstallation + properties: + binariesUninstallNodesList: + items: + type: string + type: array + inProgressNodesCount: + description: InProgressNodesCount reflects the number of nodes + that are in the process of uninstallation + type: integer + type: object + type: object + upgradeStatus: + description: Upgradestatus reflects the status of the ongoing runtime + upgrade + type: object + required: + - runtimeClass + - runtimeName + - totalNodesCount + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/install/yamls/kustomization.yaml b/install/yamls/kustomization.yaml new file mode 100644 index 000000000..10bb6134e --- /dev/null +++ b/install/yamls/kustomization.yaml @@ -0,0 +1,14 @@ +resources: + - ccruntime-peer-pods.yaml + - caa-pod.yaml + +# remove host-install configuration from ccruntime-peer-pods.yaml +patchesJSON6902: +- target: + kind: CcRuntime + name: ccruntime-sample + patch: |- + - op: remove + path: /spec/config/preInstall + - op: remove + path: /spec/config/postUninstall diff --git a/libvirt/README.md b/libvirt/README.md new file mode 100644 index 000000000..9d88f406b --- /dev/null +++ b/libvirt/README.md @@ -0,0 +1,46 @@ +# Setup instructions + +- KVM host with libvirt configured. +- Libvirt network and storage pool created +- A base storage volume created for POD VM + +## Creation of base storage volume + +- Ubuntu 20.04 VM with minimum 50GB disk and the following packages installed + - `cloud-image-utils` + - `qemu-system-x86` + +- Install packer on the VM by following the instructions in the following [link](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli) + +- Create qcow2 image by executing the following command + - [setting up authenticated registry support](../docs/registries-authentication.md) +``` +cd ../podvm +CLOUD_PROVIDER=libvirt make image +``` +- The default image uses an Ubuntu cloud image base. If instead, you would like to create a rhel based podvm image, you can +set PODVM_DISTRO when invoking build as such: +``` +export RHEL_IMAGE_URL="" <- either a local path or an url +export RHEL_IMAGE_CHECKSUM="" <- the checksum of the aforementioned image +CLOUD_PROVIDER=libvirt PODVM_DISTRO=rhel make build +``` + +The image will be available under the `output` directory + +- Copy the qcow2 image to the libvirt machine + +- Create volume +``` +export IMAGE= + +virsh vol-create-as --pool default --name podvm-base.qcow2 --capacity 20G --allocation 2G --prealloc-metadata --format qcow2 +virsh vol-upload --vol podvm-base.qcow2 $IMAGE --pool default --sparse +``` + +If you want to set default password for podvm debugging then you can use guestfish to edit the qcow2 and make any suitable changes. + +# Running cloud-api-adaptor + +Refer to the [cloud-api-adaptor deployment instructions](../install/README.md#deploy-cloud-api-adaptor) +* If your libvirt host is not configured to be passwordless make sure you set ssh-key-secret in the [kustomization file](../install/overlays/libvirt/kustomization.yaml) diff --git a/peer-pod-controller/.dockerignore b/peer-pod-controller/.dockerignore new file mode 100644 index 000000000..0f046820f --- /dev/null +++ b/peer-pod-controller/.dockerignore @@ -0,0 +1,4 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ +testbin/ diff --git a/peer-pod-controller/.gitignore b/peer-pod-controller/.gitignore new file mode 100644 index 000000000..c0a7a54ca --- /dev/null +++ b/peer-pod-controller/.gitignore @@ -0,0 +1,25 @@ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin +testbin/* + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Kubernetes Generated files - skip generated files, except for vendored files + +!vendor/**/zz_generated.* + +# editor and IDE paraphernalia +.idea +*.swp +*.swo +*~ diff --git a/peer-pod-controller/Dockerfile b/peer-pod-controller/Dockerfile new file mode 100644 index 000000000..5a355c214 --- /dev/null +++ b/peer-pod-controller/Dockerfile @@ -0,0 +1,27 @@ +# Build the manager binary +FROM golang:1.18 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY api/ api/ +COPY controllers/ controllers/ + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/peer-pod-controller/Makefile b/peer-pod-controller/Makefile new file mode 100644 index 000000000..1aa4355aa --- /dev/null +++ b/peer-pod-controller/Makefile @@ -0,0 +1,233 @@ +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +VERSION ?= 0.0.1 + +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif + +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# This variable is used to construct full image tags for bundle and catalog images. +# +# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both +# confidentialcontainers.org/peer-pod-controller-bundle:$VERSION and confidentialcontainers.org/peer-pod-controller-catalog:$VERSION. +IMAGE_TAG_BASE ?= confidentialcontainers.org/peer-pod-controller + +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command +BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + +# USE_IMAGE_DIGESTS defines if images are resolved via tags or digests +# You can enable this value if you would like to use SHA Based Digests +# To enable set flag to true +USE_IMAGE_DIGESTS ?= false +ifeq ($(USE_IMAGE_DIGESTS), true) + BUNDLE_GEN_FLAGS += --use-image-digests +endif + +# Image URL to use all building/pushing image targets +IMG ?= controller:latest +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.23 + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# This is a requirement for 'setup-envtest.sh' in the test target. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out + +##@ Build + +.PHONY: build +build: generate fmt vet ## Build manager binary. + go build -o bin/manager main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./main.go + +.PHONY: docker-build +docker-build: #test ## Build docker image with the manager. + docker build -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + docker push ${IMG} + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +.PHONY: undeploy +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +CONTROLLER_GEN = $(shell pwd)/bin/controller-gen +.PHONY: controller-gen +controller-gen: ## Download controller-gen locally if necessary. + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0) + +KUSTOMIZE = $(shell pwd)/bin/kustomize +.PHONY: kustomize +kustomize: ## Download kustomize locally if necessary. + $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) + +ENVTEST = $(shell pwd)/bin/setup-envtest +.PHONY: envtest +envtest: ## Download envtest-setup locally if necessary. + $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) + +# go-get-tool will 'go get' any package $2 and install it to $1. +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +define go-get-tool +@[ -f $(1) ] || { \ +set -e ;\ +TMP_DIR=$$(mktemp -d) ;\ +cd $$TMP_DIR ;\ +go mod init tmp ;\ +echo "Downloading $(2)" ;\ +GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ +rm -rf $$TMP_DIR ;\ +} +endef + +.PHONY: bundle +bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files. + operator-sdk generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle $(BUNDLE_GEN_FLAGS) + operator-sdk bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) docker-push IMG=$(BUNDLE_IMG) + +.PHONY: opm +OPM = ./bin/opm +opm: ## Download opm locally if necessary. +ifeq (,$(wildcard $(OPM))) +ifeq (,$(shell which opm 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPM)) ;\ + OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.19.1/$${OS}-$${ARCH}-opm ;\ + chmod +x $(OPM) ;\ + } +else +OPM = $(shell which opm) +endif +endif + +# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). +# These images MUST exist in a registry and be pull-able. +BUNDLE_IMGS ?= $(BUNDLE_IMG) + +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + +# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. +ifneq ($(origin CATALOG_BASE_IMG), undefined) +FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +endif + +# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. +# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: +# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator +.PHONY: catalog-build +catalog-build: opm ## Build a catalog image. + $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +# Push the catalog image. +.PHONY: catalog-push +catalog-push: ## Push a catalog image. + $(MAKE) docker-push IMG=$(CATALOG_IMG) diff --git a/peer-pod-controller/PROJECT b/peer-pod-controller/PROJECT new file mode 100644 index 000000000..4de8ca8e7 --- /dev/null +++ b/peer-pod-controller/PROJECT @@ -0,0 +1,18 @@ +domain: confidentialcontainers.org +layout: +- go.kubebuilder.io/v3 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: peer-pod-controller +repo: github.com/confidential-containers/cloud-api-adaptor/peer-pod-controller +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: confidentialcontainers.org + kind: PeerPodConfig + path: github.com/confidential-containers/cloud-api-adaptor/peer-pod-controller/api/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/peer-pod-controller/README.md b/peer-pod-controller/README.md new file mode 100644 index 000000000..5cba45b6b --- /dev/null +++ b/peer-pod-controller/README.md @@ -0,0 +1,139 @@ +# peer-pod-controller +peer-pod-controller is a kubernetes controller which is responsible for +watching the PeerPodConfig CRD object and manages the creation and deletion +lifecycle of all required components to run peer pods. + +## Description +This controller can be run standalone or imported into existing operators. It comes with +a CRD called PeerPodConfig that it's watching. By creating an instance of PeerPodConfig the deployment of +cloud-api-adaptor daemonset and the webhook is triggered, extended resources are advertised by +updating the node capacity fields. + +### PeerPodConfig CRD +The PeerPodConfig let's the user specify the number of peer pod vms that can be deployed. +It is spread as evenly as possible across the number of nodes. + +## Integrate with your operator +Running the peer-pod-controller as another controller embedded into an operator can be easily +done. Import the controller into your operators main.go and start it. + +For an operator-sdk generated operator it could be as simple as: + +```go + import ( + ... + peerpodcontrollers "github.com/confidential-containers/cloud-api-adaptor/peer-pod-controller/controllers" + peerpodconfig "github.com/confidential-containers/cloud-api-adaptor/peer-pod-controller/api/v1alpha1" + ... + ) + + func init() { + ... + utilruntime.Must(peerpodconfig.AddToScheme(scheme)) + ... + } + + func main() { + ... + if err = (&peerpodcontrollers.PeerPodConfigReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("RemotePodConfig"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create RemotePodConfig controller for OpenShift cluster", "controller", "RemotePodConfig") + os.Exit(1) + } + ... + } +``` + +## Getting Started +Please note that while it is possible to run the controller standalone it is not the +intended use case. + +You’ll need a Kubernetes cluster to run against. You can use [KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. +**Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). + +### Running on the cluster +1. Install Instances of Custom Resources: + +```sh +kubectl apply -f config/samples/ +``` + +2. Build and push your image to the location specified by `IMG`: + +```sh +make docker-build docker-push IMG=/peer-pod-controller:tag +``` + +3. Deploy the controller to the cluster with the image specified by `IMG`: + +```sh +make deploy IMG=/peer-pod-controller:tag +``` + +### Uninstall CRDs +To delete the CRDs from the cluster: + +```sh +make uninstall +``` + +### Undeploy controller +UnDeploy the controller to the cluster: + +```sh +make undeploy +``` + +## Contributing +// TODO(user): Add detailed information on how you would like others to contribute to this project + +### How it works +This project aims to follow the Kubernetes [Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) + +It uses [Controllers](https://kubernetes.io/docs/concepts/architecture/controller/) +which provides a reconcile function responsible for synchronizing resources until the desired state is reached on the cluster + +### Test It Out +1. Install the CRDs into the cluster: + +```sh +make install +``` + +2. Run your controller (this will run in the foreground, so switch to a new terminal if you want to leave it running): + +```sh +make run +``` + +**NOTE:** You can also run this in one step by running: `make install run` + +### Modifying the API definitions +If you are editing the API definitions, generate the manifests such as CRs or CRDs using: + +```sh +make manifests +``` + +**NOTE:** Run `make --help` for more information on all potential `make` targets + +More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) + +## License + +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/peer-pod-controller/api/v1alpha1/groupversion_info.go b/peer-pod-controller/api/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..8bd1c4d74 --- /dev/null +++ b/peer-pod-controller/api/v1alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright Confidential Containers Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the v1alpha1 API group +//+kubebuilder:object:generate=true +//+groupName=confidentialcontainers.org +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "confidentialcontainers.org", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/peer-pod-controller/api/v1alpha1/peerpodconfig_types.go b/peer-pod-controller/api/v1alpha1/peerpodconfig_types.go new file mode 100644 index 000000000..c855c0694 --- /dev/null +++ b/peer-pod-controller/api/v1alpha1/peerpodconfig_types.go @@ -0,0 +1,77 @@ +/* +Copyright Confidential Containers Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PeerPodConfigSpec defines the desired state of PeerPodConfig +type PeerPodConfigSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // InstanceType describes the name of the instance type of the chosen cloud provider + InstanceType string `json:"instanceType,omitempty"` + + // Limit is the max number of peer pods. This is exposed as expended resource on nodes + Limit string `json:"limit,omitempty"` + + // CloudSecretName is the name of the secret that holds the credentials for the cloud provider + CloudSecretName string `json:"cloudSecretName"` + + // NodeSelector selects the nodes to which the cca pods, the RuntimeClass and the MachineConfigs we use + // to deploy the full peer pod solution. + NodeSelector *metav1.LabelSelector `json:"nodeSelector"` + + // ConfigMapName is the name of the configmap that holds cloud provider specific environment Variables + ConfigMapName string `json:"configMapName"` +} + +// PeerPodConfigStatus defines the observed state of PeerPodConfig +type PeerPodConfigStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // SetupCompleted is set to true when all components have been deployed/created + SetupCompleted bool `json:"setupCompleted,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// PeerPodConfig is the Schema for the peerpodconfigs API +type PeerPodConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec PeerPodConfigSpec `json:"spec,omitempty"` + Status PeerPodConfigStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// PeerPodConfigList contains a list of PeerPodConfig +type PeerPodConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []PeerPodConfig `json:"items"` +} + +func init() { + SchemeBuilder.Register(&PeerPodConfig{}, &PeerPodConfigList{}) +} diff --git a/peer-pod-controller/api/v1alpha1/zz_generated.deepcopy.go b/peer-pod-controller/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..d77ead84c --- /dev/null +++ b/peer-pod-controller/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,121 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright Confidential Containers Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerPodConfig) DeepCopyInto(out *PeerPodConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerPodConfig. +func (in *PeerPodConfig) DeepCopy() *PeerPodConfig { + if in == nil { + return nil + } + out := new(PeerPodConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PeerPodConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerPodConfigList) DeepCopyInto(out *PeerPodConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PeerPodConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerPodConfigList. +func (in *PeerPodConfigList) DeepCopy() *PeerPodConfigList { + if in == nil { + return nil + } + out := new(PeerPodConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PeerPodConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerPodConfigSpec) DeepCopyInto(out *PeerPodConfigSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerPodConfigSpec. +func (in *PeerPodConfigSpec) DeepCopy() *PeerPodConfigSpec { + if in == nil { + return nil + } + out := new(PeerPodConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerPodConfigStatus) DeepCopyInto(out *PeerPodConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerPodConfigStatus. +func (in *PeerPodConfigStatus) DeepCopy() *PeerPodConfigStatus { + if in == nil { + return nil + } + out := new(PeerPodConfigStatus) + in.DeepCopyInto(out) + return out +} diff --git a/peer-pod-controller/config/crd/bases/confidentialcontainers.org_peerpodconfigs.yaml b/peer-pod-controller/config/crd/bases/confidentialcontainers.org_peerpodconfigs.yaml new file mode 100644 index 000000000..eb1b6546a --- /dev/null +++ b/peer-pod-controller/config/crd/bases/confidentialcontainers.org_peerpodconfigs.yaml @@ -0,0 +1,118 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: peerpodconfigs.confidentialcontainers.org +spec: + group: confidentialcontainers.org + names: + kind: PeerPodConfig + listKind: PeerPodConfigList + plural: peerpodconfigs + singular: peerpodconfig + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: PeerPodConfig is the Schema for the peerpodconfigs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: PeerPodConfigSpec defines the desired state of PeerPodConfig + properties: + cloudSecretName: + description: CloudSecretName is the name of the secret that holds + the credentials for the cloud provider + type: string + instanceType: + description: InstanceType describes the name of the instance type + of the chosen cloud provider + type: string + limit: + description: Limit is the max number of peer pods. This is exposed + as expended resource on nodes + type: string + nodeSelector: + description: NodeSelector selects the nodes to which the cca pods, + the RuntimeClass and the MachineConfigs we use to deploy the full + peer pod solution. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + required: + - cloudSecretName + - nodeSelector + type: object + status: + description: PeerPodConfigStatus defines the observed state of PeerPodConfig + properties: + setupCompleted: + description: SetupCompleted is set to true when all components have + been deployed/created + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/peer-pod-controller/config/crd/kustomization.yaml b/peer-pod-controller/config/crd/kustomization.yaml new file mode 100644 index 000000000..b98bcf49c --- /dev/null +++ b/peer-pod-controller/config/crd/kustomization.yaml @@ -0,0 +1,21 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/confidentialcontainers.org_peerpodconfigs.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_peerpodconfigs.yaml +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_peerpodconfigs.yaml +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/peer-pod-controller/config/crd/kustomizeconfig.yaml b/peer-pod-controller/config/crd/kustomizeconfig.yaml new file mode 100644 index 000000000..ec5c150a9 --- /dev/null +++ b/peer-pod-controller/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/peer-pod-controller/config/crd/patches/cainjection_in_peerpodconfigs.yaml b/peer-pod-controller/config/crd/patches/cainjection_in_peerpodconfigs.yaml new file mode 100644 index 000000000..f26f3a26f --- /dev/null +++ b/peer-pod-controller/config/crd/patches/cainjection_in_peerpodconfigs.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: peerpodconfigs.confidentialcontainers.org diff --git a/peer-pod-controller/config/crd/patches/webhook_in_peerpodconfigs.yaml b/peer-pod-controller/config/crd/patches/webhook_in_peerpodconfigs.yaml new file mode 100644 index 000000000..c391d02c8 --- /dev/null +++ b/peer-pod-controller/config/crd/patches/webhook_in_peerpodconfigs.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: peerpodconfigs.confidentialcontainers.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/peer-pod-controller/config/default/kustomization.yaml b/peer-pod-controller/config/default/kustomization.yaml new file mode 100644 index 000000000..ba829cf44 --- /dev/null +++ b/peer-pod-controller/config/default/kustomization.yaml @@ -0,0 +1,74 @@ +# Adds namespace to all resources. +namespace: peer-pod-controller-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: peer-pod-controller- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +- manager_auth_proxy_patch.yaml + +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/peer-pod-controller/config/default/manager_auth_proxy_patch.yaml b/peer-pod-controller/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 000000000..45be31887 --- /dev/null +++ b/peer-pod-controller/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,34 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/peer-pod-controller/config/default/manager_config_patch.yaml b/peer-pod-controller/config/default/manager_config_patch.yaml new file mode 100644 index 000000000..6c400155c --- /dev/null +++ b/peer-pod-controller/config/default/manager_config_patch.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--config=controller_manager_config.yaml" + volumeMounts: + - name: manager-config + mountPath: /controller_manager_config.yaml + subPath: controller_manager_config.yaml + volumes: + - name: manager-config + configMap: + name: manager-config diff --git a/peer-pod-controller/config/manager/controller_manager_config.yaml b/peer-pod-controller/config/manager/controller_manager_config.yaml new file mode 100644 index 000000000..13d91e2b4 --- /dev/null +++ b/peer-pod-controller/config/manager/controller_manager_config.yaml @@ -0,0 +1,11 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: 127.0.0.1:8080 +webhook: + port: 9443 +leaderElection: + leaderElect: true + resourceName: 02cb2ce6.confidentialcontainers.org diff --git a/peer-pod-controller/config/manager/kustomization.yaml b/peer-pod-controller/config/manager/kustomization.yaml new file mode 100644 index 000000000..c90d0843d --- /dev/null +++ b/peer-pod-controller/config/manager/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/confidential-containers/peer-pod-controller + newTag: 0.2.0 diff --git a/peer-pod-controller/config/manager/manager.yaml b/peer-pod-controller/config/manager/manager.yaml new file mode 100644 index 000000000..678025fec --- /dev/null +++ b/peer-pod-controller/config/manager/manager.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + securityContext: + runAsNonRoot: true + containers: + - command: + - /manager + args: + - --leader-elect + image: controller:latest + name: manager + securityContext: + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + env: + - name: PEERPOD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/peer-pod-controller/config/manifests/kustomization.yaml b/peer-pod-controller/config/manifests/kustomization.yaml new file mode 100644 index 000000000..ec497b6cd --- /dev/null +++ b/peer-pod-controller/config/manifests/kustomization.yaml @@ -0,0 +1,27 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- bases/peer-pod-controller.clusterserviceversion.yaml +- ../default +- ../samples +- ../scorecard + +# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. +# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. +# These patches remove the unnecessary "cert" volume and its manager container volumeMount. +#patchesJson6902: +#- target: +# group: apps +# version: v1 +# kind: Deployment +# name: controller-manager +# namespace: system +# patch: |- +# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/containers/1/volumeMounts/0 +# # Remove the "cert" volume, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing volumes in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/volumes/0 diff --git a/peer-pod-controller/config/prometheus/kustomization.yaml b/peer-pod-controller/config/prometheus/kustomization.yaml new file mode 100644 index 000000000..ed137168a --- /dev/null +++ b/peer-pod-controller/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/peer-pod-controller/config/prometheus/monitor.yaml b/peer-pod-controller/config/prometheus/monitor.yaml new file mode 100644 index 000000000..d19136ae7 --- /dev/null +++ b/peer-pod-controller/config/prometheus/monitor.yaml @@ -0,0 +1,20 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager diff --git a/peer-pod-controller/config/rbac/auth_proxy_client_clusterrole.yaml b/peer-pod-controller/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 000000000..51a75db47 --- /dev/null +++ b/peer-pod-controller/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/peer-pod-controller/config/rbac/auth_proxy_role.yaml b/peer-pod-controller/config/rbac/auth_proxy_role.yaml new file mode 100644 index 000000000..80e1857c5 --- /dev/null +++ b/peer-pod-controller/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/peer-pod-controller/config/rbac/auth_proxy_role_binding.yaml b/peer-pod-controller/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 000000000..ec7acc0a1 --- /dev/null +++ b/peer-pod-controller/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/peer-pod-controller/config/rbac/auth_proxy_service.yaml b/peer-pod-controller/config/rbac/auth_proxy_service.yaml new file mode 100644 index 000000000..71f179727 --- /dev/null +++ b/peer-pod-controller/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager diff --git a/peer-pod-controller/config/rbac/kustomization.yaml b/peer-pod-controller/config/rbac/kustomization.yaml new file mode 100644 index 000000000..731832a6a --- /dev/null +++ b/peer-pod-controller/config/rbac/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/peer-pod-controller/config/rbac/leader_election_role.yaml b/peer-pod-controller/config/rbac/leader_election_role.yaml new file mode 100644 index 000000000..4190ec805 --- /dev/null +++ b/peer-pod-controller/config/rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/peer-pod-controller/config/rbac/leader_election_role_binding.yaml b/peer-pod-controller/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 000000000..1d1321ed4 --- /dev/null +++ b/peer-pod-controller/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/peer-pod-controller/config/rbac/peerpodconfig_editor_role.yaml b/peer-pod-controller/config/rbac/peerpodconfig_editor_role.yaml new file mode 100644 index 000000000..7d2754cb0 --- /dev/null +++ b/peer-pod-controller/config/rbac/peerpodconfig_editor_role.yaml @@ -0,0 +1,30 @@ +# permissions for end users to edit peerpodconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: peerpodconfig-editor-role +rules: +- apiGroups: + - confidentialcontainers.org + resources: + - peerpodconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - confidentialcontainers.org + resources: + - peerpodconfigs/status + verbs: + - get +- apiGroups: + - '' + resources: + - 'nodes/status' + verbs: + - 'patch' diff --git a/peer-pod-controller/config/rbac/peerpodconfig_viewer_role.yaml b/peer-pod-controller/config/rbac/peerpodconfig_viewer_role.yaml new file mode 100644 index 000000000..61de5701a --- /dev/null +++ b/peer-pod-controller/config/rbac/peerpodconfig_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view peerpodconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: peerpodconfig-viewer-role +rules: +- apiGroups: + - confidentialcontainers.org + resources: + - peerpodconfigs + verbs: + - get + - list + - watch +- apiGroups: + - confidentialcontainers.org + resources: + - peerpodconfigs/status + verbs: + - get diff --git a/peer-pod-controller/config/rbac/role.yaml b/peer-pod-controller/config/rbac/role.yaml new file mode 100644 index 000000000..071d0709b --- /dev/null +++ b/peer-pod-controller/config/rbac/role.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - get + - list + - update + - watch +- apiGroups: + - confidentialcontainers.org + resources: + - peerpodconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - confidentialcontainers.org + resources: + - peerpodconfigs/finalizers + verbs: + - update +- apiGroups: + - confidentialcontainers.org + resources: + - peerpodconfigs/status + verbs: + - get + - patch + - update diff --git a/peer-pod-controller/config/rbac/role_binding.yaml b/peer-pod-controller/config/rbac/role_binding.yaml new file mode 100644 index 000000000..2070ede44 --- /dev/null +++ b/peer-pod-controller/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/peer-pod-controller/config/rbac/service_account.yaml b/peer-pod-controller/config/rbac/service_account.yaml new file mode 100644 index 000000000..7cd6025bf --- /dev/null +++ b/peer-pod-controller/config/rbac/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller-manager + namespace: system diff --git a/peer-pod-controller/config/samples/_v1alpha1_peerpodconfig.yaml b/peer-pod-controller/config/samples/_v1alpha1_peerpodconfig.yaml new file mode 100644 index 000000000..ef1635f42 --- /dev/null +++ b/peer-pod-controller/config/samples/_v1alpha1_peerpodconfig.yaml @@ -0,0 +1,10 @@ +apiVersion: confidentialcontainers.org/v1alpha1 +kind: PeerPodConfig +metadata: + name: peerpodconfig-sample +spec: + cloudSecretName: "peer-pods-secret" + limit: "20" + nodeSelector: + matchLabels: + custom-kata1: test diff --git a/peer-pod-controller/config/samples/kustomization.yaml b/peer-pod-controller/config/samples/kustomization.yaml new file mode 100644 index 000000000..f75383b1a --- /dev/null +++ b/peer-pod-controller/config/samples/kustomization.yaml @@ -0,0 +1,4 @@ +## Append samples you want in your CSV to this file as resources ## +resources: +- _v1alpha1_peerpodconfig.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/peer-pod-controller/config/scorecard/bases/config.yaml b/peer-pod-controller/config/scorecard/bases/config.yaml new file mode 100644 index 000000000..c77047841 --- /dev/null +++ b/peer-pod-controller/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/peer-pod-controller/config/scorecard/kustomization.yaml b/peer-pod-controller/config/scorecard/kustomization.yaml new file mode 100644 index 000000000..50cd2d084 --- /dev/null +++ b/peer-pod-controller/config/scorecard/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- bases/config.yaml +patchesJson6902: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +#+kubebuilder:scaffold:patchesJson6902 diff --git a/peer-pod-controller/config/scorecard/patches/basic.config.yaml b/peer-pod-controller/config/scorecard/patches/basic.config.yaml new file mode 100644 index 000000000..64c713b04 --- /dev/null +++ b/peer-pod-controller/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.20.1 + labels: + suite: basic + test: basic-check-spec-test diff --git a/peer-pod-controller/config/scorecard/patches/olm.config.yaml b/peer-pod-controller/config/scorecard/patches/olm.config.yaml new file mode 100644 index 000000000..4d9b6de08 --- /dev/null +++ b/peer-pod-controller/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.20.1 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.20.1 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.20.1 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.20.1 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.20.1 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/peer-pod-controller/controllers/peerpodconfig_controller.go b/peer-pod-controller/controllers/peerpodconfig_controller.go new file mode 100644 index 000000000..6bca5dce1 --- /dev/null +++ b/peer-pod-controller/controllers/peerpodconfig_controller.go @@ -0,0 +1,373 @@ +/* +Copyright Confidential Containers Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "encoding/json" + "errors" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + k8sclient "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "os" + "path" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "strconv" + "strings" + + ccv1alpha1 "github.com/confidential-containers/cloud-api-adaptor/peer-pod-controller/api/v1alpha1" +) + +// PeerPodConfigReconciler reconciles a PeerPodConfig object +type PeerPodConfigReconciler struct { + client.Client + Scheme *runtime.Scheme + Log logr.Logger + peerPodConfig *ccv1alpha1.PeerPodConfig +} + +var ppCloudProviderName string +var validCloudProviderNames = []string{"aws", "libvirt", "ibmcloud", "vsphere", "azure"} + +//Adding sideEffects=none as a workaround for https://github.com/kubernetes-sigs/kubebuilder/issues/1917 +//+kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpodconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpodconfigs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpodconfigs/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=nodes/status,verbs=patch +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=create;get;update;list;watch +//+kubebuilder:rbac:groups="",resources=secrets,verbs=create;get;update;list;watch +//+kubebuilder:rbac:groups="";machineconfiguration.openshift.io,resources=nodes;machineconfigs;machineconfigpools;containerruntimeconfigs;pods;services;services/finalizers;endpoints;persistentvolumeclaims;events;configmaps;secrets,verbs=get;list;watch;create;update;patch;delete + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the PeerPodConfig object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.11.0/pkg/reconcile +func (r *PeerPodConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.Log = log.FromContext(ctx) + _ = r.Log.WithValues("peerpod-controller", req.NamespacedName) + r.Log.Info("Reconciling PeerPodConfig in Kubernetes Cluster") + + // Fetch the CcRuntime instance + r.peerPodConfig = &ccv1alpha1.PeerPodConfig{} + err := r.Client.Get(context.TODO(), req.NamespacedName, r.peerPodConfig) + if err != nil { + if k8serrors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + if err := r.peerpodCloudProviderIsValid(); err != nil { + return ctrl.Result{}, err + } + + if err := r.advertiseExtendedResources(); err != nil { + return ctrl.Result{}, err + } + + // Create the cloud-api-adapter DaemonSet + ds := r.createCcaDaemonset(ppCloudProviderName) + if err := controllerutil.SetControllerReference(r.peerPodConfig, ds, r.Scheme); err != nil { + r.Log.Error(err, "Failed setting ControllerReference for cloud-api-adaptor DS") + return ctrl.Result{}, err + } + foundDs := &appsv1.DaemonSet{} + err = r.Client.Get(context.TODO(), types.NamespacedName{Name: ds.Name, Namespace: ds.Namespace}, foundDs) + if err != nil && k8serrors.IsNotFound(err) { + r.Log.Info("Creating cloud-api-adapter daemonset", "ds.Namespace", ds.Namespace, "ds.Name", ds.Name) + err = r.Client.Create(context.TODO(), ds) + if err != nil { + r.Log.Error(err, "failed to create cloud-api-adaptor") + return ctrl.Result{}, err + } + } + + r.Log.Info("Reconciling PeerPodConfig") + + return ctrl.Result{}, nil +} + +// Check if "peer-pods-secret" exists and has valid cloud provider name set in CLOUD_PROVIDER +func (r *PeerPodConfigReconciler) peerpodCloudProviderIsValid() error { + peerpodscm := corev1.ConfigMap{} + err := r.Client.Get(context.TODO(), types.NamespacedName{Name: "peer-pods-cm", Namespace: os.Getenv("PEERPODS_NAMESPACE")}, &peerpodscm) + if err != nil && k8serrors.IsNotFound(err) { + return err + } else if err != nil { + // Error reading the object - requeue the request. + return err + } else { + ppCloudProviderName = peerpodscm.Data["CLOUD_PROVIDER"] + if ppCloudProviderName == "" || !contains(validCloudProviderNames, ppCloudProviderName) { + r.Log.Info("CLOUD_PROVIDER=", "ppCloudProviderName", ppCloudProviderName) + return errors.New("set CLOUD_PROVIDER name to one supported in cloud-api-provider, found CLOUD_PROVIDER=" + ppCloudProviderName) + } + } + return nil +} + +func MountProgagationRef(mode corev1.MountPropagationMode) *corev1.MountPropagationMode { + return &mode +} + +/* + cloudProviderName needs to be verified against validCloudProviders by caller +*/ +func (r *PeerPodConfigReconciler) createCcaDaemonset(cloudProviderName string) *appsv1.DaemonSet { + var ( + runPrivileged = true + runAsUser int64 = 0 + defaultMode int32 = 0600 + sshSecretOptional = true + ) + + dsName := "peer-pod-controller-cca-daemon" + dsLabelSelectors := map[string]string{ + "name": dsName, + } + nodeSelector := map[string]string{ + "node-role.kubernetes.io/worker": "", + } + return &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: dsName, + Namespace: os.Getenv("PEERPODS_NAMESPACE"), + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: dsLabelSelectors, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: "RollingUpdate", + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: dsLabelSelectors, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "default", + NodeSelector: nodeSelector, + HostNetwork: true, + Containers: []corev1.Container{ + { + Name: "cc-runtime-install-pod", + // TODO make configurable via env var + Image: "quay.io/confidential-containers/cloud-api-adaptor-" + cloudProviderName + ":latest", + ImagePullPolicy: "Always", + SecurityContext: &corev1.SecurityContext{ + // TODO - do we really need to run as root? + Privileged: &runPrivileged, + RunAsUser: &runAsUser, + }, + Command: []string{"/usr/local/bin/entrypoint.sh"}, + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: r.peerPodConfig.Spec.CloudSecretName, + }, + }, + }, + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: r.peerPodConfig.Spec.ConfigMapName, + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "ssh", + MountPath: "/root/.ssh", + ReadOnly: true, + }, + { + MountPath: "/run/peerpod", + Name: "pods-dir", + }, + { + MountPath: "/run/netns", + MountPropagation: MountProgagationRef(corev1.MountPropagationHostToContainer), + Name: "netns", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "ssh", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "ssh-key-secret", + DefaultMode: &defaultMode, + Optional: &sshSecretOptional, + }, + }, + }, + { + Name: "pods-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/run/peerpod", + }, + }, + }, + { + Name: "netns", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/run/netns", + }, + }, + }, + }, + }, + }, + }, + } +} + +// SetupWithManager sets up the controller with the Manager. +func (r *PeerPodConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&ccv1alpha1.PeerPodConfig{}). + Complete(r) +} + +func (r *PeerPodConfigReconciler) getNodesWithLabels(nodeLabels map[string]string) (error, *corev1.NodeList) { + nodes := &corev1.NodeList{} + labelSelector := labels.SelectorFromSet(nodeLabels) + listOpts := []client.ListOption{ + client.MatchingLabelsSelector{Selector: labelSelector}, + } + + if err := r.Client.List(context.TODO(), nodes, listOpts...); err != nil { + r.Log.Error(err, "Getting list of nodes having specified labels failed") + return err, &corev1.NodeList{} + } + return nil, nodes +} + +func (r *PeerPodConfigReconciler) advertiseExtendedResources() error { + + r.Log.Info("set up extended resources") + err, nodesList := r.getNodesWithLabels(map[string]string{"node-role.kubernetes.io/worker": ""}) + if err != nil { + r.Log.Info("getting node list failed when trying to update nodes with extended resources") + return err + } + + // FIXME distribute remainder among nodes + var limitInt int64 + limitInt, err = strconv.ParseInt(r.peerPodConfig.Spec.Limit, 0, 64) + if err != nil { + r.Log.Error(err, "spec.Limit in PeerPodConfig must be an integer") + } + + limitPerNode := limitInt / int64(len(nodesList.Items)) + + for _, node := range nodesList.Items { + patches := append([]JsonPatch{}, NewJsonPatch("add", "/status/capacity", "kata.peerpods.io~1vm", + strconv.Itoa(int(limitPerNode)))) + cli, err := r.GetClient() + if err != nil { + r.Log.Error(err, "failed to get k8s client") + } + err = r.PatchNodeStatus(cli, node.Name, patches) + if err != nil { + r.Log.Error(err, "Failed to set extended resource for node", "node name", node.Name) + } + } + return nil +} + +func (r *PeerPodConfigReconciler) PatchNodeStatus(c *k8sclient.Clientset, nodeName string, patches []JsonPatch) error { + if len(patches) > 0 { + data, err := json.Marshal(patches) + if err == nil { + _, err = c.CoreV1().Nodes().Patch(context.TODO(), nodeName, types.JSONPatchType, data, metav1.PatchOptions{}, "status") + } + return err + } + r.Log.Info("empty patch for node, no change") + return nil +} + +// JsonPatch is a json marshaling helper used for patching API objects +type JsonPatch struct { + Op string `json:"op"` + Path string `json:"path"` + Value string `json:"value,omitempty"` +} + +// NewJsonPatch returns a new JsonPatch object +func NewJsonPatch(verb string, jsonpath string, key string, value string) JsonPatch { + return JsonPatch{verb, path.Join(jsonpath, strings.ReplaceAll(key, "/", "~1")), value} +} + +// GetClient creates and returns a new clientset from given config +func (r PeerPodConfigReconciler) GetClient() (*k8sclient.Clientset, error) { + Kubeconfig, err := restclient.InClusterConfig() + if err != nil { + return nil, err + } + clientset, err := k8sclient.NewForConfig(Kubeconfig) + if err != nil { + return nil, err + } + return clientset, nil +} + +func contains(s []string, str string) bool { + for _, v := range s { + if v == str { + return true + } + } + return false +} diff --git a/peer-pod-controller/controllers/suite_test.go b/peer-pod-controller/controllers/suite_test.go new file mode 100644 index 000000000..4ea34c03c --- /dev/null +++ b/peer-pod-controller/controllers/suite_test.go @@ -0,0 +1,77 @@ +/* +Copyright Confidential Containers Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + confidentialcontainersorgv1alpha1 "github.com/confidential-containers/cloud-api-adaptor/peer-pod-controller/api/v1alpha1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = confidentialcontainersorgv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/peer-pod-controller/go.mod b/peer-pod-controller/go.mod new file mode 100644 index 000000000..cf3150096 --- /dev/null +++ b/peer-pod-controller/go.mod @@ -0,0 +1,70 @@ +module github.com/confidential-containers/cloud-api-adaptor/peer-pod-controller + +go 1.18 + +require ( + github.com/go-logr/logr v1.2.3 + github.com/onsi/ginkgo/v2 v2.6.1 + github.com/onsi/gomega v1.24.2 + k8s.io/api v0.26.0 + k8s.io/apimachinery v0.26.0 + k8s.io/client-go v0.26.0 + sigs.k8s.io/controller-runtime v0.14.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/zapr v1.2.3 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/net v0.4.0 // indirect + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect + golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.26.0 // indirect + k8s.io/component-base v0.26.0 // indirect + k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect + k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/peer-pod-controller/go.sum b/peer-pod-controller/go.sum new file mode 100644 index 000000000..d2c60c75d --- /dev/null +++ b/peer-pod-controller/go.sum @@ -0,0 +1,615 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= +github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE= +github.com/onsi/gomega v1.24.2/go.mod h1:gs3J10IS7Z7r7eXRoNJIrNqU4ToQukCJhFtKrWgHWnk= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= +k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= +k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00nEdpDo= +k8s.io/apiextensions-apiserver v0.26.0/go.mod h1:7ez0LTiyW5nq3vADtK6C3kMESxadD51Bh6uz3JOlqWQ= +k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= +k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= +k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= +k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= +k8s.io/component-base v0.26.0/go.mod h1:lqHwlfV1/haa14F/Z5Zizk5QmzaVf23nQzCwVOQpfC8= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/controller-runtime v0.14.0 h1:ju2xsov5Ara6FoQuddg+az+rAxsUsTYn2IYyEKCTyDc= +sigs.k8s.io/controller-runtime v0.14.0/go.mod h1:GaRkrY8a7UZF0kqFFbUKG7n9ICiTY5T55P1RiE3UZlU= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/peer-pod-controller/hack/boilerplate.go.txt b/peer-pod-controller/hack/boilerplate.go.txt new file mode 100644 index 000000000..4155c5b5e --- /dev/null +++ b/peer-pod-controller/hack/boilerplate.go.txt @@ -0,0 +1,16 @@ +/* +Copyright Confidential Containers Contributors + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/peer-pod-controller/main.go b/peer-pod-controller/main.go new file mode 100644 index 000000000..8439abfee --- /dev/null +++ b/peer-pod-controller/main.go @@ -0,0 +1,103 @@ +/* +Copyright Confidential Containers Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + confidentialcontainersorgv1alpha1 "github.com/confidential-containers/cloud-api-adaptor/peer-pod-controller/api/v1alpha1" + "github.com/confidential-containers/cloud-api-adaptor/peer-pod-controller/controllers" + //+kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(confidentialcontainersorgv1alpha1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + Port: 9443, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "02cb2ce6.confidentialcontainers.org", + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + if err = (&controllers.PeerPodConfigReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "PeerPodConfig") + os.Exit(1) + } + //+kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/pkg/adaptor/hypervisor/aws/ec2.go b/pkg/adaptor/hypervisor/aws/ec2.go new file mode 100644 index 000000000..08ec0de03 --- /dev/null +++ b/pkg/adaptor/hypervisor/aws/ec2.go @@ -0,0 +1,78 @@ +//go:build aws + +package aws + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/ec2" +) + +type EC2CreateInstanceAPI interface { + //Create instances + //https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html + RunInstances(ctx context.Context, + params *ec2.RunInstancesInput, + optFns ...func(*ec2.Options)) (*ec2.RunInstancesOutput, error) + + CreateTags(ctx context.Context, + params *ec2.CreateTagsInput, + optFns ...func(*ec2.Options)) (*ec2.CreateTagsOutput, error) +} + +type EC2DescribeInstancesAPI interface { + DescribeInstances(ctx context.Context, + params *ec2.DescribeInstancesInput, + optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) +} + +type EC2TerminateInstancesAPI interface { + TerminateInstances(ctx context.Context, + params *ec2.TerminateInstancesInput, + optFns ...func(*ec2.Options)) (*ec2.TerminateInstancesOutput, error) +} + +func GetInstance(c context.Context, api EC2DescribeInstancesAPI, input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) { + return api.DescribeInstances(c, input) +} + +func CreateInstance(c context.Context, api EC2CreateInstanceAPI, input *ec2.RunInstancesInput) (*ec2.RunInstancesOutput, error) { + return api.RunInstances(c, input) +} + +func DeleteInstance(c context.Context, api EC2TerminateInstancesAPI, input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) { + return api.TerminateInstances(c, input) +} + +func MakeTags(c context.Context, api EC2CreateInstanceAPI, input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { + return api.CreateTags(c, input) +} + +//TODO: Use IAM role +func NewEC2Client(cloudCfg Config) (*ec2.Client, error) { + + var cfg aws.Config + var err error + + if cloudCfg.AccessKeyId != "" && cloudCfg.SecretKey != "" { + cfg, err = config.LoadDefaultConfig(context.TODO(), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cloudCfg.AccessKeyId, cloudCfg.SecretKey, "")), config.WithRegion(cloudCfg.Region)) + if err != nil { + return nil, fmt.Errorf("configuration error when using creds: %s", err) + } + + } else { + + cfg, err = config.LoadDefaultConfig(context.TODO(), + config.WithRegion(cloudCfg.Region), + config.WithSharedConfigProfile(cloudCfg.LoginProfile)) + if err != nil { + return nil, fmt.Errorf("configuration error when using shared profile: %s", err) + } + } + client := ec2.NewFromConfig(cfg) + return client, nil +} diff --git a/pkg/adaptor/hypervisor/aws/imds.go b/pkg/adaptor/hypervisor/aws/imds.go new file mode 100644 index 000000000..a6e378c17 --- /dev/null +++ b/pkg/adaptor/hypervisor/aws/imds.go @@ -0,0 +1,93 @@ +//go:build aws + +package aws + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + "io" + "strings" +) + +type MetadataRetriever struct { + client *imds.Client + mac string // save it as it commonly used in path +} + +// no error return, if something is wrong, will fail on get() +func newMetadataRetriever() *MetadataRetriever { + var r MetadataRetriever + r.client = imds.New(imds.Options{ClientEnableState: imds.ClientDefaultEnableState}) // use imds.ClientEnabled to enforce enabling + mac, err := r.get("mac") + if err != nil { + logger.Printf("NewMetadataRetriever is initialized without mac (%v)", err) + return &r + } + r.mac = mac + return &r +} + +func (r MetadataRetriever) get(path string) (string, error) { + output, err := r.client.GetMetadata(context.TODO(), &imds.GetMetadataInput{ + Path: path, + }) + if err != nil { + return "", err + } + defer output.Content.Close() + bytes, err := io.ReadAll(output.Content) + if err != nil { + return "", err + } + return string(bytes), err +} + +func retrieveMissingConfig(cfg *Config) error { + mdr := newMetadataRetriever() + if cfg.SubnetId == "" { + logger.Printf("SubnetId was not provided, trying to fetch it from IMDS") + subnetIdPath := fmt.Sprintf("network/interfaces/macs/%s/subnet-id", mdr.mac) + subnetId, err := mdr.get(subnetIdPath) + if err != nil { + return err + } + cfg.SubnetId = subnetId + logger.Printf("\"%s\" SubnetId retrieved from IMDS", subnetId) + } + if cfg.Region == "" { + logger.Printf("Region was not provided, trying to fetch it from IMDS") + region, err := mdr.get("placement/region") + if err != nil { + return err + } + cfg.Region = region + logger.Printf("\"%s\" Region retrieved from IMDS", region) + } + if cfg.KeyName == "" { + logger.Printf("KeyName was not provided, trying to fetch it from IMDS") + rawKey, err := mdr.get("public-keys") + if err != nil { + logger.Printf("failed to retrieve key, skipped: %v", err) + } + var keyName string + n, err := fmt.Sscanf(rawKey, "0=%s", &keyName) + if err != nil || n < 1 { + logger.Printf("failed to retrieve key, skipped") + } else { + cfg.KeyName = keyName + logger.Printf("\"%s\" KeyName retrieved from IMDS", keyName) + } + } + if len(cfg.SecurityGroupIds) < 1 { + logger.Printf("SecurityGroupIds was not provided, trying to fetch it from IMDS") + securityGroupIdsPath := fmt.Sprintf("network/interfaces/macs/%s/security-group-ids", mdr.mac) + securityGroupIds, err := mdr.get(securityGroupIdsPath) + if err != nil { + return err + } + cfg.SecurityGroupIds = strings.Fields(securityGroupIds) + logger.Printf("\"%s\" SecurityGroupIds retrieved from IMDS", &cfg.SecurityGroupIds) + } + return nil +} diff --git a/pkg/adaptor/hypervisor/aws/server.go b/pkg/adaptor/hypervisor/aws/server.go new file mode 100644 index 000000000..911a39127 --- /dev/null +++ b/pkg/adaptor/hypervisor/aws/server.go @@ -0,0 +1,109 @@ +//go:build aws + +package aws + +import ( + "context" + "log" + "net" + "os" + "path/filepath" + "sync" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/containerd/ttrpc" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" +) + +var logger = log.New(log.Writer(), "[helper/hypervisor] ", log.LstdFlags|log.Lmsgprefix) + +type server struct { + socketPath string + + ttRpc *ttrpc.Server + service pb.HypervisorService + + workerNode podnetwork.WorkerNode + + readyCh chan struct{} + stopCh chan struct{} + stopOnce sync.Once +} + +func NewServer(cfg hypervisor.Config, cloudCfg Config, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + + logger.Printf("hypervisor config %v", cfg) + if err := retrieveMissingConfig(&cloudCfg); err != nil { + logger.Printf("Failed to retrieve configuration, some fields may still be missing: %v", err) + } + logger.Printf("cloud config %v", cloudCfg.Redact()) + ec2Client, err := NewEC2Client(cloudCfg) + if err != nil { + return nil + } + + return &server{ + socketPath: cfg.SocketPath, + service: newService(ec2Client, &cloudCfg, &cfg, workerNode, cfg.PodsDir, daemonPort), + workerNode: workerNode, + readyCh: make(chan struct{}), + stopCh: make(chan struct{}), + } +} + +func (s *server) Start(ctx context.Context) (err error) { + + ttRpc, err := ttrpc.NewServer() + if err != nil { + return err + } + s.ttRpc = ttRpc + if err = os.MkdirAll(filepath.Dir(s.socketPath), os.ModePerm); err != nil { + return err + } + if err := os.RemoveAll(s.socketPath); err != nil { // just in case socket wasn't cleaned + return err + } + pb.RegisterHypervisorService(s.ttRpc, s.service) + listener, err := net.Listen("unix", s.socketPath) + if err != nil { + return err + } + + ttRpcErr := make(chan error) + go func() { + defer close(ttRpcErr) + if err = s.ttRpc.Serve(ctx, listener); err != nil { + ttRpcErr <- err + } + }() + defer func() { + newErr := s.ttRpc.Shutdown(context.Background()) + if newErr != nil && err == nil { + err = newErr + } + }() + + close(s.readyCh) + + select { + case <-ctx.Done(): + err = s.Shutdown() + case <-s.stopCh: + case err = <-ttRpcErr: + } + return err +} + +func (s *server) Shutdown() error { + s.stopOnce.Do(func() { + close(s.stopCh) + }) + return nil +} + +func (s *server) Ready() chan struct{} { + return s.readyCh +} diff --git a/pkg/adaptor/hypervisor/aws/service.go b/pkg/adaptor/hypervisor/aws/service.go new file mode 100644 index 000000000..1420843cc --- /dev/null +++ b/pkg/adaptor/hypervisor/aws/service.go @@ -0,0 +1,377 @@ +//go:build aws + +package aws + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/proxy" + daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/cloudinit" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/hvutil" + "github.com/containerd/containerd/pkg/cri/annotations" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" +) + +const ( + Version = "0.0.0" +) + +type hypervisorService struct { + ec2Client *ec2.Client + serviceConfig *Config + hypervisorConfig *hypervisor.Config + sandboxes map[sandboxID]*sandbox + podsDir string + daemonPort string + nodeName string + workerNode podnetwork.WorkerNode + sync.Mutex +} + +func newService(ec2Client *ec2.Client, config *Config, hypervisorConfig *hypervisor.Config, workerNode podnetwork.WorkerNode, podsDir, daemonPort string) pb.HypervisorService { + logger.Printf("service config %v", config.Redact()) + + hostname, err := os.Hostname() + if err != nil { + panic(fmt.Errorf("failed to get hostname: %w", err)) + } + + i := strings.Index(hostname, ".") + if i >= 0 { + hostname = hostname[0:i] + } + + return &hypervisorService{ + ec2Client: ec2Client, + serviceConfig: config, + hypervisorConfig: hypervisorConfig, + sandboxes: map[sandboxID]*sandbox{}, + podsDir: podsDir, + daemonPort: daemonPort, + nodeName: hostname, + workerNode: workerNode, + } +} + +type sandboxID string + +type sandbox struct { + id sandboxID + pod string + namespace string + netNSPath string + podDirPath string + vsi string + agentProxy proxy.AgentProxy + podNetworkConfig *tunneler.Config +} + +func (s *hypervisorService) Version(ctx context.Context, req *pb.VersionRequest) (*pb.VersionResponse, error) { + return &pb.VersionResponse{Version: Version}, nil +} + +func (s *hypervisorService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (*pb.CreateVMResponse, error) { + + sid := sandboxID(req.Id) + + if sid == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; exists { + return nil, fmt.Errorf("sandbox %s already exists", sid) + } + pod := hvutil.GetPodName(req.Annotations) + if pod == "" { + return nil, fmt.Errorf("pod name %s is missing in annotations", annotations.SandboxName) + } + + namespace := hvutil.GetPodNamespace(req.Annotations) + if namespace == "" { + return nil, fmt.Errorf("namespace name %s is missing in annotations", annotations.SandboxNamespace) + } + + podDirPath := filepath.Join(s.podsDir, string(sid)) + if err := os.MkdirAll(podDirPath, os.ModePerm); err != nil { + return nil, fmt.Errorf("failed to create a pod directory: %s: %w", podDirPath, err) + } + + socketPath := filepath.Join(podDirPath, proxy.SocketName) + + netNSPath := req.NetworkNamespacePath + + podNetworkConfig, err := s.workerNode.Inspect(netNSPath) + if err != nil { + return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err) + } + + agentProxy := proxy.NewAgentProxy(socketPath, s.hypervisorConfig.CriSocketPath, s.hypervisorConfig.PauseImage) + + sandbox := &sandbox{ + id: sid, + pod: pod, + namespace: namespace, + netNSPath: netNSPath, + podDirPath: podDirPath, + agentProxy: agentProxy, + podNetworkConfig: podNetworkConfig, + } + s.sandboxes[sid] = sandbox + logger.Printf("create a sandbox %s for pod %s in namespace %s (netns: %s)", req.Id, pod, namespace, sandbox.netNSPath) + return &pb.CreateVMResponse{AgentSocketPath: socketPath}, nil +} + +func (s *hypervisorService) StartVM(ctx context.Context, req *pb.StartVMRequest) (*pb.StartVMResponse, error) { + + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + daemonConfig := daemon.Config{ + PodNamespace: sandbox.namespace, + PodName: sandbox.pod, + PodNetwork: sandbox.podNetworkConfig, + } + daemonJSON, err := json.MarshalIndent(daemonConfig, "", " ") + if err != nil { + return nil, err + } + + // Store daemon.json in worker node for debugging + if err = os.WriteFile(filepath.Join(sandbox.podDirPath, "daemon.json"), daemonJSON, 0666); err != nil { + return nil, fmt.Errorf("failed to store daemon.json at %s: %w", sandbox.podDirPath, err) + } + logger.Printf("store daemon.json at %s", sandbox.podDirPath) + + cloudConfig := &cloudinit.CloudConfig{ + WriteFiles: []cloudinit.WriteFile{ + { + Path: daemon.DefaultConfigPath, + Content: string(daemonJSON), + }, + }, + } + + if authJSON, err := os.ReadFile(cloudinit.DefaultAuthfileSrcPath); err == nil { + if json.Valid(authJSON) && (len(authJSON) < cloudinit.DefaultAuthfileLimit) { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, + cloudinit.WriteFile{ + Path: cloudinit.DefaultAuthfileDstPath, + Content: cloudinit.AuthJSONToResourcesJSON(string(authJSON)), + }) + } else if len(authJSON) >= cloudinit.DefaultAuthfileLimit { + logger.Printf("Credentials file size (%d) is too large to use as userdata, ignored", len(authJSON)) + } else { + logger.Printf("Credentials file is not in a valid Json format, ignored") + } + } + + userData, err := cloudConfig.Generate() + if err != nil { + return nil, err + } + + //Convert userData to base64 + userDataEnc := base64.StdEncoding.EncodeToString([]byte(userData)) + + var input *ec2.RunInstancesInput + + if s.serviceConfig.UseLaunchTemplate { + input = &ec2.RunInstancesInput{ + MinCount: aws.Int32(1), + MaxCount: aws.Int32(1), + LaunchTemplate: &types.LaunchTemplateSpecification{ + LaunchTemplateName: aws.String(s.serviceConfig.LaunchTemplateName), + }, + UserData: &userDataEnc, + } + } else { + input = &ec2.RunInstancesInput{ + MinCount: aws.Int32(1), + MaxCount: aws.Int32(1), + ImageId: aws.String(s.serviceConfig.ImageId), + InstanceType: types.InstanceType(s.serviceConfig.InstanceType), + SecurityGroupIds: s.serviceConfig.SecurityGroupIds, + KeyName: aws.String(s.serviceConfig.KeyName), + SubnetId: aws.String(s.serviceConfig.SubnetId), + UserData: &userDataEnc, + } + } + + result, err := CreateInstance(ctx, s.ec2Client, input) + if err != nil { + return nil, fmt.Errorf("Creating instance (%v) returned error: %s", result, err) + } + + sandbox.vsi = *result.Instances[0].InstanceId + + logger.Printf("created an instance %s for sandbox %s", *result.Instances[0].PublicDnsName, req.Id) + + // TODO: Specify the maximum instance name length in AWS + vmName := hvutil.CreateInstanceName(sandbox.pod, string(sandbox.id), 0) + tagInput := &ec2.CreateTagsInput{ + Resources: []string{*result.Instances[0].InstanceId}, + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(vmName), + }, + }, + } + + _, err = MakeTags(ctx, s.ec2Client, tagInput) + if err != nil { + logger.Printf("Adding tags to the instance failed with error: %s", err) + } + + podNodeIPs, err := getIPs(result.Instances[0]) + if err != nil { + logger.Printf("failed to get IPs for the instance : %v ", err) + return nil, err + } + + if err := s.workerNode.Setup(sandbox.netNSPath, podNodeIPs, sandbox.podNetworkConfig); err != nil { + return nil, fmt.Errorf("failed to set up pod network tunnel on netns %s: %w", sandbox.netNSPath, err) + } + + serverURL := &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(podNodeIPs[0].String(), s.daemonPort), + Path: daemon.AgentURLPath, + } + + errCh := make(chan error) + go func() { + defer close(errCh) + + if err := sandbox.agentProxy.Start(context.Background(), serverURL); err != nil { + logger.Printf("error running agent proxy: %v", err) + errCh <- err + } + }() + + select { + case <-ctx.Done(): + _ = sandbox.agentProxy.Shutdown() + return nil, ctx.Err() + case err := <-errCh: + return nil, err + case <-sandbox.agentProxy.Ready(): + } + + logger.Printf("agent proxy is ready") + return &pb.StartVMResponse{}, nil +} + +func (s *hypervisorService) getSandbox(id string) (*sandbox, error) { + + sid := sandboxID(id) + + if id == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; !exists { + return nil, fmt.Errorf("sandbox %s does not exist", sid) + } + return s.sandboxes[sid], nil +} + +func (s *hypervisorService) deleteSandbox(id string) error { + sid := sandboxID(id) + if id == "" { + return errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + delete(s.sandboxes, sid) + return nil +} + +var errNotReady = errors.New("address not ready") + +func getIPs(instance types.Instance) ([]net.IP, error) { + + var podNodeIPs []net.IP + for i, nic := range instance.NetworkInterfaces { + addr := nic.PrivateIpAddress + + if addr == nil || *addr == "" || *addr == "0.0.0.0" { + return nil, errNotReady + } + + ip := net.ParseIP(*addr) + if ip == nil { + return nil, fmt.Errorf("failed to parse pod node IP %q", *addr) + } + podNodeIPs = append(podNodeIPs, ip) + + logger.Printf("podNodeIP[%d]=%s", i, ip.String()) + } + + return podNodeIPs, nil +} + +func (s *hypervisorService) deleteInstance(ctx context.Context, id string) error { + + terminateInput := &ec2.TerminateInstancesInput{ + InstanceIds: []string{ + id, + }, + } + + resp, err := DeleteInstance(ctx, s.ec2Client, terminateInput) + + if err != nil { + logger.Printf("failed to delete an instance: %v and the response is %v", err, resp) + return err + } + logger.Printf("deleted an instance %s", id) + return nil +} + +func (s *hypervisorService) StopVM(ctx context.Context, req *pb.StopVMRequest) (*pb.StopVMResponse, error) { + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + if err := sandbox.agentProxy.Shutdown(); err != nil { + logger.Printf("failed to stop agent proxy: %v", err) + } + + if err := s.deleteInstance(ctx, sandbox.vsi); err != nil { + return nil, err + } + + if err := s.workerNode.Teardown(sandbox.netNSPath, sandbox.podNetworkConfig); err != nil { + return nil, fmt.Errorf("failed to tear down netns %s: %w", sandbox.netNSPath, err) + } + err = s.deleteSandbox(req.Id) + if err != nil { + return nil, err + } + return &pb.StopVMResponse{}, nil +} diff --git a/pkg/adaptor/hypervisor/aws/types.go b/pkg/adaptor/hypervisor/aws/types.go new file mode 100644 index 000000000..032f5869e --- /dev/null +++ b/pkg/adaptor/hypervisor/aws/types.go @@ -0,0 +1,35 @@ +package aws + +import ( + "github.com/confidential-containers/cloud-api-adaptor/pkg/util" + "strings" +) + +type securityGroupIds []string + +func (i *securityGroupIds) String() string { + return strings.Join(*i, ", ") +} + +func (i *securityGroupIds) Set(value string) error { + *i = append(*i, strings.Split(value, ",")...) + return nil +} + +type Config struct { + AccessKeyId string + SecretKey string + Region string + LoginProfile string + LaunchTemplateName string + ImageId string + InstanceType string + SecurityGroupIds securityGroupIds + KeyName string + SubnetId string + UseLaunchTemplate bool +} + +func (c Config) Redact() Config { + return *util.RedactStruct(&c, "AccessKeyId", "SecretKey").(*Config) +} diff --git a/pkg/adaptor/hypervisor/aws/types_test.go b/pkg/adaptor/hypervisor/aws/types_test.go new file mode 100644 index 000000000..322f0b65d --- /dev/null +++ b/pkg/adaptor/hypervisor/aws/types_test.go @@ -0,0 +1,33 @@ +//go:build aws + +package aws + +import ( + "fmt" + "strings" + "testing" +) + +func TestAWSMasking(t *testing.T) { + secretKey := "abcdefg" + region := "eu-gb" + cloudCfg := Config{ + SecretKey: secretKey, + Region: region, + } + checkLine := func(verb string) { + logline := fmt.Sprintf(verb, cloudCfg.Redact()) + if strings.Contains(logline, secretKey) { + t.Errorf("For verb %s: %s contains the secret key: %s", verb, logline, secretKey) + } + if !strings.Contains(logline, region) { + t.Errorf("For verb %s: %s doesn't contain the region name: %s", verb, logline, region) + } + } + checkLine("%v") + checkLine("%s") + + if cloudCfg.SecretKey != secretKey { + t.Errorf("Original SecretKey field value has been overwritten") + } +} diff --git a/pkg/adaptor/hypervisor/azure/azure.go b/pkg/adaptor/hypervisor/azure/azure.go new file mode 100644 index 000000000..855554ea6 --- /dev/null +++ b/pkg/adaptor/hypervisor/azure/azure.go @@ -0,0 +1,66 @@ +//go:build azure + +package azure + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + armcompute "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v3" +) + +func CreateInstance(ctx context.Context, s *hypervisorService, parameters *armcompute.VirtualMachine) (*armcompute.VirtualMachine, error) { + vmClient, err := armcompute.NewVirtualMachinesClient(s.serviceConfig.SubscriptionId, s.azureClient, nil) + if err != nil { + return nil, fmt.Errorf("creating VM client: %w", err) + } + + vmName := *parameters.Properties.OSProfile.ComputerName + + pollerResponse, err := vmClient.BeginCreateOrUpdate(ctx, s.serviceConfig.ResourceGroupName, vmName, *parameters, nil) + if err != nil { + return nil, fmt.Errorf("beginning VM creation or update: %w", err) + } + + resp, err := pollerResponse.PollUntilDone(ctx, nil) + if err != nil { + return nil, fmt.Errorf("waiting for the VM creation: %w", err) + } + + logger.Printf("created VM successfully: %s", *resp.ID) + + return &resp.VirtualMachine, nil +} + +func DeleteInstance(ctx context.Context, s *hypervisorService, vmName string) error { + vmClient, err := armcompute.NewVirtualMachinesClient(s.serviceConfig.SubscriptionId, s.azureClient, nil) + if err != nil { + return fmt.Errorf("creating VM client: %w", err) + } + + pollerResponse, err := vmClient.BeginDelete(ctx, s.serviceConfig.ResourceGroupName, vmName, nil) + if err != nil { + return fmt.Errorf("beginning VM deletion: %w", err) + } + + if _, err = pollerResponse.PollUntilDone(ctx, nil); err != nil { + return fmt.Errorf("waiting for the VM deletion: %w", err) + } + + logger.Printf("deleted VM successfully: %s", vmName) + + return nil +} + +func NewAzureClient(cloudCfg Config) (azcore.TokenCredential, error) { + + cred, err := azidentity.NewClientSecretCredential(cloudCfg.TenantId, cloudCfg.ClientId, cloudCfg.ClientSecret, nil) + if err != nil { + return nil, err + } + //azidentity.ClientSecretCredential + return cred, nil + +} diff --git a/pkg/adaptor/hypervisor/azure/server.go b/pkg/adaptor/hypervisor/azure/server.go new file mode 100644 index 000000000..8f350c237 --- /dev/null +++ b/pkg/adaptor/hypervisor/azure/server.go @@ -0,0 +1,108 @@ +//go:build azure + +package azure + +import ( + "context" + "log" + "net" + "os" + "path/filepath" + "sync" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/containerd/ttrpc" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" +) + +var logger = log.New(log.Writer(), "[helper/hypervisor] ", log.LstdFlags|log.Lmsgprefix) + +type server struct { + socketPath string + + ttRpc *ttrpc.Server + service pb.HypervisorService + + workerNode podnetwork.WorkerNode + + readyCh chan struct{} + stopCh chan struct{} + stopOnce sync.Once +} + +func NewServer(cfg hypervisor.Config, cloudCfg Config, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + + logger.Printf("hypervisor config %v", cfg) + logger.Printf("cloud config %+v", cloudCfg.Redact()) + + azureClient, err := NewAzureClient(cloudCfg) + if err != nil { + logger.Printf("creating azure client: %v", err) + return nil + } + + return &server{ + socketPath: cfg.SocketPath, + service: newService(azureClient, &cloudCfg, &cfg, workerNode, cfg.PodsDir, daemonPort), + workerNode: workerNode, + readyCh: make(chan struct{}), + stopCh: make(chan struct{}), + } +} + +func (s *server) Start(ctx context.Context) (err error) { + + ttRpc, err := ttrpc.NewServer() + if err != nil { + return err + } + s.ttRpc = ttRpc + if err = os.MkdirAll(filepath.Dir(s.socketPath), os.ModePerm); err != nil { + return err + } + if err := os.RemoveAll(s.socketPath); err != nil { // just in case socket wasn't cleaned + return err + } + pb.RegisterHypervisorService(s.ttRpc, s.service) + listener, err := net.Listen("unix", s.socketPath) + if err != nil { + return err + } + + ttRpcErr := make(chan error) + go func() { + defer close(ttRpcErr) + if err = s.ttRpc.Serve(ctx, listener); err != nil { + ttRpcErr <- err + } + }() + defer func() { + newErr := s.ttRpc.Shutdown(context.Background()) + if newErr != nil && err == nil { + err = newErr + } + }() + + close(s.readyCh) + + select { + case <-ctx.Done(): + err = s.Shutdown() + case <-s.stopCh: + case err = <-ttRpcErr: + } + return err +} + +func (s *server) Shutdown() error { + s.stopOnce.Do(func() { + close(s.stopCh) + }) + return nil +} + +func (s *server) Ready() chan struct{} { + return s.readyCh +} diff --git a/pkg/adaptor/hypervisor/azure/service.go b/pkg/adaptor/hypervisor/azure/service.go new file mode 100644 index 000000000..8e3b5e0be --- /dev/null +++ b/pkg/adaptor/hypervisor/azure/service.go @@ -0,0 +1,483 @@ +//go:build azure + +package azure + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + armcompute "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v3" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/proxy" + daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/cloudinit" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/hvutil" + "github.com/containerd/containerd/pkg/cri/annotations" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" +) + +const ( + Version = "0.0.0" + DefaultUserName = "ubuntu" +) + +type hypervisorService struct { + azureClient azcore.TokenCredential + serviceConfig *Config + hypervisorConfig *hypervisor.Config + sandboxes map[sandboxID]*sandbox + podsDir string + daemonPort string + nodeName string + workerNode podnetwork.WorkerNode + sync.Mutex +} + +func newService(azureClient azcore.TokenCredential, config *Config, hypervisorConfig *hypervisor.Config, workerNode podnetwork.WorkerNode, podsDir, daemonPort string) pb.HypervisorService { + logger.Printf("service config %+v", config) + + hostname, err := os.Hostname() + if err != nil { + panic(fmt.Errorf("failed to get hostname: %w", err)) + } + + i := strings.Index(hostname, ".") + if i >= 0 { + hostname = hostname[0:i] + } + + return &hypervisorService{ + azureClient: azureClient, + serviceConfig: config, + hypervisorConfig: hypervisorConfig, + sandboxes: map[sandboxID]*sandbox{}, + podsDir: podsDir, + daemonPort: daemonPort, + nodeName: hostname, + workerNode: workerNode, + } +} + +type sandboxID string + +type sandbox struct { + id sandboxID + pod string + namespace string + netNSPath string + podDirPath string + vsi string + vmName string + agentProxy proxy.AgentProxy + podNetworkConfig *tunneler.Config +} + +func (s *hypervisorService) Version(ctx context.Context, req *pb.VersionRequest) (*pb.VersionResponse, error) { + return &pb.VersionResponse{Version: Version}, nil +} + +func (s *hypervisorService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (*pb.CreateVMResponse, error) { + + sid := sandboxID(req.Id) + + if sid == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; exists { + return nil, fmt.Errorf("sandbox %s already exists", sid) + } + pod := hvutil.GetPodName(req.Annotations) + if pod == "" { + return nil, fmt.Errorf("pod name %s is missing in annotations", annotations.SandboxName) + } + + namespace := hvutil.GetPodNamespace(req.Annotations) + if namespace == "" { + return nil, fmt.Errorf("namespace name %s is missing in annotations", annotations.SandboxNamespace) + } + + podDirPath := filepath.Join(s.podsDir, string(sid)) + if err := os.MkdirAll(podDirPath, os.ModePerm); err != nil { + return nil, fmt.Errorf("failed to create a pod directory: %s: %w", podDirPath, err) + } + + socketPath := filepath.Join(podDirPath, proxy.SocketName) + + netNSPath := req.NetworkNamespacePath + + podNetworkConfig, err := s.workerNode.Inspect(netNSPath) + if err != nil { + return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err) + } + + agentProxy := proxy.NewAgentProxy(socketPath, s.hypervisorConfig.CriSocketPath, s.hypervisorConfig.PauseImage) + + sandbox := &sandbox{ + id: sid, + pod: pod, + namespace: namespace, + netNSPath: netNSPath, + podDirPath: podDirPath, + agentProxy: agentProxy, + podNetworkConfig: podNetworkConfig, + } + s.sandboxes[sid] = sandbox + logger.Printf("create a sandbox %s for pod %s in namespace %s (netns: %s)", req.Id, pod, namespace, sandbox.netNSPath) + return &pb.CreateVMResponse{AgentSocketPath: socketPath}, nil +} + +func (s *hypervisorService) StartVM(ctx context.Context, req *pb.StartVMRequest) (*pb.StartVMResponse, error) { + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + daemonConfig := daemon.Config{ + PodNamespace: sandbox.namespace, + PodName: sandbox.pod, + PodNetwork: sandbox.podNetworkConfig, + } + daemonJSON, err := json.MarshalIndent(daemonConfig, "", " ") + if err != nil { + return nil, err + } + + // Store daemon.json in worker node for debugging + if err = os.WriteFile(filepath.Join(sandbox.podDirPath, "daemon.json"), daemonJSON, 0666); err != nil { + return nil, fmt.Errorf("failed to store daemon.json at %s: %w", sandbox.podDirPath, err) + } + logger.Printf("store daemon.json at %s", sandbox.podDirPath) + + cloudConfig := &cloudinit.CloudConfig{ + WriteFiles: []cloudinit.WriteFile{ + { + Path: daemon.DefaultConfigPath, + Content: string(daemonJSON), + }, + }, + } + + if authJSON, err := os.ReadFile(cloudinit.DefaultAuthfileSrcPath); err == nil { + if json.Valid(authJSON) && (len(authJSON) < cloudinit.DefaultAuthfileLimit) { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, + cloudinit.WriteFile{ + Path: cloudinit.DefaultAuthfileDstPath, + Content: cloudinit.AuthJSONToResourcesJSON(string(authJSON)), + }) + } else if len(authJSON) >= cloudinit.DefaultAuthfileLimit { + logger.Printf("Credentials file size (%d) is too large to use as userdata, ignored", len(authJSON)) + } else { + logger.Printf("Credentials file is not in a valid Json format, ignored") + } + } + + userData, err := cloudConfig.Generate() + if err != nil { + return nil, err + } + + //Convert userData to base64 + userDataEnc := base64.StdEncoding.EncodeToString([]byte(userData)) + + // TODO: Specify the maximum instance name length in Azure + vmName := hvutil.CreateInstanceName(sandbox.pod, string(sandbox.id), 0) + diskName := fmt.Sprintf("%s-disk", vmName) + nicName := fmt.Sprintf("%s-net", vmName) + + // Set the vm name to the sandbox early for cleanup purposes + sandbox.vmName = vmName + + // Get NIC using subnet and allow ports on the ssh group + vmNIC, err := s.createNetworkInterface(ctx, nicName) + if err != nil { + err = fmt.Errorf("creating VM network interface: %w", err) + logger.Printf("%v", err) + return nil, err + } + + // require ssh key for authentication on linux + sshPublicKeyPath := os.ExpandEnv(s.serviceConfig.SSHKeyPath) + var sshBytes []byte + if _, err := os.Stat(sshPublicKeyPath); err == nil { + sshBytes, err = ioutil.ReadFile(sshPublicKeyPath) + if err != nil { + err = fmt.Errorf("reading ssh public key file: %w", err) + logger.Printf("%v", err) + return nil, err + } + } else { + err = fmt.Errorf("ssh public key: %w", err) + logger.Printf("%v", err) + return nil, err + } + + vmParameters := armcompute.VirtualMachine{ + Location: to.Ptr(s.serviceConfig.Region), + Properties: &armcompute.VirtualMachineProperties{ + HardwareProfile: &armcompute.HardwareProfile{ + VMSize: to.Ptr(armcompute.VirtualMachineSizeTypes(s.serviceConfig.Size)), + }, + StorageProfile: &armcompute.StorageProfile{ + ImageReference: &armcompute.ImageReference{ + ID: to.Ptr(s.serviceConfig.ImageId), + }, + OSDisk: &armcompute.OSDisk{ + Name: to.Ptr(diskName), + CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage), + Caching: to.Ptr(armcompute.CachingTypesReadWrite), + ManagedDisk: &armcompute.ManagedDiskParameters{ + StorageAccountType: to.Ptr(armcompute.StorageAccountTypesStandardLRS), + }, + }, + }, + OSProfile: &armcompute.OSProfile{ + AdminUsername: to.Ptr(DefaultUserName), + ComputerName: to.Ptr(vmName), + CustomData: to.Ptr(userDataEnc), + LinuxConfiguration: &armcompute.LinuxConfiguration{ + DisablePasswordAuthentication: to.Ptr(true), + //TBD: replace with a suitable mechanism to use precreated SSH key + SSH: &armcompute.SSHConfiguration{ + PublicKeys: []*armcompute.SSHPublicKey{{ + Path: to.Ptr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", DefaultUserName)), + KeyData: to.Ptr(string(sshBytes)), + }}, + }, + }, + }, + NetworkProfile: &armcompute.NetworkProfile{ + NetworkInterfaces: []*armcompute.NetworkInterfaceReference{ + {ID: vmNIC.ID}, + }, + }, + }, + } + + vm, err := CreateInstance(context.TODO(), s, &vmParameters) + if err != nil { + err = fmt.Errorf("Creating instance returned error: %s", err) + logger.Printf("%v", err) + return nil, err + } + + // Set vsi to instance id + sandbox.vsi = *vm.ID + + logger.Printf("created an instance %s for sandbox %s", *vm.Name, req.Id) + + podNodeIPs, err := getIPs(vmNIC) + if err != nil { + err = fmt.Errorf("failed to get IPs for the instance : %w", err) + logger.Printf("%v", err) + return nil, err + } + + if err := s.workerNode.Setup(sandbox.netNSPath, podNodeIPs, sandbox.podNetworkConfig); err != nil { + err = fmt.Errorf("failed to set up pod network tunnel on netns %s: %w", sandbox.netNSPath, err) + logger.Printf("%v", err) + return nil, err + } + + serverURL := &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(podNodeIPs[0].String(), s.daemonPort), + Path: daemon.AgentURLPath, + } + + logger.Printf("server URL running the agent: %v", serverURL) + + errCh := make(chan error) + go func() { + defer close(errCh) + + if err := sandbox.agentProxy.Start(context.Background(), serverURL); err != nil { + logger.Printf("error running agent proxy: %v", err) + errCh <- err + } + }() + + select { + case <-ctx.Done(): + _ = sandbox.agentProxy.Shutdown() + return nil, ctx.Err() + case err := <-errCh: + return nil, err + case <-sandbox.agentProxy.Ready(): + } + + logger.Printf("agent proxy is ready") + return &pb.StartVMResponse{}, nil +} + +func (s *hypervisorService) getSandbox(id string) (*sandbox, error) { + + sid := sandboxID(id) + + if id == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; !exists { + return nil, fmt.Errorf("sandbox %s does not exist", sid) + } + return s.sandboxes[sid], nil +} + +func getIPs(nic *armnetwork.Interface) ([]net.IP, error) { + var podNodeIPs []net.IP + + for _, ipc := range nic.Properties.IPConfigurations { + podNodeIPs = append(podNodeIPs, net.ParseIP(*ipc.Properties.PrivateIPAddress)) + } + + return podNodeIPs, nil +} + +func (s *hypervisorService) deleteInstance(ctx context.Context, vmName string) error { + + if err := DeleteInstance(ctx, s, vmName); err != nil { + err = fmt.Errorf("failed to delete an instance: %w", err) + logger.Printf("%v", err) + return err + } + + logger.Printf("deleted an instance %s", vmName) + + diskName := fmt.Sprintf("%s-disk", vmName) + if err := s.deleteDisk(ctx, diskName); err != nil { + err = fmt.Errorf("failed to delete disk: %w", err) + logger.Print(err) + return err + } + + nicName := fmt.Sprintf("%s-net", vmName) + if err := s.deleteNetworkInterface(ctx, nicName); err != nil { + err = fmt.Errorf("failed to delete network interface: %w", err) + logger.Print(err) + return err + } + + return nil +} + +func (s *hypervisorService) StopVM(ctx context.Context, req *pb.StopVMRequest) (*pb.StopVMResponse, error) { + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + if err := sandbox.agentProxy.Shutdown(); err != nil { + logger.Printf("failed to stop agent proxy: %v", err) + } + + if err := s.deleteInstance(ctx, sandbox.vmName); err != nil { + return nil, err + } + + if err := s.workerNode.Teardown(sandbox.netNSPath, sandbox.podNetworkConfig); err != nil { + return nil, fmt.Errorf("failed to tear down netns %s: %w", sandbox.netNSPath, err) + } + + return &pb.StopVMResponse{}, nil +} + +func (s *hypervisorService) createNetworkInterface(ctx context.Context, nicName string) (*armnetwork.Interface, error) { + nicClient, err := armnetwork.NewInterfacesClient(s.serviceConfig.SubscriptionId, s.azureClient, nil) + if err != nil { + return nil, fmt.Errorf("creating network interfaces client: %w", err) + } + + parameters := armnetwork.Interface{ + Location: to.Ptr(s.serviceConfig.Region), + Properties: &armnetwork.InterfacePropertiesFormat{ + IPConfigurations: []*armnetwork.InterfaceIPConfiguration{ + { + Name: to.Ptr(fmt.Sprintf("%s-ipConfig", nicName)), + Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodDynamic), + Subnet: &armnetwork.Subnet{ + ID: to.Ptr(s.serviceConfig.SubnetId), + }, + }, + }, + }, + NetworkSecurityGroup: &armnetwork.SecurityGroup{ + ID: to.Ptr(s.serviceConfig.SecurityGroupId), + }, + }, + } + + pollerResponse, err := nicClient.BeginCreateOrUpdate(ctx, s.serviceConfig.ResourceGroupName, nicName, parameters, nil) + if err != nil { + return nil, fmt.Errorf("beginning creation or update of network interface: %w", err) + } + + resp, err := pollerResponse.PollUntilDone(ctx, nil) + if err != nil { + return nil, fmt.Errorf("polling network interface creation: %w", err) + } + + return &resp.Interface, nil +} + +func (s *hypervisorService) deleteNetworkInterface(ctx context.Context, nicName string) error { + nicClient, err := armnetwork.NewInterfacesClient(s.serviceConfig.SubscriptionId, s.azureClient, nil) + if err != nil { + return fmt.Errorf("creating network interfaces client: %w", err) + } + + pollerResponse, err := nicClient.BeginDelete(ctx, s.serviceConfig.ResourceGroupName, nicName, nil) + if err != nil { + return fmt.Errorf("beginning deletion of network interface: %w", err) + } + + _, err = pollerResponse.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("polling network interface deletion: %w", err) + } + + logger.Printf("deleted network interface successfully: %s", nicName) + + return nil +} + +func (s *hypervisorService) deleteDisk(ctx context.Context, diskName string) error { + diskClient, err := armcompute.NewDisksClient(s.serviceConfig.SubscriptionId, s.azureClient, nil) + if err != nil { + return fmt.Errorf("creating disk client: %w", err) + } + + pollerResponse, err := diskClient.BeginDelete(ctx, s.serviceConfig.ResourceGroupName, diskName, nil) + if err != nil { + return fmt.Errorf("beginning disk deletion: %w", err) + } + + _, err = pollerResponse.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("waiting for the disk deletion: %w", err) + } + + logger.Printf("deleted disk successfully: %s", diskName) + + return nil +} diff --git a/pkg/adaptor/hypervisor/azure/types.go b/pkg/adaptor/hypervisor/azure/types.go new file mode 100644 index 000000000..8c20fbf1b --- /dev/null +++ b/pkg/adaptor/hypervisor/azure/types.go @@ -0,0 +1,23 @@ +package azure + +import "github.com/confidential-containers/cloud-api-adaptor/pkg/util" + +type Config struct { + SubscriptionId string + ClientId string + ClientSecret string + TenantId string + ResourceGroupName string + Zone string + Region string + SubnetId string + SecurityGroupName string + SecurityGroupId string + Size string + ImageId string + SSHKeyPath string +} + +func (c Config) Redact() Config { + return *util.RedactStruct(&c, "ClientId", "TenantId", "ClientSecret").(*Config) +} diff --git a/pkg/adaptor/hypervisor/azure/types_test.go b/pkg/adaptor/hypervisor/azure/types_test.go new file mode 100644 index 000000000..8016aa90b --- /dev/null +++ b/pkg/adaptor/hypervisor/azure/types_test.go @@ -0,0 +1,40 @@ +//go:build azure + +package azure + +import ( + "fmt" + "strings" + "testing" +) + +func TestAzureMasking(t *testing.T) { + toBeRedacted := map[string]string{ + "client id": "a client id", + "tenant id": "a tenant id", + "client secret": "a client secret", + } + region := "a region" + + cloudCfg := Config{ + ClientId: toBeRedacted["client id"], + TenantId: toBeRedacted["tenant id"], + ClientSecret: toBeRedacted["client secret"], + Region: region, + } + + checkLine := func(verb string) { + logline := fmt.Sprintf(verb, cloudCfg.Redact()) + for k, v := range toBeRedacted { + if strings.Contains(logline, v) { + t.Errorf("For verb %s: %s contains the %s: %s", verb, logline, k, v) + } + } + if !strings.Contains(logline, region) { + t.Errorf("For verb %s: %s doesn't contain the region name: %s", verb, logline, region) + } + } + + checkLine("%v") + checkLine("%s") +} diff --git a/pkg/adaptor/hypervisor/ibmcloud/podvminfo.go b/pkg/adaptor/hypervisor/ibmcloud/podvminfo.go new file mode 100644 index 000000000..f18a0cb03 --- /dev/null +++ b/pkg/adaptor/hypervisor/ibmcloud/podvminfo.go @@ -0,0 +1,53 @@ +//go:build ibmcloud + +package ibmcloud + +import ( + "context" + "time" + + pb "github.com/confidential-containers/cloud-api-adaptor/proto/podvminfo" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type podVMInfoService struct { + srv *hypervisorService +} + +func newPodVMInfoService(srv *hypervisorService) pb.PodVMInfoService { + return &podVMInfoService{srv} +} + +func (s *podVMInfoService) GetInfo(ctx context.Context, req *pb.GetInfoRequest) (*pb.GetInfoResponse, error) { + + s.srv.Lock() + defer s.srv.Unlock() + + for { + + var vsi string + + for _, sandbox := range s.srv.sandboxes { + if sandbox.namespace == req.PodNamespace && sandbox.pod == req.PodName { + vsi = sandbox.vsi + break + } + } + + if vsi != "" { + return &pb.GetInfoResponse{VMID: vsi}, nil + } + + if !req.Wait { + break + } + s.srv.Unlock() + time.Sleep(5 * time.Second) // TODO: use wait/notify here + s.srv.Lock() + } + + st := status.Error(codes.NotFound, "vsi was not found") + + return nil, st +} diff --git a/pkg/adaptor/hypervisor/ibmcloud/server.go b/pkg/adaptor/hypervisor/ibmcloud/server.go new file mode 100644 index 000000000..025557e5f --- /dev/null +++ b/pkg/adaptor/hypervisor/ibmcloud/server.go @@ -0,0 +1,126 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 +//go:build ibmcloud + +package ibmcloud + +import ( + "context" + "log" + "net" + "os" + "path/filepath" + "sync" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + + "github.com/containerd/ttrpc" + pbHypervisor "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" + + pbPodVMInfo "github.com/confidential-containers/cloud-api-adaptor/proto/podvminfo" +) + +var logger = log.New(log.Writer(), "[helper/hypervisor] ", log.LstdFlags|log.Lmsgprefix) + +type server struct { + socketPath string + + ttRpc *ttrpc.Server + service pbHypervisor.HypervisorService + vmInfoService pbPodVMInfo.PodVMInfoService + + workerNode podnetwork.WorkerNode + + readyCh chan struct{} + stopCh chan struct{} + stopOnce sync.Once +} + +func NewServer(cfg hypervisor.Config, cloudCfg Config, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + + logger.Printf("hypervisor config %v", cfg) + logger.Printf("cloud config %v", cloudCfg.Redact()) + + var vpcV1 VpcV1 + if cloudCfg.ApiKey != "" { + //FIXME: Null ApiKey is used in unit tests + var err error + vpcV1, err = NewVpcV1(cloudCfg.ApiKey, cloudCfg.IamServiceURL, cloudCfg.VpcServiceURL) + if err != nil { + panic(err) + } + } + + s := &server{ + socketPath: cfg.SocketPath, + service: newService(vpcV1, &cloudCfg, &cfg, workerNode, cfg.PodsDir, daemonPort), + workerNode: workerNode, + readyCh: make(chan struct{}), + stopCh: make(chan struct{}), + } + + s.vmInfoService = newPodVMInfoService(s.service.(*hypervisorService)) // TODO: refactor not to use type casting + + return s +} + +func (s *server) Start(ctx context.Context) (err error) { + + ttRpc, err := ttrpc.NewServer() + if err != nil { + return err + } + s.ttRpc = ttRpc + if err := os.MkdirAll(filepath.Dir(s.socketPath), os.ModePerm); err != nil { + return err + } + if err := os.RemoveAll(s.socketPath); err != nil { // just in case socket wasn't cleaned + return err + } + pbHypervisor.RegisterHypervisorService(s.ttRpc, s.service) + pbPodVMInfo.RegisterPodVMInfoService(s.ttRpc, s.vmInfoService) + + listener, err := net.Listen("unix", s.socketPath) + if err != nil { + return err + } + + ttRpcErr := make(chan error) + go func() { + defer close(ttRpcErr) + if err := s.ttRpc.Serve(ctx, listener); err != nil { + ttRpcErr <- err + } + }() + defer func() { + ttRpcShutdownErr := s.ttRpc.Shutdown(context.Background()) + if ttRpcShutdownErr != nil && err == nil { + err = ttRpcShutdownErr + } + }() + + close(s.readyCh) + + select { + case <-ctx.Done(): + shutdownErr := s.Shutdown() + if shutdownErr != nil && err == nil { + err = shutdownErr + } + case <-s.stopCh: + case err = <-ttRpcErr: + } + return err +} + +func (s *server) Shutdown() error { + s.stopOnce.Do(func() { + close(s.stopCh) + }) + return nil +} + +func (s *server) Ready() chan struct{} { + return s.readyCh +} diff --git a/pkg/adaptor/hypervisor/ibmcloud/server_test.go b/pkg/adaptor/hypervisor/ibmcloud/server_test.go new file mode 100644 index 000000000..d2af665f3 --- /dev/null +++ b/pkg/adaptor/hypervisor/ibmcloud/server_test.go @@ -0,0 +1,283 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +//go:build ibmcloud +// +build ibmcloud + +package ibmcloud + +import ( + "context" + "errors" + "io/ioutil" + "log" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/IBM/go-sdk-core/v5/core" + "github.com/IBM/vpc-go-sdk/vpcv1" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/proxy" + daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/containerd/containerd/pkg/cri/annotations" + "github.com/containerd/ttrpc" + "github.com/google/uuid" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" + agent "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" +) + +func TestServerStartAndShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, dir, socketPath, client, serverErrCh := testServerStart(t, ctx) + defer testServerShutdown(t, s, socketPath, dir, serverErrCh) + if _, err := client.Version(context.Background(), &pb.VersionRequest{}); err != nil { + t.Error(err) + } +} + +func TestCreateStartAndStop(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, dir, socketPath, client, serverErrCh := testServerStart(t, ctx) + defer testServerShutdown(t, s, socketPath, dir, serverErrCh) + id := uuid.New().String() + if _, err := client.CreateVM( + context.Background(), + &pb.CreateVMRequest{ + Id: id, + Annotations: map[string]string{ + annotations.SandboxName: "test", + annotations.SandboxNamespace: "test", + }, + }, + ); err != nil { + t.Fatal(err) + } + if _, err := client.StartVM(context.Background(), &pb.StartVMRequest{Id: id}); err != nil { + t.Fatal(err) + } + + forwarderSocket := filepath.Join(dir, "pods", id, proxy.SocketName) + conn, err := net.Dial("unix", forwarderSocket) + if err != nil { + t.Fatal(err) + } + + ttrpcClient := ttrpc.NewClient(conn) + defer ttrpcClient.Close() + + agentClient := agent.NewAgentServiceClient(ttrpcClient) + + if _, err := agentClient.GetGuestDetails(ctx, &agent.GuestDetailsRequest{}); err != nil { + t.Fatal(err) + } + + if _, err := client.StopVM(context.Background(), &pb.StopVMRequest{Id: id}); err != nil { + t.Fatal(err) + } +} + +func testServerStart(t *testing.T, ctx context.Context) (hypervisor.Server, string, string, pb.HypervisorService, chan error) { + dir, err := ioutil.TempDir("", "helper") + if err != nil { + t.Fatal(err) + } + socketPath := filepath.Join(dir, "hypervisor.sock") + s := newServer(t, socketPath, filepath.Join(dir, "pods")) + + serverErrCh := make(chan error) + go func() { + defer close(serverErrCh) + if err := s.Start(ctx); err != nil { + serverErrCh <- err + } + }() + time.Sleep(1 * time.Millisecond) + select { + case err := <-serverErrCh: + t.Fatal(err) + default: + } + conn, err := net.Dial("unix", socketPath) + if err != nil { + t.Fatal(err) + } + client := pb.NewHypervisorClient(ttrpc.NewClient(conn)) + return s, dir, socketPath, client, serverErrCh +} + +func startAgentServer(t *testing.T) string { + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Expect no error, got %q", err) + } + _, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + + ttrpcServer, err := ttrpc.NewServer() + if err != nil { + t.Fatal(err) + } + + agent.RegisterAgentServiceService(ttrpcServer, newAgentService()) + agent.RegisterHealthService(ttrpcServer, &healthService{}) + + ctx := context.Background() + + go func() { + if err := ttrpcServer.Serve(ctx, listener); err != nil { + if !errors.Is(err, ttrpc.ErrServerClosed) { + t.Error(err) + } + } + }() + t.Cleanup(func() { + if err := ttrpcServer.Shutdown(context.Background()); err != nil { + t.Error(err) + } + }) + + return port +} + +func newServer(t *testing.T, socketPath, podsDir string) hypervisor.Server { + switch strings.ToLower(os.Getenv("USE_IBM_CLOUD")) { + case "", "no", "false", "0": + port := startAgentServer(t) + cfg := hypervisor.Config{ + SocketPath: socketPath, + PodsDir: podsDir, + HypProvider: "ibmcloud", + } + srv := NewServer(cfg, Config{}, &mockWorkerNode{}, port) + srv.(*server).service.(*hypervisorService).vpcV1 = &mockVpcV1{} + return srv + } + log.Print("Using IBM Cloud...") + apiKey := os.Getenv("API_KEY") + if apiKey == "" { + t.Fatal("Specify the API key as API_KEY") + } + + keyId := os.Getenv("KEY_ID") + if keyId == "" { + t.Fatal("Specify the SSH key ID as KEY_ID") + } + serviceConfig := Config{ + ApiKey: apiKey, + IamServiceURL: "https://iam.cloud.ibm.com/identity/token", + VpcServiceURL: "https://jp-tok.iaas.cloud.ibm.com/v1", + ResourceGroupID: "33e11ea6cbf340f2bfb6c8414f511b8d", + ProfileName: "bx2-2x8", + ZoneName: "us-south-2", + ImageID: "r134-d2090805-5652-4845-b287-46232e1098c3", + PrimarySubnetID: "0726-698b0a57-02db-49ee-a965-fa5f4d802fce", + PrimarySecurityGroupID: "r134-bace3bd1-6936-4126-ba6f-ff6d38775e9f", + SecondarySubnetID: "0726-bcb377d2-fccf-48c3-acd1-056d229ceb76", + SecondarySecurityGroupID: "r134-2d59206c-ac4d-4488-b9a4-a086bef59ee5", + KeyID: keyId, + VpcID: "r134-c199bf26-ec6d-4c5d-a0a2-1e74d312891f", + } + + cfg := hypervisor.Config{ + SocketPath: socketPath, + PodsDir: podsDir, + HypProvider: "ibmcloud", + } + srv := NewServer(cfg, serviceConfig, &mockWorkerNode{}, daemon.DefaultListenPort) + + return srv +} + +func testServerShutdown(t *testing.T, s hypervisor.Server, socketPath, dir string, serverErrCh chan error) { + + if err := s.Shutdown(); err != nil { + t.Error(err) + } + if err := <-serverErrCh; err != nil { + t.Error(err) + } + if _, err := os.Stat(socketPath); err == nil { + t.Errorf("Unix domain socket %s still remains\n", socketPath) + } + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } +} + +type mockVpcV1 struct { + primaryIP string + secondaryIP string +} + +func (v *mockVpcV1) GetInstance(getInstanceOptions *vpcv1.GetInstanceOptions) (result *vpcv1.Instance, response *core.DetailedResponse, err error) { + return &vpcv1.Instance{}, &core.DetailedResponse{}, nil +} + +func (v *mockVpcV1) CreateInstance(createInstanceOptions *vpcv1.CreateInstanceOptions) (result *vpcv1.Instance, response *core.DetailedResponse, err error) { + + primaryIP := v.primaryIP + if primaryIP == "" { + primaryIP = "127.0.0.1" + } + + secondaryIP := v.secondaryIP + if secondaryIP == "" { + secondaryIP = "127.0.0.1" + } + + strptr := func(s string) *string { return &s } + return &vpcv1.Instance{ + ID: strptr("mock"), + Name: strptr("mock"), + Image: &vpcv1.ImageReference{Name: strptr("mock")}, + PrimaryNetworkInterface: &vpcv1.NetworkInterfaceInstanceContextReference{ + ID: strptr("mockNIC1"), + Name: strptr("mockNIC1"), + PrimaryIpv4Address: strptr(primaryIP), + }, + NetworkInterfaces: []vpcv1.NetworkInterfaceInstanceContextReference{ + { + ID: strptr("mockNIC1"), + Name: strptr("mockNIC1"), + PrimaryIpv4Address: strptr(primaryIP), + }, + { + ID: strptr("mockNIC2"), + Name: strptr("mockNIC2"), + PrimaryIpv4Address: strptr(secondaryIP), + }, + }, + }, &core.DetailedResponse{}, nil +} + +func (v *mockVpcV1) DeleteInstance(deleteInstanceOptions *vpcv1.DeleteInstanceOptions) (response *core.DetailedResponse, err error) { + return &core.DetailedResponse{}, nil +} + +type mockWorkerNode struct{} + +func (n *mockWorkerNode) Inspect(nsPath string) (*tunneler.Config, error) { + return &tunneler.Config{}, nil +} + +func (n *mockWorkerNode) Setup(nsPath string, podNodeIPs []net.IP, config *tunneler.Config) error { + return nil +} + +func (n *mockWorkerNode) Teardown(nsPath string, config *tunneler.Config) error { + return nil +} diff --git a/pkg/adaptor/hypervisor/ibmcloud/service.go b/pkg/adaptor/hypervisor/ibmcloud/service.go new file mode 100644 index 000000000..22ac1c497 --- /dev/null +++ b/pkg/adaptor/hypervisor/ibmcloud/service.go @@ -0,0 +1,404 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 +//go:build ibmcloud + +package ibmcloud + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/IBM/vpc-go-sdk/vpcv1" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/proxy" + daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/cloudinit" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/hvutil" + "github.com/containerd/containerd/pkg/cri/annotations" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" +) + +// TODO: implement a ttrpc server to serve hypervisor RPC calls from kata shim +// https://github.com/kata-containers/kata-containers/blob/2.2.0-alpha1/src/runtime/virtcontainers/hypervisor.go#L843-L883 + +const ( + Version = "0.0.0" + maxRetries = 10 + queryInterval = 2 +) + +type hypervisorService struct { + vpcV1 VpcV1 + serviceConfig *Config + hypervisorConfig *hypervisor.Config + sandboxes map[sandboxID]*sandbox + podsDir string + daemonPort string + nodeName string + workerNode podnetwork.WorkerNode + sync.Mutex +} + +func newService(vpcV1 VpcV1, config *Config, hypervisorConfig *hypervisor.Config, workerNode podnetwork.WorkerNode, podsDir, daemonPort string) pb.HypervisorService { + + logger.Printf("service config %v", config.Redact()) + + hostname, err := os.Hostname() + if err != nil { + panic(fmt.Errorf("failed to get hostname: %w", err)) + } + + i := strings.Index(hostname, ".") + if i >= 0 { + hostname = hostname[0:i] + } + + return &hypervisorService{ + vpcV1: vpcV1, + serviceConfig: config, + hypervisorConfig: hypervisorConfig, + sandboxes: map[sandboxID]*sandbox{}, + podsDir: podsDir, + daemonPort: daemonPort, + nodeName: hostname, + workerNode: workerNode, + } +} + +type sandboxID string + +type sandbox struct { + id sandboxID + pod string + namespace string + netNSPath string + podDirPath string + vsi string + agentProxy proxy.AgentProxy + podNetworkConfig *tunneler.Config +} + +func (s *hypervisorService) Version(ctx context.Context, req *pb.VersionRequest) (*pb.VersionResponse, error) { + return &pb.VersionResponse{Version: Version}, nil +} + +func (s *hypervisorService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (*pb.CreateVMResponse, error) { + + sid := sandboxID(req.Id) + + if sid == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; exists { + return nil, fmt.Errorf("sandbox %s already exists", sid) + } + pod := hvutil.GetPodName(req.Annotations) + if pod == "" { + return nil, fmt.Errorf("pod name %s is missing in annotations", annotations.SandboxName) + } + + namespace := hvutil.GetPodNamespace(req.Annotations) + if namespace == "" { + return nil, fmt.Errorf("namespace name %s is missing in annotations", annotations.SandboxNamespace) + } + + podDirPath := filepath.Join(s.podsDir, string(sid)) + if err := os.MkdirAll(podDirPath, os.ModePerm); err != nil { + return nil, fmt.Errorf("failed to create a pod directory: %s: %w", podDirPath, err) + } + + socketPath := filepath.Join(podDirPath, proxy.SocketName) + + netNSPath := req.NetworkNamespacePath + + podNetworkConfig, err := s.workerNode.Inspect(netNSPath) + if err != nil { + return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err) + } + + agentProxy := proxy.NewAgentProxy(socketPath, s.hypervisorConfig.CriSocketPath, s.hypervisorConfig.PauseImage) + + sandbox := &sandbox{ + id: sid, + pod: pod, + namespace: namespace, + netNSPath: netNSPath, + podDirPath: podDirPath, + agentProxy: agentProxy, + podNetworkConfig: podNetworkConfig, + } + s.sandboxes[sid] = sandbox + logger.Printf("create a sandbox %s for pod %s in namespace %s (netns: %s)", req.Id, pod, namespace, sandbox.netNSPath) + return &pb.CreateVMResponse{AgentSocketPath: socketPath}, nil +} + +const maxInstanceNameLen = 63 + +func (s *hypervisorService) StartVM(ctx context.Context, req *pb.StartVMRequest) (*pb.StartVMResponse, error) { + + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + vmName := hvutil.CreateInstanceName(sandbox.pod, string(sandbox.id), maxInstanceNameLen) + + logger.Printf("StartVM: vmName: %q", vmName) + + daemonConfig := daemon.Config{ + PodNamespace: sandbox.namespace, + PodName: sandbox.pod, + PodNetwork: sandbox.podNetworkConfig, + } + daemonJSON, err := json.MarshalIndent(daemonConfig, "", " ") + if err != nil { + return nil, err + } + + // Store daemon.json in worker node for debugging + if err := os.WriteFile(filepath.Join(sandbox.podDirPath, "daemon.json"), daemonJSON, 0666); err != nil { + return nil, fmt.Errorf("failed to store daemon.json at %s: %w", sandbox.podDirPath, err) + } + logger.Printf("store daemon.json at %s", sandbox.podDirPath) + + cloudConfig := &cloudinit.CloudConfig{ + WriteFiles: []cloudinit.WriteFile{ + { + Path: daemon.DefaultConfigPath, + Content: string(daemonJSON), + }, + }, + } + + if authJSON, err := os.ReadFile(cloudinit.DefaultAuthfileSrcPath); err == nil { + if json.Valid(authJSON) && (len(authJSON) < cloudinit.DefaultAuthfileLimit) { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, + cloudinit.WriteFile{ + Path: cloudinit.DefaultAuthfileDstPath, + Content: cloudinit.AuthJSONToResourcesJSON(string(authJSON)), + }) + } else if len(authJSON) >= cloudinit.DefaultAuthfileLimit { + logger.Printf("Credentials file size (%d) is too large to use as userdata, ignored", len(authJSON)) + } else { + logger.Printf("Credentials file is not in a valid Json format, ignored") + } + } + + userData, err := cloudConfig.Generate() + if err != nil { + return nil, err + } + + prototype := &vpcv1.InstancePrototype{ + Name: &vmName, + Image: &vpcv1.ImageIdentity{ID: &s.serviceConfig.ImageID}, + UserData: &userData, + Profile: &vpcv1.InstanceProfileIdentity{Name: &s.serviceConfig.ProfileName}, + Zone: &vpcv1.ZoneIdentity{Name: &s.serviceConfig.ZoneName}, + Keys: []vpcv1.KeyIdentityIntf{ + &vpcv1.KeyIdentity{ID: &s.serviceConfig.KeyID}, + }, + VPC: &vpcv1.VPCIdentity{ID: &s.serviceConfig.VpcID}, + PrimaryNetworkInterface: &vpcv1.NetworkInterfacePrototype{ + Subnet: &vpcv1.SubnetIdentity{ID: &s.serviceConfig.PrimarySubnetID}, + SecurityGroups: []vpcv1.SecurityGroupIdentityIntf{ + &vpcv1.SecurityGroupIdentityByID{ID: &s.serviceConfig.PrimarySecurityGroupID}, + }, + }, + } + if s.serviceConfig.ResourceGroupID != "" { + prototype.ResourceGroup = &vpcv1.ResourceGroupIdentity{ID: &s.serviceConfig.ResourceGroupID} + } + if s.serviceConfig.SecondarySubnetID != "" { + prototype.NetworkInterfaces = []vpcv1.NetworkInterfacePrototype{ + { + AllowIPSpoofing: func(b bool) *bool { return &b }(true), + Subnet: &vpcv1.SubnetIdentity{ID: &s.serviceConfig.SecondarySubnetID}, + SecurityGroups: []vpcv1.SecurityGroupIdentityIntf{ + &vpcv1.SecurityGroupIdentityByID{ID: &s.serviceConfig.SecondarySecurityGroupID}, + }, + }, + } + } + + result, resp, err := s.vpcV1.CreateInstance(&vpcv1.CreateInstanceOptions{InstancePrototype: prototype}) + if err != nil { + logger.Printf("failed to create an instance : %v and the response is %s", err, resp) + return nil, err + } + + s.Lock() + sandbox.vsi = *result.ID + s.Unlock() + + logger.Printf("created an instance %s for sandbox %s", *result.Name, req.Id) + + var podNodeIPs []net.IP + + for retries := 0; retries < maxRetries; retries++ { + + ips, err := getIPs(prototype, result) + + if err == nil { + podNodeIPs = ips + break + } else if err != errNotReady { + return nil, err + } + + time.Sleep(time.Duration(queryInterval) * time.Second) + + var id string = *result.ID + getResult, resp, err := s.vpcV1.GetInstance(&vpcv1.GetInstanceOptions{ID: &id}) + if err != nil { + logger.Printf("failed to get an instance : %v and the response is %s", err, resp) + return nil, err + } + result = getResult + } + + if err := s.workerNode.Setup(sandbox.netNSPath, podNodeIPs, sandbox.podNetworkConfig); err != nil { + return nil, fmt.Errorf("failed to set up pod network tunnel on netns %s: %w", sandbox.netNSPath, err) + } + + serverURL := &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(podNodeIPs[0].String(), s.daemonPort), + Path: daemon.AgentURLPath, + } + + errCh := make(chan error) + go func() { + defer close(errCh) + + if err := sandbox.agentProxy.Start(context.Background(), serverURL); err != nil { + logger.Printf("error running agent proxy: %v", err) + errCh <- err + } + }() + + select { + case <-ctx.Done(): + _ = sandbox.agentProxy.Shutdown() + return nil, ctx.Err() + case err := <-errCh: + return nil, err + case <-sandbox.agentProxy.Ready(): + } + + logger.Printf("agent proxy is ready") + return &pb.StartVMResponse{}, nil +} + +func (s *hypervisorService) getSandbox(id string) (*sandbox, error) { + + sid := sandboxID(id) + + if id == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; !exists { + return nil, fmt.Errorf("sandbox %s does not exist", sid) + } + return s.sandboxes[sid], nil +} + +func (s *hypervisorService) deleteSandbox(id string) error { + sid := sandboxID(id) + if id == "" { + return errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + delete(s.sandboxes, sid) + return nil +} + +var errNotReady = errors.New("address not ready") + +func getIPs(prototype *vpcv1.InstancePrototype, result *vpcv1.Instance) ([]net.IP, error) { + + if len(result.NetworkInterfaces) < 1+len(prototype.NetworkInterfaces) { + return nil, errNotReady + } + + interfaces := []*vpcv1.NetworkInterfaceInstanceContextReference{result.PrimaryNetworkInterface} + for i, nic := range result.NetworkInterfaces { + if *nic.ID != *result.PrimaryNetworkInterface.ID { + interfaces = append(interfaces, &result.NetworkInterfaces[i]) + } + } + + var podNodeIPs []net.IP + + for i, nic := range interfaces { + + addr := nic.PrimaryIpv4Address + if addr == nil || *addr == "" || *addr == "0.0.0.0" { + return nil, errNotReady + } + + ip := net.ParseIP(*addr) + if ip == nil { + return nil, fmt.Errorf("failed to parse pod node IP %q", *addr) + } + podNodeIPs = append(podNodeIPs, ip) + + logger.Printf("podNodeIP[%d]=%s", i, ip.String()) + } + + return podNodeIPs, nil +} + +func (s *hypervisorService) deleteInstance(ctx context.Context, id string) error { + options := &vpcv1.DeleteInstanceOptions{} + options.SetID(id) + resp, err := s.vpcV1.DeleteInstance(options) + if err != nil { + logger.Printf("failed to delete an instance: %v and the response is %v", err, resp) + return err + } + logger.Printf("deleted an instance %s", id) + return nil +} + +func (s *hypervisorService) StopVM(ctx context.Context, req *pb.StopVMRequest) (*pb.StopVMResponse, error) { + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + if err := sandbox.agentProxy.Shutdown(); err != nil { + logger.Printf("failed to stop agent proxy: %v", err) + } + + if err := s.deleteInstance(ctx, sandbox.vsi); err != nil { + return nil, err + } + + if err := s.workerNode.Teardown(sandbox.netNSPath, sandbox.podNetworkConfig); err != nil { + return nil, fmt.Errorf("failed to tear down netns %s: %w", sandbox.netNSPath, err) + } + err = s.deleteSandbox(req.Id) + if err != nil { + return nil, err + } + return &pb.StopVMResponse{}, nil +} diff --git a/pkg/adaptor/hypervisor/ibmcloud/shim_test.go b/pkg/adaptor/hypervisor/ibmcloud/shim_test.go new file mode 100644 index 000000000..30658a0f8 --- /dev/null +++ b/pkg/adaptor/hypervisor/ibmcloud/shim_test.go @@ -0,0 +1,537 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +//go:build ibmcloud +// +build ibmcloud + +package ibmcloud + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder" + "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder/interceptor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/containerd/containerd/pkg/cri/annotations" + "github.com/containerd/ttrpc" + "github.com/gogo/protobuf/types" + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" + agent "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" +) + +func TestShim(t *testing.T) { + + dir, err := os.MkdirTemp("", "shimtest-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + helperSocketPath := os.Getenv("SHIMTEST_HELPER_SOCKET") + if helperSocketPath != "" { + if err := os.Remove(helperSocketPath); err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + } else { + helperSocketPath = filepath.Join(dir, "helper.sock") + } + + podsDir := filepath.Join(dir, "pods") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + agentDone := make(chan struct{}) + agentSocketPath := os.Getenv("SHIMTEST_AGENT_SOCKET") + if agentSocketPath != "" { + t.Logf("SHIMTEST_AGENT_SOCKET is specified\nthe agent protocol forwarder will connect to kata agent at %s", agentSocketPath) + } else { + agentSocketPath = "@" + filepath.Join(dir, "agent.sock") + go startTestAgent(t, ctx, agentSocketPath, agentDone) + } + + daemonDone := make(chan struct{}) + daemonAddr := os.Getenv("AGENT_PROTOCOL_FORWARDER_ADDRESS") + if daemonAddr == "" { + addrCh := make(chan string) + go startDaemon(t, ctx, agentSocketPath, addrCh, daemonDone) + daemonAddr = <-addrCh + } else { + defer close(daemonDone) + } + + primaryIP, port, err := net.SplitHostPort(daemonAddr) + if err != nil { + t.Fatal(err) + } + secondaryIP := os.Getenv("SHIMTEST_SECONDARY_POD_NODE_IP") + + var workerNode podnetwork.WorkerNode + + switch t := os.Getenv("SHIMTEST_TUNNEL_TYPE"); t { + case "", "mock": + workerNode = &mockWorkerNode{} + case "routing": + workerNode = podnetwork.NewWorkerNode("routing", "ens4", 0, 0) + default: + workerNode = podnetwork.NewWorkerNode(t, "", 0, 0) + } + + cfg := hypervisor.Config{ + SocketPath: helperSocketPath, + PodsDir: podsDir, + HypProvider: "ibmcloud", + } + srv := NewServer(cfg, Config{}, workerNode, port) + srv.(*server).service.(*hypervisorService).vpcV1 = &mockVpcV1{primaryIP: primaryIP, secondaryIP: secondaryIP} + + serverDone := make(chan struct{}) + go func() { + defer close(serverDone) + + if err := srv.Start(ctx); err != nil { + t.Error(err) + } + }() + + <-srv.Ready() + + clientDone := make(chan error) + + if os.Getenv("SHIMTEST_USE_REAL_SHIM") != "" { + defer close(clientDone) + + t.Logf("SHIMTEST_USE_REAL_SHIM is enabled\nhelper daemon socket is %s\nUse crictl, containerd, and kata shim to test the helper daemon", helperSocketPath) + } else { + + agentForwarderSocketPath := runMockShim(t, ctx, helperSocketPath) + + if os.Getenv("SHIMTEST_USE_AGENT_CTL") != "" { + defer close(clientDone) + t.Logf("SHIMTEST_USE_AGENT_CTL is enabled\nagent forwarder socket is %s\nUse kata-agent-ctl to test the helper daemon", agentForwarderSocketPath) + + } else { + + go func() { + defer close(clientDone) + + conn, err := net.Dial("unix", agentForwarderSocketPath) + if err != nil { + t.Error(err) + return + } + ttrpcClient := ttrpc.NewClient(conn) + defer ttrpcClient.Close() + + client := agent.NewAgentServiceClient(ttrpcClient) + + if _, err := client.GetGuestDetails(ctx, &agent.GuestDetailsRequest{}); err != nil { + t.Error(err) + return + } + }() + } + } + + select { + case <-ctx.Done(): + case <-agentDone: + case <-daemonDone: + case <-clientDone: + case <-serverDone: + } +} + +func startDaemon(t *testing.T, ctx context.Context, agentSocketPath string, portCh chan string, done chan struct{}) { + + defer close(done) + + config := &daemon.Config{} + + nsPath := os.Getenv("AGENT_PROTOCOL_FORWARDER_NAMESPACE") + interceptor := interceptor.NewInterceptor(agentSocketPath, nsPath) + + d := daemon.NewDaemon(config, "127.0.0.1:0", interceptor, &mockPodNode{}) + + daemonErr := make(chan error) + go func() { + defer close(daemonErr) + + if err := d.Start(ctx); err != nil { + daemonErr <- err + } + }() + + portCh <- d.Addr() + + select { + case <-ctx.Done(): + case err := <-daemonErr: + if err != nil { + t.Error(err) + } + } +} + +func startTestAgent(t *testing.T, ctx context.Context, agentSocketPath string, done chan struct{}) { + + defer close(done) + + ttrpcServer, err := ttrpc.NewServer() + if err != nil { + t.Error(err) + return + } + + agent.RegisterAgentServiceService(ttrpcServer, newAgentService()) + agent.RegisterHealthService(ttrpcServer, &healthService{}) + + socket := agentSocketPath + + listener, err := net.Listen("unix", socket) + if err != nil { + t.Error(err) + return + } + + ttrpcServerErr := make(chan error) + go func() { + defer close(ttrpcServerErr) + + if err := ttrpcServer.Serve(ctx, listener); err != nil { + ttrpcServerErr <- err + } + }() + defer func() { + if err := ttrpcServer.Shutdown(context.Background()); err != nil { + t.Error(err) + } + }() + + select { + case <-ctx.Done(): + case err = <-ttrpcServerErr: + } + if err != nil { + t.Error(err) + } +} + +func runMockShim(t *testing.T, ctx context.Context, helperSocketPath string) string { + + conn, err := net.Dial("unix", helperSocketPath) + if err != nil { + t.Error(err) + } + ttrpcClient := ttrpc.NewClient(conn) + defer ttrpcClient.Close() + + client := pb.NewHypervisorClient(ttrpcClient) + + podID := "123" + + req1 := &pb.CreateVMRequest{ + Id: podID, + Annotations: map[string]string{ + annotations.SandboxNamespace: "default", + annotations.SandboxName: "pod1", + }, + } + + res1, err := client.CreateVM(ctx, req1) + if err != nil { + t.Fatal(err) + } + + agentForwarderSocketPath := res1.AgentSocketPath + + req2 := &pb.StartVMRequest{ + Id: podID, + } + + if _, err := client.StartVM(ctx, req2); err != nil { + t.Fatal(err) + } + + return agentForwarderSocketPath +} + +type containerID string + +type container struct { + readCount int64 + done chan struct{} + once sync.Once +} + +type agentService struct { + containers map[containerID]*container + mutex sync.Mutex +} + +func newAgentService() *agentService { + return &agentService{ + containers: make(map[containerID]*container), + } +} + +func (s *agentService) get(cid string) (*container, error) { + + s.mutex.Lock() + c := s.containers[containerID(cid)] + s.mutex.Unlock() + + if c == nil { + return nil, fmt.Errorf("container %q not found", cid) + } + + return c, nil +} + +func (s *agentService) CreateContainer(ctx context.Context, req *agent.CreateContainerRequest) (*types.Empty, error) { + log.Printf("agent call: %T %#v", req, req) + + cid := containerID(req.ContainerId) + c := &container{ + done: make(chan struct{}), + } + + s.mutex.Lock() + defer s.mutex.Unlock() + + s.containers[cid] = c + + return &types.Empty{}, nil +} +func (s *agentService) StartContainer(ctx context.Context, req *agent.StartContainerRequest) (*types.Empty, error) { + log.Printf("agent call: %T %#v", req, req) + return &types.Empty{}, nil +} +func (s *agentService) RemoveContainer(ctx context.Context, req *agent.RemoveContainerRequest) (*types.Empty, error) { + log.Printf("agent call: %T %#v", req, req) + + cid := containerID(req.ContainerId) + + s.mutex.Lock() + defer s.mutex.Unlock() + + delete(s.containers, cid) + + return &types.Empty{}, nil +} +func (s *agentService) ExecProcess(ctx context.Context, req *agent.ExecProcessRequest) (*types.Empty, error) { + log.Printf("agent call: %T %#v", req, req) + return &types.Empty{}, nil +} +func (s *agentService) SignalProcess(ctx context.Context, req *agent.SignalProcessRequest) (*types.Empty, error) { + log.Printf("agent call: %T %#v", req, req) + + c, err := s.get(req.ContainerId) + if err != nil { + return nil, err + } + + c.once.Do(func() { + close(c.done) + }) + + return &types.Empty{}, nil +} +func (s *agentService) WaitProcess(ctx context.Context, req *agent.WaitProcessRequest) (*agent.WaitProcessResponse, error) { + log.Printf("agent call: %T %#v", req, req) + + c, err := s.get(req.ContainerId) + if err != nil { + return nil, err + } + + <-c.done + + return &agent.WaitProcessResponse{}, nil +} +func (s *agentService) UpdateContainer(ctx context.Context, req *agent.UpdateContainerRequest) (*types.Empty, error) { + log.Printf("agent call: %T %#v", req, req) + return &types.Empty{}, nil +} +func (s *agentService) StatsContainer(ctx context.Context, req *agent.StatsContainerRequest) (*agent.StatsContainerResponse, error) { + log.Printf("agent call: %T %#v", req, req) + return &agent.StatsContainerResponse{}, nil +} +func (s *agentService) PauseContainer(ctx context.Context, req *agent.PauseContainerRequest) (*types.Empty, error) { + log.Printf("agent call: %T %#v", req, req) + return &types.Empty{}, nil +} +func (s *agentService) ResumeContainer(ctx context.Context, req *agent.ResumeContainerRequest) (*types.Empty, error) { + log.Printf("agent call: %T %#v", req, req) + return &types.Empty{}, nil +} +func (s *agentService) WriteStdin(ctx context.Context, req *agent.WriteStreamRequest) (*agent.WriteStreamResponse, error) { + log.Printf("agent call: %T %#v", req, req) + return &agent.WriteStreamResponse{Len: uint32(len(req.Data))}, nil +} + +func (s *agentService) ReadStdout(ctx context.Context, req *agent.ReadStreamRequest) (*agent.ReadStreamResponse, error) { + log.Printf("agent call: ReadStdout %#v", req) + + c, err := s.get(req.ContainerId) + if err != nil { + return nil, err + } + + count := atomic.AddInt64(&c.readCount, 1) - 1 + sleep := 0 + if count > 0 { + sleep = 1 << count + } + timer := time.NewTimer(time.Duration(sleep) * time.Second) + + select { + case <-c.done: + return nil, io.EOF + + case <-timer.C: + str := fmt.Sprintf("data from agent (sleeping for %d seconds)\n", 2< 0 { + // TBD: ability to handle multiple interfaces and ips + logger.Printf("VM IP %s", domInterface[0].Addrs[0].Addr) + v.ips = append(v.ips, net.ParseIP(domInterface[0].Addrs[0].Addr)) + logger.Printf("VM IP list %v", v.ips) + } + + logger.Printf("Instance created successfully") + return &createInstanceOutput{ + instance: v, + }, nil +} + +func DeleteInstance(ctx context.Context, libvirtClient *libvirtClient, id string) (err error) { + + logger.Printf("Deleting instance (%s)", id) + idUint, _ := strconv.ParseUint(id, 10, 64) + // libvirt API takes uint32 + exists, err := checkInstanceExistsById(uint32(idUint), libvirtClient) + if err != nil { + logger.Printf("Unable to check instance (%s)", id) + return err + } + if !exists { + logger.Printf("Instance (%s) not found", id) + return err + } + // Stop and undefine domain + + // Sadly couldn't find an API to do the following + // virsh undefine --remove-all-storage + + domain, err := libvirtClient.connection.LookupDomainById(uint32(idUint)) + if err != nil { + logger.Printf("Error retrieving libvirt domain: %s", err) + return err + } + defer freeDomain(domain, &err) + + state, _, err := domain.GetState() + if err != nil { + logger.Printf("Couldn't get info about domain: %s", err) + return err + } + + if state == libvirt.DOMAIN_RUNNING || state == libvirt.DOMAIN_PAUSED { + if err = domain.Destroy(); err != nil { + logger.Printf("Couldn't destroy libvirt domain: %s", err) + return err + } + } + + // Delete volumes + domainXMLDesc, err := domain.GetXMLDesc(0) + if err != nil { + logger.Printf("Error retrieving libvirt domain XML description: %s", err) + return err + } + domainDef := libvirtxml.Domain{} + err = xml.Unmarshal([]byte(domainXMLDesc), &domainDef) + if err != nil { + logger.Printf("Unable to get the domain XML: %s", err) + } + + // Get the volume path from the XML + logger.Printf("domainDef %v", domainDef.Devices.Disks) + vol1File := domainDef.Devices.Disks[0].Source.File.File + vol2File := domainDef.Devices.Disks[1].Source.File.File + + err = deleteVolumeByPath(libvirtClient, vol1File) + if err != nil { + logger.Printf("Deleting volume (%s) returned error: %s", vol1File, err) + } + err = deleteVolumeByPath(libvirtClient, vol2File) + if err != nil { + logger.Printf("Deleting volume (%s) returned error: %s", vol2File, err) + } + // Undefine the domain + if err := domain.UndefineFlags(libvirt.DOMAIN_UNDEFINE_NVRAM); err != nil { + if e := err.(libvirt.Error); e.Code == libvirt.ERR_NO_SUPPORT || e.Code == libvirt.ERR_INVALID_ARG { + logger.Printf("libvirt does not support undefine flags: will try again without flags") + if err = domain.Undefine(); err != nil { + logger.Printf("couldn't undefine libvirt domain: %v", err) + return err + } + } else { + logger.Printf("couldn't undefine libvirt domain with flags: %v", err) + return err + } + } + + return nil +} + +func NewLibvirtClient(libvirtCfg Config) (*libvirtClient, error) { + + // Define Domain via XML created before. + conn, err := libvirt.NewConnect(libvirtCfg.URI) + if err != nil { + return nil, err + } + + pool, err := conn.LookupStoragePoolByName(libvirtCfg.PoolName) + if err != nil { + return nil, fmt.Errorf("can't find storage pool %q: %v", libvirtCfg.PoolName, err) + } + + fmt.Printf("Created libvirt connection") + + return &libvirtClient{ + connection: conn, + pool: pool, + poolName: libvirtCfg.PoolName, + networkName: libvirtCfg.NetworkName, + dataDir: libvirtCfg.DataDir, + }, nil +} + +// freeDomain releases the domain pointer. If the operation fail and the error +// context is nil then it gets updated, otherwise it preserve the pointer to +// keep any previous error reported. +func freeDomain(domain *libvirt.Domain, errCtx *error) { + newErr := domain.Free() + if newErr != nil && *errCtx == nil { + *errCtx = newErr + } +} diff --git a/pkg/adaptor/hypervisor/libvirt/server.go b/pkg/adaptor/hypervisor/libvirt/server.go new file mode 100644 index 000000000..2c2e25b9c --- /dev/null +++ b/pkg/adaptor/hypervisor/libvirt/server.go @@ -0,0 +1,110 @@ +//go:build libvirt +// +build libvirt + +package libvirt + +import ( + "context" + "log" + "net" + "os" + "path/filepath" + "sync" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/containerd/ttrpc" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" +) + +var logger = log.New(log.Writer(), "[helper/hypervisor] ", log.LstdFlags|log.Lmsgprefix) + +type server struct { + socketPath string + + ttRpc *ttrpc.Server + service pb.HypervisorService + + workerNode podnetwork.WorkerNode + + readyCh chan struct{} + stopCh chan struct{} + stopOnce sync.Once +} + +func NewServer(cfg hypervisor.Config, cloudCfg Config, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + + logger.Printf("hypervisor config %v", cfg) + logger.Printf("cloud config %v", cloudCfg) + + libvirtClient, err := NewLibvirtClient(cloudCfg) + if err != nil { + logger.Printf("Unable to create libvirt connection: %v", err) + return nil + } + return &server{ + socketPath: cfg.SocketPath, + service: newService(libvirtClient, &cloudCfg, &cfg, workerNode, cfg.PodsDir, daemonPort), + workerNode: workerNode, + readyCh: make(chan struct{}), + stopCh: make(chan struct{}), + } +} + +func (s *server) Start(ctx context.Context) (err error) { + + ttRpc, err := ttrpc.NewServer() + if err != nil { + return err + } + s.ttRpc = ttRpc + if err = os.MkdirAll(filepath.Dir(s.socketPath), os.ModePerm); err != nil { + return err + } + if err := os.RemoveAll(s.socketPath); err != nil { // just in case socket wasn't cleaned + return err + } + pb.RegisterHypervisorService(s.ttRpc, s.service) + listener, err := net.Listen("unix", s.socketPath) + if err != nil { + return err + } + + ttRpcErr := make(chan error) + go func() { + defer close(ttRpcErr) + if err = s.ttRpc.Serve(ctx, listener); err != nil { + ttRpcErr <- err + } + }() + defer func() { + newErr := s.ttRpc.Shutdown(context.Background()) + if newErr != nil && err == nil { + err = newErr + } + }() + + close(s.readyCh) + + select { + case <-ctx.Done(): + err = s.Shutdown() + case <-s.stopCh: + case err = <-ttRpcErr: + } + return err +} + +func (s *server) Shutdown() error { + s.stopOnce.Do(func() { + close(s.stopCh) + //TBD: Close libvirt connection + }) + + return nil +} + +func (s *server) Ready() chan struct{} { + return s.readyCh +} diff --git a/pkg/adaptor/hypervisor/libvirt/service.go b/pkg/adaptor/hypervisor/libvirt/service.go new file mode 100644 index 000000000..95eb0c5e5 --- /dev/null +++ b/pkg/adaptor/hypervisor/libvirt/service.go @@ -0,0 +1,310 @@ +//go:build libvirt +// +build libvirt + +package libvirt + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/proxy" + daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/cloudinit" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/hvutil" + "github.com/containerd/containerd/pkg/cri/annotations" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" +) + +const ( + Version = "0.0.0" +) + +type hypervisorService struct { + libvirtClient *libvirtClient + serviceConfig *Config + hypervisorConfig *hypervisor.Config + sandboxes map[sandboxID]*sandbox + podsDir string + daemonPort string + nodeName string + workerNode podnetwork.WorkerNode + sync.Mutex +} + +func newService(libvirtClient *libvirtClient, config *Config, hypervisorConfig *hypervisor.Config, workerNode podnetwork.WorkerNode, podsDir, daemonPort string) pb.HypervisorService { + logger.Printf("service config %v", config) + + hostname, err := os.Hostname() + if err != nil { + panic(fmt.Errorf("failed to get hostname: %w", err)) + } + + i := strings.Index(hostname, ".") + if i >= 0 { + hostname = hostname[0:i] + } + + return &hypervisorService{ + libvirtClient: libvirtClient, + serviceConfig: config, + hypervisorConfig: hypervisorConfig, + sandboxes: map[sandboxID]*sandbox{}, + podsDir: podsDir, + daemonPort: daemonPort, + nodeName: hostname, + workerNode: workerNode, + } +} + +type sandboxID string + +type sandbox struct { + id sandboxID + pod string + namespace string + netNSPath string + podDirPath string + vsi string + agentProxy proxy.AgentProxy + podNetworkConfig *tunneler.Config +} + +func (s *hypervisorService) Version(ctx context.Context, req *pb.VersionRequest) (*pb.VersionResponse, error) { + return &pb.VersionResponse{Version: Version}, nil +} + +func (s *hypervisorService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (*pb.CreateVMResponse, error) { + + sid := sandboxID(req.Id) + + if sid == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; exists { + return nil, fmt.Errorf("sandbox %s already exists", sid) + } + pod := hvutil.GetPodName(req.Annotations) + if pod == "" { + return nil, fmt.Errorf("pod name %s is missing in annotations", annotations.SandboxName) + } + + namespace := hvutil.GetPodNamespace(req.Annotations) + if namespace == "" { + return nil, fmt.Errorf("namespace name %s is missing in annotations", annotations.SandboxNamespace) + } + + podDirPath := filepath.Join(s.podsDir, string(sid)) + if err := os.MkdirAll(podDirPath, os.ModePerm); err != nil { + return nil, fmt.Errorf("failed to create a pod directory: %s: %w", podDirPath, err) + } + + socketPath := filepath.Join(podDirPath, proxy.SocketName) + + netNSPath := req.NetworkNamespacePath + + podNetworkConfig, err := s.workerNode.Inspect(netNSPath) + if err != nil { + return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err) + } + + agentProxy := proxy.NewAgentProxy(socketPath, s.hypervisorConfig.CriSocketPath, s.hypervisorConfig.PauseImage) + + sandbox := &sandbox{ + id: sid, + pod: pod, + namespace: namespace, + netNSPath: netNSPath, + podDirPath: podDirPath, + agentProxy: agentProxy, + podNetworkConfig: podNetworkConfig, + } + s.sandboxes[sid] = sandbox + logger.Printf("create a sandbox %s for pod %s in namespace %s (netns: %s)", req.Id, pod, namespace, sandbox.netNSPath) + return &pb.CreateVMResponse{AgentSocketPath: socketPath}, nil +} + +func (s *hypervisorService) StartVM(ctx context.Context, req *pb.StartVMRequest) (*pb.StartVMResponse, error) { + + logger.Printf("Starting VM") + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + daemonConfig := daemon.Config{ + PodNamespace: sandbox.namespace, + PodName: sandbox.pod, + PodNetwork: sandbox.podNetworkConfig, + } + daemonJSON, err := json.MarshalIndent(daemonConfig, "", " ") + if err != nil { + return nil, err + } + + // Store daemon.json in worker node for debugging + if err = os.WriteFile(filepath.Join(sandbox.podDirPath, "daemon.json"), daemonJSON, 0666); err != nil { + return nil, fmt.Errorf("failed to store daemon.json at %s: %w", sandbox.podDirPath, err) + } + logger.Printf("store daemon.json at %s", sandbox.podDirPath) + + cloudConfig := &cloudinit.CloudConfig{ + WriteFiles: []cloudinit.WriteFile{ + { + Path: daemon.DefaultConfigPath, + Content: string(daemonJSON), + }, + }, + } + + if authJSON, err := os.ReadFile(cloudinit.DefaultAuthfileSrcPath); err == nil { + if json.Valid(authJSON) && (len(authJSON) < cloudinit.DefaultAuthfileLimit) { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, + cloudinit.WriteFile{ + Path: cloudinit.DefaultAuthfileDstPath, + Content: cloudinit.AuthJSONToResourcesJSON(string(authJSON)), + }) + } else if len(authJSON) >= cloudinit.DefaultAuthfileLimit { + logger.Printf("Credentials file size (%d) is too large to use as userdata, ignored", len(authJSON)) + } else { + logger.Printf("Credentials file is not in a valid Json format, ignored") + } + } + + userData, err := cloudConfig.Generate() + if err != nil { + return nil, err + } + + // TODO: Specify the maximum instance name length in Libvirt + vmName := hvutil.CreateInstanceName(sandbox.pod, string(sandbox.id), 0) + vm := &vmConfig{name: vmName, userData: userData} + result, err := CreateInstance(ctx, s.libvirtClient, vm) + if err != nil { + logger.Printf("failed to create an instance : %v", err) + return nil, err + } + + sandbox.vsi = result.instance.instanceId + + logger.Printf("created an instance(%s) with id(%s) for sandbox %s", result.instance.name, sandbox.vsi, req.Id) + + //Get Libvirt VM IP + podNodeIPs, err := getIPs(result.instance) + if err != nil { + logger.Printf("failed to get IPs for the instance : %v ", err) + return nil, err + } + + if err := s.workerNode.Setup(sandbox.netNSPath, podNodeIPs, sandbox.podNetworkConfig); err != nil { + return nil, fmt.Errorf("failed to set up pod network tunnel on netns %s: %w", sandbox.netNSPath, err) + } + + serverURL := &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(podNodeIPs[0].String(), s.daemonPort), + Path: daemon.AgentURLPath, + } + + errCh := make(chan error) + go func() { + defer close(errCh) + + if err := sandbox.agentProxy.Start(context.Background(), serverURL); err != nil { + logger.Printf("error running agent proxy: %v", err) + errCh <- err + } + }() + + select { + case <-ctx.Done(): + _ = sandbox.agentProxy.Shutdown() + return nil, ctx.Err() + case err := <-errCh: + return nil, err + case <-sandbox.agentProxy.Ready(): + } + + logger.Printf("agent proxy is ready") + return &pb.StartVMResponse{}, nil +} + +func (s *hypervisorService) getSandbox(id string) (*sandbox, error) { + + sid := sandboxID(id) + + if id == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; !exists { + return nil, fmt.Errorf("sandbox %s does not exist", sid) + } + return s.sandboxes[sid], nil +} + +func (s *hypervisorService) deleteSandbox(id string) error { + sid := sandboxID(id) + if id == "" { + return errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + delete(s.sandboxes, sid) + return nil +} + +func getIPs(instance *vmConfig) ([]net.IP, error) { + + return instance.ips, nil +} + +func (s *hypervisorService) deleteInstance(ctx context.Context, id string) error { + + err := DeleteInstance(ctx, s.libvirtClient, id) + if err != nil { + logger.Printf("failed to delete the instance (%s): %v", id, err) + return err + } + logger.Printf("deleted the instance (%s)", id) + return nil +} + +func (s *hypervisorService) StopVM(ctx context.Context, req *pb.StopVMRequest) (*pb.StopVMResponse, error) { + logger.Printf("Stopping VM") + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + if err := sandbox.agentProxy.Shutdown(); err != nil { + logger.Printf("failed to stop agent proxy: %v", err) + } + + if err := s.deleteInstance(ctx, sandbox.vsi); err != nil { + return nil, err + } + + if err := s.workerNode.Teardown(sandbox.netNSPath, sandbox.podNetworkConfig); err != nil { + return nil, fmt.Errorf("failed to tear down netns %s: %w", sandbox.netNSPath, err) + } + err = s.deleteSandbox(req.Id) + if err != nil { + return nil, err + } + return &pb.StopVMResponse{}, nil +} diff --git a/pkg/adaptor/hypervisor/libvirt/stream.go b/pkg/adaptor/hypervisor/libvirt/stream.go new file mode 100644 index 000000000..26b3bca74 --- /dev/null +++ b/pkg/adaptor/hypervisor/libvirt/stream.go @@ -0,0 +1,38 @@ +//go:build libvirt +// +build libvirt + +package libvirt + +// Code copied from https://github.com/openshift/cluster-api-provider-libvirt + +import ( + "io" + libvirt "libvirt.org/go/libvirt" +) + +// StreamIO libvirt struct +type streamIO struct { + stream libvirt.Stream +} + +var _ io.Writer = &streamIO{} +var _ io.Reader = &streamIO{} +var _ io.Closer = &streamIO{} + +// NewStreamIO returns libvirt StreamIO +func newStreamIO(s libvirt.Stream) *streamIO { + return &streamIO{stream: s} +} + +func (sio *streamIO) Read(p []byte) (int, error) { + return sio.stream.Recv(p) +} + +func (sio *streamIO) Write(p []byte) (int, error) { + return sio.stream.Send(p) +} + +// Close closes the stream +func (sio *streamIO) Close() error { + return sio.stream.Finish() +} diff --git a/pkg/adaptor/hypervisor/libvirt/types.go b/pkg/adaptor/hypervisor/libvirt/types.go new file mode 100644 index 000000000..80b9947c8 --- /dev/null +++ b/pkg/adaptor/hypervisor/libvirt/types.go @@ -0,0 +1,28 @@ +package libvirt + +import ( + "net" +) + +type Config struct { + URI string + PoolName string + NetworkName string + DataDir string +} + +type vmConfig struct { + name string + num uint8 + cpu uint + mem uint + rootDiskSize uint64 + userData string + metaData string + ips []net.IP + instanceId string //keeping it consistent with sandbox.vsi +} + +type createInstanceOutput struct { + instance *vmConfig +} diff --git a/pkg/adaptor/hypervisor/libvirt/volume.go b/pkg/adaptor/hypervisor/libvirt/volume.go new file mode 100644 index 000000000..2ab4045d6 --- /dev/null +++ b/pkg/adaptor/hypervisor/libvirt/volume.go @@ -0,0 +1,361 @@ +//go:build libvirt +// +build libvirt + +package libvirt + +// Code copied from https://github.com/openshift/cluster-api-provider-libvirt + +import ( + "encoding/xml" + "errors" + "fmt" + "io" + libvirt "libvirt.org/go/libvirt" + libvirtxml "libvirt.org/go/libvirtxml" + "strconv" + "strings" + "time" +) + +// ErrVolumeNotFound is returned when a domain is not found +var ErrVolumeNotFound = errors.New("Domain not found") + +var waitSleepInterval = 1 * time.Second + +// waitTimeout time +var waitTimeout = 5 * time.Minute + +// waitForSuccess wait for success and timeout after 5 minutes. +func waitForSuccess(errorMessage string, f func() error) error { + start := time.Now() + for { + err := f() + if err == nil { + return nil + } + logger.Printf("%s. Re-trying.\n", err) + + time.Sleep(waitSleepInterval) + if time.Since(start) > waitTimeout { + return fmt.Errorf("%s: %s", errorMessage, err) + } + } +} + +func newDefVolume(name string) libvirtxml.StorageVolume { + return libvirtxml.StorageVolume{ + Name: name, + Target: &libvirtxml.StorageVolumeTarget{ + Format: &libvirtxml.StorageVolumeTargetFormat{ + Type: "qcow2", + }, + Permissions: &libvirtxml.StorageVolumeTargetPermissions{ + Mode: "644", + }, + }, + Capacity: &libvirtxml.StorageVolumeSize{ + Unit: "bytes", + Value: 1, + }, + } +} + +func newDefBackingStoreFromLibvirt(baseVolume *libvirt.StorageVol) (libvirtxml.StorageVolumeBackingStore, error) { + baseVolumeDef, err := newDefVolumeFromLibvirt(baseVolume) + if err != nil { + return libvirtxml.StorageVolumeBackingStore{}, fmt.Errorf("could not get volume: %s", err) + } + baseVolPath, err := baseVolume.GetPath() + if err != nil { + return libvirtxml.StorageVolumeBackingStore{}, fmt.Errorf("could not get base image path: %s", err) + } + backingStoreDef := libvirtxml.StorageVolumeBackingStore{ + Path: baseVolPath, + Format: &libvirtxml.StorageVolumeTargetFormat{ + Type: baseVolumeDef.Target.Format.Type, + }, + } + return backingStoreDef, nil +} + +func newDefVolumeFromLibvirt(volume *libvirt.StorageVol) (libvirtxml.StorageVolume, error) { + name, err := volume.GetName() + if err != nil { + return libvirtxml.StorageVolume{}, fmt.Errorf("could not get name for volume: %s", err) + } + volumeDefXML, err := volume.GetXMLDesc(0) + if err != nil { + return libvirtxml.StorageVolume{}, fmt.Errorf("could not get XML description for volume %s: %s", name, err) + } + volumeDef, err := newDefVolumeFromXML(volumeDefXML) + if err != nil { + return libvirtxml.StorageVolume{}, fmt.Errorf("could not get a volume definition from XML for %s: %s", volumeDef.Name, err) + } + return volumeDef, nil +} + +// Creates a volume definition from a XML +func newDefVolumeFromXML(s string) (libvirtxml.StorageVolume, error) { + var volumeDef libvirtxml.StorageVolume + err := xml.Unmarshal([]byte(s), &volumeDef) + if err != nil { + return libvirtxml.StorageVolume{}, err + } + return volumeDef, nil +} + +func timeFromEpoch(str string) time.Time { + var s, ns int + + ts := strings.Split(str, ".") + if len(ts) == 2 { + ns, _ = strconv.Atoi(ts[1]) + } + s, _ = strconv.Atoi(ts[0]) + + return time.Unix(int64(s), int64(ns)) +} + +func uploadVolume(libvirtClient *libvirtClient, volumeDef libvirtxml.StorageVolume, img image) (volumeKey string, err error) { + + // Refresh the pool of the volume so that libvirt knows it is + // not longer in use. + err = waitForSuccess("Error refreshing pool for volume", func() error { + return libvirtClient.pool.Refresh(0) + }) + if err != nil { + return "", fmt.Errorf("timeout when calling waitForSuccess: %v", err) + } + + volumeDefXML, err := xml.Marshal(volumeDef) + if err != nil { + return "", fmt.Errorf("Error serializing libvirt volume: %s", err) + } + // create the volume + volume, err := libvirtClient.pool.StorageVolCreateXML(string(volumeDefXML), 0) + if err != nil { + return "", fmt.Errorf("Error creating libvirt volume for device %s: %s", volumeDef.Name, err) + } + defer freeVolume(volume, &err) + + // upload ISO file + err = img.importImage(newCopier(libvirtClient.connection, volume, volumeDef.Capacity.Value), volumeDef) + if err != nil { + return "", fmt.Errorf("Error while uploading volume %s: %s", img.string(), err) + } + + volumeKey, err = volume.GetKey() + if err != nil { + return "", fmt.Errorf("Error retrieving volume key: %s", err) + } + logger.Printf("Volume ID: %s", volumeKey) + return volumeKey, nil +} + +func newCopier(conn *libvirt.Connect, volume *libvirt.StorageVol, size uint64) func(src io.Reader) error { + copier := func(src io.Reader) (err error) { + var bytesCopied int64 + + stream, err := conn.NewStream(0) + if err != nil { + return err + } + + defer func() { + var newErr error + if uint64(bytesCopied) != size { + newErr = stream.Abort() + } else { + newErr = stream.Finish() + } + if newErr != nil && err == nil { + err = newErr + } + newErr = stream.Free() + if newErr != nil && err == nil { + err = newErr + } + }() + + if err = volume.Upload(stream, 0, size, 0); err != nil { + return err + } + + sio := newStreamIO(*stream) + + bytesCopied, err = io.Copy(sio, src) + if err != nil { + return err + } + logger.Printf("%d bytes uploaded\n", bytesCopied) + return nil + } + return copier +} + +func createVolume(volName string, volSize uint64, baseVolName string, libvirtClient *libvirtClient) (err error) { + volumeDef := newDefVolume(volName) + volumeDef.Target.Format.Type = "qcow2" + + baseVolume, err := getVolume(libvirtClient, baseVolName) + + if err != nil { + return fmt.Errorf("Can't retrieve volume %s", baseVolName) + } + defer freeVolume(baseVolume, &err) + + var baseVolumeInfo *libvirt.StorageVolInfo + baseVolumeInfo, err = baseVolume.GetInfo() + if err != nil { + return fmt.Errorf("Can't retrieve volume info %s", baseVolName) + } + + if baseVolumeInfo.Capacity > volSize { + volumeDef.Capacity.Value = baseVolumeInfo.Capacity + } else { + volumeDef.Capacity.Value = volSize + } + + backingStoreDef, err := newDefBackingStoreFromLibvirt(baseVolume) + if err != nil { + return fmt.Errorf("Could not retrieve backing store %s", baseVolName) + } + volumeDef.BackingStore = &backingStoreDef + + volumeDefXML, err := xml.Marshal(volumeDef) + if err != nil { + return fmt.Errorf("Error serializing libvirt volume: %s", err) + } + + // create the volume + // Refresh the pool of the volume so that libvirt knows it is + // not longer in use. + err = waitForSuccess("error refreshing pool for volume", func() error { + return libvirtClient.pool.Refresh(0) + }) + if err != nil { + return fmt.Errorf("can't find storage pool '%s'", libvirtClient.poolName) + } + + volume, err := libvirtClient.pool.StorageVolCreateXML(string(volumeDefXML), 0) + if err != nil { + return fmt.Errorf("Error creating libvirt volume: %s", err) + } + defer freeVolume(volume, &err) + + // we use the key as the id + key, err := volume.GetKey() + if err != nil { + return fmt.Errorf("Error retrieving volume key: %s", err) + } + + logger.Printf("Uploaded volume key %s", key) + return nil + +} + +func getVolume(libvirtClient *libvirtClient, volumeName string) (*libvirt.StorageVol, error) { + // Check whether the storage volume exists. Its name needs to be + // unique. + volume, err := libvirtClient.pool.LookupStorageVolByName(volumeName) + if err != nil { + // Let's try by ID in case of older Installer + volume, err = libvirtClient.connection.LookupStorageVolByKey(volumeName) + if err != nil { + return nil, fmt.Errorf("can't retrieve volume %q: %v", volumeName, err) + } + } + return volume, nil +} + +// VolumeExists checks if a volume exists +func volumeExists(libvirtClient *libvirtClient, volumeName string) (exist bool, err error) { + + logger.Printf("Check if %s volume exists", volumeName) + volume, err := getVolume(libvirtClient, volumeName) + if err != nil { + return false, nil + } + defer freeVolume(volume, &err) + + return true, nil +} + +func deleteVolumeByPath(libvirtClient *libvirtClient, path string) (err error) { + + // Get volume name from path + + volume, err := libvirtClient.connection.LookupStorageVolByPath(path) + if err != nil { + logger.Printf("can't retrieve volume %q: %v", path, err) + return err + } + + defer freeVolume(volume, &err) + + // Get name + name, err := volume.GetName() + if err != nil { + logger.Printf("Error retrieving volume name: %s", err) + return err + } + + return deleteVolume(libvirtClient, name) + +} + +func deleteVolume(libvirtClient *libvirtClient, name string) (err error) { + exists, err := volumeExists(libvirtClient, name) + if err != nil { + logger.Printf("Unable to check if volume (%s) exists", name) + return err + } + if !exists { + logger.Printf("Volume %s does not exists", name) + return ErrVolumeNotFound + } + logger.Printf("Deleting volume %s", name) + + volume, err := getVolume(libvirtClient, name) + if err != nil { + return fmt.Errorf("Can't retrieve volume %s", name) + } + defer freeVolume(volume, &err) + + // Refresh the pool of the volume so that libvirt knows it is + // not longer in use. + volPool, err := volume.LookupPoolByVolume() + if err != nil { + return fmt.Errorf("Error retrieving pool for volume: %s", err) + } + defer func() { + newErr := volPool.Free() + if newErr != nil && err == nil { + err = newErr + } + }() + + err = waitForSuccess("Error refreshing pool for volume", func() error { + return volPool.Refresh(0) + }) + if err != nil { + return fmt.Errorf("timeout when calling waitForSuccess: %v", err) + } + + err = volume.Delete(0) + if err != nil { + return fmt.Errorf("Can't delete volume %s: %s", name, err) + } + + return nil +} + +// freeVolume releases the volume pointer. If the operation fail and the error +// context is nil then it gets updated, otherwise it preserve the pointer to +// keep any previous error reported. +func freeVolume(volume *libvirt.StorageVol, errCtx *error) { + newErr := volume.Free() + if newErr != nil && *errCtx == nil { + *errCtx = newErr + } +} diff --git a/pkg/adaptor/hypervisor/registry/aws.go b/pkg/adaptor/hypervisor/registry/aws.go new file mode 100644 index 000000000..358e25c85 --- /dev/null +++ b/pkg/adaptor/hypervisor/registry/aws.go @@ -0,0 +1,13 @@ +// +build aws + +package registry + +import ( + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/aws" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" +) + +func newServer(cfg hypervisor.Config, cloudConfig interface{}, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + return aws.NewServer(cfg, cloudConfig.(aws.Config), workerNode, daemonPort) +} diff --git a/pkg/adaptor/hypervisor/registry/azure.go b/pkg/adaptor/hypervisor/registry/azure.go new file mode 100644 index 000000000..32a2580b6 --- /dev/null +++ b/pkg/adaptor/hypervisor/registry/azure.go @@ -0,0 +1,15 @@ +//go:build azure +// +build azure + +package registry + +import ( + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/azure" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" +) + +//nolint:typecheck +func newServer(cfg hypervisor.Config, cloudConfig interface{}, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + return azure.NewServer(cfg, cloudConfig.(azure.Config), workerNode, daemonPort) +} diff --git a/pkg/adaptor/hypervisor/registry/ibmcloud.go b/pkg/adaptor/hypervisor/registry/ibmcloud.go new file mode 100644 index 000000000..d8c56369c --- /dev/null +++ b/pkg/adaptor/hypervisor/registry/ibmcloud.go @@ -0,0 +1,13 @@ +// +build ibmcloud + +package registry + +import ( + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/ibmcloud" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" +) + +func newServer(cfg hypervisor.Config, cloudConfig interface{}, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + return ibmcloud.NewServer(cfg, cloudConfig.(ibmcloud.Config), workerNode, daemonPort) +} diff --git a/pkg/adaptor/hypervisor/registry/libvirt.go b/pkg/adaptor/hypervisor/registry/libvirt.go new file mode 100644 index 000000000..3aa81183a --- /dev/null +++ b/pkg/adaptor/hypervisor/registry/libvirt.go @@ -0,0 +1,13 @@ +// +build libvirt + +package registry + +import ( + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/libvirt" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" +) + +func newServer(cfg hypervisor.Config, cloudConfig interface{}, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + return libvirt.NewServer(cfg, cloudConfig.(libvirt.Config), workerNode, daemonPort) +} diff --git a/pkg/adaptor/hypervisor/registry/register.go b/pkg/adaptor/hypervisor/registry/register.go new file mode 100644 index 000000000..68c97a0bc --- /dev/null +++ b/pkg/adaptor/hypervisor/registry/register.go @@ -0,0 +1,10 @@ +package registry + +import ( + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" +) + +func NewServer(cfg hypervisor.Config, cloudConfig interface{}, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + return newServer(cfg, cloudConfig, workerNode, daemonPort) +} diff --git a/pkg/adaptor/hypervisor/registry/vsphere.go b/pkg/adaptor/hypervisor/registry/vsphere.go new file mode 100644 index 000000000..8e3fcc6ee --- /dev/null +++ b/pkg/adaptor/hypervisor/registry/vsphere.go @@ -0,0 +1,14 @@ +//go:build vsphere +// +build vsphere + +package registry + +import ( + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor/vsphere" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" +) + +func newServer(cfg hypervisor.Config, cloudConfig interface{}, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + return vsphere.NewServer(cfg, cloudConfig.(vsphere.Config), workerNode, daemonPort) +} diff --git a/pkg/adaptor/hypervisor/types.go b/pkg/adaptor/hypervisor/types.go new file mode 100644 index 000000000..fcf22f4f8 --- /dev/null +++ b/pkg/adaptor/hypervisor/types.go @@ -0,0 +1,25 @@ +package hypervisor + +import ( + "context" +) + +type Server interface { + Start(ctx context.Context) error + Shutdown() error + Ready() chan struct{} +} + +const ( + DefaultSocketPath = "/run/peerpod/hypervisor.sock" + DefaultPodsDir = "/run/peerpod/pods" + DefaultPauseImage = "k8s.gcr.io/pause:3.7" +) + +type Config struct { + SocketPath string + CriSocketPath string + PauseImage string + PodsDir string + HypProvider string +} diff --git a/pkg/adaptor/hypervisor/vsphere/server.go b/pkg/adaptor/hypervisor/vsphere/server.go new file mode 100644 index 000000000..abbf2b77c --- /dev/null +++ b/pkg/adaptor/hypervisor/vsphere/server.go @@ -0,0 +1,114 @@ +package vsphere + +import ( + "context" + "log" + "net" + "os" + "path/filepath" + "sync" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/containerd/ttrpc" + "github.com/vmware/govmomi" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" +) + +var logger = log.New(log.Writer(), "[helper/hypervisor] ", log.LstdFlags|log.Lmsgprefix) + +type server struct { + socketPath string + + ttRpc *ttrpc.Server + service pb.HypervisorService + + gclient *govmomi.Client + + workerNode podnetwork.WorkerNode + + readyCh chan struct{} + stopCh chan struct{} + stopOnce sync.Once +} + +func NewServer(cfg hypervisor.Config, vmcfg Config, workerNode podnetwork.WorkerNode, daemonPort string) hypervisor.Server { + + logger.Printf("hypervisor config %v", cfg) + logger.Printf("cloud config %v", vmcfg.Redact()) + + govmomiClient, err := NewGovmomiClient(vmcfg) + if err != nil { + logger.Printf("Error creating vcenter session for new server: %s", err) + return nil + } + + return &server{ + socketPath: cfg.SocketPath, + service: newService(govmomiClient, &vmcfg, &cfg, workerNode, cfg.PodsDir, daemonPort), + gclient: govmomiClient, + workerNode: workerNode, + readyCh: make(chan struct{}), + stopCh: make(chan struct{}), + } +} + +func (s *server) Start(ctx context.Context) (err error) { + + ttRpc, err := ttrpc.NewServer() + if err != nil { + return err + } + s.ttRpc = ttRpc + if err = os.MkdirAll(filepath.Dir(s.socketPath), os.ModePerm); err != nil { + return err + } + if err := os.RemoveAll(s.socketPath); err != nil { // just in case socket wasn't cleaned + return err + } + pb.RegisterHypervisorService(s.ttRpc, s.service) + listener, err := net.Listen("unix", s.socketPath) + if err != nil { + return err + } + + ttRpcErr := make(chan error) + go func() { + defer close(ttRpcErr) + if err = s.ttRpc.Serve(ctx, listener); err != nil { + ttRpcErr <- err + } + }() + defer func() { + newErr := s.ttRpc.Shutdown(context.Background()) + if newErr != nil && err == nil { + err = newErr + } + }() + + close(s.readyCh) + + select { + case <-ctx.Done(): + err = s.Shutdown() + case <-s.stopCh: + case err = <-ttRpcErr: + } + return err +} + +func (s *server) Shutdown() error { + + DeleteGovmomiClient(s.gclient) + + s.stopOnce.Do(func() { + close(s.stopCh) + }) + + return nil +} + +func (s *server) Ready() chan struct{} { + return s.readyCh +} diff --git a/pkg/adaptor/hypervisor/vsphere/service.go b/pkg/adaptor/hypervisor/vsphere/service.go new file mode 100644 index 000000000..9eb08fd8a --- /dev/null +++ b/pkg/adaptor/hypervisor/vsphere/service.go @@ -0,0 +1,324 @@ +package vsphere + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/vmware/govmomi" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/hypervisor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/proxy" + daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/cloudinit" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/hvutil" + "github.com/containerd/containerd/pkg/cri/annotations" + + pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" +) + +const ( + Version = "0.0.0" + VIM25LaunchTemplateName = "kata" +) + +type hypervisorService struct { + govmomiClient *govmomi.Client + serviceConfig *Config + hypervisorConfig *hypervisor.Config + sandboxes map[sandboxID]*sandbox + podsDir string + daemonPort string + nodeName string + workerNode podnetwork.WorkerNode + sync.Mutex +} + +func newService(govmomiClient *govmomi.Client, config *Config, hypervisorConfig *hypervisor.Config, workerNode podnetwork.WorkerNode, podsDir, daemonPort string) pb.HypervisorService { + logger.Printf("service config %v", config) + hostname, err := os.Hostname() + if err != nil { + panic(fmt.Errorf("failed to get hostname: %w", err)) + } + + i := strings.Index(hostname, ".") + if i >= 0 { + hostname = hostname[0:i] + } + + return &hypervisorService{ + govmomiClient: govmomiClient, + serviceConfig: config, + hypervisorConfig: hypervisorConfig, + sandboxes: map[sandboxID]*sandbox{}, + podsDir: podsDir, + daemonPort: daemonPort, + nodeName: hostname, + workerNode: workerNode, + } +} + +type sandboxID string + +type sandbox struct { + id sandboxID + pod string + namespace string + netNSPath string + podDirPath string + vsi string + agentProxy proxy.AgentProxy + podNetworkConfig *tunneler.Config +} + +func (s *hypervisorService) Version(ctx context.Context, req *pb.VersionRequest) (*pb.VersionResponse, error) { + return &pb.VersionResponse{Version: Version}, nil +} + +func (s *hypervisorService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (*pb.CreateVMResponse, error) { + + logger.Printf("CreateVM") + + sid := sandboxID(req.Id) + + if sid == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; exists { + return nil, fmt.Errorf("sandbox %s already exists", sid) + } + pod := hvutil.GetPodName(req.Annotations) + if pod == "" { + return nil, fmt.Errorf("pod name %s is missing in annotations", annotations.SandboxName) + } + + namespace := hvutil.GetPodNamespace(req.Annotations) + if namespace == "" { + return nil, fmt.Errorf("namespace name %s is missing in annotations", annotations.SandboxNamespace) + } + + podDirPath := filepath.Join(s.podsDir, string(sid)) + if err := os.MkdirAll(podDirPath, os.ModePerm); err != nil { + return nil, fmt.Errorf("failed to create a pod directory: %s: %w", podDirPath, err) + } + + socketPath := filepath.Join(podDirPath, proxy.SocketName) + + netNSPath := req.NetworkNamespacePath + + podNetworkConfig, err := s.workerNode.Inspect(netNSPath) + if err != nil { + return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err) + } + + agentProxy := proxy.NewAgentProxy(socketPath, s.hypervisorConfig.CriSocketPath, s.hypervisorConfig.PauseImage) + + sandbox := &sandbox{ + id: sid, + pod: pod, + namespace: namespace, + netNSPath: netNSPath, + podDirPath: podDirPath, + agentProxy: agentProxy, + podNetworkConfig: podNetworkConfig, + } + + s.sandboxes[sid] = sandbox + + // TODO: Specify the maximum instance name length in vSphere + vmname := hvutil.CreateInstanceName(sandbox.pod, string(sandbox.id), 0) + + logger.Printf("create a sandbox %s for pod %s in namespace %s (netns: %s)", req.Id, pod, namespace, sandbox.netNSPath) + + logger.Printf("CreateVM %s done", vmname) + + return &pb.CreateVMResponse{AgentSocketPath: socketPath}, nil +} + +func (s *hypervisorService) StartVM(ctx context.Context, req *pb.StartVMRequest) (*pb.StartVMResponse, error) { + + err := CheckSessionWithRestore(ctx, s.serviceConfig, s.govmomiClient) + if err != nil { + logger.Printf("StartVM cannot find or create a new vcenter session") + return nil, err + } + + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + // TODO: Specify the maximum instance name length in vSphere + vmname := hvutil.CreateInstanceName(sandbox.pod, string(sandbox.id), 0) + logger.Printf("StartVM %s", vmname) + + daemonConfig := daemon.Config{ + PodNamespace: sandbox.namespace, + PodName: sandbox.pod, + PodNetwork: sandbox.podNetworkConfig, + } + daemonJSON, err := json.MarshalIndent(daemonConfig, "", " ") + if err != nil { + return nil, err + } + + // Store daemon.json in worker node for debugging + if err = os.WriteFile(filepath.Join(sandbox.podDirPath, "daemon.json"), daemonJSON, 0666); err != nil { + return nil, fmt.Errorf("failed to store daemon.json at %s: %w", sandbox.podDirPath, err) + } + + cloudConfig := &cloudinit.CloudConfig{ + WriteFiles: []cloudinit.WriteFile{ + { + Path: daemon.DefaultConfigPath, + Content: string(daemonJSON), + }, + }, + } + + if authJSON, err := os.ReadFile(cloudinit.DefaultAuthfileSrcPath); err == nil { + if json.Valid(authJSON) && (len(authJSON) < cloudinit.DefaultAuthfileLimit) { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, + cloudinit.WriteFile{ + Path: cloudinit.DefaultAuthfileDstPath, + Content: cloudinit.AuthJSONToResourcesJSON(string(authJSON)), + }) + } else if len(authJSON) >= cloudinit.DefaultAuthfileLimit { + logger.Printf("Credentials file size (%d) is too large to use as userdata, ignored", len(authJSON)) + } else { + logger.Printf("Credentials file is not in a valid Json format, ignored") + } + } + + userData, err := cloudConfig.Generate() + if err != nil { + return nil, err + } + + result, err := CreateInstance(ctx, s.govmomiClient.Client, s.serviceConfig, vmname, userData) + if err != nil { + return nil, fmt.Errorf("creating instance for vm %s returned error: %s", vmname, err) + } + + sandbox.vsi = result.uuid + + if err := s.workerNode.Setup(sandbox.netNSPath, result.ips, sandbox.podNetworkConfig); err != nil { + return nil, fmt.Errorf("failed to set up pod network tunnel on netns %s: %w", sandbox.netNSPath, err) + } + + serverURL := &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(result.ips[0].String(), s.daemonPort), + Path: daemon.AgentURLPath, + } + + errCh := make(chan error) + go func() { + defer close(errCh) + + if err := sandbox.agentProxy.Start(context.Background(), serverURL); err != nil { + logger.Printf("error running agent proxy: %v", err) + errCh <- err + } + }() + + select { + case <-ctx.Done(): + _ = sandbox.agentProxy.Shutdown() + return nil, ctx.Err() + case err := <-errCh: + return nil, err + case <-sandbox.agentProxy.Ready(): + } + + logger.Printf("StartVM %s done", vmname) + + return &pb.StartVMResponse{}, nil +} + +func (s *hypervisorService) getSandbox(id string) (*sandbox, error) { + + sid := sandboxID(id) + + if id == "" { + return nil, errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + if _, exists := s.sandboxes[sid]; !exists { + return nil, fmt.Errorf("sandbox %s does not exist", sid) + } + return s.sandboxes[sid], nil +} + +func (s *hypervisorService) deleteSandbox(id string) error { + sid := sandboxID(id) + if id == "" { + return errors.New("empty sandbox id") + } + s.Lock() + defer s.Unlock() + delete(s.sandboxes, sid) + return nil +} + +func (s *hypervisorService) deleteInstance(ctx context.Context, id string, vmname string) error { + + err := DeleteInstance(ctx, s.govmomiClient.Client, s.serviceConfig, vmname) + if err != nil { + logger.Printf("failed to delete the instance (%s): %v", id, err) + return err + } + + return nil +} + +func (s *hypervisorService) StopVM(ctx context.Context, req *pb.StopVMRequest) (*pb.StopVMResponse, error) { + + err := CheckSessionWithRestore(ctx, s.serviceConfig, s.govmomiClient) + if err != nil { + logger.Printf("StopVM cannot find or create a new vcenter session") + return nil, err + } + + sandbox, err := s.getSandbox(req.Id) + if err != nil { + return nil, err + } + + // TODO: Specify the maximum instance name length in vSphere + vmname := hvutil.CreateInstanceName(sandbox.pod, string(sandbox.id), 0) + + logger.Printf("StopVM %s", vmname) + + if err := sandbox.agentProxy.Shutdown(); err != nil { + logger.Printf("failed to stop agent proxy: %v", err) + } + + if err := s.deleteInstance(ctx, sandbox.vsi, vmname); err != nil { + return nil, err + } + + if err := s.workerNode.Teardown(sandbox.netNSPath, sandbox.podNetworkConfig); err != nil { + return nil, fmt.Errorf("failed to tear down netns %s: %w", sandbox.netNSPath, err) + } + + err = s.deleteSandbox(req.Id) + if err != nil { + return nil, err + } + + logger.Printf("StopVM %s done", vmname) + return &pb.StopVMResponse{}, nil +} diff --git a/pkg/adaptor/hypervisor/vsphere/types.go b/pkg/adaptor/hypervisor/vsphere/types.go new file mode 100644 index 000000000..96a96c791 --- /dev/null +++ b/pkg/adaptor/hypervisor/vsphere/types.go @@ -0,0 +1,29 @@ +package vsphere + +import ( + "net" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/util" +) + +type Config struct { + VcenterURL string + UserName string + Password string + Insecure bool + Datacenter string + Vcluster string + Datastore string + Resourcepool string + Deployfolder string + Template string // template will be hardcoded podvm-template for now +} + +type createInstanceOutput struct { + uuid string + ips []net.IP +} + +func (c Config) Redact() Config { + return *util.RedactStruct(&c, "UserName", "Password").(*Config) +} diff --git a/pkg/adaptor/hypervisor/vsphere/vsphere.go b/pkg/adaptor/hypervisor/vsphere/vsphere.go new file mode 100644 index 000000000..b400b9d3b --- /dev/null +++ b/pkg/adaptor/hypervisor/vsphere/vsphere.go @@ -0,0 +1,352 @@ +package vsphere + +import ( + "context" + "encoding/base64" + "fmt" + "net" + "net/url" + "path" + "strings" + "sync" + "time" + + "github.com/vmware/govmomi" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/session/keepalive" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/methods" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" +) + +var glock sync.Mutex + +func NewGovmomiClient(vmcfg Config) (*govmomi.Client, error) { + + ctx := context.TODO() + + urlinfo, err := soap.ParseURL(vmcfg.VcenterURL) + if err != nil { + return nil, err + } + + // TODO make credentials secure + urlinfo.User = url.UserPassword(vmcfg.UserName, vmcfg.Password) + + soapClient := soap.NewClient(urlinfo, vmcfg.Insecure) + + vim25Client, err := vim25.NewClient(ctx, soapClient) + if err != nil { + return nil, err + } + + vim25Client.RoundTripper = keepalive.NewHandlerSOAP(vim25Client.RoundTripper, 1*time.Minute, soapKeepAliveHandler(ctx, vim25Client)) + + manager := session.NewManager(vim25Client) + err = manager.Login(ctx, urlinfo.User) + if err != nil { + return nil, err + } + + logger.Printf("Created session for user %s", vmcfg.UserName) + + gclient := govmomi.Client{ + Client: vim25Client, + SessionManager: manager, + } + + return &gclient, nil +} + +func soapKeepAliveHandler(ctx context.Context, c *vim25.Client) func() error { + + return func() error { + + _, err := methods.GetCurrentTime(ctx, c) + if err != nil { + logger.Printf("SOAP keep-alive handler error %s", err) + return err + } + + return nil + } +} + +func DeleteGovmomiClient(gclient *govmomi.Client) { + + glock.Lock() + + defer glock.Unlock() + + err := gclient.SessionManager.Logout(context.Background()) + if err != nil { + logger.Printf("Vcenter logout failed error: %s", err) + } +} + +func CheckSessionWithRestore(ctx context.Context, vmcfg *Config, gclient *govmomi.Client) error { + + glock.Lock() + + defer glock.Unlock() + + active, err := gclient.SessionManager.SessionIsActive(ctx) + + if active { + if err == nil { + return nil + } + } + + if err != nil { + logger.Printf("Creating new sesssion for user %s due to current session error: %s", vmcfg.UserName, err) + } + + _ = gclient.SessionManager.Logout(ctx) // Cleanup purposes + + urlinfo, _ := soap.ParseURL(vmcfg.VcenterURL) + + urlinfo.User = url.UserPassword(vmcfg.UserName, vmcfg.Password) + + err = gclient.SessionManager.Login(ctx, urlinfo.User) + if err != nil { + logger.Printf("Vcenter login failed error: %s", err) + return err + } + + logger.Printf("Created new session for user %s", vmcfg.UserName) + + return nil +} + +type VmConfig []types.BaseOptionValue + +func CreateInstance(ctx context.Context, vim25Client *vim25.Client, vmcfg *Config, vmname string, userData string) (*createInstanceOutput, error) { + + // this creates and starts the VM + + finder := find.NewFinder(vim25Client) + + // If vmcfg.Datacenter is null DatacenterOrDefault will return the default datacenter + dc, err := finder.DatacenterOrDefault(ctx, vmcfg.Datacenter) + if err != nil { + logger.Printf("Cannot find vcenter datacenter %s error: %s", vmcfg.Datacenter, err) + return nil, err + } + + vmcfg.Datacenter = dc.Name() + + logger.Printf("Datacenter now is %s", vmcfg.Datacenter) + + finder.SetDatacenter(dc) + + // TODO change to tpl, err := template.FindTemplate(ctx, ctx.VSphereVM.Spec.Template) + // "github.com/vmware/govmomi/template" + + vm, err := finder.VirtualMachine(ctx, vmcfg.Template) + if err != nil { + logger.Printf("Cannot find VM template %s error: %s", vmcfg.Template, err) + return nil, err + } + + // TODO for DRS ResourcePoolOrDefault() + pool, err := finder.DefaultResourcePool(ctx) + if err != nil { + return nil, err + } + poolref := types.NewReference(pool.Reference()) + + // Logical ( not physical ) vm folder placement. If no folder exists it + // will be created in the current inventory path ie /your-current-datacenter/vm/newfolder/ + + inventory_path := path.Join(dc.InventoryPath, "vm") + + vmfolder, err := finder.Folder(ctx, inventory_path) + if err != nil { + logger.Printf("Cannot find inventory folder %s error: %s", inventory_path, err) + return nil, err + } + + deploy_path_dirs := strings.Split(vmcfg.Deployfolder, "/") + deploy_path := inventory_path + + for _, dir := range deploy_path_dirs { + deploy_path = path.Join(deploy_path, dir) + current_folder, err := finder.Folder(ctx, deploy_path) + if err != nil { + if _, ok := err.(*find.NotFoundError); ok { + current_folder, err = vmfolder.CreateFolder(ctx, dir) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + vmfolder = current_folder + } + + vmfolderref := vmfolder.Reference() + + relocateSpec := types.VirtualMachineRelocateSpec{ + Folder: &vmfolderref, + Pool: poolref, + } + + data := []byte(userData) + + for { + decoded, err := base64.StdEncoding.DecodeString(string(data)) + if err != nil { + break + } + data = decoded + } + + userDataEnc := base64.StdEncoding.EncodeToString(data) + + var extraconfig VmConfig + + extraconfig = append(extraconfig, + &types.OptionValue{ + Key: "guestinfo.userdata", + Value: userDataEnc, + }, + &types.OptionValue{ + Key: "guestinfo.userdata.encoding", + Value: "base64", + }, + ) + + configSpec := types.VirtualMachineConfigSpec{ + ExtraConfig: extraconfig, + } + + if vmcfg.Datastore != "" { + datastorepath := fmt.Sprintf("/%s/datastore/%s", vmcfg.Datacenter, vmcfg.Datastore) + datastore, err := finder.Datastore(ctx, datastorepath) + if err != nil { + return nil, err + } + datastoreref := types.NewReference(datastore.Reference()) + relocateSpec.Datastore = datastoreref + } + + cloneSpec := &types.VirtualMachineCloneSpec{ + Location: relocateSpec, + PowerOn: true, + Template: false, + Config: &configSpec, + } + + task, err := vm.Clone(ctx, vmfolder, vmname, *cloneSpec) + if err != nil { + logger.Printf("Can't clone to vm %s: %s", vmname, err) + return nil, err + } + + info, err := task.WaitForResult(ctx, nil) // TODO Fix to have a timeout + if err != nil { + logger.Printf("wait for clone task failed") + return nil, err + } + + clone := object.NewVirtualMachine(vim25Client, info.Result.(types.ManagedObjectReference)) + name, err := clone.ObjectName(ctx) + if err != nil { + return nil, err + } + + logger.Printf("clone %s, Clone UUID %s created", name, clone.UUID(ctx)) + + //uuid := strings.Replace(mvm.Config.Uuid, "-", "", -1) + + podNodeIPs, err := getIPs(clone) // TODO Fix to get all ips + if err != nil { + logger.Printf("failed to get IPs for the instance : %v ", err) + return nil, err + } + + return &createInstanceOutput{ + uuid: clone.UUID(ctx), + ips: podNodeIPs, + }, nil +} + +func getIPs(vm *object.VirtualMachine) ([]net.IP, error) { // TODO Fix to get all ips + var podNodeIPs []net.IP + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(300*time.Second)) + defer cancel() + + logger.Printf("start waiting for cloned vm ip") + ip, err := vm.WaitForIP(ctx, true) + if err != nil { + return nil, err + } + + logger.Printf("VM IP=%s", ip) + ip_item := net.ParseIP(ip) + if ip_item == nil { + return nil, fmt.Errorf("failed to parse pod node IP %q", ip) + } + podNodeIPs = append(podNodeIPs, ip_item) + + return podNodeIPs, nil +} + +func DeleteInstance(ctx context.Context, vim25Client *vim25.Client, vmcfg *Config, vmname string) (err error) { + + var ( + task *object.Task + state types.VirtualMachinePowerState + ) + + finder := find.NewFinder(vim25Client) + + // If vmcfg.Datacenter is null DatacenterOrDefault will return the default datacenter + dc, err := finder.DatacenterOrDefault(ctx, vmcfg.Datacenter) + + if err != nil { + logger.Printf("can't get vcenter datacenter %s", vmcfg.Datacenter) + return err + } + + finder.SetDatacenter(dc) + + vm_path := path.Join(dc.InventoryPath, "vm", vmcfg.Deployfolder, vmname) + + vm, err := finder.VirtualMachine(ctx, vm_path) //TODO may need find by UUID + if err != nil { + logger.Printf("can't find VM %s to delete it", vm_path) + return err + } + + state, err = vm.PowerState(ctx) + if err != nil { + return err + } + + if state == types.VirtualMachinePowerStatePoweredOn { + task, err = vm.PowerOff(ctx) + if err != nil { + return err + } + + // Ignore error since the VM may already been in powered off state. + // vm.Destroy will fail if the VM is still powered on. + _ = task.Wait(ctx) + } + + task, err = vm.Destroy(ctx) + if err != nil { + return err + } + + _ = task.Wait(ctx) + + return nil +} diff --git a/pkg/adaptor/proxy/proxy.go b/pkg/adaptor/proxy/proxy.go new file mode 100644 index 000000000..48f1f7bc3 --- /dev/null +++ b/pkg/adaptor/proxy/proxy.go @@ -0,0 +1,218 @@ +// Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package proxy + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "net/url" + "os" + "path/filepath" + "sync" + "time" + + "github.com/containerd/ttrpc" + pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + criapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" +) + +const ( + SocketName = "agent.ttrpc" + + defaultMaxRetries = 20 + defaultRetryInterval = 10 * time.Second +) + +var logger = log.New(log.Writer(), "[adaptor/proxy] ", log.LstdFlags|log.Lmsgprefix) + +type criClient struct { + criapi.ImageServiceClient +} + +type AgentProxy interface { + Start(ctx context.Context, serverURL *url.URL) error + Ready() chan struct{} + Shutdown() error +} + +type agentProxy struct { + readyCh chan struct{} + stopCh chan struct{} + stopOnce sync.Once + socketPath string + criSocketPath string + maxRetries int + retryInterval time.Duration + pauseImage string +} + +func NewAgentProxy(socketPath, criSocketPath string, pauseImage string) AgentProxy { + + return &agentProxy{ + socketPath: socketPath, + criSocketPath: criSocketPath, + readyCh: make(chan struct{}), + stopCh: make(chan struct{}), + maxRetries: defaultMaxRetries, + retryInterval: defaultRetryInterval, + pauseImage: pauseImage, + } +} + +func (p *agentProxy) dial(ctx context.Context, address string) (net.Conn, error) { + + var conn net.Conn + + maxRetries := defaultMaxRetries + count := 1 + for { + var err error + + func() { + ctx, cancel := context.WithTimeout(ctx, p.retryInterval) + defer cancel() + + // TODO: Support TLS + conn, err = (&net.Dialer{}).DialContext(ctx, "tcp", address) + + if err == nil || count == maxRetries { + return + } + <-ctx.Done() + }() + + if err == nil { + break + } + if count == maxRetries { + err := fmt.Errorf("reaches max retry count. gave up establishing agent proxy connection to %s: %w", address, err) + logger.Print(err) + return nil, err + } + logger.Printf("failed to establish agent proxy connection to %s: %v. (retrying... %d/%d)", address, err, count, p.maxRetries) + + count++ + } + + logger.Printf("established agent proxy connection to %s", address) + + return conn, nil +} + +func initCriClient(ctx context.Context, target string) (*criClient, error) { + if target != "" { + conn, err := grpc.DialContext(ctx, target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock(), + grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", target) + }), + ) + if err != nil { + return nil, err + } + + criClient := &criClient{ + ImageServiceClient: criapi.NewImageServiceClient(conn), + } + + logger.Printf("established cri uds connection to %s", target) + return criClient, nil + } + + return nil, fmt.Errorf("cri runtime endpoint is not specified, it is used to get the image name from image digest.") +} + +func (p *agentProxy) Start(ctx context.Context, serverURL *url.URL) error { + + if err := os.MkdirAll(filepath.Dir(p.socketPath), os.ModePerm); err != nil { + return fmt.Errorf("failed to create parent directories for socket: %s", p.socketPath) + } + if err := os.Remove(p.socketPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove %s: %w", p.socketPath, err) + } + + logger.Printf("Listening on %s\n", p.socketPath) + + listener, err := net.Listen("unix", p.socketPath) + if err != nil { + return fmt.Errorf("failed to listen on %s: %w", p.socketPath, err) + } + + dialer := func(ctx context.Context) (net.Conn, error) { + return p.dial(ctx, serverURL.Host) + } + + criClient, err := initCriClient(ctx, p.criSocketPath) + if err != nil { + // cri client is optional currently, we ignore any errors here + logger.Printf("failed to init cri client, the err: %v", err) + } + + proxyService := newProxyService(dialer, criClient, p.pauseImage) + defer func() { + if err := proxyService.Close(); err != nil { + logger.Printf("error closing agent proxy connection: %v", err) + } + }() + + if err := proxyService.Connect(ctx); err != nil { + return fmt.Errorf("error connecting to agent: %v", err) + } + + ttrpcServer, err := ttrpc.NewServer() + if err != nil { + return fmt.Errorf("failed to create TTRPC server: %w", err) + } + + pb.RegisterAgentServiceService(ttrpcServer, proxyService) + pb.RegisterImageService(ttrpcServer, proxyService) + pb.RegisterHealthService(ttrpcServer, proxyService) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + ttrpcServerErr := make(chan error) + go func() { + defer close(ttrpcServerErr) + + if err := ttrpcServer.Serve(ctx, listener); err != nil && !errors.Is(err, ttrpc.ErrServerClosed) { + ttrpcServerErr <- err + } + }() + defer func() { + if err := ttrpcServer.Shutdown(ctx); err != nil { + logger.Printf("error shutting down TTRPC server: %v", err) + } + }() + + close(p.readyCh) + + select { + case <-ctx.Done(): + if err := p.Shutdown(); err != nil { + logger.Printf("error on shutdown: %v", err) + } + case <-p.stopCh: + case err := <-ttrpcServerErr: + return err + } + + return nil +} + +func (p *agentProxy) Ready() chan struct{} { + return p.readyCh +} + +func (p *agentProxy) Shutdown() error { + logger.Printf("shutting down socket forwarder") + p.stopOnce.Do(func() { + close(p.stopCh) + }) + return nil +} diff --git a/pkg/adaptor/proxy/proxy_test.go b/pkg/adaptor/proxy/proxy_test.go new file mode 100644 index 000000000..1cacceb82 --- /dev/null +++ b/pkg/adaptor/proxy/proxy_test.go @@ -0,0 +1,341 @@ +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package proxy + +import ( + "context" + "errors" + "net" + "net/url" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/containerd/ttrpc" + "github.com/gogo/protobuf/types" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" + pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" +) + +func TestNewAgentProxy(t *testing.T) { + + socketPath := "/run/dummy.sock" + + proxy := NewAgentProxy(socketPath, "", "") + p, ok := proxy.(*agentProxy) + if !ok { + t.Fatalf("expect %T, got %T", &agentProxy{}, proxy) + } + if e, a := socketPath, p.socketPath; e != a { + t.Fatalf("expect %q, got %q", e, a) + } +} + +func TestStartStop(t *testing.T) { + + dir := t.TempDir() + + socketPath := filepath.Join(dir, "test.sock") + + agentServer, err := ttrpc.NewServer() + if err != nil { + t.Fatalf("expect no error, got %q", err) + } + pb.RegisterAgentServiceService(agentServer, &agentMock{}) + pb.RegisterImageService(agentServer, &agentMock{}) + pb.RegisterHealthService(agentServer, &agentMock{}) + + agentListener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("expect no error, got %q", err) + } + defer func() { + err := agentListener.Close() + if e, a := net.ErrClosed, err; !errors.Is(a, e) { + t.Fatalf("expect %q, got %q", e, a) + } + }() + + agentServerErrCh := make(chan error) + go func() { + defer close(agentServerErrCh) + + err := agentServer.Serve(context.Background(), agentListener) + if err != nil { + agentServerErrCh <- err + return + } + }() + defer func() { + err := agentServer.Shutdown(context.Background()) + if err != nil { + t.Fatalf("expect no error, got %q", err) + } + }() + + serverURL := &url.URL{ + Scheme: "grpc", + Host: agentListener.Addr().String(), + } + + proxy := NewAgentProxy(socketPath, "", "") + p, ok := proxy.(*agentProxy) + if !ok { + t.Fatalf("expect %T, got %T", &agentProxy{}, proxy) + } + + proxyErrCh := make(chan error) + go func() { + defer close(proxyErrCh) + + if err := proxy.Start(context.Background(), serverURL); err != nil { + proxyErrCh <- err + } + }() + defer func() { + if err := p.Shutdown(); err != nil { + t.Fatalf("expect no error, got %q", err) + } + }() + + select { + case err := <-proxyErrCh: + t.Fatalf("expect no error, got %q", err) + case <-proxy.Ready(): + } + + conn, err := net.Dial("unix", socketPath) + if err != nil { + t.Fatalf("expect no error, got %q", err) + } + + ttrpcClient := ttrpc.NewClient(conn) + + client := struct { + pb.AgentServiceService + pb.ImageService + pb.HealthService + }{ + AgentServiceService: pb.NewAgentServiceClient(ttrpcClient), + ImageService: pb.NewImageClient(ttrpcClient), + HealthService: pb.NewHealthClient(ttrpcClient), + } + + { + res, err := client.PullImage(context.Background(), &pb.PullImageRequest{Image: "abc", ContainerId: "123"}) + if err != nil { + t.Fatalf("expect no error, got %q", err) + } + if res == nil { + t.Fatal("expect non nil, got nil") + } + } + + { + res, err := client.CreateContainer(context.Background(), &pb.CreateContainerRequest{ContainerId: "123", OCI: &pb.Spec{Annotations: map[string]string{"aaa": "111"}}}) + if err != nil { + t.Fatalf("expect no error, got %q", err) + } + if res == nil { + t.Fatal("expect non nil, got nil") + } + } + + select { + case err := <-agentServerErrCh: + t.Fatalf("expect no error, got %q", err) + case err := <-proxyErrCh: + t.Fatalf("expect no error, got %q", err) + default: + } +} + +func TestDialerSuccess(t *testing.T) { + p := &agentProxy{ + maxRetries: 20, + retryInterval: 100 * time.Millisecond, + } + + for { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("expect no error, got %q", err) + } + + address := listener.Addr().String() + + if err := listener.Close(); err != nil { + t.Fatalf("expect no error, got %q", err) + } + + listenerErrCh := make(chan error) + go func() { + defer close(listenerErrCh) + + time.Sleep(250 * time.Millisecond) + + var err error + // Open the same port + listener, err = net.Listen("tcp", address) + if err != nil { + listenerErrCh <- err + } + }() + + conn, err := p.dial(context.Background(), address) + if err == nil { + listener.Close() + break + } + defer conn.Close() + + if e := <-listenerErrCh; e != nil { + // A rare case occurs. Retry the test. + t.Logf("%v", e) + continue + } + + listener.Close() + if err != nil { + t.Fatalf("expect no error, got %q", err) + } + break + } +} + +func TestDialerFailure(t *testing.T) { + p := &agentProxy{ + maxRetries: 5, + retryInterval: 100 * time.Millisecond, + } + + address := "0.0.0.0:0" + conn, err := p.dial(context.Background(), address) + if err == nil { + conn.Close() + t.Fatal("expect error, got nil") + } + + if e, a := "reaches max retry count", err.Error(); !strings.Contains(a, e) { + t.Fatalf("expect %q, got %q", e, a) + } +} + +type agentMock struct{} + +func (m *agentMock) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) ExecProcess(ctx context.Context, req *pb.ExecProcessRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) SignalProcess(ctx context.Context, req *pb.SignalProcessRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest) (*pb.WaitProcessResponse, error) { + return &pb.WaitProcessResponse{}, nil +} +func (m *agentMock) UpdateContainer(ctx context.Context, req *pb.UpdateContainerRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) StatsContainer(ctx context.Context, req *pb.StatsContainerRequest) (*pb.StatsContainerResponse, error) { + return &pb.StatsContainerResponse{}, nil +} +func (m *agentMock) PauseContainer(ctx context.Context, req *pb.PauseContainerRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) ResumeContainer(ctx context.Context, req *pb.ResumeContainerRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) WriteStdin(ctx context.Context, req *pb.WriteStreamRequest) (*pb.WriteStreamResponse, error) { + return &pb.WriteStreamResponse{}, nil +} +func (m *agentMock) ReadStdout(ctx context.Context, req *pb.ReadStreamRequest) (*pb.ReadStreamResponse, error) { + return &pb.ReadStreamResponse{}, nil +} +func (m *agentMock) ReadStderr(ctx context.Context, req *pb.ReadStreamRequest) (*pb.ReadStreamResponse, error) { + return &pb.ReadStreamResponse{}, nil +} +func (m *agentMock) CloseStdin(ctx context.Context, req *pb.CloseStdinRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) TtyWinResize(ctx context.Context, req *pb.TtyWinResizeRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) UpdateInterface(ctx context.Context, req *pb.UpdateInterfaceRequest) (*protocols.Interface, error) { + return &protocols.Interface{}, nil +} +func (m *agentMock) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) { + return &pb.Routes{}, nil +} +func (m *agentMock) ListInterfaces(ctx context.Context, req *pb.ListInterfacesRequest) (*pb.Interfaces, error) { + return &pb.Interfaces{}, nil +} +func (m *agentMock) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) (*pb.Routes, error) { + return &pb.Routes{}, nil +} +func (m *agentMock) AddARPNeighbors(ctx context.Context, req *pb.AddARPNeighborsRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) GetIPTables(ctx context.Context, req *pb.GetIPTablesRequest) (*pb.GetIPTablesResponse, error) { + return &pb.GetIPTablesResponse{}, nil +} +func (m *agentMock) SetIPTables(ctx context.Context, req *pb.SetIPTablesRequest) (*pb.SetIPTablesResponse, error) { + return &pb.SetIPTablesResponse{}, nil +} +func (m *agentMock) GetMetrics(ctx context.Context, req *pb.GetMetricsRequest) (*pb.Metrics, error) { + return &pb.Metrics{}, nil +} +func (m *agentMock) CreateSandbox(ctx context.Context, req *pb.CreateSandboxRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) DestroySandbox(ctx context.Context, req *pb.DestroySandboxRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) ReseedRandomDev(ctx context.Context, req *pb.ReseedRandomDevRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) GetGuestDetails(ctx context.Context, req *pb.GuestDetailsRequest) (*pb.GuestDetailsResponse, error) { + return &pb.GuestDetailsResponse{}, nil +} +func (m *agentMock) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByProbeRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) SetGuestDateTime(ctx context.Context, req *pb.SetGuestDateTimeRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) CopyFile(ctx context.Context, req *pb.CopyFileRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) GetOOMEvent(ctx context.Context, req *pb.GetOOMEventRequest) (*pb.OOMEvent, error) { + return &pb.OOMEvent{}, nil +} +func (m *agentMock) AddSwap(ctx context.Context, req *pb.AddSwapRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) GetVolumeStats(ctx context.Context, req *pb.VolumeStatsRequest) (*pb.VolumeStatsResponse, error) { + return &pb.VolumeStatsResponse{}, nil +} +func (m *agentMock) ResizeVolume(ctx context.Context, req *pb.ResizeVolumeRequest) (*types.Empty, error) { + return &types.Empty{}, nil +} +func (m *agentMock) PullImage(ctx context.Context, req *pb.PullImageRequest) (*pb.PullImageResponse, error) { + return &pb.PullImageResponse{}, nil +} +func (m *agentMock) Check(ctx context.Context, req *pb.CheckRequest) (*pb.HealthCheckResponse, error) { + return &pb.HealthCheckResponse{}, nil +} +func (m *agentMock) Version(ctx context.Context, req *pb.CheckRequest) (*pb.VersionCheckResponse, error) { + return &pb.VersionCheckResponse{}, nil +} diff --git a/pkg/adaptor/proxy/service.go b/pkg/adaptor/proxy/service.go new file mode 100644 index 000000000..205926332 --- /dev/null +++ b/pkg/adaptor/proxy/service.go @@ -0,0 +1,222 @@ +// Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package proxy + +import ( + "context" + "fmt" + "net" + "strings" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/agentproto" + cri "github.com/containerd/containerd/pkg/cri/annotations" + crio "github.com/containers/podman/v4/pkg/annotations" + "github.com/gogo/protobuf/types" + pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" + criapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" +) + +type proxyService struct { + agentproto.Redirector + criClient *criClient + pauseImage string +} + +func newProxyService(dialer func(context.Context) (net.Conn, error), criClient *criClient, pauseImage string) *proxyService { + + redirector := agentproto.NewRedirector(dialer) + + return &proxyService{ + Redirector: redirector, + criClient: criClient, + pauseImage: pauseImage, + } +} + +func (s *proxyService) getImageFromDigest(ctx context.Context, digest string) (string, error) { + if s.criClient == nil { + return "", fmt.Errorf("getImageFromDigest: criClient is nil.") + } + + req := &criapi.ListImagesRequest{} + resp, err := s.criClient.ImageServiceClient.ListImages(ctx, req) + if err != nil { + return "", err + } + + images := resp.GetImages() + for _, img := range images { + logger.Printf("imageTag: %s, image digest: %s", img.RepoTags[0], img.Id) + if img.Id == digest { + return img.RepoTags[0], nil + } + } + return "", fmt.Errorf("Did not find imageTag from image digest %s", digest) +} + +func (s *proxyService) getImageName(annotations map[string]string) (string, error) { + + logger.Printf("getImageName: check if its a sandbox type container") + for containerType, containerTypeSandbox := range map[string]string{ + cri.ContainerType: cri.ContainerTypeSandbox, + crio.ContainerType: crio.ContainerTypeSandbox, + } { + if annotations[containerType] == containerTypeSandbox { + logger.Printf("getImageName: pause image: %s", s.pauseImage) + return s.pauseImage, nil + } + } + + for _, a := range []string{cri.ImageName, crio.ImageName} { + if image, ok := annotations[a]; ok { + logger.Printf("getImageName: image: %s", image) + return image, nil + } + } + + return "", fmt.Errorf("container image name is not specified in annotations: %#v", annotations) +} + +// AgentServiceService methods + +func (s *proxyService) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (*types.Empty, error) { + + logger.Printf("CreateContainer: containerID:%s", req.ContainerId) + if len(req.OCI.Annotations) > 0 { + logger.Print(" annotations:") + for k, v := range req.OCI.Annotations { + logger.Printf(" %s: %s", k, v) + } + } + if len(req.OCI.Mounts) > 0 { + logger.Print(" mounts:") + for _, m := range req.OCI.Mounts { + logger.Printf(" destination:%s source:%s type:%s", m.Destination, m.Source, m.Type) + } + } + if len(req.Storages) > 0 { + logger.Print(" storages:") + for _, s := range req.Storages { + logger.Printf(" mount_point:%s source:%s fstype:%s driver:%s", s.MountPoint, s.Source, s.Fstype, s.Driver) + } + } + if len(req.Devices) > 0 { + logger.Print(" devices:") + for _, d := range req.Devices { + logger.Printf(" container_path:%s vm_path:%s type:%s", d.ContainerPath, d.VmPath, d.Type) + } + } + imageName, err := s.getImageName(req.OCI.Annotations) + if err != nil { + logger.Printf("CreateContainer: image name is not available in CreateContainerRequest: %v", err) + } else { + // Get the imageName from digest + if strings.HasPrefix(imageName, "sha256:") { + digest := imageName + logger.Printf("CreateContainer: get imageName from digest %q", digest) + imageName, err = s.getImageFromDigest(ctx, digest) + if err != nil { + return nil, err + } + } + + logger.Printf("CreateContainer: calling PullImage for %q before CreateContainer (cid: %q)", imageName, req.ContainerId) + + pullImageReq := &pb.PullImageRequest{ + Image: imageName, + } + + pullImageRes, pullImageErr := s.Redirector.PullImage(ctx, pullImageReq) + + if pullImageErr != nil { + logger.Printf("CreateContainer: failed to call PullImage, probably because the image has already been pulled. ignored: %v", pullImageErr) + } else { + logger.Printf("CreateContainer: successfully pulled image %q", pullImageRes.ImageRef) + } + + // kata-agent uses this annotation to fix the image bundle path + // https://github.com/kata-containers/kata-containers/blob/8ad86e2ec9d26d2ef07f3bf794352a3fda7597e5/src/agent/src/rpc.rs#L694-L696 + req.OCI.Annotations[cri.ImageName] = imageName + } + + res, err := s.Redirector.CreateContainer(ctx, req) + + if err != nil { + logger.Printf("CreateContainer fails: %v", err) + } + + return res, err +} + +func (s *proxyService) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*types.Empty, error) { + + logger.Printf("StartContainer: containerID:%s", req.ContainerId) + + res, err := s.Redirector.StartContainer(ctx, req) + + if err != nil { + logger.Printf("StartContainer fails: %v", err) + } + + return res, err +} + +func (s *proxyService) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*types.Empty, error) { + + logger.Printf("RemoveContainer: containerID:%s", req.ContainerId) + + res, err := s.Redirector.RemoveContainer(ctx, req) + + if err != nil { + logger.Printf("RemoveContainer fails: %v", err) + } + + return res, err +} + +func (s *proxyService) CreateSandbox(ctx context.Context, req *pb.CreateSandboxRequest) (*types.Empty, error) { + + logger.Printf("CreateSandbox: hostname:%s sandboxId:%s", req.Hostname, req.SandboxId) + + if len(req.Storages) > 0 { + logger.Print(" storages:") + for _, s := range req.Storages { + logger.Printf(" mountpoint:%s source:%s fstype:%s driver:%s", s.MountPoint, s.Source, s.Fstype, s.Driver) + } + } + + res, err := s.Redirector.CreateSandbox(ctx, req) + + if err != nil { + logger.Printf("CreateSandbox fails: %v", err) + } + + return res, err +} + +func (s *proxyService) DestroySandbox(ctx context.Context, req *pb.DestroySandboxRequest) (*types.Empty, error) { + + logger.Printf("DestroySandbox") + + res, err := s.Redirector.DestroySandbox(ctx, req) + + if err != nil { + logger.Printf("DestroySandbox fails: %v", err) + } + + return res, err +} + +func (s *proxyService) PullImage(ctx context.Context, req *pb.PullImageRequest) (*pb.PullImageResponse, error) { + + logger.Printf("PullImage: image:%s containerID:%s", req.Image, req.ContainerId) + + res, err := s.Redirector.PullImage(ctx, req) + + if err != nil { + logger.Printf("PullImage fails: %v", err) + } + + return res, err +} diff --git a/pkg/forwarder/forwarder.go b/pkg/forwarder/forwarder.go new file mode 100644 index 000000000..0a3601ac6 --- /dev/null +++ b/pkg/forwarder/forwarder.go @@ -0,0 +1,140 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package forwarder + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "sync" + + "github.com/containerd/ttrpc" + pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder/interceptor" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" +) + +var logger = log.New(log.Writer(), "[forwarder] ", log.LstdFlags|log.Lmsgprefix) + +const ( + DefaultListenHost = "0.0.0.0" + DefaultListenPort = "15150" + DefaultListenAddr = DefaultListenHost + ":" + DefaultListenPort + DefaultConfigPath = "/peerpod/daemon.json" + DefaultPodNetworkSpecPath = "/peerpod/podnetwork.json" + DefaultKataAgentSocketPath = "/run/kata-containers/agent.sock" + DefaultKataAgentNamespace = "" + AgentURLPath = "/agent" +) + +type Config struct { + PodNamespace string `json:"pod-namespace"` + PodName string `json:"pod-name"` + PodNetwork *tunneler.Config `json:"pod-network"` +} + +type Daemon interface { + Start(ctx context.Context) error + Shutdown() error + Addr() string +} + +type daemon struct { + listenAddr string + interceptor interceptor.Interceptor + podNode podnetwork.PodNode + + readyCh chan struct{} + stopCh chan struct{} + stopOnce sync.Once +} + +func NewDaemon(spec *Config, listenAddr string, interceptor interceptor.Interceptor, podNode podnetwork.PodNode) Daemon { + + daemon := &daemon{ + listenAddr: listenAddr, + interceptor: interceptor, + podNode: podNode, + readyCh: make(chan struct{}), + stopCh: make(chan struct{}), + } + + return daemon +} + +func (d *daemon) Start(ctx context.Context) error { + + // Set up pod network + + if err := d.podNode.Setup(); err != nil { + return fmt.Errorf("failed to set up pod network: %w", err) + } + defer func() { + if err := d.podNode.Teardown(); err != nil { + logger.Printf("failed to tear down pod network: %v", err) + } + }() + + // Set up agent protocol interceptor + + listener, err := net.Listen("tcp", d.listenAddr) + if err != nil { + return err + } + d.listenAddr = listener.Addr().String() + + ttrpcServer, err := ttrpc.NewServer() + if err != nil { + return fmt.Errorf("failed to create TTRPC server: %w", err) + } + + pb.RegisterAgentServiceService(ttrpcServer, d.interceptor) + pb.RegisterImageService(ttrpcServer, d.interceptor) + pb.RegisterHealthService(ttrpcServer, d.interceptor) + + ttrpcServerErr := make(chan error) + go func() { + defer close(ttrpcServerErr) + + if err := ttrpcServer.Serve(ctx, listener); err != nil && !errors.Is(err, ttrpc.ErrServerClosed) { + ttrpcServerErr <- fmt.Errorf("error running TTRPC server for kata agent interceptor: %w", err) + } + }() + defer func() { + if err := ttrpcServer.Shutdown(ctx); err != nil { + logger.Printf("error shutting down TTRPC server: %v", err) + } + if err := d.interceptor.Close(); err != nil { + logger.Printf("error shutting down kata agent interceptor: %v", err) + } + }() + + close(d.readyCh) + + select { + case <-ctx.Done(): + return d.Shutdown() + case <-d.stopCh: + case err := <-ttrpcServerErr: + return err + } + + return nil +} + +func (d *daemon) Shutdown() error { + d.stopOnce.Do(func() { + close(d.stopCh) + }) + return nil +} + +func (d *daemon) Addr() string { + <-d.readyCh + return d.listenAddr +} diff --git a/pkg/forwarder/forwarder_test.go b/pkg/forwarder/forwarder_test.go new file mode 100644 index 000000000..9620a09a8 --- /dev/null +++ b/pkg/forwarder/forwarder_test.go @@ -0,0 +1,100 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package forwarder + +import ( + "context" + "net" + "testing" + "time" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/agentproto" +) + +type mockConn struct{} + +func (*mockConn) Read(b []byte) (n int, err error) { return 0, nil } +func (*mockConn) Write(b []byte) (n int, err error) { return 0, nil } +func (*mockConn) Close() error { return nil } +func (*mockConn) LocalAddr() net.Addr { return nil } +func (*mockConn) RemoteAddr() net.Addr { return nil } +func (*mockConn) SetDeadline(t time.Time) error { return nil } +func (*mockConn) SetReadDeadline(t time.Time) error { return nil } +func (*mockConn) SetWriteDeadline(t time.Time) error { return nil } + +func dummyDialer(ctx context.Context) (net.Conn, error) { + return &mockConn{}, nil +} + +func TestNew(t *testing.T) { + + config := &Config{} + + ret := NewDaemon(config, DefaultListenAddr, agentproto.NewRedirector(dummyDialer), &mockPodNode{}) + if ret == nil { + t.Fatal("Expect non nil, got nil") + } + d, ok := ret.(*daemon) + if !ok { + t.Fatalf("Expect *daemon, got %T", d) + } + if d.interceptor == nil { + t.Fatal("Expect non nil, got nil") + } + if d.stopCh == nil { + t.Fatal("Expect non nil, got nil") + } + select { + case <-d.stopCh: + t.Fatal("channel is closed") + default: + } +} + +func TestStart(t *testing.T) { + + d := &daemon{ + interceptor: agentproto.NewRedirector(dummyDialer), + podNode: &mockPodNode{}, + readyCh: make(chan struct{}), + stopCh: make(chan struct{}), + } + + errCh := make(chan error) + go func() { + defer close(errCh) + + if err := d.Start(context.Background()); err != nil { + errCh <- err + } + }() + + select { + case err := <-errCh: + t.Fatalf("Expect no error, got %q", err) + default: + } + + if err := d.Shutdown(); err != nil { + t.Fatalf("Expect no error, got %q", err) + } + + select { + case err := <-errCh: + if err != nil { + t.Fatalf("Expect no error, got %q", err) + } + default: + } +} + +type mockPodNode struct{} + +func (n *mockPodNode) Setup() error { + return nil +} + +func (n *mockPodNode) Teardown() error { + return nil +} diff --git a/pkg/forwarder/interceptor/interceptor.go b/pkg/forwarder/interceptor/interceptor.go new file mode 100644 index 000000000..fff3e31bf --- /dev/null +++ b/pkg/forwarder/interceptor/interceptor.go @@ -0,0 +1,197 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package interceptor + +import ( + "context" + "fmt" + "log" + "net" + "os" + "time" + + "github.com/gogo/protobuf/types" + pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" + "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/agentproto" +) + +var logger = log.New(log.Writer(), "[forwarder/interceptor] ", log.LstdFlags|log.Lmsgprefix) + +type Interceptor interface { + agentproto.Redirector +} + +type interceptor struct { + agentproto.Redirector + + nsPath string +} + +func dial(ctx context.Context, agentSocket string) (net.Conn, error) { + + var conn net.Conn + + maxRetries := 30 + retryInterval := 5 * time.Second + + count := 1 + for { + var err error + + func() { + ctx, cancel := context.WithTimeout(ctx, retryInterval) + defer cancel() + + conn, err = (&net.Dialer{}).DialContext(ctx, "unix", agentSocket) + + if err == nil || count == maxRetries { + return + } + <-ctx.Done() + }() + + if err == nil { + break + } + if count == maxRetries { + err := fmt.Errorf("reaches max retry count. gave up establishing agent connection to %s: %w", agentSocket, err) + logger.Print(err) + return nil, err + } + logger.Printf("failed to establish agent connection to %s: %v. (retrying... %d/%d)", agentSocket, err, count, maxRetries) + + count++ + } + + logger.Printf("established agent connection to %s", agentSocket) + + return conn, nil +} + +func NewInterceptor(agentSocket, nsPath string) Interceptor { + + agentDialer := func(ctx context.Context) (net.Conn, error) { + return dial(ctx, agentSocket) + } + + redirector := agentproto.NewRedirector(agentDialer) + + return &interceptor{ + Redirector: redirector, + nsPath: nsPath, + } +} + +func (i *interceptor) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (*types.Empty, error) { + + logger.Printf("CreateContainer: containerID:%s", req.ContainerId) + + // Specify the network namespace path in the container spec + req.OCI.Linux.Namespaces = append(req.OCI.Linux.Namespaces, pb.LinuxNamespace{ + Type: string(specs.NetworkNamespace), + Path: i.nsPath, + }) + + logger.Printf(" namespaces:") + for _, ns := range req.OCI.Linux.Namespaces { + logger.Printf(" %s: %q", ns.Type, ns.Path) + } + + if len(req.OCI.Mounts) > 0 { + for _, m := range req.OCI.Mounts { + if _, err := os.Stat(m.Source); os.IsNotExist(err) && m.Type == "bind" { + logger.Printf("mount source %s doesn't exist, try to create", m.Source) + if err = os.MkdirAll(m.Source, os.ModePerm); err != nil { + logger.Printf("Failed to create dir: %v", err) + } + } + } + } + + res, err := i.Redirector.CreateContainer(ctx, req) + + if err != nil { + logger.Printf("CreateContainer failed with error: %v", err) + } + + return res, err +} + +func (i *interceptor) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*types.Empty, error) { + + logger.Printf("StartContainer: containerID:%s", req.ContainerId) + + res, err := i.Redirector.StartContainer(ctx, req) + + if err != nil { + logger.Printf("StartContainer failed with error: %v", err) + } + + return res, err +} + +func (i *interceptor) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*types.Empty, error) { + + logger.Printf("RemoveContainer: containerID:%s", req.ContainerId) + + res, err := i.Redirector.RemoveContainer(ctx, req) + + if err != nil { + logger.Printf("RemoveContainer failed with error: %v", err) + } + return res, err +} + +func (i *interceptor) CreateSandbox(ctx context.Context, req *pb.CreateSandboxRequest) (*types.Empty, error) { + + logger.Printf("CreateSandbox: hostname:%s sandboxId:%s", req.Hostname, req.SandboxId) + + if len(req.Dns) > 0 { + logger.Print(" dns:") + for _, d := range req.Dns { + logger.Printf(" %s", d) + } + + logger.Print(" Eliminated the DNS setting above from CreateSandboxRequest to stop updating /etc/resolv.conf on the peer pod VM") + logger.Print(" See https://github.com/confidential-containers/cloud-api-adaptor/issues/98 for the details.") + logger.Println() + req.Dns = nil + } + + res, err := i.Redirector.CreateSandbox(ctx, req) + + if err != nil { + logger.Printf("CreateSandbox failed with error: %v", err) + } + + return res, err +} + +func (i *interceptor) DestroySandbox(ctx context.Context, req *pb.DestroySandboxRequest) (*types.Empty, error) { + + logger.Printf("DestroySandbox") + + res, err := i.Redirector.DestroySandbox(ctx, req) + + if err != nil { + logger.Printf("DestroySandbox failed with error: %v", err) + } + + return res, err +} + +func (i *interceptor) PullImage(ctx context.Context, req *pb.PullImageRequest) (*pb.PullImageResponse, error) { + + logger.Printf("PullImage: image: %q, containerID: %q", req.Image, req.ContainerId) + + res, err := i.Redirector.PullImage(ctx, req) + + if err != nil { + logger.Printf("PullImage failed with error: %v", err) + } + + return res, err +} diff --git a/pkg/forwarder/interceptor/interceptor_test.go b/pkg/forwarder/interceptor/interceptor_test.go new file mode 100644 index 000000000..d27d70fcb --- /dev/null +++ b/pkg/forwarder/interceptor/interceptor_test.go @@ -0,0 +1,18 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package interceptor + +import ( + "testing" +) + +func TestNewInterceptor(t *testing.T) { + + socketName := "dummy.sock" + + i := NewInterceptor(socketName, "") + if i == nil { + t.Fatal("Expect non nil, got nil") + } +} diff --git a/pkg/internal/testing/testutils.go b/pkg/internal/testing/testutils.go new file mode 100644 index 000000000..0e499702d --- /dev/null +++ b/pkg/internal/testing/testutils.go @@ -0,0 +1,26 @@ +// (C) Copyright Red Hat 2022. +// SPDX-License-Identifier: Apache-2.0 + +// Package testutils provides utilities for testing. +package testutils + +import ( + "os" + "testing" +) + +// SkipTestIfNotRoot skips the test if not running as root user. +func SkipTestIfNotRoot(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("This test requires root privileges. Skipping.") + } +} + +// SkipTestIfRunningInCI skips the test if running in CI environment. +func SkipTestIfRunningInCI(t *testing.T) { + value, exported := os.LookupEnv("CI") + + if exported && value == "true" { + t.Skip("This test is disabled on CI. Skipping.") + } +} diff --git a/pkg/podnetwork/common.go b/pkg/podnetwork/common.go new file mode 100644 index 000000000..a02263632 --- /dev/null +++ b/pkg/podnetwork/common.go @@ -0,0 +1,61 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package podnetwork + +import ( + "fmt" + "log" + "net" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler/routing" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" +) + +var logger = log.New(log.Writer(), "[podnetwork] ", log.LstdFlags|log.Lmsgprefix) + +func init() { + tunneler.Register("routing", routing.NewWorkerNodeTunneler, routing.NewPodNodeTunneler) + tunneler.Register("vxlan", vxlan.NewWorkerNodeTunneler, vxlan.NewPodNodeTunneler) +} + +func getRoutes(ns *netops.NS) ([]*netops.Route, string, error) { + + routes, err := ns.GetRoutes() + if err != nil { + return nil, "", fmt.Errorf("failed to get routes on namespace %q: %w", ns.Path, err) + } + + logger.Printf("routes on netns %s", ns.Path) + for _, r := range routes { + var dst, gw, dev string + if r.Dst != nil { + dst = r.Dst.String() + } else { + dst = "default" + } + if r.GW != nil { + gw = "via " + r.GW.String() + } + if r.Dev != "" { + dev = "dev " + r.Dev + } + logger.Printf(" %s %s %s", dst, gw, dev) + } + + for _, r := range routes { + if r.Dst == nil || r.Dst.IP == nil { + return routes, r.Dev, nil + } + if r.Dst.IP.Equal(net.IPv4zero) && r.Dst.Mask != nil { + ones, bits := r.Dst.Mask.Size() + if bits != 0 && ones == 0 { + return routes, r.Dev, nil + } + } + } + + return nil, "", fmt.Errorf("failed to identify destination interface of default gateway on network namespace %q", ns.Path) +} diff --git a/pkg/podnetwork/podnetwork_test.go b/pkg/podnetwork/podnetwork_test.go new file mode 100644 index 000000000..c8d8c63e6 --- /dev/null +++ b/pkg/podnetwork/podnetwork_test.go @@ -0,0 +1,224 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package podnetwork + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" + + testutils "github.com/confidential-containers/cloud-api-adaptor/pkg/internal/testing" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tuntest" +) + +type mockWorkerNodeTunneler struct{} + +func newMockWorkerNodeTunneler() tunneler.Tunneler { + return &mockWorkerNodeTunneler{} +} + +func (t *mockWorkerNodeTunneler) Setup(nsPath string, podNodeIPs []net.IP, config *tunneler.Config) error { + return nil +} + +func (t *mockWorkerNodeTunneler) Teardown(nsPath, hostInterface string, config *tunneler.Config) error { + return nil +} + +type mockPodNodeTunneler struct{} + +func newMockPodNodeTunneler() tunneler.Tunneler { + return &mockPodNodeTunneler{} +} + +func (t *mockPodNodeTunneler) Setup(nsPath string, podNodeIPs []net.IP, config *tunneler.Config) error { + return nil +} + +func (t *mockPodNodeTunneler) Teardown(nsPath, hostInterface string, config *tunneler.Config) error { + return nil +} + +func TestWorkerNode(t *testing.T) { + testutils.SkipTestIfNotRoot(t) + + mockTunnelType := "mock" + tunneler.Register(mockTunnelType, newMockWorkerNodeTunneler, newMockPodNodeTunneler) + + workerNodeNS := tuntest.NewNamedNS(t, "test-workernode") + defer tuntest.DeleteNamedNS(t, workerNodeNS) + + tuntest.BridgeAdd(t, workerNodeNS, "ens0") + tuntest.AddrAdd(t, workerNodeNS, "ens0", "192.168.0.2/24") + tuntest.BridgeAdd(t, workerNodeNS, "ens1") + tuntest.AddrAdd(t, workerNodeNS, "ens1", "192.168.1.2/24") + tuntest.RouteAdd(t, workerNodeNS, "", "192.168.0.1", "ens0") + + workerPodNS := tuntest.NewNamedNS(t, "test-workerpod") + defer tuntest.DeleteNamedNS(t, workerPodNS) + + tuntest.BridgeAdd(t, workerPodNS, "eth0") + tuntest.AddrAdd(t, workerPodNS, "eth0", "172.16.0.2/24") + tuntest.RouteAdd(t, workerPodNS, "", "172.16.0.1", "eth0") + + for hostInterface, expected := range map[string]struct { + podNodeIP string + workerNodeIP string + }{ + "": { + workerNodeIP: "192.168.0.2/24", + }, + "ens0": { + workerNodeIP: "192.168.0.2/24", + }, + "ens1": { + workerNodeIP: "192.168.1.2/24", + }, + } { + + err := workerNodeNS.Run(func() error { + + workerNode := NewWorkerNode(mockTunnelType, hostInterface, 0, 0) + require.NotNil(t, workerNode, "hostInterface=%q", hostInterface) + + config, err := workerNode.Inspect(workerPodNS.Path) + require.Nil(t, err, "hostInterface=%q", hostInterface) + + err = workerNode.Setup(workerPodNS.Path, []net.IP{net.ParseIP("192.168.0.3"), net.ParseIP("192.168.0.3")}, config) + require.Nil(t, err, "hostInterface=%q", hostInterface) + + require.Equal(t, "172.16.0.2/24", config.PodIP, "hostInterface=%q", hostInterface) + require.Equal(t, "eth0", config.InterfaceName, "hostInterface=%q", hostInterface) + require.Equal(t, 1500, config.MTU, "hostInterface=%q", hostInterface) + require.Equal(t, hostInterface == "ens1", config.Dedicated, "hostInterface=%q", hostInterface) + require.Equal(t, expected.workerNodeIP, config.WorkerNodeIP, "hostInterface=%q", hostInterface) + require.Equal(t, mockTunnelType, config.TunnelType, "hostInterface=%q", hostInterface) + + require.Equal(t, len(config.Routes), 1, "hostInterface=%q", hostInterface) + require.Empty(t, config.Routes[0].Dst, "hostInterface=%q", hostInterface) + require.Equal(t, config.Routes[0].GW, "172.16.0.1", "hostInterface=%q", hostInterface) + require.Equal(t, config.Routes[0].Dev, "eth0", "hostInterface=%q", hostInterface) + + err = workerNode.Teardown(workerPodNS.Path, config) + require.Nil(t, err, "hostInterface=%q", hostInterface) + + return nil + }) + require.Nil(t, err, "hostInterface=%q", hostInterface) + } +} + +func TestPodNode(t *testing.T) { + testutils.SkipTestIfNotRoot(t) + + mockTunnelType := "mock" + tunneler.Register(mockTunnelType, newMockWorkerNodeTunneler, newMockPodNodeTunneler) + + podNodeNS := tuntest.NewNamedNS(t, "test-podnode") + defer tuntest.DeleteNamedNS(t, podNodeNS) + + tuntest.BridgeAdd(t, podNodeNS, "ens0") + tuntest.AddrAdd(t, podNodeNS, "ens0", "192.168.0.3/24") + tuntest.BridgeAdd(t, podNodeNS, "ens1") + tuntest.AddrAdd(t, podNodeNS, "ens1", "192.168.1.3/24") + tuntest.RouteAdd(t, podNodeNS, "", "192.168.0.1", "ens0") + + podNS := tuntest.NewNamedNS(t, "test-pod") + defer tuntest.DeleteNamedNS(t, podNS) + + for hostInterface, expected := range map[string]struct { + podNodeIP string + workerNodeIP string + }{ + "": { + podNodeIP: "192.168.0.3", + workerNodeIP: "192.168.0.2", + }, + "ens0": { + podNodeIP: "192.168.0.3", + workerNodeIP: "192.168.0.2", + }, + "ens1": { + podNodeIP: "192.168.1.3", + workerNodeIP: "192.168.1.2", + }, + } { + + err := podNodeNS.Run(func() error { + + config := &tunneler.Config{ + PodIP: "172.16.0.2", + Routes: []*tunneler.Route{ + { + Dst: "", + GW: "172.16.0.1", + Dev: "eth0", + }, + }, + InterfaceName: "eth0", + MTU: 1500, + WorkerNodeIP: expected.workerNodeIP, + TunnelType: mockTunnelType, + Dedicated: hostInterface == "ens1", + } + + podNode := NewPodNode(podNS.Path, hostInterface, config) + require.NotNil(t, podNode, "hostInterface=%q", hostInterface) + + err := podNode.Setup() + require.Nil(t, err, "hostInterface=%q", hostInterface) + + err = podNode.Teardown() + require.Nil(t, err, "hostInterface=%q", hostInterface) + + return nil + }) + require.Nil(t, err, "hostInterface=%q", hostInterface) + } +} + +func TestPluginDetectHostInterface(t *testing.T) { + testutils.SkipTestIfNotRoot(t) + + hostNS := tuntest.NewNamedNS(t, "test-host") + defer tuntest.DeleteNamedNS(t, hostNS) + + tuntest.BridgeAdd(t, hostNS, "eth0") + tuntest.BridgeAdd(t, hostNS, "eth1") + + tuntest.AddrAdd(t, hostNS, "eth0", "10.10.0.2/24") + + errCh := make(chan error) + go func() { + defer close(errCh) + _, err := detectIP(hostNS, "eth1", 1*time.Second) + errCh <- err + }() + + select { + case <-time.After(2 * time.Second): + t.Fatal("timeout does not occur") + case err := <-errCh: + if e, a := fmt.Sprintf("failed to identify IP address assigned to host interface eth1 on netns %s", hostNS.Path), err.Error(); e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + } + + go func() { + time.Sleep(500 * time.Millisecond) + tuntest.AddrAdd(t, hostNS, "eth1", "192.168.0.2/24") + }() + + ip, err := detectIP(hostNS, "eth1", 1500*time.Millisecond) + if err != nil { + t.Fatalf("Expect nil, got %v", err) + } + if e, a := ip.String(), "192.168.0.2"; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } +} diff --git a/pkg/podnetwork/podnode.go b/pkg/podnetwork/podnode.go new file mode 100644 index 000000000..57ca09c0b --- /dev/null +++ b/pkg/podnetwork/podnode.go @@ -0,0 +1,185 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package podnetwork + +import ( + "fmt" + "net" + "time" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" +) + +type PodNode interface { + Setup() error + Teardown() error +} + +type podNode struct { + nsPath string + hostInterface string + config *tunneler.Config +} + +func NewPodNode(nsPath string, hostInterface string, config *tunneler.Config) PodNode { + + podNode := &podNode{ + nsPath: nsPath, + hostInterface: hostInterface, + config: config, + } + + return podNode +} + +func (n *podNode) Setup() error { + + tun, err := tunneler.PodNodeTunneler(n.config.TunnelType) + if err != nil { + return fmt.Errorf("failed to get tunneler: %w", err) + } + + hostNS, err := netops.GetNS() + if err != nil { + return fmt.Errorf("failed to open the host network namespace: %w", err) + } + defer func() { + if err := hostNS.Close(); err != nil { + logger.Printf("failed to close the host network namespace: %v", err) + } + }() + + hostPrimaryInterface, err := detectPrimaryInterface(hostNS, 3*time.Minute) + if err != nil { + return err + } + + primaryPodNodeIP, err := detectIP(hostNS, hostPrimaryInterface, 3*time.Minute) + if err != nil { + return err + } + + podNodeIPs := []net.IP{primaryPodNodeIP} + + hostInterface := n.hostInterface + if hostInterface == "" { + hostInterface = hostPrimaryInterface + } + + if n.config.Dedicated { + if hostInterface == hostPrimaryInterface { + return fmt.Errorf("%s is not a dedicated interface", hostInterface) + } + + dedicatePodNodeIP, err := detectIP(hostNS, hostInterface, 3*time.Minute) + if err != nil { + return err + } + + podNodeIPs = append(podNodeIPs, dedicatePodNodeIP) + } + + podNS, err := netops.NewNSFromPath(n.nsPath) + if err != nil { + return fmt.Errorf("failed to open network namespace %q: %w", n.nsPath, err) + } + defer func() { + if err := podNS.Close(); err != nil { + logger.Printf("failed to close a network namespace: %q", podNS.Path) + } + }() + + if err := tun.Setup(n.nsPath, podNodeIPs, n.config); err != nil { + return fmt.Errorf("failed to set up tunnel %q: %w", n.config.TunnelType, err) + } + + return nil +} + +func (n *podNode) Teardown() error { + + tun, err := tunneler.PodNodeTunneler(n.config.TunnelType) + if err != nil { + return fmt.Errorf("failed to get tunneler: %w", err) + } + + hostNS, err := netops.GetNS() + if err != nil { + return fmt.Errorf("failed to open the host network namespace: %w", err) + } + defer func() { + if err := hostNS.Close(); err != nil { + logger.Printf("failed to close the host network namespace: %v", err) + } + }() + + hostInterface := n.hostInterface + if hostInterface == "" { + _, hostPrimaryInterface, err := getRoutes(hostNS) + if err != nil { + return fmt.Errorf("failed to identify the host primary interface: %w", err) + } + hostInterface = hostPrimaryInterface + } + + if err := tun.Teardown(n.nsPath, hostInterface, n.config); err != nil { + return fmt.Errorf("failed to tear down tunnel %q: %w", n.config.TunnelType, err) + } + + return nil +} + +func detectPrimaryInterface(hostNS *netops.NS, timeout time.Duration) (string, error) { + + timeoutCh := time.After(timeout) + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + + _, hostPrimaryInterface, err := getRoutes(hostNS) + if err == nil { + return hostPrimaryInterface, nil + } + + select { + case <-timeoutCh: + return "", fmt.Errorf("failed to identify primary interface on netns %s", hostNS.Path) + case <-ticker.C: + } + + logger.Printf("failed to identify the host primary interface: %v (retrying...)", err) + } +} + +func detectIP(hostNS *netops.NS, hostInterface string, timeout time.Duration) (net.IP, error) { + + // An IP address of the second network interface of an IBM Cloud VPC instance is assigned by DHCP + // several seconds after the first interface gets an IP address. + + timeoutCh := time.After(timeout) + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + + ips, err := hostNS.GetIP(hostInterface) + if err != nil { + return nil, fmt.Errorf("failed to get addresses assigned %s on netns %s: %w", hostInterface, hostNS.Path, err) + } + if len(ips) > 1 { + return nil, fmt.Errorf("more than one IP address assigned on %s (netns: %s)", hostInterface, hostNS.Path) + } + if len(ips) == 1 { + return ips[0], nil + } + + select { + case <-timeoutCh: + return nil, fmt.Errorf("failed to identify IP address assigned to host interface %s on netns %s", hostInterface, hostNS.Path) + case <-ticker.C: + } + } +} diff --git a/pkg/podnetwork/tunneler/routing/common.go b/pkg/podnetwork/tunneler/routing/common.go new file mode 100644 index 000000000..1558efa42 --- /dev/null +++ b/pkg/podnetwork/tunneler/routing/common.go @@ -0,0 +1,21 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "net" +) + +const ( + localTableOriginalPriority = 0 + localTableNewPriority = 32765 + podTablePriority = 0 +) + +func mask32(ip net.IP) *net.IPNet { + return &net.IPNet{ + IP: ip, + Mask: net.CIDRMask(8*net.IPv4len, 8*net.IPv4len), + } +} diff --git a/pkg/podnetwork/tunneler/routing/iptables.go b/pkg/podnetwork/tunneler/routing/iptables.go new file mode 100644 index 000000000..8050933c5 --- /dev/null +++ b/pkg/podnetwork/tunneler/routing/iptables.go @@ -0,0 +1,97 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "fmt" + "strings" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" + "github.com/coreos/go-iptables/iptables" +) + +const ( + chainName = "PEERPOD" + ruleComment = "peerpod" +) + +type iptablesRule struct { + table string + chain string + spec []string +} + +func setIPTablesRules(ns *netops.NS, hostInterface string) error { + + var iptablesRules = []iptablesRule{ + { + table: "raw", + chain: chainName, + spec: []string{"-i", vrf1Name, "-m", "comment", "--comment", ruleComment, "-j", "NOTRACK"}, + }, + { + table: "raw", + chain: chainName, + spec: []string{"-i", vrf2Name, "-m", "comment", "--comment", ruleComment, "-j", "NOTRACK"}, + }, + { + table: "raw", + chain: chainName, + spec: []string{"-i", hostInterface, "-m", "comment", "--comment", ruleComment, "-j", "NOTRACK"}, + }, + { + table: "raw", + chain: "PREROUTING", + spec: []string{"-j", chainName}, + }, + { + table: "filter", + chain: chainName, + spec: []string{"-i", vrf1Name, "-m", "comment", "--comment", ruleComment, "-j", "ACCEPT"}, + }, + { + table: "filter", + chain: chainName, + spec: []string{"-i", vrf2Name, "-m", "comment", "--comment", ruleComment, "-j", "ACCEPT"}, + }, + { + table: "filter", + chain: chainName, + spec: []string{"-i", hostInterface, "-m", "comment", "--comment", ruleComment, "-j", "ACCEPT"}, + }, + { + table: "filter", + chain: "FORWARD", + spec: []string{"-j", chainName}, + }, + } + + return ns.Run(func() error { + + ipt, err := iptables.New(iptables.IPFamily(iptables.ProtocolIPv4)) + if err != nil { + return fmt.Errorf("failed to initialize iptables: %w", err) + } + + for _, rule := range iptablesRules { + + exists, err := ipt.ChainExists(rule.table, rule.chain) + if err != nil { + return fmt.Errorf("failed to check the existence of iptables chain %q: %w", rule.chain, err) + } + + if !exists { + if err := ipt.NewChain(rule.table, rule.chain); err != nil { + return fmt.Errorf("failed to create iptables chain %q: %w", rule.chain, err) + } + } + + if err := ipt.AppendUnique(rule.table, rule.chain, rule.spec...); err != nil { + return fmt.Errorf("failed to add iptables rule \"-t %s -A %s %s\": %w", rule.table, rule.chain, strings.Join(rule.spec, " "), err) + } + } + + return nil + }) +} diff --git a/pkg/podnetwork/tunneler/routing/iptables_test.go b/pkg/podnetwork/tunneler/routing/iptables_test.go new file mode 100644 index 000000000..1b5d1af30 --- /dev/null +++ b/pkg/podnetwork/tunneler/routing/iptables_test.go @@ -0,0 +1,98 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "testing" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/internal/testing" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tuntest" + "github.com/coreos/go-iptables/iptables" +) + +func TestIPTables(t *testing.T) { + testutils.SkipTestIfNotRoot(t) + + workerNS := tuntest.NewNamedNS(t, "test-host") + defer tuntest.DeleteNamedNS(t, workerNS) + + ipt, err := iptables.New(iptables.IPFamily(iptables.ProtocolIPv4)) + if err != nil { + t.Fatalf("Expect no error, got %q", err) + } + + if err := workerNS.Run(func() error { + + return ipt.ChangePolicy("filter", "FORWARD", "DROP") + + }); err != nil { + t.Fatalf("Expect no error, got %q", err) + } + + hostInterface := "ens4" + + if err := setIPTablesRules(workerNS, hostInterface); err != nil { + t.Fatalf("Expect no error, got %q", err) + } + + if err := workerNS.Run(func() error { + + if exists, err := ipt.ChainExists("filter", chainName); err != nil { + return err + } else if e, a := true, exists; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + + if exists, err := ipt.Exists("filter", chainName, "-i", vrf1Name, "-m", "comment", "--comment", ruleComment, "-j", "ACCEPT"); err != nil { + return err + } else if e, a := true, exists; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + + if exists, err := ipt.Exists("filter", chainName, "-i", vrf2Name, "-m", "comment", "--comment", ruleComment, "-j", "ACCEPT"); err != nil { + return err + } else if e, a := true, exists; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + + if exists, err := ipt.Exists("filter", chainName, "-i", hostInterface, "-m", "comment", "--comment", ruleComment, "-j", "ACCEPT"); err != nil { + return err + } else if e, a := true, exists; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + + if exists, err := ipt.ChainExists("raw", chainName); err != nil { + return err + } else if e, a := true, exists; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + + if exists, err := ipt.Exists("raw", chainName, "-i", vrf1Name, "-m", "comment", "--comment", ruleComment, "-j", "NOTRACK"); err != nil { + return err + } else if e, a := true, exists; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + + if exists, err := ipt.Exists("raw", chainName, "-i", vrf2Name, "-m", "comment", "--comment", ruleComment, "-j", "NOTRACK"); err != nil { + return err + } else if e, a := true, exists; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + + if exists, err := ipt.Exists("raw", chainName, "-i", hostInterface, "-m", "comment", "--comment", ruleComment, "-j", "NOTRACK"); err != nil { + return err + } else if e, a := true, exists; e != a { + t.Fatalf("Expect %v, got %v", e, a) + } + + return nil + + }); err != nil { + t.Fatalf("Expect no error, got %q", err) + } + // Check idempotency + if err := setIPTablesRules(workerNS, hostInterface); err != nil { + t.Fatalf("Expect no error, got %q", err) + } +} diff --git a/pkg/podnetwork/tunneler/routing/keepalive.go b/pkg/podnetwork/tunneler/routing/keepalive.go new file mode 100644 index 000000000..e8839335e --- /dev/null +++ b/pkg/podnetwork/tunneler/routing/keepalive.go @@ -0,0 +1,284 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sync" + "syscall" + "time" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" + "golang.org/x/sys/unix" +) + +// Keep alive server is only used to work around the following issue on IBM Cloud. +// +// Conditions: +// * Two VM instances A and B are provisioned with two network interfaces each +// * A network namespace is configured on each instance +// * Veth interfaces and routing tables are configured to enable network communication +// between the two network namespaces via the second network interfaces of the instances. +// * "Allow IP Spoofing" option is enabled for each network interface +// * No communication is made before an experiment between the second network interfaces +// of the two instances. +// +// Step to reproduce the issue: +// * Open a TCP connection from A and B +// Note that this connection requires routing, and requires "Allow IP spoofing" to be enabled. +// +// Expected result: +// * A TCP connection is successfully established +// +// Actual result: +// * A SYN packet from A is successfully delivered to B +// * Then, a SYN+ACK packet from B is successfully delivered to A +// * Then, an ACK packet from A is lost and NOT delivered to B +// +// Cause of the issue: +// * Not yet identified +// +// Work around: +// * Periodically establish TCP connections that does not require routing +// using the IP addresses assigned to the second network interfaces +// before establishing TCP connections that requires routing. +// +// Keep alive sever and client implement this work around solution + +const ( + defaultKeepAliveListenPort = "10345" + defaultKeepAliveListenAddr = "0.0.0.0:" + defaultKeepAliveListenPort +) + +var keepAlive = struct { + clientStopCh map[string]chan struct{} + serverStopCh chan struct{} + mutex sync.Mutex +}{ + clientStopCh: make(map[string]chan struct{}), +} + +func getKey(podIP net.IP, atWorkerNode bool) string { + var prefix string + if atWorkerNode { + prefix = "worker:" + } else { + prefix = "pod:" + } + return prefix + podIP.String() +} + +func startKeepAlive(ns *netops.NS, podIP net.IP, address string, atWorkerNode bool) error { + + keepAlive.mutex.Lock() + defer keepAlive.mutex.Unlock() + + if _, ok := keepAlive.clientStopCh[getKey(podIP, atWorkerNode)]; ok { + return fmt.Errorf("keep alive for pod IP %s is already configured", podIP) + } + + if atWorkerNode && len(keepAlive.clientStopCh) == 0 { + keepAlive.serverStopCh = make(chan struct{}) + if err := launchKeepAliveServer(ns, true, keepAlive.serverStopCh); err != nil { + return err + } + } + + clientStopCh := make(chan struct{}) + keepAlive.clientStopCh[getKey(podIP, atWorkerNode)] = clientStopCh + logger.Printf("keep alive for pod IP %s registered", podIP) + + return launchKeepAliveClient(ns, address, clientStopCh, withVRF(atWorkerNode)) +} + +func stopKeepAlive(podIP net.IP, atWorkerNode bool) error { + + keepAlive.mutex.Lock() + defer keepAlive.mutex.Unlock() + + clientStopCh, ok := keepAlive.clientStopCh[getKey(podIP, atWorkerNode)] + if !ok { + return fmt.Errorf("keep alive for pod IP %s is not found", podIP) + } + + close(clientStopCh) + delete(keepAlive.clientStopCh, getKey(podIP, atWorkerNode)) + + if atWorkerNode && len(keepAlive.clientStopCh) == 0 { + close(keepAlive.serverStopCh) + } + + return nil +} + +func control(network string, address string, c syscall.RawConn) (err error) { + + e := c.Control(func(fd uintptr) { + err = unix.SetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_BINDTODEVICE, vrf1Name) + }) + if e != nil { + return e + } + + return err +} + +type keepAliveClientConfig struct { + interval time.Duration + timeout time.Duration + useVRF bool +} + +func withVRF(useVRF bool) func(*keepAliveClientConfig) { + return func(cfg *keepAliveClientConfig) { + cfg.useVRF = useVRF + } +} + +func withInterval(interval time.Duration) func(*keepAliveClientConfig) { + return func(cfg *keepAliveClientConfig) { + cfg.interval = interval + } +} + +func withTimeout(timeout time.Duration) func(*keepAliveClientConfig) { + return func(cfg *keepAliveClientConfig) { + cfg.timeout = timeout + } +} + +func launchKeepAliveClient(targetNS *netops.NS, serverAddr string, stopCh chan struct{}, options ...func(*keepAliveClientConfig)) error { + + cfg := keepAliveClientConfig{ + interval: time.Minute * 5, + timeout: 5 * time.Second, + } + + for _, f := range options { + f(&cfg) + } + + dialer := &net.Dialer{} + if cfg.useVRF { + dialer.Control = control + } + + ns, err := targetNS.Clone() + if err != nil { + return fmt.Errorf("failed to clone a network namespace: %w", err) + } + + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, err error) { + runErr := ns.Run(func() error { + conn, err = dialer.DialContext(ctx, network, addr) + return err + }) + if err == nil && runErr != nil { + err = runErr + } + return + }, + }, + } + + serverURL := (&url.URL{Scheme: "http", Host: serverAddr, Path: "/"}).String() + + req, err := http.NewRequest("GET", serverURL, nil) + if err != nil { + return fmt.Errorf("failed to create a HTTP request for keep alive: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-stopCh + cancel() + }() + + go func() { + ticker := time.NewTicker(cfg.interval) + defer ticker.Stop() + + defer ns.Close() + + for { + select { + case <-stopCh: + return + default: + } + + ctx, cancel := context.WithTimeout(ctx, cfg.timeout) + + logger.Printf("connecting to keep alive server %s", serverURL) + + res, err := client.Do(req.WithContext(ctx)) + if err != nil { + logger.Printf("failed to connect to a keep alive server: %s: %v", serverURL, err) + <-ctx.Done() + cancel() + continue + } + cancel() + + res.Body.Close() + + <-ticker.C + } + }() + + return nil +} + +func launchKeepAliveServer(targetNS *netops.NS, useVRF bool, stopCh chan struct{}) error { + + listenAddr := defaultKeepAliveListenAddr + + listenConfig := &net.ListenConfig{} + + if useVRF { + listenConfig.Control = control + } + + ns, err := targetNS.Clone() + if err != nil { + return fmt.Errorf("failed to clone a network namespace: %w", err) + } + + var listener net.Listener + if err := ns.Run(func() error { + var err error + listener, err = listenConfig.Listen(context.Background(), "tcp", listenAddr) + return err + }); err != nil { + return fmt.Errorf("failed to listen on %s for keep alive: %w", listenAddr, err) + } + + httpServer := http.Server{ + Addr: listenAddr, + } + + go func() { + defer ns.Close() + + if err := httpServer.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Printf("error running http server for keep alive: %v", err) + } + }() + + go func() { + <-stopCh + if err := httpServer.Shutdown(context.Background()); err != nil { + logger.Printf("error on shutdown of http server for keep alive: %v", err) + } + }() + + return nil +} diff --git a/pkg/podnetwork/tunneler/routing/keepalive_test.go b/pkg/podnetwork/tunneler/routing/keepalive_test.go new file mode 100644 index 000000000..033cb6b5c --- /dev/null +++ b/pkg/podnetwork/tunneler/routing/keepalive_test.go @@ -0,0 +1,75 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "net" + "net/http" + "testing" + "time" + + testutils "github.com/confidential-containers/cloud-api-adaptor/pkg/internal/testing" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tuntest" + "github.com/vishvananda/netlink" +) + +func TestKeepAlive(t *testing.T) { + // TODO: enable this test once https://github.com/confidential-containers/cloud-api-adaptor/issues/52 is fixed + testutils.SkipTestIfRunningInCI(t) + testutils.SkipTestIfNotRoot(t) + + workerNS := tuntest.NewNamedNS(t, "test-host") + defer tuntest.DeleteNamedNS(t, workerNS) + + vm1NS := tuntest.NewNamedNS(t, "test-podvm1") + defer tuntest.DeleteNamedNS(t, vm1NS) + + ifName := "eth1" + tuntest.VethAdd(t, workerNS, ifName, vm1NS, ifName) + tuntest.AddrAdd(t, workerNS, ifName, "192.168.0.1/24") + tuntest.AddrAdd(t, vm1NS, ifName, "192.168.0.2/24") + + if err := workerNS.LinkAdd(vrf1Name, &netlink.Vrf{Table: vrf1TableID}); err != nil { + t.Fatalf("failed to add vrf %s: %v", vrf1Name, err) + } + + if err := workerNS.LinkSetUp(vrf1Name); err != nil { + t.Fatalf("failed to set vrf %s up: %v", vrf1Name, err) + } + + if err := workerNS.LinkSetMaster(ifName, vrf1Name); err != nil { + t.Fatalf("failed to set master of %s to vrf %s: %v", ifName, vrf1Name, err) + } + + stopCh := make(chan struct{}) + + if err := launchKeepAliveClient(workerNS, net.JoinHostPort("192.168.0.2", daemonListenPort), stopCh, withVRF(true), withInterval(200*time.Millisecond), withTimeout(100*time.Millisecond)); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + if err := launchKeepAliveServer(workerNS, true, stopCh); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + time.Sleep(500 * time.Millisecond) + + if err := launchKeepAliveClient(vm1NS, net.JoinHostPort("192.168.0.1", defaultKeepAliveListenPort), stopCh, withVRF(false), withInterval(200*time.Millisecond), withTimeout(100*time.Millisecond)); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + go func() { + if err := vm1NS.Run(func() error { + httpServer := http.Server{ + Addr: net.JoinHostPort("0.0.0.0", daemonListenPort), + } + return httpServer.ListenAndServe() + }); err != nil { + t.Logf("Expect no error, got %v", err) + t.Fail() + } + }() + + time.Sleep(time.Second) + close(stopCh) +} diff --git a/pkg/podnetwork/tunneler/routing/podnode.go b/pkg/podnetwork/tunneler/routing/podnode.go new file mode 100644 index 000000000..0754393b9 --- /dev/null +++ b/pkg/podnetwork/tunneler/routing/podnode.go @@ -0,0 +1,222 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "errors" + "fmt" + "net" + "os" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" + "golang.org/x/sys/unix" +) + +type podNodeTunneler struct { + keepAliveStopCh chan struct{} +} + +func NewPodNodeTunneler() tunneler.Tunneler { + return &podNodeTunneler{ + keepAliveStopCh: make(chan struct{}), + } +} + +const ( + podVethName = "eth0" + hostVethName = "veth0" + + podTableID = 45001 + sourceTableID = 45002 + sourceTablePriority = 505 +) + +func checkDefaultRoute(dst *net.IPNet) bool { + + if dst == nil || dst.IP == nil { + return true + } + + if !dst.IP.Equal(net.IPv4zero) { + return false + } + + if dst.Mask == nil { + return false + } + + ones, bits := dst.Mask.Size() + if bits == 0 { + return false + } + + return ones == 0 +} + +func (t *podNodeTunneler) Setup(nsPath string, podNodeIPs []net.IP, config *tunneler.Config) error { + + if !config.Dedicated { + return errors.New("shared subnet is not supported") + } + + if len(podNodeIPs) != 2 { + return errors.New("secondary pod node IP is not available") + } + + podNodeIP := podNodeIPs[1] + + podIP, podIPNet, err := net.ParseCIDR(config.PodIP) + if err != nil { + return fmt.Errorf("failed to parse pod IP %s: %w", config.PodIP, err) + } + podIPNet.IP = podIP + + nodeIP, _, err := net.ParseCIDR(config.WorkerNodeIP) + if err != nil { + return fmt.Errorf("failed to parse node IP %s: %w", config.WorkerNodeIP, err) + } + + hostNS, err := netops.GetNS() + if err != nil { + return fmt.Errorf("failed to get host network namespace: %w", err) + } + defer hostNS.Close() + + hostInterface, err := hostNS.LinkNameByAddr(podNodeIP) + if err != nil { + return fmt.Errorf("failed to identify host interface that has %s on netns %s", podNodeIP.String(), hostNS.Path) + } + + podNS, err := netops.NewNSFromPath(nsPath) + if err != nil { + return fmt.Errorf("failed to get a pod network namespace: %s: %w", nsPath, err) + } + defer podNS.Close() + + if err := hostNS.RuleAdd(nil, "", localTableNewPriority, unix.RT_TABLE_LOCAL); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add local table at priority %d: %w", localTableNewPriority, err) + } + + if err = hostNS.RuleDel(nil, "", localTableOriginalPriority, unix.RT_TABLE_LOCAL); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to delete local table at priority %d: %w", localTableOriginalPriority, err) + } + + if err := hostNS.VethAdd(hostVethName, podNS, podVethName); err != nil { + return fmt.Errorf("failed to create a veth pair: %s and %s on %s: %w", hostVethName, podVethName, nsPath, err) + } + + mtu := int(config.MTU) + if err := podNS.SetMTU(podVethName, mtu); err != nil { + return fmt.Errorf("failed to set MTU of %s to %d on %s: %w", podVethName, mtu, nsPath, err) + } + + if err := podNS.AddrAdd(podVethName, podIPNet); err != nil { + return fmt.Errorf("failed to add pod IP %s to %s on %s: %w", podIPNet, podVethName, nsPath, err) + } + + if err := podNS.LinkSetUp(podVethName); err != nil { + return fmt.Errorf("failed to set %s up on %s: %w", podVethName, nsPath, err) + } + + if err := hostNS.LinkSetUp(hostVethName); err != nil { + return fmt.Errorf("failed to set %s up on host network namespace: %w", hostVethName, err) + } + + var defaultRouteGateway net.IP + + // We need to process routes without gateway address first. Processing routes with a gateway causes an error if the gateway is not reachable. + // Calico sets up routes with this pattern. + // https://github.com/projectcalico/cni-plugin/blob/7495c0279c34faac315b82c1838bca638e23dbbe/pkg/dataplane/linux/dataplane_linux.go#L158-L167 + + var first, second []*tunneler.Route + for _, route := range config.Routes { + if route.GW == "" { + first = append(first, route) + } else { + second = append(second, route) + } + } + routes := append(first, second...) + + for _, route := range routes { + var dst *net.IPNet + if route.Dst != "" { + var err error + _, dst, err = net.ParseCIDR(route.Dst) + if err != nil { + return fmt.Errorf("failed to add route destination %s: %w", route.Dst, err) + } + } + var gw net.IP + if route.GW != "" { + gw = net.ParseIP(route.GW) + if gw == nil { + return fmt.Errorf("failed to parse GW IP: %s", route.GW) + } + } + + if err := podNS.RouteAdd(0, dst, gw, podVethName); err != nil { + return fmt.Errorf("failed to add a route to %s via %s on pod network namespace %s: %w", dst, gw, nsPath, err) + } + + if checkDefaultRoute(dst) { + defaultRouteGateway = gw + } + } + + if defaultRouteGateway == nil { + return errors.New("no default route gateway is specified") + } + + if err := hostNS.AddrAdd(hostVethName, mask32(defaultRouteGateway)); err != nil { + return fmt.Errorf("failed to add GW IP %s to %s on host network namespace: %w", defaultRouteGateway, hostVethName, err) + } + + if err := hostNS.RouteAdd(podTableID, mask32(podIP), nil, hostVethName); err != nil { + return fmt.Errorf("failed to add route table %d to pod %s IP on host network namespace: %w", podTableID, podIP, err) + } + + if err := hostNS.RouteAdd(sourceTableID, nil, nodeIP, hostInterface); err != nil { + return fmt.Errorf("failed to add route table %d to pod %s IP on host network namespace: %w", sourceTableID, podIP, err) + } + + if err := hostNS.RuleAdd(nil, "", podTablePriority, podTableID); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add route table %d for pod IP at priority %d: %w", podTableID, podTablePriority, err) + } + + if err := hostNS.RuleAdd(mask32(podIP), hostVethName, sourceTablePriority, sourceTableID); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add route table %d for source routing at priority %d: %w", sourceTableID, sourceTablePriority, err) + } + + for key, val := range map[string]string{ + "net/ipv4/ip_forward": "1", + fmt.Sprintf("net/ipv4/conf/%s/proxy_arp", hostVethName): "1", + fmt.Sprintf("net/ipv4/neigh/%s/proxy_delay", hostVethName): "0", + } { + if err := hostNS.SysctlSet(key, val); err != nil { + return err + } + } + + if err := startKeepAlive(hostNS, podIP, net.JoinHostPort(nodeIP.String(), defaultKeepAliveListenPort), false); err != nil { + return err + } + + return nil +} + +func (t *podNodeTunneler) Teardown(nsPath, hostInterface string, config *tunneler.Config) error { + + podIP, _, err := net.ParseCIDR(config.PodIP) + if err != nil { + return fmt.Errorf("failed to parse pod IP: %w", err) + } + + if err := stopKeepAlive(podIP, false); err != nil { + logger.Printf("failed to stop keep alive: %v", err) + } + + return nil +} diff --git a/pkg/podnetwork/tunneler/routing/tunnel_test.go b/pkg/podnetwork/tunneler/routing/tunnel_test.go new file mode 100644 index 000000000..3f728a776 --- /dev/null +++ b/pkg/podnetwork/tunneler/routing/tunnel_test.go @@ -0,0 +1,19 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "testing" + + testutils "github.com/confidential-containers/cloud-api-adaptor/pkg/internal/testing" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tuntest" +) + +func TestRouting(t *testing.T) { + // TODO: enable this test once https://github.com/confidential-containers/cloud-api-adaptor/issues/52 is fixed + testutils.SkipTestIfRunningInCI(t) + + tuntest.RunTunnelTest(t, "routing", NewWorkerNodeTunneler, NewPodNodeTunneler, true) + +} diff --git a/pkg/podnetwork/tunneler/routing/workernode.go b/pkg/podnetwork/tunneler/routing/workernode.go new file mode 100644 index 000000000..8b0cae219 --- /dev/null +++ b/pkg/podnetwork/tunneler/routing/workernode.go @@ -0,0 +1,297 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "errors" + "fmt" + "log" + "net" + "os" + "time" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +var logger = log.New(log.Writer(), "[tunneler/routing] ", log.LstdFlags|log.Lmsgprefix) + +const ( + sourceRouteTablePriority = 505 + + secondPodInterface = "eth1" + + vrf1Name = "ppvrf1" + vrf2Name = "ppvrf2" + + vrf1TableID = 49001 + vrf2TableID = 49002 + minTableID = 50000 + maxTableID = 59999 + + daemonListenPort = "15150" +) + +const ( + vethPrefix = "ppveth" +) + +type workerNodeTunneler struct { +} + +func NewWorkerNodeTunneler() tunneler.Tunneler { + return &workerNodeTunneler{} +} + +func (t *workerNodeTunneler) Setup(nsPath string, podNodeIPs []net.IP, config *tunneler.Config) error { + + if !config.Dedicated { + return errors.New("shared subnet is not supported") + } + + if len(podNodeIPs) != 2 { + return errors.New("secondary pod node IP is not available") + } + + podNodeIP := podNodeIPs[1] + + hostNS, err := netops.GetNS() + if err != nil { + return fmt.Errorf("failed to get current network namespace: %w", err) + } + defer func() { + if e := hostNS.Close(); e != nil { + err = fmt.Errorf("failed to close the original network namespace: %w (previous error: %v)", e, err) + } + }() + + workerNodeIP, _, err := net.ParseCIDR(config.WorkerNodeIP) + if err != nil { + return fmt.Errorf("failed to parse worker node IP %q", config.WorkerNodeIP) + } + hostInterface, err := hostNS.LinkNameByAddr(workerNodeIP) + if err != nil { + return fmt.Errorf("failed to identify host interface that has %s on netns %s", workerNodeIP.String(), hostNS.Path) + } + + logger.Print("Ensure routing table entries and VRF devices on host") + + if err := hostNS.RuleAdd(nil, "", localTableNewPriority, unix.RT_TABLE_LOCAL); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add local table at priority %d: %w", localTableNewPriority, err) + } + if err = hostNS.RuleDel(nil, "", localTableOriginalPriority, unix.RT_TABLE_LOCAL); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to delete local table at priority %d: %w", localTableOriginalPriority, err) + } + + if err := hostNS.LinkAdd(vrf1Name, &netlink.Vrf{Table: vrf1TableID}); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add vrf %s: %w", vrf1Name, err) + } + + if err := hostNS.LinkSetUp(vrf1Name); err != nil { + return fmt.Errorf("failed to set vrf %s up: %w", vrf1Name, err) + } + + if err := hostNS.LinkSetMaster(hostInterface, vrf1Name); err != nil { + return fmt.Errorf("failed to set master of %s to vrf %s: %w", hostInterface, vrf1Name, err) + } + + if err := hostNS.LinkAdd(vrf2Name, &netlink.Vrf{Table: vrf2TableID}); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add vrf %s: %w", vrf2Name, err) + } + + if err := hostNS.LinkSetUp(vrf2Name); err != nil { + return fmt.Errorf("failed to set vrf %s up: %w", vrf2Name, err) + } + + podNS, err := netops.NewNSFromPath(nsPath) + if err != nil { + return fmt.Errorf("failed to get a network namespace: %s: %w", nsPath, err) + } + defer func() { + if e := podNS.Close(); e != nil { + err = fmt.Errorf("failed to close the pod network namespace: %w (previous error: %v)", e, err) + } + }() + + vethName, err := hostNS.VethAddPrefix(vethPrefix, podNS, secondPodInterface) + if err != nil { + return err + } + if err := podNS.LinkSetUp(secondPodInterface); err != nil { + return err + } + + logger.Printf("Create a veth pair between host and Pod network namespace %s", nsPath) + logger.Printf(" Host: %s", vethName) + logger.Printf(" Pod: %s", secondPodInterface) + + podIP, _, err := net.ParseCIDR(config.PodIP) + if err != nil { + return fmt.Errorf("failed to parse PodIP: %w", err) + } + + podInterface := config.InterfaceName + + logger.Printf("Add tc redirect filters between %s and %s on pod network namespace %s", podInterface, secondPodInterface, nsPath) + + if err := podNS.RedirectAdd(podInterface, secondPodInterface); err != nil { + return fmt.Errorf("failed to add a tc redirect filter from %s to %s: %w", podInterface, secondPodInterface, err) + } + + if err := podNS.RedirectAdd(secondPodInterface, podInterface); err != nil { + return fmt.Errorf("failed to add a tc redirect filter from %s to %s: %w", secondPodInterface, podInterface, err) + } + + if err := hostNS.LinkSetMaster(vethName, vrf2Name); err != nil { + return err + } + + hwAddr, err := podNS.GetHardwareAddr(podInterface) + if err != nil { + return fmt.Errorf("failed to get hardware address of pod interface %s (netns %s): %w", podInterface, podNS.Path, err) + } + + if err := hostNS.SetHardwareAddr(vethName, hwAddr); err != nil { + return fmt.Errorf("failed to set hardware address %q to veth interface %s (netns %s): %w", hwAddr, vethName, hostNS.Path, err) + } + + if err := hostNS.LinkSetUp(vethName); err != nil { + return err + } + + logger.Printf("Add a routing table entry to route traffic to Pod IP %s to PodVM IP %s", podIP, podNodeIP) + + // TODO: remove this sleep. + // Without this sleep, add route fails due to "failed to create a route: network is unreachable", + // when pod network is created for the first time + time.Sleep(time.Second) + + if err := hostNS.RouteAdd(vrf2TableID, mask32(podIP), podNodeIP, hostInterface); err != nil { + return fmt.Errorf("failed to add a route to pod VM: %w", err) + } + + logger.Printf("Add Pod IP %s to %s and delete local route", podIP, vethName) + // FIXME: Proxy arp does not become effective when no IP address is added to the interface, so we add pod IP to this interface, and delete its local route. + if err := hostNS.AddrAdd(vethName, mask32(podIP)); err != nil { + return err + } + if err := hostNS.LocalRouteDel(vrf2TableID, mask32(podIP), vethName); err != nil { + return err + } + + tableID := minTableID + for { + var err error + tableID, err = hostNS.GetAvailableTableID(vrf1Name, sourceRouteTablePriority, tableID, maxTableID) + if err != nil { + return err + } + if err := hostNS.RouteAddOnlink(tableID, nil, net.ParseIP(config.Routes[0].GW), vethName); err == nil { + break + } else if !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add a route from a pod VM to a pod proxy: %w", err) + } + tableID++ + } + logger.Printf("Add a routing table entry to route traffic from Pod VM %s back to pod network namespace %s", podNodeIP, nsPath) + if err := hostNS.RuleAdd(mask32(podIP), vrf1Name, sourceRouteTablePriority, tableID); err != nil { + return err + } + + logger.Printf("Enable proxy ARP on %s", vethName) + for key, val := range map[string]string{ + "net/ipv4/ip_forward": "1", + fmt.Sprintf("net/ipv4/conf/%s/accept_local", vethName): "1", + fmt.Sprintf("net/ipv4/conf/%s/proxy_arp", vethName): "1", + fmt.Sprintf("net/ipv4/neigh/%s/proxy_delay", vethName): "0", + } { + if err := hostNS.SysctlSet(key, val); err != nil { + return err + } + } + + if err := setIPTablesRules(hostNS, hostInterface); err != nil { + return err + } + + if err := startKeepAlive(hostNS, podIP, net.JoinHostPort(podNodeIP.String(), daemonListenPort), true); err != nil { + return err + } + + return nil +} + +func (t *workerNodeTunneler) Teardown(nsPath, hostInterface string, config *tunneler.Config) error { + + hostNS, err := netops.GetNS() + if err != nil { + return fmt.Errorf("failed to get current network namespace: %w", err) + } + defer func() { + if e := hostNS.Close(); e != nil { + err = fmt.Errorf("failed to close the original network namespace: %w (previous error: %v)", e, err) + } + }() + + podNS, err := netops.NewNSFromPath(nsPath) + if err != nil { + return fmt.Errorf("failed to get a network namespace: %s: %w", nsPath, err) + } + defer func() { + if e := podNS.Close(); e != nil { + err = fmt.Errorf("Failed close the pod network namespace: %w (previous error: %v)", e, err) + } + }() + + podIP, _, err := net.ParseCIDR(config.PodIP) + if err != nil { + return fmt.Errorf("failed to parse pod IP: %w", err) + } + + logger.Printf("Delete routing table entries for Pod IP %s", podIP) + + if err := hostNS.RouteDel(vrf2TableID, mask32(podIP), nil, hostInterface); err != nil { + return err + } + rules, err := hostNS.RuleList(mask32(podIP), vrf1Name, sourceRouteTablePriority, 0) + if err != nil { + return err + } + if len(rules) == 0 { + return fmt.Errorf("failed to identify rule %s vrf %s pref %d", podIP, vrf1Name, sourceRouteTablePriority) + } + for _, rule := range rules { + if rule.Table == 0 { + return fmt.Errorf("failed to identify table ID for rule %s vrf %s pref %d", podIP, vrf1Name, sourceRouteTablePriority) + } + err := hostNS.RuleDel(mask32(podIP), vrf1Name, sourceRouteTablePriority, rule.Table) + if err != nil { + return fmt.Errorf("failed to delete a rule %s vrf %s pref %d table %d: %w", podIP, vrf1Name, sourceRouteTablePriority, rule.Table, err) + } + } + + logger.Printf("Delete tc redirect filters on %s and %s in the network namespace %s", config.InterfaceName, hostInterface, nsPath) + + if err := podNS.RedirectDel(config.InterfaceName); err != nil { + return fmt.Errorf("failed to delete a tc redirect filter from %s to %s: %w", config.InterfaceName, secondPodInterface, err) + } + + if err := podNS.RedirectDel(secondPodInterface); err != nil { + return fmt.Errorf("failed to delete a tc redirect filter from %s to %s: %w", secondPodInterface, config.InterfaceName, err) + } + + logger.Printf("Delete veth %s in the network namespace %s", secondPodInterface, nsPath) + + if err := podNS.LinkDel(secondPodInterface); err != nil { + return fmt.Errorf("failed to delete a veth interface %s at %s: %w", secondPodInterface, podNS.Name, err) + } + + if err := stopKeepAlive(podIP, true); err != nil { + logger.Printf("failed to stop keep alive: %v", err) + } + + return nil +} diff --git a/pkg/podnetwork/tunneler/tunneler.go b/pkg/podnetwork/tunneler/tunneler.go new file mode 100644 index 000000000..5c49c5d07 --- /dev/null +++ b/pkg/podnetwork/tunneler/tunneler.go @@ -0,0 +1,75 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package tunneler + +import ( + "fmt" + "net" +) + +type Tunneler interface { + Setup(nsPath string, podNodeIPs []net.IP, config *Config) error + Teardown(nsPath, hostInterface string, config *Config) error +} + +type Config struct { + PodIP string + PodHwAddr string `json:"pod-hw-addr"` + Routes []*Route `json:"routes"` + InterfaceName string `json:"interface"` + MTU int `json:"mtu"` + WorkerNodeIP string `json:"worker-node-ip"` + TunnelType string `json:"tunnel-type"` + Dedicated bool `json:"dedicated"` + Index int `json:"index"` + VXLANPort int `json:"vxlan-port,omitempty"` + VXLANID int `json:"vxlan-id,omitempty"` +} + +type Route struct { + Dst string + GW string + Dev string +} + +type driver struct { + newWorkerNodeTunneler func() Tunneler + newPodNodeTunneler func() Tunneler +} + +var drivers = make(map[string]*driver) + +func Register(tunnelType string, newWorkerNodeTunneler, newPodNodeTunneler func() Tunneler) { + drivers[tunnelType] = &driver{ + newWorkerNodeTunneler: newWorkerNodeTunneler, + newPodNodeTunneler: newPodNodeTunneler, + } +} + +func getDriver(tunnelType string) (*driver, error) { + + driver, ok := drivers[tunnelType] + if !ok { + return nil, fmt.Errorf("unknown tunnel type: %q", tunnelType) + } + return driver, nil +} + +func WorkerNodeTunneler(tunnelType string) (Tunneler, error) { + + driver, err := getDriver(tunnelType) + if err != nil { + return nil, err + } + return driver.newWorkerNodeTunneler(), nil +} + +func PodNodeTunneler(tunnelType string) (Tunneler, error) { + + driver, err := getDriver(tunnelType) + if err != nil { + return nil, err + } + return driver.newPodNodeTunneler(), nil +} diff --git a/pkg/podnetwork/tunneler/vxlan/podnode.go b/pkg/podnetwork/tunneler/vxlan/podnode.go new file mode 100644 index 000000000..4d6b861d4 --- /dev/null +++ b/pkg/podnetwork/tunneler/vxlan/podnode.go @@ -0,0 +1,126 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package vxlan + +import ( + "fmt" + "net" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" + "github.com/vishvananda/netlink" +) + +const ( + podVxlanInterface = "vxlan0" + maxMTU = 1450 +) + +type podNodeTunneler struct { +} + +func NewPodNodeTunneler() tunneler.Tunneler { + return &podNodeTunneler{} +} + +func (t *podNodeTunneler) Setup(nsPath string, podNodeIPs []net.IP, config *tunneler.Config) error { + + nodeIP, _, err := net.ParseCIDR(config.WorkerNodeIP) + if err != nil { + return fmt.Errorf("failed to parse node IP %s: %w", config.WorkerNodeIP, err) + } + + podIP, podIPNet, err := net.ParseCIDR(config.PodIP) + if err != nil { + return fmt.Errorf("failed to parse pod IP %s: %w", config.PodIP, err) + } + podIPNet.IP = podIP + + hostNS, err := netops.GetNS() + if err != nil { + return fmt.Errorf("failed to get host network namespace: %w", err) + } + defer hostNS.Close() + + podNS, err := netops.NewNSFromPath(nsPath) + if err != nil { + return fmt.Errorf("failed to get a pod network namespace: %s: %w", nsPath, err) + } + defer podNS.Close() + + vxlanLink := &netlink.Vxlan{ + Group: nodeIP, + VxlanId: config.VXLANID, + Port: config.VXLANPort, + } + if err := hostNS.LinkAdd(podVxlanInterface, vxlanLink); err != nil { + return fmt.Errorf("failed to add vxlan interface %s: %w", podVxlanInterface, err) + } + + if err := hostNS.LinkSetNS(podVxlanInterface, podNS); err != nil { + return fmt.Errorf("failed to move vxlan interface %s to netns %s: %w", podVxlanInterface, podNS.Path, err) + } + + if err := podNS.SetHardwareAddr(podVxlanInterface, config.PodHwAddr); err != nil { + return fmt.Errorf("failed to set pod Mac %s on %s: %w", config.PodHwAddr, podVxlanInterface, err) + } + + mtu := int(config.MTU) + if mtu > maxMTU { + mtu = maxMTU + } + if err := podNS.SetMTU(podVxlanInterface, mtu); err != nil { + return fmt.Errorf("failed to set MTU of %s to %d on %s: %w", podVxlanInterface, mtu, nsPath, err) + } + + if err := podNS.AddrAdd(podVxlanInterface, podIPNet); err != nil { + return fmt.Errorf("failed to add pod IP %s to %s on %s: %w", podIPNet, podVxlanInterface, nsPath, err) + } + + if err := podNS.LinkSetUp(podVxlanInterface); err != nil { + return err + } + + // We need to process routes without gateway address first. Processing routes with a gateway causes an error if the gateway is not reachable. + // Calico sets up routes with this pattern. + // https://github.com/projectcalico/cni-plugin/blob/7495c0279c34faac315b82c1838bca638e23dbbe/pkg/dataplane/linux/dataplane_linux.go#L158-L167 + + var first, second []*tunneler.Route + for _, route := range config.Routes { + if route.GW == "" { + first = append(first, route) + } else { + second = append(second, route) + } + } + routes := append(first, second...) + + for _, route := range routes { + var dst *net.IPNet + if route.Dst != "" { + var err error + _, dst, err = net.ParseCIDR(route.Dst) + if err != nil { + return fmt.Errorf("failed to add route destination %s: %w", route.Dst, err) + } + } + var gw net.IP + if route.GW != "" { + gw = net.ParseIP(route.GW) + if gw == nil { + return fmt.Errorf("failed to parse GW IP: %s", route.GW) + } + } + + if err := podNS.RouteAdd(0, dst, gw, podVxlanInterface); err != nil { + return fmt.Errorf("failed to add a route to %s via %s on pod network namespace %s: %w", dst, gw, nsPath, err) + } + } + + return nil +} + +func (t *podNodeTunneler) Teardown(nsPath, hostInterface string, config *tunneler.Config) error { + return nil +} diff --git a/pkg/podnetwork/tunneler/vxlan/tunnel_test.go b/pkg/podnetwork/tunneler/vxlan/tunnel_test.go new file mode 100644 index 000000000..cb72ca4fa --- /dev/null +++ b/pkg/podnetwork/tunneler/vxlan/tunnel_test.go @@ -0,0 +1,16 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package vxlan + +import ( + "testing" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tuntest" +) + +func TestVXLAN(t *testing.T) { + + tuntest.RunTunnelTest(t, "vxlan", NewWorkerNodeTunneler, NewPodNodeTunneler, false) + +} diff --git a/pkg/podnetwork/tunneler/vxlan/workernode.go b/pkg/podnetwork/tunneler/vxlan/workernode.go new file mode 100644 index 000000000..0718753e7 --- /dev/null +++ b/pkg/podnetwork/tunneler/vxlan/workernode.go @@ -0,0 +1,173 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package vxlan + +import ( + "errors" + "fmt" + "log" + "net" + "os" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" + "github.com/vishvananda/netlink" +) + +var logger = log.New(log.Writer(), "[tunneler/vxlan] ", log.LstdFlags|log.Lmsgprefix) + +const ( + DefaultVXLANPort = 4789 + DefaultVXLANMinID = 555000 + hostVxlanInterfacePrefix = "ppvxlan" + secondPodInterface = "vxlan1" +) + +type workerNodeTunneler struct { +} + +func NewWorkerNodeTunneler() tunneler.Tunneler { + return &workerNodeTunneler{} +} + +func (t *workerNodeTunneler) Setup(nsPath string, podNodeIPs []net.IP, config *tunneler.Config) error { + + var dstAddr net.IP + + if config.Dedicated { + dstAddr = podNodeIPs[1] + } else { + dstAddr = podNodeIPs[0] + } + + hostNS, err := netops.GetNS() + if err != nil { + return fmt.Errorf("failed to get current network namespace: %w", err) + } + defer func() { + if e := hostNS.Close(); e != nil { + err = fmt.Errorf("failed to close the original network namespace: %w (previous error: %v)", e, err) + } + }() + + podNS, err := netops.NewNSFromPath(nsPath) + if err != nil { + return fmt.Errorf("failed to get a network namespace: %s: %w", nsPath, err) + } + defer func() { + if e := podNS.Close(); e != nil { + err = fmt.Errorf("failed to close the pod network namespace: %w (previous error: %v)", e, err) + } + }() + + index := 1 + + links, err := hostNS.LinkList() + if err != nil { + return fmt.Errorf("failed to get interfaces on host: %w", err) + } + + var hostVxlanInterface string + + for { + hostVxlanInterface = fmt.Sprintf("%s%d", hostVxlanInterfacePrefix, index) + var found bool + for _, link := range links { + if link.Attrs().Name == hostVxlanInterface { + found = true + break + } + } + + if !found { + + vxlanLink := &netlink.Vxlan{ + Group: dstAddr, + VxlanId: config.VXLANID, + Port: config.VXLANPort, + } + logger.Printf("vxlan %s (remote %s:%d, id: %d) created at %s", hostVxlanInterface, dstAddr.String(), config.VXLANPort, config.VXLANID, hostNS.Path) + err := hostNS.LinkAdd(hostVxlanInterface, vxlanLink) + if err == nil { + logger.Printf("vxlan %s created at %s", hostVxlanInterface, hostNS.Path) + break + } + logger.Printf("vxlan %s created at %s: %v", hostVxlanInterface, hostNS.Path, err) + if !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add vxlan interface %s: %w", hostVxlanInterface, err) + } + } + index++ + if index > 5 { + return fmt.Errorf("failed to create vxlan interface %s: too many", hostVxlanInterface) + } + } + + if err := hostNS.LinkSetNS(hostVxlanInterface, podNS); err != nil { + return fmt.Errorf("failed to move vxlan interface %s to netns %s: %w", hostVxlanInterface, podNS.Path, err) + } + logger.Printf("vxlan %s is moved to %s", hostVxlanInterface, podNS.Path) + + if err := podNS.LinkSetName(hostVxlanInterface, secondPodInterface); err != nil { + return fmt.Errorf("failed to change vxlan interface name %s on netns %s to %s: %w", hostVxlanInterface, podNS.Path, secondPodInterface, err) + } + + if err := podNS.LinkSetUp(secondPodInterface); err != nil { + return err + } + + podInterface := config.InterfaceName + + logger.Printf("Add tc redirect filters between %s and %s on pod network namespace %s", podInterface, secondPodInterface, nsPath) + + if err := podNS.RedirectAdd(podInterface, secondPodInterface); err != nil { + return fmt.Errorf("failed to add a tc redirect filter from %s to %s: %w", podInterface, secondPodInterface, err) + } + + if err := podNS.RedirectAdd(secondPodInterface, podInterface); err != nil { + return fmt.Errorf("failed to add a tc redirect filter from %s to %s: %w", secondPodInterface, podInterface, err) + } + + return nil +} + +func (t *workerNodeTunneler) Teardown(nsPath, hostInterface string, config *tunneler.Config) error { + + hostNS, err := netops.GetNS() + if err != nil { + return fmt.Errorf("failed to get current network namespace: %w", err) + } + defer func() { + if e := hostNS.Close(); e != nil { + err = fmt.Errorf("failed to close the original network namespace: %w (previous error: %v)", e, err) + } + }() + + podNS, err := netops.NewNSFromPath(nsPath) + if err != nil { + return fmt.Errorf("failed to get a network namespace: %s: %w", nsPath, err) + } + defer func() { + if e := podNS.Close(); e != nil { + err = fmt.Errorf("failed close the pod network namespace: %w (previous error: %v)", e, err) + } + }() + + logger.Printf("Delete tc redirect filters on %s and %s in the network namespace %s", config.InterfaceName, hostInterface, nsPath) + + if err := podNS.RedirectDel(config.InterfaceName); err != nil { + return fmt.Errorf("failed to delete a tc redirect filter from %s to %s: %w", config.InterfaceName, secondPodInterface, err) + } + + if err := podNS.RedirectDel(secondPodInterface); err != nil { + return fmt.Errorf("failed to delete a tc redirect filter from %s to %s: %w", secondPodInterface, config.InterfaceName, err) + } + + logger.Printf("Delete vxlan interface %s in the network namespace %s", secondPodInterface, nsPath) + + if err := podNS.LinkDel(secondPodInterface); err != nil { + return fmt.Errorf("failed to delete vxlan interface %s at %s: %w", secondPodInterface, podNS.Name, err) + } + return nil +} diff --git a/pkg/podnetwork/tuntest/tuntest.go b/pkg/podnetwork/tuntest/tuntest.go new file mode 100644 index 000000000..5c9dbc49d --- /dev/null +++ b/pkg/podnetwork/tuntest/tuntest.go @@ -0,0 +1,210 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package tuntest + +import ( + "fmt" + "net" + "net/http" + "testing" + + testutils "github.com/confidential-containers/cloud-api-adaptor/pkg/internal/testing" + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" + "github.com/coreos/go-iptables/iptables" +) + +type testPod struct { + podAddr string + podHwAddr string + podNodePrimaryAddr string + podNodeSecondaryAddr string + workerPodNS, podNS, podNodeNS *netops.NS + config *tunneler.Config + hostInterface string + workerNodeTunneler tunneler.Tunneler + podNodeTunneler tunneler.Tunneler +} + +func getIP(t *testing.T, addr string) string { + t.Helper() + + ip, _, err := net.ParseCIDR(addr) + if err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + return ip.String() +} + +func RunTunnelTest(t *testing.T, tunnelType string, newWorkerNodeTunneler, newPodNodeTunneler func() tunneler.Tunneler, dedicated bool) { + testutils.SkipTestIfNotRoot(t) + + const ( + gatewayIP = "10.128.0.1" + gatewayAddr = gatewayIP + "/24" + workerPrimaryAddr = "10.10.0.1/16" + workerSecondaryAddr = "192.168.0.1/24" + ) + + pods := []*testPod{ + {podAddr: "10.128.0.2/24", podHwAddr: "0a:58:0a:84:03:ce", podNodePrimaryAddr: "10.10.1.2/16", podNodeSecondaryAddr: "192.168.0.2/24"}, + {podAddr: "10.128.0.3/24", podHwAddr: "0a:58:0a:84:03:cf", podNodePrimaryAddr: "10.10.1.3/16", podNodeSecondaryAddr: "192.168.0.3/24"}, + } + + bridgeNS := NewNamedNS(t, "test-bridge") + defer DeleteNamedNS(t, bridgeNS) + + workerNS := NewNamedNS(t, "test-worker") + defer DeleteNamedNS(t, workerNS) + + if err := workerNS.Run(func() error { + ipt, err := iptables.New(iptables.IPFamily(iptables.ProtocolIPv4)) + if err != nil { + return err + } + if err := ipt.Append("filter", "FORWARD", "-i", "cni0", "-j", "ACCEPT"); err != nil { + return err + } + return ipt.ChangePolicy("filter", "FORWARD", "DROP") + }); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + BridgeAdd(t, bridgeNS, "br0") + BridgeAdd(t, bridgeNS, "br1") + + VethAdd(t, workerNS, "enc0", bridgeNS, "worker-eth0") + VethAdd(t, workerNS, "enc1", bridgeNS, "worker-eth1") + + LinkSetMaster(t, bridgeNS, "worker-eth0", "br0") + LinkSetMaster(t, bridgeNS, "worker-eth1", "br1") + + BridgeAdd(t, workerNS, "cni0") + + AddrAdd(t, workerNS, "cni0", gatewayAddr) + AddrAdd(t, workerNS, "enc0", workerPrimaryAddr) + AddrAdd(t, workerNS, "enc1", workerSecondaryAddr) + + RouteAdd(t, workerNS, "", "10.10.254.1", "enc0") + AddrAdd(t, bridgeNS, "br0", "10.10.254.1/16") + + for i, pod := range pods { + + pod.workerNodeTunneler = newWorkerNodeTunneler() + pod.podNodeTunneler = newPodNodeTunneler() + + pod.workerPodNS = NewNamedNS(t, fmt.Sprintf("test-workerpod%d", i)) + defer DeleteNamedNS(t, pod.workerPodNS) + + veth := fmt.Sprintf("veth%d", i) + VethAdd(t, workerNS, veth, pod.workerPodNS, "eth0") + LinkSetMaster(t, workerNS, veth, "cni0") + + AddrAdd(t, pod.workerPodNS, "eth0", pod.podAddr) + HwAddrAdd(t, pod.workerPodNS, "eth0", pod.podHwAddr) + RouteAdd(t, pod.workerPodNS, "", gatewayIP, "eth0") + + pod.podNodeNS = NewNamedNS(t, fmt.Sprintf("test-podvm%d", i)) + defer DeleteNamedNS(t, pod.podNodeNS) + + vmEth0 := fmt.Sprintf("podvm%d-eth0", i) + vmEth1 := fmt.Sprintf("podvm%d-eth1", i) + + VethAdd(t, pod.podNodeNS, "enc0", bridgeNS, vmEth0) + VethAdd(t, pod.podNodeNS, "enc1", bridgeNS, vmEth1) + AddrAdd(t, pod.podNodeNS, "enc0", pod.podNodePrimaryAddr) + AddrAdd(t, pod.podNodeNS, "enc1", pod.podNodeSecondaryAddr) + LinkSetMaster(t, bridgeNS, vmEth0, "br0") + LinkSetMaster(t, bridgeNS, vmEth1, "br1") + + pod.podNS = NewNamedNS(t, fmt.Sprintf("test-pod%d", i)) + defer DeleteNamedNS(t, pod.podNS) + } + + for i, pod := range pods { + + pod.config = &tunneler.Config{ + PodIP: pod.podAddr, + PodHwAddr: pod.podHwAddr, + Routes: []*tunneler.Route{{Dst: "", GW: "10.128.0.1", Dev: ""}}, + InterfaceName: "eth0", + MTU: 1500, + TunnelType: tunnelType, + Dedicated: dedicated, + Index: i, + } + + if tunnelType == "vxlan" { + pod.config.VXLANPort = 4789 // vxlan.DefaultVXLANPort + pod.config.VXLANID = 555000 + i // vxlan.DefaultVXLANMinID + index + } + + podNodeIPs := []net.IP{net.ParseIP(getIP(t, pod.podNodePrimaryAddr))} + + if dedicated { + podNodeIPs = append(podNodeIPs, net.ParseIP(getIP(t, pod.podNodeSecondaryAddr))) + pod.hostInterface = "enc1" + pod.config.WorkerNodeIP = workerSecondaryAddr + } else { + pod.hostInterface = "enc0" + pod.config.WorkerNodeIP = workerPrimaryAddr + } + + if err := workerNS.Run(func() error { + return pod.workerNodeTunneler.Setup(pod.workerPodNS.Path, podNodeIPs, pod.config) + + }); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + go func() { + if err := pod.podNodeNS.Run(func() error { + httpServer := http.Server{ + Addr: net.JoinHostPort("0.0.0.0", "15150"), + } + return httpServer.ListenAndServe() + }); err != nil { + t.Logf("Expect no error, got %v", err) + t.Fail() + } + }() + + if err := pod.podNodeNS.Run(func() error { + return pod.podNodeTunneler.Setup(pod.podNS.Path, podNodeIPs, pod.config) + + }); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + } + + for _, pod := range pods { + httpServer := StartHTTPServer(t, pod.podNS, fmt.Sprintf("%s:8080", getIP(t, pod.podAddr))) + defer httpServer.Shutdown(t) + } + + for i, pod := range pods { + ConnectToHTTPServer(t, workerNS, fmt.Sprintf("%s:8080", getIP(t, pod.podAddr)), getIP(t, gatewayAddr)) + ConnectToHTTPServer(t, pod.podNS, fmt.Sprintf("%s:8080", getIP(t, pods[(i+1)%len(pods)].podAddr)), getIP(t, pod.podAddr)) + } + + for _, pod := range pods { + + if err := workerNS.Run(func() error { + + return pod.workerNodeTunneler.Teardown(pod.workerPodNS.Path, pod.hostInterface, pod.config) + + }); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + if err := pod.podNodeNS.Run(func() error { + + return pod.podNodeTunneler.Teardown(pod.podNS.Path, pod.hostInterface, pod.config) + + }); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + } +} diff --git a/pkg/podnetwork/tuntest/util.go b/pkg/podnetwork/tuntest/util.go new file mode 100644 index 000000000..c7604c7e6 --- /dev/null +++ b/pkg/podnetwork/tuntest/util.go @@ -0,0 +1,279 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package tuntest + +import ( + "context" + "errors" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "testing" + "time" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" +) + +func NewNamedNS(t *testing.T, prefix string) *netops.NS { + t.Helper() + + var ns *netops.NS + name := prefix + index := 1 + for { + var err error + ns, err = netops.NewNamedNS(name) + if err == nil { + break + } + if !errors.Is(err, os.ErrExist) { + t.Fatalf("failed to create a named network namespace %s: %v", name, err) + } + index++ + name = fmt.Sprintf("%s-%d", prefix, index) + } + + if err := ns.LinkSetUp("lo"); err != nil { + t.Fatalf("failed to set the link up: lo") + } + + return ns +} + +func DeleteNamedNS(t *testing.T, ns *netops.NS) { + t.Helper() + + if err := ns.Close(); err != nil { + t.Fatal("failed to close a network namespace") + } + if err := ns.Delete(); err != nil { + t.Fatalf("failed to delete a named network namespace: %s: %v", ns.Name, err) + } +} + +func BridgeAdd(t *testing.T, ns *netops.NS, name string) { + t.Helper() + + attrs := netlink.NewLinkAttrs() + attrs.Namespace = ns + if err := ns.LinkAdd(name, &netlink.Bridge{LinkAttrs: attrs}); err != nil { + t.Fatalf("failed to create a bridge: %v", err) + } + if err := ns.LinkSetUp(name); err != nil { + t.Fatalf("failed to set the link up: %s: %v", name, err) + } +} + +func LinkSetMaster(t *testing.T, ns *netops.NS, name, masterName string) { + t.Helper() + + if err := ns.LinkSetMaster(name, masterName); err != nil { + t.Fatalf("failed to set the master link of %s to %s: %v", name, masterName, err) + } +} + +func VethAdd(t *testing.T, ns *netops.NS, name string, peer *netops.NS, peerName string) { + t.Helper() + + if err := ns.VethAdd(name, peer, peerName); err != nil { + t.Fatalf("failed to create a veth pair between two namespaces: %v", err) + } + if err := ns.LinkSetUp(name); err != nil { + t.Fatalf("failed to set the link up: %s: %v", name, err) + } + if err := peer.LinkSetUp(peerName); err != nil { + t.Fatalf("failed to set the link up: %s: %v", peerName, err) + } +} + +func AddrAdd(t *testing.T, ns *netops.NS, name, addr string) { + t.Helper() + + ip, err := netlink.ParseIPNet(addr) + if err != nil { + t.Fatalf("failed to parse IP %s: %v", addr, err) + } + if err := ns.AddrAdd(name, ip); err != nil { + t.Fatalf("failed to add %s to %s: %v", addr, name, err) + } +} + +func HwAddrAdd(t *testing.T, ns *netops.NS, name, hwAddr string) { + t.Helper() + + if err := ns.SetHardwareAddr(name, hwAddr); err != nil { + t.Fatalf("failed to add %s to %s: %v", hwAddr, name, err) + } +} + +func RouteAdd(t *testing.T, ns *netops.NS, dest, gw, dev string) { + t.Helper() + + if dest == "" { + dest = "0.0.0.0/0" + } + _, destNet, err := net.ParseCIDR(dest) + if err != nil { + t.Fatalf("failed to parse CIDR %s: %v", dest, err) + } + var gwIP net.IP + if gw != "" { + gwIP = net.ParseIP(gw) + if gwIP == nil { + t.Fatalf("failed to parse IP %s: %v", gw, err) + } + } + if err := ns.RouteAdd(0, destNet, gwIP, dev); err != nil { + t.Fatalf("failed to add a route to %s via %s: %v", dest, gw, err) + } +} + +type httpHandler string + +const testHTTPHandler = httpHandler("Hello") + +func (h httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "%s", string(h)) +} + +type TestHTTPServer struct { + ns *netops.NS + listener net.Listener + server *http.Server +} + +func StartHTTPServer(t *testing.T, ns *netops.NS, addr string) *TestHTTPServer { + t.Helper() + s := &TestHTTPServer{ + ns: ns, + } + + err := ns.Run(func() error { + + var err error + s.server = &http.Server{ + Handler: testHTTPHandler, + } + s.listener, err = net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("failed to listen on %s: %v", addr, err) + } + go func() { + err := s.server.Serve(s.listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + fmt.Fprintf(os.Stderr, "failed to start an HTTP server: %v", err) + } + }() + return nil + }) + if err != nil { + t.Fatalf("failed to run a function at a network namespace: %v", err) + } + return s +} + +func (s *TestHTTPServer) Shutdown(t *testing.T) { + t.Helper() + + err := s.ns.Run(func() error { + err := s.server.Shutdown(context.Background()) + s.listener.Close() + return err + }) + if err != nil { + t.Fatal("failed to run a function at a network namespace") + } +} + +func ConnectToHTTPServer(t *testing.T, ns *netops.NS, addr, localAddr string) { + t.Helper() + + var tcpAddr net.Addr + if localAddr != "" { + var err error + tcpAddr, err = net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:0", localAddr)) + if err != nil { + t.Fatalf("failed to get TCP address: %s", localAddr) + } + } + + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, address string) (conn net.Conn, err error) { + if e := ns.Run(func() error { + d := &net.Dialer{ + LocalAddr: tcpAddr, + } + conn, err = d.DialContext(ctx, network, address) + return nil + }); e != nil { + t.Fatalf("failed to run a dialer at network namespace %s", ns.Name) + } + return conn, err + }, + }, + } + + if err := ns.Run(func() error { + + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s", addr), nil) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(req.Context(), 5*time.Second) + defer cancel() + + req = req.WithContext(ctx) + + res, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to get an http response at %s from http://%s : %v", ns.Name, addr, err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("failed to read an http response at %s from http://%s:: %v", ns.Name, addr, err) + } + content := string(body) + if content != string(testHTTPHandler) { + return fmt.Errorf("unexpected response at %s from the HTTP server: %s", ns.Name, content) + } + return nil + + }); err != nil { + t.Fatalf("failed to run a HTTP client at a network namespace: %v", err) + } +} + +func CheckVRF(t *testing.T, ns *netops.NS) { + t.Helper() + + name := "vrf0dummy" + + if err := ns.LinkAdd(name, &netlink.Vrf{Table: 12345}); err != nil { + + if errors.Is(err, unix.ENOTSUP) { + t.Log( + "vrf is not enabled in the Linux kernel\n", + "========================================================\n", + " Please load the vrf module.\n", + " In Ubuntu, run the following commands.\n", + " apt-get install \"linux-modules-extra-$(uname -r)\"\n", + " modprobe vrf\n", + "========================================================\n", + ) + } + + t.Fatalf("failed to create a VRF interface: %v\n", err) + } + + if err := ns.LinkDel("vrf0dummy"); err != nil { + t.Fatalf("failed to delete a VRF interface %s: %v\n", name, err) + } +} diff --git a/pkg/podnetwork/workernode.go b/pkg/podnetwork/workernode.go new file mode 100644 index 000000000..ae3f85cde --- /dev/null +++ b/pkg/podnetwork/workernode.go @@ -0,0 +1,221 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package podnetwork + +import ( + "fmt" + "net" + "sync" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork/tunneler" + "github.com/confidential-containers/cloud-api-adaptor/pkg/util/netops" +) + +const DefaultTunnelType = "vxlan" + +type WorkerNode interface { + Inspect(nsPath string) (*tunneler.Config, error) + Setup(nsPath string, podNodeIPs []net.IP, config *tunneler.Config) error + Teardown(nsPath string, config *tunneler.Config) error +} + +type workerNode struct { + tunnelType string + hostInterface string + vxlanPort int + vxlanMinID int +} + +// TODO: Pod index is reset when this process restarts. +// We need to manage a persistent unique index number for each pod VM +var podIndexManager podIndex + +type podIndex struct { + index int + mutex sync.Mutex +} + +func (p *podIndex) Get() int { + p.mutex.Lock() + defer p.mutex.Unlock() + + index := p.index + p.index++ + return index +} + +func NewWorkerNode(tunnelType, hostInterface string, vxlanPort, vxlanMinID int) WorkerNode { + + return &workerNode{ + tunnelType: tunnelType, + hostInterface: hostInterface, + vxlanPort: vxlanPort, + vxlanMinID: vxlanMinID, + } +} + +func (n *workerNode) Inspect(nsPath string) (*tunneler.Config, error) { + + config := &tunneler.Config{ + TunnelType: n.tunnelType, + Index: podIndexManager.Get(), + } + + hostNS, err := netops.GetNS() + if err != nil { + return nil, fmt.Errorf("failed to open the host network namespace: %w", err) + } + defer func() { + if err := hostNS.Close(); err != nil { + logger.Printf("failed to close the host network namespace: %v", err) + } + }() + + _, hostPrimaryInterface, err := getRoutes(hostNS) + if err != nil { + return nil, fmt.Errorf("failed to identify the host primary interface: %w", err) + } + + hostInterface := n.hostInterface + if hostInterface == "" { + hostInterface = hostPrimaryInterface + } else if hostInterface != hostPrimaryInterface { + config.Dedicated = true + } + + addrs, err := hostNS.GetIPNet(hostInterface) + if err != nil { + return nil, fmt.Errorf("failed to get IP address on %s (netns: %s): %w", hostInterface, hostNS.Path, err) + } + if len(addrs) != 1 { + logger.Printf("more than one IP address (%v) assigned on %s (netns: %s)", addrs, hostInterface, hostNS.Path) + } + // Use the first IP as the workerNodeIP + // TBD: Might be faster to retrieve using K8s downward API + config.WorkerNodeIP = addrs[0].String() + + podNS, err := netops.NewNSFromPath(nsPath) + if err != nil { + return nil, fmt.Errorf("failed to open network namespace %q: %w", nsPath, err) + } + defer func() { + if err := podNS.Close(); err != nil { + logger.Printf("failed to close a network namespace: %q", podNS.Path) + } + }() + + routes, podInterface, err := getRoutes(podNS) + if err != nil { + return nil, err + } + + podIP, err := getPodIP(podNS, podInterface) + if err != nil { + return nil, err + } + + config.PodIP = podIP + config.PodHwAddr, err = podNS.GetHardwareAddr(podInterface) + if err != nil { + logger.Printf("failed to get Mac address of the Pod interface") + return nil, fmt.Errorf("failed to get Mac address for Pod interface %s: %w", podInterface, err) + } + + config.InterfaceName = podInterface + + mtu, err := podNS.GetMTU(podInterface) + if err != nil { + return nil, fmt.Errorf("failed to get MTU size of %s: %w", podInterface, err) + } + config.MTU = mtu + + for _, route := range routes { + r := &tunneler.Route{ + Dev: route.Dev, + } + if route.Dst != nil { + r.Dst = route.Dst.String() + } + if route.GW != nil { + r.GW = route.GW.String() + } + config.Routes = append(config.Routes, r) + } + + if n.tunnelType == "vxlan" { + config.VXLANPort = n.vxlanPort + config.VXLANID = n.vxlanMinID + config.Index + } + + return config, nil +} + +func (n *workerNode) Setup(nsPath string, podNodeIPs []net.IP, config *tunneler.Config) error { + + tun, err := tunneler.WorkerNodeTunneler(n.tunnelType) + if err != nil { + return fmt.Errorf("failed to get tunneler: %w", err) + } + + if err := tun.Setup(nsPath, podNodeIPs, config); err != nil { + return fmt.Errorf("failed to set up tunnel %q: %w", config.TunnelType, err) + } + + return nil +} + +func (n *workerNode) Teardown(nsPath string, config *tunneler.Config) error { + + tun, err := tunneler.WorkerNodeTunneler(n.tunnelType) + if err != nil { + return fmt.Errorf("failed to get tunneler: %w", err) + } + + hostNS, err := netops.GetNS() + if err != nil { + return fmt.Errorf("failed to open the host network namespace: %w", err) + } + defer func() { + if err := hostNS.Close(); err != nil { + logger.Printf("failed to close the host network namespace: %v", err) + } + }() + + hostInterface := n.hostInterface + if hostInterface == "" { + _, hostPrimaryInterface, err := getRoutes(hostNS) + if err != nil { + return fmt.Errorf("failed to identify the host primary interface: %w", err) + } + hostInterface = hostPrimaryInterface + } + + if err := tun.Teardown(nsPath, hostInterface, config); err != nil { + return fmt.Errorf("failed to tear down tunnel %q: %w", config.TunnelType, err) + } + + return nil +} + +func getPodIP(podNS *netops.NS, podInterface string) (string, error) { + + ipNets, err := podNS.GetIPNet(podInterface) + if err != nil { + return "", fmt.Errorf("failed to get IP address on %s of netns %s: %w", podInterface, podNS.Path, err) + } + + var ips []string + for _, ipNet := range ipNets { + if ipNet.IP.To4() != nil { + ips = append(ips, ipNet.String()) + } + } + if len(ips) < 1 { + return "", fmt.Errorf("no IPv4 address found on %s of netns %s", podInterface, podNS.Path) + } + if len(ips) > 1 { + return "", fmt.Errorf("more than one IPv4 addresses found on %s of netns %s", podInterface, podNS.Path) + } + return ips[0], nil +} diff --git a/pkg/util/agentproto/redirector.go b/pkg/util/agentproto/redirector.go new file mode 100644 index 000000000..f9224208d --- /dev/null +++ b/pkg/util/agentproto/redirector.go @@ -0,0 +1,397 @@ +// Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package agentproto + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + + "github.com/containerd/ttrpc" + "github.com/gogo/protobuf/types" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" + + pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" +) + +type Redirector interface { + pb.AgentServiceService + pb.ImageService + pb.HealthService + + Connect(ctx context.Context) error + Close() error +} + +type redirector struct { + agentClient *client + ttrpcClient *ttrpc.Client + dialer func(context.Context) (net.Conn, error) + once sync.Once +} + +type client struct { + pb.AgentServiceService + pb.ImageService + pb.HealthService +} + +func NewRedirector(dialer func(context.Context) (net.Conn, error)) Redirector { + + return &redirector{ + dialer: dialer, + } +} + +func (s *redirector) Connect(ctx context.Context) error { + + var err error + + s.once.Do(func() { + + conn, e := s.dialer(ctx) + if e != nil { + err = e + return + } + + s.ttrpcClient = ttrpc.NewClient(conn) + + s.agentClient = &client{ + AgentServiceService: pb.NewAgentServiceClient(s.ttrpcClient), + ImageService: pb.NewImageClient(s.ttrpcClient), + HealthService: pb.NewHealthClient(s.ttrpcClient), + } + }) + + if err != nil { + return fmt.Errorf("agent connection is not established: %w", err) + } + + if s.agentClient == nil { + return errors.New("agent connection is not established") + } + + return nil +} + +func (s *redirector) Close() error { + client := s.ttrpcClient + if client == nil { + return nil + } + return client.Close() +} + +// AgentServiceService methods + +func (s *redirector) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.CreateContainer(ctx, req) +} + +func (s *redirector) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.StartContainer(ctx, req) +} + +func (s *redirector) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.RemoveContainer(ctx, req) +} + +func (s *redirector) ExecProcess(ctx context.Context, req *pb.ExecProcessRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.ExecProcess(ctx, req) +} + +func (s *redirector) SignalProcess(ctx context.Context, req *pb.SignalProcessRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.SignalProcess(ctx, req) +} + +func (s *redirector) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest) (res *pb.WaitProcessResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.WaitProcess(ctx, req) +} + +func (s *redirector) UpdateContainer(ctx context.Context, req *pb.UpdateContainerRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.UpdateContainer(ctx, req) +} + +func (s *redirector) StatsContainer(ctx context.Context, req *pb.StatsContainerRequest) (res *pb.StatsContainerResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.StatsContainer(ctx, req) +} + +func (s *redirector) PauseContainer(ctx context.Context, req *pb.PauseContainerRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.PauseContainer(ctx, req) +} + +func (s *redirector) ResumeContainer(ctx context.Context, req *pb.ResumeContainerRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.ResumeContainer(ctx, req) +} + +func (s *redirector) WriteStdin(ctx context.Context, req *pb.WriteStreamRequest) (res *pb.WriteStreamResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.WriteStdin(ctx, req) +} + +func (s *redirector) ReadStdout(ctx context.Context, req *pb.ReadStreamRequest) (res *pb.ReadStreamResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.ReadStdout(ctx, req) +} + +func (s *redirector) ReadStderr(ctx context.Context, req *pb.ReadStreamRequest) (res *pb.ReadStreamResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.ReadStderr(ctx, req) +} + +func (s *redirector) CloseStdin(ctx context.Context, req *pb.CloseStdinRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.CloseStdin(ctx, req) +} + +func (s *redirector) TtyWinResize(ctx context.Context, req *pb.TtyWinResizeRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.TtyWinResize(ctx, req) +} + +func (s *redirector) UpdateInterface(ctx context.Context, req *pb.UpdateInterfaceRequest) (res *protocols.Interface, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.UpdateInterface(ctx, req) +} + +func (s *redirector) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (res *pb.Routes, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.UpdateRoutes(ctx, req) +} + +func (s *redirector) ListInterfaces(ctx context.Context, req *pb.ListInterfacesRequest) (res *pb.Interfaces, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.ListInterfaces(ctx, req) +} + +func (s *redirector) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) (res *pb.Routes, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.ListRoutes(ctx, req) +} + +func (s *redirector) AddARPNeighbors(ctx context.Context, req *pb.AddARPNeighborsRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.AddARPNeighbors(ctx, req) +} + +func (s *redirector) GetIPTables(ctx context.Context, req *pb.GetIPTablesRequest) (res *pb.GetIPTablesResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.GetIPTables(ctx, req) +} + +func (s *redirector) SetIPTables(ctx context.Context, req *pb.SetIPTablesRequest) (res *pb.SetIPTablesResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.SetIPTables(ctx, req) +} + +func (s *redirector) GetMetrics(ctx context.Context, req *pb.GetMetricsRequest) (res *pb.Metrics, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.GetMetrics(ctx, req) +} + +func (s *redirector) CreateSandbox(ctx context.Context, req *pb.CreateSandboxRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.CreateSandbox(ctx, req) +} + +func (s *redirector) DestroySandbox(ctx context.Context, req *pb.DestroySandboxRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.DestroySandbox(ctx, req) +} + +func (s *redirector) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.OnlineCPUMem(ctx, req) +} + +func (s *redirector) ReseedRandomDev(ctx context.Context, req *pb.ReseedRandomDevRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.ReseedRandomDev(ctx, req) +} + +func (s *redirector) GetGuestDetails(ctx context.Context, req *pb.GuestDetailsRequest) (res *pb.GuestDetailsResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.GetGuestDetails(ctx, req) +} + +func (s *redirector) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByProbeRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.MemHotplugByProbe(ctx, req) +} + +func (s *redirector) SetGuestDateTime(ctx context.Context, req *pb.SetGuestDateTimeRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.SetGuestDateTime(ctx, req) +} + +func (s *redirector) CopyFile(ctx context.Context, req *pb.CopyFileRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.CopyFile(ctx, req) +} + +func (s *redirector) GetOOMEvent(ctx context.Context, req *pb.GetOOMEventRequest) (res *pb.OOMEvent, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.GetOOMEvent(ctx, req) +} + +func (s *redirector) AddSwap(ctx context.Context, req *pb.AddSwapRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.AddSwap(ctx, req) +} + +func (s *redirector) GetVolumeStats(ctx context.Context, req *pb.VolumeStatsRequest) (res *pb.VolumeStatsResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.GetVolumeStats(ctx, req) +} + +func (s *redirector) ResizeVolume(ctx context.Context, req *pb.ResizeVolumeRequest) (res *types.Empty, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.ResizeVolume(ctx, req) +} + +// ImageService method + +func (s *redirector) PullImage(ctx context.Context, req *pb.PullImageRequest) (res *pb.PullImageResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.PullImage(ctx, req) +} + +// HealthService methods + +func (s *redirector) Check(ctx context.Context, req *pb.CheckRequest) (res *pb.HealthCheckResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.Check(ctx, req) +} + +func (s *redirector) Version(ctx context.Context, req *pb.CheckRequest) (res *pb.VersionCheckResponse, err error) { + + if err := s.Connect(ctx); err != nil { + return nil, err + } + return s.agentClient.Version(ctx, req) +} diff --git a/pkg/util/cloudinit/cloudconfig.go b/pkg/util/cloudinit/cloudconfig.go new file mode 100644 index 000000000..6678b70af --- /dev/null +++ b/pkg/util/cloudinit/cloudconfig.go @@ -0,0 +1,105 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package cloudinit + +import ( + "bytes" + "encoding/base64" + "fmt" + "strings" + "text/template" +) + +const ( + DefaultAuthfileSrcPath = "/root/containers/auth.json" + // image-rs fixed dst path for support at the agent, we convert it explictly to the resources file format + // e.g. https://github.com/confidential-containers/attestation-agent/blob/main/src/kbc_modules/offline_fs_kbc/aa-offline_fs_kbc-resources.json + DefaultAuthfileDstPath = "/etc/aa-offline_fs_kbc-resources.json" + DefaultAuthfileLimit = 12288 // TODO: use a whole userdata limit mechanism instead of limiting authfile +) + +// https://cloudinit.readthedocs.io/en/latest/topics/format.html#cloud-config-data + +type CloudConfig struct { + WriteFiles []WriteFile `yaml:"write_files"` +} + +// https://cloudinit.readthedocs.io/en/latest/topics/modules.html#write-files +type WriteFile struct { + Path string `yaml:"path"` + Content string `yaml:"content,omitempty"` + Owner string `yaml:"owner,omitempty"` + Permissions string `yaml:"permissions,omitempty"` + Encoding string `yaml:"encoding,omitempty"` + Append string `yaml:"append,omitempty"` +} + +const cloudInitText = `{{/* Template for cloud-config */ -}} +#cloud-config +{{- if .WriteFiles }} + +write_files: +{{- range .WriteFiles }} + - path: {{ .Path }} +{{- if .Owner }} + owner: {{ .Owner }} +{{- end }} +{{- if .Permissions }} + permissions: {{ .Permissions }} +{{- end }} +{{- if .Encoding }} + encoding: {{ .Encoding }} +{{- end }} +{{- if .Append }} + append: {{ .Append }} +{{- end }} +{{- if .Content }} + content: | +{{- range splitLines .Content }} + {{ . }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +` + +var templateFuncMap = template.FuncMap{ + "splitLines": splitLines, +} + +func splitLines(text string) []string { + lines := strings.Split(text, "\n") + if len(lines) == 0 { + return nil + } + if lines[len(lines)-1] == "" { + return lines[:len(lines)-1] + } + return lines +} + +func (config *CloudConfig) Generate() (string, error) { + + tpl, err := template.New("base").Funcs(templateFuncMap).Parse(cloudInitText) + if err != nil { + return "", fmt.Errorf("Error initializing a template for cloudinit userdata: %w", err) + } + + var buf bytes.Buffer + + if err := tpl.Execute(&buf, config); err != nil { + return "", fmt.Errorf("Error executing a template for cloudinit userdata: %w", err) + } + + return buf.String(), nil +} + +func AuthJSONToResourcesJSON(text string) string { + var buf bytes.Buffer + tpl := template.Must(template.New("cerdTpl").Parse("{\"Credential\":\"{{.EncodedAuth}}\"}")) + if err := tpl.Execute(&buf, struct{ EncodedAuth string }{base64.StdEncoding.EncodeToString([]byte(text))}); err != nil { + return "" + } + return buf.String() +} diff --git a/pkg/util/cloudinit/cloudconfig_test.go b/pkg/util/cloudinit/cloudconfig_test.go new file mode 100644 index 000000000..6b73b01f7 --- /dev/null +++ b/pkg/util/cloudinit/cloudconfig_test.go @@ -0,0 +1,42 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package cloudinit + +import ( + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v2" +) + +func TestUserData(t *testing.T) { + cloudConfig := &CloudConfig{ + WriteFiles: []WriteFile{ + {Path: "/123", Content: "Hello\n"}, + {Path: "/456", Content: "Hello\nWorld\n", Owner: "root:root"}, + }, + } + + userData, err := cloudConfig.Generate() + if err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + firstLine := userData[0:strings.Index(userData, "\n")] + + if e, a := "#cloud-config", firstLine; e != a { + t.Fatalf("Expect %q, got %q", e, a) + } + + var output CloudConfig + + if err := yaml.Unmarshal([]byte(userData), &output); err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + if e, a := cloudConfig, &output; !reflect.DeepEqual(e, a) { + t.Fatalf("Expect %#v, got %#v", e, a) + } +} diff --git a/pkg/util/hvutil/hvutil.go b/pkg/util/hvutil/hvutil.go new file mode 100644 index 000000000..16ac28965 --- /dev/null +++ b/pkg/util/hvutil/hvutil.go @@ -0,0 +1,61 @@ +package hvutil + +import ( + "fmt" + "strings" + + cri "github.com/containerd/containerd/pkg/cri/annotations" +) + +func GetPodName(annotations map[string]string) string { + + sandboxName := annotations[cri.SandboxName] + + // cri-o stores the sandbox name in the form of k8s____0 + // Extract the pod name from it. + if tmp := strings.Split(sandboxName, "_"); len(tmp) > 1 && tmp[0] == "k8s" { + return tmp[1] + } + + return sandboxName +} + +func GetPodNamespace(annotations map[string]string) string { + + return annotations[cri.SandboxNamespace] +} + +func sanitize(input string) string { + + var output string + + for _, c := range strings.ToLower(input) { + if !(('a' <= c && c <= 'z') || ('0' <= c && c <= '9') || c == '-') { + c = '-' + } + output += string(c) + } + + return output +} + +var podvmNamePrefix = "podvm" + +func CreateInstanceName(podName, sandboxID string, podvmNameMax int) string { + + podName = sanitize(podName) + sandboxID = sanitize(sandboxID) + + podNameLen := len(podName) + if podvmNameMax > 0 && len(podvmNamePrefix)+podNameLen+10 > podvmNameMax { + podNameLen = podvmNameMax - len(podvmNamePrefix) - 10 + if podNameLen < 0 { + panic(fmt.Errorf("podvmNameMax is too small: %d", podvmNameMax)) + } + fmt.Printf("podNameLen: %d", podNameLen) + } + + vmName := fmt.Sprintf("%s-%.*s-%.8s", podvmNamePrefix, podNameLen, podName, sandboxID) + + return vmName +} diff --git a/pkg/util/hvutil/hvutil_test.go b/pkg/util/hvutil/hvutil_test.go new file mode 100644 index 000000000..9809a5e17 --- /dev/null +++ b/pkg/util/hvutil/hvutil_test.go @@ -0,0 +1,17 @@ +package hvutil + +import "testing" + +func TestCreateInstanceName(t *testing.T) { + + pod := "podname1" + sid := "d38339f99f605f18b4bcbce983147ad2d270ba479668d80b3bfa69a6b0237aa7" + + if e, a := "podvm-podname1-d38339f9", CreateInstanceName(pod, sid, 0); e != a { + t.Errorf("expected %s, got %s", e, a) + } + + if e, a := "podvm-podna-d38339f9", CreateInstanceName(pod, sid, 20); e != a { + t.Errorf("expected %s, got %s", e, a) + } +} diff --git a/pkg/util/netops/netops.go b/pkg/util/netops/netops.go new file mode 100644 index 000000000..2de6a6ab6 --- /dev/null +++ b/pkg/util/netops/netops.go @@ -0,0 +1,888 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package netops + +import ( + "errors" + "fmt" + "log" + "net" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" + + "github.com/containernetworking/plugins/pkg/utils/sysctl" +) + +var logger = log.New(log.Writer(), "[util/netops] ", log.LstdFlags|log.Lmsgprefix) + +// NS represents a network namespace +type NS struct { + Name string + Path string + nsHandle netns.NsHandle + handle *netlink.Handle +} + +// NewNSFromPath returns an NS specified by a namespace path +func NewNSFromPath(nsPath string) (*NS, error) { + + netnsDir := "/run/netns" + altNetnsDir := "/var/run/netns" + + if filepath.Dir(nsPath) == altNetnsDir { + nsPath = filepath.Join(netnsDir, filepath.Base(nsPath)) + } + + if filepath.Dir(nsPath) != netnsDir { + return nil, fmt.Errorf("%s is not in %s", nsPath, netnsDir) + } + + name := filepath.Base(nsPath) + + nsHandle, err := netns.GetFromPath(nsPath) + if err != nil { + return nil, fmt.Errorf("failed to get network namespace %s: %w", nsPath, err) + } + handle, err := netlink.NewHandleAt(nsHandle) + if err != nil { + return nil, fmt.Errorf("failed to get a handle for network namespace %s: %w", nsPath, err) + } + ns := &NS{ + Name: name, + Path: nsPath, + nsHandle: nsHandle, + handle: handle, + } + return ns, nil +} + +// GetNS returns an NS of the current network namespace +func GetNS() (*NS, error) { + + pid := os.Getpid() + tid := unix.Gettid() + path := fmt.Sprintf("/proc/%d/task/%d/ns/net", pid, tid) + + nsHandle, err := netns.GetFromPath(path) + if err != nil { + return nil, fmt.Errorf("failed to get current network namespace: %w", err) + } + handle, err := netlink.NewHandleAt(nsHandle) + if err != nil { + return nil, fmt.Errorf("failed to get a netlink handle for the current network namespace: %w", err) + } + ns := &NS{ + nsHandle: nsHandle, + handle: handle, + Path: path, + } + return ns, nil +} + +// NewNamedNS returns an NS of the current network namespace +func NewNamedNS(name string) (ns *NS, err error) { + runtime.LockOSThread() + defer runtime.LockOSThread() + + old, err := netns.Get() + if err != nil { + return nil, fmt.Errorf("failed to get the current network namespace: %w", err) + } + defer func() { + if e := netns.Set(old); e != nil { + err = fmt.Errorf("failed to set back to the old network namespace: %w (previous error %v)", e, err) + } + }() + + nsHandle, err := netns.NewNamed(name) + if err != nil { + return nil, fmt.Errorf("failed to get current network namespace: %w", err) + } + handle, err := netlink.NewHandleAt(nsHandle) + if err != nil { + return nil, fmt.Errorf("failed to get a netlink handle for the current network namespace: %w", err) + } + ns = &NS{ + Name: name, + Path: filepath.Join("/run/netns", name), + nsHandle: nsHandle, + handle: handle, + } + return ns, nil +} + +// Clone returns a copy of a network namespace. A returned network namespace need to be closed separately. +func (ns *NS) Clone() (*NS, error) { + + nsHandle, err := netns.GetFromPath(ns.Path) + if err != nil { + return nil, fmt.Errorf("failed to get network namespace at %q: %w", ns.Path, err) + } + handle, err := netlink.NewHandleAt(nsHandle) + if err != nil { + return nil, fmt.Errorf("failed to get a netlink handle for network namespace at %q: %w", ns.Path, err) + } + + clone := &NS{ + nsHandle: nsHandle, + handle: handle, + Path: ns.Path, + } + return clone, nil +} + +// FD returns a file descriptor of the network namespace +func (ns *NS) FD() int { + return int(ns.nsHandle) +} + +// Close closes an NS +func (ns *NS) Close() error { + + ns.handle.Close() + + if err := ns.nsHandle.Close(); err != nil { + return fmt.Errorf("failed to close network namespace %v: %w", ns.handle, err) + } + + return nil +} + +// Delete deletes a named NS +func (ns *NS) Delete() error { + if err := netns.DeleteNamed(ns.Name); err != nil { + return fmt.Errorf("failed to delete a named network namespace %s: %w", ns.Name, err) + } + return nil +} + +// Run calls a function in a network namespace +func (ns *NS) Run(fn func() error) (err error) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + oldNS, err := netns.Get() + if err != nil { + return fmt.Errorf("failed to get current network namespace: %w", err) + } + defer func() { + if e := oldNS.Close(); e != nil { + err = fmt.Errorf("failed to close the original network namespace: %w (previous error: %v)", e, err) + } + }() + + if ns.nsHandle != oldNS { + if err = netns.Set(ns.nsHandle); err != nil { + return fmt.Errorf("failed to set a network namespace: %w", err) + } + defer func() { + if e := netns.Set(oldNS); e != nil { + err = fmt.Errorf("failed to set back to the host network namespace: %w (previous error %v)", err, e) + } + }() + } + + return fn() +} + +// SysctlGet returns a kernel parameter value specified by key +func (ns *NS) SysctlGet(key string) (string, error) { + var val string + err := ns.Run(func() error { + var err error + + val, err = sysctl.Sysctl(key) + if err != nil { + return fmt.Errorf("failed to get a value of sysctl parameter %q: %w", key, err) + } + return err + }) + return val, err +} + +// SysctlSet returns a kernel parameter value specified by key +func (ns *NS) SysctlSet(key string, val string) error { + + err := ns.Run(func() error { + if _, err := sysctl.Sysctl(key, val); err != nil { + return fmt.Errorf("failed to set sysctl parameter %q to %q: %w", key, val, err) + } + return nil + }) + return err +} + +// GetIP returns a list of IP addresses assigned to a link +func (ns *NS) GetIP(linkName string) ([]net.IP, error) { + + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return nil, fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + addrs, err := ns.handle.AddrList(link, netlink.FAMILY_V4) + if err != nil { + return nil, fmt.Errorf("failed to get addressess assigned to %s (type: %s, netns: %s): %w", linkName, link.Type(), ns.Path, err) + } + + var ips []net.IP + for _, addr := range addrs { + ips = append(ips, addr.IP) + } + + return ips, nil +} + +// GetIPNet returns a list of IPNets assigned to a link +func (ns *NS) GetIPNet(linkName string) ([]*net.IPNet, error) { + + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return nil, fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + addrs, err := ns.handle.AddrList(link, netlink.FAMILY_V4) + if err != nil { + return nil, fmt.Errorf("failed to delete an interface of %s: %s: %w", linkName, link.Type(), err) + } + + var ipNets []*net.IPNet + for _, addr := range addrs { + ipNets = append(ipNets, addr.IPNet) + } + + return ipNets, nil +} + +// GetMTU returns MTU size of a link +func (ns *NS) GetMTU(linkName string) (int, error) { + + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return 0, fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + + mtu := link.Attrs().MTU + + return mtu, nil +} + +// SetMTU sets MTU size of a link +func (ns *NS) SetMTU(linkName string, mtu int) error { + + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + + if err := ns.handle.LinkSetMTU(link, mtu); err != nil { + return fmt.Errorf("failed to set MTU of %s to %d: %w", linkName, mtu, err) + } + + return nil +} + +func (ns *NS) GetHardwareAddr(linkName string) (string, error) { + + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return "", fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + + hwAddr := link.Attrs().HardwareAddr.String() + + return hwAddr, nil +} + +func (ns *NS) SetHardwareAddr(linkName, hwAddr string) error { + + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + + mac, err := net.ParseMAC(hwAddr) + if err != nil { + return fmt.Errorf("failed to parse hardware address %q: %w", hwAddr, err) + + } + + if err := ns.handle.LinkSetHardwareAddr(link, mac); err != nil { + return fmt.Errorf("failed to set hardware address %q to %s (netns: %s): %w", hwAddr, linkName, ns.Path, err) + } + + return nil +} + +func (ns *NS) LinkNameByAddr(ip net.IP) (string, error) { + + links, err := ns.handle.LinkList() + if err != nil { + return "", fmt.Errorf("failed to get a list of interfaces (netns: %s): %w", ns.Path, err) + } + + var names []string + for _, link := range links { + name := link.Attrs().Name + addrs, err := ns.GetIP(name) + if err != nil { + return "", fmt.Errorf("failed to obtain addresses assigned to %s (netns: %s)", name, ns.Path) + } + for _, addr := range addrs { + if addr.Equal(ip) { + names = append(names, name) + break + } + } + } + + if len(names) == 0 { + return "", fmt.Errorf("failed to find interface that has %s on netns %s", ip.String(), ns.Path) + } + if len(names) > 1 { + return "", fmt.Errorf("multile interfaces have %s on netns %s: %s", ip.String(), ns.Path, strings.Join(names, ", ")) + } + + return names[0], nil +} + +func (ns *NS) LinkList() ([]netlink.Link, error) { + + links, err := ns.handle.LinkList() + if err != nil { + return nil, fmt.Errorf("failed to get a list of interfaces: %w", err) + } + + return links, nil +} + +func (ns *NS) LinkByName(name string) (netlink.Link, error) { + + link, err := ns.handle.LinkByName(name) + if err != nil { + return nil, fmt.Errorf("failed to find interface %s: %w", name, err) + } + + return link, nil +} + +// LinkAdd creates a new link with an attribute specified by link +func (ns *NS) LinkAdd(linkName string, link netlink.Link) error { + attrs := link.Attrs() + *attrs = netlink.NewLinkAttrs() + attrs.Name = linkName + if err := ns.handle.LinkAdd(link); err != nil { + return fmt.Errorf("failed to create an interface of %s: %s: %w", link.Type(), linkName, err) + } + return nil +} + +// LinkDel deletes a link +func (ns *NS) LinkDel(linkName string) error { + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + if err := ns.handle.LinkDel(link); err != nil { + return fmt.Errorf("failed to delete an interface of %s: %s: %w", linkName, link.Type(), err) + } + return nil +} + +// LinkSetMaster sets a master device of a link +func (ns *NS) LinkSetMaster(linkName, masterName string) error { + master, err := ns.handle.LinkByName(masterName) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", masterName, err) + } + + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + if err := ns.handle.LinkSetMaster(link, master); err != nil { + return fmt.Errorf("failed to set master device of %s to %s: %w", linkName, masterName, err) + } + return nil +} + +// LinkSetNS changes network namespace of a link +func (ns *NS) LinkSetNS(linkName string, targetNS *NS) error { + + link, err := ns.LinkByName(linkName) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + + if err := ns.handle.LinkSetNsFd(link, targetNS.FD()); err != nil { + return fmt.Errorf("failed to change network namespace of interface %s from %s to %s: %w", linkName, ns.Path, targetNS.Path, err) + } + + return nil +} + +// LinkSetName changes name of a link +func (ns *NS) LinkSetName(linkName, newName string) error { + + link, err := ns.LinkByName(linkName) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + + if err := ns.handle.LinkSetName(link, newName); err != nil { + return fmt.Errorf("failed to change name of interface %s on %s to %s: %w", linkName, ns.Path, newName, err) + } + + return nil +} + +// AddrAdd adds an IP address to a link +func (ns *NS) AddrAdd(linkName string, addr *net.IPNet) error { + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + if err := ns.handle.AddrAdd(link, &netlink.Addr{IPNet: addr}); err != nil { + return fmt.Errorf("failed to assign an IP address to %s: %w", linkName, err) + } + return nil +} + +// LinkSetUp makes the link status up +func (ns *NS) LinkSetUp(linkName string) error { + link, err := ns.handle.LinkByName(linkName) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", linkName, err) + } + if err := ns.handle.LinkSetUp(link); err != nil { + return fmt.Errorf("failed to set link state up: %s: %w", linkName, err) + } + return nil +} + +// RouteAdd adds a new route +func (ns *NS) RouteAdd(table int, dest *net.IPNet, gw net.IP, dev string) error { + logger.Printf("RouteAdd details: table(%d), dest(%v), gw(%v), dev(%s)", table, dest, gw, dev) + + if dest == nil { + _, dest, _ = net.ParseCIDR("0.0.0.0/0") + } + route := &netlink.Route{ + Table: table, + Dst: dest, + Gw: gw, + } + if dev != "" { + link, err := ns.handle.LinkByName(dev) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", dev, err) + } + route.LinkIndex = link.Attrs().Index + } + if gw == nil { + route.Scope = netlink.SCOPE_LINK + } + if err := ns.handle.RouteAdd(route); err != nil { + return fmt.Errorf("failed to create a route: %w", err) + } + return nil +} + +func (ns *NS) RouteAddOnlink(table int, dest *net.IPNet, gw net.IP, dev string) error { + if dest == nil { + _, dest, _ = net.ParseCIDR("0.0.0.0/0") + } + route := &netlink.Route{ + Table: table, + Dst: dest, + Gw: gw, + Flags: int(netlink.FLAG_ONLINK), + } + if dev != "" { + link, err := ns.handle.LinkByName(dev) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", dev, err) + } + route.LinkIndex = link.Attrs().Index + } + if gw == nil { + route.Scope = netlink.SCOPE_LINK + } + if err := ns.handle.RouteAdd(route); err != nil { + return fmt.Errorf("failed to create a route (table: %d, dest: %s, gw: %s) with flags %d: %w", route.Table, route.Dst.String(), route.Gw.String(), route.Flags, err) + } + return nil +} + +// RouteDel deletes routes +func (ns *NS) RouteDel(table int, dest *net.IPNet, gw net.IP, dev string) error { + filterMask := netlink.RT_FILTER_DST + if dest == nil { + _, dest, _ = net.ParseCIDR("0.0.0.0/0") + } + route := &netlink.Route{ + Dst: dest, + } + if table != 0 { + route.Table = table + filterMask |= netlink.RT_FILTER_TABLE + } + if gw != nil { + route.Gw = gw + filterMask |= netlink.RT_FILTER_GW + } + if dev != "" { + link, err := ns.handle.LinkByName(dev) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", dev, err) + } + route.LinkIndex = link.Attrs().Index + filterMask |= netlink.RT_FILTER_OIF + } + routes, err := ns.handle.RouteListFiltered(netlink.FAMILY_V4, route, filterMask) + if err != nil { + return fmt.Errorf("failed to get a list of routes: table %d, dest: %s, gw: %s, dev %s: %w", table, dest, gw, dev, err) + } + for _, route := range routes { + if err := ns.handle.RouteDel(&route); err != nil { + return fmt.Errorf("failed to delete a route: table %d, dest: %s, gw: %s, dev %s: %w", table, dest, gw, dev, err) + } + } + return nil +} + +type Route struct { + Dst *net.IPNet + GW net.IP + Dev string +} + +// GetRoutes gets a list of routes on the main table +func (ns *NS) GetRoutes() ([]*Route, error) { + + routeList, err := ns.handle.RouteList(nil, netlink.FAMILY_V4) + if err != nil { + return nil, fmt.Errorf("failed to get routes on namespace %q: %w", ns.Name, err) + } + + var routes []*Route + for _, r := range routeList { + + var dev string + if r.LinkIndex > 0 { + link, err := ns.handle.LinkByIndex(r.LinkIndex) + if err != nil { + return nil, fmt.Errorf("failed to get a link with index %d of a route: %w", r.LinkIndex, err) + } + dev = link.Attrs().Name + } + + if r.Type != unix.RTN_UNICAST { + logger.Printf("route (dst:%s, gw:%s: dev:%s) has unexpected type %d. ignoring", r.Dst, r.Gw, dev, r.Type) + continue + } + + if r.Table != unix.RT_TABLE_MAIN { + logger.Printf("route (dst:%s, gw:%s: dev:%s) has unexpected table %d. ignoring", r.Dst, r.Gw, dev, r.Table) + continue + } + + if r.Flags != 0 { + logger.Printf("route (dst:%s, gw:%s: dev:%s) has unexpected flags %d. ignoring", r.Dst, r.Gw, dev, r.Flags) + continue + } + + switch r.Protocol { + case unix.RTPROT_STATIC: + case unix.RTPROT_BOOT: + case unix.RTPROT_DHCP: + case unix.RTPROT_KERNEL: + continue + default: + logger.Printf("route (dst:%s, gw:%s: dev:%s) has unexpected protocol %d. ignoring", r.Dst, r.Gw, dev, r.Protocol) + continue + } + + switch r.Scope { + case netlink.SCOPE_UNIVERSE: + case netlink.SCOPE_LINK: + case netlink.SCOPE_HOST: + default: + logger.Printf("route (dst:%s, gw:%s: dev:%s) has unexpected scope %d. ignoring", r.Dst, r.Gw, dev, r.Scope) + continue + } + + route := &Route{ + Dst: r.Dst, + GW: r.Gw, + Dev: dev, + } + routes = append(routes, route) + } + + return routes, nil +} + +// LocalRouteDel deletes a route of the local local +func (ns *NS) LocalRouteDel(table int, dest *net.IPNet, dev string) error { + + link, err := ns.handle.LinkByName(dev) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", dev, err) + } + route := &netlink.Route{ + Table: table, + Dst: dest, + Src: dest.IP, + Type: unix.RTN_LOCAL, + Protocol: unix.RTPROT_KERNEL, + Scope: unix.RT_SCOPE_HOST, + LinkIndex: link.Attrs().Index, + } + if err := ns.handle.RouteDel(route); err != nil { + return fmt.Errorf("failed to delete a route: table %d, dest %s, dev %s: %w", table, dest, dev, err) + } + return nil +} + +// GetAvailableTableID returns a table ID that is not currently used +func (ns *NS) GetAvailableTableID(iif string, priority, min, max int) (int, error) { + rule := netlink.NewRule() + rule.Priority = priority + rule.IifName = iif + + rules, err := ns.handle.RuleListFiltered(netlink.FAMILY_V4, nil, netlink.RT_FILTER_PRIORITY|netlink.RT_FILTER_IIF) + if err != nil { + return 0, fmt.Errorf("failed to get rules: %w", err) + } + + used := make(map[int]bool) + for _, rule := range rules { + used[rule.Table] = true + } + + for id := min; id <= max; id++ { + if !used[id] { + return id, nil + } + } + return 0, fmt.Errorf("No table ID is available") +} + +// RuleAdd adds a new rule in the routing policy database +func (ns *NS) RuleAdd(src *net.IPNet, iif string, priority int, table int) error { + rule := netlink.NewRule() + rule.Src = src + rule.IifName = iif + rule.Priority = priority + rule.Table = table + + if err := ns.handle.RuleAdd(rule); err != nil { + return fmt.Errorf("failed to add a rule: %w", err) + } + return nil +} + +// RuleDel deletes a rule in the routing policy database +func (ns *NS) RuleDel(src *net.IPNet, iif string, priority int, table int) error { + rule := netlink.NewRule() + rule.Src = src + rule.IifName = iif + rule.Priority = priority + rule.Table = table + + if err := ns.handle.RuleDel(rule); err != nil { + return fmt.Errorf("failed to delete a rule: %w", err) + } + return nil +} + +// RuleList gets a list of rules in the routing policy database +func (ns *NS) RuleList(src *net.IPNet, iif string, priority int, table int) ([]netlink.Rule, error) { + rule := netlink.NewRule() + var filterMask uint64 + if src != nil { + rule.Src = src + filterMask |= netlink.RT_FILTER_SRC + } + if iif != "" { + rule.IifName = iif + filterMask |= netlink.RT_FILTER_IIF + } + if priority != 0 { + rule.Priority = priority + filterMask |= netlink.RT_FILTER_PRIORITY + } + rules, err := ns.handle.RuleListFiltered(netlink.FAMILY_V4, rule, filterMask) + if err != nil { + return nil, fmt.Errorf("failed to get rules: %w", err) + } + return rules, nil +} + +// DetectPodIP returns IP and link of the default route device +func (ns *NS) DetectPodIP() (net.IP, string, error) { + routes, err := ns.handle.RouteList(nil, netlink.FAMILY_V4) + if err != nil { + return nil, "", fmt.Errorf("failed to get routes on the pod namespace: %w", err) + } + + var defaultRoute *netlink.Route + for _, route := range routes { + if route.Dst == nil { + defaultRoute = &route + break + } + } + if defaultRoute == nil { + return nil, "", fmt.Errorf("failed to get default route on the pod namespace") + } + podLinks, err := ns.handle.LinkList() + if err != nil { + return nil, "", fmt.Errorf("failed to get interfaces on the pod namespace: %w", err) + } + var defaultLink netlink.Link + var podIP net.IP + for _, link := range podLinks { + if link.Attrs().Index == defaultRoute.LinkIndex { + defaultLink = link + addrs, err := ns.handle.AddrList(link, netlink.FAMILY_V4) + if err != nil { + return nil, "", fmt.Errorf("failed to get IP address of interface %s on the pod namespace: %w", link.Attrs().Name, err) + } + if len(addrs) == 0 { + return nil, "", fmt.Errorf("found no IPv4 addresses on the interface %s on the pod namespace", link.Attrs().Name) + } + if len(addrs) > 1 { + return nil, "", fmt.Errorf("found multiple IPv4 addresses on the interface %s on the pod namespace", link.Attrs().Name) + } + podIP = addrs[0].IP + break + } + } + + return podIP, defaultLink.Attrs().Name, nil +} + +// VethAdd adds a veth pair +func (ns *NS) VethAdd(name string, peerNS *NS, peerName string) error { + if err := ns.LinkAdd(name, &netlink.Veth{PeerName: peerName, PeerNamespace: netlink.NsFd(peerNS.nsHandle)}); err != nil { + return fmt.Errorf("failed to add veth pair %s and %s: %w", name, peerName, err) + } + return nil +} + +// VethAddPrefix adds a veth pair. One endpoint is create at ns, and its name begins with vethPrefix. +// The other endpoint is created at peerNS and its name is peerName. +func (ns *NS) VethAddPrefix(vethPrefix string, peerNS *NS, peerName string) (string, error) { + links, err := ns.handle.LinkList() + if err != nil { + return "", fmt.Errorf("failed to get interfaces on host: %w", err) + } + index := 1 + for { + vethName := fmt.Sprintf("%s%d", vethPrefix, index) + var found bool + for _, link := range links { + if link.Attrs().Name == vethName { + found = true + break + } + } + if !found { + err := ns.LinkAdd(vethName, &netlink.Veth{PeerName: peerName, PeerNamespace: netlink.NsFd(peerNS.nsHandle)}) + if err == nil { + return vethName, nil + } + if !errors.Is(err, os.ErrExist) { + return "", fmt.Errorf("failed to add veth pair %s and %s: %w", vethName, peerName, err) + } + } + index++ + } +} + +// RedirectAdd adds a tc ingress qdisc and redirect filter that redirects all traffic from src to dst +func (ns *NS) RedirectAdd(src, dst string) error { + srcLink, err := ns.handle.LinkByName(src) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", src, err) + } + + dstLink, err := ns.handle.LinkByName(dst) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", dst, err) + } + + qdisc := &netlink.Ingress{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: srcLink.Attrs().Index, + Parent: netlink.HANDLE_INGRESS, + }, + } + if err := ns.handle.QdiscAdd(qdisc); err != nil { + return fmt.Errorf("failed to add qdisc to %s: %w", src, err) + } + + filter := &netlink.U32{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: srcLink.Attrs().Index, + Parent: netlink.MakeHandle(0xffff, 0), + Protocol: unix.ETH_P_ALL, + }, + Actions: []netlink.Action{ + &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_EGRESS_REDIR, + Ifindex: dstLink.Attrs().Index, + }, + }, + } + + if err := ns.handle.FilterAdd(filter); err != nil { + return fmt.Errorf("failed to add a filter to %s : %w", src, err) + } + + return nil +} + +// RedirectDel deletes a tc ingress qdisc and redirect filters on src +func (ns *NS) RedirectDel(src string) error { + srcLink, err := ns.handle.LinkByName(src) + if err != nil { + return fmt.Errorf("failed to get interface %s: %w", src, err) + } + + filters, err := ns.handle.FilterList(srcLink, netlink.MakeHandle(0xffff, 0)) + if err != nil { + return fmt.Errorf("failed to get a list of filters on %s: %w", src, err) + } + for _, filter := range filters { + if _, ok := filter.(*netlink.U32); ok { + if err = ns.handle.FilterDel(filter); err != nil { + return fmt.Errorf("failed to delete a filter to %s : %w", src, err) + } + } + } + + qdiscs, err := ns.handle.QdiscList(srcLink) + if err != nil { + return err + } + for _, qdisc := range qdiscs { + if _, ok := qdisc.(*netlink.Ingress); ok { + if err := ns.handle.QdiscDel(qdisc); err != nil { + return fmt.Errorf("failed to delete a qdisc on %s : %w", src, err) + } + } + } + + return nil +} diff --git a/pkg/util/netops/netops_test.go b/pkg/util/netops/netops_test.go new file mode 100644 index 000000000..6d40cee75 --- /dev/null +++ b/pkg/util/netops/netops_test.go @@ -0,0 +1,56 @@ +// (C) Copyright IBM Corp. 2022. +// SPDX-License-Identifier: Apache-2.0 + +package netops + +import ( + "runtime" + "testing" + + "github.com/confidential-containers/cloud-api-adaptor/pkg/internal/testing" + "github.com/vishvananda/netns" +) + +func TestRoute(t *testing.T) { + testutils.SkipTestIfNotRoot(t) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + oldns, err := netns.Get() + if err != nil { + t.Fatalf("Failed to get the current network namespace: %v", err) + } + + podns, err := netns.New() + if err != nil { + t.Fatalf("Failed to create network namespace: %v", err) + } + defer func() { + if err := netns.Set(oldns); err != nil { + t.Fatalf("Failed to set a network namespace: %v", err) + } + if err := podns.Close(); err != nil { + t.Fatalf("Failed to close a network namespace: %v", err) + } + }() +} + +func TestRouteList(t *testing.T) { + testutils.SkipTestIfNotRoot(t) + + ns, err := GetNS() + if err != nil { + t.Fatalf("Expect no error, got %v", err) + } + defer ns.Close() + + routes, err := ns.GetRoutes() + if err != nil { + t.Fatalf("Expect no error, got %v", err) + } + + for _, route := range routes { + t.Logf("Route: %#v", route) + } +} diff --git a/pkg/util/redacting.go b/pkg/util/redacting.go new file mode 100644 index 000000000..4d14ef4d4 --- /dev/null +++ b/pkg/util/redacting.go @@ -0,0 +1,24 @@ +package util + +import ( + "fmt" + "reflect" +) + +const replacement = "**********" + +func RedactStruct(struc interface{}, fields ...string) interface{} { + v := reflect.ValueOf(struc).Elem() + if v.Type().Kind() != reflect.Struct { + panic(fmt.Sprintf("Unsupported type, %v", v.Type().String())) + } + for _, field := range fields { + f := v.FieldByName(field) + if f.Kind() == reflect.String { + f.SetString(replacement) + } else { + f.Set(reflect.Zero(v.Type())) + } + } + return struc +} diff --git a/podvm/Dockerfile.podvm b/podvm/Dockerfile.podvm new file mode 100644 index 000000000..178063260 --- /dev/null +++ b/podvm/Dockerfile.podvm @@ -0,0 +1,29 @@ +# Copyright Confidential Containers Contributors +# +# SPDX-License-Identifier: Apache-2.0 +# +# Builds pod vm image inside container +# +ARG BUILDER_IMG="quay.io/confidential-containers/cloud-api-adaptor-podvm-builder" + +FROM ${BUILDER_IMG} AS podvm_builder + +ARG CLOUD_PROVIDER=libvirt +ARG PODVM_DISTRO=ubuntu + +# If these are not provided as build args then the +# default values assigned in the Makefile will get used +ARG UBUNTU_IMAGE_URL +ARG UBUNTU_IMAGE_CHECKSUM + + +# Building binaries separately allows for reuse in case of error +# with image building step +RUN cd cloud-api-adaptor/podvm && \ + CLOUD_PROVIDER=$CLOUD_PROVIDER make binaries + +RUN cd cloud-api-adaptor/podvm && \ + CLOUD_PROVIDER=$CLOUD_PROVIDER PODVM_DISTRO=$PODVM_DISTRO make image + +FROM scratch +COPY --from=podvm_builder /src/cloud-api-adaptor/podvm/output/*.qcow2 / diff --git a/podvm/Dockerfile.podvm.centos b/podvm/Dockerfile.podvm.centos new file mode 100644 index 000000000..3cdb775c4 --- /dev/null +++ b/podvm/Dockerfile.podvm.centos @@ -0,0 +1,32 @@ +# Copyright Confidential Containers Contributors +# +# SPDX-License-Identifier: Apache-2.0 +# +# Builds pod vm image inside container +# +ARG BUILDER_IMG="quay.io/confidential-containers/cloud-api-adaptor-podvm-builder" + +FROM ${BUILDER_IMG} AS podvm_builder + +ARG CLOUD_PROVIDER=libvirt +ARG PODVM_DISTRO=centos + +# Default to CentOS 8-stream x86_64 image. These variables can be overriden as needed +ARG CENTOS_IMAGE_URL=https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20200113.0.x86_64.qcow2 +ARG CENTOS_IMAGE_CHECKSUM=b80ca9ccad8715aa1aeee582906096d3190b91cebe05fcb3ac7cb197fd68a7fe + + +# Building binaries separately allows for reuse in case of error +# with image building step +RUN cd cloud-api-adaptor/podvm && \ + CLOUD_PROVIDER=$CLOUD_PROVIDER LIBC=gnu make binaries + +# workaround to ensure hashicorp packer is called instead +# of cracklib packer which is installed by default +ENV PATH="/usr/bin:${PATH}" + +RUN cd cloud-api-adaptor/podvm && \ + CLOUD_PROVIDER=$CLOUD_PROVIDER PODVM_DISTRO=$PODVM_DISTRO LIBC=gnu make image + +FROM scratch +COPY --from=podvm_builder /src/cloud-api-adaptor/podvm/output/*.qcow2 / diff --git a/podvm/Dockerfile.podvm_builder b/podvm/Dockerfile.podvm_builder new file mode 100644 index 000000000..e1bc57134 --- /dev/null +++ b/podvm/Dockerfile.podvm_builder @@ -0,0 +1,57 @@ +# Copyright Confidential Containers Contributors +# +# SPDX-License-Identifier: Apache-2.0 +# +# Creates a builder container image that should be used to build the Pod VM +# disk inside a container. +# +FROM ubuntu:20.04 + +ARG GO_VERSION="1.18.7" +ARG PROTOC_VERSION="3.11.4" +ARG RUST_VERSION="1.62.0" + + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update -y && \ + apt-get install -y build-essential cloud-image-utils curl git gnupg \ + libdevmapper-dev libgpgme-dev lsb-release pkg-config qemu-kvm \ + musl-tools unzip wget git && \ + curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - && \ + echo "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | tee -a /etc/apt/sources.list && \ + apt-get update && apt-get install -y packer && \ + curl https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz -o go${GO_VERSION}.linux-amd64.tar.gz && \ + rm -rf /usr/local/go && tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz && \ + rm -f go${GO_VERSION}.linux-amd64.tar.gz + + +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain "${RUST_VERSION}" + +ENV PATH "/root/.cargo/bin:/usr/local/go/bin:$PATH" + +RUN echo $PATH + +RUN rustup target add x86_64-unknown-linux-musl && ln -sf /usr/bin/g++ /bin/musl-g++ + + +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip && \ + unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local && rm -f protoc-${PROTOC_VERSION}-linux-x86_64.zip + +WORKDIR /src + +ARG CAA_SRC="https://github.com/confidential-containers/cloud-api-adaptor" +ARG CAA_SRC_BRANCH="staging" + +ARG KATA_SRC="https://github.com/kata-containers/kata-containers" +ARG KATA_SRC_BRANCH="CCv0" + +RUN echo $CAA_SRC + +RUN echo $CAA_SRC_BRANCH + +RUN git clone ${CAA_SRC} -b ${CAA_SRC_BRANCH} cloud-api-adaptor +RUN git clone ${KATA_SRC} -b ${KATA_SRC_BRANCH} kata-containers + + +ENV GOPATH /src diff --git a/podvm/Dockerfile.podvm_builder.centos b/podvm/Dockerfile.podvm_builder.centos new file mode 100644 index 000000000..d921c99e5 --- /dev/null +++ b/podvm/Dockerfile.podvm_builder.centos @@ -0,0 +1,57 @@ +# Copyright Confidential Containers Contributors +# +# SPDX-License-Identifier: Apache-2.0 +# +# Creates a builder container image that should be used to build the Pod VM +# disk inside a container. +# +FROM quay.io/centos/centos:stream8 + +ARG GO_VERSION="1.18.7" +ARG PROTOC_VERSION="3.11.4" +ARG RUST_VERSION="1.62.0" + + +RUN dnf config-manager --set-enabled powertools && \ + dnf install -y epel-release epel-next-release && dnf update -y && \ + dnf groupinstall -y 'Development Tools' && \ + dnf install -y yum-utils gnupg git curl pkg-config libseccomp-devel gpgme-devel \ + device-mapper-devel qemu-kvm unzip wget libassuan-devel golang-github-cpuguy83-md2man \ + genisoimage cloud-utils-growpart cloud-init && \ + yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo && \ + dnf install -y packer && \ + curl https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz -o go${GO_VERSION}.linux-amd64.tar.gz && \ + rm -rf /usr/local/go && tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz && \ + rm -f go${GO_VERSION}.linux-amd64.tar.gz + + +# cloud-utils package is not available for centos. +RUN git clone https://github.com/canonical/cloud-utils +RUN cd cloud-utils && make install + +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain "${RUST_VERSION}" + +ENV PATH "/root/.cargo/bin:/usr/local/go/bin:$PATH" + +RUN echo $PATH + + +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip && \ + unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local && rm -f protoc-${PROTOC_VERSION}-linux-x86_64.zip + +WORKDIR /src + +ARG CAA_SRC="https://github.com/confidential-containers/cloud-api-adaptor" +ARG CAA_SRC_BRANCH="staging" + +ARG KATA_SRC="https://github.com/kata-containers/kata-containers" +ARG KATA_SRC_BRANCH="CCv0" + +RUN echo $CAA_SRC + +RUN echo $CAA_SRC_BRANCH + +RUN git clone ${CAA_SRC} -b ${CAA_SRC_BRANCH} cloud-api-adaptor +RUN git clone ${KATA_SRC} -b ${KATA_SRC_BRANCH} kata-containers + +ENV GOPATH /src diff --git a/podvm/Makefile b/podvm/Makefile new file mode 100644 index 000000000..e466b61ef --- /dev/null +++ b/podvm/Makefile @@ -0,0 +1,70 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# + +include Makefile.inc + +.PHONY: image clean + +UBUNTU_IMAGE_URL ?= https://cloud-images.ubuntu.com/$(UBUNTU_RELEASE)/current/$(UBUNTU_RELEASE)-server-cloudimg-$(ARCH).img +UBUNTU_IMAGE_CHECKSUM ?= $(shell curl -LO https://cloud-images.ubuntu.com/"$(UBUNTU_RELEASE)"/current/SHA256SUMS && \ + grep "$(UBUNTU_RELEASE)"-server-cloudimg-"$(ARCH)".img SHA256SUMS | awk '{print $$1}' && \ + rm -f SHA256SUMS) + +IMAGE_SUFFIX := .qcow2 +PODVM_DISTRO ?= ubuntu +KATA_AGENT_SRC := ../../kata-containers/src/agent +STATIC_LIBSECCOMP_BUILDER := ../../kata-containers/ci/install_libseccomp.sh +AGENT_PROTOCOL_FORWARDER_SRC := ../ + +image: $(IMAGE_FILE) + +setopts: +ifeq ($(PODVM_DISTRO),ubuntu) + @echo defined + $(eval OPTS := -var qemu_image_name=${IMAGE_FILE} \ + -var cloud_image_url=${UBUNTU_IMAGE_URL} \ + -var cloud_image_checksum=${UBUNTU_IMAGE_CHECKSUM} qcow2/ubuntu) +else ifeq ($(PODVM_DISTRO),rhel) +ifndef RHEL_IMAGE_URL + $(error "RHEL_IMAGE_URL is not defined") +endif +ifndef RHEL_IMAGE_CHECKSUM + $(error "RHEL_IMAGE_CHECKSUM is not defined") +endif + $(eval OPTS := -var qemu_image_name=${IMAGE_FILE} \ + -var cloud_image_url=${RHEL_IMAGE_URL} \ + -var disk_size=11144 \ + -var ssh_username=cloud-user \ + -var cloud_image_checksum=${RHEL_IMAGE_CHECKSUM} qcow2/rhel) + +else ifeq ($(PODVM_DISTRO),centos) +ifndef CENTOS_IMAGE_URL + $(error "CENTOS_IMAGE_URL is not defined") +endif +ifndef CENTOS_IMAGE_CHECKSUM + $(error "CENTOS_IMAGE_CHECKSUM is not defined") +endif + $(eval OPTS := -var qemu_image_name=${IMAGE_FILE} \ + -var cloud_image_url=${CENTOS_IMAGE_URL} \ + -var cloud_image_checksum=${CENTOS_IMAGE_CHECKSUM} qcow2/centos) + +else + $(error PODVM_DISTRO is invalid or not defined) +endif + +$(IMAGE_FILE): $(BINARIES) $(FILES) setopts + rm -fr output + rm -f cloud-init.img + cloud-localds cloud-init.img qcow2/userdata.cfg + mkdir -p toupload + packer build ${OPTS} + rm -fr toupload + rm -f cloud-init.img + +clean: + rm -f "$(IMAGE_FILE)" "$(UBUNTU_IMAGE_FILE)" $(BINARIES) + rm -fr "$(SKOPEO_SRC)" "$(UMOCI_SRC)" "$(PAUSE_SRC)" "$(FILES_DIR)/$(PAUSE_BUNDLE)" + +.PHONY: force +force: diff --git a/podvm/Makefile.inc b/podvm/Makefile.inc new file mode 100644 index 000000000..d3d69f269 --- /dev/null +++ b/podvm/Makefile.inc @@ -0,0 +1,130 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This file contains the common build steps across the providers +# Including building the required binaries from source +# It is to be included via `include ../../podvm/Makefile.inc` in +# each of the cloud provider podvm image Makefiles + +UBUNTU_RELEASE = focal +SKOPEO_VERSION = 1.5.0 +UMOCI_VERSION = 0.4.7 + +IMAGE_PREFIX := podvm + +ARCH := $(subst x86_64,amd64,$(shell uname -m)) + +FILES_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))/files + +ifndef IMAGE_NAME +COMMIT := $(shell commit=$$(git describe --match '' --dirty --always) && \ + if [ -n "$$(git ls-files --other --exclude-per-directory .gitignore "$(FILES_DIR)")" ]; then \ + commit="$${commit%-dirty}-dirty"; \ + fi && \ + echo "$$commit") +ifndef COMMIT +$(error Failed to derive an image name. Explicitly define IMAGE_NAME) +endif +IMAGE_NAME := $(IMAGE_PREFIX)-$(COMMIT)-$(ARCH) +endif + +IMAGE_SUFFIX ?= +IMAGE_FILE = $(IMAGE_NAME)$(IMAGE_SUFFIX) + +LIBC_ARCH := $(shell uname -m) +LIBC ?= $(if $(findstring s390x,$(LIBC_ARCH)),gnu,musl) + +AGENT_PROTOCOL_FORWARDER = $(FILES_DIR)/usr/local/bin/agent-protocol-forwarder +KATA_AGENT = $(FILES_DIR)/usr/local/bin/kata-agent +PAUSE = $(FILES_DIR)/$(PAUSE_BUNDLE)/rootfs/pause +SKOPEO = $(FILES_DIR)/usr/bin/skopeo +UMOCI = $(FILES_DIR)/usr/local/bin/umoci +ATTESTATION_AGENT = $(FILES_DIR)/usr/local/bin/attestation-agent + +BINARIES = $(AGENT_PROTOCOL_FORWARDER) $(KATA_AGENT) $(PAUSE) + +ifdef USE_SKOPEO +BINARIES += $(SKOPEO) $(UMOCI) +endif + +KBC_URI ?= "null" +ifdef AA_KBC +BINARIES += $(ATTESTATION_AGENT) +$(shell sed -i "s/\(aa_kbc_params = \)\"[^\"]*\"/\1\"${AA_KBC}::${KBC_URI}\"/g" $(FILES_DIR)/etc/agent-config.toml) +endif + +AGENT_PROTOCOL_FORWARDER_SRC = ../.. + +KATA_AGENT_SRC = ../../../kata-containers/src/agent +KATA_AGENT_BUILD_TYPE = release + +SKOPEO_SRC = skopeo +SKOPEO_REPO = https://github.com/containers/skopeo + +UMOCI_SRC = umoci +UMOCI_REPO = https://github.com/opencontainers/umoci + +# Embed the pause container image +# https://github.com/arronwy/kata-containers/commit/75b9f3fa3caaae62f49b4733f65cbab0cc87dbee +PAUSE_SRC = pause +PAUSE_REPO = docker://k8s.gcr.io/pause +PAUSE_VERSION = 3.6 +PAUSE_BUNDLE = pause_bundle + +# Static libseccomp is necessary for kata-agent +# https://github.com/kata-containers/kata-containers/issues/5044#issuecomment-1239773921 +STATIC_LIBSECCOMP_BUILDER = ../../../kata-containers/ci/install_libseccomp.sh +STATIC_LIBSECCOMP_DIR = $(abspath staticlib) +STATIC_LIBSECCOMP = $(STATIC_LIBSECCOMP_DIR)/kata-libseccomp/lib/libseccomp.a + +ATTESTATION_AGENT_SRC = ../../../attestation-agent +ATTESTATION_AGENT_REPO = https://github.com/confidential-containers/attestation-agent + +binaries: $(BINARIES) + +$(AGENT_PROTOCOL_FORWARDER): force + cd "$(AGENT_PROTOCOL_FORWARDER_SRC)" && $(MAKE) agent-protocol-forwarder + install -D --compare "$(AGENT_PROTOCOL_FORWARDER_SRC)/agent-protocol-forwarder" "$@" + +$(KATA_AGENT): force $(STATIC_LIBSECCOMP) + cd "$(KATA_AGENT_SRC)/../libs" && $(MAKE) LIBC=$(LIBC) BUILD_TYPE=$(KATA_AGENT_BUILD_TYPE) LIBSECCOMP_LINK_TYPE=static LIBSECCOMP_LIB_PATH=$(dir $(STATIC_LIBSECCOMP)) + cd "$(KATA_AGENT_SRC)" && $(MAKE) LIBC=$(LIBC) BUILD_TYPE=$(KATA_AGENT_BUILD_TYPE) LIBSECCOMP_LINK_TYPE=static LIBSECCOMP_LIB_PATH=$(dir $(STATIC_LIBSECCOMP)) + install -D --compare "$(KATA_AGENT_SRC)/target/$(shell uname -m)-unknown-linux-$(LIBC)/$(KATA_AGENT_BUILD_TYPE)/$(@F)" "$@" + +$(STATIC_LIBSECCOMP): + $(STATIC_LIBSECCOMP_BUILDER) $(STATIC_LIBSECCOMP_DIR)/kata-libseccomp $(STATIC_LIBSECCOMP_DIR)/kata-gperf + +# Skoepo package packages are available in RHEL/CentOS 8 or later and Ubuntu 20.10 or later +$(SKOPEO_SRC): + git clone -b "v$(SKOPEO_VERSION)" "$(SKOPEO_REPO)" "$(SKOPEO_SRC)" + +$(SKOPEO_SRC)/bin/skopeo: $(SKOPEO_SRC) + cd "$(SKOPEO_SRC)" && make bin/skopeo + +$(SKOPEO): $(SKOPEO_SRC)/bin/skopeo + install -D --compare "$(SKOPEO_SRC)/bin/skopeo" "$@" + +# The umoci release page only publishes amd64 binaries. https://github.com/opencontainers/umoci/releases +$(UMOCI_SRC): + git clone -b "v$(UMOCI_VERSION)" "$(UMOCI_REPO)" "$(UMOCI_SRC)" + +$(UMOCI_SRC)/umoci: $(UMOCI_SRC) + cd "$(UMOCI_SRC)" && make + +$(UMOCI): $(UMOCI_SRC)/umoci + install -D --compare "$(UMOCI_SRC)/umoci" "$@" + +$(PAUSE_SRC): $(SKOPEO_SRC)/bin/skopeo + $(SKOPEO_SRC)/bin/skopeo --policy "$(FILES_DIR)/etc/containers/policy.json" copy "$(PAUSE_REPO):$(PAUSE_VERSION)" "oci:$(PAUSE_SRC):$(PAUSE_VERSION)" + +$(PAUSE): | $(PAUSE_SRC) $(UMOCI_SRC)/umoci + $(UMOCI_SRC)/umoci unpack --rootless --image "$(PAUSE_SRC):$(PAUSE_VERSION)" "${FILES_DIR}/$(PAUSE_BUNDLE)" + +$(ATTESTATION_AGENT_SRC): + git clone "$(ATTESTATION_AGENT_REPO)" "$(ATTESTATION_AGENT_SRC)" + +$(ATTESTATION_AGENT): $(ATTESTATION_AGENT_SRC) + cd "$(ATTESTATION_AGENT_SRC)" && make KBC="$(AA_KBC)" LIBC="$(LIBC)" + mkdir -p "$(@D)" + install --compare "$(ATTESTATION_AGENT_SRC)/app/target/$(LIBC_ARCH)-unknown-linux-$(LIBC)/release/attestation-agent" "$@" diff --git a/podvm/files/etc/agent-config.toml b/podvm/files/etc/agent-config.toml new file mode 100644 index 000000000..c58b62aa2 --- /dev/null +++ b/podvm/files/etc/agent-config.toml @@ -0,0 +1,48 @@ +# This disables signature verification which now defaults to true. +# We should consider a better solution. See #331 for more info +enable_signature_verification=false + +# When using the agent-config.toml the KATA_AGENT_SERVER_ADDR env var seems to be ignored, so set it here +server_addr="unix:///run/kata-containers/agent.sock" + +# This field sets up the KBC that attestation agent uses +# This is replaced in the makefile steps so do not set it manually +aa_kbc_params = "" + +# temp workaround for kata-containers/kata-containers#5590 +[endpoints] +allowed = [ +"AddARPNeighborsRequest", +"AddSwapRequest", +"CloseStdinRequest", +"CopyFileRequest", +"CreateContainerRequest", +"CreateSandboxRequest", +"DestroySandboxRequest", +"ExecProcessRequest", +"GetMetricsRequest", +"GetOOMEventRequest", +"GuestDetailsRequest", +"ListInterfacesRequest", +"ListRoutesRequest", +"MemHotplugByProbeRequest", +"OnlineCPUMemRequest", +"PauseContainerRequest", +"PullImageRequest", +"ReadStreamRequest", +"RemoveContainerRequest", +"ReseedRandomDevRequest", +"ResumeContainerRequest", +"SetGuestDateTimeRequest", +"SignalProcessRequest", +"StartContainerRequest", +"StartTracingRequest", +"StatsContainerRequest", +"StopTracingRequest", +"TtyWinResizeRequest", +"UpdateContainerRequest", +"UpdateInterfaceRequest", +"UpdateRoutesRequest", +"WaitProcessRequest", +"WriteStreamRequest" +] \ No newline at end of file diff --git a/podvm/files/etc/containers/policy.json b/podvm/files/etc/containers/policy.json new file mode 100644 index 000000000..e0b40fba5 --- /dev/null +++ b/podvm/files/etc/containers/policy.json @@ -0,0 +1,3 @@ +{ + "default": [{"type": "insecureAcceptAnything"}] +} diff --git a/podvm/files/etc/systemd/system/agent-protocol-forwarder.service b/podvm/files/etc/systemd/system/agent-protocol-forwarder.service new file mode 100644 index 000000000..129faf091 --- /dev/null +++ b/podvm/files/etc/systemd/system/agent-protocol-forwarder.service @@ -0,0 +1,14 @@ +[Unit] +Description=Agent Protocol Forwarder +After=cloud-init.target +Wants=cloud-init.target +DefaultDependencies=no + + +[Service] +ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -kata-agent-socket /run/kata-containers/agent.sock +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/podvm/files/etc/systemd/system/kata-agent.service b/podvm/files/etc/systemd/system/kata-agent.service new file mode 100644 index 000000000..d536f223f --- /dev/null +++ b/podvm/files/etc/systemd/system/kata-agent.service @@ -0,0 +1,16 @@ +[Unit] +Description=Kata Agent +After=network.target + +[Service] +ExecStart=/usr/local/bin/kata-agent --config /etc/agent-config.toml +ExecStartPre=-umount /sys/fs/cgroup/misc +ExecStartPre=ip netns add podns +ExecStartPre=ip netns exec podns ip link set lo up +ExecStopPost=ip netns delete podns +ExecStopPost=/usr/local/bin/kata-agent-clean --config /etc/agent-config.toml +# Now specified in the agent-config.toml Environment="KATA_AGENT_SERVER_ADDR=unix:///run/kata-containers/agent.sock" +SyslogIdentifier=kata-agent + +[Install] +WantedBy=multi-user.target diff --git a/podvm/files/etc/systemd/system/multi-user.target.wants/agent-protocol-forwarder.service b/podvm/files/etc/systemd/system/multi-user.target.wants/agent-protocol-forwarder.service new file mode 120000 index 000000000..9ff49ce02 --- /dev/null +++ b/podvm/files/etc/systemd/system/multi-user.target.wants/agent-protocol-forwarder.service @@ -0,0 +1 @@ +../agent-protocol-forwarder.service \ No newline at end of file diff --git a/podvm/files/etc/systemd/system/multi-user.target.wants/kata-agent.service b/podvm/files/etc/systemd/system/multi-user.target.wants/kata-agent.service new file mode 120000 index 000000000..b498ebe4a --- /dev/null +++ b/podvm/files/etc/systemd/system/multi-user.target.wants/kata-agent.service @@ -0,0 +1 @@ +../kata-agent.service \ No newline at end of file diff --git a/podvm/files/etc/systemd/system/multi-user.target.wants/run-image.mount b/podvm/files/etc/systemd/system/multi-user.target.wants/run-image.mount new file mode 120000 index 000000000..07254bd41 --- /dev/null +++ b/podvm/files/etc/systemd/system/multi-user.target.wants/run-image.mount @@ -0,0 +1 @@ +../run-image.mount \ No newline at end of file diff --git "a/podvm/files/etc/systemd/system/multi-user.target.wants/run-kata\\x2dcontainers.mount" "b/podvm/files/etc/systemd/system/multi-user.target.wants/run-kata\\x2dcontainers.mount" new file mode 120000 index 000000000..b92a05dad --- /dev/null +++ "b/podvm/files/etc/systemd/system/multi-user.target.wants/run-kata\\x2dcontainers.mount" @@ -0,0 +1 @@ +../run-kata\x2dcontainers.mount \ No newline at end of file diff --git a/podvm/files/etc/systemd/system/run-image.mount b/podvm/files/etc/systemd/system/run-image.mount new file mode 100644 index 000000000..f4afe119c --- /dev/null +++ b/podvm/files/etc/systemd/system/run-image.mount @@ -0,0 +1,12 @@ +[Unit] +Description=Mount unit for /run/image +Before=kata-agent.service + +[Mount] +What=/image +Where=/run/image +Type=none +Options=bind + +[Install] +WantedBy=multi-user.target diff --git "a/podvm/files/etc/systemd/system/run-kata\\x2dcontainers.mount" "b/podvm/files/etc/systemd/system/run-kata\\x2dcontainers.mount" new file mode 100644 index 000000000..64d5e27f6 --- /dev/null +++ "b/podvm/files/etc/systemd/system/run-kata\\x2dcontainers.mount" @@ -0,0 +1,12 @@ +[Unit] +Description=Mount unit for /run/kata-containers +Before=kata-agent.service + +[Mount] +What=/kata-containers +Where=/run/kata-containers +Type=none +Options=bind + +[Install] +WantedBy=multi-user.target diff --git a/podvm/files/usr/local/bin/kata-agent-clean b/podvm/files/usr/local/bin/kata-agent-clean new file mode 100755 index 000000000..4cae32d20 --- /dev/null +++ b/podvm/files/usr/local/bin/kata-agent-clean @@ -0,0 +1,36 @@ +#!/bin/bash +# Removes any parts of the kata-agent configuration that +# maybe left over after shutdown/restarts. +# List of items currently handled by this are: +# - ttrpc server sock (e.g. /run/kata-containers/agent.sock) +# + +function usage() { + echo "Usage: $0 --config " +} + +while (( "$#" )); do + case "$1" in + --config) config_path=$2 ;; + --help) usage; exit 0 ;; + *) usage 1>&2; exit 1;; + esac + shift 2 +done + +if [[ -z "${config_path-}" ]]; then + usage 1>&2 + exit 1 +fi + +# ttrpc server sock removal +server_addr_regex='server_addr="unix://([^"]+)"' +server_addr_line=$(grep "server_addr" "${config_path}") +[[ $server_addr_line =~ $server_addr_regex ]] +server_addr="${BASH_REMATCH[1]}" +if [ -S "$server_addr" ]; then + echo "Removing $server_addr" + rm -f "$server_addr" +else + echo "Can't remove $server_addr as it doesn't exist" +fi diff --git a/podvm/qcow2/centos/qemu-centos.pkr.hcl b/podvm/qcow2/centos/qemu-centos.pkr.hcl new file mode 100644 index 000000000..211cf4f8e --- /dev/null +++ b/podvm/qcow2/centos/qemu-centos.pkr.hcl @@ -0,0 +1,76 @@ +source "qemu" "centos" { + boot_command = [""] + disk_compression = true + disk_image = true + disk_size = "${var.disk_size}" + format = "qcow2" + headless = true + iso_checksum = "${var.cloud_image_checksum}" + iso_url = "${var.cloud_image_url}" + output_directory = "output" + qemuargs = [["-m", "${var.memory}"], ["-smp", "cpus=${var.cpus}"], ["-cdrom", "${var.cloud_init_image}"], ["-serial", "mon:stdio"]] + ssh_password = "${var.ssh_password}" + ssh_port = 22 + ssh_username = "${var.ssh_username}" + ssh_wait_timeout = "300s" + vm_name = "${var.qemu_image_name}" + shutdown_command = "sudo shutdown -h now" + qemu_binary = "/usr/libexec/qemu-kvm" +} + +build { + sources = ["source.qemu.centos"] + + provisioner "shell-local" { + command = "tar cf toupload/files.tar files" + } + + provisioner "file" { + source = "./toupload" + destination = "/tmp/" + } + + provisioner "shell" { + inline = [ + "cd /tmp && tar xf toupload/files.tar", + "rm toupload/files.tar" + ] + } + + provisioner "file" { + source = "qcow2/copy-files.sh" + destination = "~/copy-files.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/copy-files.sh" + ] + } + + provisioner "file" { + source = "qcow2/selinux_relabel.sh" + destination = "~/selinux_relabel.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/selinux_relabel.sh" + ] + } + + provisioner "file" { + source = "qcow2/misc-settings.sh" + destination = "~/misc-settings.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/misc-settings.sh" + ] + } + +} diff --git a/podvm/qcow2/centos/variables.pkr.hcl b/podvm/qcow2/centos/variables.pkr.hcl new file mode 100644 index 000000000..4c35d0a95 --- /dev/null +++ b/podvm/qcow2/centos/variables.pkr.hcl @@ -0,0 +1,50 @@ +// variables.pkr.hcl + +// For those variables that you don't provide a default for, you must +// set them from the command line, a var-file, or the environment. + +variable "cloud_init_image" { + type = string + default = "cloud-init.img" +} + +variable "cpus" { + type = string + default = "2" +} + +variable "disk_size" { + type = string + # This is the default virtual size of CentOS cloud image (qcow2) + default = "10240" +} + +variable "cloud_image_checksum" { + type = string + default = "b80ca9ccad8715aa1aeee582906096d3190b91cebe05fcb3ac7cb197fd68a7fe" +} + +variable "cloud_image_url" { + type = string + default = "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20200113.0.x86_64.qcow2" +} + +variable "memory" { + type = string + default = "2048M" +} + +variable "ssh_password" { + type = string + default = "PeerP0d" +} + +variable "ssh_username" { + type = string + default = "peerpod" +} + +variable "qemu_image_name" { + type = string + default = "peer-pod" +} diff --git a/podvm/qcow2/copy-files.sh b/podvm/qcow2/copy-files.sh new file mode 100755 index 000000000..6dbdadd53 --- /dev/null +++ b/podvm/qcow2/copy-files.sh @@ -0,0 +1,16 @@ +# copy-files.sh is used to copy required files into +# the correct location on the podvm image + +sudo mkdir -p /etc/containers +sudo cp /tmp/files/etc/agent-config.toml /etc/agent-config.toml +sudo cp -a /tmp/files/etc/containers/* /etc/containers/ +sudo cp -a /tmp/files/etc/systemd/* /etc/systemd/ +if [ -e /tmp/files/etc/aa-offline_fs_kbc-resources.json ]; then + sudo cp /tmp/files/etc/aa-offline_fs_kbc-resources.json /etc/aa-offline_fs_kbc-resources.json +fi + + +sudo mkdir -p /usr/local/bin +sudo cp -a /tmp/files/usr/* /usr/ + +sudo cp -a /tmp/files/pause_bundle / diff --git a/podvm/qcow2/misc-settings.sh b/podvm/qcow2/misc-settings.sh new file mode 100755 index 000000000..665077d6d --- /dev/null +++ b/podvm/qcow2/misc-settings.sh @@ -0,0 +1,16 @@ +# Uncomment the awk statement if you want to use mac as +# dhcp-identifier in Ubuntu +#awk '/^[[:space:]-]*dhcp4/{ split($0,arr,/dhcp4.*/) +# gsub(/-/," ", arr[1]) +# rep=arr[1] +# print $0} +# rep{ printf "%s%s\n", rep, "dhcp-identifier: mac" +# rep="" +# next} 1' /etc/netplan/50-cloud-init.yaml | sudo tee /etc/netplan/50-cloud-init.yaml + +# This ensures machine-id is generated during first boot and a unique +# dhcp IP is assigned to the VM +sudo echo -n > /etc/machine-id + +#Lock password for the ssh user (peerpod) to disallow logins +sudo passwd -l peerpod diff --git a/podvm/qcow2/rhel/qemu-rhel.pkr.hcl b/podvm/qcow2/rhel/qemu-rhel.pkr.hcl new file mode 100644 index 000000000..8875f0f1b --- /dev/null +++ b/podvm/qcow2/rhel/qemu-rhel.pkr.hcl @@ -0,0 +1,74 @@ +source "qemu" "rhel" { + boot_command = [""] + disk_compression = true + disk_image = true + disk_size = "${var.disk_size}" + format = "qcow2" + headless = true + iso_checksum = "${var.cloud_image_checksum}" + iso_url = "${var.cloud_image_url}" + output_directory = "output" + qemuargs = [["-m", "${var.memory}"], ["-smp", "cpus=${var.cpus}"], ["-cdrom", "${var.cloud_init_image}"], ["-serial", "mon:stdio"], ["-cpu", "Cascadelake-Server"]] + ssh_password = "${var.ssh_password}" + ssh_port = 22 + ssh_username = "${var.ssh_username}" + ssh_wait_timeout = "300s" + vm_name = "${var.qemu_image_name}" + shutdown_command = "sudo shutdown -h now" +} + +build { + sources = ["source.qemu.rhel"] + + provisioner "shell-local" { + command = "tar cf toupload/files.tar files" + } + + provisioner "file" { + source = "./toupload" + destination = "/tmp/" + } + + provisioner "shell" { + inline = [ + "cd /tmp && tar xf toupload/files.tar", + "rm toupload/files.tar" + ] + } + + provisioner "file" { + source = "qcow2/copy-files.sh" + destination = "~/copy-files.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/copy-files.sh" + ] + } + + provisioner "file" { + source = "qcow2/misc-settings.sh" + destination = "~/misc-settings.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/misc-settings.sh" + ] + } + + provisioner "file" { + source = "qcow2/selinux_relabel.sh" + destination = "~/selinux_relabel.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/selinux_relabel.sh" + ] + } +} diff --git a/podvm/qcow2/rhel/variables.pkr.hcl b/podvm/qcow2/rhel/variables.pkr.hcl new file mode 100644 index 000000000..be495adf1 --- /dev/null +++ b/podvm/qcow2/rhel/variables.pkr.hcl @@ -0,0 +1,49 @@ +// variables.pkr.hcl + +// For those variables that you don't provide a default for, you must +// set them from the command line, a var-file, or the environment. + +variable "cloud_init_image" { + type = string + default = "cloud-init.img" +} + +variable "cpus" { + type = string + default = "2" +} + +variable "disk_size" { + type = string + default = "6144" +} + +variable "cloud_image_checksum" { + type = string + default = "d96622d77bcbab5526fd42e7d933ee851d239327946992a018b0bfc9fad777e7" +} + +variable "cloud_image_url" { + type = string + default = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" +} + +variable "memory" { + type = string + default = "2048M" +} + +variable "ssh_password" { + type = string + default = "PeerP0d" +} + +variable "ssh_username" { + type = string + default = "peerpod" +} + +variable "qemu_image_name" { + type = string + default = "peer-pod" +} diff --git a/podvm/qcow2/selinux_relabel.sh b/podvm/qcow2/selinux_relabel.sh new file mode 100644 index 000000000..ed31afc99 --- /dev/null +++ b/podvm/qcow2/selinux_relabel.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +FOLDERS="/etc /usr/local/bin /pause_bundle" + +for entry in $FOLDERS +do + sudo restorecon -p -r $entry +done diff --git a/podvm/qcow2/ubuntu/qemu-ubuntu.pkr.hcl b/podvm/qcow2/ubuntu/qemu-ubuntu.pkr.hcl new file mode 100644 index 000000000..cc083c6ee --- /dev/null +++ b/podvm/qcow2/ubuntu/qemu-ubuntu.pkr.hcl @@ -0,0 +1,63 @@ +source "qemu" "ubuntu" { + boot_command = [""] + disk_compression = true + disk_image = true + disk_size = "${var.disk_size}" + format = "qcow2" + headless = true + iso_checksum = "${var.cloud_image_checksum}" + iso_url = "${var.cloud_image_url}" + output_directory = "output" + qemuargs = [["-m", "${var.memory}"], ["-smp", "cpus=${var.cpus}"], ["-cdrom", "${var.cloud_init_image}"], ["-serial", "mon:stdio"]] + ssh_password = "${var.ssh_password}" + ssh_port = 22 + ssh_username = "${var.ssh_username}" + ssh_wait_timeout = "300s" + vm_name = "${var.qemu_image_name}" + shutdown_command = "sudo shutdown -h now" +} + +build { + sources = ["source.qemu.ubuntu"] + + provisioner "shell-local" { + command = "tar cf toupload/files.tar files" + } + + provisioner "file" { + source = "./toupload" + destination = "/tmp/" + } + + provisioner "shell" { + inline = [ + "cd /tmp && tar xf toupload/files.tar", + "rm toupload/files.tar" + ] + } + + provisioner "file" { + source = "qcow2/copy-files.sh" + destination = "~/copy-files.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/copy-files.sh" + ] + } + + provisioner "file" { + source = "qcow2/misc-settings.sh" + destination = "~/misc-settings.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/misc-settings.sh" + ] + } + +} diff --git a/podvm/qcow2/ubuntu/variables.pkr.hcl b/podvm/qcow2/ubuntu/variables.pkr.hcl new file mode 100644 index 000000000..be495adf1 --- /dev/null +++ b/podvm/qcow2/ubuntu/variables.pkr.hcl @@ -0,0 +1,49 @@ +// variables.pkr.hcl + +// For those variables that you don't provide a default for, you must +// set them from the command line, a var-file, or the environment. + +variable "cloud_init_image" { + type = string + default = "cloud-init.img" +} + +variable "cpus" { + type = string + default = "2" +} + +variable "disk_size" { + type = string + default = "6144" +} + +variable "cloud_image_checksum" { + type = string + default = "d96622d77bcbab5526fd42e7d933ee851d239327946992a018b0bfc9fad777e7" +} + +variable "cloud_image_url" { + type = string + default = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" +} + +variable "memory" { + type = string + default = "2048M" +} + +variable "ssh_password" { + type = string + default = "PeerP0d" +} + +variable "ssh_username" { + type = string + default = "peerpod" +} + +variable "qemu_image_name" { + type = string + default = "peer-pod" +} diff --git a/podvm/qcow2/userdata.cfg b/podvm/qcow2/userdata.cfg new file mode 100644 index 000000000..1f650a5ea --- /dev/null +++ b/podvm/qcow2/userdata.cfg @@ -0,0 +1,18 @@ +#cloud-config +ssh_pwauth: true +chpasswd: + expire: false + +locale: en_US.UTF-8 +locale_configfile: /etc/default/locale + +users: + - name: peerpod + # This is the hash of the default password: PeerP0d created using + # openssl passwd -1 PeerP0d + # This user/password will be locked post image creation + # If you want to change the user and password, ensure you change + # the ssh_username and ssh_password in the variables + passwd: $1$u4tXfju6$lvXwBnkFT4S0depRtcTjo/ + lock_passwd: false + sudo: ALL=(ALL) NOPASSWD:ALL diff --git a/proto/podvminfo/podvminfo.pb.go b/proto/podvminfo/podvminfo.pb.go new file mode 100644 index 000000000..805e16078 --- /dev/null +++ b/proto/podvminfo/podvminfo.pb.go @@ -0,0 +1,645 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: podvminfo/podvminfo.proto + +package podvminfo + +import ( + context "context" + fmt "fmt" + github_com_containerd_ttrpc "github.com/containerd/ttrpc" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type GetInfoRequest struct { + PodName string `protobuf:"bytes,1,opt,name=PodName,proto3" json:"PodName,omitempty"` + PodNamespace string `protobuf:"bytes,2,opt,name=PodNamespace,proto3" json:"PodNamespace,omitempty"` + Wait bool `protobuf:"varint,3,opt,name=Wait,proto3" json:"Wait,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetInfoRequest) Reset() { *m = GetInfoRequest{} } +func (*GetInfoRequest) ProtoMessage() {} +func (*GetInfoRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_b5023c5fde235d5c, []int{0} +} +func (m *GetInfoRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *GetInfoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_GetInfoRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *GetInfoRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetInfoRequest.Merge(m, src) +} +func (m *GetInfoRequest) XXX_Size() int { + return m.Size() +} +func (m *GetInfoRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetInfoRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetInfoRequest proto.InternalMessageInfo + +type GetInfoResponse struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,proto3" json:"VMID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetInfoResponse) Reset() { *m = GetInfoResponse{} } +func (*GetInfoResponse) ProtoMessage() {} +func (*GetInfoResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_b5023c5fde235d5c, []int{1} +} +func (m *GetInfoResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *GetInfoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_GetInfoResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *GetInfoResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetInfoResponse.Merge(m, src) +} +func (m *GetInfoResponse) XXX_Size() int { + return m.Size() +} +func (m *GetInfoResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetInfoResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetInfoResponse proto.InternalMessageInfo + +func init() { + proto.RegisterType((*GetInfoRequest)(nil), "podvminfo.GetInfoRequest") + proto.RegisterType((*GetInfoResponse)(nil), "podvminfo.GetInfoResponse") +} + +func init() { proto.RegisterFile("podvminfo/podvminfo.proto", fileDescriptor_b5023c5fde235d5c) } + +var fileDescriptor_b5023c5fde235d5c = []byte{ + // 264 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0x41, 0x4b, 0xc3, 0x30, + 0x1c, 0xc5, 0x1b, 0x15, 0x67, 0x83, 0x28, 0xe4, 0xd4, 0xed, 0x10, 0x46, 0x41, 0xd8, 0xa5, 0x2d, + 0xe8, 0x37, 0x18, 0x03, 0xd9, 0x61, 0x3a, 0x7a, 0x98, 0xe0, 0x41, 0x48, 0x93, 0x54, 0x03, 0x6b, + 0xfe, 0xb1, 0x49, 0x3d, 0xfb, 0xf1, 0x76, 0xf4, 0xe8, 0xd1, 0xf5, 0x93, 0x88, 0xb1, 0x76, 0x14, + 0xbc, 0xfd, 0xf2, 0xfe, 0x2f, 0xbc, 0xc7, 0xc3, 0x63, 0x03, 0xe2, 0xad, 0x52, 0xba, 0x84, 0xac, + 0xa7, 0xd4, 0xd4, 0xe0, 0x80, 0x84, 0xbd, 0x10, 0x17, 0xf8, 0xe2, 0x56, 0xba, 0xa5, 0x2e, 0x21, + 0x97, 0xaf, 0x8d, 0xb4, 0x8e, 0x44, 0x78, 0xb4, 0x06, 0x71, 0xc7, 0x2a, 0x19, 0xa1, 0x29, 0x9a, + 0x85, 0xf9, 0xdf, 0x93, 0xc4, 0xf8, 0xbc, 0x43, 0x6b, 0x18, 0x97, 0xd1, 0x91, 0x3f, 0x0f, 0x34, + 0x42, 0xf0, 0xc9, 0x03, 0x53, 0x2e, 0x3a, 0x9e, 0xa2, 0xd9, 0x59, 0xee, 0x39, 0xbe, 0xc2, 0x97, + 0x7d, 0x86, 0x35, 0xa0, 0xad, 0xb7, 0x6d, 0x56, 0xcb, 0x45, 0x97, 0xe0, 0xf9, 0xfa, 0x1e, 0x87, + 0x6b, 0x10, 0x9b, 0xd5, 0x8f, 0x91, 0xcc, 0xf1, 0xa8, 0xfb, 0x43, 0xc6, 0xe9, 0xa1, 0xff, 0xb0, + 0xeb, 0x64, 0xf2, 0xdf, 0xe9, 0x37, 0x22, 0x0e, 0xe6, 0x4f, 0xbb, 0x3d, 0x0d, 0x3e, 0xf7, 0x34, + 0x78, 0x6f, 0x29, 0xda, 0xb5, 0x14, 0x7d, 0xb4, 0x14, 0x7d, 0xb5, 0x14, 0x3d, 0x2e, 0x9e, 0x95, + 0x7b, 0x69, 0x8a, 0x94, 0x43, 0x95, 0x71, 0xd0, 0xa5, 0x12, 0x52, 0x3b, 0xc5, 0xb6, 0x09, 0x07, + 0xed, 0x98, 0xd2, 0xb2, 0xb6, 0x19, 0xdf, 0x42, 0x23, 0x12, 0x66, 0x54, 0xc2, 0x04, 0x33, 0x0e, + 0xea, 0xcc, 0x8f, 0x77, 0x18, 0xb3, 0x38, 0xf5, 0xc2, 0xcd, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x47, 0x91, 0x92, 0x20, 0x6a, 0x01, 0x00, 0x00, +} + +func (m *GetInfoRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GetInfoRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GetInfoRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Wait { + i-- + if m.Wait { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } + if len(m.PodNamespace) > 0 { + i -= len(m.PodNamespace) + copy(dAtA[i:], m.PodNamespace) + i = encodeVarintPodvminfo(dAtA, i, uint64(len(m.PodNamespace))) + i-- + dAtA[i] = 0x12 + } + if len(m.PodName) > 0 { + i -= len(m.PodName) + copy(dAtA[i:], m.PodName) + i = encodeVarintPodvminfo(dAtA, i, uint64(len(m.PodName))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *GetInfoResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GetInfoResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GetInfoResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.VMID) > 0 { + i -= len(m.VMID) + copy(dAtA[i:], m.VMID) + i = encodeVarintPodvminfo(dAtA, i, uint64(len(m.VMID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintPodvminfo(dAtA []byte, offset int, v uint64) int { + offset -= sovPodvminfo(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *GetInfoRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.PodName) + if l > 0 { + n += 1 + l + sovPodvminfo(uint64(l)) + } + l = len(m.PodNamespace) + if l > 0 { + n += 1 + l + sovPodvminfo(uint64(l)) + } + if m.Wait { + n += 2 + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *GetInfoResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.VMID) + if l > 0 { + n += 1 + l + sovPodvminfo(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovPodvminfo(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozPodvminfo(x uint64) (n int) { + return sovPodvminfo(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *GetInfoRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&GetInfoRequest{`, + `PodName:` + fmt.Sprintf("%v", this.PodName) + `,`, + `PodNamespace:` + fmt.Sprintf("%v", this.PodNamespace) + `,`, + `Wait:` + fmt.Sprintf("%v", this.Wait) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func (this *GetInfoResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&GetInfoResponse{`, + `VMID:` + fmt.Sprintf("%v", this.VMID) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func valueToStringPodvminfo(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} + +type PodVMInfoService interface { + GetInfo(ctx context.Context, req *GetInfoRequest) (*GetInfoResponse, error) +} + +func RegisterPodVMInfoService(srv *github_com_containerd_ttrpc.Server, svc PodVMInfoService) { + srv.Register("podvminfo.PodVMInfo", map[string]github_com_containerd_ttrpc.Method{ + "GetInfo": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req GetInfoRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.GetInfo(ctx, &req) + }, + }) +} + +type podVMInfoClient struct { + client *github_com_containerd_ttrpc.Client +} + +func NewPodVMInfoClient(client *github_com_containerd_ttrpc.Client) PodVMInfoService { + return &podVMInfoClient{ + client: client, + } +} + +func (c *podVMInfoClient) GetInfo(ctx context.Context, req *GetInfoRequest) (*GetInfoResponse, error) { + var resp GetInfoResponse + if err := c.client.Call(ctx, "podvminfo.PodVMInfo", "GetInfo", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} +func (m *GetInfoRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPodvminfo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GetInfoRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GetInfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PodName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPodvminfo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPodvminfo + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPodvminfo + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PodName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PodNamespace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPodvminfo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPodvminfo + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPodvminfo + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PodNamespace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Wait", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPodvminfo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Wait = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipPodvminfo(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPodvminfo + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GetInfoResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPodvminfo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GetInfoResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GetInfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VMID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPodvminfo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPodvminfo + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPodvminfo + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VMID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPodvminfo(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPodvminfo + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipPodvminfo(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPodvminfo + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPodvminfo + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPodvminfo + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthPodvminfo + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupPodvminfo + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthPodvminfo + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthPodvminfo = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowPodvminfo = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupPodvminfo = fmt.Errorf("proto: unexpected end of group") +) diff --git a/proto/podvminfo/podvminfo.proto b/proto/podvminfo/podvminfo.proto new file mode 100644 index 000000000..478462b1a --- /dev/null +++ b/proto/podvminfo/podvminfo.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package podvminfo; + +option go_package = "github.com/confidential-containers/cloud-api-adaptor/proto/podvminfo"; + +service PodVMInfo { + rpc GetInfo(GetInfoRequest) returns (GetInfoResponse) {} +} + +message GetInfoRequest { + string PodName = 1; + string PodNamespace = 2; + + bool Wait = 3; +} + +message GetInfoResponse { + string VMID = 1; +} diff --git a/vsphere/README.md b/vsphere/README.md new file mode 100644 index 000000000..5aad9bb89 --- /dev/null +++ b/vsphere/README.md @@ -0,0 +1,45 @@ +# Setup instructions + +## Prerequisites: + +- A configured vCenter instance +- datacenter, datastore and cluster are required fields + +## Creating the vsphere template: +- Install packer on your system by following the instructions in the following [link](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli) +- Create the vsphere template + - [setting up authenticated registry support](../docs/registries-authentication.md) +``` +cd vsphere/image && make +``` + +For a rhel podvm image, try: +``` +cd vsphere/image && PODVM_DISTRO=rhel make +``` + +If successful, you can find the image on your vcenter deployment + +## File structure +- **vcenter.auto.pkrvars.hcl** - The main template. +- **settings.auto.pkrvars.hcl** - Guest settings referenced by the template file +- **vcenter.auto.pkrvars.hcl** - Your vCenter config + +If settings.auto.pkrvars.hcl and vcenter.auto.pkrvars.hcl are not present, they will be created +when you first run make. You can also choose to create these files if you prefer. + +Please take a look at variables.pkr.hcl for all the variables supported. + +Here are a few that you might find useful to configure the guest: +- *vm_cpu_count* + This field configures the number of vcpus for the guest. +- *vm_guest_size* + Determines the amount of guest memory. +- *vm_disk_size* + The size of the virtual hard disk. Please note that installation might fail if the disk size is too small. +- *vm_network_name* + The virtualized network adapter to use. vmxnet3 is the default. + +## Potential issues +The start of the installation uses automated keyboard input. Timing issues may prevent entering +the shell. Please try experimenting with vm_boot_wait if you encounter this problem. diff --git a/vsphere/image/Makefile b/vsphere/image/Makefile new file mode 100644 index 000000000..729979640 --- /dev/null +++ b/vsphere/image/Makefile @@ -0,0 +1,142 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +.PHONY: all check cleanconfig config build clean force +.DEFAULT_GOAL := all +SKOPEO_VERSION = 1.5.0 +UMOCI_VERSION = 0.4.7 + +ARCH := $(subst x86_64,amd64,$(shell uname -m)) + +FILES_DIR := files +FILES = $(shell find "$(FILES_DIR)" -type f -o -type l) + +AGENT_PROTOCOL_FORWARDER = $(FILES_DIR)/usr/local/bin/agent-protocol-forwarder +KATA_AGENT = $(FILES_DIR)/usr/local/bin/kata-agent +PAUSE = $(FILES_DIR)/$(PAUSE_BUNDLE)/rootfs/pause +SKOPEO = $(FILES_DIR)/usr/bin/skopeo +UMOCI = $(FILES_DIR)/usr/local/bin/umoci + +BINARIES = $(AGENT_PROTOCOL_FORWARDER) $(KATA_AGENT) $(PAUSE) + +ifdef USE_SKOPEO +BINARIES += $(SKOPEO) $(UMOCI) +endif + +AGENT_PROTOCOL_FORWARDER_SRC = ../.. + +KATA_AGENT_SRC = ../../../kata-containers/src/agent +KATA_AGENT_BUILD_TYPE = release + +SKOPEO_SRC = skopeo +SKOPEO_REPO = https://github.com/containers/skopeo + +UMOCI_SRC = umoci +UMOCI_REPO = https://github.com/opencontainers/umoci + +# Embed the pause container image +# https://github.com/arronwy/kata-containers/commit/75b9f3fa3caaae62f49b4733f65cbab0cc87dbee +PAUSE_SRC = pause +PAUSE_REPO = docker://k8s.gcr.io/pause +PAUSE_VERSION = 3.6 +PAUSE_BUNDLE = pause_bundle + +# Static libseccomp is necessary for kata-agent +# https://github.com/kata-containers/kata-containers/issues/5044#issuecomment-1239773921 +STATIC_LIB_BUILDER = ../../../kata-containers/ci/install_libseccomp.sh +STATIC_LIB_DIR = $(abspath staticlib) +STATIC_LIB = $(STATIC_LIB_DIR)/kata-libseccomp/lib/libseccomp.a + +GUESTCONFIG = settings.auto.pkrvars.hcl +VCENTERCONFIG = vcenter.auto.pkrvars.hcl +OVERWRITE = +ifndef PODVM_DISTRO +PODVM_DISTRO = ubuntu +endif + +check: +ifeq (, $(shell which packer)) + $(error "packer executable is not found, I looked here: $(PATH)") +endif +ifeq (, $(shell which go)) + $(error "go executable is not found, I looked here: $(PATH)") +endif + $(if $(strip $(GOPATH)),,$(error "GOPATH is unset, try "export GOPATH=/path/to/go"")) +ifeq ($(FORCE),1) + $(eval OVERWRITE := -force) +endif +ifeq ($(PODVM_DISTRO),ubuntu) + $(eval OPTS := ubuntu) +else ifeq ($(PODVM_DISTRO),rhel) + $(eval OPTS := rhel) +else + $(error PODVM_DISTRO is invalid or not defined) +endif + +cleanconfig: +ifeq ($(PODVM_DISTRO),ubuntu) + $(eval OPTS := ubuntu) +else ifeq ($(PODVM_DISTRO),rhel) + $(eval OPTS := rhel) +else + $(error PODVM_DISTRO is invalid or not defined) +endif + rm -rf ${OPTS}/$(VCENTERCONFIG) ${OPTS}/$(GUESTCONFIG) + + +config: | check + ./${OPTS}/scripts/createconfig.sh "guest" "${OPTS}/$(GUESTCONFIG)" + ./${OPTS}/scripts/createconfig.sh "vcenter" "${OPTS}/$(VCENTERCONFIG)" + +build: | config $(BINARIES) $(FILES) + rm -rf files.tar + cd files && tar cf ../files.tar * + packer build ${OVERWRITE} ${OPTS} + +$(AGENT_PROTOCOL_FORWARDER): force + cd "$(AGENT_PROTOCOL_FORWARDER_SRC)" && CLOUD_PROVIDER=vsphere $(MAKE) agent-protocol-forwarder + install -D --compare "$(AGENT_PROTOCOL_FORWARDER_SRC)/agent-protocol-forwarder" "$@" + +$(KATA_AGENT): force $(STATIC_LIB) + cd "$(KATA_AGENT_SRC)/../libs" && $(MAKE) BUILD_TYPE=$(KATA_AGENT_BUILD_TYPE) LIBSECCOMP_LINK_TYPE=static LIBSECCOMP_LIB_PATH=$(dir $(STATIC_LIB)) + cd "$(KATA_AGENT_SRC)" && $(MAKE) BUILD_TYPE=$(KATA_AGENT_BUILD_TYPE) LIBSECCOMP_LINK_TYPE=static LIBSECCOMP_LIB_PATH=$(dir $(STATIC_LIB)) + install -D --compare "$(KATA_AGENT_SRC)/target/$(shell uname -m)-unknown-linux-$(if $(findstring s390x,$(shell uname -m)),gnu,musl)/$(KATA_AGENT_BUILD_TYPE)/$(@F)" "$@" + +$(STATIC_LIB): + $(STATIC_LIB_BUILDER) $(STATIC_LIB_DIR)/kata-libseccomp $(STATIC_LIB_DIR)/kata-gperf + +# Skoepo package packages are available in RHEL/CentOS 8 or later and Ubuntu 20.10 or later +$(SKOPEO_SRC): + git clone -b "v$(SKOPEO_VERSION)" "$(SKOPEO_REPO)" "$(SKOPEO_SRC)" + +$(SKOPEO_SRC)/bin/skopeo: $(SKOPEO_SRC) + cd "$(SKOPEO_SRC)" && make bin/skopeo + +$(SKOPEO): $(SKOPEO_SRC)/bin/skopeo + install -D --compare "$(SKOPEO_SRC)/bin/skopeo" "$@" + +# The umoci release page only publishes amd64 binaries. https://github.com/opencontainers/umoci/releases +$(UMOCI_SRC): + git clone -b "v$(UMOCI_VERSION)" "$(UMOCI_REPO)" "$(UMOCI_SRC)" + +$(UMOCI_SRC)/umoci: $(UMOCI_SRC) + cd "$(UMOCI_SRC)" && make + +$(UMOCI): $(UMOCI_SRC)/umoci + install -D --compare "$(UMOCI_SRC)/umoci" "$@" + +$(PAUSE_SRC): $(SKOPEO_SRC)/bin/skopeo + $(SKOPEO_SRC)/bin/skopeo --policy "$(FILES_DIR)/etc/containers/policy.json" copy "$(PAUSE_REPO):$(PAUSE_VERSION)" "oci:$(PAUSE_SRC):$(PAUSE_VERSION)" + +$(PAUSE): | $(PAUSE_SRC) $(UMOCI_SRC)/umoci + $(UMOCI_SRC)/umoci unpack --rootless --image "$(PAUSE_SRC):$(PAUSE_VERSION)" "${FILES_DIR}/$(PAUSE_BUNDLE)" + +clean: cleanconfig + rm -f $(BINARIES) + rm -rf staticlib + rm -rf files.tar + rm -fr "$(SKOPEO_SRC)" "$(UMOCI_SRC)" "$(PAUSE_SRC)" "$(FILES_DIR)/$(PAUSE_BUNDLE)" + +all: build + +force: diff --git a/vsphere/image/files/etc/agent-config.toml b/vsphere/image/files/etc/agent-config.toml new file mode 100644 index 000000000..915189305 --- /dev/null +++ b/vsphere/image/files/etc/agent-config.toml @@ -0,0 +1,45 @@ + +# This disables signature verification which now defaults to true. +# We should consider a better solution. See #331 for more info +enable_signature_verification=false + +# When using the agent-config.toml the KATA_AGENT_SERVER_ADDR env var seems to be ignored, so set it here +server_addr="unix:///run/kata-containers/agent.sock" + +# temp workaround for kata-containers/kata-containers#5590 +[endpoints] +allowed = [ +"AddARPNeighborsRequest", +"AddSwapRequest", +"CloseStdinRequest", +"CopyFileRequest", +"CreateContainerRequest", +"CreateSandboxRequest", +"DestroySandboxRequest", +"ExecProcessRequest", +"GetMetricsRequest", +"GetOOMEventRequest", +"GuestDetailsRequest", +"ListInterfacesRequest", +"ListRoutesRequest", +"MemHotplugByProbeRequest", +"OnlineCPUMemRequest", +"PauseContainerRequest", +"PullImageRequest", +"ReadStreamRequest", +"RemoveContainerRequest", +"ReseedRandomDevRequest", +"ResumeContainerRequest", +"SetGuestDateTimeRequest", +"SignalProcessRequest", +"StartContainerRequest", +"StartTracingRequest", +"StatsContainerRequest", +"StopTracingRequest", +"TtyWinResizeRequest", +"UpdateContainerRequest", +"UpdateInterfaceRequest", +"UpdateRoutesRequest", +"WaitProcessRequest", +"WriteStreamRequest" +] diff --git a/vsphere/image/files/etc/cloud/cloud.cfg.d/06_podvm.cfg b/vsphere/image/files/etc/cloud/cloud.cfg.d/06_podvm.cfg new file mode 100644 index 000000000..6acc9cc43 --- /dev/null +++ b/vsphere/image/files/etc/cloud/cloud.cfg.d/06_podvm.cfg @@ -0,0 +1 @@ +datasource_list: [ VMware, None ] diff --git a/vsphere/image/files/etc/containers/policy.json b/vsphere/image/files/etc/containers/policy.json new file mode 100644 index 000000000..e0b40fba5 --- /dev/null +++ b/vsphere/image/files/etc/containers/policy.json @@ -0,0 +1,3 @@ +{ + "default": [{"type": "insecureAcceptAnything"}] +} diff --git a/vsphere/image/files/etc/systemd/system/agent-protocol-forwarder.service b/vsphere/image/files/etc/systemd/system/agent-protocol-forwarder.service new file mode 100644 index 000000000..8c51a31ab --- /dev/null +++ b/vsphere/image/files/etc/systemd/system/agent-protocol-forwarder.service @@ -0,0 +1,12 @@ +[Unit] +Description=Agent Protocol Forwarder +After=cloud-init.target +Wants=cloud-init.target +DefaultDependencies=no + + +[Service] +ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -kata-agent-socket /run/kata-containers/agent.sock + +[Install] +WantedBy=multi-user.target diff --git a/vsphere/image/files/etc/systemd/system/kata-agent.service b/vsphere/image/files/etc/systemd/system/kata-agent.service new file mode 100644 index 000000000..9d2c4af9f --- /dev/null +++ b/vsphere/image/files/etc/systemd/system/kata-agent.service @@ -0,0 +1,15 @@ +[Unit] +Description=Kata Agent +After=network.target + +[Service] +ExecStart=/usr/local/bin/kata-agent --config /etc/agent-config.toml +ExecStartPre=-umount /sys/fs/cgroup/misc +ExecStartPre=ip netns add podns +ExecStartPre=ip netns exec podns ip link set lo up +ExecStopPost=ip netns delete podns +#Environment="KATA_AGENT_SERVER_ADDR=unix:///run/kata-containers/agent.sock" +SyslogIdentifier=kata-agent + +[Install] +WantedBy=multi-user.target diff --git a/vsphere/image/files/etc/systemd/system/multi-user.target.wants/agent-protocol-forwarder.service b/vsphere/image/files/etc/systemd/system/multi-user.target.wants/agent-protocol-forwarder.service new file mode 120000 index 000000000..9ff49ce02 --- /dev/null +++ b/vsphere/image/files/etc/systemd/system/multi-user.target.wants/agent-protocol-forwarder.service @@ -0,0 +1 @@ +../agent-protocol-forwarder.service \ No newline at end of file diff --git a/vsphere/image/files/etc/systemd/system/multi-user.target.wants/kata-agent.service b/vsphere/image/files/etc/systemd/system/multi-user.target.wants/kata-agent.service new file mode 120000 index 000000000..b498ebe4a --- /dev/null +++ b/vsphere/image/files/etc/systemd/system/multi-user.target.wants/kata-agent.service @@ -0,0 +1 @@ +../kata-agent.service \ No newline at end of file diff --git a/vsphere/image/files/etc/systemd/system/multi-user.target.wants/run-image.mount b/vsphere/image/files/etc/systemd/system/multi-user.target.wants/run-image.mount new file mode 120000 index 000000000..07254bd41 --- /dev/null +++ b/vsphere/image/files/etc/systemd/system/multi-user.target.wants/run-image.mount @@ -0,0 +1 @@ +../run-image.mount \ No newline at end of file diff --git "a/vsphere/image/files/etc/systemd/system/multi-user.target.wants/run-kata\\x2dcontainers.mount" "b/vsphere/image/files/etc/systemd/system/multi-user.target.wants/run-kata\\x2dcontainers.mount" new file mode 120000 index 000000000..b92a05dad --- /dev/null +++ "b/vsphere/image/files/etc/systemd/system/multi-user.target.wants/run-kata\\x2dcontainers.mount" @@ -0,0 +1 @@ +../run-kata\x2dcontainers.mount \ No newline at end of file diff --git a/vsphere/image/files/etc/systemd/system/run-image.mount b/vsphere/image/files/etc/systemd/system/run-image.mount new file mode 100644 index 000000000..f4afe119c --- /dev/null +++ b/vsphere/image/files/etc/systemd/system/run-image.mount @@ -0,0 +1,12 @@ +[Unit] +Description=Mount unit for /run/image +Before=kata-agent.service + +[Mount] +What=/image +Where=/run/image +Type=none +Options=bind + +[Install] +WantedBy=multi-user.target diff --git "a/vsphere/image/files/etc/systemd/system/run-kata\\x2dcontainers.mount" "b/vsphere/image/files/etc/systemd/system/run-kata\\x2dcontainers.mount" new file mode 100644 index 000000000..64d5e27f6 --- /dev/null +++ "b/vsphere/image/files/etc/systemd/system/run-kata\\x2dcontainers.mount" @@ -0,0 +1,12 @@ +[Unit] +Description=Mount unit for /run/kata-containers +Before=kata-agent.service + +[Mount] +What=/kata-containers +Where=/run/kata-containers +Type=none +Options=bind + +[Install] +WantedBy=multi-user.target diff --git a/vsphere/image/misc-settings.sh b/vsphere/image/misc-settings.sh new file mode 100644 index 000000000..74267e85d --- /dev/null +++ b/vsphere/image/misc-settings.sh @@ -0,0 +1,13 @@ +# Uncomment the awk statement if you want to use mac as +# dhcp-identifier in Ubuntu +#awk '/^[[:space:]-]*dhcp4/{ split($0,arr,/dhcp4.*/) +# gsub(/-/," ", arr[1]) +# rep=arr[1] +# print $0} +# rep{ printf "%s%s\n", rep, "dhcp-identifier: mac" +# rep="" +# next} 1' /etc/netplan/50-cloud-init.yaml | sudo tee /etc/netplan/50-cloud-init.yaml + +# This ensures machine-id is generated during first boot and a unique +# dhcp IP is assigned to the VM +sudo echo -n > /etc/machine-id diff --git a/vsphere/image/rhel/data/ks.pkr.hcl b/vsphere/image/rhel/data/ks.pkr.hcl new file mode 100644 index 000000000..b1e74883a --- /dev/null +++ b/vsphere/image/rhel/data/ks.pkr.hcl @@ -0,0 +1,76 @@ + +# Red Hat Enterprise Linux Server 9 + +### Installs from the first attached CD-ROM/DVD on the system. +cdrom + +### Performs the kickstart installation in text mode. +text + +### Accepts the End User License Agreement. +eula --agreed + +### Sets the language to use during installation and the default language to use on the installed system. +lang ${vm_guest_os_language} + +### Sets the default keyboard type for the system. +keyboard ${vm_guest_os_keyboard} + +### bootproto dhcp and enabled at boot +network --bootproto=dhcp --onboot=yes --hostname=${vm_guest_hostname} + +### user with sudo privileges +user --name=${build_username} --plaintext --password=${build_password} --groups=wheel + +### firewall is disabled +firewall --disabled + +### selinux is permissive +selinux --permissive + +### Sets the system time zone. +timezone ${vm_guest_os_timezone} + +### Sets how the boot loader should be installed. +bootloader --location=mbr + +### Initialize any invalid partition tables found on disks. +zerombr + +### Removes partitions from the system, prior to creation of new partitions. +clearpart --all --initlabel + +# diskconfig +part /boot --fstype xfs --size=1024 --label=BOOTFS +part /boot/efi --fstype vfat --size=1024 --label=EFIFS +part pv.01 --size=100 --grow +volgroup sysvg --pesize=4096 pv.01 +logvol swap --fstype swap --name=lv_swap --vgname=sysvg --size=1024 --label=SWAPFS +logvol / --fstype xfs --name=lv_root --vgname=sysvg --percent=100 --label=ROOTFS + + +### Modifies the default set of services that will run under the default runlevel. +services --enabled=NetworkManager,sshd,vmtoolsd + +### Do not configure X on the installed system. +skipx + +### Packages selection. +%packages --ignoremissing --excludedocs +@core +-iwl*firmware +open-vm-tools +cloud-init +%end + +### Post-installation commands. +%post +echo "${build_username} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/${build_username} +sed -i "s/^.*requiretty/#Defaults requiretty/" /etc/sudoers +## Disable cloud-init for packer provisioners to succeed, cloud-init reconfigures ssh keys +touch /etc/cloud/cloud-init.disabled +%end + +### Reboot after the installation is complete. +### --eject attempt to eject the media before rebooting. +reboot --eject diff --git a/vsphere/image/rhel/scripts/createconfig.sh b/vsphere/image/rhel/scripts/createconfig.sh new file mode 100755 index 000000000..908bea206 --- /dev/null +++ b/vsphere/image/rhel/scripts/createconfig.sh @@ -0,0 +1,82 @@ +#!/bin/bash +ACTION="$1" +CONF="$2" +GREEN='\033[0;32m' +NOCOLOR='\033[0m' +default_guest_config=$(cat < $CONF +//vCenter configuration settings +vcenter_server = "${vcenter}" +datacenter = "${datacenter}" +vcenter_username = "${username}" +vcenter_password = "${password}" +datastore = "${datastore}" +cluster = "${cluster}" +template = "${template}" +EOF + printf "${GREEN} Created vcenter config file $CONF with user provided values" + cat "$CONF" + printf "${NOCOLOR}" + ;; + "guest") + [ -e "$CONF" ] && echo "$CONF already exists, not overwriting" && exit 0 + echo "Writing default guest config to $CONF" + echo -e "$default_guest_config" > $CONF + printf "${GREEN} Created guest config file $CONF with default values \n" + cat $CONF + printf "${NOCOLOR}" + ;; + *) + echo "invalid input" + exit 1 + ;; +esac diff --git a/vsphere/image/rhel/variables.pkr.hcl b/vsphere/image/rhel/variables.pkr.hcl new file mode 100644 index 000000000..f1d068be7 --- /dev/null +++ b/vsphere/image/rhel/variables.pkr.hcl @@ -0,0 +1,170 @@ +/* + DESCRIPTION: + Ubuntu Server 20.04 LTS variables using the Packer Builder for VMware vSphere (vsphere-iso). +*/ + +// vSphere Credentials + +variable "vcenter_server" { + type = string + description = "The fully qualified domain name or IP address of the vCenter Server instance" + default = null +} + +variable "vcenter_username" { + type = string + description = "vCenter login, do not use vsphere credentials here!" + default = null +} + +variable "datacenter" { + type = string + description = "The name of the target datacenter" + default = null +} + +variable "vcenter_password" { + type = string + description = "vCenter password" + default = null +} + +variable "datastore" { + type = string + description = "The name of the target datastore" + default = null +} + +variable "cluster" { + type = string + description = "The name of the target cluster" + default = null +} + +variable "template" { + type = string + description = "The name of the template to use" + default = null +} + +variable "vm_guest_os_type" { + type = string + description = "Guest OS type" + default = null +} + +variable "vm_guest_os_language" { + type = string + description = "Guest OS language" + default = "en_US" +} + +variable "vm_guest_os_keyboard" { + type = string + description = "Guest OS keyboard" + default = "us" +} + +variable "vm_guest_os_timezone" { + type = string + description = "Guest OS timezone" + default = "UTC" +} + +variable "vm_firmware" { + type = string + description = "Guest os bios - legacy or efi" + default = "efi" +} + +variable "vm_cpu_count" { + type = number + description = "Number of guest cpus" +} + +variable "vm_mem_size" { + type = number + description = "Guest memory" +} + +variable "vm_disk_size" { + type = number + description = "Guest disk size" +} + +variable "vm_disk_thin_provisioned" { + type = string + description = "Thin provisioning" + default = "true" +} + +variable "vm_interface_name" { + type = string + description = "interface name" + default = null +} + +variable "vm_network_name" { + type = string + description = "Network name" + default = "VM Network" +} + +variable "iso_url" { + type = string + description = "URL of installation iso" + default = null +} + +variable "iso_checksum_value" { + type = string + description = "Checksum of installation iso" + default = null +} + +variable "vm_boot_wait" { + type = string + description = "Wait time for guest keyboard input" + default = "2s" +} + +variable "ssh_port" { + type = number + description = "ssh port" + default = 22 +} + +variable "ssh_timeout" { + type = string + description = "time to wait for ssh to be alive in the guest" +} + +variable "convert_to_template" { + type = bool + description = "time to wait for ssh to be alive in the guest" + default = true +} + +variable "insecure_connection" { + type = bool + description = "time to wait for ssh to be alive in the guest" + default = true +} + +variable "vm_hostname" { + type = string + description = "name of the podvm guest, by default time is suffixed" + default = "podvm" +} + +variable "vm_username" { + type = string + description = "podvm username" + default = "peerpod" +} + +variable "vm_password" { + type = string + description = "podvm password" + default = "peerp0d" +} diff --git a/vsphere/image/rhel/vsphere-rhel.pkr.hcl b/vsphere/image/rhel/vsphere-rhel.pkr.hcl new file mode 100644 index 000000000..47341e6ce --- /dev/null +++ b/vsphere/image/rhel/vsphere-rhel.pkr.hcl @@ -0,0 +1,97 @@ +// BLOCK: source +// Defines the builder configuration blocks. + +// BLOCK: build +// Defines the builders to run, provisioners, and post-processors. + +locals { + data_source_content = { + "/ks.cfg" = templatefile("${abspath(path.root)}/data/ks.pkr.hcl", { + build_username = "${var.vm_username}" + build_password = "${var.vm_password}" + vm_guest_os_language = "${var.vm_guest_os_language}" + vm_guest_os_keyboard = "${var.vm_guest_os_keyboard}" + vm_guest_os_timezone = "${var.vm_guest_os_timezone}" + vm_guest_hostname = "${var.vm_hostname}" + }) + } + data_source_command = "inst.ks=cdrom:/ks.cfg" +} + +source "vsphere-iso" "rhel" { +// vcenter settings + vcenter_server = "${var.vcenter_server}" + username = "${var.vcenter_username}" + datacenter = "${var.datacenter}" + password = "${var.vcenter_password}" + datastore = "${var.datastore}" + cluster = "${var.cluster}" + insecure_connection = "${var.insecure_connection}" + guest_os_type = "${var.vm_guest_os_type}" + +// whether to create a template and if yes, the name + vm_name = "${var.template}" + convert_to_template = "${var.convert_to_template}" + +// ssh user/pass to the guest, same as cloudinit data + ssh_username = "${var.vm_username}" + ssh_password = "${var.vm_password}" + ssh_timeout = "${var.ssh_timeout}" + +// VM resources + CPUs = var.vm_cpu_count + RAM = var.vm_mem_size + RAM_reserve_all = true + + disk_controller_type = ["pvscsi"] + storage { + disk_size = var.vm_disk_size + disk_thin_provisioned = var.vm_disk_thin_provisioned + } + + network_adapters { + network = "${var.vm_network_name}" + network_card = "${var.vm_interface_name}" + } + +// Attach cloudinit config as a disk + cd_content = local.data_source_content + firmware = "${var.vm_firmware}" + +// iso path and checksum + iso_url = "${var.iso_url}" + iso_checksum = "${var.iso_checksum_value}" + +// boot command for autoinstall + boot_wait = "2s" + boot_command = [ + "", + "e", + "", + " text ${local.data_source_command}", + "x" + ] +} + +build { + sources = ["source.vsphere-iso.rhel"] + + provisioner "file" { + source = "./files.tar" + destination = "/tmp/" + } + + provisioner "file" { + source = "misc-settings.sh" + destination = "~/misc-settings.sh" + } + + provisioner "shell" { + inline = [ + "cd /tmp && sudo tar xf files.tar -C /", + "rm /tmp/files.tar", + "sudo bash ~/misc-settings.sh", + "sudo rm -rf /etc/cloud/cloud-init.disabled" + ] + } +} diff --git a/vsphere/image/ubuntu/data/meta-data b/vsphere/image/ubuntu/data/meta-data new file mode 100644 index 000000000..e69de29bb diff --git a/vsphere/image/ubuntu/data/user-data.pkr.hcl b/vsphere/image/ubuntu/data/user-data.pkr.hcl new file mode 100644 index 000000000..399e37d89 --- /dev/null +++ b/vsphere/image/ubuntu/data/user-data.pkr.hcl @@ -0,0 +1,107 @@ +#cloud-config + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# Ubuntu Server 20.04 LTS + +autoinstall: + version: 1 + apt: + geoip: true + preserve_sources_list: false + primary: + - arches: [amd64, i386] + uri: http://us.archive.ubuntu.com/ubuntu + - arches: [default] + uri: http://ports.ubuntu.com/ubuntu-ports + early-commands: + - sudo systemctl stop ssh + locale: en_US.UTF-8 + keyboard: + layout: ${vm_guest_os_keyboard} + storage: + config: + - ptable: gpt + path: /dev/sda + wipe: superblock + type: disk + id: disk-sda + - device: disk-sda + size: 500M + wipe: superblock + flag: boot + number: 1 + grub_device: true + type: partition + id: partition-0 + - fstype: fat32 + volume: partition-0 + label: EFIFS + type: format + id: format-efi + - device: disk-sda + size: 500M + wipe: superblock + number: 2 + type: partition + id: partition-1 + - fstype: xfs + volume: partition-1 + label: BOOTFS + type: format + id: format-boot + - device: disk-sda + size: -1 + wipe: superblock + number: 3 + type: partition + id: partition-2 + - name: sysvg + devices: + - partition-2 + type: lvm_volgroup + id: lvm_volgroup-0 + - name: root + volgroup: lvm_volgroup-0 + size: -1 + wipe: superblock + type: lvm_partition + id: lvm_partition-root + - fstype: xfs + volume: lvm_partition-root + type: format + label: ROOTFS + id: format-root + - path: / + device: format-root + type: mount + id: mount-root + - path: /boot + device: format-boot + type: mount + id: mount-boot + - path: /boot/efi + device: format-efi + type: mount + id: mount-efi + identity: + hostname: ${vm_guest_hostname} + username: ${build_username} + password: ${build_password_encrypted} + ssh: + install-server: true + allow-pw: true + packages: + - openssh-server + - open-vm-tools + - cloud-init + user-data: + disable_root: false + timezone: ${vm_guest_os_timezone} + late-commands: + - sed -i -e 's/^#\?PasswordAuthentication.*/PasswordAuthentication yes/g' /target/etc/ssh/sshd_config + - echo '${build_username} ALL=(ALL) NOPASSWD:ALL' > /target/etc/sudoers.d/${build_username} + - curtin in-target --target=/target -- chmod 440 /etc/sudoers.d/${build_username} diff --git a/vsphere/image/ubuntu/scripts/createconfig.sh b/vsphere/image/ubuntu/scripts/createconfig.sh new file mode 100755 index 000000000..9920c3272 --- /dev/null +++ b/vsphere/image/ubuntu/scripts/createconfig.sh @@ -0,0 +1,78 @@ +#!/bin/bash +ACTION="$1" +CONF="$2" +GREEN='\033[0;32m' +NOCOLOR='\033[0m' +default_guest_config=$(cat < $CONF +//vCenter configuration settings +vcenter_server = "${vcenter}" +datacenter = "${datacenter}" +vcenter_username = "${username}" +vcenter_password = "${password}" +datastore = "${datastore}" +cluster = "${cluster}" +template = "${template}" +EOF + printf "${GREEN} Created vcenter config file $CONF with user provided values" + cat "$CONF" + printf "${NOCOLOR}" + ;; + "guest") + [ -e "$CONF" ] && echo "$CONF already exists, not overwriting" && exit 0 + echo "Writing default guest config to $CONF" + echo -e "$default_guest_config" > $CONF + printf "${GREEN} Created guest config file $CONF with default values \n" + cat $CONF + printf "${NOCOLOR}" + ;; + *) + echo "invalid input" + exit 1 + ;; +esac diff --git a/vsphere/image/ubuntu/variables.pkr.hcl b/vsphere/image/ubuntu/variables.pkr.hcl new file mode 100644 index 000000000..f8b334089 --- /dev/null +++ b/vsphere/image/ubuntu/variables.pkr.hcl @@ -0,0 +1,158 @@ +/* + DESCRIPTION: + Ubuntu Server 20.04 LTS variables using the Packer Builder for VMware vSphere (vsphere-iso). +*/ + +// vSphere Credentials + +variable "vcenter_server" { + type = string + description = "The fully qualified domain name or IP address of the vCenter Server instance" + default = null +} + +variable "vcenter_username" { + type = string + description = "vCenter login, do not use vsphere credentials here!" + default = null +} + +variable "datacenter" { + type = string + description = "The name of the target datacenter" + default = null +} + +variable "vcenter_password" { + type = string + description = "vCenter password" + default = null +} + +variable "datastore" { + type = string + description = "The name of the target datastore" + default = null +} + +variable "cluster" { + type = string + description = "The name of the target cluster" + default = null +} + +variable "template" { + type = string + description = "The name of the template to use" + default = null +} + +variable "vm_guest_os_type" { + type = string + description = "Guest OS type" + default = null +} + +variable "vm_guest_os_language" { + type = string + description = "Guest OS language" + default = "en_US" +} + +variable "vm_guest_os_keyboard" { + type = string + description = "Guest OS keyboard" + default = "us" +} + +variable "vm_guest_os_timezone" { + type = string + description = "Guest OS timezone" + default = "UTC" +} + +variable "vm_firmware" { + type = string + description = "Guest os bios - legacy or efi" + default = "efi" +} + +variable "vm_cpu_count" { + type = number + description = "Number of guest cpus" +} + +variable "vm_mem_size" { + type = number + description = "Guest memory" +} + +variable "vm_disk_size" { + type = number + description = "Guest disk size" +} + +variable "vm_disk_thin_provisioned" { + type = string + description = "Thin provisioning" + default = "true" +} + +variable "vm_interface_name" { + type = string + description = "interface name" + default = null +} + +variable "vm_network_name" { + type = string + description = "Network name" + default = "VM Network" +} + +variable "iso_url" { + type = string + description = "URL of installation iso" + default = null +} + +variable "iso_checksum_value" { + type = string + description = "Checksum of installation iso" + default = null +} + +variable "vm_boot_wait" { + type = string + description = "Wait time for guest keyboard input" + default = "2s" +} + +variable "ssh_port" { + type = number + description = "ssh port" + default = 22 +} + +variable "ssh_timeout" { + type = string + description = "time to wait for ssh to be alive in the guest" +} + +variable "convert_to_template" { + type = bool + description = "time to wait for ssh to be alive in the guest" + default = true +} + +variable "insecure_connection" { + type = bool + description = "time to wait for ssh to be alive in the guest" + default = true +} + +variable "vm_hostname" { + type = string + description = "name of the podvm guest, by default time is suffixed" + default = "podvm" +} diff --git a/vsphere/image/ubuntu/vsphere-ubuntu.pkr.hcl b/vsphere/image/ubuntu/vsphere-ubuntu.pkr.hcl new file mode 100644 index 000000000..5b42ddbc0 --- /dev/null +++ b/vsphere/image/ubuntu/vsphere-ubuntu.pkr.hcl @@ -0,0 +1,111 @@ +// BLOCK: source +// Defines the builder configuration blocks. + +// BLOCK: build +// Defines the builders to run, provisioners, and post-processors. + +locals { + data_source_content = { + "/meta-data" = file("${abspath(path.root)}/data/meta-data") + "/user-data" = templatefile("${abspath(path.root)}/data/user-data.pkr.hcl", { + build_username = "peerpod" + build_password = "peerp0d" + build_password_encrypted = "$y$j9T$ifwkDEP8Rb9c47stpEGuF1$wT/v1.7ci8lEnD7rLmEqkzFmXuHwpHqhvjg3HQp7hm8" + vm_guest_os_language = "${var.vm_guest_os_language}" + vm_guest_os_keyboard = "${var.vm_guest_os_keyboard}" + vm_guest_os_timezone = "${var.vm_guest_os_timezone}" + vm_guest_hostname = "${var.vm_hostname}" + }) + } + data_source_command = "ds=\"nocloud\"" +} + +source "vsphere-iso" "ubuntu" { +// vcenter settings + vcenter_server = "${var.vcenter_server}" + username = "${var.vcenter_username}" + datacenter = "${var.datacenter}" + password = "${var.vcenter_password}" + datastore = "${var.datastore}" + cluster = "${var.cluster}" + insecure_connection = "${var.insecure_connection}" + guest_os_type = "${var.vm_guest_os_type}" + +// whether to create a template and if yes, the name + vm_name = "${var.template}" + convert_to_template = "${var.convert_to_template}" + +// ssh user/pass to the guest, same as cloudinit data + ssh_username = "peerpod" + ssh_password = "peerp0d" + ssh_timeout = "${var.ssh_timeout}" + +// VM resources + CPUs = var.vm_cpu_count + RAM = var.vm_mem_size + RAM_reserve_all = true + + disk_controller_type = ["pvscsi"] + storage { + disk_size = var.vm_disk_size + disk_thin_provisioned = var.vm_disk_thin_provisioned + } + + network_adapters { + network = "${var.vm_network_name}" + network_card = "${var.vm_interface_name}" + } + + +// Attach cloudinit config as a disk + cd_content = local.data_source_content + cd_label = "cidata" + firmware = "${var.vm_firmware}" + +// iso path and checksum + iso_url = "${var.iso_url}" + iso_checksum = "${var.iso_checksum_value}" + +// boot command for autoinstall + boot_wait = "2s" + boot_command = [ + "c", + "linux /casper/vmlinuz --- autoinstall ${local.data_source_command}", + "", + "initrd /casper/initrd", + "", + "boot", + "" + ] +} + + +build { + sources = ["source.vsphere-iso.ubuntu"] + + provisioner "file" { + source = "./files.tar" + destination = "/tmp/" + } + + provisioner "shell" { + inline = [ +// delete preinstalled configs so that we can choose our own datasource + "cd /etc/cloud/cloud.cfg.d && sudo rm -rf 99-installer.cfg && sudo rm -rf 90_dpkg.cfg", + "cd /tmp && sudo tar xf files.tar -C /", + "rm /tmp/files.tar" + ] + } + + provisioner "file" { + source = "misc-settings.sh" + destination = "~/misc-settings.sh" + } + + provisioner "shell" { + remote_folder = "~" + inline = [ + "sudo bash ~/misc-settings.sh" + ] + } +} diff --git a/webhook/Dockerfile b/webhook/Dockerfile new file mode 100644 index 000000000..d1f93c9e8 --- /dev/null +++ b/webhook/Dockerfile @@ -0,0 +1,28 @@ +# Build the manager binary +FROM golang:1.18 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +#COPY api/ api/ +#COPY controllers/ controllers/ +COPY pkg/ pkg/ + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/webhook/LICENSE b/webhook/LICENSE new file mode 100644 index 000000000..5c389317e --- /dev/null +++ b/webhook/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2020 Red Hat, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/webhook/Makefile b/webhook/Makefile new file mode 100644 index 000000000..c5bcecc95 --- /dev/null +++ b/webhook/Makefile @@ -0,0 +1,148 @@ + +# Image URL to use all building/pushing image targets +IMG ?= controller:latest +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.24.2 + +KIND_CLUSTER_NAME ?= webhook + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# This is a requirement for 'setup-envtest.sh' in the test target. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out + +.PHONY: test-e2e +test-e2e: ## Run the end-to-end tests on a local kind cluster. + bash ./tests/e2e/run-local.sh + +##@ Build + +.PHONY: build +build: generate fmt vet ## Build manager binary. + go build -o bin/manager main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./main.go + +.PHONY: docker-build +docker-build: test ## Build docker image with the manager. + docker build -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + docker push ${IMG} + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +.PHONY: undeploy +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: kind-cluster +kind-cluster: ## create a kind cluster for testing + kind create cluster --image "kindest/node:v$(ENVTEST_K8S_VERSION)" \ + --wait 120s --name "$(KIND_CLUSTER_NAME)" + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.1/cert-manager.yaml + +.PHONY: kind-delete +kind-delete: ## delete the kind test cluster + kind delete cluster --name "$(KIND_CLUSTER_NAME)" + +.PHONY: kind-load +kind-load: ## load the image in the local kind cluster + kind load docker-image ${IMG} --name webhook + +.PHONY: kind-deploy +kind-deploy: docker-build kind-load deploy ## deploy the webhook in the local kind cluster + +##@ Build Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest + +## Tool Versions +KUSTOMIZE_VERSION ?= v3.8.7 +CONTROLLER_TOOLS_VERSION ?= v0.9.2 + +KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + test -s $(LOCALBIN)/kustomize || { curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN); } + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): $(LOCALBIN) + test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest diff --git a/webhook/PROJECT b/webhook/PROJECT new file mode 100644 index 000000000..1d07b8543 --- /dev/null +++ b/webhook/PROJECT @@ -0,0 +1,6 @@ +domain: confidential-containers +layout: +- go.kubebuilder.io/v3 +projectName: peer-pods-webhook +repo: confidential-containers/peer-pods-webhook +version: "3" diff --git a/webhook/README.md b/webhook/README.md new file mode 100644 index 000000000..b8354cd7e --- /dev/null +++ b/webhook/README.md @@ -0,0 +1,32 @@ +# Introduction +This mutating webhook modifies a POD spec using specific runtimeclass to remove all `resources` entries and replace it with peer-pod extended resource. + +## Need for mutating webhook +A peer-pod uses resources at two places: +- Kubernetes Worker Node: Peer-Pod metadata, Kata shim resources, remote-hypervisor/cloud-api-adaptor resources, vxlan etc +- Cloud Instance: The actual peer-pod VM running in the cloud (eg. EC2 instance in AWS, or Azure VM instance) + +For peer-pods case the resources are really consumed outside of the worker node. It’s external to the Kubernetes cluster. + +This creates two problems: +1. Peer-pod scheduling can fail due to the unavailability of required resources on the worker node even though the peer-pod will not consume the requested resources from the worker node. + +2. Cluster-admin have no way to view the actual peer-pods VM capacity and consumption. + + +A simple solution to the above problems is to advertise peer-pod capacity as Kubernetes extended resources and let Kubernetes scheduler handle the peer-pod capacity tracking and accounting. Additionally, POD overhead can be used to account for actual `cpu` and `mem` resource requirements on the Kubernetes worker node. +The mutating webhook removes any `resources` entries from the Pod spec and adds the peer-pods extended resources. + + +![](https://i.imgur.com/MYwSQaX.png) + + + +## Installation + +Please refer to the following [instructions](docs/INSTALL.md) + +## Development + +Please refer to the following [instructions](docs/DEVELOPMENT.md) + diff --git a/webhook/config/certmanager/certificate.yaml b/webhook/config/certmanager/certificate.yaml new file mode 100644 index 000000000..52d866183 --- /dev/null +++ b/webhook/config/certmanager/certificate.yaml @@ -0,0 +1,25 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/webhook/config/certmanager/kustomization.yaml b/webhook/config/certmanager/kustomization.yaml new file mode 100644 index 000000000..bebea5a59 --- /dev/null +++ b/webhook/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/webhook/config/certmanager/kustomizeconfig.yaml b/webhook/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 000000000..90d7c313c --- /dev/null +++ b/webhook/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/webhook/config/default/kustomization.yaml b/webhook/config/default/kustomization.yaml new file mode 100644 index 000000000..508375c28 --- /dev/null +++ b/webhook/config/default/kustomization.yaml @@ -0,0 +1,74 @@ +# Adds namespace to all resources. +namespace: peer-pods-webhook-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: peer-pods-webhook- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +#- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +- manager_auth_proxy_patch.yaml + +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace +- name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml +- name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace +- name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service diff --git a/webhook/config/default/manager_auth_proxy_patch.yaml b/webhook/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 000000000..131a31429 --- /dev/null +++ b/webhook/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,34 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.11.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/webhook/config/default/manager_config_patch.yaml b/webhook/config/default/manager_config_patch.yaml new file mode 100644 index 000000000..6c400155c --- /dev/null +++ b/webhook/config/default/manager_config_patch.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--config=controller_manager_config.yaml" + volumeMounts: + - name: manager-config + mountPath: /controller_manager_config.yaml + subPath: controller_manager_config.yaml + volumes: + - name: manager-config + configMap: + name: manager-config diff --git a/webhook/config/default/manager_webhook_patch.yaml b/webhook/config/default/manager_webhook_patch.yaml new file mode 100644 index 000000000..738de350b --- /dev/null +++ b/webhook/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/webhook/config/default/webhookcainjection_patch.yaml b/webhook/config/default/webhookcainjection_patch.yaml new file mode 100644 index 000000000..28b1826d4 --- /dev/null +++ b/webhook/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,8 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/webhook/config/manager/controller_manager_config.yaml b/webhook/config/manager/controller_manager_config.yaml new file mode 100644 index 000000000..1220baff7 --- /dev/null +++ b/webhook/config/manager/controller_manager_config.yaml @@ -0,0 +1,11 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: 127.0.0.1:8080 +webhook: + port: 9443 +leaderElection: + leaderElect: true + resourceName: a3663802.confidential-containers diff --git a/webhook/config/manager/kustomization.yaml b/webhook/config/manager/kustomization.yaml new file mode 100644 index 000000000..bcc9bdc6a --- /dev/null +++ b/webhook/config/manager/kustomization.yaml @@ -0,0 +1,15 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/confidential-containers/peer-pods-webhook diff --git a/webhook/config/manager/manager.yaml b/webhook/config/manager/manager.yaml new file mode 100644 index 000000000..95bf6888c --- /dev/null +++ b/webhook/config/manager/manager.yaml @@ -0,0 +1,67 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 2 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + securityContext: + runAsNonRoot: true + containers: + - command: + - /manager + args: + - --leader-elect + image: controller:latest + name: manager + env: + - name: TARGET_RUNTIMECLASS + value: kata-remote-cc + - name: POD_VM_INSTANCE_TYPE + value: t2.small + - name: POD_VM_EXTENDED_RESOURCE + value: kata.peerpods.io/vm + securityContext: + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/webhook/config/prometheus/kustomization.yaml b/webhook/config/prometheus/kustomization.yaml new file mode 100644 index 000000000..ed137168a --- /dev/null +++ b/webhook/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/webhook/config/prometheus/monitor.yaml b/webhook/config/prometheus/monitor.yaml new file mode 100644 index 000000000..d19136ae7 --- /dev/null +++ b/webhook/config/prometheus/monitor.yaml @@ -0,0 +1,20 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager diff --git a/webhook/config/rbac/auth_proxy_client_clusterrole.yaml b/webhook/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 000000000..51a75db47 --- /dev/null +++ b/webhook/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/webhook/config/rbac/auth_proxy_role.yaml b/webhook/config/rbac/auth_proxy_role.yaml new file mode 100644 index 000000000..80e1857c5 --- /dev/null +++ b/webhook/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/webhook/config/rbac/auth_proxy_role_binding.yaml b/webhook/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 000000000..ec7acc0a1 --- /dev/null +++ b/webhook/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/webhook/config/rbac/auth_proxy_service.yaml b/webhook/config/rbac/auth_proxy_service.yaml new file mode 100644 index 000000000..71f179727 --- /dev/null +++ b/webhook/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager diff --git a/webhook/config/rbac/kustomization.yaml b/webhook/config/rbac/kustomization.yaml new file mode 100644 index 000000000..731832a6a --- /dev/null +++ b/webhook/config/rbac/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/webhook/config/rbac/leader_election_role.yaml b/webhook/config/rbac/leader_election_role.yaml new file mode 100644 index 000000000..4190ec805 --- /dev/null +++ b/webhook/config/rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/webhook/config/rbac/leader_election_role_binding.yaml b/webhook/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 000000000..1d1321ed4 --- /dev/null +++ b/webhook/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/webhook/config/rbac/role.yaml b/webhook/config/rbac/role.yaml new file mode 100644 index 000000000..98157e2b7 --- /dev/null +++ b/webhook/config/rbac/role.yaml @@ -0,0 +1,20 @@ + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/webhook/config/rbac/role_binding.yaml b/webhook/config/rbac/role_binding.yaml new file mode 100644 index 000000000..2070ede44 --- /dev/null +++ b/webhook/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/webhook/config/rbac/service_account.yaml b/webhook/config/rbac/service_account.yaml new file mode 100644 index 000000000..7cd6025bf --- /dev/null +++ b/webhook/config/rbac/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller-manager + namespace: system diff --git a/webhook/config/webhook/kustomization.yaml b/webhook/config/webhook/kustomization.yaml new file mode 100644 index 000000000..9cf26134e --- /dev/null +++ b/webhook/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/webhook/config/webhook/kustomizeconfig.yaml b/webhook/config/webhook/kustomizeconfig.yaml new file mode 100644 index 000000000..66781c6cb --- /dev/null +++ b/webhook/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,18 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/webhook/config/webhook/manifests.yaml b/webhook/config/webhook/manifests.yaml new file mode 100644 index 000000000..de51c2c85 --- /dev/null +++ b/webhook/config/webhook/manifests.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-v1-pod + failurePolicy: Fail + name: mwebhook.peerpods.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: None + namespaceSelector: + matchExpressions: + - key: peerpods + operator: In + values: ["true"] diff --git a/webhook/config/webhook/service.yaml b/webhook/config/webhook/service.yaml new file mode 100644 index 000000000..c9f09d4b8 --- /dev/null +++ b/webhook/config/webhook/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/webhook/docs/DEVELOPMENT.md b/webhook/docs/DEVELOPMENT.md new file mode 100644 index 000000000..b21827de1 --- /dev/null +++ b/webhook/docs/DEVELOPMENT.md @@ -0,0 +1,98 @@ +# Introduction +These instructions should help you to build a custom version of the webhook with your +changes + +## Prerequisites +- Golang (1.18.x) +- Operator SDK version (1.23.x+) +- podman and podman-docker or docker +- Access to Kubernetes cluster (1.24+) +- Container registry to store images + + +### Using kind cluster +For `kind` clusters, you can use the following Makefile targets + +Create kind cluster +``` +make kind-cluster +``` +Deploy the webhook in the kind cluster +``` +make kind-deploy IMG=//peer-pods-webhook: +``` + +If not using `kind`, the follow these steps to deploy the webhook + +### Deploy cert-manager +``` +kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.9.1/cert-manager.yaml +``` + +### Running on the cluster +1. Build and push your image to the location specified by `IMG`: + +```sh +make docker-build docker-push IMG=//peer-pods-webhook: +``` + +2. Deploy the controller to the cluster with the image specified by `IMG`: + +```sh +make deploy IMG=//peer-pods-webhook: +``` +3. To delete the webhook from the cluster: + +```sh +make undeploy IMG=//peer-pods-webhook: +``` + +### Testing +You can manually test your changes by following the steps: + +1. Create the runtimeclass +```sh +kubectl apply -f hack/rc.yaml +``` +2. Create the pod +```sh +kubectl apply -f hack/pod.yaml +``` +3. View the mutated pod +```sh +kubectl get -f hack/pod.yaml -o yaml | grep kata.peerpods +``` + +You can see that that the `hack/pod.yaml` has Kubernetes resources specified in the spec: +``` +resources: + requests: + cpu: 1 + memory: 1Gi + limits: + cpu: 1 + memory: 2Gi +``` +In the mutated pod these have been removed and the pod overhead +``` + overhead: + cpu: 250m + memory: 120Mi +``` + +and peer pod vm limit added. +``` + resources: + limits: + kata.peerpods.io/vm: "1" + requests: + kata.peerpods.io/vm: "1" +``` + +However, before opening a pull request with your changes, it is recommended that you run +the end-to-end tests locally. Ensure that you have [bats](https://bats-core.readthedocs.io), +docker (or podman-docker), and [kind](https://kind.sigs.k8s.io/) installed in your system; +and then run: +```sh +make test-e2e +``` diff --git a/webhook/docs/INSTALL.md b/webhook/docs/INSTALL.md new file mode 100644 index 000000000..d2e0ddf3f --- /dev/null +++ b/webhook/docs/INSTALL.md @@ -0,0 +1,76 @@ +## Getting Started +You’ll need a Kubernetes cluster to run against. You can use [KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. + +You'll also need to advertise the extended resource `kata.peerpods.io/vm`. + +A simple daemonset is provided under the following [directory](../hack/extended-resources/). + +``` +cd ../hack/extended-resources +./setup.sh +``` + +The above command advertises `20` pod VM resources. You can modify the spec as needed. + +To verify, check the node object +``` +kubectl get node $NODENAME -o=jsonpath='{.status.allocatable} | jq' +``` + +You should see a similar output like the one below: +``` +{ + "attachable-volumes-aws-ebs": "25", + "cpu": "2", + "ephemeral-storage": "56147256023", + "hugepages-1Gi": "0", + "hugepages-2Mi": "0", + "kata.peerpods.io/vm": "20", + "memory": "7935928Ki", + "pods": "110" +} +``` + +### Using kind cluster +For `kind` clusters, you can use the following Makefile targets + +Create kind cluster +``` +make kind-cluster +``` +Deploy the webhook in the kind cluster +``` +make kind-deploy IMG=quay.io/confidential-containers/peer-pods-webhook +``` + +If not using `kind`, the follow these steps to deploy the webhook + +### Deploy cert-manager +``` +kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.9.1/cert-manager.yaml +``` + +### Deploy webhook +``` +kubectl apply -f hack/webhook-deploy.yaml +``` + +The default `RuntimeClass` that the webhook monitors is `kata-remote-cc`. +The default `RuntimeClass` can be changed by modifying the `TARGET_RUNTIMECLASS` environment variable. +For example, executing the following command changes it to `kata` + +``` +kubectl set env deployment/peer-pods-webhook-controller-manager -n peer-pods-webhook-system TARGET_RUNTIMECLASS=kata +``` + +The default Pod VM instance type is `t2.small` and can be changed by modifying the `POD_VM_INSTANCE_TYPE` environment variable. + +### Label namespace for mutation + +The mutation needs to be opt-in to avoid any errors or conflicts with critical control-plane pods. +For any namespace where you want to create peer pods, add the following label `peerpods: true` + +``` +export NAMESPACE="REPLACE_ME" +kubectl label ns $NAMESPACE peerpods=true +``` diff --git a/webhook/go.mod b/webhook/go.mod new file mode 100644 index 000000000..38581f338 --- /dev/null +++ b/webhook/go.mod @@ -0,0 +1,79 @@ +module confidential-containers/peer-pods-webhook + +go 1.18 + +require ( + k8s.io/api v0.24.2 + k8s.io/apimachinery v0.24.2 + k8s.io/client-go v0.24.2 + sigs.k8s.io/controller-runtime v0.12.2 +) + +require ( + cloud.google.com/go v0.81.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful v2.9.5+incompatible // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-logr/logr v1.2.0 // indirect + github.com/go-logr/zapr v1.2.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/go-cmp v0.5.5 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.12.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.19.1 // indirect + golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/apiextensions-apiserver v0.24.2 // indirect + k8s.io/component-base v0.24.2 // indirect + k8s.io/klog/v2 v2.60.1 // indirect + k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect + sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/webhook/go.sum b/webhook/go.sum new file mode 100644 index 000000000..b81635ead --- /dev/null +++ b/webhook/go.sum @@ -0,0 +1,963 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= +github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= +k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= +k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= +k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= +k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM= +k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= +k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA= +k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= +k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= +k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= +k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= +sigs.k8s.io/controller-runtime v0.12.2 h1:nqV02cvhbAj7tbt21bpPpTByrXGn2INHRsi39lXy9sE= +sigs.k8s.io/controller-runtime v0.12.2/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/webhook/hack/boilerplate.go.txt b/webhook/hack/boilerplate.go.txt new file mode 100644 index 000000000..29c55ecda --- /dev/null +++ b/webhook/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/webhook/hack/deploy.yaml b/webhook/hack/deploy.yaml new file mode 100644 index 000000000..14e16746e --- /dev/null +++ b/webhook/hack/deploy.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-remote-cc + containers: + - name: nginx + image: bitnami/nginx diff --git a/webhook/hack/extended-resources/ext-res-cm.yaml b/webhook/hack/extended-resources/ext-res-cm.yaml new file mode 100644 index 000000000..0e90894bd --- /dev/null +++ b/webhook/hack/extended-resources/ext-res-cm.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +data: + script: |- + #!/bin/sh + patch_node() { + curl -k --header "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ + --header "Content-Type: application/json-patch+json" \ + --request PATCH \ + --data '[{"op": "add", "path": "/status/capacity/kata.peerpods.io~1vm", "value": "'$1'"}]' \ + https://kubernetes.default.svc.cluster.local/api/v1/nodes/${MY_NODE_NAME}/status + } + patch_node 20 + while [ 1 ] + do + sleep 10 + done + +kind: ConfigMap +metadata: + name: ext-res-updater + namespace: kube-system diff --git a/webhook/hack/extended-resources/ext-res-ds.yaml b/webhook/hack/extended-resources/ext-res-ds.yaml new file mode 100644 index 000000000..fd588d790 --- /dev/null +++ b/webhook/hack/extended-resources/ext-res-ds.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: ext-res-updater + name: ext-res-updater + namespace: kube-system +spec: + selector: + matchLabels: + app: ext-res-updater + template: + metadata: + labels: + app: ext-res-updater + spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + serviceAccountName: ext-res-updater + containers: + - image: curlimages/curl + command: ["sh"] + args: ["/ext-res-updater.sh"] + name: updater + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: configmap + mountPath: /ext-res-updater.sh + subPath: ext-res-updater.sh + volumes: + - name: configmap + configMap: + name: ext-res-updater + items: + - key: script + path: ext-res-updater.sh diff --git a/webhook/hack/extended-resources/ext-res-rbac.yaml b/webhook/hack/extended-resources/ext-res-rbac.yaml new file mode 100644 index 000000000..ce4d2925c --- /dev/null +++ b/webhook/hack/extended-resources/ext-res-rbac.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ext-res-updater + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ext-res-updater +rules: +- apiGroups: + - '' + resources: + - 'nodes/status' + verbs: + - 'patch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ext-res-updater +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ext-res-updater +subjects: +- kind: ServiceAccount + name: ext-res-updater + namespace: kube-system diff --git a/webhook/hack/extended-resources/setup.sh b/webhook/hack/extended-resources/setup.sh new file mode 100755 index 000000000..fa601a62b --- /dev/null +++ b/webhook/hack/extended-resources/setup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +kubectl apply -f ext-res-rbac.yaml +kubectl apply -f ext-res-cm.yaml +kubectl apply -f ext-res-ds.yaml diff --git a/webhook/hack/pod.yaml b/webhook/hack/pod.yaml new file mode 100644 index 000000000..f1d868c3a --- /dev/null +++ b/webhook/hack/pod.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + run: nginx + name: nginx +spec: + containers: + - image: bitnami/nginx + name: nginx + resources: + requests: + cpu: 1 + memory: 1Gi + limits: + cpu: 1 + memory: 2Gi + runtimeClassName: kata-remote-cc diff --git a/webhook/hack/rc.yaml b/webhook/hack/rc.yaml new file mode 100644 index 000000000..f5c3e258b --- /dev/null +++ b/webhook/hack/rc.yaml @@ -0,0 +1,10 @@ +apiVersion: node.k8s.io/v1 +handler: kata-remote-cc +kind: RuntimeClass +metadata: + name: kata-remote-cc +overhead: + podFixed: + memory: "120Mi" + cpu: "250m" + diff --git a/webhook/hack/webhook-deploy.yaml b/webhook/hack/webhook-deploy.yaml new file mode 100644 index 000000000..b3ff37b6a --- /dev/null +++ b/webhook/hack/webhook-deploy.yaml @@ -0,0 +1,332 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: peer-pods-webhook-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: peer-pods-webhook-controller-manager + namespace: peer-pods-webhook-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: peer-pods-webhook-leader-election-role + namespace: peer-pods-webhook-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: peer-pods-webhook-manager-role +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: peer-pods-webhook-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: peer-pods-webhook-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: peer-pods-webhook-leader-election-rolebinding + namespace: peer-pods-webhook-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: peer-pods-webhook-leader-election-role +subjects: +- kind: ServiceAccount + name: peer-pods-webhook-controller-manager + namespace: peer-pods-webhook-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: peer-pods-webhook-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: peer-pods-webhook-manager-role +subjects: +- kind: ServiceAccount + name: peer-pods-webhook-controller-manager + namespace: peer-pods-webhook-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: peer-pods-webhook-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: peer-pods-webhook-proxy-role +subjects: +- kind: ServiceAccount + name: peer-pods-webhook-controller-manager + namespace: peer-pods-webhook-system +--- +apiVersion: v1 +data: + controller_manager_config.yaml: | + apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 + kind: ControllerManagerConfig + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: 127.0.0.1:8080 + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: a3663802.confidential-containers +kind: ConfigMap +metadata: + name: peer-pods-webhook-manager-config + namespace: peer-pods-webhook-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: peer-pods-webhook-controller-manager-metrics-service + namespace: peer-pods-webhook-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + name: peer-pods-webhook-webhook-service + namespace: peer-pods-webhook-system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + control-plane: controller-manager + name: peer-pods-webhook-controller-manager + namespace: peer-pods-webhook-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.11.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + - --leader-elect + command: + - /manager + image: quay.io/confidential-containers/peer-pods-webhook:latest + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + env: + - name: TARGET_RUNTIMECLASS + value: kata-remote-cc + - name: POD_VM_INSTANCE_TYPE + value: t2.small + - name: POD_VM_EXTENDED_RESOURCE + value: kata.peerpods.io/vm + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + securityContext: + runAsNonRoot: true + serviceAccountName: peer-pods-webhook-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: peer-pods-webhook-serving-cert + namespace: peer-pods-webhook-system +spec: + dnsNames: + - peer-pods-webhook-webhook-service.peer-pods-webhook-system.svc + - peer-pods-webhook-webhook-service.peer-pods-webhook-system.svc.cluster.local + issuerRef: + kind: Issuer + name: peer-pods-webhook-selfsigned-issuer + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: peer-pods-webhook-selfsigned-issuer + namespace: peer-pods-webhook-system +spec: + selfSigned: {} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: peer-pods-webhook-system/peer-pods-webhook-serving-cert + name: peer-pods-webhook-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: peer-pods-webhook-webhook-service + namespace: peer-pods-webhook-system + path: /mutate-v1-pod + failurePolicy: Fail + name: mwebhook.peerpods.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: None + namespaceSelector: + matchExpressions: + - key: peerpods + operator: In + values: ["true"] diff --git a/webhook/main.go b/webhook/main.go new file mode 100644 index 000000000..83dce6805 --- /dev/null +++ b/webhook/main.go @@ -0,0 +1,98 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "confidential-containers/peer-pods-webhook/pkg/mutating_webhook" + "flag" + "os" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/webhook" + //+kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + //+kubebuilder:scaffold:scheme +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + Port: 9443, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "a3663802.confidential-containers", + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + setupLog.Info("Setting up webhook server") + mgr.GetWebhookServer().Register("/mutate-v1-pod", &webhook.Admission{Handler: &mutating_webhook.PodMutator{Client: mgr.GetClient()}}) + + //+kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/webhook/pkg/mutating_webhook/mutating-webhook.go b/webhook/pkg/mutating_webhook/mutating-webhook.go new file mode 100644 index 000000000..001dab642 --- /dev/null +++ b/webhook/pkg/mutating_webhook/mutating-webhook.go @@ -0,0 +1,62 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutating_webhook + +import ( + "context" + "encoding/json" + "net/http" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:admissionReviewVersions=v1,path=/mutate-v1-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=mwebhook.peerpods.io,sideEffects=None + +// podMutator mutates Pods +type PodMutator struct { + Client client.Client + decoder *admission.Decoder +} + +// podMutator adds peer-pod extended resource to the pod spec add removes all other resource specs +func (a *PodMutator) Handle(ctx context.Context, req admission.Request) admission.Response { + pod := &corev1.Pod{} + + err := a.decoder.Decode(req, pod) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + mutatedPod, _ := removePodResourceSpec(pod) + marshaledPod, err := json.Marshal(mutatedPod) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) +} + +// podMutator implements admission.DecoderInjector. +// A decoder will be automatically injected. + +// InjectDecoder injects the decoder. +func (a *PodMutator) InjectDecoder(d *admission.Decoder) error { + a.decoder = d + return nil +} diff --git a/webhook/pkg/mutating_webhook/remove-resourcespec.go b/webhook/pkg/mutating_webhook/remove-resourcespec.go new file mode 100644 index 000000000..970db2023 --- /dev/null +++ b/webhook/pkg/mutating_webhook/remove-resourcespec.go @@ -0,0 +1,69 @@ +package mutating_webhook + +import ( + "os" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +const ( + RUNTIME_CLASS_NAME_DEFAULT = "kata-remote-cc" + POD_VM_ANNOTATION_INSTANCE_TYPE = "kata.peerpods.io/instance_type" + POD_VM_INSTANCE_TYPE_DEFAULT = "t2.small" + POD_VM_EXTENDED_RESOURCE_DEFAULT = "kata.peerpods.io/vm" +) + +// remove the POD resource spec +func removePodResourceSpec(pod *corev1.Pod) (*corev1.Pod, error) { + var runtimeClassName string + mpod := pod.DeepCopy() + + if runtimeClassName = os.Getenv("TARGET_RUNTIMECLASS"); runtimeClassName == "" { + runtimeClassName = RUNTIME_CLASS_NAME_DEFAULT + } + // Mutate only if the POD is using specific runtimeClass + if mpod.Spec.RuntimeClassName == nil || *mpod.Spec.RuntimeClassName != runtimeClassName { + return mpod, nil + } + + var podVmInstanceType string + if podVmInstanceType = os.Getenv("POD_VM_INSTANCE_TYPE"); podVmInstanceType == "" { + podVmInstanceType = POD_VM_INSTANCE_TYPE_DEFAULT + } + + if mpod.Annotations == nil { + mpod.Annotations = map[string]string{} + } + + mpod.Annotations[POD_VM_ANNOTATION_INSTANCE_TYPE] = podVmInstanceType + + // Remove all resource specs + for idx := range mpod.Spec.Containers { + mpod.Spec.Containers[idx].Resources = corev1.ResourceRequirements{} + } + + for idx := range mpod.Spec.InitContainers { + mpod.Spec.InitContainers[idx].Resources = corev1.ResourceRequirements{} + } + + // Add peer-pod resource to one container + mpod.Spec.Containers[0].Resources = defaultContainerResourceRequirements() + return mpod, nil +} + +// defaultContainerResourceRequirements returns the default requirements for a container +func defaultContainerResourceRequirements() corev1.ResourceRequirements { + requirements := corev1.ResourceRequirements{} + requirements.Requests = corev1.ResourceList{} + requirements.Limits = corev1.ResourceList{} + + var podVmExtResource string + if podVmExtResource = os.Getenv("POD_VM_EXTENDED_RESOURCE"); podVmExtResource == "" { + podVmExtResource = POD_VM_EXTENDED_RESOURCE_DEFAULT + } + + requirements.Requests[corev1.ResourceName(podVmExtResource)] = resource.MustParse("1") + requirements.Limits[corev1.ResourceName(podVmExtResource)] = resource.MustParse("1") + return requirements +} diff --git a/webhook/pkg/utils/utils.go b/webhook/pkg/utils/utils.go new file mode 100644 index 000000000..48eb2d59d --- /dev/null +++ b/webhook/pkg/utils/utils.go @@ -0,0 +1,119 @@ +package utils + +import ( + "fmt" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// GetResourceRequestQuantity finds and returns the request quantity for a specific resource. +func GetResourceRequestQuantity(pod *corev1.Pod, resourceName corev1.ResourceName) resource.Quantity { + requestQuantity := resource.Quantity{} + + switch resourceName { + case corev1.ResourceCPU: + requestQuantity = resource.Quantity{Format: resource.DecimalSI} + case corev1.ResourceMemory: + requestQuantity = resource.Quantity{Format: resource.BinarySI} + default: + requestQuantity = resource.Quantity{Format: resource.DecimalSI} + } + + for _, container := range pod.Spec.Containers { + if rQuantity, ok := container.Resources.Requests[resourceName]; ok { + requestQuantity.Add(rQuantity) + } + } + + for _, container := range pod.Spec.InitContainers { + if rQuantity, ok := container.Resources.Requests[resourceName]; ok { + if requestQuantity.Cmp(rQuantity) < 0 { + requestQuantity = rQuantity.DeepCopy() + } + } + } + + // Don't add PodOverhead to the total requests + //if pod.Spec.Overhead != nil { + // if podOverhead, ok := pod.Spec.Overhead[resourceName]; ok && !requestQuantity.IsZero() { + // requestQuantity.Add(podOverhead) + // } + //} + + return requestQuantity +} + +// GetResourceRequest finds and returns the request value for a specific resource. +func GetResourceRequest(pod *corev1.Pod, resource corev1.ResourceName) int64 { + + requestQuantity := GetResourceRequestQuantity(pod, resource) + + if resource == corev1.ResourceCPU { + return requestQuantity.MilliValue() + } + + return requestQuantity.Value() +} + +// MergePodResourceRequirements merges enumerated requirements with default requirements +// it annotates the pod with information about what requirements were modified +func MergePodResourceRequirements(pod *corev1.Pod, defaultRequirements *corev1.ResourceRequirements) { + annotations := []string{} + + for i := range pod.Spec.Containers { + annotations = mergeContainerResources(&pod.Spec.Containers[i], defaultRequirements, "container", annotations) + } + + for i := range pod.Spec.InitContainers { + annotations = mergeContainerResources(&pod.Spec.InitContainers[i], defaultRequirements, "init container", annotations) + } + + if len(annotations) > 0 { + if pod.ObjectMeta.Annotations == nil { + pod.ObjectMeta.Annotations = make(map[string]string) + } + val := strings.Join(annotations, "; ") + pod.ObjectMeta.Annotations["kata.peerpods.io/changes"] = val + } +} + +// mergeContainerResources handles defaulting all of the resources on a container. +func mergeContainerResources(container *corev1.Container, defaultRequirements *corev1.ResourceRequirements, annotationPrefix string, annotations []string) []string { + setRequests := []string{} + setLimits := []string{} + if container.Resources.Limits == nil { + container.Resources.Limits = corev1.ResourceList{} + } + if container.Resources.Requests == nil { + container.Resources.Requests = corev1.ResourceList{} + } + + for k, v := range defaultRequirements.Limits { + _, found := container.Resources.Limits[k] + if !found { + container.Resources.Limits[k] = v.DeepCopy() + setLimits = append(setLimits, string(k)) + } + } + for k, v := range defaultRequirements.Requests { + _, found := container.Resources.Requests[k] + if !found { + container.Resources.Requests[k] = v.DeepCopy() + setRequests = append(setRequests, string(k)) + } + } + if len(setRequests) > 0 { + sort.Strings(setRequests) + a := strings.Join(setRequests, ", ") + fmt.Sprintf(" request for %s %s", annotationPrefix, container.Name) + annotations = append(annotations, a) + } + if len(setLimits) > 0 { + sort.Strings(setLimits) + a := strings.Join(setLimits, ", ") + fmt.Sprintf(" limit for %s %s", annotationPrefix, container.Name) + annotations = append(annotations, a) + } + return annotations +} diff --git a/webhook/tests/e2e/run-local.sh b/webhook/tests/e2e/run-local.sh new file mode 100755 index 000000000..485ddd8d9 --- /dev/null +++ b/webhook/tests/e2e/run-local.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# +# Copyright Confidential Containers Contributors +# +# SPDX-License-Identifier: Apache-2.0 +# +# Use this script to run the end-to-end tests locally. +# +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly webhook_dir="$(cd "${script_dir}/../../" && pwd)" + +# Whether to run this script in debug mode or not. +debug=0 +export IMG="peer-pods-webhook:test" + +cleanup () { + if [ $debug -eq 1 ]; then + echo "INFO: running in debug mode. Do not clean up the test environment." + return + fi + + echo "INFO: clean up the test environment" + pushd "$webhook_dir" >/dev/null + make kind-delete || true + docker rmi "$IMG" || true +} + +# Start the cluster and ensure it is ready to use. +# +cluster_up() { + pushd "$webhook_dir" >/dev/null + make kind-cluster + popd >/dev/null + + local cert_manager_ns="cert-manager" + local cert_manager_pods="$(kubectl get pods -n "$cert_manager_ns" 2>/dev/null | \ + grep cert-manager-webhook |awk '{ print $1}')" + + if [ -z "$cert_manager_pods" ]; then + echo "ERROR: failed to get the certification manager webhook pods" + exit 1 + fi + + local pod + for pod in $cert_manager_pods; do + kubectl wait --for=condition=Ready --timeout=60s \ + -n "$cert_manager_ns" "pod/$pod" + done +} + +# Install the webhook and ensure it is ready to use. +# +install_webhook() { + local ns="peer-pods-webhook-system" + + pushd "$webhook_dir" >/dev/null + make deploy + popd >/dev/null + + local webhook_pods="$(kubectl get pods -n "$ns" 2>/dev/null | \ + grep peer-pods-webhook-controller-manager |awk '{ print $1}')" + + if [ -z "$webhook_pods" ];then + echo "ERROR: failed to get the webhook controller manager pods" + exit 1 + fi + + local pod + for pod in $webhook_pods; do + kubectl wait --for=condition=Ready --timeout=60s -n "$ns" \ + "pod/$pod" + done +} + +main() { + parse_args $@ + + for cmd in bats docker kind; do + if ! command -v "$cmd" &>/dev/null; then + echo "ERROR: $cmd command is required for this script" + exit 1 + fi + done + + trap cleanup EXIT + + echo "INFO: start the test cluster" + cluster_up + + pushd "$webhook_dir" >/dev/null + + echo "INFO: build the webhook" + make docker-build + + echo "INFO: load the $IMG image into the cluster" + make kind-load + + echo "INFO: install the webhook in the cluster" + install_webhook + + echo "INFO: run tests" + bats tests/e2e/webhook_tests.bats + + popd >/dev/null +} + +parse_args() { + while getopts "dh" opt; do + case $opt in + d) debug=1;; + h) usage && exit 0;; + *) usage && exit 1;; + esac + done +} + +usage() { + cat <<-EOF + Start a k8s cluster with kind, build and install the webhook then + run end-to-end tests. + + It requires bats, docker and kind to run. + + Use: $0 [-d] [-h], where: + -d: debug mode. It will leave created resources (cluster, image, and etc...) + -h: show this usage + EOF +} + +main "$@" diff --git a/webhook/tests/e2e/webhook_tests.bats b/webhook/tests/e2e/webhook_tests.bats new file mode 100644 index 000000000..8831491ca --- /dev/null +++ b/webhook/tests/e2e/webhook_tests.bats @@ -0,0 +1,99 @@ +#!/usr/bin/env bats +# +# Copyright Confidential Containers Contributors +# +# SPDX-License-Identifier: Apache-2.0 +# +# End-to-end tests. +# +test_tags="[webhook]" + +# Assert that the pod mutated as expected. +# +# Parameters: +# $1: the expected instance type +# $2: the expected VM limits +# $3: the expected VM requests +# +# Global variables: +# $pod_file: path to the pod configuration file. +# +assert_pod_mutated() { + local expect_instance_type="$1" + local expect_vm_limits="$2" + local expect_vm_requests="$3" + + local actual_instance_type=$(kubectl get -f "$pod_file" \ + -o jsonpath='{.metadata.annotations.kata\.peerpods\.io/instance_type}') + echo "Instance type expected: $expect_instance_type, actual: $actual_instance_type" + [ "$expect_instance_type" == "$actual_instance_type" ] + + local actual_vm_limits=$(kubectl get -f "$pod_file" \ + -o jsonpath='{.spec.containers[0].resources.limits.kata\.peerpods\.io/vm}') + echo "VM limits expected: $expect_vm_limits, actual: $actual_vm_limits" + [ $expect_vm_limits -eq $actual_vm_limits ] + + local actual_vm_requests=$(kubectl get -f "$pod_file" \ + -o jsonpath='{.spec.containers[0].resources.requests.kata\.peerpods\.io/vm}') + echo "VM requests expected: $expect_vm_requests, actual: $actual_vm_requests" + [ $expect_vm_requests -eq $actual_vm_requests ] +} + +setup_file() { + export project_dir="$(cd ${BATS_TEST_DIRNAME}/../.. && pwd)" + echo "Create runtimeClass" + kubectl apply -f "$project_dir/hack/rc.yaml" +} + +setup() { + export pod_file="$project_dir/hack/pod.yaml" +} + +teardown() { + kubectl delete -f "$pod_file" || true +} + +@test "$test_tags test it can mutate a pod" { + kubectl apply -f "$pod_file" + assert_pod_mutated "t2.small" 1 1 +} + +@test "$test_tags test it should not mutate non-peerpods" { + echo "Create a pod without runtimeClassName" + cat "$pod_file" | sed -e 's/^\s*runtimeClassName:.*//' | \ + kubectl apply -f - + + ! kubectl get -f ../../hack/pod.yaml -o json | \ + grep kata\.peerpods +} + +@test "$test_tags test default parameters can be changed" { + skip "TODO: This test is not passing" + local runtimeclass="kata-wh-test" + local instance_type='t2.micro' + + # Create a dummy runtimeClass to use on this test. + cat <<-EOF | kubectl apply -f - + apiVersion: node.k8s.io/v1 + handler: ${runtimeclass} + kind: RuntimeClass + metadata: + name: ${runtimeclass} + overhead: + podFixed: + memory: "120Mi" + cpu: "250m" + EOF + + kubectl set env deployment/peer-pods-webhook-controller-manager \ + -n peer-pods-webhook-system TARGET_RUNTIMECLASS="$runtimeclass" + + kubectl set env deployment/peer-pods-webhook-controller-manager \ + -n peer-pods-webhook-system POD_VM_INSTANCE_TYPE="$instance_type" + + cat "$pod_file" | sed -e 's/^\(\s*runtimeClassName:\).*/\1 '${runtimeclass}'/' | \ + kubectl apply -f - + + kubectl get -f $pod_file -o json + assert_pod_mutated "$instance_type" 1 1 +}