diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e3d9fbc9a183..f37f267c7bdd 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,6 +22,7 @@ cmd/configschema/ @open-telemetry/collector-contrib-approvers @mx-psi @dmitryax @pmcollins cmd/mdatagen/ @open-telemetry/collector-contrib-approvers @dmitryax +cmd/opampsupervisor/ @open-telemetry/collector-contrib-approvers @evan-bradley @atoulme @tigrannajaryan cmd/otelcontribcol/ @open-telemetry/collector-contrib-approvers cmd/oteltestbedcol/ @open-telemetry/collector-contrib-approvers cmd/telemetrygen/ @open-telemetry/collector-contrib-approvers @mx-psi @codeboten diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 4107accc67e5..68316bc0df41 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -20,6 +20,7 @@ body: # Start Collector components list - cmd/configschema - cmd/mdatagen + - cmd/opampsupervisor - cmd/otelcontribcol - cmd/oteltestbedcol - cmd/telemetrygen diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 9680ffa19da9..1c9d4773bc55 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -14,6 +14,7 @@ body: # Start Collector components list - cmd/configschema - cmd/mdatagen + - cmd/opampsupervisor - cmd/otelcontribcol - cmd/oteltestbedcol - cmd/telemetrygen diff --git a/.github/ISSUE_TEMPLATE/other.yaml b/.github/ISSUE_TEMPLATE/other.yaml index 1746a9f0c376..3c6069627cc0 100644 --- a/.github/ISSUE_TEMPLATE/other.yaml +++ b/.github/ISSUE_TEMPLATE/other.yaml @@ -14,6 +14,7 @@ body: # Start Collector components list - cmd/configschema - cmd/mdatagen + - cmd/opampsupervisor - cmd/otelcontribcol - cmd/oteltestbedcol - cmd/telemetrygen diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3d76b7033704..472a1cb63107 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,6 +17,11 @@ updates: schedule: interval: "weekly" day: "wednesday" + - package-ecosystem: "gomod" + directory: "/cmd/opampsupervisor" + schedule: + interval: "weekly" + day: "wednesday" - package-ecosystem: "gomod" directory: "/cmd/otelcontribcol" schedule: @@ -1097,8 +1102,3 @@ updates: schedule: interval: "weekly" day: "wednesday" - - package-ecosystem: "gomod" - directory: "/receiver/tcplogreceiver" - schedule: - interval: "weekly" - day: "wednesday" diff --git a/cmd/opampsupervisor/.gitignore b/cmd/opampsupervisor/.gitignore new file mode 100644 index 000000000000..fbaf96e91182 --- /dev/null +++ b/cmd/opampsupervisor/.gitignore @@ -0,0 +1,2 @@ +effective.yaml +agent.log diff --git a/cmd/opampsupervisor/Makefile b/cmd/opampsupervisor/Makefile new file mode 100644 index 000000000000..ded7a36092dc --- /dev/null +++ b/cmd/opampsupervisor/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/cmd/opampsupervisor/README.md b/cmd/opampsupervisor/README.md new file mode 100644 index 000000000000..3601a0732bcd --- /dev/null +++ b/cmd/opampsupervisor/README.md @@ -0,0 +1,24 @@ +# OpAMP Supervisor for the OpenTelemetry Collector + +This is an implementation of an OpAMP Supervisor that runs a Collector instance using configuration provided from an OpAMP server. This implementation +is following a design specified [here](./specification/README.md). +The design is still undergoing changes, and as such this implementation may change as well. + +## Experimenting with the supervisor + +The supervisor is currently undergoing heavy development and is not ready for any serious use. However, if you would like to test it, you can follow the steps below: + +1. Download the [opamp-go](https://github.com/open-telemetry/opamp-go) repository, and run the OpAMP example server in the `internal/examples/server` directory. +2. From the Collector contrib repository root, build the Collector: + + ```shell + make otelcontribcol + ``` + +3. Run the supervisor, substituting `` for your platform: + + ```shell + go run . --config testdata/supervisor_.yaml + ``` + +4. The supervisor should connect to the OpAMP server and start a Collector instance. diff --git a/cmd/opampsupervisor/go.mod b/cmd/opampsupervisor/go.mod new file mode 100644 index 000000000000..f904fe846433 --- /dev/null +++ b/cmd/opampsupervisor/go.mod @@ -0,0 +1,27 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/cmd/opampsupervisor + +go 1.19 + +require ( + github.com/cenkalti/backoff/v4 v4.2.0 + github.com/knadh/koanf v1.5.0 + github.com/oklog/ulid/v2 v2.1.0 + github.com/open-telemetry/opamp-go v0.6.0 + go.uber.org/zap v1.24.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.8.2 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/sys v0.5.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/cmd/opampsupervisor/go.sum b/cmd/opampsupervisor/go.sum new file mode 100644 index 000000000000..dd758b3fcf0c --- /dev/null +++ b/cmd/opampsupervisor/go.sum @@ -0,0 +1,414 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= +github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= +github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= +github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= +github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs= +github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf v1.5.0 h1:q2TSd/3Pyc/5yP9ldIrSdIz26MCcyNQzW0pEAugLPNs= +github.com/knadh/koanf v1.5.0/go.mod h1:Hgyjp4y8v44hpZtPzs7JZfRAW5AhN7KfZcwv1RYggDs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= +github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/open-telemetry/opamp-go v0.6.0 h1:pTnVvIp9FT3qEQ8P+evENhvaQGkF8i6vu73gawBPRLQ= +github.com/open-telemetry/opamp-go v0.6.0/go.mod h1:IMdeuHGVc5CjKSu5/oNV0o+UmiXuahoHvoZ4GOmAI9M= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/cmd/opampsupervisor/main.go b/cmd/opampsupervisor/main.go new file mode 100644 index 000000000000..b93579842c09 --- /dev/null +++ b/cmd/opampsupervisor/main.go @@ -0,0 +1,33 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "flag" + "os" + "os/signal" + + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/opampsupervisor/supervisor" +) + +func main() { + configFlag := flag.String("config", "", "Path to a supervisor configuration file") + flag.Parse() + + logger, _ := zap.NewDevelopment() + + supervisor, err := supervisor.NewSupervisor(logger, *configFlag) + if err != nil { + logger.Error(err.Error()) + os.Exit(-1) + return + } + + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + <-interrupt + supervisor.Shutdown() +} diff --git a/cmd/opampsupervisor/specification/README.md b/cmd/opampsupervisor/specification/README.md new file mode 100644 index 000000000000..5ca8f9f62889 --- /dev/null +++ b/cmd/opampsupervisor/specification/README.md @@ -0,0 +1,469 @@ +# OpAMP for OpenTelemetry Collector + +Author: Tigran Najaryan + +## Introduction + +[OpAMP](https://github.com/open-telemetry/opamp-spec) has been one of +the most requested capabilities at Kubecon 2022. We can implement OpAMP +for the Collector in 2 different ways: + +- As a Collector extension, with limited functionality, +- As an external Supervisor, that implements all or most of OpAMP + capabilities. + +In discussions with users and Collector contributors we found that both +of these approaches are wanted. This document describes how to implement +both while minimizing duplicate work. + +The main idea is to implement a Collector extension with a limited set +of OpAMP capabilities, where the extension can be used on its own, then +additionally create an external Supervisor that uses the exact same +extension as a helper and implements the remaining OpAMP capabilities on +top of what the extension implements. This way most of OpAMP-related +functionality is implemented once only, there is virtually no code +duplication. + +## Supervised Operation + +This section describes a specialized Supervisor made for OpenTelemetry +Collector. The Supervisor will have the benefit of knowing for example +how to inject desirable values into the Collector's configuration file. + +Here is how a Supervisor-based management works: + +![Supervisor architecture diagram](supervisor-diagram.png) + +The Supervisor process does the following: + +- Implements the client-side of OpAMP protocol and communicates with + the OpAMP Backend. +- Starts/stops the Collector process as necessary. +- Receives configuration from the OpAMP Backend and pushes it to the + Collector, using the Collector config.yaml file as an intermediary, + restarting the Collector process as necessary. +- Serves as a watchdog, restarts the Collector process if the + Collector crashes. +- Accepts an OpAMP connection from Collectors' [*opamp + extension*](#collectors-opamp-extension), receives the Collector's + AgentDescription, HealthStatus and EffectiveConfig messages and + forwards them to the OpAMP Backend. +- Optionally: downloads Collector executable packages offered by the + Backend and performs the Collector updates. +- Optionally: configures Collector to collect Collector's own metrics + and report the metrics to the OTLP telemetry backend requested by + OpAMP Backend. +- Optionally: collects Collector logs and sends them to the Telemetry + Backend via OTLP. + +Supervisor is implemented as a Go library that may be customized and +rebuilt by vendors with useful default configurations, such as the OpAMP +Backend endpoint to connect to, in order to minimize the manual +configuration required. + +*Important: the Supervisor needs to be highly stable, so we need to keep +its complexity and functionality to minimum. The features listed in this +section need a critical review and may be removed (responsibility moved +elsewhere, e.g. to the Collector itself).* + +### Supervisor Configuration + +The Supervisor is configured via a yaml config file: + +```yaml +# OpAMP backend server settings. +server: + # endpoint is a URL and is mandatory. + # ws,wss,http,https schemes are supported. + # Other connection settings, e.g. TLS cert, etc. + endpoint: wss://example.com/opamp + +# Keys with boolean true/false values that enable a particular +# OpAMP capability. +# The Supervisor will accept remote configuration from the Server. +# If enabled the Supervisor will also report RemoteConfig status +# to the Server. +capabilities: + AcceptsRemoteConfig: # false if unspecified + + # The Supervisor will report EffectiveConfig to the Server. + ReportsEffectiveConfig: # true if unspecified + + # The Supervisor can accept Collector executable package updates. + # If enabled the Supervisor will also report package status to the + # Server. + AcceptsPackages: # false if unspecified + + # The Collector will report own metrics to the destination specified by + # the Server. + ReportsOwnMetrics: # true if unspecified + + # The Collector will report own logs to the destination specified by + # the Server. + ReportsOwnLogs: # true if unspecified + + # The Collector will accept connections settings for exporters + # from the Server. + AcceptsOtherConnectionSettings: # false if unspecified + + # The Supervisor will accept restart requests. + AcceptsRestartCommand: # true if unspecified + + # The Collector will report Health. + ReportsHealth: # true if unspecified + +storage: + # A writable directory where the Supervisor can store data + # (e.g. cached remote config). + # defaults to /var/lib/otelcol/supervisor on posix systems + # and %ProgramData%/Otelcol/Supervisor on Windows. + directory: /path/to/dir + +collector: + # Path to Collector executable. Required. + executable: /opt/otelcol/bin/otelcol + + # Extra command line flags to pass to the Collector executable. + args: + + # Extra environment variables to set when executing the Collector. + env: + + # Optional user name to drop the privileges to when running the + # Collector process. + run_as: myuser + # Path to optional local Collector config file to be merged with the + # config provided by the OpAMP server. + config_file: /etc/otelcol/config.yaml + # Optional directories that are allowed to be read/written by the + # Collector. + # If unspecified then NO access to the filesystem is allowed. + access_dirs: + read: + allow: \[/var/log\] + deny: \[/var/log/secret_logs\] + write: + allow: \[/var/otelcol\] +``` + +### Executing Collector + +The Supervisor starts and stops the Collector process as necessary. When +run_as setting is provided, the Supervisor will execute the Collector +process as the specified user. This is highly recommended in situations +when the Supervisor itself is running as root and it is desirable to +drop the root privileges and run the Collector as a more restricted +user. + +#### Stopping Collector + +To stop the Collector the Supervisor will issue a SIGTERM signal first, +wait for the process to exit, then force exit via SIGKILL if necessary. + +### Collector Config + +The Supervisor creates a Collector config file in a temp directory and +passes to the Collector via --config command line option. The config +file is created by merging the local config file (optional) and the +remote config received from OpAMP backend (also optional). The +configuration file merging rules match the merging rules that are +already in place in the Collector codebase (key-by-key, recursively). + +#### Remote Configuration + +Note: this capability must be manually enabled by the user via a +AcceptsRemoteConfig setting in the supervisor config file and is +disabled by default. + +The Supervisor receives [*Remote +Configuration*](https://github.com/open-telemetry/opamp-spec/blob/main/specification.md#configuration) +from the OpAMP Backend, merges it with an optional local config file and +writes it to the Collector's config file, then restarts the Collector. + +In the future once config file watching is implemented the Collector can +reload the config without the need for the Supervisor to restart the +Collector process. + +The Supervisor will report to the OpAMP Backend the status of all these +operations via RemoteConfigStatus message. + +#### Sanitizing Configuration + +The Supervisor will sanitize the configuration of the components that +access the local filesystem according to the access_dirs config setting +to only allow specified directories and their subdirectories. This +applies for example to \`include\` setting of the \`filelog\` receiver +or to \`directory\` setting of the \`file_storage\` extension. + +The Supervisor will locate all such entries while building the Collector +config file and will delete the ones which are prohibited by the access +control settings. + +*Open Question: if after sanitizing the component's directory setting +the configuration becomes invalid what do we do?* + +*The sanitizing logic is hard-coded in the Supervisor and works for +specific components only. In the future we will consider implementing a +more generic safety mechanism that does not depend on the knowledge +about specific component behavior.* + +#### Bootstrapping + +In order to obtain the remote configuration from the OpAMP Backend the +Supervisor must send an AgentDescription to the Backend. Initially the +Supervisor doesn't have this information. The AgentDescription becomes +available only after the Collector process is started and the +AgentDescription is sent from the opamp extension to the Supervisor. +However, it is impossible to start the Collector without a +configuration. + +To overcome this problem the Supervisor starts the Collector with an +"noop" configuration that collects nothing but allows the opamp +extension to be started. The "noop" configuration is a single pipeline +with a filelog receiver that points to a non-existing file and a logging +exporter and the opamp extension. The purpose of the "noop" +configuration is to make sure the Collector starts and the opamp +extension communicates with the Supervisor. + +Once the initial Collector launch is successful and the remote +configuration is received by the Supervisor the Supervisor restarts the +Collector with the new config. The new config is also cached by the +Supervisor in a local file, so that subsequent restarts no longer need +to start the Collector using the "noop" configuration. Caching of the +last config also allows the Supervisor to subsequently start the +Collector without waiting for the OpAMP Backend to provide the remote +config and mitigates OpAMP Backend unavailability. + +#### Reverting + +One of the challenges when the Supervisor updates the configuration is +knowing whether the configuration is successfully applied or not. There +may not exist a definite signal that indicates that all of the +configuration is good and applied by the Collector. One possible +approach is to set a certain period after which the Collector is +expected to be up and running in "healthy" status and if that does not +happen (i.e. the Collector crashes or "healthy" status is not seen) then +the configuration is reverted to the last one. + +The reverting likely needs to be an optional feature that the user can +enable. TODO: add a Supervisor config setting for this option. + +### Watchdog + +The Supervisor will monitor the Collector process it started. If the +Collector process terminates unexpectedly the Supervisor will restart +the Collector (with a backoff). The Supervisor can also report restart +or crash-related metrics along with other telemetry it reports for the +Collector. + +### Collector Instance UID + +The Supervisor maintains a Collector instance_uid (a +[ULID](https://github.com/open-telemetry/opamp-spec/blob/main/specification.md#agenttoserverinstance_uid)). +The instance_uid is generated by the Supervisor on the first run or +during the Supervisor installation and remains unchanged thereafter. The +instance_uid will be used in OpAMP communication. + +The value is injected into the Collector config in [opamp +extension's](#collectors-opamp-extension) instance_uid setting and as +service.instance.id attribute under the service.telemetry.resource +setting. + +Note: if [Make generated service.instance.id available to +extensions](https://github.com/open-telemetry/opentelemetry-collector/issues/6599) +is implemented then the instance_uid setting in the opamp extension is +no longer necessary. + +### Collector's Own Telemetry + +#### Own Metrics + +Supervisor will configure Collector to report its own metrics to the +OTLP destination requested by the OpAMP Backend. See how it is done in +[this example +here](https://github.com/open-telemetry/opamp-go/blob/efddaa260895e7ebb2431deeefef3a3380d8d902/internal/examples/supervisor/supervisor/supervisor.go#L277). + +Eventually, when the Collector's internal metrics are migrated to Otel +SDK we should be able to configure the SDK to send to this destination +without configuring a Collector pipeline. + +#### Own Logs + +The Supervisor will collect Collector's logs (the stdout and stderr) and +send them to the log collection endpoint offered by the OpAMP Backend. + +The Supervisor will configure the Collector output logs in json format +(e.g. [like +this](https://github.com/open-telemetry/opamp-go/blob/efddaa260895e7ebb2431deeefef3a3380d8d902/internal/examples/supervisor/supervisor/supervisor.go#LL220C21-L220C21)) +and will parse them, then send to the OTLP destination requested by the +OpAMP Backend. The Supervisor will enrich the logs using the attributes +it received previously in AgentDescription message from the [opamp +extension](#collectors-opamp-extension). + +Note that for logs we are not using the same approach as we do for +Collector's own metrics. The reason is that collecting logs may produce +logs and that may result in catastrophic amplification of generated +logs. To safeguard from that we collect logs using the Supervisor. + +The additional benefit of using Supervisor for log collection is that if +the Collector crashes the Supervisor will still be able to collect all +log output up until the process termination, which can be crucial for +understanding the cause of crash (using Collector to collect its logs +won't guarantee this). + +The Supervisor will also write the Collector's log to a local log file. +The path to the Collector log files will be printed in the Supervisor +output. + +*Open Question: instead of writing to a local log file do we want to +pipe Collector logs to Supervisor's log output?* + +### Collector Executable Updates + +Note: this capability must be manually enabled by the user via the +AcceptsPackages setting in the supervisor config file and is disabled by +default. + +This capability should be only implemented after we introduce the code +signing process for the Collector and all released Collector executables +are signed, the Supervisor is capable of verifying the signature of +executables and other OpAMP security recommendations are followed. + +The Supervisor will download Collector package updates when offered so +by the Backend. The Supervisor will verify the integrity of the packages +and will install them. This requires stopping the Collector, overwriting +the Collector executable file with the newly downloaded version and +starting the Collector. Before overwriting the executable the Supervisor +will save it in case it is necessary for reverting. + +If after the restart the Collector does not become healthy the +Supervisor will revert the update, by stopping the Collector, reverting +the Collector executable file and starting the Collector again. The +failed update attempt will be reported to the Backend and the failed +Collector package version will be marked as "bad" to avoid trying it +again even if offered by the Backend. + +Note: cached local config must be invalidated after executable updates +to make sure a fresh AgentDescription is obtained by the Supervisor on +the next Collector start (at the minimum the version number to be +included in AgentDescription is expected to change after the executable +is updated). + +### Addons Management + +The Collector currently does not have a concept of addons so this OpAMP +capability is not implemented by the Supervisor. + +### Exporter Connection Settings + +The Supervisor will populate the connection settings for exporters in +the Collector configuration file based on the "other_connections" named +settings it receives from the OpAMP Backend. Each named connection +setting corresponds to the exporter with the same name. The Supervisor +will populate exporter settings from OpAMP ConnectionSettings message +the following way: + +| **ConnectionSettings** | **Exporter setting** | +|---------------------------|----------------------| +| destination_endpoint | endpoint | +| headers | headers | +| certificate.public_key | tls.cert_file | +| certificate.private_key | tls.key_file | +| certificate.ca_public_key | tls.ca_file | + +The received certificate will be written to local files and the paths to +the files containing the keys will be populated in the corresponding +sections under the \`tls\` setting. + +## Collector's opamp Extension + +The Supervisor automatically injects the opamp extension in the +Collector's configuration. The opamp extension implements an OpAMP +client with a small subset of OpAMP agent capabilities: + +- ReportsStatus. The extension reports agent description and status. + This is the first message from the client to the server in the OpAMP + protocol that is essential for beginning OpAMP message exchange. +- ReportsEffectiveConfig. The extension reports the Collector's + effective config on startup and any time the config changes. In + order to do this the opamp extension needs [access to the effective + config](https://github.com/open-telemetry/opentelemetry-collector/issues/6596). +- ReportsHealth. The extension reports Collector's health on startup + and any time the health changes. In order to do this the opamp + extension needs access to the health of the Collector. The very + basic health capability can be replicated by mirroring the + functionality of the healthcheck extension, a more advanced + capability depends on the [component status + reporting](https://github.com/open-telemetry/opentelemetry-collector/pull/6560). + +The messages received from the opamp extension are forwarded by the +Supervisor to the destination OpAMP Backend and replies to these +messages that the Supervisor receives from the OpAMP Backend are +forwarded in the opposite direction to the opamp extension. The +Supervisor also peeks into AgentDescription and keeps it for its own use +as necessary (e.g. to use as [log attributes](#own-logs)). + +The opamp extension in the Collector will accept the following +configuration: + +```yaml +extensions: + opamp: + # OpAMP server URL. Supports WS or plain http transport, + # based on the scheme of the URL (ws,wss,http,https). + # Any other settings defined in HTTPClientSettings, squashed. This + # includes ability to specify an "auth" setting that refers + # to an extension that implements the Authentication interface. + endpoint: + + # ULID formatted as a 26 character string in canonical + # representation. Auto-generated on start if missing. + # Injected by Supervisor. + # Note: can be deprecated after Collector issue #6599 + # is implemented. + instance_uid: +``` + +The extension uses an OpAMP connection to the Supervisor when used with +the Supervisor model. + +The extensions' configuration cannot be overridden by the remote +configuration. + +The same extension can be used to connect directly to the OpAMP Server, +without the Supervisor: + +![OpAMP extension architecture diagram](extension-diagram.png) + +This is the simplified operation model where only status reporting is +required. Only the 3 capabilities that the extension implements work. No +other OpAMP capabilities are available in this operation model. + +When the opamp extension is used without the Supervisor it will need to +know what service.instance.id the Collector's telemetry is using. There +is an [open +issue](https://github.com/open-telemetry/opentelemetry-collector/issues/6599) +to allow this. + +*Open Question: when used with Supervisor do we want the Supervisor to +actively periodically query the health of the Collector or we can rely +on opamp extension to report the health when it changes?* + +## Future Work + +- Decide if we want to have Supervisor-less AcceptsRemoteConfig + capability in the Collector. This currently can't be done by using + just an extension. At the minimum it requires a config Provider. +- Consider extending the Supervisor to be able to manage multiple + Collector instances. + +## References + +- OpAMP Specification: + [https://github.com/open-telemetry/opamp-spec/blob/main/specification.md](https://github.com/open-telemetry/opamp-spec/blob/main/specification.md) +- OpAMP client and server implementation in Go: + [https://github.com/open-telemetry/opamp-go](https://github.com/open-telemetry/opamp-go) +- Example Supervisor implementation: + [https://github.com/open-telemetry/opamp-go/tree/main/internal/examples/supervisor](https://github.com/open-telemetry/opamp-go/tree/main/internal/examples/supervisor) +- OpAMP Milestone in the Collector: + [https://github.com/open-telemetry/opentelemetry-collector/milestone/29](https://github.com/open-telemetry/opentelemetry-collector/milestone/29) diff --git a/cmd/opampsupervisor/specification/extension-diagram.png b/cmd/opampsupervisor/specification/extension-diagram.png new file mode 100644 index 000000000000..9f754812a8a6 Binary files /dev/null and b/cmd/opampsupervisor/specification/extension-diagram.png differ diff --git a/cmd/opampsupervisor/specification/supervisor-diagram.png b/cmd/opampsupervisor/specification/supervisor-diagram.png new file mode 100644 index 000000000000..cb10dd2eb0d8 Binary files /dev/null and b/cmd/opampsupervisor/specification/supervisor-diagram.png differ diff --git a/cmd/opampsupervisor/supervisor/commander/commander.go b/cmd/opampsupervisor/supervisor/commander/commander.go new file mode 100644 index 000000000000..23ef147343b6 --- /dev/null +++ b/cmd/opampsupervisor/supervisor/commander/commander.go @@ -0,0 +1,179 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package commander + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "sync/atomic" + "syscall" + "time" + + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/opampsupervisor/supervisor/config" +) + +// Commander can start/stop/restart the Agent executable and also watch for a signal +// for the Agent process to finish. +type Commander struct { + logger *zap.Logger + cfg *config.Agent + args []string + cmd *exec.Cmd + doneCh chan struct{} + running *atomic.Int64 +} + +func NewCommander(logger *zap.Logger, cfg *config.Agent, args ...string) (*Commander, error) { + if cfg.Executable == "" { + return nil, errors.New("agent.executable config option must be specified") + } + + return &Commander{ + logger: logger, + cfg: cfg, + args: args, + running: &atomic.Int64{}, + }, nil +} + +// Start the Agent and begin watching the process. +// Agent's stdout and stderr are written to a file. +// Calling this method when a command is already running +// is a no-op. +func (c *Commander) Start(ctx context.Context) error { + if c.running.Load() == 1 { + // Already started, nothing to do + return nil + } + + c.logger.Debug("Starting agent", zap.String("agent", c.cfg.Executable)) + + logFilePath := "agent.log" + logFile, err := os.Create(logFilePath) + if err != nil { + return fmt.Errorf("cannot create %s: %w", logFilePath, err) + } + + c.cmd = exec.CommandContext(ctx, c.cfg.Executable, c.args...) // #nosec G204 + + // Capture standard output and standard error. + // https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21072 + c.cmd.Stdout = logFile + c.cmd.Stderr = logFile + + c.doneCh = make(chan struct{}) + + if err := c.cmd.Start(); err != nil { + return err + } + + c.logger.Debug("Agent process started", zap.Int("pid", c.cmd.Process.Pid)) + c.running.Store(1) + + go c.watch() + + return nil +} + +func (c *Commander) Restart(ctx context.Context) error { + if err := c.Stop(ctx); err != nil { + return err + } + + return c.Start(ctx) +} + +func (c *Commander) watch() { + err := c.cmd.Wait() + + // cmd.Wait returns an exec.ExitError when the Collector exits unsuccessfully or stops + // after receiving a signal. The Commander caller will handle these cases, so we filter + // them out here. + var exitError *exec.ExitError + if ok := errors.As(err, &exitError); err != nil && !ok { + c.logger.Error("An error occurred while watching the agent process", zap.Error(err)) + } + + c.running.Store(0) + close(c.doneCh) +} + +// Done returns a channel that will send a signal when the Agent process is finished. +func (c *Commander) Done() <-chan struct{} { + return c.doneCh +} + +// Pid returns Agent process PID if it is started or 0 if it is not. +func (c *Commander) Pid() int { + if c.cmd == nil || c.cmd.Process == nil { + return 0 + } + return c.cmd.Process.Pid +} + +// ExitCode returns Agent process exit code if it exited or 0 if it is not. +func (c *Commander) ExitCode() int { + if c.cmd == nil || c.cmd.ProcessState == nil { + return 0 + } + return c.cmd.ProcessState.ExitCode() +} + +func (c *Commander) IsRunning() bool { + return c.running.Load() != 0 +} + +// Stop the Agent process. Sends SIGTERM to the process and wait for up 10 seconds +// and if the process does not finish kills it forcedly by sending SIGKILL. +// Returns after the process is terminated. +func (c *Commander) Stop(ctx context.Context) error { + if c.running.Load() == 0 { + // Not started, nothing to do. + return nil + } + + c.logger.Debug("Stopping agent process", zap.Int("pid", c.cmd.Process.Pid)) + + // Gracefully signal process to stop. + if err := c.cmd.Process.Signal(syscall.SIGTERM); err != nil { + return err + } + + waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + + // Setup a goroutine to wait a while for process to finish and send kill signal + // to the process if it doesn't finish. + var innerErr error + go func() { + <-waitCtx.Done() + + if !errors.Is(waitCtx.Err(), context.DeadlineExceeded) { + c.logger.Debug("Agent process successfully stopped.", zap.Int("pid", c.cmd.Process.Pid)) + return + } + + // Time is out. Kill the process. + c.logger.Debug( + "Agent process is not responding to SIGTERM. Sending SIGKILL to kill forcedly.", + zap.Int("pid", c.cmd.Process.Pid)) + if innerErr = c.cmd.Process.Signal(syscall.SIGKILL); innerErr != nil { + return + } + }() + + // Wait for process to terminate + <-c.doneCh + + c.running.Store(0) + + // Let goroutine know process is finished. + cancel() + + return innerErr +} diff --git a/cmd/opampsupervisor/supervisor/config/config.go b/cmd/opampsupervisor/supervisor/config/config.go new file mode 100644 index 000000000000..31edfce10f2e --- /dev/null +++ b/cmd/opampsupervisor/supervisor/config/config.go @@ -0,0 +1,18 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package config + +// Supervisor is the Supervisor config file format. +type Supervisor struct { + Server *OpAMPServer + Agent *Agent +} + +type OpAMPServer struct { + Endpoint string +} + +type Agent struct { + Executable string +} diff --git a/cmd/opampsupervisor/supervisor/healthchecker/healthchecker.go b/cmd/opampsupervisor/supervisor/healthchecker/healthchecker.go new file mode 100644 index 000000000000..e941284daef3 --- /dev/null +++ b/cmd/opampsupervisor/supervisor/healthchecker/healthchecker.go @@ -0,0 +1,45 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package healthchecker + +import ( + "context" + "fmt" + "net/http" + "time" +) + +// TODO: Support more settings +// https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/23344 +type HTTPHealthChecker struct { + endpoint string +} + +func NewHTTPHealthChecker(endpoint string) *HTTPHealthChecker { + return &HTTPHealthChecker{ + endpoint: endpoint, + } +} + +func (h *HTTPHealthChecker) Check(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, "GET", h.endpoint, nil) + if err != nil { + return err + } + + client := http.Client{ + Timeout: time.Second * 10, + } + + resp, err := client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("health check on %s returned %d", h.endpoint, resp.StatusCode) + } + + return nil +} diff --git a/cmd/opampsupervisor/supervisor/supervisor.go b/cmd/opampsupervisor/supervisor/supervisor.go new file mode 100644 index 000000000000..4287e2c88c23 --- /dev/null +++ b/cmd/opampsupervisor/supervisor/supervisor.go @@ -0,0 +1,745 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package supervisor + +import ( + "context" + "errors" + "fmt" + "math/rand" + "net" + "os" + "runtime" + "sort" + "sync/atomic" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/knadh/koanf" + "github.com/knadh/koanf/parsers/yaml" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/rawbytes" + "github.com/oklog/ulid/v2" + "github.com/open-telemetry/opamp-go/client" + "github.com/open-telemetry/opamp-go/client/types" + "github.com/open-telemetry/opamp-go/protobufs" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/opampsupervisor/supervisor/commander" + "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/opampsupervisor/supervisor/config" + "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/opampsupervisor/supervisor/healthchecker" +) + +// This Supervisor is developed specifically for the OpenTelemetry Collector. +const agentType = "io.opentelemetry.collector" + +// Supervisor implements supervising of OpenTelemetry Collector and uses OpAMPClient +// to work with an OpAMP Server. +type Supervisor struct { + logger *zap.Logger + + // Commander that starts/stops the Agent process. + commander *commander.Commander + + startedAt time.Time + + healthCheckTicker *backoff.Ticker + healthChecker *healthchecker.HTTPHealthChecker + lastHealthCheckErr error + + // Supervisor's own config. + config config.Supervisor + + // Agent's instance id. + instanceID ulid.ULID + + // The version of the agent. + agentVersion string + + // A config section to be added to the Collector's config to fetch its own metrics. + // TODO: store this persistently so that when starting we can compose the effective + // config correctly. + // https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21078 + agentConfigOwnMetricsSection *atomic.Value + + // agentHealthCheckEndpoint is the endpoint the Collector's health check extension + // will listen on for health check requests from the Supervisor. + agentHealthCheckEndpoint string + + // Final effective config of the Collector. + effectiveConfig *atomic.Value + + // Location of the effective config file. + effectiveConfigFilePath string + + // Last received remote config. + remoteConfig *protobufs.AgentRemoteConfig + + // A channel to indicate there is a new config to apply. + hasNewConfig chan struct{} + + // The OpAMP client to connect to the OpAMP Server. + opampClient client.OpAMPClient + + shuttingDown bool + + agentHasStarted bool + agentStartHealthCheckAttempts int +} + +func NewSupervisor(logger *zap.Logger, configFile string) (*Supervisor, error) { + s := &Supervisor{ + logger: logger, + hasNewConfig: make(chan struct{}, 1), + effectiveConfigFilePath: "effective.yaml", + agentConfigOwnMetricsSection: &atomic.Value{}, + effectiveConfig: &atomic.Value{}, + } + + if err := s.loadConfig(configFile); err != nil { + return nil, fmt.Errorf("error loading config: %w", err) + } + + if err := s.getBootstrapInfo(); err != nil { + s.logger.Error("Couldn't get agent version", zap.Error(err)) + } + + port, err := s.findRandomPort() + + if err != nil { + return nil, fmt.Errorf("could not find port for health check: %w", err) + } + + s.agentHealthCheckEndpoint = fmt.Sprintf("localhost:%d", port) + + id, err := s.createInstanceID() + + if err != nil { + return nil, err + } + + s.instanceID = id + + logger.Debug("Supervisor starting", + zap.String("id", s.instanceID.String()), zap.String("type", agentType), zap.String("version", s.agentVersion)) + + s.loadAgentEffectiveConfig() + + if err = s.startOpAMP(); err != nil { + return nil, fmt.Errorf("cannot start OpAMP client: %w", err) + } + + s.commander, err = commander.NewCommander( + s.logger, + s.config.Agent, + "--config", s.effectiveConfigFilePath, + ) + if err != nil { + return nil, err + } + + s.startHealthCheckTicker() + go s.runAgentProcess() + + return s, nil +} + +func (s *Supervisor) loadConfig(configFile string) error { + if configFile == "" { + return errors.New("path to config file cannot be empty") + } + + k := koanf.New("::") + if err := k.Load(file.Provider(configFile), yaml.Parser()); err != nil { + return err + } + + if err := k.Unmarshal("", &s.config); err != nil { + return fmt.Errorf("cannot parse %v: %w", configFile, err) + } + + return nil +} + +// TODO: Implement bootstrapping https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21071 +// nolint: unparam +func (s *Supervisor) getBootstrapInfo() (err error) { + s.agentVersion = "1.0.0" + + return nil +} + +func (s *Supervisor) startOpAMP() error { + s.opampClient = client.NewWebSocket(s.logger.Sugar()) + + settings := types.StartSettings{ + OpAMPServerURL: s.config.Server.Endpoint, + InstanceUid: s.instanceID.String(), + Callbacks: types.CallbacksStruct{ + OnConnectFunc: func() { + s.logger.Debug("Connected to the server.") + }, + OnConnectFailedFunc: func(err error) { + s.logger.Error("Failed to connect to the server", zap.Error(err)) + }, + OnErrorFunc: func(err *protobufs.ServerErrorResponse) { + s.logger.Error("Server returned an error response", zap.String("message", err.ErrorMessage)) + }, + OnMessageFunc: s.onMessage, + OnOpampConnectionSettingsFunc: func(ctx context.Context, settings *protobufs.OpAMPConnectionSettings) error { + // TODO: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21043 + s.logger.Debug("Received ConnectionSettings request") + return nil + }, + OnOpampConnectionSettingsAcceptedFunc: func(settings *protobufs.OpAMPConnectionSettings) { + // TODO: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21043 + s.logger.Debug("ConnectionSettings accepted") + }, + OnCommandFunc: func(command *protobufs.ServerToAgentCommand) error { + cmdType := command.GetType() + if *cmdType.Enum() == protobufs.CommandType_CommandType_Restart { + // TODO: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21077 + s.logger.Debug("Received restart command") + } + return nil + }, + SaveRemoteConfigStatusFunc: func(ctx context.Context, status *protobufs.RemoteConfigStatus) { + // TODO: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21079 + }, + GetEffectiveConfigFunc: func(ctx context.Context) (*protobufs.EffectiveConfig, error) { + return s.createEffectiveConfigMsg(), nil + }, + }, + // TODO: Make capabilities configurable + Capabilities: protobufs.AgentCapabilities_AgentCapabilities_AcceptsRemoteConfig | + protobufs.AgentCapabilities_AgentCapabilities_ReportsRemoteConfig | + protobufs.AgentCapabilities_AgentCapabilities_ReportsEffectiveConfig | + protobufs.AgentCapabilities_AgentCapabilities_ReportsOwnMetrics | + protobufs.AgentCapabilities_AgentCapabilities_ReportsHealth, + } + err := s.opampClient.SetAgentDescription(s.createAgentDescription()) + if err != nil { + return err + } + + err = s.opampClient.SetHealth(&protobufs.AgentHealth{Healthy: false}) + if err != nil { + return err + } + + s.logger.Debug("Starting OpAMP client...") + + err = s.opampClient.Start(context.Background(), settings) + if err != nil { + return err + } + + s.logger.Debug("OpAMP Client started.") + + return nil +} + +// TODO: Persist instance ID. https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21073 +func (s *Supervisor) createInstanceID() (ulid.ULID, error) { + entropy := ulid.Monotonic(rand.New(rand.NewSource(0)), 0) + id, err := ulid.New(ulid.Timestamp(time.Now()), entropy) + + if err != nil { + return ulid.ULID{}, err + } + + return id, nil + +} + +func keyVal(key, val string) *protobufs.KeyValue { + return &protobufs.KeyValue{ + Key: key, + Value: &protobufs.AnyValue{ + Value: &protobufs.AnyValue_StringValue{StringValue: val}, + }, + } +} + +func (s *Supervisor) createAgentDescription() *protobufs.AgentDescription { + hostname, _ := os.Hostname() + + return &protobufs.AgentDescription{ + IdentifyingAttributes: []*protobufs.KeyValue{ + keyVal("service.name", agentType), + keyVal("service.version", s.agentVersion), + keyVal("service.instance.id", s.instanceID.String()), + }, + NonIdentifyingAttributes: []*protobufs.KeyValue{ + keyVal("os.family", runtime.GOOS), + keyVal("host.name", hostname), + }, + } +} + +func (s *Supervisor) composeExtraLocalConfig() string { + return fmt.Sprintf(` +service: + telemetry: + logs: + # Enables JSON log output for the Agent. + encoding: json + resource: + # Set resource attributes required by OpAMP spec. + # See https://github.com/open-telemetry/opamp-spec/blob/main/specification.md#agentdescriptionidentifying_attributes + service.name: %s + service.version: %s + service.instance.id: %s + + # Enable extension to allow the Supervisor to check health. + extensions: [health_check] + +extensions: + health_check: + endpoint: %s +`, + agentType, + s.agentVersion, + s.instanceID.String(), + s.agentHealthCheckEndpoint, + ) +} + +func (s *Supervisor) loadAgentEffectiveConfig() { + var effectiveConfigBytes []byte + + effFromFile, err := os.ReadFile(s.effectiveConfigFilePath) + if err == nil { + // We have an effective config file. + effectiveConfigBytes = effFromFile + } else { + // No effective config file, just use the initial config. + effectiveConfigBytes = []byte(s.composeExtraLocalConfig()) + } + + s.effectiveConfig.Store(string(effectiveConfigBytes)) +} + +// createEffectiveConfigMsg create an EffectiveConfig with the content of the +// current effective config. +func (s *Supervisor) createEffectiveConfigMsg() *protobufs.EffectiveConfig { + cfgStr, ok := s.effectiveConfig.Load().(string) + if !ok { + cfgStr = "" + } + + cfg := &protobufs.EffectiveConfig{ + ConfigMap: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "": {Body: []byte(cfgStr)}, + }, + }, + } + + return cfg +} + +func (s *Supervisor) setupOwnMetrics(_ context.Context, settings *protobufs.TelemetryConnectionSettings) (configChanged bool) { + var cfg string + if settings.DestinationEndpoint == "" { + // No destination. Disable metric collection. + s.logger.Debug("Disabling own metrics pipeline in the config") + cfg = "" + } else { + s.logger.Debug("Enabling own metrics pipeline in the config") + + port, err := s.findRandomPort() + + if err != nil { + s.logger.Error("Could not setup own metrics", zap.Error(err)) + return + } + + cfg = fmt.Sprintf( + ` +receivers: + # Collect own metrics + prometheus/own_metrics: + config: + scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 10s + static_configs: + - targets: ['0.0.0.0:%d'] +exporters: + otlphttp/own_metrics: + metrics_endpoint: %s + +service: + telemetry: + metrics: + address: :%d + pipelines: + metrics/own_metrics: + receivers: [prometheus/own_metrics] + exporters: [otlphttp/own_metrics] +`, + port, + settings.DestinationEndpoint, + port, + ) + } + + s.agentConfigOwnMetricsSection.Store(cfg) + + // Need to recalculate the Agent config so that the metric config is included in it. + configChanged, err := s.recalcEffectiveConfig() + if err != nil { + return + } + + return configChanged +} + +// composeEffectiveConfig composes the effective config from multiple sources: +// 1) the remote config from OpAMP Server +// 2) the own metrics config section +// 3) the local override config that is hard-coded in the Supervisor. +func (s *Supervisor) composeEffectiveConfig(config *protobufs.AgentRemoteConfig) (configChanged bool, err error) { + var k = koanf.New(".") + + // Begin with empty config. We will merge received configs on top of it. + if err = k.Load(rawbytes.Provider([]byte{}), yaml.Parser()); err != nil { + return false, err + } + + // Sort to make sure the order of merging is stable. + var names []string + for name := range config.Config.ConfigMap { + if name == "" { + // skip instance config + continue + } + names = append(names, name) + } + + sort.Strings(names) + + // Append instance config as the last item. + names = append(names, "") + + // Merge received configs. + for _, name := range names { + item := config.Config.ConfigMap[name] + var k2 = koanf.New(".") + err = k2.Load(rawbytes.Provider(item.Body), yaml.Parser()) + if err != nil { + return false, fmt.Errorf("cannot parse config named %s: %w", name, err) + } + err = k.Merge(k2) + if err != nil { + return false, fmt.Errorf("cannot merge config named %s: %w", name, err) + } + } + + // Merge own metrics config. + ownMetricsCfg, ok := s.agentConfigOwnMetricsSection.Load().(string) + if ok { + if err = k.Load(rawbytes.Provider([]byte(ownMetricsCfg)), yaml.Parser()); err != nil { + return false, err + } + } + + // Merge local config last since it has the highest precedence. + if err = k.Load(rawbytes.Provider([]byte(s.composeExtraLocalConfig())), yaml.Parser()); err != nil { + return false, err + } + + // The merged final result is our effective config. + effectiveConfigBytes, err := k.Marshal(yaml.Parser()) + if err != nil { + return false, err + } + + // Check if effective config is changed. + newEffectiveConfig := string(effectiveConfigBytes) + configChanged = false + if s.effectiveConfig.Load().(string) != newEffectiveConfig { + s.logger.Debug("Effective config changed.") + s.effectiveConfig.Store(newEffectiveConfig) + configChanged = true + } + + return configChanged, nil +} + +// Recalculate the Agent's effective config and if the config changes, signal to the +// background goroutine that the config needs to be applied to the Agent. +func (s *Supervisor) recalcEffectiveConfig() (configChanged bool, err error) { + configChanged, err = s.composeEffectiveConfig(s.remoteConfig) + if err != nil { + s.logger.Error("Error composing effective config. Ignoring received config", zap.Error(err)) + return configChanged, err + } + + return configChanged, nil +} + +func (s *Supervisor) startAgent() { + err := s.commander.Start(context.Background()) + if err != nil { + s.logger.Error("Cannot start the agent", zap.Error(err)) + err = s.opampClient.SetHealth(&protobufs.AgentHealth{Healthy: false, LastError: fmt.Sprintf("Cannot start the agent: %v", err)}) + + if err != nil { + s.logger.Error("Failed to report OpAMP client health", zap.Error(err)) + } + + return + } + + s.agentHasStarted = false + s.agentStartHealthCheckAttempts = 0 + s.startedAt = time.Now() + s.startHealthCheckTicker() + + s.healthChecker = healthchecker.NewHTTPHealthChecker(fmt.Sprintf("http://%s", s.agentHealthCheckEndpoint)) +} + +func (s *Supervisor) startHealthCheckTicker() { + // Prepare health checker + healthCheckBackoff := backoff.NewExponentialBackOff() + healthCheckBackoff.MaxInterval = 60 * time.Second + healthCheckBackoff.MaxElapsedTime = 0 // Never stop + if s.healthCheckTicker != nil { + s.healthCheckTicker.Stop() + } + s.healthCheckTicker = backoff.NewTicker(healthCheckBackoff) +} + +func (s *Supervisor) healthCheck() { + if !s.commander.IsRunning() { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + + err := s.healthChecker.Check(ctx) + cancel() + + if errors.Is(err, s.lastHealthCheckErr) { + // No difference from last check. Nothing new to report. + return + } + + // Prepare OpAMP health report. + health := &protobufs.AgentHealth{ + StartTimeUnixNano: uint64(s.startedAt.UnixNano()), + } + + if err != nil { + health.Healthy = false + if !s.agentHasStarted && s.agentStartHealthCheckAttempts < 10 { + health.LastError = "Agent is starting" + s.agentStartHealthCheckAttempts++ + } else { + health.LastError = err.Error() + s.logger.Error("Agent is not healthy", zap.Error(err)) + } + } else { + s.agentHasStarted = true + health.Healthy = true + s.logger.Debug("Agent is healthy.") + } + + // Report via OpAMP. + if err2 := s.opampClient.SetHealth(health); err2 != nil { + s.logger.Error("Could not report health to OpAMP server", zap.Error(err2)) + return + } + + s.lastHealthCheckErr = err +} + +func (s *Supervisor) runAgentProcess() { + if _, err := os.Stat(s.effectiveConfigFilePath); err == nil { + // We have an effective config file saved previously. Use it to start the agent. + s.startAgent() + } + + restartTimer := time.NewTimer(0) + restartTimer.Stop() + + for { + select { + case <-s.hasNewConfig: + restartTimer.Stop() + s.stopAgentApplyConfig() + s.startAgent() + + case <-s.commander.Done(): + if s.shuttingDown { + break + } + + s.logger.Debug("Agent process exited unexpectedly. Will restart in a bit...", zap.Int("pid", s.commander.Pid()), zap.Int("exit_code", s.commander.ExitCode())) + errMsg := fmt.Sprintf( + "Agent process PID=%d exited unexpectedly, exit code=%d. Will restart in a bit...", + s.commander.Pid(), s.commander.ExitCode(), + ) + err := s.opampClient.SetHealth(&protobufs.AgentHealth{Healthy: false, LastError: errMsg}) + + if err != nil { + s.logger.Error("Could not report health to OpAMP server", zap.Error(err)) + } + + // TODO: decide why the agent stopped. If it was due to bad config, report it to server. + // https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21079 + + // Wait 5 seconds before starting again. + restartTimer.Stop() + restartTimer.Reset(5 * time.Second) + + case <-restartTimer.C: + s.startAgent() + + case <-s.healthCheckTicker.C: + s.healthCheck() + } + } +} + +func (s *Supervisor) stopAgentApplyConfig() { + s.logger.Debug("Stopping the agent to apply new config") + cfg := s.effectiveConfig.Load().(string) + err := s.commander.Stop(context.Background()) + + if err != nil { + s.logger.Error("Could not stop agent process", zap.Error(err)) + } + + s.writeEffectiveConfigToFile(cfg, s.effectiveConfigFilePath) +} + +func (s *Supervisor) writeEffectiveConfigToFile(cfg string, filePath string) { + f, err := os.Create(filePath) + if err != nil { + s.logger.Error("Cannot create effective config file", zap.Error(err)) + } + defer f.Close() + + _, err = f.WriteString(cfg) + + if err != nil { + s.logger.Error("Cannot write effective config file", zap.Error(err)) + } +} + +func (s *Supervisor) Shutdown() { + s.logger.Debug("Supervisor shutting down...") + s.shuttingDown = true + if s.commander != nil { + err := s.commander.Stop(context.Background()) + + if err != nil { + s.logger.Error("Could not stop agent process", zap.Error(err)) + } + } + + if s.opampClient != nil { + err := s.opampClient.SetHealth( + &protobufs.AgentHealth{ + Healthy: false, LastError: "Supervisor is shutdown", + }, + ) + + if err != nil { + s.logger.Error("Could not report health to OpAMP server", zap.Error(err)) + } + + err = s.opampClient.Stop(context.Background()) + + if err != nil { + s.logger.Error("Could not stop the OpAMP client", zap.Error(err)) + } + } +} + +func (s *Supervisor) onMessage(ctx context.Context, msg *types.MessageData) { + configChanged := false + if msg.RemoteConfig != nil { + s.remoteConfig = msg.RemoteConfig + s.logger.Debug("Received remote config from server", zap.String("hash", fmt.Sprintf("%x", s.remoteConfig.ConfigHash))) + + var err error + configChanged, err = s.recalcEffectiveConfig() + if err != nil { + err = s.opampClient.SetRemoteConfigStatus(&protobufs.RemoteConfigStatus{ + LastRemoteConfigHash: msg.RemoteConfig.ConfigHash, + Status: protobufs.RemoteConfigStatuses_RemoteConfigStatuses_FAILED, + ErrorMessage: err.Error(), + }) + if err != nil { + s.logger.Error("Could not report failed OpAMP remote config status", zap.Error(err)) + } + } else { + err = s.opampClient.SetRemoteConfigStatus(&protobufs.RemoteConfigStatus{ + LastRemoteConfigHash: msg.RemoteConfig.ConfigHash, + Status: protobufs.RemoteConfigStatuses_RemoteConfigStatuses_APPLIED, + }) + if err != nil { + s.logger.Error("Could not report applied OpAMP remote config status", zap.Error(err)) + } + } + } + + if msg.OwnMetricsConnSettings != nil { + configChanged = s.setupOwnMetrics(ctx, msg.OwnMetricsConnSettings) || configChanged + } + + if msg.AgentIdentification != nil { + newInstanceID, err := ulid.Parse(msg.AgentIdentification.NewInstanceUid) + if err != nil { + s.logger.Error("Failed to parse instance ULID", zap.Error(err)) + } + + s.logger.Debug("Agent identity is changing", + zap.String("old_id", s.instanceID.String()), + zap.String("new_id", newInstanceID.String())) + s.instanceID = newInstanceID + err = s.opampClient.SetAgentDescription(s.createAgentDescription()) + if err != nil { + s.logger.Error("Failed to send agent description to OpAMP server") + } + + configChanged = true + } + + if configChanged { + err := s.opampClient.UpdateEffectiveConfig(ctx) + if err != nil { + s.logger.Error("The OpAMP client failed to update the effective config", zap.Error(err)) + } + + s.logger.Debug("Config is changed. Signal to restart the agent") + // Signal that there is a new config. + select { + case s.hasNewConfig <- struct{}{}: + default: + } + } +} + +func (s *Supervisor) findRandomPort() (int, error) { + l, err := net.Listen("tcp", "localhost:0") + + if err != nil { + return 0, err + } + + port := l.Addr().(*net.TCPAddr).Port + + err = l.Close() + + if err != nil { + return 0, err + } + + return port, nil +} diff --git a/cmd/opampsupervisor/testdata/supervisor_darwin.yaml b/cmd/opampsupervisor/testdata/supervisor_darwin.yaml new file mode 100644 index 000000000000..9b278fedbcd8 --- /dev/null +++ b/cmd/opampsupervisor/testdata/supervisor_darwin.yaml @@ -0,0 +1,5 @@ +server: + endpoint: ws://127.0.0.1:4320/v1/opamp + +agent: + executable: ../../bin/otelcontribcol_darwin_amd64 diff --git a/cmd/opampsupervisor/testdata/supervisor_linux.yaml b/cmd/opampsupervisor/testdata/supervisor_linux.yaml new file mode 100644 index 000000000000..9cb491ff2436 --- /dev/null +++ b/cmd/opampsupervisor/testdata/supervisor_linux.yaml @@ -0,0 +1,5 @@ +server: + endpoint: ws://127.0.0.1:4320/v1/opamp + +agent: + executable: ../../bin/otelcontribcol_linux_amd64 diff --git a/cmd/opampsupervisor/testdata/supervisor_windows.yaml b/cmd/opampsupervisor/testdata/supervisor_windows.yaml new file mode 100644 index 000000000000..0aefc0c2b727 --- /dev/null +++ b/cmd/opampsupervisor/testdata/supervisor_windows.yaml @@ -0,0 +1,5 @@ +server: + endpoint: ws://127.0.0.1:4320/v1/opamp + +agent: + executable: ../../bin/otelcontribcol_windows_amd64.exe diff --git a/versions.yaml b/versions.yaml index 1b3f9a824697..3e10a94d55db 100644 --- a/versions.yaml +++ b/versions.yaml @@ -8,6 +8,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib - github.com/open-telemetry/opentelemetry-collector-contrib/cmd/configschema - github.com/open-telemetry/opentelemetry-collector-contrib/cmd/mdatagen + - github.com/open-telemetry/opentelemetry-collector-contrib/cmd/opampsupervisor - github.com/open-telemetry/opentelemetry-collector-contrib/cmd/telemetrygen - github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provider/s3provider - github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector