From d08e910e4c6648680d1059ff362264dc0ce4d5b7 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 7 Sep 2022 18:57:06 -0500 Subject: [PATCH] External Locking DB: Redis (#2491) * Initial build out of redis backend * Regenerate locking mocks and matchers * More test cleanup * More test fixes * Added tests for redis * Test fix * Linting fix * Dcos update * Update redis.go * Update server.go * Adding nolint to RedisDB struct --- .gitignore | 1 + CONTRIBUTING.md | 2 + cmd/server.go | 26 + cmd/server_test.go | 1 + docker-compose.yml | 16 +- go.mod | 15 +- go.sum | 29 + runatlantis.io/docs/server-configuration.md | 28 + .../events/events_controller_e2e_test.go | 7 +- server/controllers/jobs_controller.go | 4 +- server/controllers/locks_controller.go | 5 +- server/controllers/locks_controller_test.go | 27 +- server/core/locking/locking.go | 4 + .../locking/mocks/matchers/command_name.go | 33 + .../matchers/locking_applycommandlock.go | 3 +- .../mocks/matchers/locking_trylockresponse.go | 3 +- .../map_of_string_to_models_projectlock.go | 3 +- .../locking/mocks/matchers/models_project.go | 3 +- .../mocks/matchers/models_projectlock.go | 3 +- .../matchers/models_projectplanstatus.go | 33 + .../mocks/matchers/models_pullrequest.go | 3 +- .../mocks/matchers/models_pullstatus.go | 33 + .../locking/mocks/matchers/models_user.go | 3 +- .../mocks/matchers/ptr_to_command_lock.go | 33 + .../matchers/ptr_to_models_projectlock.go | 3 +- .../matchers/ptr_to_models_pullstatus.go | 33 + .../slice_of_command_projectresult.go | 33 + .../matchers/slice_of_models_projectlock.go | 3 +- .../core/locking/mocks/matchers/time_time.go | 3 +- .../locking/mocks/mock_apply_lock_checker.go | 5 +- .../core/locking/mocks/mock_apply_locker.go | 5 +- server/core/locking/mocks/mock_backend.go | 199 ++++- server/core/locking/mocks/mock_locker.go | 5 +- server/core/redis/redis.go | 391 +++++++++ server/core/redis/redis_test.go | 760 ++++++++++++++++++ server/events/apply_command_runner.go | 7 +- server/events/command_runner_test.go | 22 +- server/events/db_updater.go | 6 +- server/events/delete_lock_command.go | 5 +- server/events/delete_lock_command_test.go | 2 +- server/events/pull_closed_executor.go | 8 +- server/events/pull_closed_executor_test.go | 10 +- server/server.go | 44 +- server/user_config.go | 4 + 44 files changed, 1764 insertions(+), 102 deletions(-) create mode 100644 server/core/locking/mocks/matchers/command_name.go create mode 100644 server/core/locking/mocks/matchers/models_projectplanstatus.go create mode 100644 server/core/locking/mocks/matchers/models_pullstatus.go create mode 100644 server/core/locking/mocks/matchers/ptr_to_command_lock.go create mode 100644 server/core/locking/mocks/matchers/ptr_to_models_pullstatus.go create mode 100644 server/core/locking/mocks/matchers/slice_of_command_projectresult.go create mode 100644 server/core/redis/redis.go create mode 100644 server/core/redis/redis_test.go diff --git a/.gitignore b/.gitignore index b47a8d08a8..5fe3071760 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ helm/test-values.yaml golangci-lint atlantis .devcontainer +atlantis.env # gitreleaser dist/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6874d8faab..864293f65c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -130,6 +130,8 @@ We use [pegomock](https://github.com/petergtz/pegomock) for mocking. If you're modifying any interfaces that are mocked, you'll need to regen the mocks for that interface. +Install using `go get github.com/petergtz/pegomock/pegomock` + If you see errors like: ``` # github.com/runatlantis/atlantis/server/events [github.com/runatlantis/atlantis/server/events.test] diff --git a/cmd/server.go b/cmd/server.go index ac855774ed..4ffbc53a3f 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -80,11 +80,15 @@ const ( GitlabWebhookSecretFlag = "gitlab-webhook-secret" // nolint: gosec APISecretFlag = "api-secret" HidePrevPlanComments = "hide-prev-plan-comments" + LockingDBType = "locking-db-type" LogLevelFlag = "log-level" ParallelPoolSize = "parallel-pool-size" StatsNamespace = "stats-namespace" AllowDraftPRs = "allow-draft-prs" PortFlag = "port" + RedisHost = "redis-host" + RedisPassword = "redis-password" + RedisPort = "redis-port" RepoConfigFlag = "repo-config" RepoConfigJSONFlag = "repo-config-json" // RepoWhitelistFlag is deprecated for RepoAllowlistFlag. @@ -123,10 +127,12 @@ const ( DefaultDataDir = "~/.atlantis" DefaultGHHostname = "github.com" DefaultGitlabHostname = "gitlab.com" + DefaultLockingDBType = "boltdb" DefaultLogLevel = "info" DefaultParallelPoolSize = 15 DefaultStatsNamespace = "atlantis" DefaultPort = 4141 + DefaultRedisPort = 6379 DefaultTFDownloadURL = "https://releases.hashicorp.com" DefaultTFEHostname = "app.terraform.io" DefaultVCSStatusName = "atlantis" @@ -264,6 +270,10 @@ var stringFlags = map[string]stringFlag{ APISecretFlag: { description: "Secret to validate requests made to the API", }, + LockingDBType: { + description: "The locking database type to use for storing plan and apply locks.", + defaultValue: DefaultLockingDBType, + }, LogLevelFlag: { description: "Log level. Either debug, info, warn, or error.", defaultValue: DefaultLogLevel, @@ -272,6 +282,12 @@ var stringFlags = map[string]stringFlag{ description: "Namespace for aggregating stats.", defaultValue: DefaultStatsNamespace, }, + RedisHost: { + description: "The Redis Hostname for when using a Locking DB type of 'redis'.", + }, + RedisPassword: { + description: "The Redis Password for when using a Locking DB type of 'redis'.", + }, RepoConfigFlag: { description: "Path to a repo config file, used to customize how Atlantis runs on each repo. See runatlantis.io/docs for more details.", }, @@ -450,6 +466,10 @@ var intFlags = map[string]intFlag{ description: "Port to bind to.", defaultValue: DefaultPort, }, + RedisPort: { + description: "The Redis Port for when using a Locking DB type of 'redis'.", + defaultValue: DefaultRedisPort, + }, } var int64Flags = map[string]int64Flag{ @@ -672,6 +692,9 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig) { if c.BitbucketBaseURL == "" { c.BitbucketBaseURL = DefaultBitbucketBaseURL } + if c.LockingDBType == "" { + c.LockingDBType = DefaultLockingDBType + } if c.LogLevel == "" { c.LogLevel = DefaultLogLevel } @@ -684,6 +707,9 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig) { if c.Port == 0 { c.Port = DefaultPort } + if c.RedisPort == 0 { + c.RedisPort = DefaultRedisPort + } if c.TFDownloadURL == "" { c.TFDownloadURL = DefaultTFDownloadURL } diff --git a/cmd/server_test.go b/cmd/server_test.go index e57febf175..c8110dfa48 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -84,6 +84,7 @@ var testFlags = map[string]interface{}{ GitlabTokenFlag: "gitlab-token", GitlabUserFlag: "gitlab-user", GitlabWebhookSecretFlag: "gitlab-secret", + LockingDBType: "boltdb", LogLevelFlag: "debug", StatsNamespace: "atlantis", AllowDraftPRs: true, diff --git a/docker-compose.yml b/docker-compose.yml index a1eafa657a..2ea622fa4c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,17 @@ services: NGROK_PORT: atlantis:4141 depends_on: - atlantis + redis: + image: redis:6.2-alpine + restart: always + ports: + - '6379:6379' + command: redis-server --save 20 1 --loglevel warning --requirepass test123 + volumes: + - redis:/data atlantis: + depends_on: + - redis build: context: . dockerfile: Dockerfile.dev @@ -23,4 +33,8 @@ services: - ./:/atlantis/src # Contains the flags that atlantis uses in env var form env_file: - - ./atlantis.env \ No newline at end of file + - ./atlantis.env + +volumes: + redis: + driver: local \ No newline at end of file diff --git a/go.mod b/go.mod index 574078dd09..f77b8e9dec 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/elazarl/go-bindata-assetfs v1.0.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/go-ozzo/ozzo-validation v0.0.0-20170913164239-85dcd8368eba + github.com/go-redis/redis/v8 v8.11.5 github.com/go-test/deep v1.0.8 github.com/golang-jwt/jwt/v4 v4.4.2 github.com/google/go-github/v31 v31.0.0 @@ -92,7 +93,7 @@ require ( github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect - github.com/onsi/gomega v1.10.1 // indirect + github.com/onsi/gomega v1.20.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect @@ -135,14 +136,24 @@ require ( require ( cloud.google.com/go/compute v1.6.1 // indirect cloud.google.com/go/iam v0.3.0 // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect + github.com/alicebob/miniredis/v2 v2.23.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/go-redis/redis/v9 v9.0.0-beta.2 // indirect github.com/google/go-github/v45 v45.2.0 // indirect github.com/m3db/prometheus_client_golang v0.8.1 // indirect github.com/m3db/prometheus_client_model v0.1.0 // indirect github.com/m3db/prometheus_common v0.1.0 // indirect github.com/m3db/prometheus_procfs v0.8.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/onsi/ginkgo v1.14.0 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 // indirect + golang.org/x/tools v0.1.5 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gotest.tools/v3 v3.3.0 // indirect ) diff --git a/go.sum b/go.sum index 3325a74447..04749e0323 100644 --- a/go.sum +++ b/go.sum @@ -74,10 +74,17 @@ github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.23.0 h1:+lwAJYjvvdIVg6doFHuotFjueJ/7KY10xo/vm3X3Scw= +github.com/alicebob/miniredis/v2 v2.23.0/go.mod h1:XNqvJdQJv5mSuVMc0ynneafpnL/zv52acZ6kqeS0t88= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0= @@ -113,6 +120,8 @@ github.com/cactus/go-statsd-client/statsd v0.0.0-20200623234511-94959e3146b2 h1: github.com/cactus/go-statsd-client/statsd v0.0.0-20200623234511-94959e3146b2/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -129,6 +138,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= @@ -162,8 +173,13 @@ github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotf github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-redis/redis/v9 v9.0.0-beta.2 h1:ZSr84TsnQyKMAg8gnV+oawuQezeJR11/09THcWCQzr4= +github.com/go-redis/redis/v9 v9.0.0-beta.2/go.mod h1:Bldcd/M/bm9HbnNPi/LUtYBSD8ttcZYBMupwMXhdU0o= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= @@ -439,13 +455,19 @@ github.com/nlopes/slack v0.4.0 h1:OVnHm7lv5gGT5gkcHsZAyw++oHVFihbjWbL3UceUpiA= github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= +github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -556,6 +578,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 h1:k/gmLsJDWwWqbLCur2yWnJzwQEKRcAHXo6seXGuSwWw= +github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= github.com/zclconf/go-cty v1.5.1 h1:oALUZX+aJeEBUe2a1+uD2+UTaYfEjnKFDEMRydkGvWE= @@ -723,6 +747,7 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -769,6 +794,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -869,6 +895,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -877,6 +904,7 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1029,6 +1057,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/runatlantis.io/docs/server-configuration.md b/runatlantis.io/docs/server-configuration.md index d3a92a5071..6e8ed3b70e 100644 --- a/runatlantis.io/docs/server-configuration.md +++ b/runatlantis.io/docs/server-configuration.md @@ -422,6 +422,16 @@ Values are chosen in this order: Hide previous plan comments to declutter PRs. This is only supported in GitHub currently. +### `--locking-db-type` + ```bash + atlantis server --locking-db-type="" + ``` + The locking database type to use for storing plan and apply locks. Defaults to `boltdb`. + + Notes: + * If set to `boltdb`, only one process may have access to the boltdb instance. + * If set to `redis`, then `--redis-host`, `--redis-port`, and `--redis-password` must be set. + ### `--log-level` ```bash atlantis server --log-level="" @@ -440,6 +450,24 @@ Values are chosen in this order: ``` Port to bind to. Defaults to `4141`. +### `--redis-host` + ```bash + atlantis server --redis-host="localhost" + ``` + The Redis Hostname for when using a Locking DB type of `redis`. + +### `--redis-password` + ```bash + atlantis server --redis-password="password123" + ``` + The Redis Password for when using a Locking DB type of `redis`. + +### `--redis-port` + ```bash + atlantis server --redis-port=6379 + ``` + The Redis Port for when using a Locking DB type of `redis`. Defaults to `6379`. + ### `--repo-config` ```bash atlantis server --repo-config="path/to/repos.yaml" diff --git a/server/controllers/events/events_controller_e2e_test.go b/server/controllers/events/events_controller_e2e_test.go index cf8b2ceb50..7ccab8c9a2 100644 --- a/server/controllers/events/events_controller_e2e_test.go +++ b/server/controllers/events/events_controller_e2e_test.go @@ -857,6 +857,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl Ok(t, err) boltdb, err := db.New(dataDir) Ok(t, err) + backend := boltdb lockingClient := locking.NewClient(boltdb) applyLocker = locking.NewApplyClient(boltdb, userConfig.DisableApply) projectLocker := &events.DefaultProjectLocker{ @@ -986,7 +987,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl } dbUpdater := &events.DBUpdater{ - DB: boltdb, + Backend: backend, } pullUpdater := &events.PullUpdater{ @@ -1093,7 +1094,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl Drainer: drainer, PreWorkflowHooksCommandRunner: preWorkflowHooksCommandRunner, PostWorkflowHooksCommandRunner: postWorkflowHooksCommandRunner, - PullStatusFetcher: boltdb, + PullStatusFetcher: backend, } repoAllowlistChecker, err := events.NewRepoAllowlistChecker("*") @@ -1106,7 +1107,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl Locker: lockingClient, VCSClient: e2eVCSClient, WorkingDir: workingDir, - DB: boltdb, + Backend: backend, PullClosedTemplate: &events.PullClosedEventTemplate{}, LogStreamResourceCleaner: projectCmdOutputHandler, }, diff --git a/server/controllers/jobs_controller.go b/server/controllers/jobs_controller.go index a41e44446e..fe7023e5b2 100644 --- a/server/controllers/jobs_controller.go +++ b/server/controllers/jobs_controller.go @@ -8,7 +8,7 @@ import ( "github.com/gorilla/mux" "github.com/runatlantis/atlantis/server/controllers/templates" "github.com/runatlantis/atlantis/server/controllers/websocket" - "github.com/runatlantis/atlantis/server/core/db" + "github.com/runatlantis/atlantis/server/core/locking" "github.com/runatlantis/atlantis/server/logging" "github.com/runatlantis/atlantis/server/metrics" "github.com/uber-go/tally" @@ -31,7 +31,7 @@ type JobsController struct { Logger logging.SimpleLogging ProjectJobsTemplate templates.TemplateWriter ProjectJobsErrorTemplate templates.TemplateWriter - Db *db.BoltDB + Backend locking.Backend WsMux *websocket.Multiplexor KeyGenerator JobIDKeyGenerator StatsScope tally.Scope diff --git a/server/controllers/locks_controller.go b/server/controllers/locks_controller.go index 7d147f9f7a..d32b10dd2f 100644 --- a/server/controllers/locks_controller.go +++ b/server/controllers/locks_controller.go @@ -6,7 +6,6 @@ import ( "net/url" "github.com/runatlantis/atlantis/server/controllers/templates" - "github.com/runatlantis/atlantis/server/core/db" "github.com/gorilla/mux" "github.com/runatlantis/atlantis/server/core/locking" @@ -27,7 +26,7 @@ type LocksController struct { LockDetailTemplate templates.TemplateWriter WorkingDir events.WorkingDir WorkingDirLocker events.WorkingDirLocker - DB *db.BoltDB + Backend locking.Backend DeleteLockCommand events.DeleteLockCommand } @@ -137,7 +136,7 @@ func (l *LocksController) DeleteLock(w http.ResponseWriter, r *http.Request) { l.Logger.Err("unable to delete workspace: %s", err) } } - if err := l.DB.UpdateProjectStatus(lock.Pull, lock.Workspace, lock.Project.Path, models.DiscardedPlanStatus); err != nil { + if err := l.Backend.UpdateProjectStatus(lock.Pull, lock.Workspace, lock.Project.Path, models.DiscardedPlanStatus); err != nil { l.Logger.Err("unable to update project status: %s", err) } diff --git a/server/controllers/locks_controller_test.go b/server/controllers/locks_controller_test.go index 01dddf6dd5..b3d34e2373 100644 --- a/server/controllers/locks_controller_test.go +++ b/server/controllers/locks_controller_test.go @@ -298,12 +298,13 @@ func TestDeleteLock_UpdateProjectStatus(t *testing.T) { RepoFullName: repoName, }, }, nil) + var backend locking.Backend tmp, cleanup := TempDir(t) defer cleanup() - db, err := db.New(tmp) + backend, err := db.New(tmp) Ok(t, err) // Seed the DB with a successful plan for that project (that is later discarded). - _, err = db.UpdatePullWithResults(pull, []command.ProjectResult{ + _, err = backend.UpdatePullWithResults(pull, []command.ProjectResult{ { Command: command.Plan, RepoRelDir: projectPath, @@ -321,14 +322,14 @@ func TestDeleteLock_UpdateProjectStatus(t *testing.T) { VCSClient: cp, WorkingDirLocker: workingDirLocker, WorkingDir: workingDir, - DB: db, + Backend: backend, } req, _ := http.NewRequest("GET", "", bytes.NewBuffer(nil)) req = mux.SetURLVars(req, map[string]string{"id": "id"}) w := httptest.NewRecorder() lc.DeleteLock(w, req) ResponseContains(t, w, http.StatusOK, "Deleted lock id \"id\"") - status, err := db.GetPullStatus(pull) + status, err := backend.GetPullStatus(pull) Ok(t, err) Assert(t, status.Projects != nil, "status projects was nil") Equals(t, []models.ProjectStatus{ @@ -352,18 +353,19 @@ func TestDeleteLock_CommentFailed(t *testing.T) { cp := vcsmocks.NewMockClient() workingDir := mocks2.NewMockWorkingDir() workingDirLocker := events.NewDefaultWorkingDirLocker() - When(cp.CreateComment(AnyRepo(), AnyInt(), AnyString(), AnyString())).ThenReturn(errors.New("err")) + var backend locking.Backend tmp, cleanup := TempDir(t) defer cleanup() - db, err := db.New(tmp) + backend, err := db.New(tmp) Ok(t, err) + When(cp.CreateComment(AnyRepo(), AnyInt(), AnyString(), AnyString())).ThenReturn(errors.New("err")) lc := controllers.LocksController{ DeleteLockCommand: dlc, Logger: logging.NewNoopLogger(t), VCSClient: cp, WorkingDir: workingDir, WorkingDirLocker: workingDirLocker, - DB: db, + Backend: backend, } req, _ := http.NewRequest("GET", "", bytes.NewBuffer(nil)) req = mux.SetURLVars(req, map[string]string{"id": "id"}) @@ -379,6 +381,11 @@ func TestDeleteLock_CommentSuccess(t *testing.T) { dlc := mocks2.NewMockDeleteLockCommand() workingDir := mocks2.NewMockWorkingDir() workingDirLocker := events.NewDefaultWorkingDirLocker() + var backend locking.Backend + tmp, cleanup := TempDir(t) + defer cleanup() + backend, err := db.New(tmp) + Ok(t, err) pull := models.PullRequest{ BaseRepo: models.Repo{FullName: "owner/repo"}, } @@ -390,15 +397,11 @@ func TestDeleteLock_CommentSuccess(t *testing.T) { RepoFullName: "owner/repo", }, }, nil) - tmp, cleanup := TempDir(t) - defer cleanup() - db, err := db.New(tmp) - Ok(t, err) lc := controllers.LocksController{ DeleteLockCommand: dlc, Logger: logging.NewNoopLogger(t), VCSClient: cp, - DB: db, + Backend: backend, WorkingDir: workingDir, WorkingDirLocker: workingDirLocker, } diff --git a/server/core/locking/locking.go b/server/core/locking/locking.go index 2abb0a4dc6..f001b2b2ae 100644 --- a/server/core/locking/locking.go +++ b/server/core/locking/locking.go @@ -33,6 +33,10 @@ type Backend interface { List() ([]models.ProjectLock, error) GetLock(project models.Project, workspace string) (*models.ProjectLock, error) UnlockByPull(repoFullName string, pullNum int) ([]models.ProjectLock, error) + UpdateProjectStatus(pull models.PullRequest, workspace string, repoRelDir string, newStatus models.ProjectPlanStatus) error + GetPullStatus(pull models.PullRequest) (*models.PullStatus, error) + DeletePullStatus(pull models.PullRequest) error + UpdatePullWithResults(pull models.PullRequest, newResults []command.ProjectResult) (models.PullStatus, error) LockCommand(cmdName command.Name, lockTime time.Time) (*command.Lock, error) UnlockCommand(cmdName command.Name) error diff --git a/server/core/locking/mocks/matchers/command_name.go b/server/core/locking/mocks/matchers/command_name.go new file mode 100644 index 0000000000..9dcc26d1e8 --- /dev/null +++ b/server/core/locking/mocks/matchers/command_name.go @@ -0,0 +1,33 @@ +// Code generated by pegomock. DO NOT EDIT. +package matchers + +import ( + "github.com/petergtz/pegomock" + "reflect" + + command "github.com/runatlantis/atlantis/server/events/command" +) + +func AnyCommandName() command.Name { + pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*(command.Name))(nil)).Elem())) + var nullValue command.Name + return nullValue +} + +func EqCommandName(value command.Name) command.Name { + pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) + var nullValue command.Name + return nullValue +} + +func NotEqCommandName(value command.Name) command.Name { + pegomock.RegisterMatcher(&pegomock.NotEqMatcher{Value: value}) + var nullValue command.Name + return nullValue +} + +func CommandNameThat(matcher pegomock.ArgumentMatcher) command.Name { + pegomock.RegisterMatcher(matcher) + var nullValue command.Name + return nullValue +} diff --git a/server/core/locking/mocks/matchers/locking_applycommandlock.go b/server/core/locking/mocks/matchers/locking_applycommandlock.go index 07bbd23e29..4e56270623 100644 --- a/server/core/locking/mocks/matchers/locking_applycommandlock.go +++ b/server/core/locking/mocks/matchers/locking_applycommandlock.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" locking "github.com/runatlantis/atlantis/server/core/locking" ) diff --git a/server/core/locking/mocks/matchers/locking_trylockresponse.go b/server/core/locking/mocks/matchers/locking_trylockresponse.go index 02aaa00983..53e9bb19d5 100644 --- a/server/core/locking/mocks/matchers/locking_trylockresponse.go +++ b/server/core/locking/mocks/matchers/locking_trylockresponse.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" locking "github.com/runatlantis/atlantis/server/core/locking" ) diff --git a/server/core/locking/mocks/matchers/map_of_string_to_models_projectlock.go b/server/core/locking/mocks/matchers/map_of_string_to_models_projectlock.go index e541f2b227..eb1b54416e 100644 --- a/server/core/locking/mocks/matchers/map_of_string_to_models_projectlock.go +++ b/server/core/locking/mocks/matchers/map_of_string_to_models_projectlock.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" models "github.com/runatlantis/atlantis/server/events/models" ) diff --git a/server/core/locking/mocks/matchers/models_project.go b/server/core/locking/mocks/matchers/models_project.go index 0cc4104e5a..a5a87e6f0d 100644 --- a/server/core/locking/mocks/matchers/models_project.go +++ b/server/core/locking/mocks/matchers/models_project.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" models "github.com/runatlantis/atlantis/server/events/models" ) diff --git a/server/core/locking/mocks/matchers/models_projectlock.go b/server/core/locking/mocks/matchers/models_projectlock.go index 64127a9247..182266c5e6 100644 --- a/server/core/locking/mocks/matchers/models_projectlock.go +++ b/server/core/locking/mocks/matchers/models_projectlock.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" models "github.com/runatlantis/atlantis/server/events/models" ) diff --git a/server/core/locking/mocks/matchers/models_projectplanstatus.go b/server/core/locking/mocks/matchers/models_projectplanstatus.go new file mode 100644 index 0000000000..bf2f410b93 --- /dev/null +++ b/server/core/locking/mocks/matchers/models_projectplanstatus.go @@ -0,0 +1,33 @@ +// Code generated by pegomock. DO NOT EDIT. +package matchers + +import ( + "github.com/petergtz/pegomock" + "reflect" + + models "github.com/runatlantis/atlantis/server/events/models" +) + +func AnyModelsProjectPlanStatus() models.ProjectPlanStatus { + pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*(models.ProjectPlanStatus))(nil)).Elem())) + var nullValue models.ProjectPlanStatus + return nullValue +} + +func EqModelsProjectPlanStatus(value models.ProjectPlanStatus) models.ProjectPlanStatus { + pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) + var nullValue models.ProjectPlanStatus + return nullValue +} + +func NotEqModelsProjectPlanStatus(value models.ProjectPlanStatus) models.ProjectPlanStatus { + pegomock.RegisterMatcher(&pegomock.NotEqMatcher{Value: value}) + var nullValue models.ProjectPlanStatus + return nullValue +} + +func ModelsProjectPlanStatusThat(matcher pegomock.ArgumentMatcher) models.ProjectPlanStatus { + pegomock.RegisterMatcher(matcher) + var nullValue models.ProjectPlanStatus + return nullValue +} diff --git a/server/core/locking/mocks/matchers/models_pullrequest.go b/server/core/locking/mocks/matchers/models_pullrequest.go index db2666f02f..9ae2a7e920 100644 --- a/server/core/locking/mocks/matchers/models_pullrequest.go +++ b/server/core/locking/mocks/matchers/models_pullrequest.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" models "github.com/runatlantis/atlantis/server/events/models" ) diff --git a/server/core/locking/mocks/matchers/models_pullstatus.go b/server/core/locking/mocks/matchers/models_pullstatus.go new file mode 100644 index 0000000000..cbf59c5af1 --- /dev/null +++ b/server/core/locking/mocks/matchers/models_pullstatus.go @@ -0,0 +1,33 @@ +// Code generated by pegomock. DO NOT EDIT. +package matchers + +import ( + "github.com/petergtz/pegomock" + "reflect" + + models "github.com/runatlantis/atlantis/server/events/models" +) + +func AnyModelsPullStatus() models.PullStatus { + pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*(models.PullStatus))(nil)).Elem())) + var nullValue models.PullStatus + return nullValue +} + +func EqModelsPullStatus(value models.PullStatus) models.PullStatus { + pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) + var nullValue models.PullStatus + return nullValue +} + +func NotEqModelsPullStatus(value models.PullStatus) models.PullStatus { + pegomock.RegisterMatcher(&pegomock.NotEqMatcher{Value: value}) + var nullValue models.PullStatus + return nullValue +} + +func ModelsPullStatusThat(matcher pegomock.ArgumentMatcher) models.PullStatus { + pegomock.RegisterMatcher(matcher) + var nullValue models.PullStatus + return nullValue +} diff --git a/server/core/locking/mocks/matchers/models_user.go b/server/core/locking/mocks/matchers/models_user.go index e9bf1384ba..0aa92b5d88 100644 --- a/server/core/locking/mocks/matchers/models_user.go +++ b/server/core/locking/mocks/matchers/models_user.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" models "github.com/runatlantis/atlantis/server/events/models" ) diff --git a/server/core/locking/mocks/matchers/ptr_to_command_lock.go b/server/core/locking/mocks/matchers/ptr_to_command_lock.go new file mode 100644 index 0000000000..9d47963546 --- /dev/null +++ b/server/core/locking/mocks/matchers/ptr_to_command_lock.go @@ -0,0 +1,33 @@ +// Code generated by pegomock. DO NOT EDIT. +package matchers + +import ( + "github.com/petergtz/pegomock" + "reflect" + + command "github.com/runatlantis/atlantis/server/events/command" +) + +func AnyPtrToCommandLock() *command.Lock { + pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*(*command.Lock))(nil)).Elem())) + var nullValue *command.Lock + return nullValue +} + +func EqPtrToCommandLock(value *command.Lock) *command.Lock { + pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) + var nullValue *command.Lock + return nullValue +} + +func NotEqPtrToCommandLock(value *command.Lock) *command.Lock { + pegomock.RegisterMatcher(&pegomock.NotEqMatcher{Value: value}) + var nullValue *command.Lock + return nullValue +} + +func PtrToCommandLockThat(matcher pegomock.ArgumentMatcher) *command.Lock { + pegomock.RegisterMatcher(matcher) + var nullValue *command.Lock + return nullValue +} diff --git a/server/core/locking/mocks/matchers/ptr_to_models_projectlock.go b/server/core/locking/mocks/matchers/ptr_to_models_projectlock.go index c33537f97d..7b0b6f1084 100644 --- a/server/core/locking/mocks/matchers/ptr_to_models_projectlock.go +++ b/server/core/locking/mocks/matchers/ptr_to_models_projectlock.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" models "github.com/runatlantis/atlantis/server/events/models" ) diff --git a/server/core/locking/mocks/matchers/ptr_to_models_pullstatus.go b/server/core/locking/mocks/matchers/ptr_to_models_pullstatus.go new file mode 100644 index 0000000000..207e4ccb08 --- /dev/null +++ b/server/core/locking/mocks/matchers/ptr_to_models_pullstatus.go @@ -0,0 +1,33 @@ +// Code generated by pegomock. DO NOT EDIT. +package matchers + +import ( + "github.com/petergtz/pegomock" + "reflect" + + models "github.com/runatlantis/atlantis/server/events/models" +) + +func AnyPtrToModelsPullStatus() *models.PullStatus { + pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*(*models.PullStatus))(nil)).Elem())) + var nullValue *models.PullStatus + return nullValue +} + +func EqPtrToModelsPullStatus(value *models.PullStatus) *models.PullStatus { + pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) + var nullValue *models.PullStatus + return nullValue +} + +func NotEqPtrToModelsPullStatus(value *models.PullStatus) *models.PullStatus { + pegomock.RegisterMatcher(&pegomock.NotEqMatcher{Value: value}) + var nullValue *models.PullStatus + return nullValue +} + +func PtrToModelsPullStatusThat(matcher pegomock.ArgumentMatcher) *models.PullStatus { + pegomock.RegisterMatcher(matcher) + var nullValue *models.PullStatus + return nullValue +} diff --git a/server/core/locking/mocks/matchers/slice_of_command_projectresult.go b/server/core/locking/mocks/matchers/slice_of_command_projectresult.go new file mode 100644 index 0000000000..be168d19bf --- /dev/null +++ b/server/core/locking/mocks/matchers/slice_of_command_projectresult.go @@ -0,0 +1,33 @@ +// Code generated by pegomock. DO NOT EDIT. +package matchers + +import ( + "github.com/petergtz/pegomock" + "reflect" + + command "github.com/runatlantis/atlantis/server/events/command" +) + +func AnySliceOfCommandProjectResult() []command.ProjectResult { + pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*([]command.ProjectResult))(nil)).Elem())) + var nullValue []command.ProjectResult + return nullValue +} + +func EqSliceOfCommandProjectResult(value []command.ProjectResult) []command.ProjectResult { + pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) + var nullValue []command.ProjectResult + return nullValue +} + +func NotEqSliceOfCommandProjectResult(value []command.ProjectResult) []command.ProjectResult { + pegomock.RegisterMatcher(&pegomock.NotEqMatcher{Value: value}) + var nullValue []command.ProjectResult + return nullValue +} + +func SliceOfCommandProjectResultThat(matcher pegomock.ArgumentMatcher) []command.ProjectResult { + pegomock.RegisterMatcher(matcher) + var nullValue []command.ProjectResult + return nullValue +} diff --git a/server/core/locking/mocks/matchers/slice_of_models_projectlock.go b/server/core/locking/mocks/matchers/slice_of_models_projectlock.go index 16932f9a11..f510db6e8e 100644 --- a/server/core/locking/mocks/matchers/slice_of_models_projectlock.go +++ b/server/core/locking/mocks/matchers/slice_of_models_projectlock.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" models "github.com/runatlantis/atlantis/server/events/models" ) diff --git a/server/core/locking/mocks/matchers/time_time.go b/server/core/locking/mocks/matchers/time_time.go index 755cf1bf89..461e1dd6d0 100644 --- a/server/core/locking/mocks/matchers/time_time.go +++ b/server/core/locking/mocks/matchers/time_time.go @@ -2,9 +2,8 @@ package matchers import ( - "reflect" - "github.com/petergtz/pegomock" + "reflect" time "time" ) diff --git a/server/core/locking/mocks/mock_apply_lock_checker.go b/server/core/locking/mocks/mock_apply_lock_checker.go index 9234c4d32c..e133e9994a 100644 --- a/server/core/locking/mocks/mock_apply_lock_checker.go +++ b/server/core/locking/mocks/mock_apply_lock_checker.go @@ -4,11 +4,10 @@ package mocks import ( - "reflect" - "time" - pegomock "github.com/petergtz/pegomock" locking "github.com/runatlantis/atlantis/server/core/locking" + "reflect" + "time" ) type MockApplyLockChecker struct { diff --git a/server/core/locking/mocks/mock_apply_locker.go b/server/core/locking/mocks/mock_apply_locker.go index 88e3c3897e..d8608ec1f3 100644 --- a/server/core/locking/mocks/mock_apply_locker.go +++ b/server/core/locking/mocks/mock_apply_locker.go @@ -4,11 +4,10 @@ package mocks import ( - "reflect" - "time" - pegomock "github.com/petergtz/pegomock" locking "github.com/runatlantis/atlantis/server/core/locking" + "reflect" + "time" ) type MockApplyLocker struct { diff --git a/server/core/locking/mocks/mock_backend.go b/server/core/locking/mocks/mock_backend.go index edfb550558..94aea8e857 100644 --- a/server/core/locking/mocks/mock_backend.go +++ b/server/core/locking/mocks/mock_backend.go @@ -4,12 +4,11 @@ package mocks import ( - "reflect" - "time" - pegomock "github.com/petergtz/pegomock" - "github.com/runatlantis/atlantis/server/events/command" + command "github.com/runatlantis/atlantis/server/events/command" models "github.com/runatlantis/atlantis/server/events/models" + "reflect" + "time" ) type MockBackend struct { @@ -126,6 +125,74 @@ func (mock *MockBackend) UnlockByPull(repoFullName string, pullNum int) ([]model return ret0, ret1 } +func (mock *MockBackend) UpdateProjectStatus(pull models.PullRequest, workspace string, repoRelDir string, newStatus models.ProjectPlanStatus) error { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockBackend().") + } + params := []pegomock.Param{pull, workspace, repoRelDir, newStatus} + result := pegomock.GetGenericMockFrom(mock).Invoke("UpdateProjectStatus", params, []reflect.Type{reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(error) + } + } + return ret0 +} + +func (mock *MockBackend) GetPullStatus(pull models.PullRequest) (*models.PullStatus, error) { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockBackend().") + } + params := []pegomock.Param{pull} + result := pegomock.GetGenericMockFrom(mock).Invoke("GetPullStatus", params, []reflect.Type{reflect.TypeOf((**models.PullStatus)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 *models.PullStatus + var ret1 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(*models.PullStatus) + } + if result[1] != nil { + ret1 = result[1].(error) + } + } + return ret0, ret1 +} + +func (mock *MockBackend) DeletePullStatus(pull models.PullRequest) error { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockBackend().") + } + params := []pegomock.Param{pull} + result := pegomock.GetGenericMockFrom(mock).Invoke("DeletePullStatus", params, []reflect.Type{reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(error) + } + } + return ret0 +} + +func (mock *MockBackend) UpdatePullWithResults(pull models.PullRequest, newResults []command.ProjectResult) (models.PullStatus, error) { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockBackend().") + } + params := []pegomock.Param{pull, newResults} + result := pegomock.GetGenericMockFrom(mock).Invoke("UpdatePullWithResults", params, []reflect.Type{reflect.TypeOf((*models.PullStatus)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 models.PullStatus + var ret1 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(models.PullStatus) + } + if result[1] != nil { + ret1 = result[1].(error) + } + } + return ret0, ret1 +} + func (mock *MockBackend) LockCommand(cmdName command.Name, lockTime time.Time) (*command.Lock, error) { if mock == nil { panic("mock must not be nil. Use myMock := NewMockBackend().") @@ -353,6 +420,130 @@ func (c *MockBackend_UnlockByPull_OngoingVerification) GetAllCapturedArguments() return } +func (verifier *VerifierMockBackend) UpdateProjectStatus(pull models.PullRequest, workspace string, repoRelDir string, newStatus models.ProjectPlanStatus) *MockBackend_UpdateProjectStatus_OngoingVerification { + params := []pegomock.Param{pull, workspace, repoRelDir, newStatus} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "UpdateProjectStatus", params, verifier.timeout) + return &MockBackend_UpdateProjectStatus_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockBackend_UpdateProjectStatus_OngoingVerification struct { + mock *MockBackend + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockBackend_UpdateProjectStatus_OngoingVerification) GetCapturedArguments() (models.PullRequest, string, string, models.ProjectPlanStatus) { + pull, workspace, repoRelDir, newStatus := c.GetAllCapturedArguments() + return pull[len(pull)-1], workspace[len(workspace)-1], repoRelDir[len(repoRelDir)-1], newStatus[len(newStatus)-1] +} + +func (c *MockBackend_UpdateProjectStatus_OngoingVerification) GetAllCapturedArguments() (_param0 []models.PullRequest, _param1 []string, _param2 []string, _param3 []models.ProjectPlanStatus) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]models.PullRequest, len(c.methodInvocations)) + for u, param := range params[0] { + _param0[u] = param.(models.PullRequest) + } + _param1 = make([]string, len(c.methodInvocations)) + for u, param := range params[1] { + _param1[u] = param.(string) + } + _param2 = make([]string, len(c.methodInvocations)) + for u, param := range params[2] { + _param2[u] = param.(string) + } + _param3 = make([]models.ProjectPlanStatus, len(c.methodInvocations)) + for u, param := range params[3] { + _param3[u] = param.(models.ProjectPlanStatus) + } + } + return +} + +func (verifier *VerifierMockBackend) GetPullStatus(pull models.PullRequest) *MockBackend_GetPullStatus_OngoingVerification { + params := []pegomock.Param{pull} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "GetPullStatus", params, verifier.timeout) + return &MockBackend_GetPullStatus_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockBackend_GetPullStatus_OngoingVerification struct { + mock *MockBackend + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockBackend_GetPullStatus_OngoingVerification) GetCapturedArguments() models.PullRequest { + pull := c.GetAllCapturedArguments() + return pull[len(pull)-1] +} + +func (c *MockBackend_GetPullStatus_OngoingVerification) GetAllCapturedArguments() (_param0 []models.PullRequest) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]models.PullRequest, len(c.methodInvocations)) + for u, param := range params[0] { + _param0[u] = param.(models.PullRequest) + } + } + return +} + +func (verifier *VerifierMockBackend) DeletePullStatus(pull models.PullRequest) *MockBackend_DeletePullStatus_OngoingVerification { + params := []pegomock.Param{pull} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "DeletePullStatus", params, verifier.timeout) + return &MockBackend_DeletePullStatus_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockBackend_DeletePullStatus_OngoingVerification struct { + mock *MockBackend + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockBackend_DeletePullStatus_OngoingVerification) GetCapturedArguments() models.PullRequest { + pull := c.GetAllCapturedArguments() + return pull[len(pull)-1] +} + +func (c *MockBackend_DeletePullStatus_OngoingVerification) GetAllCapturedArguments() (_param0 []models.PullRequest) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]models.PullRequest, len(c.methodInvocations)) + for u, param := range params[0] { + _param0[u] = param.(models.PullRequest) + } + } + return +} + +func (verifier *VerifierMockBackend) UpdatePullWithResults(pull models.PullRequest, newResults []command.ProjectResult) *MockBackend_UpdatePullWithResults_OngoingVerification { + params := []pegomock.Param{pull, newResults} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "UpdatePullWithResults", params, verifier.timeout) + return &MockBackend_UpdatePullWithResults_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockBackend_UpdatePullWithResults_OngoingVerification struct { + mock *MockBackend + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockBackend_UpdatePullWithResults_OngoingVerification) GetCapturedArguments() (models.PullRequest, []command.ProjectResult) { + pull, newResults := c.GetAllCapturedArguments() + return pull[len(pull)-1], newResults[len(newResults)-1] +} + +func (c *MockBackend_UpdatePullWithResults_OngoingVerification) GetAllCapturedArguments() (_param0 []models.PullRequest, _param1 [][]command.ProjectResult) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]models.PullRequest, len(c.methodInvocations)) + for u, param := range params[0] { + _param0[u] = param.(models.PullRequest) + } + _param1 = make([][]command.ProjectResult, len(c.methodInvocations)) + for u, param := range params[1] { + _param1[u] = param.([]command.ProjectResult) + } + } + return +} + func (verifier *VerifierMockBackend) LockCommand(cmdName command.Name, lockTime time.Time) *MockBackend_LockCommand_OngoingVerification { params := []pegomock.Param{cmdName, lockTime} methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "LockCommand", params, verifier.timeout) diff --git a/server/core/locking/mocks/mock_locker.go b/server/core/locking/mocks/mock_locker.go index 2346124d32..645ca45528 100644 --- a/server/core/locking/mocks/mock_locker.go +++ b/server/core/locking/mocks/mock_locker.go @@ -4,12 +4,11 @@ package mocks import ( - "reflect" - "time" - pegomock "github.com/petergtz/pegomock" locking "github.com/runatlantis/atlantis/server/core/locking" models "github.com/runatlantis/atlantis/server/events/models" + "reflect" + "time" ) type MockLocker struct { diff --git a/server/core/redis/redis.go b/server/core/redis/redis.go new file mode 100644 index 0000000000..062660243a --- /dev/null +++ b/server/core/redis/redis.go @@ -0,0 +1,391 @@ +// Package redis handles our remote database layer. +package redis + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/go-redis/redis/v9" + "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/events/command" + "github.com/runatlantis/atlantis/server/events/models" +) + +var ctx = context.Background() + +// Redis is a database using Redis 6 +type RedisDB struct { // nolint: revive + client *redis.Client +} + +const ( + pullKeySeparator = "::" +) + +func New(hostname string, port int, password string) (*RedisDB, error) { + + fmt.Println(hostname, port) + rdb := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("%s:%d", hostname, port), + Password: password, + DB: 0, // use default DB + }) + + return &RedisDB{ + client: rdb, + }, nil +} + +// NewWithClient is used for testing. +func NewWithClient(client *redis.Client, bucket string, globalBucket string) (*RedisDB, error) { + return &RedisDB{ + client: client, + }, nil +} + +// TryLock attempts to create a new lock. If the lock is +// acquired, it will return true and the lock returned will be newLock. +// If the lock is not acquired, it will return false and the current +// lock that is preventing this lock from being acquired. +func (r *RedisDB) TryLock(newLock models.ProjectLock) (bool, models.ProjectLock, error) { + var currLock models.ProjectLock + key := r.lockKey(newLock.Project, newLock.Workspace) + newLockSerialized, _ := json.Marshal(newLock) + + val, err := r.client.Get(ctx, key).Result() + // if there is no run at that key then we're free to create the lock + if err == redis.Nil { + err := r.client.Set(ctx, key, newLockSerialized, 0).Err() + if err != nil { + return false, currLock, errors.Wrap(err, "db transaction failed") + } + return true, newLock, nil + } else if err != nil { + // otherwise the lock fails, return to caller the run that's holding the lock + return false, currLock, errors.Wrap(err, "db transaction failed") + } else { + if err := json.Unmarshal([]byte(val), &currLock); err != nil { + return false, currLock, errors.Wrap(err, "failed to deserialize current lock") + } + return false, currLock, nil + } +} + +// Unlock attempts to unlock the project and workspace. +// If there is no lock, then it will return a nil pointer. +// If there is a lock, then it will delete it, and then return a pointer +// to the deleted lock. +func (r *RedisDB) Unlock(project models.Project, workspace string) (*models.ProjectLock, error) { + var lock models.ProjectLock + key := r.lockKey(project, workspace) + + val, err := r.client.Get(ctx, key).Result() + if err == redis.Nil { + return nil, nil + } else if err != nil { + return nil, errors.Wrap(err, "db transaction failed") + } else { + if err := json.Unmarshal([]byte(val), &lock); err != nil { + return nil, errors.Wrap(err, "failed to deserialize current lock") + } + r.client.Del(ctx, key) + return &lock, nil + } +} + +// List lists all current locks. +func (r *RedisDB) List() ([]models.ProjectLock, error) { + var locks []models.ProjectLock + iter := r.client.Scan(ctx, 0, "pr*", 0).Iterator() + for iter.Next(ctx) { + var lock models.ProjectLock + val, err := r.client.Get(ctx, iter.Val()).Result() + if err != nil { + return nil, errors.Wrap(err, "db transaction failed") + } + if err := json.Unmarshal([]byte(val), &lock); err != nil { + return locks, errors.Wrap(err, fmt.Sprintf("failed to deserialize lock at key '%s'", iter.Val())) + } + locks = append(locks, lock) + } + if err := iter.Err(); err != nil { + return locks, errors.Wrap(err, "db transaction failed") + } + + return locks, nil +} + +// GetLock returns a pointer to the lock for that project and workspace. +// If there is no lock, it returns a nil pointer. +func (r *RedisDB) GetLock(project models.Project, workspace string) (*models.ProjectLock, error) { + key := r.lockKey(project, workspace) + + val, err := r.client.Get(ctx, key).Result() + if err == redis.Nil { + return nil, nil + } else if err != nil { + return nil, errors.Wrap(err, "db transaction failed") + } else { + var lock models.ProjectLock + if err := json.Unmarshal([]byte(val), &lock); err != nil { + return nil, errors.Wrapf(err, "deserializing lock at key %q", key) + } + // need to set it to Local after deserialization due to https://github.com/golang/go/issues/19486 + lock.Time = lock.Time.Local() + return &lock, nil + } +} + +// UnlockByPull deletes all locks associated with that pull request and returns them. +func (r *RedisDB) UnlockByPull(repoFullName string, pullNum int) ([]models.ProjectLock, error) { + var locks []models.ProjectLock + + iter := r.client.Scan(ctx, 0, fmt.Sprintf("pr/%s*", repoFullName), 0).Iterator() + for iter.Next(ctx) { + var lock models.ProjectLock + val, err := r.client.Get(ctx, iter.Val()).Result() + if err != nil { + return nil, errors.Wrap(err, "db transaction failed") + } + if err := json.Unmarshal([]byte(val), &lock); err != nil { + return locks, errors.Wrap(err, fmt.Sprintf("failed to deserialize lock at key '%s'", iter.Val())) + } + if lock.Pull.Num == pullNum { + locks = append(locks, lock) + if _, err := r.Unlock(lock.Project, lock.Workspace); err != nil { + return locks, errors.Wrapf(err, "unlocking repo %s, path %s, workspace %s", lock.Project.RepoFullName, lock.Project.Path, lock.Workspace) + } + } + } + + if err := iter.Err(); err != nil { + return locks, errors.Wrap(err, "db transaction failed") + } + + return locks, nil +} + +func (r *RedisDB) LockCommand(cmdName command.Name, lockTime time.Time) (*command.Lock, error) { + + lock := command.Lock{ + CommandName: cmdName, + LockMetadata: command.LockMetadata{ + UnixTime: lockTime.Unix(), + }, + } + + cmdLockKey := r.commandLockKey(cmdName) + + newLockSerialized, _ := json.Marshal(lock) + + _, err := r.client.Get(ctx, cmdLockKey).Result() + if err == redis.Nil { + err = r.client.Set(ctx, cmdLockKey, newLockSerialized, 0).Err() + return &lock, errors.Wrap(err, "db transaction failed") + } else if err != nil { + return nil, errors.Wrap(err, "db transaction failed") + } else { + return nil, errors.New("db transaction failed: lock already exists") + } +} + +func (r *RedisDB) UnlockCommand(cmdName command.Name) error { + cmdLockKey := r.commandLockKey(cmdName) + _, err := r.client.Get(ctx, cmdLockKey).Result() + if err == redis.Nil { + return errors.New("db transaction failed: no lock exists") + } else if err != nil { + return errors.Wrap(err, "db transaction failed") + } else { + return r.client.Del(ctx, cmdLockKey).Err() + } +} + +func (r *RedisDB) CheckCommandLock(cmdName command.Name) (*command.Lock, error) { + cmdLock := command.Lock{} + + cmdLockKey := r.commandLockKey(cmdName) + val, err := r.client.Get(ctx, cmdLockKey).Result() + if err == redis.Nil { + return nil, nil + } else if err != nil { + return nil, errors.Wrap(err, "db transaction failed") + } else { + if err := json.Unmarshal([]byte(val), &cmdLock); err != nil { + return nil, errors.Wrap(err, "failed to deserialize Lock") + } + return &cmdLock, err + } +} + +// UpdatePullWithResults updates pull's status with the latest project results. +// It returns the new PullStatus object. +func (r *RedisDB) UpdateProjectStatus(pull models.PullRequest, workspace string, repoRelDir string, newStatus models.ProjectPlanStatus) error { + key, err := r.pullKey(pull) + if err != nil { + return err + } + + currStatusPtr, err := r.getPull(key) + if err != nil { + return err + } + if currStatusPtr == nil { + return nil + } + currStatus := *currStatusPtr + + // Update the status. + for i := range currStatus.Projects { + // NOTE: We're using a reference here because we are + // in-place updating its Status field. + proj := &currStatus.Projects[i] + if proj.Workspace == workspace && proj.RepoRelDir == repoRelDir { + proj.Status = newStatus + break + } + } + + err = r.writePull(key, currStatus) + return errors.Wrap(err, "db transaction failed") +} + +func (r *RedisDB) GetPullStatus(pull models.PullRequest) (*models.PullStatus, error) { + key, err := r.pullKey(pull) + if err != nil { + return nil, err + } + + pullStatus, err := r.getPull(key) + + return pullStatus, errors.Wrap(err, "db transaction failed") +} + +func (r *RedisDB) DeletePullStatus(pull models.PullRequest) error { + key, err := r.pullKey(pull) + if err != nil { + return err + } + return errors.Wrap(r.deletePull(key), "db transaction failed") +} + +func (r *RedisDB) UpdatePullWithResults(pull models.PullRequest, newResults []command.ProjectResult) (models.PullStatus, error) { + key, err := r.pullKey(pull) + if err != nil { + return models.PullStatus{}, err + } + + var newStatus models.PullStatus + currStatus, err := r.getPull(key) + if err != nil { + return newStatus, errors.Wrap(err, "db transaction failed") + } + + // If there is no pull OR if the pull we have is out of date, we + // just write a new pull. + if currStatus == nil || currStatus.Pull.HeadCommit != pull.HeadCommit { + var statuses []models.ProjectStatus + for _, res := range newResults { + statuses = append(statuses, r.projectResultToProject(res)) + } + newStatus = models.PullStatus{ + Pull: pull, + Projects: statuses, + } + } else { + // If there's an existing pull at the right commit then we have to + // merge our project results with the existing ones. We do a merge + // because it's possible a user is just applying a single project + // in this command and so we don't want to delete our data about + // other projects that aren't affected by this command. + newStatus = *currStatus + for _, res := range newResults { + // First, check if we should update any existing projects. + updatedExisting := false + for i := range newStatus.Projects { + // NOTE: We're using a reference here because we are + // in-place updating its Status field. + proj := &newStatus.Projects[i] + if res.Workspace == proj.Workspace && + res.RepoRelDir == proj.RepoRelDir && + res.ProjectName == proj.ProjectName { + + proj.Status = res.PlanStatus() + updatedExisting = true + break + } + } + + if !updatedExisting { + // If we didn't update an existing project, then we need to + // add this because it's a new one. + newStatus.Projects = append(newStatus.Projects, r.projectResultToProject(res)) + } + } + } + + // Now, we overwrite the key with our new status. + return newStatus, errors.Wrap(r.writePull(key, newStatus), "db transaction failed") +} + +func (r *RedisDB) getPull(key string) (*models.PullStatus, error) { + val, err := r.client.Get(ctx, key).Result() + if err == redis.Nil { + return nil, nil + } else if err != nil { + return nil, errors.Wrap(err, "db transaction failed") + } else { + var p models.PullStatus + if err := json.Unmarshal([]byte(val), &p); err != nil { + return nil, errors.Wrapf(err, "deserializing pull at %q with contents %q", key, val) + } + return &p, nil + } +} + +func (r *RedisDB) writePull(key string, pull models.PullStatus) error { + serialized, err := json.Marshal(pull) + if err != nil { + return errors.Wrap(err, "serializing") + } + err = r.client.Set(ctx, key, serialized, 0).Err() + return errors.Wrap(err, "DB Transaction failed") +} + +func (r *RedisDB) deletePull(key string) error { + err := r.client.Del(ctx, key).Err() + return errors.Wrap(err, "DB Transaction failed") +} + +func (r *RedisDB) lockKey(p models.Project, workspace string) string { + return fmt.Sprintf("pr/%s/%s/%s", p.RepoFullName, p.Path, workspace) +} + +func (r *RedisDB) commandLockKey(cmdName command.Name) string { + return fmt.Sprintf("global/%s/lock", cmdName) +} + +func (r *RedisDB) pullKey(pull models.PullRequest) (string, error) { + hostname := pull.BaseRepo.VCSHost.Hostname + if strings.Contains(hostname, pullKeySeparator) { + return "", fmt.Errorf("vcs hostname %q contains illegal string %q", hostname, pullKeySeparator) + } + repo := pull.BaseRepo.FullName + if strings.Contains(repo, pullKeySeparator) { + return "", fmt.Errorf("repo name %q contains illegal string %q", hostname, pullKeySeparator) + } + + return fmt.Sprintf("%s::%s::%d", hostname, repo, pull.Num), nil +} + +func (r *RedisDB) projectResultToProject(p command.ProjectResult) models.ProjectStatus { + return models.ProjectStatus{ + Workspace: p.Workspace, + RepoRelDir: p.RepoRelDir, + ProjectName: p.ProjectName, + Status: p.PlanStatus(), + } +} diff --git a/server/core/redis/redis_test.go b/server/core/redis/redis_test.go new file mode 100644 index 0000000000..24231b27f3 --- /dev/null +++ b/server/core/redis/redis_test.go @@ -0,0 +1,760 @@ +package redis_test + +import ( + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/core/redis" + "github.com/runatlantis/atlantis/server/events/command" + "github.com/runatlantis/atlantis/server/events/models" + + . "github.com/runatlantis/atlantis/testing" +) + +var project = models.NewProject("owner/repo", "parent/child") +var workspace = "default" +var pullNum = 1 +var lock = models.ProjectLock{ + Pull: models.PullRequest{ + Num: pullNum, + }, + User: models.User{ + Username: "lkysow", + }, + Workspace: workspace, + Project: project, + Time: time.Now(), +} + +func TestLockCommandNotSet(t *testing.T) { + t.Log("retrieving apply lock when there are none should return empty LockCommand") + s := miniredis.RunT(t) + r := newTestRedis(s) + exists, err := r.CheckCommandLock(command.Apply) + Ok(t, err) + Assert(t, exists == nil, "exp nil") +} + +func TestLockCommandEnabled(t *testing.T) { + t.Log("setting the apply lock") + s := miniredis.RunT(t) + r := newTestRedis(s) + timeNow := time.Now() + _, err := r.LockCommand(command.Apply, timeNow) + Ok(t, err) + + config, err := r.CheckCommandLock(command.Apply) + Ok(t, err) + Equals(t, true, config.IsLocked()) +} + +func TestLockCommandFail(t *testing.T) { + t.Log("setting the apply lock") + s := miniredis.RunT(t) + r := newTestRedis(s) + timeNow := time.Now() + _, err := r.LockCommand(command.Apply, timeNow) + Ok(t, err) + + _, err = r.LockCommand(command.Apply, timeNow) + ErrEquals(t, "db transaction failed: lock already exists", err) +} + +func TestUnlockCommandDisabled(t *testing.T) { + t.Log("unsetting the apply lock") + s := miniredis.RunT(t) + r := newTestRedis(s) + timeNow := time.Now() + _, err := r.LockCommand(command.Apply, timeNow) + Ok(t, err) + + config, err := r.CheckCommandLock(command.Apply) + Ok(t, err) + Equals(t, true, config.IsLocked()) + + err = r.UnlockCommand(command.Apply) + Ok(t, err) + + config, err = r.CheckCommandLock(command.Apply) + Ok(t, err) + Assert(t, config == nil, "exp nil object") +} + +func TestUnlockCommandFail(t *testing.T) { + t.Log("setting the apply lock") + s := miniredis.RunT(t) + r := newTestRedis(s) + err := r.UnlockCommand(command.Apply) + ErrEquals(t, "db transaction failed: no lock exists", err) +} + +func TestMixedLocksPresent(t *testing.T) { + s := miniredis.RunT(t) + r := newTestRedis(s) + timeNow := time.Now() + _, err := r.LockCommand(command.Apply, timeNow) + Ok(t, err) + + _, _, err = r.TryLock(lock) + Ok(t, err) + + ls, err := r.List() + Ok(t, err) + Equals(t, 1, len(ls)) +} + +func TestListNoLocks(t *testing.T) { + t.Log("listing locks when there are none should return an empty list") + s := miniredis.RunT(t) + r := newTestRedis(s) + ls, err := r.List() + Ok(t, err) + Equals(t, 0, len(ls)) +} + +func TestListOneLock(t *testing.T) { + t.Log("listing locks when there is one should return it") + s := miniredis.RunT(t) + r := newTestRedis(s) + _, _, err := r.TryLock(lock) + Ok(t, err) + ls, err := r.List() + Ok(t, err) + Equals(t, 1, len(ls)) +} + +func TestListMultipleLocks(t *testing.T) { + t.Log("listing locks when there are multiple should return them") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + + // add multiple locks + repos := []string{ + "owner/repo1", + "owner/repo2", + "owner/repo3", + "owner/repo4", + } + + for _, r := range repos { + newLock := lock + newLock.Project = models.NewProject(r, "path") + _, _, err := rdb.TryLock(newLock) + Ok(t, err) + } + ls, err := rdb.List() + Ok(t, err) + Equals(t, 4, len(ls)) + for _, r := range repos { + found := false + for _, l := range ls { + if l.Project.RepoFullName == r { + found = true + } + } + Assert(t, found, "expected %s in %v", r, ls) + } +} + +func TestListAddRemove(t *testing.T) { + t.Log("listing after adding and removing should return none") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + _, _, err := rdb.TryLock(lock) + Ok(t, err) + _, err = rdb.Unlock(project, workspace) + Ok(t, err) + + ls, err := rdb.List() + Ok(t, err) + Equals(t, 0, len(ls)) +} + +func TestLockingNoLocks(t *testing.T) { + t.Log("with no locks yet, lock should succeed") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + acquired, currLock, err := rdb.TryLock(lock) + Ok(t, err) + Equals(t, true, acquired) + Equals(t, lock, currLock) +} + +func TestLockingExistingLock(t *testing.T) { + t.Log("if there is an existing lock, lock should...") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + _, _, err := rdb.TryLock(lock) + Ok(t, err) + + t.Log("...succeed if the new project has a different path") + { + newLock := lock + newLock.Project = models.NewProject(project.RepoFullName, "different/path") + acquired, currLock, err := rdb.TryLock(newLock) + Ok(t, err) + Equals(t, true, acquired) + Equals(t, pullNum, currLock.Pull.Num) + } + + t.Log("...succeed if the new project has a different workspace") + { + newLock := lock + newLock.Workspace = "different-workspace" + acquired, currLock, err := rdb.TryLock(newLock) + Ok(t, err) + Equals(t, true, acquired) + Equals(t, newLock, currLock) + } + + t.Log("...succeed if the new project has a different repoName") + { + newLock := lock + newLock.Project = models.NewProject("different/repo", project.Path) + acquired, currLock, err := rdb.TryLock(newLock) + Ok(t, err) + Equals(t, true, acquired) + Equals(t, newLock, currLock) + } + + t.Log("...not succeed if the new project only has a different pullNum") + { + newLock := lock + newLock.Pull.Num = lock.Pull.Num + 1 + acquired, currLock, err := rdb.TryLock(newLock) + Ok(t, err) + Equals(t, false, acquired) + Equals(t, currLock.Pull.Num, pullNum) + } +} + +func TestUnlockingNoLocks(t *testing.T) { + t.Log("unlocking with no locks should succeed") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + _, err := rdb.Unlock(project, workspace) + + Ok(t, err) +} + +func TestUnlocking(t *testing.T) { + t.Log("unlocking with an existing lock should succeed") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + + _, _, err := rdb.TryLock(lock) + Ok(t, err) + _, err = rdb.Unlock(project, workspace) + Ok(t, err) + + // should be no locks listed + ls, err := rdb.List() + Ok(t, err) + Equals(t, 0, len(ls)) + + // should be able to re-lock that repo with a new pull num + newLock := lock + newLock.Pull.Num = lock.Pull.Num + 1 + acquired, currLock, err := rdb.TryLock(newLock) + Ok(t, err) + Equals(t, true, acquired) + Equals(t, newLock, currLock) +} + +func TestUnlockingMultiple(t *testing.T) { + t.Log("unlocking and locking multiple locks should succeed") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + + _, _, err := rdb.TryLock(lock) + Ok(t, err) + + new := lock + new.Project.RepoFullName = "new/repo" + _, _, err = rdb.TryLock(new) + Ok(t, err) + + new2 := lock + new2.Project.Path = "new/path" + _, _, err = rdb.TryLock(new2) + Ok(t, err) + + new3 := lock + new3.Workspace = "new-workspace" + _, _, err = rdb.TryLock(new3) + Ok(t, err) + + // now try and unlock them + _, err = rdb.Unlock(new3.Project, new3.Workspace) + Ok(t, err) + _, err = rdb.Unlock(new2.Project, workspace) + Ok(t, err) + _, err = rdb.Unlock(new.Project, workspace) + Ok(t, err) + _, err = rdb.Unlock(project, workspace) + Ok(t, err) + + // should be none left + ls, err := rdb.List() + Ok(t, err) + Equals(t, 0, len(ls)) +} + +func TestUnlockByPullNone(t *testing.T) { + t.Log("UnlockByPull should be successful when there are no locks") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + + _, err := rdb.UnlockByPull("any/repo", 1) + Ok(t, err) +} + +func TestUnlockByPullOne(t *testing.T) { + t.Log("with one lock, UnlockByPull should...") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + _, _, err := rdb.TryLock(lock) + Ok(t, err) + + t.Log("...delete nothing when its the same repo but a different pull") + { + _, err := rdb.UnlockByPull(project.RepoFullName, pullNum+1) + Ok(t, err) + ls, err := rdb.List() + Ok(t, err) + Equals(t, 1, len(ls)) + } + t.Log("...delete nothing when its the same pull but a different repo") + { + _, err := rdb.UnlockByPull("different/repo", pullNum) + Ok(t, err) + ls, err := rdb.List() + Ok(t, err) + Equals(t, 1, len(ls)) + } + t.Log("...delete the lock when its the same repo and pull") + { + _, err := rdb.UnlockByPull(project.RepoFullName, pullNum) + Ok(t, err) + ls, err := rdb.List() + Ok(t, err) + Equals(t, 0, len(ls)) + } +} + +func TestUnlockByPullAfterUnlock(t *testing.T) { + t.Log("after locking and unlocking, UnlockByPull should be successful") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + _, _, err := rdb.TryLock(lock) + Ok(t, err) + _, err = rdb.Unlock(project, workspace) + Ok(t, err) + + _, err = rdb.UnlockByPull(project.RepoFullName, pullNum) + Ok(t, err) + ls, err := rdb.List() + Ok(t, err) + Equals(t, 0, len(ls)) +} + +func TestUnlockByPullMatching(t *testing.T) { + t.Log("UnlockByPull should delete all locks in that repo and pull num") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + _, _, err := rdb.TryLock(lock) + Ok(t, err) + + // add additional locks with the same repo and pull num but different paths/workspaces + new := lock + new.Project.Path = "dif/path" + _, _, err = rdb.TryLock(new) + Ok(t, err) + new2 := lock + new2.Workspace = "new-workspace" + _, _, err = rdb.TryLock(new2) + Ok(t, err) + + // there should now be 3 + ls, err := rdb.List() + Ok(t, err) + Equals(t, 3, len(ls)) + + // should all be unlocked + _, err = rdb.UnlockByPull(project.RepoFullName, pullNum) + Ok(t, err) + ls, err = rdb.List() + Ok(t, err) + Equals(t, 0, len(ls)) +} + +func TestGetLockNotThere(t *testing.T) { + t.Log("getting a lock that doesn't exist should return a nil pointer") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + l, err := rdb.GetLock(project, workspace) + Ok(t, err) + Equals(t, (*models.ProjectLock)(nil), l) +} + +func TestGetLock(t *testing.T) { + t.Log("getting a lock should return the lock") + s := miniredis.RunT(t) + rdb := newTestRedis(s) + _, _, err := rdb.TryLock(lock) + Ok(t, err) + + l, err := rdb.GetLock(project, workspace) + Ok(t, err) + // can't compare against time so doing each field + Equals(t, lock.Project, l.Project) + Equals(t, lock.Workspace, l.Workspace) + Equals(t, lock.Pull, l.Pull) + Equals(t, lock.User, l.User) +} + +// Test we can create a status and then getCommandLock it. +func TestPullStatus_UpdateGet(t *testing.T) { + s := miniredis.RunT(t) + rdb := newTestRedis(s) + + pull := models.PullRequest{ + Num: 1, + HeadCommit: "sha", + URL: "url", + HeadBranch: "head", + BaseBranch: "base", + Author: "lkysow", + State: models.OpenPullState, + BaseRepo: models.Repo{ + FullName: "runatlantis/atlantis", + Owner: "runatlantis", + Name: "atlantis", + CloneURL: "clone-url", + SanitizedCloneURL: "clone-url", + VCSHost: models.VCSHost{ + Hostname: "github.com", + Type: models.Github, + }, + }, + } + status, err := rdb.UpdatePullWithResults( + pull, + []command.ProjectResult{ + { + Command: command.Plan, + RepoRelDir: ".", + Workspace: "default", + Failure: "failure", + }, + }) + Ok(t, err) + + maybeStatus, err := rdb.GetPullStatus(pull) + Ok(t, err) + Equals(t, pull, maybeStatus.Pull) // nolint: staticcheck + Equals(t, []models.ProjectStatus{ + { + Workspace: "default", + RepoRelDir: ".", + ProjectName: "", + Status: models.ErroredPlanStatus, + }, + }, status.Projects) +} + +// Test we can create a status, delete it, and then we shouldn't be able to getCommandLock +// it. +func TestPullStatus_UpdateDeleteGet(t *testing.T) { + s := miniredis.RunT(t) + rdb := newTestRedis(s) + + pull := models.PullRequest{ + Num: 1, + HeadCommit: "sha", + URL: "url", + HeadBranch: "head", + BaseBranch: "base", + Author: "lkysow", + State: models.OpenPullState, + BaseRepo: models.Repo{ + FullName: "runatlantis/atlantis", + Owner: "runatlantis", + Name: "atlantis", + CloneURL: "clone-url", + SanitizedCloneURL: "clone-url", + VCSHost: models.VCSHost{ + Hostname: "github.com", + Type: models.Github, + }, + }, + } + _, err := rdb.UpdatePullWithResults( + pull, + []command.ProjectResult{ + { + RepoRelDir: ".", + Workspace: "default", + Failure: "failure", + }, + }) + Ok(t, err) + + err = rdb.DeletePullStatus(pull) + Ok(t, err) + + maybeStatus, err := rdb.GetPullStatus(pull) + Ok(t, err) + Assert(t, maybeStatus == nil, "exp nil") +} + +// Test we can create a status, update a specific project's status within that +// pull status, and when we getCommandLock all the project statuses, that specific project +// should be updated. +func TestPullStatus_UpdateProject(t *testing.T) { + s := miniredis.RunT(t) + rdb := newTestRedis(s) + + pull := models.PullRequest{ + Num: 1, + HeadCommit: "sha", + URL: "url", + HeadBranch: "head", + BaseBranch: "base", + Author: "lkysow", + State: models.OpenPullState, + BaseRepo: models.Repo{ + FullName: "runatlantis/atlantis", + Owner: "runatlantis", + Name: "atlantis", + CloneURL: "clone-url", + SanitizedCloneURL: "clone-url", + VCSHost: models.VCSHost{ + Hostname: "github.com", + Type: models.Github, + }, + }, + } + _, err := rdb.UpdatePullWithResults( + pull, + []command.ProjectResult{ + { + RepoRelDir: ".", + Workspace: "default", + Failure: "failure", + }, + { + RepoRelDir: ".", + Workspace: "staging", + ApplySuccess: "success!", + }, + }) + Ok(t, err) + + err = rdb.UpdateProjectStatus(pull, "default", ".", models.DiscardedPlanStatus) + Ok(t, err) + + status, err := rdb.GetPullStatus(pull) + Ok(t, err) + Equals(t, pull, status.Pull) // nolint: staticcheck + Equals(t, []models.ProjectStatus{ + { + Workspace: "default", + RepoRelDir: ".", + ProjectName: "", + Status: models.DiscardedPlanStatus, + }, + { + Workspace: "staging", + RepoRelDir: ".", + ProjectName: "", + Status: models.AppliedPlanStatus, + }, + }, status.Projects) // nolint: staticcheck +} + +// Test that if we update an existing pull status and our new status is for a +// different HeadSHA, that we just overwrite the old status. +func TestPullStatus_UpdateNewCommit(t *testing.T) { + s := miniredis.RunT(t) + rdb := newTestRedis(s) + + pull := models.PullRequest{ + Num: 1, + HeadCommit: "sha", + URL: "url", + HeadBranch: "head", + BaseBranch: "base", + Author: "lkysow", + State: models.OpenPullState, + BaseRepo: models.Repo{ + FullName: "runatlantis/atlantis", + Owner: "runatlantis", + Name: "atlantis", + CloneURL: "clone-url", + SanitizedCloneURL: "clone-url", + VCSHost: models.VCSHost{ + Hostname: "github.com", + Type: models.Github, + }, + }, + } + _, err := rdb.UpdatePullWithResults( + pull, + []command.ProjectResult{ + { + RepoRelDir: ".", + Workspace: "default", + Failure: "failure", + }, + }) + Ok(t, err) + + pull.HeadCommit = "newsha" + status, err := rdb.UpdatePullWithResults(pull, + []command.ProjectResult{ + { + RepoRelDir: ".", + Workspace: "staging", + ApplySuccess: "success!", + }, + }) + + Ok(t, err) + Equals(t, 1, len(status.Projects)) + + maybeStatus, err := rdb.GetPullStatus(pull) + Ok(t, err) + Equals(t, pull, maybeStatus.Pull) + Equals(t, []models.ProjectStatus{ + { + Workspace: "staging", + RepoRelDir: ".", + ProjectName: "", + Status: models.AppliedPlanStatus, + }, + }, maybeStatus.Projects) +} + +// Test that if we update an existing pull status and our new status is for a +// the same commit, that we merge the statuses. +func TestPullStatus_UpdateMerge(t *testing.T) { + s := miniredis.RunT(t) + rdb := newTestRedis(s) + + pull := models.PullRequest{ + Num: 1, + HeadCommit: "sha", + URL: "url", + HeadBranch: "head", + BaseBranch: "base", + Author: "lkysow", + State: models.OpenPullState, + BaseRepo: models.Repo{ + FullName: "runatlantis/atlantis", + Owner: "runatlantis", + Name: "atlantis", + CloneURL: "clone-url", + SanitizedCloneURL: "clone-url", + VCSHost: models.VCSHost{ + Hostname: "github.com", + Type: models.Github, + }, + }, + } + _, err := rdb.UpdatePullWithResults( + pull, + []command.ProjectResult{ + { + Command: command.Plan, + RepoRelDir: "mergeme", + Workspace: "default", + Failure: "failure", + }, + { + Command: command.Plan, + RepoRelDir: "projectname", + Workspace: "default", + ProjectName: "projectname", + Failure: "failure", + }, + { + Command: command.Plan, + RepoRelDir: "staythesame", + Workspace: "default", + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "tf out", + LockURL: "lock-url", + RePlanCmd: "plan command", + ApplyCmd: "apply command", + }, + }, + }) + Ok(t, err) + + updateStatus, err := rdb.UpdatePullWithResults(pull, + []command.ProjectResult{ + { + Command: command.Apply, + RepoRelDir: "mergeme", + Workspace: "default", + ApplySuccess: "applied!", + }, + { + Command: command.Apply, + RepoRelDir: "projectname", + Workspace: "default", + ProjectName: "projectname", + Error: errors.New("apply error"), + }, + { + Command: command.Apply, + RepoRelDir: "newresult", + Workspace: "default", + ApplySuccess: "success!", + }, + }) + Ok(t, err) + + getStatus, err := rdb.GetPullStatus(pull) + Ok(t, err) + + // Test both the pull state returned from the update call *and* the getCommandLock + // call. + for _, s := range []models.PullStatus{updateStatus, *getStatus} { + Equals(t, pull, s.Pull) + Equals(t, []models.ProjectStatus{ + { + RepoRelDir: "mergeme", + Workspace: "default", + Status: models.AppliedPlanStatus, + }, + { + RepoRelDir: "projectname", + Workspace: "default", + ProjectName: "projectname", + Status: models.ErroredApplyStatus, + }, + { + RepoRelDir: "staythesame", + Workspace: "default", + Status: models.PlannedPlanStatus, + }, + { + RepoRelDir: "newresult", + Workspace: "default", + Status: models.AppliedPlanStatus, + }, + }, updateStatus.Projects) + } +} + +func newTestRedis(mr *miniredis.Miniredis) *redis.RedisDB { + r, err := redis.New(mr.Host(), mr.Server().Addr().Port, "") + if err != nil { + panic(errors.Wrap(err, "failed to create test redis client")) + } + return r +} diff --git a/server/events/apply_command_runner.go b/server/events/apply_command_runner.go index d6b6832cee..35be6ed876 100644 --- a/server/events/apply_command_runner.go +++ b/server/events/apply_command_runner.go @@ -1,7 +1,6 @@ package events import ( - "github.com/runatlantis/atlantis/server/core/db" "github.com/runatlantis/atlantis/server/core/locking" "github.com/runatlantis/atlantis/server/events/command" "github.com/runatlantis/atlantis/server/events/models" @@ -18,7 +17,7 @@ func NewApplyCommandRunner( autoMerger *AutoMerger, pullUpdater *PullUpdater, dbUpdater *DBUpdater, - db *db.BoltDB, + backend locking.Backend, parallelPoolSize int, SilenceNoProjects bool, silenceVCSStatusNoProjects bool, @@ -35,7 +34,7 @@ func NewApplyCommandRunner( autoMerger: autoMerger, pullUpdater: pullUpdater, dbUpdater: dbUpdater, - DB: db, + Backend: backend, parallelPoolSize: parallelPoolSize, SilenceNoProjects: SilenceNoProjects, silenceVCSStatusNoProjects: silenceVCSStatusNoProjects, @@ -46,7 +45,7 @@ func NewApplyCommandRunner( type ApplyCommandRunner struct { DisableApplyAll bool - DB *db.BoltDB + Backend locking.Backend locker locking.ApplyLockChecker vcsClient vcs.Client commitStatusUpdater CommitStatusUpdater diff --git a/server/events/command_runner_test.go b/server/events/command_runner_test.go index 832e12b188..19f46e13de 100644 --- a/server/events/command_runner_test.go +++ b/server/events/command_runner_test.go @@ -91,7 +91,7 @@ func setup(t *testing.T) *vcsmocks.MockClient { applyLockChecker = lockingmocks.NewMockApplyLockChecker() dbUpdater = &events.DBUpdater{ - DB: defaultBoltDB, + Backend: defaultBoltDB, } pullUpdater = &events.PullUpdater{ @@ -537,8 +537,8 @@ func TestRunAutoplanCommand_DeletePlans(t *testing.T) { defer cleanup() boltDB, err := db.New(tmp) Ok(t, err) - dbUpdater.DB = boltDB - applyCommandRunner.DB = boltDB + dbUpdater.Backend = boltDB + applyCommandRunner.Backend = boltDB autoMerger.GlobalAutomerge = true defer func() { autoMerger.GlobalAutomerge = false }() @@ -584,8 +584,8 @@ func TestFailedApprovalCreatesFailedStatusUpdate(t *testing.T) { defer cleanup() boltDB, err := db.New(tmp) Ok(t, err) - dbUpdater.DB = boltDB - applyCommandRunner.DB = boltDB + dbUpdater.Backend = boltDB + applyCommandRunner.Backend = boltDB autoMerger.GlobalAutomerge = true defer func() { autoMerger.GlobalAutomerge = false }() @@ -630,8 +630,8 @@ func TestApprovedPoliciesUpdateFailedPolicyStatus(t *testing.T) { defer cleanup() boltDB, err := db.New(tmp) Ok(t, err) - dbUpdater.DB = boltDB - applyCommandRunner.DB = boltDB + dbUpdater.Backend = boltDB + applyCommandRunner.Backend = boltDB autoMerger.GlobalAutomerge = true defer func() { autoMerger.GlobalAutomerge = false }() @@ -686,8 +686,8 @@ func TestApplyMergeablityWhenPolicyCheckFails(t *testing.T) { defer cleanup() boltDB, err := db.New(tmp) Ok(t, err) - dbUpdater.DB = boltDB - applyCommandRunner.DB = boltDB + dbUpdater.Backend = boltDB + applyCommandRunner.Backend = boltDB autoMerger.GlobalAutomerge = true defer func() { autoMerger.GlobalAutomerge = false }() @@ -765,8 +765,8 @@ func TestRunApply_DiscardedProjects(t *testing.T) { defer cleanup() boltDB, err := db.New(tmp) Ok(t, err) - dbUpdater.DB = boltDB - applyCommandRunner.DB = boltDB + dbUpdater.Backend = boltDB + applyCommandRunner.Backend = boltDB pull := fixtures.Pull pull.BaseRepo = fixtures.GithubRepo _, err = boltDB.UpdatePullWithResults(pull, []command.ProjectResult{ diff --git a/server/events/db_updater.go b/server/events/db_updater.go index eaa0712dc5..3da8534092 100644 --- a/server/events/db_updater.go +++ b/server/events/db_updater.go @@ -1,13 +1,13 @@ package events import ( - "github.com/runatlantis/atlantis/server/core/db" + "github.com/runatlantis/atlantis/server/core/locking" "github.com/runatlantis/atlantis/server/events/command" "github.com/runatlantis/atlantis/server/events/models" ) type DBUpdater struct { - DB *db.BoltDB + Backend locking.Backend } func (c *DBUpdater) updateDB(ctx *command.Context, pull models.PullRequest, results []command.ProjectResult) (models.PullStatus, error) { @@ -23,5 +23,5 @@ func (c *DBUpdater) updateDB(ctx *command.Context, pull models.PullRequest, resu filtered = append(filtered, r) } ctx.Log.Debug("updating DB with pull results") - return c.DB.UpdatePullWithResults(pull, filtered) + return c.Backend.UpdatePullWithResults(pull, filtered) } diff --git a/server/events/delete_lock_command.go b/server/events/delete_lock_command.go index 98c37f8cca..4d28994d75 100644 --- a/server/events/delete_lock_command.go +++ b/server/events/delete_lock_command.go @@ -1,7 +1,6 @@ package events import ( - "github.com/runatlantis/atlantis/server/core/db" "github.com/runatlantis/atlantis/server/core/locking" "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/logging" @@ -21,7 +20,7 @@ type DefaultDeleteLockCommand struct { Logger logging.SimpleLogging WorkingDir WorkingDir WorkingDirLocker WorkingDirLocker - DB *db.BoltDB + Backend locking.Backend } // DeleteLock handles deleting the lock at id @@ -76,7 +75,7 @@ func (l *DefaultDeleteLockCommand) deleteWorkingDir(lock models.ProjectLock) { l.Logger.Err("unable to delete workspace: %s", err) } } - if err := l.DB.UpdateProjectStatus(lock.Pull, lock.Workspace, lock.Project.Path, models.DiscardedPlanStatus); err != nil { + if err := l.Backend.UpdateProjectStatus(lock.Pull, lock.Workspace, lock.Project.Path, models.DiscardedPlanStatus); err != nil { l.Logger.Err("unable to delete project status: %s", err) } } diff --git a/server/events/delete_lock_command_test.go b/server/events/delete_lock_command_test.go index 1b467b1eb5..0aee40385b 100644 --- a/server/events/delete_lock_command_test.go +++ b/server/events/delete_lock_command_test.go @@ -79,7 +79,7 @@ func TestDeleteLock_Success(t *testing.T) { dlc := events.DefaultDeleteLockCommand{ Locker: l, Logger: logging.NewNoopLogger(t), - DB: db, + Backend: db, WorkingDirLocker: workingDirLocker, WorkingDir: workingDir, } diff --git a/server/events/pull_closed_executor.go b/server/events/pull_closed_executor.go index 27e2836de4..8ee1827e9f 100644 --- a/server/events/pull_closed_executor.go +++ b/server/events/pull_closed_executor.go @@ -21,8 +21,6 @@ import ( "strings" "text/template" - "github.com/runatlantis/atlantis/server/core/db" - "github.com/runatlantis/atlantis/server/logging" "github.com/pkg/errors" @@ -54,7 +52,7 @@ type PullClosedExecutor struct { VCSClient vcs.Client WorkingDir WorkingDir Logger logging.SimpleLogging - DB *db.BoltDB + Backend locking.Backend PullClosedTemplate PullCleanupTemplate LogStreamResourceCleaner ResourceCleaner } @@ -81,7 +79,7 @@ func (t *PullClosedEventTemplate) Execute(wr io.Writer, data interface{}) error // CleanUpPull cleans up after a closed pull request. func (p *PullClosedExecutor) CleanUpPull(repo models.Repo, pull models.PullRequest) error { - pullStatus, err := p.DB.GetPullStatus(pull) + pullStatus, err := p.Backend.GetPullStatus(pull) if err != nil { // Log and continue to clean up other resources. p.Logger.Err("retrieving pull status: %s", err) @@ -112,7 +110,7 @@ func (p *PullClosedExecutor) CleanUpPull(repo models.Repo, pull models.PullReque } // Delete pull from DB. - if err := p.DB.DeletePullStatus(pull); err != nil { + if err := p.Backend.DeletePullStatus(pull); err != nil { p.Logger.Err("deleting pull from db: %s", err) } diff --git a/server/events/pull_closed_executor_test.go b/server/events/pull_closed_executor_test.go index 2c0f43ac71..80e2a0ecb3 100644 --- a/server/events/pull_closed_executor_test.go +++ b/server/events/pull_closed_executor_test.go @@ -47,7 +47,7 @@ func TestCleanUpPullWorkspaceErr(t *testing.T) { pce := events.PullClosedExecutor{ WorkingDir: w, PullClosedTemplate: &events.PullClosedEventTemplate{}, - DB: db, + Backend: db, } err = errors.New("err") When(w.Delete(fixtures.GithubRepo, fixtures.Pull)).ThenReturn(err) @@ -67,7 +67,7 @@ func TestCleanUpPullUnlockErr(t *testing.T) { pce := events.PullClosedExecutor{ Locker: l, WorkingDir: w, - DB: db, + Backend: db, PullClosedTemplate: &events.PullClosedEventTemplate{}, } err = errors.New("err") @@ -90,7 +90,7 @@ func TestCleanUpPullNoLocks(t *testing.T) { Locker: l, VCSClient: cp, WorkingDir: w, - DB: db, + Backend: db, } When(l.UnlockByPull(fixtures.GithubRepo.FullName, fixtures.Pull.Num)).ThenReturn(nil, nil) err = pce.CleanUpPull(fixtures.GithubRepo, fixtures.Pull) @@ -176,7 +176,7 @@ func TestCleanUpPullComments(t *testing.T) { Locker: l, VCSClient: cp, WorkingDir: w, - DB: db, + Backend: db, } t.Log("testing: " + c.Description) When(l.UnlockByPull(fixtures.GithubRepo.FullName, fixtures.Pull.Num)).ThenReturn(c.Locks, nil) @@ -254,7 +254,7 @@ func TestCleanUpLogStreaming(t *testing.T) { pullClosedExecutor := events.PullClosedExecutor{ Locker: locker, WorkingDir: workingDir, - DB: db, + Backend: db, VCSClient: client, PullClosedTemplate: &events.PullClosedEventTemplate{}, LogStreamResourceCleaner: prjCmdOutHandler, diff --git a/server/server.go b/server/server.go index ce8dbffe88..037d5a7ed3 100644 --- a/server/server.go +++ b/server/server.go @@ -35,6 +35,7 @@ import ( cfg "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/core/db" + "github.com/runatlantis/atlantis/server/core/redis" "github.com/runatlantis/atlantis/server/jobs" "github.com/runatlantis/atlantis/server/metrics" "github.com/runatlantis/atlantis/server/scheduled" @@ -397,18 +398,33 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { EnableDiffMarkdownFormat: userConfig.EnableDiffMarkdownFormat, } - boltdb, err := db.New(userConfig.DataDir) - if err != nil { - return nil, err - } var lockingClient locking.Locker var applyLockingClient locking.ApplyLocker + var backend locking.Backend + + switch dbtype := userConfig.LockingDBType; dbtype { + case "redis": + logger.Info("Utilizing Redis DB") + backend, err = redis.New(userConfig.RedisHost, userConfig.RedisPort, userConfig.RedisPassword) + if err != nil { + return nil, err + } + case "boltdb": + logger.Info("Utilizing BoltDB") + backend, err = db.New(userConfig.DataDir) + if err != nil { + return nil, err + } + } + if userConfig.DisableRepoLocking { + logger.Info("Repo Locking is disabled") lockingClient = locking.NewNoOpLocker() } else { - lockingClient = locking.NewClient(boltdb) + lockingClient = locking.NewClient(backend) } - applyLockingClient = locking.NewApplyClient(boltdb, userConfig.DisableApply) + + applyLockingClient = locking.NewApplyClient(backend, userConfig.DisableApply) workingDirLocker := events.NewDefaultWorkingDirLocker() var workingDir events.WorkingDir = &events.FileWorkspace{ @@ -437,7 +453,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { Logger: logger, WorkingDir: workingDir, WorkingDirLocker: workingDirLocker, - DB: boltdb, + Backend: backend, } pullClosedExecutor := events.NewInstrumentedPullClosedExecutor( @@ -447,7 +463,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { Locker: lockingClient, WorkingDir: workingDir, Logger: logger, - DB: boltdb, + Backend: backend, PullClosedTemplate: &events.PullClosedEventTemplate{}, LogStreamResourceCleaner: projectCmdOutputHandler, VCSClient: vcsClient, @@ -575,7 +591,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { } dbUpdater := &events.DBUpdater{ - DB: boltdb, + Backend: backend, } pullUpdater := &events.PullUpdater{ @@ -622,7 +638,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { autoMerger, userConfig.ParallelPoolSize, userConfig.SilenceNoProjects, - boltdb, + backend, ) pullReqStatusFetcher := vcs.NewPullReqStatusFetcher(vcsClient) @@ -636,7 +652,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { autoMerger, pullUpdater, dbUpdater, - boltdb, + backend, userConfig.ParallelPoolSize, userConfig.SilenceNoProjects, userConfig.SilenceVCSStatusNoProjects, @@ -703,7 +719,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { Drainer: drainer, PreWorkflowHooksCommandRunner: preWorkflowHooksCommandRunner, PostWorkflowHooksCommandRunner: postWorkflowHooksCommandRunner, - PullStatusFetcher: boltdb, + PullStatusFetcher: backend, TeamAllowlistChecker: githubTeamAllowlistChecker, VarFileAllowlistChecker: varFileAllowlistChecker, } @@ -721,7 +737,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { LockDetailTemplate: templates.LockTemplate, WorkingDir: workingDir, WorkingDirLocker: workingDirLocker, - DB: boltdb, + Backend: backend, DeleteLockCommand: deleteLockCommand, } @@ -737,7 +753,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { Logger: logger, ProjectJobsTemplate: templates.ProjectJobsTemplate, ProjectJobsErrorTemplate: templates.ProjectJobsErrorTemplate, - Db: boltdb, + Backend: backend, WsMux: wsMux, KeyGenerator: controllers.JobIDKeyGenerator{}, StatsScope: statsScope.SubScope("api"), diff --git a/server/user_config.go b/server/user_config.go index af3e5e1c02..88805a0c69 100644 --- a/server/user_config.go +++ b/server/user_config.go @@ -49,11 +49,15 @@ type UserConfig struct { GitlabWebhookSecret string `mapstructure:"gitlab-webhook-secret"` APISecret string `mapstructure:"api-secret"` HidePrevPlanComments bool `mapstructure:"hide-prev-plan-comments"` + LockingDBType string `mapstructure:"locking-db-type"` LogLevel string `mapstructure:"log-level"` ParallelPoolSize int `mapstructure:"parallel-pool-size"` StatsNamespace string `mapstructure:"stats-namespace"` PlanDrafts bool `mapstructure:"allow-draft-prs"` Port int `mapstructure:"port"` + RedisHost string `mapstructure:"redis-host"` + RedisPassword string `mapstructure:"redis-password"` + RedisPort int `mapstructure:"redis-port"` RepoConfig string `mapstructure:"repo-config"` RepoConfigJSON string `mapstructure:"repo-config-json"` RepoAllowlist string `mapstructure:"repo-allowlist"`