diff --git a/modules/project_cleanup/README.md b/modules/project_cleanup/README.md index d462f922..052a89a0 100644 --- a/modules/project_cleanup/README.md +++ b/modules/project_cleanup/README.md @@ -21,12 +21,17 @@ The following services must be enabled on the project housing the cleanup functi | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| job\_schedule | Cleaner function run frequency, in cron syntax | string | `"*/5 * * * *"` | no | | max\_project\_age\_in\_hours | The maximum number of hours that a GCP project, selected by `target_tag_name` and `target_tag_value`, can exist | number | `"6"` | no | | organization\_id | The organization ID whose projects to clean up | string | n/a | yes | | project\_id | The project ID to host the scheduled function in | string | n/a | yes | | region | The region the project is in (App Engine specific) | string | n/a | yes | -| target\_tag\_name | The name of a tag to filter GCP projects on for consideration by the cleanup utility | string | `"cft-ephemeral"` | no | -| target\_tag\_value | The value of a tag to filter GCP projects on for consideration by the cleanup utility | string | `"true"` | no | +| target\_excluded\_labels | Map of project lablels that won't be deleted. | map(string) | `` | no | +| target\_folder\_id | Folder ID to delete all projects under. | string | `""` | no | +| target\_included\_labels | Map of project lablels that will be deleted. | map(string) | `` | no | +| target\_tag\_name | The name of a tag to filter GCP projects on for consideration by the cleanup utility (legacy, use `target_included_labels` map instead). | string | `""` | no | +| target\_tag\_value | The value of a tag to filter GCP projects on for consideration by the cleanup utility (legacy, use `target_included_labels` map instead). | string | `""` | no | +| topic\_name | Name of pubsub topic connecting the scheduled projects cleanup function | string | `"pubsub_scheduled_project_cleaner"` | no | ## Outputs diff --git a/modules/project_cleanup/function_source/README.md b/modules/project_cleanup/function_source/README.md index 61498345..52526e38 100644 --- a/modules/project_cleanup/function_source/README.md +++ b/modules/project_cleanup/function_source/README.md @@ -3,7 +3,9 @@ This is a simple utility that scans a GCP organization for projects matching certain criteria, and enqueues such projects for deletion. Currently supported criteria are the combination of: - **Age:** Only projects older than the configured age, in hours, will be marked for deletion. -- **Key-Value Pair:** Only projects whose labels contain the provided key-value pair will be marked for deletion. +- **Key-Value Pair Include:** Only projects whose labels contain the provided key-value pair will be marked for deletion. +- **Key-Value Pair Exclude:** Projects whose labels contain the provided key-value pair won't be marked for deletion. +- **Folder ID:** Only projects under this Folder ID will be marked for deletion. Both of these criteria must be met for a project to be deleted. @@ -13,9 +15,10 @@ The following environment variables may be specified to configure the cleanup ut | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| `TARGET_TAG_NAME` | The tag name to match on for identifying projects to delete | string | n/a | yes | -| `TARGET_TAG_VALUE` | The tag value to match on for identifying projects to delete | string | n/a | yes | -| `MAX_PROJECT_AGE_HOURS` | The project age, in hours, at which point deletion should be considered | integer | n/a | yes | +| `TARGET_EXCLUDED_LABELS` | Labels to match on for identifying projects to avoid deletion | string | n/a | no | +| `TARGET_FOLDER_ID` | Folder ID to delete prjojects under | string | n/a | yes | +| `TARGET_INCLUDED_LABELS` | Labels to match on for identifying projects to delete | string | n/a | no | +| `MAX_PROJECT_AGE_HOURS` | The project age, in hours, at which point deletion should be considered | integer | n/a | no | ## Required Permissions diff --git a/modules/project_cleanup/function_source/go.sum b/modules/project_cleanup/function_source/go.sum deleted file mode 100644 index c24552fc..00000000 --- a/modules/project_cleanup/function_source/go.sum +++ /dev/null @@ -1,115 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= -go.opencensus.io v0.19.1/go.mod h1:gug0GbSHa8Pafr0d2urOSgoXHZ6x/RUlaiT0d9pqb4A= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190318221613-d196dffd7c2b h1:ZWpVMTsK0ey5WJCu+vVdfMldWq7/ezaOcjnKWIHWVkE= -golang.org/x/net v0.0.0-20190318221613-d196dffd7c2b/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI= -google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= -google.golang.org/api v0.2.0 h1:B5VXkdjt7K2Gm6fGBC9C9a1OAKJDT95cTqwet+2zib0= -google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= -google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/modules/project_cleanup/function_source/main.go b/modules/project_cleanup/function_source/main.go index 9471ffcd..22a04012 100644 --- a/modules/project_cleanup/function_source/main.go +++ b/modules/project_cleanup/function_source/main.go @@ -17,99 +17,192 @@ limitations under the License. package project_cleaner import ( + "encoding/json" "fmt" + "golang.org/x/net/context" + "golang.org/x/oauth2/google" + "google.golang.org/api/cloudresourcemanager/v1" "log" "os" + "regexp" "strconv" "time" - - "golang.org/x/net/context" - "golang.org/x/oauth2/google" - "google.golang.org/api/cloudresourcemanager/v1" ) var ( logger = log.New(os.Stdout, "", 0) ) -// PubSubMessage wraps the message sent to the background Cloud Function by GCP PubSub. type PubSubMessage struct { Data []byte `json:"data"` } -// CleanUpProjects is the entry point of the scheduled function. -func CleanUpProjects(ctx context.Context, m PubSubMessage) error { - targetTag := os.Getenv("TARGET_TAG_NAME") - targetValue := os.Getenv("TARGET_TAG_VALUE") - maxAgeInHoursStr := os.Getenv("MAX_PROJECT_AGE_HOURS") - maxAgeInHours, err := strconv.ParseInt(maxAgeInHoursStr, 10, 0) - if err != nil { - logger.Fatal(fmt.Sprintf("Could not convert %s to integer", maxAgeInHoursStr)) - return err +const LifecycleStateActiveRequested = "ACTIVE" +const TargetExcludedLabels = "TARGET_EXCLUDED_LABELS" +const TargetIncludedLabels = "TARGET_INCLUDED_LABELS" +const TargetFolderId = "TARGET_FOLDER_ID" +const MaxProjectAgeHours = "MAX_PROJECT_AGE_HOURS" + +func activeProjectFilter(project *cloudresourcemanager.Project) bool { + return project.LifecycleState == LifecycleStateActiveRequested +} + +func tooOldTime(i int64) time.Time { + return time.Unix(time.Now().Unix()-i, 0) +} + +func processProjectsResponsePage(removeProjectById func(projectId string)) func(page *cloudresourcemanager.ListProjectsResponse) error { + excludedLabelsMap := getLabelsMapFromEnv(TargetExcludedLabels) + excludeProjectByOneOfLabelsFilter := func(project *cloudresourcemanager.Project) bool { + return !checkIfAtLeastOneLabelPresentIfAny(project, excludedLabelsMap, true) } - logger.Println(fmt.Sprintf("Launching project cleanup for projects older than %d hours, with tag %s=%s", maxAgeInHours, targetTag, targetValue)) - err = deleteProjectsMatchingTag(ctx, targetTag, targetValue, maxAgeInHours) + + includedLabelsMap := getLabelsMapFromEnv(TargetIncludedLabels) + includeProjectByOneOfLabelsFilter := func(project *cloudresourcemanager.Project) bool { + return checkIfAtLeastOneLabelPresentIfAny(project, includedLabelsMap, false) + } + + resourceCreationCutoff := tooOldTime(int64(getCorrectMaxAgeInHoursOrTerminateExecution()) * 60 * 60) + ageFilter := func(project *cloudresourcemanager.Project) bool { + projectCreatedAt, err := time.Parse(time.RFC3339, project.CreateTime) + if err != nil { + logger.Println(fmt.Sprintf("Fail to parse CreateTime for [%s], skip it. Error [%s]", project.Name, err.Error())) + return false + } + return projectCreatedAt.Before(resourceCreationCutoff) + } + + combinedProjectFilter := func(project *cloudresourcemanager.Project) bool { + return activeProjectFilter(project) && ageFilter(project) && includeProjectByOneOfLabelsFilter(project) && excludeProjectByOneOfLabelsFilter(project) + } + + return func(page *cloudresourcemanager.ListProjectsResponse) error { + for _, project := range page.Projects { + if combinedProjectFilter(project) { + projectId := project.ProjectId + logger.Println(fmt.Sprintf("Got project %s ", projectId)) + removeProjectById(projectId) + } + } + return nil + } +} + +func getCorrectMaxAgeInHoursOrTerminateExecution() int64 { + maxAgeInHoursStr := os.Getenv(MaxProjectAgeHours) + maxAgeInHours, err := strconv.ParseInt(os.Getenv(MaxProjectAgeHours), 10, 0) if err != nil { - logger.Fatal(err) + logger.Fatal(fmt.Sprintf("Could not convert [%s] to integer. Specify correct value, please.", maxAgeInHoursStr)) } - return err + return maxAgeInHours } -func deleteProjectsMatchingTag(ctx context.Context, key string, value string, acceptableAgeInHours int64) error { - logger.Println("Initializing Google client") - c, err := google.DefaultClient(ctx, cloudresourcemanager.CloudPlatformScope) +func checkIfAtLeastOneLabelPresentIfAny(project *cloudresourcemanager.Project, labels map[string]string, isExcludeCheck bool) bool { + if len(labels) == 0 { + return !isExcludeCheck + } + result := false + projectLabels := project.Labels + for key, value := range labels { + if !result { + result = projectLabels[key] == value + } + } + return result +} + +func getLabelsMapFromEnv(envVariableName string) map[string]string { + targetExcludedLabels := os.Getenv(envVariableName) + logger.Println("Trying to get labels map") + labels := make(map[string]string) + err := json.Unmarshal([]byte(targetExcludedLabels), &labels) if err != nil { - return err + logger.Println(fmt.Sprintf("Fail to get labels map from [%s] env variable, error [%s]", envVariableName, err.Error())) + } else { + logger.Println(fmt.Sprintf("Got labels map [%s] from [%s] env variable", labels, envVariableName)) } + return labels +} - cloudResourceManagerService, err := cloudresourcemanager.New(c) +func getResourceManagerServiceOrTerminateExecution(ctx context.Context) *cloudresourcemanager.Service { + logger.Println("Trying to initialize Google client") + client, err := google.DefaultClient(ctx, cloudresourcemanager.CloudPlatformScope) + if err != nil { + logger.Println("Fail to initialize Google client, terminate execution") + logger.Fatal(err) + } + logger.Println("Initialized Google client") + logger.Println("Trying to get Cloud Resource Manager") + cloudResourceManagerService, err := cloudresourcemanager.New(client) if err != nil { - return err + logger.Println("Fail to get Cloud Resource Manager, terminate execution") + logger.Fatal(err) } + logger.Println("Got Cloud Resource Manager") + return cloudResourceManagerService +} - resourceCreationCutoff := tooOldTime(int64(acceptableAgeInHours * 60 * 60)) +func invoke(ctx context.Context) { + cloudResourceManagerService := getResourceManagerServiceOrTerminateExecution(ctx) - logger.Println("Looking through projects") - req := cloudResourceManagerService.Projects.List() - if err := req.Pages(ctx, func(page *cloudresourcemanager.ListProjectsResponse) error { - for _, project := range page.Projects { - if val, ok := project.Labels[key]; !(ok && val == value) { - logger.Println(fmt.Sprintf("Rejecting project %s because of missing label", project.ProjectId)) - continue - } + removeLien := func(name string) { + logger.Println(fmt.Sprintf("Trying to remove lien [%s]", name)) + _, err := cloudResourceManagerService.Liens.Delete(name).Context(ctx).Do() + if err != nil { + logger.Println(fmt.Sprintf("Fail to remove lien [%s], error [%s]", name, err.Error())) + } else { + logger.Println(fmt.Sprintf("Removed lien [%s]", name)) + } - if project.LifecycleState != "ACTIVE" { - logger.Println(fmt.Sprintf("Rejecting project %s because lifecycle state is not ACTIVE", project.ProjectId)) - continue - } + } - projectCreatedAt, err := time.Parse(time.RFC3339, project.CreateTime) - if err != nil { - return err - } + removeProjectById := func(projectId string) { + logger.Println(fmt.Sprintf("Trying to remove project [%s]", projectId)) + _, err := cloudResourceManagerService.Projects.Delete(projectId).Context(ctx).Do() + if err != nil { + logger.Println(fmt.Sprintf("Fail to remove project [%s], error [%s]", projectId, err.Error())) + } else { + logger.Println(fmt.Sprintf("Removed project [%s]", projectId)) + } + } - if !projectCreatedAt.Before(resourceCreationCutoff) { - logger.Println(fmt.Sprintf("Rejecting project %s because it was created too recently", project.ProjectId)) - continue + removeProjectWithLiens := func(projectId string) { + logger.Println(fmt.Sprintf("Trying to get all liens for the project [%s]", projectId)) + parent := fmt.Sprintf("projects/%s", projectId) + req := cloudResourceManagerService.Liens.List().Parent(parent) + if err := req.Pages(ctx, func(page *cloudresourcemanager.ListLiensResponse) error { + logger.Println(fmt.Sprintf("Got [%d] liens for the project [%s]", len(page.Liens), projectId)) + for _, lien := range page.Liens { + removeLien(lien.Name) } + removeProjectById(projectId) + return nil + }); err != nil { + logger.Println(fmt.Sprintf("Fail to get all liens for the project [%s], error [%s]", projectId, err.Error())) + } + } - logger.Println(fmt.Sprintf("Project %s was created at %s, is active, and is older than %d hours. Deleting.", project.ProjectId, project.CreateTime, acceptableAgeInHours)) + folderId := getCorrectFolderIdOrTerminateExecution() + logger.Println(fmt.Sprintf("Trying to get projects from folder with id [%s] and process them", folderId)) + requestFilter := fmt.Sprintf("parent.type:folder parent.id:%s", folderId) + req := cloudResourceManagerService.Projects.List().Filter(requestFilter) + if err := req.Pages(ctx, processProjectsResponsePage(removeProjectWithLiens)); err != nil { + logger.Println(fmt.Sprintf("Fail to get projects for the forlder with id [%s], error [%s]", folderId, err.Error())) + } +} - _, err = cloudResourceManagerService.Projects.Delete(project.ProjectId).Do() - if err != nil { - logger.Fatal(err) - return err - } - logger.Println(fmt.Sprintf("Requested deletion of project %s.", project.ProjectId)) - } - return nil - }); err != nil { - return err +func getCorrectFolderIdOrTerminateExecution() string { + targetFolderIdString := os.Getenv(TargetFolderId) + matched, err := regexp.MatchString(`^[0-9]+$`, targetFolderIdString) + if err != nil || !matched { + logger.Fatal(fmt.Sprintf("Invalid folder id [%s]. Specify correct value, please.", targetFolderIdString)) } - logger.Println("Considered all projects") - return nil + return targetFolderIdString } -func tooOldTime(i int64) time.Time { - return time.Unix(time.Now().Unix()-i, 0) +func CleanUpProjects(ctx context.Context, m PubSubMessage) error { + getCorrectMaxAgeInHoursOrTerminateExecution() + getCorrectFolderIdOrTerminateExecution() + invoke(ctx) + return nil } diff --git a/modules/project_cleanup/main.tf b/modules/project_cleanup/main.tf index 040928df..3c2bf608 100644 --- a/modules/project_cleanup/main.tf +++ b/modules/project_cleanup/main.tf @@ -14,6 +14,10 @@ * limitations under the License. */ +locals { + target_included_labels = var.target_tag_name != "" && var.target_tag_value != "" ? merge({ "${var.target_tag_name}" = "${var.target_tag_value}" }, var.target_included_labels) : var.target_included_labels +} + resource "google_service_account" "project_cleaner_function" { project = var.project_id account_id = "project-cleaner-function" @@ -30,20 +34,21 @@ module "scheduled_project_cleaner" { source = "../../" project_id = var.project_id job_name = "project-cleaner" - job_schedule = "*/5 * * * *" + job_schedule = var.job_schedule function_entry_point = "CleanUpProjects" function_source_directory = "${path.module}/function_source" function_name = "old-project-cleaner" region = var.region - topic_name = "pubsub_scheduled_project_cleaner" + topic_name = var.topic_name function_available_memory_mb = 128 function_description = "Clean up GCP projects older than ${var.max_project_age_in_hours} hours matching particular tags" function_runtime = "go111" - function_service_account_email = "${google_service_account.project_cleaner_function.email}" + function_service_account_email = google_service_account.project_cleaner_function.email function_environment_variables = { - TARGET_TAG_NAME = var.target_tag_name - TARGET_TAG_VALUE = var.target_tag_value - MAX_PROJECT_AGE_HOURS = var.max_project_age_in_hours + TARGET_EXCLUDED_LABELS = jsonencode(var.target_excluded_labels) + TARGET_FOLDER_ID = var.target_folder_id + TARGET_INCLUDED_LABELS = jsonencode(local.target_included_labels) + MAX_PROJECT_AGE_HOURS = var.max_project_age_in_hours } } diff --git a/modules/project_cleanup/variables.tf b/modules/project_cleanup/variables.tf index 7841d1d3..5c2f7343 100644 --- a/modules/project_cleanup/variables.tf +++ b/modules/project_cleanup/variables.tf @@ -29,16 +29,28 @@ variable "region" { description = "The region the project is in (App Engine specific)" } +variable "job_schedule" { + type = string + description = "Cleaner function run frequency, in cron syntax" + default = "*/5 * * * *" +} + +variable "topic_name" { + type = string + description = "Name of pubsub topic connecting the scheduled projects cleanup function" + default = "pubsub_scheduled_project_cleaner" +} + variable "target_tag_name" { type = string - description = "The name of a tag to filter GCP projects on for consideration by the cleanup utility" - default = "cft-ephemeral" + description = "The name of a tag to filter GCP projects on for consideration by the cleanup utility (legacy, use `target_included_labels` map instead)." + default = "" } variable "target_tag_value" { type = string - description = "The value of a tag to filter GCP projects on for consideration by the cleanup utility" - default = "true" + description = "The value of a tag to filter GCP projects on for consideration by the cleanup utility (legacy, use `target_included_labels` map instead)." + default = "" } variable "max_project_age_in_hours" { @@ -46,3 +58,21 @@ variable "max_project_age_in_hours" { description = "The maximum number of hours that a GCP project, selected by `target_tag_name` and `target_tag_value`, can exist" default = 6 } + +variable "target_excluded_labels" { + type = map(string) + description = "Map of project lablels that won't be deleted." + default = {} +} + +variable "target_included_labels" { + type = map(string) + description = "Map of project lablels that will be deleted." + default = {} +} + +variable "target_folder_id" { + type = string + description = "Folder ID to delete all projects under." + default = "" +}