793 upgrade password hash (#876)

* Update config.py

* Update crypto imports to fideslib and remove legacy file and tests

* Update Identity salt

* checkpoint, lowercasing all of the things

* update more config values to lowercase

* fix linting errors, fix config validators

* update the configs to use lowercase keys

* lowercase the allowed keys

* Update create_test_data.py

* bump fideslib version

* Lowercase config variables

* Fix linting issues

* Fix some test failures

* Fix application fixtures

* Remove old celery config options

* Sort script imports

* Update changelog

Co-authored-by: Thomas <[email protected]>

CHANGELOG.md

@@ -18,9 +18,11 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.6.3...main)
 
-### Added
+### Breaking Changes
+
+* Update fidesops to use bcrypt for hashing [#876](https://github.com/ethyca/fidesops/pull/876)
+
 
-* ...
 
 
 ## [1.6.3](https://github.com/ethyca/fidesops/compare/1.6.2...1.6.3)

Makefile

@@ -36,7 +36,7 @@ init-db: compose-build
 	python -c "\
 	from fidesops.db.database import init_db; \
 	from fidesops.core.config import config; \
-	init_db(config.database.SQLALCHEMY_DATABASE_URI);"
+	init_db(config.database.sqlalchemy_database_uri);"
 
 reset-db:
 	@echo "Resetting and re-initializing the application db..."
@@ -112,7 +112,7 @@ check-migrations: compose-build
 	python -c "\
 	from fidesops.db.database import check_missing_migrations; \
 	from fidesops.core.config import config; \
-	check_missing_migrations(config.database.SQLALCHEMY_DATABASE_URI);"
+	check_missing_migrations(config.database.sqlalchemy_database_uri);"
 	@make teardown
 
 isort-ci:

data/config/fidesops.toml

@@ -1,40 +1,40 @@
-PORT = 8080
+port = 8080
 
 [database]
-SERVER = "testserver"
-USER = "testuser"
-PASSWORD = "testpassword"
-DB = "testdb"
-TEST_DB = "test_testdb"
-ENABLED = true
+server = "testserver"
+user = "testuser"
+password = "testpassword"
+db = "testdb"
+test_db = "test_testdb"
+enabled = true
 
 [redis]
-HOST = "testredis"
-PASSWORD = "testpassword"
-PORT = 1234
-CHARSET = "utf8"
-DEFAULT_TTL_SECONDS = 1000
-DB_INDEX = 0
-ENABLED = true
+host = "testredis"
+password = "testpassword"
+port = 1234
+charset = "utf8"
+default_ttl_seconds = 1000
+db_index = 0
+enabled = true
 
 [security]
-APP_ENCRYPTION_KEY = "atestencryptionkeythatisvalidlen"
-CORS_ORIGINS = [ "http://test.com", "https://test.com",]
-OAUTH_ROOT_CLIENT_ID = "testrootclientid"
-OAUTH_ROOT_CLIENT_SECRET = "testrootclientsecret"
-DRP_JWT_SECRET = "testdrpsecret"
-LOG_LEVEL = "DEBUG"
+app_encryption_key = "atestencryptionkeythatisvalidlen"
+cors_origins = [ "http://test.com", "https://test.com",]
+oauth_root_client_id = "testrootclientid"
+oauth_root_client_secret = "testrootclientsecret"
+drp_jwt_secret = "testdrpsecret"
+log_level = "DEBUG"
 
 [execution]
-TASK_RETRY_COUNT = 0
-TASK_RETRY_DELAY = 1
-TASK_RETRY_BACKOFF = 1
-REQUIRE_MANUAL_REQUEST_APPROVAL = false
-MASKING_STRICT = true
+task_retry_count = 0
+task_retry_delay = 1
+task_retry_backoff = 1
+require_manual_request_approval = false
+masking_strict = true
 
 [root_user]
-ANALYTICS_OPT_OUT = true
-ANALYTICS_ID = "internal"
+analytics_opt_out = true
+analytics_id = "internal"
 
 [admin_ui]
-ENABLED = true
+enabled = true

docs/fidesops/docs/guides/policy_webhooks.md

@@ -5,15 +5,14 @@
 A Policy webhook is an HTTPS Callback that you've defined on a Policy to call an external
 REST API endpoint *before* or *after* a Privacy Request executes.
 
-You can define as many webhooks as you'd like.  Webhooks can be `one_way`, where we will just ping your API and move on, 
-or `two_way`, where we will wait for a response. Any `derived_identities` returned from a `two_way` webhook will be saved 
+You can define as many webhooks as you'd like.  Webhooks can be `one_way`, where we will just ping your API and move on,
+or `two_way`, where we will wait for a response. Any `derived_identities` returned from a `two_way` webhook will be saved
 and can be used to locate other user information.  For example, a webhook might take a known `email` `identity` and
 use that to find a `phone_number` `derived)identity`.
 
 Another use case for a Policy Webhook might be to log a user out of your mobile app after you've cleared
 their data from your system.  In this case, you'd create a `Policy` and a `ConnectionConfig` to describe the URL to hit
-to clear the cache. You'd then create a `one-way` `PolicyPostWebhook` to run after your PrivacyRequest executes. 
-
+to clear the cache. You'd then create a `one-way` `PolicyPostWebhook` to run after your PrivacyRequest executes.
 
 ## Configuration
 
@@ -24,7 +23,7 @@ You will then define a `PolicyPreWebhook` or a `PolicyPostWebhook`for a specific
 
 The information that describes how to connect to your API endpoint lives on a `ConnectionConfig`. We also use
 `ConnectionConfigs` to connect to databases like `PostgreSQL` and `MongoDB`.  This same construct can help us store
-how to connect to an external API endpoint. 
+how to connect to an external API endpoint.
 
 For more information on ConnectionConfigs, see how to [Create a ConnectionConfig.](/fidesops/api/#operations-Connections-put_connections_api_v1_connection_put)
 
@@ -54,11 +53,12 @@ See API docs on how to [Set a ConnectionConfig's Secrets](/fidesops/api#operatio
 ```
 
 ### Define pre-execution or post-execution webhooks
+
 After you've defined a `ConnectionConfig`, you can create lists of webhooks to run *before* (`PolicyPreWebhooks`)
 or *after* (`PolicyPostWebhooks`) a PrivacyRequest is executed.
 
 If you are defining PolicyPreWebhooks, all desired PolicyPreWebhooks should be included in the request
-body in the desired order.  Any PolicyPreWebhooks on the Policy *not* included in the request, will be removed from the 
+body in the desired order.  Any PolicyPreWebhooks on the Policy *not* included in the request, will be removed from the
 Policy. The same applies for PolicyPostWebhooks.
 
 To update your list of PolicyPreWebhooks:
@@ -88,19 +88,17 @@ Similarly, to update your list of Post-Execution webhooks on a Policy:
 PUT /policy/<policy_key>/webhook/post_execution
 ```
 
-
 See API docs for more information on how to [Update PolicyPreWebhooks](/fidesops/api#operations-Policy_Webhooks-create_or_update_pre_execution_webhooks_api_v1_policy__policy_key__webhook_pre_execution_put)
 and how to [Update PolicyPostWebhooks](/fidesops/api#operations-Policy_Webhooks-create_or_update_post_execution_webhooks_api_v1_policy__policy_key__webhook_post_execution_put).
 
-
 ### Update a single webhook
 
 To update a single PolicyPreWebhook or PolicyPostWebhook, send a PATCH request to update selected attributes.
 Note that updates to order can likewise update the order of related webhooks.
 
-The following example will update the PolicyPreWebhook with key `webhook_hook` to be `two_way` instead of 
-`one_way` and will update its order from 0 to 1.  Because we've defined two PolicyPreWebhooks, this causes the 
-webhook at position 1 to move to position 0. 
+The following example will update the PolicyPreWebhook with key `webhook_hook` to be `two_way` instead of
+`one_way` and will update its order from 0 to 1.  Because we've defined two PolicyPreWebhooks, this causes the
+webhook at position 1 to move to position 0.
 
 ```json title="<code>PATCH /policy/<policy_key>/webhook/pre-execution/wake_up_snowflake_db</code>"
 {
@@ -109,8 +107,9 @@ webhook at position 1 to move to position 0.
 }
 ```
 
-Because this PATCH request updated the order of other webhooks, a reordered summary is included under the 
+Because this PATCH request updated the order of other webhooks, a reordered summary is included under the
 `new_order` attribute:
+
 ```json title="Response"
 {
     "resource": {
@@ -142,10 +141,8 @@ PATCH /policy/<policy_key>/webhook/post_execution/<post_execution_key>
 See API docs for more information on how to [PATCH a PolicyPreWebhook](/fidesops/api#operations-Policy_Webhooks-update_pre_execution_webhook_api_v1_policy__policy_key__webhook_pre_execution__pre_webhook_key__patch)
 and how to [PATCH a PolicyPostWebhook](/fidesops/api#operations-Policy_Webhooks-update_post_execution_webhook_api_v1_policy__policy_key__webhook_post_execution__post_webhook_key__patch).
 
-
 ## Webhook request format
 
-
 Before and after running access or erasure requests, fidesops will send requests to any configured webhooks in sequential order
 with the following request body:
 
@@ -161,11 +158,11 @@ with the following request body:
 }
 ```
 
-Most of these attributes were configured by you: the `direction`, the `callback_type` ("pre" for `PolicyPreWebhook`s that will run 
+Most of these attributes were configured by you: the `direction`, the `callback_type` ("pre" for `PolicyPreWebhook`s that will run
 before PrivacyRequest execution or "post" for `PolicyPostWebhook`s that will run after PrivacyRequestExecution).
 Known identities are also embedded in the request.
 
-For `two-way` `PolicyPreWebhooks`, we include specific headers in case you need to pause PrivacyRequest 
+For `two-way` `PolicyPreWebhooks`, we include specific headers in case you need to pause PrivacyRequest
 execution while you take care of additional processing on your end.
 
 ```json
@@ -174,18 +171,19 @@ execution while you take care of additional processing on your end.
   "reply-to-token": "<jwe_token>"
 }
 ```
+
  To resume, you should send a request back to the `reply-to` URL with the `reply-to-token`.  The `reply-to-token` will
-expire when your redis cache expires: `config.redis.DEFAULT_TTL_SECONDS` (Ffdesops uses the redis cache to temporarily
+expire when your redis cache expires: `config.redis.default_ttl_seconds` (Fidesops uses the redis cache to temporarily
  store identity data).  At this point, your PrivacyRequest will be given an `error` status, and you would have to resubmit
 the PrivacyRequest.
 
 ## Webhook response format
 
-Your webhook should respond immediately. If more processing time is needed, either make sure it is configured as a 
+Your webhook should respond immediately. If more processing time is needed, either make sure it is configured as a
 `one-way` webhook, or reply with `halt=True` if you want to pause execution and wait for your processing to finish.
-Note that only `PolicyPreWebhooks` can pause execution. 
+Note that only `PolicyPreWebhooks` can pause execution.
 
-We don't expect a response from `one-way` webhooks, but `two-way` webhooks should respond with the following: 
+We don't expect a response from `one-way` webhooks, but `two-way` webhooks should respond with the following:
 
 ```json
 {
@@ -201,7 +199,7 @@ Derived identity is optional: a returned email or phone number will replace curr
 
 ## Resuming request execution
 
-If your webhook needed more processing time, once completed, send a request to the `reply-to` URL 
+If your webhook needed more processing time, once completed, send a request to the `reply-to` URL
 given to you in the original request header with the `reply-to-token` auth token.
 
 ```json title="<code>POST privacy_request/<privacy-request-id>/resume</code>"
@@ -216,6 +214,6 @@ given to you in the original request header with the `reply-to-token` auth token
 
 If there are no derived identities, an empty `{}` request body will suffice.
 
-The `reply-to-token` is a JWE containing the current webhook id, scopes to access the callback endpoint, 
-and the datetime the token is issued.  We unpack this and resume the privacy request execution after the 
+The `reply-to-token` is a JWE containing the current webhook id, scopes to access the callback endpoint,
+and the datetime the token is issued.  We unpack this and resume the privacy request execution after the
 specified webhook. The `reply-to-token` expires after a set amount of time, specified by the `config.execution.PRIVACY_REQUEST_DELAY_TIMEOUT` config variable. Once the redis cache expires, fidesops no longer has the original identity data and the privacy request should be resubmitted.

fidesops-integration.toml

@@ -1,23 +1,23 @@
 [postgres_example]
-SERVER="postgres_example"
-USER="postgres"
-PASSWORD="postgres"
-DB="postgres_example"
-PORT= 5432
+server="postgres_example"
+user="postgres"
+password="postgres"
+db="postgres_example"
+port= 5432
 
 [mongodb_example]
-SERVER= "mongodb_example"
-USER="mongo_user"
-PASSWORD="mongo_pass"
-DB="mongo_test"
-PORT=27017
+server= "mongodb_example"
+user="mongo_user"
+password="mongo_pass"
+db="mongo_test"
+port=27017
 
 [mysql_example]
-SERVER="mysql_example"
-USER="mysql_user"
-PASSWORD="mysql_pw"
-DB="mysql_example"
-PORT=3306
+server="mysql_example"
+user="mysql_user"
+password="mysql_pw"
+db="mysql_example"
+port=3306
 
 [redshift]
 external_uri=""
@@ -32,15 +32,15 @@ dataset=""
 [bigquery.keyfile_creds]
 
 [mssql_example]
-SERVER="mssql_example"
-USER="sa"
-PASSWORD="Mssql_pw1"
-DB="mssql_example"
-PORT=1433
+server="mssql_example"
+user="sa"
+password="Mssql_pw1"
+db="mssql_example"
+port=1433
 
 [mariadb_example]
-SERVER="mariadb_example"
-USER="mariadb_user"
-PASSWORD="mariadb_pw"
-DB="mariadb_example"
-PORT=3306
+server="mariadb_example"
+user="mariadb_user"
+password="mariadb_pw"
+db="mariadb_example"
+port=3306

fidesops.toml

@@ -1,43 +1,43 @@
-PORT = 8080
+port = 8080
 
 [database]
-SERVER = "db"
-USER = "postgres"
-PASSWORD = "216f4b49bea5da4f84f05288258471852c3e325cd336821097e1e65ff92b528a"
-DB = "app"
-TEST_DB = "test"
-ENABLED = true
+server = "db"
+user = "postgres"
+password = "216f4b49bea5da4f84f05288258471852c3e325cd336821097e1e65ff92b528a"
+db = "app"
+test_db = "test"
+enabled = true
 
 [redis]
-HOST = "redis"
-PASSWORD = "testpassword"
-PORT = 6379
-CHARSET = "utf8"
-DEFAULT_TTL_SECONDS = 604800
-DB_INDEX = 0
-ENABLED = true
-SSL = false
-SSL_CERT_REQS = "required"
+host = "redis"
+password = "testpassword"
+port = 6379
+charset = "utf8"
+default_ttl_seconds = 604800
+db_index = 0
+enabled = true
+ssl = false
+ssl_cert_reqs = "required"
 
 [security]
-APP_ENCRYPTION_KEY = "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
-CORS_ORIGINS = [ "http://localhost", "http://localhost:8080", "http://localhost:3000", "http://localhost:3001",]
-ENCODING = "UTF-8"
-OAUTH_ROOT_CLIENT_ID = "fidesopsadmin"
-OAUTH_ROOT_CLIENT_SECRET = "fidesopsadminsecret"
-DRP_JWT_SECRET = "secret"
-LOG_LEVEL = "INFO"
+app_encryption_key = "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
+cors_origins = [ "http://localhost", "http://localhost:8080", "http://localhost:3000", "http://localhost:3001",]
+encoding = "UTF-8"
+oauth_root_client_id = "fidesopsadmin"
+oauth_root_client_secret = "fidesopsadminsecret"
+drp_jwt_secret = "secret"
+log_level = "INFO"
 
 [execution]
-MASKING_STRICT = true
-REQUIRE_MANUAL_REQUEST_APPROVAL = false
-TASK_RETRY_COUNT = 0
-TASK_RETRY_DELAY = 1
-TASK_RETRY_BACKOFF = 1
-WORKER_ENABLED = false
+masking_strict = true
+require_manual_request_approval = false
+task_retry_count = 0
+task_retry_delay = 1
+task_retry_backoff = 1
+worker_enabled = false
 
 [root_user]
-ANALYTICS_OPT_OUT = false
+analytics_opt_out = false
 
 [admin_ui]
-ENABLED = true
+enabled = true

requirements.txt

@@ -10,7 +10,7 @@ fastapi-caching[redis]
 fastapi-pagination[sqlalchemy]~= 0.9.3
 fastapi[all]==0.78.0
 fideslang==1.0.0
-fideslib==2.2.2
+fideslib==3.0.0
 fideslog==1.2.1
 multidimensional_urlencode==0.0.4
 pandas==1.3.3