Publish fidesops to PyPI (ethyca#491)

* Add missing `__init__.py` files

* Publish to PyPI on new tag creation

* Alphabetize `requirements.txt`

There are no version number or constraint changes - only ordering.

* Add `versioneer`

To automate version numbers

* Fix a metric ton of linter errors

These errors are being surfaced due to the new `__init__.py` files.

* Publish to PyPI on published releases

* Test that the `request_id` param actually filters

* Update references to the `id` query param to `request_id`

* Test publishes to PyPI

* Publish to testpypi prior to production

.gitattributes

@@ -0,0 +1 @@
+src/fidesops/_version.py export-subst

.github/workflows/publish.yml → .github/workflows/publish_to_dockerhub.yml

@@ -1,4 +1,4 @@
-name: Publish
+name: Publish to DockerHub
 
 on:
   push:

.github/workflows/publish_to_pypi.yml

@@ -0,0 +1,30 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+env:
+  TWINE_USERNAME: __token__
+
+jobs:
+  publish_to_pypi:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Twine
+        run: pip install twine
+
+      - name: Create Source Distribution
+        run: python setup.py sdist
+
+      - name: Test Twine Upload
+        run: twine upload --repository testpypi dist/*
+        env:
+          TWINE_PASSWORD: ${{ secrets.TESTPYPI_TOKEN }}
+
+      - name: Production Twine Upload
+        run: twine upload dist/*
+        env:
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}

.github/workflows/safe_pr_checks.yml

@@ -36,4 +36,4 @@ jobs:
 
       - name: Integration Tests
         run: make pytest-integration
-        timeout-minutes: 20
+        timeout-minutes: 20

MANIFEST.in

@@ -0,0 +1,7 @@
+include LICENSE
+include README.md
+include requirements.txt
+include dev-requirements.txt
+include versioneer.py
+include src/fidesops/alembic.ini
+include src/fidesops/_version.py

docs/fidesops/docs/guides/privacy_requests.md

@@ -89,7 +89,7 @@ An optional denial reason can be provided when denying a Privacy Request:
 ## How do I monitor Privacy Requests as they execute?
 Privacy Requests can be monitored at any time throughout their execution by submitting any of the following requests:
 
-`GET api/v1/privacy-request?id=<privacy_request_id>`
+`GET api/v1/privacy-request?request_id=<privacy_request_id>`
 
 `GET api/v1/privacy-request?external_id=<external_id>`
 

docs/fidesops/docs/guides/reporting.md

@@ -45,7 +45,7 @@ Check out the [API docs here](/fidesops/api#operations-Privacy_Requests-get_requ
 
 Use the `id` query param to view the high level status of a single privacy request.
 
-`GET api/v1/privacy-request?id=<privacy_request_id>`
+`GET api/v1/privacy-request?request_id=<privacy_request_id>`
 
 If an `external_id` was provided at request creation, we can also track the privacy request using:
 
@@ -98,7 +98,7 @@ logs for `my-postgres-db` (when the `order` collection is starting and finishing
 that were potentially returned or masked based on the Rules you've specified on the Policy. The embedded execution logs 
 are automatically truncated at 50 logs, so to view the entire list of logs, visit the execution logs endpoint separately.
 
-`GET api/v1/privacy-request?id={privacy_request_id}&verbose=True`
+`GET api/v1/privacy-request?request_id={privacy_request_id}&verbose=True`
 
 ```json
 {

docs/fidesops/docs/postman/Fidesops.postman_collection.json

@@ -774,7 +774,7 @@
 						"method": "GET",
 						"header": [],
 						"url": {
-							"raw": "{{host}}/privacy-request/?id={{privacy_request_id}}&verbose=True",
+							"raw": "{{host}}/privacy-request/?request_id={{privacy_request_id}}&verbose=True",
 							"host": [
 								"{{host}}"
 							],
@@ -784,7 +784,7 @@
 							],
 							"query": [
 								{
-									"key": "id",
+									"key": "request_id",
 									"value": "{{privacy_request_id}}"
 								},
 								{

pyproject.toml

@@ -39,22 +39,28 @@ exclude = '''
 [tool.pylint.messages_control]
 disable=[
     "bad-option-value",
-    "line-too-long",
-    "invalid-name",
-    "too-few-public-methods",
-    "no-self-argument",
+    "broad-except",
     "dangerous-default-value",
     "duplicate-code",
-    "logging-fstring-interpolation",
+    "fixme",
     "import-error",
-    "unused-argument",
-    "no-self-use",
     "import-outside-toplevel",
-    "unsubscriptable-object", # Otherwise throws errors on certain Type annotations
+    "invalid-name",
+    "line-too-long",
+    "logging-fstring-interpolation",
+    "missing-class-docstring",
+    "missing-function-docstring",
     "missing-module-docstring",
+    "no-self-argument",
+    "no-self-use",
     "raise-missing-from",
+    "too-few-public-methods",
     "too-many-ancestors",
-    "fixme"]
+    "too-many-arguments",
+    "unnecessary-comprehension",
+    "unsubscriptable-object", # Otherwise throws errors on certain Type annotations
+    "unused-argument",
+]
 
 [tool.pylint.reports]
 reports="no"

requirements.txt

@@ -1,35 +1,36 @@
+alembic==1.6.5
 APScheduler==3.8.0
+bcrypt~=3.2.0
+boto3~=1.18.14
+click==7.1.2
+cryptography~=3.4.8
+dask==2021.10.0
+email-validator
+emails
+fastapi-caching[redis]
+fastapi-pagination[sqlalchemy]~= 0.8.3
 fastapi[all]==0.68.1
-sqlalchemy==1.4.14
-alembic==1.6.5
 fidesctl==0.9.8.4
-python-jose[cryptography]==3.3.0
+multidimensional_urlencode==0.0.4
+pandas==1.3.3
 passlib[bcrypt]==1.7.2
 psycopg2-binary==2.9.1
-email-validator
-emails
-fastapi-caching[redis]
-sqlalchemy-stubs==0.4
-SQLAlchemy-Utils==0.37.8
-redis==3.5.3
 pydantic~=1.8.2
-starlette~=0.14.2
-python-dotenv~=0.19.0
-bcrypt~=3.2.0
-Unidecode==1.2.0
-uvicorn~=0.13.4
 pydash==5.0.2
-boto3~=1.18.14
-cryptography~=3.4.8
-fastapi-pagination[sqlalchemy]~= 0.8.3
-dask==2021.10.0
-requests~=2.25.0
+pyjwt
 pymongo==3.12.0
-pandas==1.3.3
-click==7.1.2
 PyMySQL==1.0.2
-sqlalchemy-redshift==0.8.8
+python-dotenv~=0.19.0
+python-jose[cryptography]==3.3.0
+redis==3.5.3
+requests~=2.25.0
 snowflake-sqlalchemy==1.3.2
 sqlalchemy-bigquery==1.3.0
-multidimensional_urlencode==0.0.4
-pyjwt
+sqlalchemy-redshift==0.8.8
+sqlalchemy-stubs==0.4
+SQLAlchemy-Utils==0.37.8
+sqlalchemy==1.4.14
+starlette~=0.14.2
+Unidecode==1.2.0
+uvicorn~=0.13.4
+versioneer==0.19

setup.cfg

@@ -0,0 +1,11 @@
+[versioneer]
+VCS = git
+style = pep440
+versionfile_source = src/fidesops/_version.py
+versionfile_build = fidesops/_version.py
+tag_prefix =
+parentdir_prefix =
+
+[mypy]
+[mypy-src.fidesops._version]
+ignore_errors = True

setup.py

@@ -1,5 +1,8 @@
 import pathlib
-from setuptools import setup, find_packages
+
+from setuptools import find_packages, setup
+
+import versioneer
 
 here = pathlib.Path(__file__).parent.resolve()
 long_description = open("README.md").read()
@@ -10,6 +13,8 @@
 
 setup(
     name="fidesops",
+    version=versioneer.get_version(),
+    cmdclass=versioneer.get_cmdclass(),
     description="Automation engine for privacy requests",
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -18,6 +23,7 @@
     python_requires=">=3.7, <4",
     package_dir={"": "src"},
     packages=find_packages(where="src"),
+    package_data={"fidesops": ["alembic.ini"]},
     include_package_data=True,
     author="Ethyca, Inc.",
     author_email="[email protected]",

src/fidesops/__init__.py

@@ -0,0 +1,4 @@
+from ._version import get_versions
+
+__version__ = get_versions()["version"]
+del get_versions