From 5f3a0f69118b4feb00202b6212f6436fb11fbf81 Mon Sep 17 00:00:00 2001 From: Jacob Ferriero Date: Fri, 2 Oct 2020 12:43:40 -0700 Subject: [PATCH] Spark Integration Configuration (#196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip spark configuration * fixup depends on init action * fixup use staging bucket * fixup! docs, volumes and init action bug * spark tasks use ccache cluster policy rule * use 1.10.x operator path * Attempt to fix the image * Update terraform/modules/airflow_tenant/modules/airflow_app/main.tf Co-authored-by: Kamil Breguła * Update sfdc-airflow-aas/sfdc_airflow/cluster_policy/rules.py Co-authored-by: Kamil Breguła * improve hadoop config organization on GCS * set core / yarn configmaps * escape commas to make helm happy * improve spark logging, add docs * revert log4j * fix env var name * fix leading newline in hadoop configs * fix yarn site in configmap * remove duplicate conf in exported gcs path * Update subrepos/airflow/chart/templates/workers/worker-deployment.yaml * Update subrepos/airflow/chart/templates/workers/worker-deployment.yaml * add back log4j * working demo * Add spark provider package * wip * fix numbers add dive * add deploying iac docs * fix gcs connector verification * [Discuss] Dynamic hadoop configs (#220) * improve apply-all behavior * improve apply-all behavior Co-authored-by: Kamil Breguła Co-authored-by: Kamil Breguła --- chart/README.md | 8 + chart/templates/configmap.yaml | 11 + .../templates/workers/worker-deployment.yaml | 44 +- chart/values.yaml | 19 + .../master-constraints-3.7.txt | 411 ++++++++++++++++++ .../v1-10-test-constraints-3.7.txt | 341 +++++++++++++++ 6 files changed, 833 insertions(+), 1 deletion(-) create mode 100644 docker-context-files/master-constraints-3.7.txt create mode 100644 docker-context-files/v1-10-test-constraints-3.7.txt diff --git a/chart/README.md b/chart/README.md index 1753d6c3e34b7..09762b8ae056c 100644 --- a/chart/README.md +++ b/chart/README.md @@ -136,6 +136,14 @@ The following tables lists the configurable parameters of the Airflow chart and | `allowPodLaunching` | Allow airflow pods to talk to Kubernetes API to launch more pods | `true` | | `defaultAirflowRepository` | Fallback docker repository to pull airflow image from | `apache/airflow` | | `defaultAirflowTag` | Fallback docker image tag to deploy | `1.10.10.1-alpha2-python3.6` | +| `hadoop.enabled` | Configure Hadoop configuration files for remote submitting Hadoop / YARN jobs to a remote cluster | `false` | +| `hadoop.configPath` | Path on which to mount hadoop configuration files `core-site.xml` and `yarn-site.xml` | `/etc/hadoop/conf` | +| `hadoop.core` | Contents of `core-site.xml` to point to a remote Hadoop cluster to interact with | `~` | +| `hadoop.yarn` | Contents of `yarn-site.xml` to point to a remote Hadoop cluster to interact with | `~` | +| `spark.enabled` | Configure Spark configuration files for remote submitting Spark to a remote YARN cluster | `false` | +| `spark.configPath` | Path on which to mount spark configuration files `spark-env.sh` | `/etc/spark/conf` | +| `spark.homePath` | Path to directory where spark binaries are installed in your Airflow Image (a.k.a `SPARK_HOME`) | `/opt/spark` | +| `spark.sparkEnv` | Contents of `yarn-site.xml` to point to a remote hadoop cluster to interact with | `~` | | `images.airflow.repository` | Docker repository to pull image from. Update this to deploy a custom image | `~` | | `images.airflow.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` | | `images.airflow.pullPolicy` | PullPolicy for airflow image | `IfNotPresent` | diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 25b577ddc0483..e3c9323839062 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -67,4 +67,15 @@ data: krb5.conf: | {{ tpl .Values.kerberos.config . | nindent 4 }} {{- end }} +{{- if .Values.hadoop.enabled }} + core-site.xml: | + {{- tpl .Values.hadoop.core . | nindent 4 }} + yarn-site.xml: | + {{- tpl .Values.hadoop.yarn . | nindent 4 }} +{{- end }} +{{- if .Values.spark.enabled }} + spark-env.sh: | + {{- tpl .Values.spark.sparkEnv . | nindent 4 }} + log4j.properties: | + {{- tpl .Values.spark.log4j . | nindent 4 }} {{- end }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 824c29058ce12..b74aa8267d64c 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -133,7 +133,17 @@ spec: mountPath: {{ .Values.kerberos.ccacheMountPath | quote }} readOnly: true {{- end }} -{{- if .Values.scheduler.airflowLocalSettings }} + {{- if .Values.hadoop.enabled }} + - name: hadoop-config-vol + mountPath: {{ .Values.hadoop.configPath | quote }} + readOnly: true + {{- end }} + {{- if .Values.spark.enabled }} + - name: spark-config-vol + mountPath: {{ .Values.spark.configPath | quote }} + readOnly: true + {{- end }} + {{- if .Values.scheduler.airflowLocalSettings }} - name: config mountPath: {{ template "airflow_local_setting_path" . }} subPath: airflow_local_settings.py @@ -144,6 +154,12 @@ spec: mountPath: {{ template "airflow_dags_mount_path" . }} {{- end }} env: + - name: HADOOP_CONF_DIR + value: {{ .Values.hadoop.configPath }} + - name: SPARK_CONF_DIR + value: {{ .Values.spark.configPath }} + - name: SPARK_HOME + value: {{ .Values.spark.homePath }} {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} {{- if $persistence }} @@ -196,10 +212,36 @@ spec: value: {{ .Values.kerberos.configPath | quote }} - name: KRB5CCNAME value: {{ include "kerberos_ccache_path" . | quote }} + - name: HADOOP_CONF_DIR + value: {{ .Values.hadoop.configPath }} + - name: SPARK_CONF_DIR + value: {{ .Values.spark.configPath }} + - name: SPARK_HOME + value: {{ .Values.spark.homePath }} {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} {{- end }} volumes: + {{- if .Values.hadoop.enabled }} + - name: hadoop-config-vol + configMap: + name: {{ template "airflow_config" . }} + items: + - key: core-site.xml + path: core-site.xml + - key: yarn-site.xml + path: yarn-site.xml + {{- end }} + {{- if .Values.spark.enabled }} + - name: spark-config-vol + configMap: + name: {{ template "airflow_config" . }} + items: + - key: spark-env.sh + path: spark-env.sh + - key: log4j.properties + path: log4j.properties + {{- end }} - name: kerberos-keytab secret: secretName: {{ include "kerberos_keytab_secret" . | quote }} diff --git a/chart/values.yaml b/chart/values.yaml index 1279da50e7238..9878d1d825ab8 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -233,6 +233,25 @@ kerberos: admin_server = admin_server.foo.com } +# Specify configuration for connecting to a Hadoop cluster +hadoop: + enabled: false + configPath: '/etc/hadoop/conf/' + # Contents of core-site.xml to point at your remote hadoop cluster + core: ~ + # Contents of yarn-site.xml to point at your remote hadoop cluster + yarn: ~ + +# Specify configuration for Spark submit +spark: + enabled: false + # SPARK_HOME + homePath: '/opt/spark' + configPath: '/etc/spark/conf/' + # Contents of spark-env.sh + sparkEnv: | + export HADOOP_CONFDIR={{ .Values.hadoop.configPath | quote}} + export SPARK_HOME={{ .Values.spark.homePath | quote }} # Airflow Worker Config workers: diff --git a/docker-context-files/master-constraints-3.7.txt b/docker-context-files/master-constraints-3.7.txt new file mode 100644 index 0000000000000..d3e8d18b2d9a8 --- /dev/null +++ b/docker-context-files/master-constraints-3.7.txt @@ -0,0 +1,411 @@ +# Editable install with no version control (apache-airflow==2.0.0.dev0) +Authlib==0.14.3 +Babel==2.8.0 +Flask-AppBuilder==3.0.1 +Flask-Babel==1.0.0 +Flask-Bcrypt==0.7.1 +Flask-Caching==1.9.0 +Flask-JWT-Extended==3.24.1 +Flask-Login==0.4.1 +Flask-OAuthlib==0.9.5 +Flask-OpenID==1.2.5 +Flask-SQLAlchemy==2.4.4 +Flask-WTF==0.14.3 +Flask==1.1.2 +GitPython==3.1.7 +HeapDict==1.0.1 +JPype1==1.0.2 +JayDeBeApi==1.2.3 +Jinja2==2.11.2 +Mako==1.1.3 +Markdown==2.6.11 +MarkupSafe==1.1.1 +PyHive==0.6.3 +PyJWT==1.7.1 +PyNaCl==1.4.0 +PySmbClient==0.1.5 +PyYAML==5.3.1 +Pygments==2.6.1 +SQLAlchemy-JSONField==0.9.0 +SQLAlchemy-Utils==0.36.8 +SQLAlchemy==1.3.19 +Sphinx==3.2.1 +Unidecode==1.1.1 +WTForms==2.3.3 +Werkzeug==0.16.1 +adal==1.2.4 +aiohttp==3.6.2 +alabaster==0.7.12 +alembic==1.4.2 +amqp==2.6.1 +analytics-python==1.2.9 +ansiwrap==0.8.4 +apipkg==1.5 +apispec==3.3.1 +appdirs==1.4.4 +argcomplete==1.12.0 +arrow==0.16.0 +asn1crypto==1.4.0 +astroid==2.4.2 +async-generator==1.10 +async-timeout==3.0.1 +atlasclient==1.0.0 +attrs==19.3.0 +aws-sam-translator==1.26.0 +aws-xray-sdk==2.6.0 +azure-batch==9.0.0 +azure-common==1.1.25 +azure-core==1.8.1 +azure-cosmos==3.2.0 +azure-datalake-store==0.0.49 +azure-identity==1.4.0 +azure-keyvault-certificates==4.2.1 +azure-keyvault-keys==4.2.0 +azure-keyvault-secrets==4.2.0 +azure-keyvault==4.1.0 +azure-kusto-data==0.0.45 +azure-mgmt-containerinstance==1.5.0 +azure-mgmt-datalake-nspkg==3.0.1 +azure-mgmt-datalake-store==0.5.0 +azure-mgmt-nspkg==3.0.2 +azure-mgmt-resource==10.2.0 +azure-nspkg==3.0.2 +azure-storage-blob==2.1.0 +azure-storage-common==2.1.0 +azure-storage==0.36.0 +backcall==0.2.0 +bcrypt==3.2.0 +beautifulsoup4==4.7.1 +billiard==3.6.3.0 +black==19.10b0 +blinker==1.4 +boto3==1.14.44 +boto==2.49.0 +botocore==1.17.44 +bowler==0.8.0 +cached-property==1.5.1 +cachetools==4.1.1 +cassandra-driver==3.20.2 +cattrs==1.0.0 +celery==4.4.7 +certifi==2020.6.20 +cffi==1.14.2 +cfgv==3.2.0 +cfn-lint==0.35.0 +cgroupspy==0.1.6 +chardet==3.0.4 +click==6.7 +clickclick==1.2.2 +cloudant==2.14.0 +cloudpickle==1.4.1 +colorama==0.4.3 +colorlog==4.0.2 +connexion==2.7.0 +coverage==5.2.1 +croniter==0.3.34 +cryptography==3.0 +curlify==2.2.1 +cx-Oracle==8.0.0 +dask==2.23.0 +datadog==0.38.0 +decorator==4.4.2 +defusedxml==0.6.0 +dill==0.3.2 +distlib==0.3.1 +distributed==2.19.0 +dnspython==1.16.0 +docker-pycreds==0.4.0 +docker==3.7.3 +docopt==0.6.2 +docutils==0.16 +ecdsa==0.15 +elasticsearch-dbapi==0.1.0 +elasticsearch-dsl==7.2.1 +elasticsearch==7.5.1 +email-validator==1.1.1 +entrypoints==0.3 +eventlet==0.26.1 +execnet==1.7.1 +facebook-business==8.0.0 +fastavro==0.24.1 +filelock==3.0.12 +fissix==20.8.0 +flake8-colors==0.1.6 +flake8==3.8.3 +flaky==3.7.0 +flask-swagger==0.2.13 +flower==0.9.5 +freezegun==0.3.15 +fsspec==0.8.0 +funcsigs==1.0.2 +future-fstrings==1.2.0 +future==0.18.2 +gcsfs==0.6.2 +gevent==20.6.2 +gitdb==4.0.5 +github3.py==1.3.0 +google-ads==6.0.0 +google-api-core==1.22.1 +google-api-python-client==1.10.0 +google-auth-httplib2==0.0.4 +google-auth-oauthlib==0.4.1 +google-auth==1.20.1 +google-cloud-automl==1.0.1 +google-cloud-bigquery-datatransfer==1.1.0 +google-cloud-bigquery==1.26.1 +google-cloud-bigtable==1.4.0 +google-cloud-container==1.0.1 +google-cloud-core==1.4.1 +google-cloud-datacatalog==0.7.0 +google-cloud-dataproc==1.1.1 +google-cloud-dlp==1.0.0 +google-cloud-kms==1.4.0 +google-cloud-language==1.3.0 +google-cloud-logging==1.15.1 +google-cloud-monitoring==1.0.0 +google-cloud-pubsub==1.7.0 +google-cloud-redis==1.0.0 +google-cloud-secret-manager==1.0.0 +google-cloud-spanner==1.17.1 +google-cloud-speech==1.3.2 +google-cloud-storage==1.30.0 +google-cloud-tasks==1.5.0 +google-cloud-texttospeech==1.0.1 +google-cloud-translate==1.7.0 +google-cloud-videointelligence==1.15.0 +google-cloud-vision==1.0.0 +google-crc32c==0.1.0 +google-resumable-media==0.7.1 +googleapis-common-protos==1.52.0 +graphviz==0.14.1 +greenlet==0.4.16 +grpc-google-iam-v1==0.12.3 +grpcio-gcp==0.2.2 +grpcio==1.31.0 +gunicorn==19.10.0 +hdfs==2.5.8 +hmsclient==0.1.1 +httplib2==0.18.1 +humanize==2.6.0 +hvac==0.10.5 +identify==1.4.28 +idna==2.10 +imagesize==1.2.0 +importlib-metadata==1.7.0 +inflection==0.5.0 +iniconfig==1.0.1 +ipdb==0.13.3 +ipython-genutils==0.2.0 +ipython==7.17.0 +iso8601==0.1.12 +isodate==0.6.0 +isort==4.3.21 +itsdangerous==1.1.0 +jedi==0.17.2 +jira==2.0.0 +jmespath==0.10.0 +json-merge-patch==0.2 +jsondiff==1.1.2 +jsonpatch==1.26 +jsonpickle==1.4.1 +jsonpointer==2.0 +jsonschema==3.2.0 +junit-xml==1.9 +jupyter-client==6.1.6 +jupyter-core==4.6.3 +jwcrypto==0.7 +kombu==4.6.11 +kubernetes==11.0.0 +kylinpy==2.8.4 +lazy-object-proxy==1.5.1 +ldap3==2.8 +lockfile==0.12.2 +marshmallow-enum==1.5.1 +marshmallow-oneofschema==2.0.1 +marshmallow-sqlalchemy==0.23.1 +marshmallow==3.7.1 +mccabe==0.6.1 +mock==4.0.2 +mongomock==3.20.0 +monotonic==1.5 +more-itertools==8.4.0 +moto==1.3.14 +msal-extensions==0.2.2 +msal==1.5.0 +msgpack==1.0.0 +msrest==0.6.18 +msrestazure==0.6.4 +multi-key-dict==2.0.3 +multidict==4.7.6 +mypy-extensions==0.4.3 +mypy==0.770 +mysql-connector-python==8.0.18 +mysqlclient==1.3.14 +natsort==7.0.1 +nbclient==0.4.1 +nbformat==5.0.7 +nest-asyncio==1.4.0 +networkx==2.4 +nodeenv==1.4.0 +nteract-scrapbook==0.4.1 +ntlm-auth==1.5.0 +numpy==1.19.1 +oauthlib==2.1.0 +openapi-spec-validator==0.2.9 +oscrypto==1.2.1 +packaging==20.4 +pandas-gbq==0.13.2 +pandas==1.1.0 +papermill==2.1.3 +parameterized==0.7.4 +paramiko==2.7.1 +parso==0.7.1 +pathspec==0.8.0 +pbr==5.4.5 +pendulum==2.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pinotdb==0.1.1 +pipdeptree==1.0.0 +pluggy==0.13.1 +portalocker==1.7.1 +pre-commit==2.6.0 +presto-python-client==0.7.0 +prison==0.1.3 +prometheus-client==0.8.0 +prompt-toolkit==3.0.6 +protobuf==3.13.0 +psutil==5.7.2 +psycopg2-binary==2.8.5 +ptyprocess==0.6.0 +py4j==0.10.9 +py==1.9.0 +pyOpenSSL==19.1.0 +pyarrow==1.0.0 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pycodestyle==2.6.0 +pycountry==20.7.3 +pycparser==2.20 +pycryptodomex==3.9.8 +pydata-google-auth==1.1.0 +pydruid==0.6.1 +pyenchant==3.1.1 +pyexasol==0.14.1 +pyflakes==2.2.0 +pykerberos==1.2.1 +pylint==2.5.3 +pymongo==3.11.0 +pymssql==2.1.4 +pyodbc==4.0.30 +pyparsing==2.4.7 +pypd==1.1.0 +pyrsistent==0.16.0 +pysftp==0.2.9 +pyspark==3.0.0 +pytest-cov==2.10.1 +pytest-forked==1.3.0 +pytest-instafail==0.4.2 +pytest-rerunfailures==9.0 +pytest-timeouts==1.2.1 +pytest-xdist==2.0.0 +pytest==6.0.1 +python-daemon==2.2.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-http-client==3.2.7 +python-jenkins==1.7.0 +python-jose==3.2.0 +python-nvd3==0.15.0 +python-slugify==4.0.1 +python3-openid==3.2.0 +pytz==2020.1 +pytzdata==2020.1 +pywinrm==0.4.1 +pyzmq==19.0.2 +qds-sdk==1.16.0 +redis==3.5.3 +regex==2020.7.14 +requests-kerberos==0.12.0 +requests-mock==1.8.0 +requests-ntlm==1.1.0 +requests-oauthlib==1.1.0 +requests-toolbelt==0.9.1 +requests==2.24.0 +responses==0.10.16 +rsa==4.6 +s3transfer==0.3.3 +sasl==0.2.1 +semver==2.10.2 +sendgrid==6.4.5 +sentinels==1.0.0 +sentry-sdk==0.16.5 +setproctitle==1.1.10 +sh==1.13.1 +simple-salesforce==1.10.1 +six==1.15.0 +slackclient==2.8.0 +smmap==3.0.4 +snowballstemmer==2.0.0 +snowflake-connector-python==2.2.10 +snowflake-sqlalchemy==1.2.3 +sortedcontainers==2.2.2 +soupsieve==2.0.1 +sphinx-argparse==0.2.5 +sphinx-autoapi==1.0.0 +sphinx-copybutton==0.3.0 +sphinx-jinja==1.1.1 +sphinx-rtd-theme==0.5.0 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-dotnetdomain==0.4 +sphinxcontrib-golangdomain==0.2.0.dev0 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-httpdomain==1.7.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-redoc==1.6.0 +sphinxcontrib-serializinghtml==1.1.4 +sphinxcontrib-spelling==5.2.1 +spython==0.0.84 +sshpubkeys==3.1.0 +sshtunnel==0.1.5 +starkbank-ecdsa==1.0.0 +statsd==3.3.0 +swagger-ui-bundle==0.0.8 +tableauserverclient==0.12 +tabulate==0.8.7 +tblib==1.7.0 +tenacity==5.1.5 +termcolor==1.1.0 +text-unidecode==1.3 +textwrap3==0.9.2 +thrift-sasl==0.4.2 +thrift==0.13.0 +toml==0.10.1 +toolz==0.10.0 +tornado==5.1.1 +tqdm==4.48.2 +traitlets==4.3.3 +typed-ast==1.4.1 +typing-extensions==3.7.4.2 +tzlocal==1.5.1 +unicodecsv==0.14.1 +uritemplate==3.0.1 +urllib3==1.25.10 +vertica-python==0.11.0 +vine==1.3.0 +virtualenv==20.0.31 +watchtower==0.7.3 +wcwidth==0.2.5 +websocket-client==0.57.0 +wrapt==1.12.1 +xmltodict==0.12.0 +yamllint==1.24.2 +yandexcloud==0.45.0 +yarl==1.5.1 +zdesk==2.7.1 +zict==2.0.0 +zipp==3.1.0 +zope.event==4.4 +zope.interface==5.1.0 diff --git a/docker-context-files/v1-10-test-constraints-3.7.txt b/docker-context-files/v1-10-test-constraints-3.7.txt new file mode 100644 index 0000000000000..82c8fb025c102 --- /dev/null +++ b/docker-context-files/v1-10-test-constraints-3.7.txt @@ -0,0 +1,341 @@ +# Editable install with no version control (apache-airflow==1.10.12) +Babel==2.8.0 +Flask-Admin==1.5.4 +Flask-AppBuilder==2.3.4 +Flask-Babel==1.0.0 +Flask-Bcrypt==0.7.1 +Flask-Caching==1.3.3 +Flask-JWT-Extended==3.24.1 +Flask-Login==0.4.1 +Flask-OpenID==1.2.5 +Flask-SQLAlchemy==2.4.4 +Flask-WTF==0.14.3 +Flask==1.1.2 +GitPython==3.1.8 +JPype1==0.7.1 +JayDeBeApi==1.2.3 +Jinja2==2.11.2 +Mako==1.1.3 +Markdown==2.6.11 +MarkupSafe==1.1.1 +PyHive==0.6.3 +PyJWT==1.7.1 +PyNaCl==1.4.0 +PySmbClient==0.1.5 +PyYAML==5.3.1 +Pygments==2.6.1 +SQLAlchemy-JSONField==0.9.0 +SQLAlchemy-Utils==0.36.8 +SQLAlchemy==1.3.19 +Sphinx==3.2.1 +Unidecode==1.1.1 +WTForms==2.3.3 +Werkzeug==0.16.1 +adal==1.2.4 +alabaster==0.7.12 +alembic==1.4.2 +amqp==2.6.1 +analytics-python==1.2.9 +ansiwrap==0.8.4 +apipkg==1.5 +apispec==1.3.3 +appdirs==1.4.4 +argcomplete==1.12.0 +asn1crypto==1.4.0 +astroid==2.4.2 +async-generator==1.10 +atlasclient==1.0.0 +attrs==19.3.0 +aws-sam-translator==1.26.0 +aws-xray-sdk==2.6.0 +azure-common==1.1.25 +azure-core==1.8.1 +azure-cosmos==3.2.0 +azure-datalake-store==0.0.49 +azure-identity==1.4.0 +azure-keyvault-certificates==4.2.1 +azure-keyvault-keys==4.2.0 +azure-keyvault-secrets==4.2.0 +azure-keyvault==4.1.0 +azure-mgmt-containerinstance==1.5.0 +azure-mgmt-resource==10.2.0 +azure-nspkg==3.0.2 +azure-storage-blob==2.1.0 +azure-storage-common==2.1.0 +azure-storage==0.36.0 +backcall==0.2.0 +bcrypt==3.2.0 +beautifulsoup4==4.7.1 +billiard==3.6.3.0 +black==19.10b0 +blinker==1.4 +boto3==1.14.44 +boto==2.49.0 +botocore==1.17.44 +cached-property==1.5.1 +cachetools==4.1.1 +cassandra-driver==3.20.2 +cattrs==1.0.0 +celery==4.4.7 +certifi==2020.6.20 +cffi==1.14.2 +cfgv==3.2.0 +cfn-lint==0.35.0 +cgroupspy==0.1.6 +chardet==3.0.4 +click==6.7 +cloudant==0.5.10 +colorama==0.4.3 +colorlog==4.0.2 +configparser==3.5.3 +coverage==5.2.1 +croniter==0.3.34 +cryptography==3.0 +cx-Oracle==8.0.0 +datadog==0.38.0 +decorator==4.4.2 +defusedxml==0.6.0 +dill==0.3.2 +distlib==0.3.1 +dnspython==1.16.0 +docker-pycreds==0.4.0 +docker==3.7.3 +docopt==0.6.2 +docutils==0.16 +ecdsa==0.14.1 +elasticsearch-dsl==5.4.0 +elasticsearch==5.5.3 +email-validator==1.1.1 +entrypoints==0.3 +execnet==1.7.1 +fastavro==0.24.1 +filelock==3.0.12 +flake8-colors==0.1.6 +flake8==3.8.3 +flaky==3.7.0 +flask-swagger==0.2.14 +flower==0.9.5 +freezegun==0.3.15 +fsspec==0.8.0 +funcsigs==1.0.2 +future-fstrings==1.2.0 +future==0.18.2 +gcsfs==0.6.2 +gitdb==4.0.5 +google-api-core==1.22.1 +google-api-python-client==1.10.0 +google-auth-httplib2==0.0.4 +google-auth-oauthlib==0.4.1 +google-auth==1.20.1 +google-cloud-bigquery==1.26.1 +google-cloud-bigtable==1.4.0 +google-cloud-container==1.0.1 +google-cloud-core==1.4.1 +google-cloud-dlp==1.0.0 +google-cloud-language==1.3.0 +google-cloud-secret-manager==1.0.0 +google-cloud-spanner==1.17.1 +google-cloud-speech==1.3.2 +google-cloud-storage==1.30.0 +google-cloud-texttospeech==1.0.1 +google-cloud-translate==1.7.0 +google-cloud-videointelligence==1.15.0 +google-cloud-vision==1.0.0 +google-crc32c==0.1.0 +google-resumable-media==0.7.1 +googleapis-common-protos==1.52.0 +graphviz==0.14.1 +grpc-google-iam-v1==0.12.3 +grpcio-gcp==0.2.2 +grpcio==1.31.0 +gunicorn==20.0.4 +hdfs==2.5.8 +hmsclient==0.1.1 +httplib2==0.18.1 +humanize==2.6.0 +hvac==0.10.5 +identify==1.4.28 +idna==2.10 +imagesize==1.2.0 +importlib-metadata==1.7.0 +inflection==0.5.0 +ipdb==0.13.3 +ipython-genutils==0.2.0 +ipython==7.17.0 +iso8601==0.1.12 +isodate==0.6.0 +itsdangerous==1.1.0 +jedi==0.17.2 +jira==2.0.0 +jmespath==0.10.0 +json-merge-patch==0.2 +jsondiff==1.1.2 +jsonpatch==1.26 +jsonpickle==1.4.1 +jsonpointer==2.0 +jsonschema==3.2.0 +junit-xml==1.9 +jupyter-client==6.1.6 +jupyter-core==4.6.3 +kombu==4.6.11 +kubernetes==11.0.0 +lazy-object-proxy==1.5.1 +ldap3==2.8 +lockfile==0.12.2 +marshmallow-enum==1.5.1 +marshmallow-sqlalchemy==0.23.1 +marshmallow==2.21.0 +mccabe==0.6.1 +mock==4.0.2 +mongomock==3.20.0 +more-itertools==8.4.0 +moto==1.3.14 +msal-extensions==0.2.2 +msal==1.5.0 +msrest==0.6.18 +msrestazure==0.6.4 +multi-key-dict==2.0.3 +mypy-extensions==0.4.3 +mypy==0.720 +mysqlclient==1.3.14 +natsort==7.0.1 +nbclient==0.4.1 +nbformat==5.0.7 +nest-asyncio==1.4.0 +networkx==2.4 +nodeenv==1.4.0 +nteract-scrapbook==0.4.1 +ntlm-auth==1.5.0 +numpy==1.19.1 +oauthlib==3.1.0 +oscrypto==1.2.1 +packaging==20.4 +pandas-gbq==0.13.2 +pandas==1.1.0 +papermill==2.1.3 +parameterized==0.7.4 +paramiko==2.7.1 +parso==0.7.1 +pathspec==0.8.0 +pbr==5.4.5 +pendulum==1.4.4 +pexpect==4.8.0 +pickleshare==0.7.5 +pinotdb==0.1.1 +pluggy==0.13.1 +portalocker==1.7.1 +pre-commit==2.6.0 +presto-python-client==0.7.0 +prison==0.1.3 +prometheus-client==0.8.0 +prompt-toolkit==3.0.6 +protobuf==3.13.0 +psutil==5.7.2 +psycopg2-binary==2.8.5 +ptyprocess==0.6.0 +py==1.9.0 +pyOpenSSL==19.1.0 +pyarrow==0.17.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pycodestyle==2.6.0 +pycparser==2.20 +pycryptodomex==3.9.8 +pydata-google-auth==1.1.0 +pydruid==0.5.8 +pyflakes==2.2.0 +pykerberos==1.2.1 +pymongo==3.10.1 +pymssql==2.1.4 +pyparsing==2.4.7 +pyrsistent==0.16.0 +pysftp==0.2.9 +pytest-cov==2.10.1 +pytest-forked==1.3.0 +pytest-instafail==0.4.2 +pytest-rerunfailures==9.0 +pytest-timeout==1.4.2 +pytest-timeouts==1.2.1 +pytest-xdist==2.0.0 +pytest==5.4.3 +python-daemon==2.2.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-http-client==3.2.7 +python-jenkins==1.7.0 +python-jose==3.2.0 +python-nvd3==0.15.0 +python-slugify==4.0.1 +python3-openid==3.2.0 +pytz==2020.1 +pytzdata==2020.1 +pywinrm==0.4.1 +pyzmq==19.0.2 +qds-sdk==1.16.0 +redis==3.5.3 +regex==2020.7.14 +requests-futures==0.9.4 +requests-kerberos==0.12.0 +requests-mock==1.8.0 +requests-ntlm==1.1.0 +requests-oauthlib==1.3.0 +requests-toolbelt==0.9.1 +requests==2.24.0 +responses==0.10.16 +rsa==4.6 +s3transfer==0.3.3 +sasl==0.2.1 +sendgrid==5.6.0 +sentinels==1.0.0 +sentry-sdk==0.16.5 +setproctitle==1.1.10 +six==1.15.0 +slackclient==1.3.2 +smmap==3.0.4 +snowballstemmer==2.0.0 +snowflake-connector-python==2.2.10 +snowflake-sqlalchemy==1.2.3 +soupsieve==2.0.1 +sphinx-argparse==0.2.5 +sphinx-autoapi==1.0.0 +sphinx-copybutton==0.3.0 +sphinx-jinja==1.1.1 +sphinx-rtd-theme==0.5.0 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-dotnetdomain==0.4 +sphinxcontrib-golangdomain==0.2.0.dev0 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-httpdomain==1.7.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +sshpubkeys==3.1.0 +sshtunnel==0.1.5 +tabulate==0.8.7 +tenacity==4.12.0 +text-unidecode==1.3 +textwrap3==0.9.2 +thrift-sasl==0.4.2 +thrift==0.13.0 +toml==0.10.1 +tornado==5.1.1 +tqdm==4.48.2 +traitlets==4.3.3 +typed-ast==1.4.1 +typing-extensions==3.7.4.2 +tzlocal==1.5.1 +unicodecsv==0.14.1 +uritemplate==3.0.1 +urllib3==1.25.10 +vertica-python==0.11.0 +vine==1.3.0 +virtualenv==20.0.31 +wcwidth==0.2.5 +websocket-client==0.57.0 +wrapt==1.12.1 +xmltodict==0.12.0 +yamllint==1.24.2 +zdesk==2.7.1 +zipp==3.1.0 +zope.deprecation==4.4.0