diff --git a/.gitignore b/.gitignore index d1025d4ef4c..e3ebed6763f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ # Distribution / packaging .Python env/ +dbt_env/ build/ develop-eggs/ dist/ @@ -83,3 +84,11 @@ target/ # pycharm .idea/ + +# AWS credentials +.aws/ + +.DS_Store + +# vscode +.vscode/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 897deb3d93f..be93433e3e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ Contributors: - Previously, dbt put macros from all installed plugins into the namespace. This version of dbt will not include adapter plugin macros unless they are from the currently-in-use adapter or one of its dependencies [#2590](https://github.com/fishtown-analytics/dbt/pull/2590) ### Features +- Added option "--adapter" to `dbt init` to create a sample `profiles.yml` based on the chosen adapter ([#2533](https://github.com/fishtown-analytics/dbt/issues/2533), [#2594](https://github.com/fishtown-analytics/dbt/pull/2594)) - Added support for Snowflake query tags at the connection and model level ([#1030](https://github.com/fishtown-analytics/dbt/issues/1030), [#2555](https://github.com/fishtown-analytics/dbt/pull/2555/)) - Added new node selector methods (`config`, `test_type`, `test_name`, `package`) ([#2425](https://github.com/fishtown-analytics/dbt/issues/2425), [#2629](https://github.com/fishtown-analytics/dbt/pull/2629)) - Added option to specify profile when connecting to Redshift via IAM ([#2437](https://github.com/fishtown-analytics/dbt/issues/2437), [#2581](https://github.com/fishtown-analytics/dbt/pull/2581)) @@ -38,7 +39,7 @@ Contributors: - Compile assets as part of docs generate ([#2072](https://github.com/fishtown-analytics/dbt/issues/2072), [#2623](https://github.com/fishtown-analytics/dbt/pull/2623)) Contributors: -- [@brunomurino](https://github.com/brunomurino) ([#2437](https://github.com/fishtown-analytics/dbt/pull/2581)) +- [@brunomurino](https://github.com/brunomurino) ([#2581](https://github.com/fishtown-analytics/dbt/pull/2581), [#2594](https://github.com/fishtown-analytics/dbt/pull/2594)) - [@DrMcTaco](https://github.com/DrMcTaco) ([#1030](https://github.com/fishtown-analytics/dbt/issues/1030)),[#2555](https://github.com/fishtown-analytics/dbt/pull/2555/)) - [@kning](https://github.com/kning) ([#2627](https://github.com/fishtown-analytics/dbt/pull/2627)) - [@azhard](https://github.com/azhard) ([#2588](https://github.com/fishtown-analytics/dbt/pull/2588)) diff --git a/core/dbt/main.py b/core/dbt/main.py index d4344f1cbed..ba8d3136518 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -364,6 +364,14 @@ def _build_init_subparser(subparsers, base_subparser): Name of the new project ''', ) + sub.add_argument( + '--adapter', + default='redshift', + type=str, + help=''' + Write sample profiles.yml for which adapter + ''', + ) sub.set_defaults(cls=init_task.InitTask, which='init', rpc_method=None) return sub diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index df01d8c94d9..98cad30a25f 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -1,8 +1,11 @@ import os +import shutil import dbt.config import dbt.clients.git import dbt.clients.system +from dbt.adapters.factory import load_plugin, get_include_paths +from dbt.exceptions import RuntimeException from dbt.logger import GLOBAL_LOGGER as logger @@ -11,12 +14,12 @@ STARTER_REPO = 'https://github.com/fishtown-analytics/dbt-starter-project.git' STARTER_BRANCH = 'dbt-yml-config-version-2' DOCS_URL = 'https://docs.getdbt.com/docs/configure-your-profile' -SAMPLE_PROFILES_YML_FILE = 'https://docs.getdbt.com/docs/profile' # noqa ON_COMPLETE_MESSAGE = """ Your new dbt project "{project_name}" was created! If this is your first time -using dbt, you'll need to set up your profiles.yml file -- this file will -tell dbt how to connect to your database. You can find this file by running: +using dbt, you'll need to set up your profiles.yml file (we've created a sample +file for you to connect to {sample_adapter}) -- this file will tell dbt how +to connect to your database. You can find this file by running: {open_cmd} {profiles_path} @@ -32,34 +35,6 @@ """ -STARTER_PROFILE = """ -# For more information on how to configure this file, please see: -# {profiles_sample} - -default: - outputs: - dev: - type: redshift - threads: 1 - host: 127.0.0.1 - port: 5439 - user: alice - pass: pa55word - dbname: warehouse - schema: dbt_alice - prod: - type: redshift - threads: 1 - host: 127.0.0.1 - port: 5439 - user: alice - pass: pa55word - dbname: warehouse - schema: analytics - target: dev -""".format(profiles_sample=SAMPLE_PROFILES_YML_FILE) - - class InitTask(BaseTask): def clone_starter_repo(self, project_name): dbt.clients.git.clone( @@ -76,34 +51,48 @@ def create_profiles_dir(self, profiles_dir): return True return False - def create_profiles_file(self, profiles_file): + def create_profiles_file(self, profiles_file, sample_adapter): + # Line below raises an exception if the specified adapter is not found + load_plugin(sample_adapter) + adapter_path = get_include_paths(sample_adapter)[0] + sample_profiles_path = adapter_path / 'sample_profiles.yml' + + if not sample_profiles_path.exists(): + raise RuntimeException(f'No sample profile for {sample_adapter}') + if not os.path.exists(profiles_file): - dbt.clients.system.make_file(profiles_file, STARTER_PROFILE) + shutil.copyfile(sample_profiles_path, profiles_file) return True + return False - def get_addendum(self, project_name, profiles_path): + def get_addendum(self, project_name, profiles_path, sample_adapter): open_cmd = dbt.clients.system.open_dir_cmd() return ON_COMPLETE_MESSAGE.format( open_cmd=open_cmd, project_name=project_name, + sample_adapter=sample_adapter, profiles_path=profiles_path, docs_url=DOCS_URL ) def run(self): project_dir = self.args.project_name + sample_adapter = self.args.adapter profiles_dir = dbt.config.PROFILES_DIR profiles_file = os.path.join(profiles_dir, 'profiles.yml') - self.create_profiles_dir(profiles_dir) - self.create_profiles_file(profiles_file) - msg = "Creating dbt configuration folder at {}" logger.info(msg.format(profiles_dir)) + msg = "With sample profiles.yml for {}" + logger.info(msg.format(sample_adapter)) + + self.create_profiles_dir(profiles_dir) + self.create_profiles_file(profiles_file, sample_adapter) + if os.path.exists(project_dir): raise RuntimeError("directory {} already exists!".format( project_dir @@ -111,5 +100,5 @@ def run(self): self.clone_starter_repo(project_dir) - addendum = self.get_addendum(project_dir, profiles_dir) + addendum = self.get_addendum(project_dir, profiles_dir, sample_adapter) logger.info(addendum) diff --git a/plugins/bigquery/dbt/include/bigquery/sample_profiles.yml b/plugins/bigquery/dbt/include/bigquery/sample_profiles.yml new file mode 100644 index 00000000000..36f47fdbcad --- /dev/null +++ b/plugins/bigquery/dbt/include/bigquery/sample_profiles.yml @@ -0,0 +1,26 @@ +default: + outputs: + + dev: + type: bigquery + method: oauth + project: [GCP project id] + dataset: [the name of your dbt dataset] # You can also use "schema" here + threads: [1 or more] + timeout_seconds: 300 + location: US # Optional, one of US or EU + priority: interactive + retries: 1 + + prod: + type: bigquery + method: service-account + project: [GCP project id] + dataset: [the name of your dbt dataset] + threads: [1 or more] + keyfile: [/path/to/bigquery/keyfile.json] + timeout_seconds: 300 + priority: interactive + retries: 1 + + target: dev \ No newline at end of file diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py index b51a0372434..ed01fd9b4ff 100644 --- a/plugins/bigquery/setup.py +++ b/plugins/bigquery/setup.py @@ -34,6 +34,7 @@ package_data={ 'dbt': [ 'include/bigquery/dbt_project.yml', + 'include/bigquery/sample_profiles.yml', 'include/bigquery/macros/*.sql', 'include/bigquery/macros/**/*.sql', ] diff --git a/plugins/postgres/dbt/include/postgres/sample_profiles.yml b/plugins/postgres/dbt/include/postgres/sample_profiles.yml new file mode 100644 index 00000000000..567f3912893 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/sample_profiles.yml @@ -0,0 +1,24 @@ +default: + outputs: + + dev: + type: postgres + threads: [1 or more] + host: [host] + port: [port] + user: [dev_username] + pass: [dev_password] + dbname: [dbname] + schema: [dev_schema] + + prod: + type: postgres + threads: [1 or more] + host: [host] + port: [port] + user: [prod_username] + pass: [prod_password] + dbname: [dbname] + schema: [prod_schema] + + target: dev diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py index 19969cf2a5b..337e5e531ef 100644 --- a/plugins/postgres/setup.py +++ b/plugins/postgres/setup.py @@ -57,6 +57,7 @@ def _dbt_psycopg2_name(): package_data={ 'dbt': [ 'include/postgres/dbt_project.yml', + 'include/postgres/sample_profiles.yml', 'include/postgres/macros/*.sql', 'include/postgres/macros/**/*.sql', ] diff --git a/plugins/redshift/dbt/include/redshift/sample_profiles.yml b/plugins/redshift/dbt/include/redshift/sample_profiles.yml new file mode 100644 index 00000000000..46cd979344c --- /dev/null +++ b/plugins/redshift/dbt/include/redshift/sample_profiles.yml @@ -0,0 +1,25 @@ +default: + outputs: + + dev: + type: redshift + threads: [1 or more] + host: [host] + port: [port] + user: [dev_username] + pass: [dev_password] + dbname: [dbname] + schema: [dev_schema] + + prod: + type: redshift + method: iam + cluster_id: [cluster_id] + threads: [1 or more] + host: [host] + port: [port] + user: [prod_user] + dbname: [dbname] + schema: [prod_schema] + + target: dev diff --git a/plugins/redshift/setup.py b/plugins/redshift/setup.py index c5462ddbc28..bb12f0f68ae 100644 --- a/plugins/redshift/setup.py +++ b/plugins/redshift/setup.py @@ -34,6 +34,7 @@ package_data={ 'dbt': [ 'include/redshift/dbt_project.yml', + 'include/redshift/sample_profiles.yml', 'include/redshift/macros/*.sql', 'include/redshift/macros/**/*.sql', ] diff --git a/plugins/snowflake/dbt/include/snowflake/sample_profiles.yml b/plugins/snowflake/dbt/include/snowflake/sample_profiles.yml new file mode 100644 index 00000000000..22dca47c586 --- /dev/null +++ b/plugins/snowflake/dbt/include/snowflake/sample_profiles.yml @@ -0,0 +1,29 @@ +default: + outputs: + + dev: # User-Password config + type: snowflake + account: [account id + region (if applicable)] + user: [username] + password: [password] + role: [user role] + database: [database name] + warehouse: [warehouse name] + schema: [dbt schema] + threads: [1 or more] + client_session_keep_alive: False + + prod: # Keypair config + type: snowflake + account: [account id + region (if applicable)] + user: [username] + role: [user role] + private_key_path: [path/to/private.key] + private_key_passphrase: [passphrase for the private key, if key is encrypted] + database: [database name] + warehouse: [warehouse name] + schema: [dbt schema] + threads: [1 or more] + client_session_keep_alive: False + + target: dev \ No newline at end of file diff --git a/plugins/snowflake/setup.py b/plugins/snowflake/setup.py index 0dfacb11289..a9b2e81bb38 100644 --- a/plugins/snowflake/setup.py +++ b/plugins/snowflake/setup.py @@ -34,6 +34,7 @@ package_data={ 'dbt': [ 'include/snowflake/dbt_project.yml', + 'include/snowflake/sample_profiles.yml', 'include/snowflake/macros/*.sql', 'include/snowflake/macros/**/*.sql', ] diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index 89828e37bca..c7b5a85363d 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -28,7 +28,7 @@ def models(self): @use_profile('postgres') def test_postgres_init_task(self): project_name = self.get_project_name() - self.run_dbt(['init', project_name]) + self.run_dbt(['init', project_name, '--adapter', 'postgres']) assert os.path.exists(project_name) project_file = os.path.join(project_name, 'dbt_project.yml')