Merge pull request #2594 from brunomurino/issue-2533

added option --adapter to dbt init, to create sample profiles.yml bas…

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
+dbt_env/
 build/
 develop-eggs/
 dist/
@@ -83,3 +84,11 @@ target/
 
 # pycharm
 .idea/
+
+# AWS credentials
+.aws/
+
+.DS_Store
+
+# vscode
+.vscode/

CHANGELOG.md

@@ -27,6 +27,7 @@ Contributors:
 - Previously, dbt put macros from all installed plugins into the namespace. This version of dbt will not include adapter plugin macros unless they are from the currently-in-use adapter or one of its dependencies [#2590](https://github.com/fishtown-analytics/dbt/pull/2590)
 
 ### Features
+- Added option "--adapter" to `dbt init` to create a sample `profiles.yml` based on the chosen adapter ([#2533](https://github.com/fishtown-analytics/dbt/issues/2533), [#2594](https://github.com/fishtown-analytics/dbt/pull/2594))
 - Added support for Snowflake query tags at the connection and model level ([#1030](https://github.com/fishtown-analytics/dbt/issues/1030), [#2555](https://github.com/fishtown-analytics/dbt/pull/2555/))
 - Added new node selector methods (`config`, `test_type`, `test_name`, `package`) ([#2425](https://github.com/fishtown-analytics/dbt/issues/2425), [#2629](https://github.com/fishtown-analytics/dbt/pull/2629))
 - Added option to specify profile when connecting to Redshift via IAM ([#2437](https://github.com/fishtown-analytics/dbt/issues/2437), [#2581](https://github.com/fishtown-analytics/dbt/pull/2581))
@@ -38,7 +39,7 @@ Contributors:
 - Compile assets as part of docs generate ([#2072](https://github.com/fishtown-analytics/dbt/issues/2072), [#2623](https://github.com/fishtown-analytics/dbt/pull/2623))
 
 Contributors:
-- [@brunomurino](https://github.com/brunomurino) ([#2437](https://github.com/fishtown-analytics/dbt/pull/2581))
+- [@brunomurino](https://github.com/brunomurino) ([#2581](https://github.com/fishtown-analytics/dbt/pull/2581), [#2594](https://github.com/fishtown-analytics/dbt/pull/2594))
 - [@DrMcTaco](https://github.com/DrMcTaco) ([#1030](https://github.com/fishtown-analytics/dbt/issues/1030)),[#2555](https://github.com/fishtown-analytics/dbt/pull/2555/))
 - [@kning](https://github.com/kning) ([#2627](https://github.com/fishtown-analytics/dbt/pull/2627))
 - [@azhard](https://github.com/azhard) ([#2588](https://github.com/fishtown-analytics/dbt/pull/2588))

core/dbt/main.py

@@ -364,6 +364,14 @@ def _build_init_subparser(subparsers, base_subparser):
         Name of the new project
         ''',
     )
+    sub.add_argument(
+        '--adapter',
+        default='redshift',
+        type=str,
+        help='''
+        Write sample profiles.yml for which adapter
+        ''',
+    )
     sub.set_defaults(cls=init_task.InitTask, which='init', rpc_method=None)
     return sub
 

core/dbt/task/init.py

@@ -1,8 +1,11 @@
 import os
+import shutil
 
 import dbt.config
 import dbt.clients.git
 import dbt.clients.system
+from dbt.adapters.factory import load_plugin, get_include_paths
+from dbt.exceptions import RuntimeException
 
 from dbt.logger import GLOBAL_LOGGER as logger
 
@@ -11,12 +14,12 @@
 STARTER_REPO = 'https://github.com/fishtown-analytics/dbt-starter-project.git'
 STARTER_BRANCH = 'dbt-yml-config-version-2'
 DOCS_URL = 'https://docs.getdbt.com/docs/configure-your-profile'
-SAMPLE_PROFILES_YML_FILE = 'https://docs.getdbt.com/docs/profile'  # noqa
 
 ON_COMPLETE_MESSAGE = """
 Your new dbt project "{project_name}" was created! If this is your first time
-using dbt, you'll need to set up your profiles.yml file -- this file will
-tell dbt how to connect to your database. You can find this file by running:
+using dbt, you'll need to set up your profiles.yml file (we've created a sample
+file for you to connect to {sample_adapter}) -- this file will tell dbt how
+to connect to your database. You can find this file by running:
 
   {open_cmd} {profiles_path}
 
@@ -32,34 +35,6 @@
 """
 
 
-STARTER_PROFILE = """
-# For more information on how to configure this file, please see:
-# {profiles_sample}
-
-default:
-  outputs:
-    dev:
-      type: redshift
-      threads: 1
-      host: 127.0.0.1
-      port: 5439
-      user: alice
-      pass: pa55word
-      dbname: warehouse
-      schema: dbt_alice
-    prod:
-      type: redshift
-      threads: 1
-      host: 127.0.0.1
-      port: 5439
-      user: alice
-      pass: pa55word
-      dbname: warehouse
-      schema: analytics
-  target: dev
-""".format(profiles_sample=SAMPLE_PROFILES_YML_FILE)
-
-
 class InitTask(BaseTask):
     def clone_starter_repo(self, project_name):
         dbt.clients.git.clone(
@@ -76,40 +51,54 @@ def create_profiles_dir(self, profiles_dir):
             return True
         return False
 
-    def create_profiles_file(self, profiles_file):
+    def create_profiles_file(self, profiles_file, sample_adapter):
+        # Line below raises an exception if the specified adapter is not found
+        load_plugin(sample_adapter)
+        adapter_path = get_include_paths(sample_adapter)[0]
+        sample_profiles_path = adapter_path / 'sample_profiles.yml'
+
+        if not sample_profiles_path.exists():
+            raise RuntimeException(f'No sample profile for {sample_adapter}')
+
         if not os.path.exists(profiles_file):
-            dbt.clients.system.make_file(profiles_file, STARTER_PROFILE)
+            shutil.copyfile(sample_profiles_path, profiles_file)
             return True
+
         return False
 
-    def get_addendum(self, project_name, profiles_path):
+    def get_addendum(self, project_name, profiles_path, sample_adapter):
         open_cmd = dbt.clients.system.open_dir_cmd()
 
         return ON_COMPLETE_MESSAGE.format(
             open_cmd=open_cmd,
             project_name=project_name,
+            sample_adapter=sample_adapter,
             profiles_path=profiles_path,
             docs_url=DOCS_URL
         )
 
     def run(self):
         project_dir = self.args.project_name
+        sample_adapter = self.args.adapter
 
         profiles_dir = dbt.config.PROFILES_DIR
         profiles_file = os.path.join(profiles_dir, 'profiles.yml')
 
-        self.create_profiles_dir(profiles_dir)
-        self.create_profiles_file(profiles_file)
-
         msg = "Creating dbt configuration folder at {}"
         logger.info(msg.format(profiles_dir))
 
+        msg = "With sample profiles.yml for {}"
+        logger.info(msg.format(sample_adapter))
+
+        self.create_profiles_dir(profiles_dir)
+        self.create_profiles_file(profiles_file, sample_adapter)
+
         if os.path.exists(project_dir):
             raise RuntimeError("directory {} already exists!".format(
                 project_dir
             ))
 
         self.clone_starter_repo(project_dir)
 
-        addendum = self.get_addendum(project_dir, profiles_dir)
+        addendum = self.get_addendum(project_dir, profiles_dir, sample_adapter)
         logger.info(addendum)

plugins/bigquery/dbt/include/bigquery/sample_profiles.yml

@@ -0,0 +1,26 @@
+default:
+  outputs:
+
+    dev:
+      type: bigquery
+      method: oauth
+      project: [GCP project id]
+      dataset: [the name of your dbt dataset] # You can also use "schema" here
+      threads: [1 or more]
+      timeout_seconds: 300
+      location: US # Optional, one of US or EU
+      priority: interactive
+      retries: 1
+
+    prod:
+      type: bigquery
+      method: service-account
+      project: [GCP project id]
+      dataset: [the name of your dbt dataset]
+      threads: [1 or more]
+      keyfile: [/path/to/bigquery/keyfile.json]
+      timeout_seconds: 300
+      priority: interactive
+      retries: 1
+
+  target: dev

plugins/bigquery/setup.py

@@ -34,6 +34,7 @@
     package_data={
         'dbt': [
             'include/bigquery/dbt_project.yml',
+            'include/bigquery/sample_profiles.yml',
             'include/bigquery/macros/*.sql',
             'include/bigquery/macros/**/*.sql',
         ]

plugins/postgres/dbt/include/postgres/sample_profiles.yml

@@ -0,0 +1,24 @@
+default:
+  outputs:
+
+    dev:
+      type: postgres
+      threads: [1 or more]
+      host: [host]
+      port: [port]
+      user: [dev_username]
+      pass: [dev_password]
+      dbname: [dbname]
+      schema: [dev_schema]
+
+    prod:
+      type: postgres
+      threads: [1 or more]
+      host: [host]
+      port: [port]
+      user: [prod_username]
+      pass: [prod_password]
+      dbname: [dbname]
+      schema: [prod_schema]
+
+  target: dev

plugins/postgres/setup.py

@@ -57,6 +57,7 @@ def _dbt_psycopg2_name():
     package_data={
         'dbt': [
             'include/postgres/dbt_project.yml',
+            'include/postgres/sample_profiles.yml',
             'include/postgres/macros/*.sql',
             'include/postgres/macros/**/*.sql',
         ]

plugins/redshift/dbt/include/redshift/sample_profiles.yml

@@ -0,0 +1,25 @@
+default:
+  outputs:
+
+    dev:
+      type: redshift
+      threads: [1 or more]
+      host: [host]
+      port: [port]
+      user: [dev_username]
+      pass: [dev_password]
+      dbname: [dbname]
+      schema: [dev_schema]
+
+    prod:
+      type: redshift
+      method: iam
+      cluster_id: [cluster_id]
+      threads: [1 or more]
+      host: [host]
+      port: [port]
+      user: [prod_user]
+      dbname: [dbname]
+      schema: [prod_schema]
+
+  target: dev

plugins/redshift/setup.py

@@ -34,6 +34,7 @@
     package_data={
         'dbt': [
             'include/redshift/dbt_project.yml',
+            'include/redshift/sample_profiles.yml',
             'include/redshift/macros/*.sql',
             'include/redshift/macros/**/*.sql',
         ]

plugins/snowflake/dbt/include/snowflake/sample_profiles.yml

@@ -0,0 +1,29 @@
+default:
+  outputs:
+
+    dev: # User-Password config
+      type: snowflake
+      account: [account id + region (if applicable)]
+      user: [username]
+      password: [password]
+      role: [user role]
+      database: [database name]
+      warehouse: [warehouse name]
+      schema: [dbt schema]
+      threads: [1 or more]
+      client_session_keep_alive: False
+
+    prod: # Keypair config
+      type: snowflake
+      account: [account id + region (if applicable)]
+      user: [username]
+      role: [user role]
+      private_key_path: [path/to/private.key]
+      private_key_passphrase: [passphrase for the private key, if key is encrypted]
+      database: [database name]
+      warehouse: [warehouse name]
+      schema: [dbt schema]
+      threads: [1 or more]
+      client_session_keep_alive: False
+
+  target: dev

plugins/snowflake/setup.py

@@ -34,6 +34,7 @@
     package_data={
         'dbt': [
             'include/snowflake/dbt_project.yml',
+            'include/snowflake/sample_profiles.yml',
             'include/snowflake/macros/*.sql',
             'include/snowflake/macros/**/*.sql',
         ]

test/integration/040_init_test/test_init.py

@@ -28,7 +28,7 @@ def models(self):
     @use_profile('postgres')
     def test_postgres_init_task(self):
         project_name = self.get_project_name()
-        self.run_dbt(['init', project_name])
+        self.run_dbt(['init', project_name, '--adapter', 'postgres'])
 
         assert os.path.exists(project_name)
         project_file = os.path.join(project_name, 'dbt_project.yml')