Rework sources to be dbt models rather than manually created

.github/workflows/ci_test_package.yml

@@ -115,7 +115,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
 
       - name: Install Python packages
-        run: python -m pip install dbt-snowflake~=1.1.0 sqlfluff-templater-dbt
+        run: python -m pip install dbt-snowflake~=1.2.0 sqlfluff-templater-dbt
 
       - name: Test database connection
         run: dbt debug

.github/workflows/main_test_package.yml

@@ -98,7 +98,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Install Python packages
-        run: python -m pip install dbt-snowflake~=1.1.0 sqlfluff-templater-dbt
+        run: python -m pip install dbt-snowflake~=1.2.0 sqlfluff-templater-dbt
 
       - name: Test database connection
         run: dbt debug

.gitignore

@@ -11,3 +11,4 @@ Pipfile.lock
 env.sh
 
 .python_version
+.vscode

README.md

@@ -42,20 +42,18 @@ packages:
 
 ## Configuration
 
-The following configuration can be used to specify where the raw data is uploaded, and where the dbt models are created:
+The following configuration can be used to specify where the raw (sources) data is uploaded, and where the dbt models are created:
 
 ```yml
-vars:
-  dbt_artifacts_database: your_db # optional, default is your target database
-  dbt_artifacts_schema: your_schema # optional, default is your target schema
-  dbt_artifacts_create_schema: true|false # optional, set to false if you don't have privileges to create schema, default is true
-
 models:
   ...
   dbt_artifacts:
     +schema: your_destination_schema # optional, default is your target database
     staging:
       +schema: your_destination_schema # optional, default is your target schema
+    sources:
+        +database: your_sources_database # optional, default is your target database
+        +schema: your sources_database # optional, default is your target schema
   ...
 ```
 

dbt_project.yml

@@ -1,7 +1,7 @@
 name: 'dbt_artifacts'
 version: '1.2.0'
 config-version: 2
-require-dbt-version: ">=1.0.0"
+require-dbt-version: ">=1.2.0"
 profile: "dbt_artifacts"
 
 clean-targets: # folders to be removed by `dbt clean`
@@ -13,3 +13,6 @@ models:
   dbt_artifacts:
     +materialized: view
     +file_format: delta
+    sources:
+      +materialized: incremental
+      +on_schema_change: append_new_columns