diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..24eb65c
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+workspaces/*
+
+!.gitkeep
diff --git a/config/.env.app b/config/.env.app
new file mode 100644
index 0000000..a564457
--- /dev/null
+++ b/config/.env.app
@@ -0,0 +1,3 @@
+APP_IDENTITY_DOMAIN=${APP_IDENTITY_DOMAIN:-}
+APP_IDENTITY_CLIENT_ID=${APP_IDENTITY_CLIENT_ID:-}
+APP_IDENTITY_CONNECTION=${APP_IDENTITY_CONNECTION:-}
diff --git a/config/.env.catalog b/config/.env.catalog
new file mode 100644
index 0000000..1beaa2a
--- /dev/null
+++ b/config/.env.catalog
@@ -0,0 +1,7 @@
+CATALOG_AUTH_IDPS_PRIMARY_ISSUER_URI=${CATALOG_AUTH_IDPS_PRIMARY_ISSUER_URI:-}
+CATALOG_AUTH_IDPS_PRIMARY_JWK_SET_URI=${CATALOG_AUTH_IDPS_PRIMARY_JWK_SET_URI:-}
+CATALOG_AUTH_IDPS_SECONDARY_ISSUER_URI=${CATALOG_AUTH_IDPS_SECONDARY_ISSUER_URI:-}
+CATALOG_AUTH_IDPS_SECONDARY_JWK_SET_URI=${CATALOG_AUTH_IDPS_SECONDARY_JWK_SET_URI:-}
+
+# Generate a 256-bit (32-char) secret key and set it here. This is used to generate access tokens for the Matatika App and API.
+MATATIKA_AUTH_LOCAL_SECRET=${MATATIKA_AUTH_LOCAL_SECRET:-matatikaauthlocalsecret_changeme}
diff --git a/docker-compose.yml b/docker-compose.yml
index 092bf3d..e6c33a6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -13,6 +13,7 @@ services:
limits:
memory: 3g
tty: true
+ env_file: config/.env.catalog
environment:
# Define DEVELOPMENT mode environments like debug settings etc
- ACTIVE_PROFILES=default,distribution
@@ -25,8 +26,6 @@ services:
- ELASTICSEARCH_HOST=search
- MATATIKA_WORKSPACES_HOME=/workspaces
- MATATIKA_PLUGINS_HOME=file:/plugins
- # Generate a 256-bit (32-char) secret key and set it here. This is used to generate access tokens for the Matatika App and API.
- - MATATIKA_AUTH_LOCAL_SECRET=matatikaauthlocalsecret_changeme
depends_on: {"search-setup": {"condition": "service_completed_successfully"}, "search": {"condition": "service_healthy"}, "db": {"condition": "service_healthy"}}
healthcheck:
test: curl -s http://localhost:8080/api/actuator/health | grep "UP" >/dev/null; if [[ $$? == 0 ]]; then echo 0; else echo 1; fi
@@ -44,6 +43,7 @@ services:
- '3080:80'
- '3443:443'
tty: true
+ env_file: config/.env.app
environment:
# Define DEVELOPMENT mode environments like debug settings etc
- REACT_APP_ENV=distribution
@@ -104,7 +104,6 @@ services:
# Short lived container that creates the certificates
search-createcerts:
- container_name: search-createcerts
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
environment:
- ES_HOME=/usr/share/elasticsearch
@@ -150,7 +149,6 @@ services:
db:
image: postgres:11
- container_name: db
volumes:
- matatika-catalog_postgres:/var/lib/postgresql/data
environment:
diff --git a/workspaces/.gitignore b/workspaces/.gitignore
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/project/analyze/.gitkeep b/workspaces/.gitkeep
similarity index 100%
rename from workspaces/project/analyze/.gitkeep
rename to workspaces/.gitkeep
diff --git a/workspaces/project/.gitignore b/workspaces/project/.gitignore
deleted file mode 100644
index 15e24c3..0000000
--- a/workspaces/project/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-/venv
-/.meltano
-.env
diff --git a/workspaces/project/README.md b/workspaces/project/README.md
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/project/extract/.gitkeep b/workspaces/project/extract/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/project/load/.gitkeep b/workspaces/project/load/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/project/meltano.yml b/workspaces/project/meltano.yml
deleted file mode 100644
index a6051e7..0000000
--- a/workspaces/project/meltano.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-version: 1
-default_environment: dev
-project_id: 8268c1b5-4c4e-43e6-811f-6d9b769d47ee
-environments:
-- name: dev
-- name: staging
-- name: prod
diff --git a/workspaces/project/notebook/.gitkeep b/workspaces/project/notebook/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/project/orchestrate/.gitkeep b/workspaces/project/orchestrate/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/project/output/.gitignore b/workspaces/project/output/.gitignore
deleted file mode 100644
index d6b7ef3..0000000
--- a/workspaces/project/output/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*
-!.gitignore
diff --git a/workspaces/project/requirements.txt b/workspaces/project/requirements.txt
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/project/transform/.gitkeep b/workspaces/project/transform/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/project/workspace.yml b/workspaces/project/workspace.yml
deleted file mode 100644
index 5d0a9a6..0000000
--- a/workspaces/project/workspace.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-version: workspaces/v0.1
-name: project
-app_properties:
- WELCOME_DATASET_ALIAS: welcome
-dataset_paths:
-- analyze/datasets
-- .meltano/analyze/datasets
-channel_paths:
-- analyze/channels
-pipeline_paths:
-- pipelines
-plugin_paths:
-- plugins
diff --git a/workspaces/testrepo-joeiagq/.gitignore b/workspaces/testrepo-joeiagq/.gitignore
deleted file mode 100644
index 15e24c3..0000000
--- a/workspaces/testrepo-joeiagq/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-/venv
-/.meltano
-.env
diff --git a/workspaces/testrepo-joeiagq/README.md b/workspaces/testrepo-joeiagq/README.md
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/testrepo-joeiagq/analyze/.gitkeep b/workspaces/testrepo-joeiagq/analyze/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/testrepo-joeiagq/analyze/channels/Github.yml b/workspaces/testrepo-joeiagq/analyze/channels/Github.yml
deleted file mode 100644
index 390db1a..0000000
--- a/workspaces/testrepo-joeiagq/analyze/channels/Github.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-# This file is managed by the 'analyze-github' file bundle and updated automatically when `meltano upgrade` is run.
-# To prevent any manual changes from being overwritten, remove the file bundle from `meltano.yml` or disable automatic updates:
-# meltano config --plugin-type=files analyze-github set _update analyze/channels/Github.yml false
-
-version: channels/v0.1
-description: Github
-picture: https://app.matatika.com/assets/images/datasource/tap-github.png
\ No newline at end of file
diff --git a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-commits-per-day-last-30-days.yml b/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-commits-per-day-last-30-days.yml
deleted file mode 100644
index dd0f034..0000000
--- a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-commits-per-day-last-30-days.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-# This file is managed by the 'analyze-github' file bundle and updated automatically when `meltano upgrade` is run.
-# To prevent any manual changes from being overwritten, remove the file bundle from `meltano.yml` or disable automatic updates:
-# meltano config --plugin-type=files analyze-github set _update analyze/datasets/tap-github/github-commits-per-day-last-30-days.yml false
-
-version: datasets/v0.2
-source: Github
-title: Commits per Day for the Last 30 Days
-description: |-
- Number of commits per day for the last 30 Days.
-
- #github
-metadata: |-
- {
- "name": "commits",
- "label": "Total Commits per Day for the Last 30 Days",
- "related_table": {
- "columns": [
- {"name": "date_day", "label": "Date", "description": "Date"}
- ],
- "aggregates": [
- {"name": "commit_count", "label": "Commit Count", "description": "Commit Count"}
- ]
- }
- }
-visualisation: |-
- {"chartjs-chart":
- {"chartType": "line",
- "options": {
- "scales": {
- "y": {
- "beginAtZero": true
- }
- }
- }
- }
- }
-query: |-
- with recursive dim_date as (
- select (current_date - interval '30 day')::date as date_day --start
- union all
- select (date_day + INTERVAL '1 day')::date
- from dim_date
- where date_day < (current_date - interval '1 day')::date --end
- )
- select date_day "commits.date_day"
- , count(*) - 1 "commits.commit_count"
- from dim_date dd
- left join commits c on c.commit__author__date::date = dd.date_day
- GROUP BY date_day
- ORDER BY date_day
diff --git a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-commits-per-repo-by-month-last-12-months.yml b/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-commits-per-repo-by-month-last-12-months.yml
deleted file mode 100644
index 59220b3..0000000
--- a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-commits-per-repo-by-month-last-12-months.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-# This file is managed by the 'analyze-github' file bundle and updated automatically when `meltano upgrade` is run.
-# To prevent any manual changes from being overwritten, remove the file bundle from `meltano.yml` or disable automatic updates:
-# meltano config --plugin-type=files analyze-github set _update analyze/datasets/tap-github/github-commits-per-repo-by-month-last-12-months.yml false
-
-version: datasets/v0.2
-source: Github
-title: Commits Per Repo Last 12 Months
-description: |-
- Total commits per repo for the last 12 months
-
- #github
-metadata: |-
- {
- "name": "commits",
- "label": "Commits Per Repo Last 12 Months",
- "related_table": {
- "columns": [
- {"name": "year", "label": "Year", "description": "Year"},
- {"name": "month", "label": "Month", "description": "Month"},
- {"name": "repo", "label": "Repository", "description": "Repository"}
- ],
- "aggregates": [
- {"name": "commit_count", "label": "Commits", "description": "Commits"}
- ]
- }
- }
-visualisation: |-
- {"chartjs-chart":
- {"chartType": "bar",
- "options": {
- "scales": {
- "x": {
- "stacked": true
- },
- "y": {
- "stacked": true,
- "title": {
- "display": true,
- "text": "Spend"
- }
- }
- }
- }
- }
- }
-query: |-
- SELECT
- date_part('year', commit__author__date::DATE) as "commits.year",
- date_part('month', commit__author__date::DATE) as "commits.month",
- repo as "commits.repo",
- COUNT(*) AS "commits.commit_count"
- FROM commits
- WHERE commit__author__date >= CURRENT_DATE - INTERVAL '1 YEAR'
- GROUP BY 1, 2, 3
- ORDER BY 1, 2, 3
diff --git a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-latest-10-issues.yml b/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-latest-10-issues.yml
deleted file mode 100644
index d67adc2..0000000
--- a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-latest-10-issues.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-# This file is managed by the 'analyze-github' file bundle and updated automatically when `meltano upgrade` is run.
-# To prevent any manual changes from being overwritten, remove the file bundle from `meltano.yml` or disable automatic updates:
-# meltano config --plugin-type=files analyze-github set _update analyze/datasets/tap-github/github-latest-10-issues.yml false
-
-version: datasets/v0.2
-source: Github
-title: Latest 10 Issues
-description: |-
- Latest 10 issues.
-
- #github
-metadata: |-
- {
- "name": "issues",
- "label": "Latest 10 Issues",
- "related_table": {
- "columns": [
- {"name": "created_at", "label": "Created At", "description": "Created At"},
- {"name": "state", "label": "State", "description": "State"},
- {"name": "closed_at", "label": "Closed At", "description": "Closed At"},
- {"name": "org", "label": "Organization", "description": "Organization"},
- {"name": "repo", "label": "Repository", "description": "Repository"},
- {"name": "title", "label": "Title", "description": "Title"}
- ]
- }
- }
-visualisation: '{"google-chart": {"chartType": "Table"}}'
-query: |-
- SELECT
- created_at "issues.created_at",
- state "issues.state",
- closed_at "issues.closed_at",
- org "issues.org",
- repo "issues.repo",
- title "issues.title"
- FROM issues
- ORDER BY created_at DESC
- LIMIT 10
diff --git a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-latest-10-prs.yml b/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-latest-10-prs.yml
deleted file mode 100644
index 7d963b0..0000000
--- a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-latest-10-prs.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-# This file is managed by the 'analyze-github' file bundle and updated automatically when `meltano upgrade` is run.
-# To prevent any manual changes from being overwritten, remove the file bundle from `meltano.yml` or disable automatic updates:
-# meltano config --plugin-type=files analyze-github set _update analyze/datasets/tap-github/github-latest-10-prs.yml false
-
-version: datasets/v0.2
-source: Github
-title: Latest 10 Pull Requests
-description: |-
- Latest 10 pull requests.
-
- #github
-metadata: |-
- {
- "name": "pull_requests",
- "label": "Latest 10 Pull Requests",
- "related_table": {
- "columns": [
- {"name": "created_at", "label": "Created At", "description": "Created At"},
- {"name": "state", "label": "State", "description": "State"},
- {"name": "closed_at", "label": "Closed At", "description": "Closed At"},
- {"name": "org", "label": "Organization", "description": "Organization"},
- {"name": "repo", "label": "Repository", "description": "Repository"},
- {"name": "title", "label": "Title", "description": "Title"}
- ]
- }
- }
-visualisation: '{"google-chart": {"chartType": "Table"}}'
-query: |-
- SELECT
- created_at "pull_requests.created_at",
- state "pull_requests.state",
- closed_at "pull_requests.closed_at",
- org "pull_requests.org",
- repo "pull_requests.repo",
- title "pull_requests.title"
- FROM pull_requests
- ORDER BY created_at DESC
- LIMIT 10
diff --git a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-total-commits-by-users-last-year.yml b/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-total-commits-by-users-last-year.yml
deleted file mode 100644
index 927305c..0000000
--- a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-total-commits-by-users-last-year.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-# This file is managed by the 'analyze-github' file bundle and updated automatically when `meltano upgrade` is run.
-# To prevent any manual changes from being overwritten, remove the file bundle from `meltano.yml` or disable automatic updates:
-# meltano config --plugin-type=files analyze-github set _update analyze/datasets/tap-github/github-total-commits-by-users-last-year.yml false
-
-version: datasets/v0.2
-source: Github
-title: Total Commits per User for the Last Year
-description: |-
- Number of commits by each user for the last year.
-
- #github
-metadata: |-
- {
- "name": "commits",
- "label": "Total Commits per User for the Last Year",
- "related_table": {
- "columns": [
- {"name": "committer_name", "label": "Committer Name", "description": "Committer Name"}
- ],
- "aggregates": [
- {"name": "commit_count", "label": "Commit Count", "description": "Commit Count"}
- ]
- }
- }
-visualisation: |-
- {"chartjs-chart":
- {"chartType": "bar",
- "options": {
- "scales": {
- "y": {
- "beginAtZero": true
- }
- }
- }
- }
- }
-query: |-
- SELECT
- COUNT(*) "commits.commit_count",
- commit__committer__name "commits.committer_name"
- FROM commits
- GROUP BY commit__committer__name
- ORDER BY "commits.commit_count" DESC
diff --git a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-total-commits-per-repo-last-year-to-date.yml b/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-total-commits-per-repo-last-year-to-date.yml
deleted file mode 100644
index 4bbaada..0000000
--- a/workspaces/testrepo-joeiagq/analyze/datasets/tap-github/github-total-commits-per-repo-last-year-to-date.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-# This file is managed by the 'analyze-github' file bundle and updated automatically when `meltano upgrade` is run.
-# To prevent any manual changes from being overwritten, remove the file bundle from `meltano.yml` or disable automatic updates:
-# meltano config --plugin-type=files analyze-github set _update analyze/datasets/tap-github/github-total-commits-per-repo-last-year-to-date.yml false
-
-version: datasets/v0.2
-source: Github
-title: Total Commits per Repo Last Year to Date
-description: |-
- The number of commits per repo for the last year to date
-
- #github
-metadata: |-
- {
- "name": "commits",
- "label": "Total Commits per User for the Last Year",
- "related_table": {
- "columns": [
- {"name": "org_and_repo", "label": "Repository", "description": "Repository"}
- ],
- "aggregates": [
- {"name": "commit_count", "label": "Commit Count", "description": "Commit Count"}
- ]
- }
- }
-visualisation: |-
- {"chartjs-chart":
- {"chartType": "bar",
- "options": {
- "scales": {
- "y": {
- "beginAtZero": true
- }
- }
- }
- }
- }
-query: |-
- SELECT
- count(*) "commits.commit_count"
- , concat(org, '/', repo) "commits.org_and_repo"
- from commits
- where commit__author__date >= current_date - interval '1 year'
- group by org, repo
- order by "commits.commit_count" DESC
diff --git a/workspaces/testrepo-joeiagq/analyze/datasets/welcome.yml b/workspaces/testrepo-joeiagq/analyze/datasets/welcome.yml
deleted file mode 100644
index 4e0057a..0000000
--- a/workspaces/testrepo-joeiagq/analyze/datasets/welcome.yml
+++ /dev/null
@@ -1,8 +0,0 @@
-version: datasets/v0.2
-title: Welcome to the workspace
-description: |-
-
- This is the beginning of your personal feed.
-
- What are you waiting for? Click # Explore and watch some datasets!
-
diff --git a/workspaces/testrepo-joeiagq/extract/.gitkeep b/workspaces/testrepo-joeiagq/extract/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/testrepo-joeiagq/load/.gitkeep b/workspaces/testrepo-joeiagq/load/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/testrepo-joeiagq/meltano.yml b/workspaces/testrepo-joeiagq/meltano.yml
deleted file mode 100644
index c91c64d..0000000
--- a/workspaces/testrepo-joeiagq/meltano.yml
+++ /dev/null
@@ -1,27 +0,0 @@
-version: 1
-default_environment: dev
-project_id: 36cdd140-a645-461d-839e-f379152c19e7
-environments:
-- name: dev
-- name: staging
-- name: prod
-plugins:
- extractors:
- - name: tap-github
- variant: meltanolabs
- pip_url: git+https://github.com/MeltanoLabs/tap-github.git
- loaders:
- - name: target-postgres
- variant: transferwise
- pip_url: git+https://github.com/Matatika/pipelinewise-target-postgres@v0.1.0
- transformers:
- - name: dbt
- variant: dbt-labs
- pip_url: "dbt-core~=1.3.0 dbt-postgres~=1.3.0 dbt-snowflake~=1.3.0\n"
- files:
- - name: analyze-github
- variant: matatika
- pip_url: git+https://github.com/Matatika/analyze-github
- - name: files-dbt
- variant: matatika
- pip_url: git+https://github.com/Matatika/files-dbt@v1.0.x.1
diff --git a/workspaces/testrepo-joeiagq/notebook/.gitkeep b/workspaces/testrepo-joeiagq/notebook/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/testrepo-joeiagq/orchestrate/.gitkeep b/workspaces/testrepo-joeiagq/orchestrate/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/testrepo-joeiagq/output/.gitignore b/workspaces/testrepo-joeiagq/output/.gitignore
deleted file mode 100644
index d6b7ef3..0000000
--- a/workspaces/testrepo-joeiagq/output/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*
-!.gitignore
diff --git a/workspaces/testrepo-joeiagq/pipelines/GitHub.yml b/workspaces/testrepo-joeiagq/pipelines/GitHub.yml
deleted file mode 100644
index fd73b07..0000000
--- a/workspaces/testrepo-joeiagq/pipelines/GitHub.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-version: pipelines/v0.1
-name: GitHub
-timeout: 0
-data_components:
-- tap-github
-- Warehouse
-- dbt
diff --git a/workspaces/testrepo-joeiagq/plugins/extractors/tap-github--meltanolabs.lock b/workspaces/testrepo-joeiagq/plugins/extractors/tap-github--meltanolabs.lock
deleted file mode 100644
index 444d1d3..0000000
--- a/workspaces/testrepo-joeiagq/plugins/extractors/tap-github--meltanolabs.lock
+++ /dev/null
@@ -1,39 +0,0 @@
-{
- "plugin_type": "extractors",
- "name": "tap-github",
- "namespace": "tap_github",
- "variant": "meltanolabs",
- "label": "GitHub",
- "docs": "http://localhost:4000/docs/data-sources/tap-github/",
- "repo": "https://github.com/MeltanoLabs/tap-github",
- "pip_url": "git+https://github.com/MeltanoLabs/tap-github.git",
- "description": "GitHub is an online software development platform. It's used for storing, tracking, and collaborating on software projects.\n\n## Prerequisites\n\nYou will need a Github `auth_token` scoped with full access to repo. For help with this you can follow the GitHub [Creating a Personal Access Token Docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)\n\n## Auth Token\n\nGithub token to authenticate with.\n\n## Rate Limit Buffer\n\nAdd a buffer to avoid consuming all query points for the token at hand.\n\n## Repositories\n\nAn array of strings containing the github repos to be included.\n\nExample: `[\"Matatika/tap-spotify\", \"Matatika/tap-auth0\"]`\n## Learn more\n\n[Matatika GitHub Data Source Documentation](http://localhost:4000/docs/data-sources/tap-github)\n\n[Meltano Hub GitHub Documentation](https://hub.meltano.com/extractors/tap-github/)",
- "logo_url": "https://localhost:3443/assets/images/datasource/tap-github.png",
- "capabilities": [
- "discover",
- "state",
- "about",
- "catalog"
- ],
- "settings": [
- {
- "name": "auth_token",
- "kind": "password",
- "label": "Auth Token",
- "protected": false,
- "required": "true"
- },
- {
- "name": "rate_limit_buffer",
- "kind": "integer",
- "label": "Rate Limit Buffer",
- "protected": false
- },
- {
- "name": "repositories",
- "kind": "array",
- "label": "Repositories",
- "protected": false
- }
- ]
-}
\ No newline at end of file
diff --git a/workspaces/testrepo-joeiagq/plugins/files/analyze-github--matatika.lock b/workspaces/testrepo-joeiagq/plugins/files/analyze-github--matatika.lock
deleted file mode 100644
index 5961c4d..0000000
--- a/workspaces/testrepo-joeiagq/plugins/files/analyze-github--matatika.lock
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "plugin_type": "files",
- "name": "analyze-github",
- "namespace": "tap_github",
- "variant": "matatika",
- "label": "Github Insights",
- "repo": "https://github.com/Matatika/analyze-github",
- "pip_url": "git+https://github.com/Matatika/analyze-github",
- "description": "Instant insights on commits, issues and PRs from Github.",
- "logo_url": "https://localhost:3443/assets/images/datasource/tap-github.png",
- "update": {
- "*.yml": "true"
- }
-}
\ No newline at end of file
diff --git a/workspaces/testrepo-joeiagq/plugins/files/files-dbt--matatika.lock b/workspaces/testrepo-joeiagq/plugins/files/files-dbt--matatika.lock
deleted file mode 100644
index d01beb9..0000000
--- a/workspaces/testrepo-joeiagq/plugins/files/files-dbt--matatika.lock
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "plugin_type": "files",
- "name": "files-dbt",
- "namespace": "dbt",
- "variant": "matatika",
- "label": "files-dbt",
- "repo": "https://github.com/Matatika/files-dbt",
- "pip_url": "git+https://github.com/Matatika/files-dbt@v1.0.x.1",
- "logo_url": "https://localhost:3443null",
- "update": {
- "transform/profile/profiles.yml": "true"
- }
-}
\ No newline at end of file
diff --git a/workspaces/testrepo-joeiagq/plugins/loaders/target-postgres--transferwise.lock b/workspaces/testrepo-joeiagq/plugins/loaders/target-postgres--transferwise.lock
deleted file mode 100644
index 7228e73..0000000
--- a/workspaces/testrepo-joeiagq/plugins/loaders/target-postgres--transferwise.lock
+++ /dev/null
@@ -1,181 +0,0 @@
-{
- "plugin_type": "loaders",
- "name": "target-postgres",
- "namespace": "postgres_transferwise",
- "variant": "transferwise",
- "label": "Postgres Warehouse",
- "repo": "git+https://github.com/Matatika/pipelinewise-target-postgres@v0.1.0",
- "pip_url": "git+https://github.com/Matatika/pipelinewise-target-postgres@v0.1.0",
- "description": "Loader for Postgres Data Warehouse\n\nThe World's Most Advanced Open Source Relational Database\n\n## Settings\n\n### User\nPostgres user\n\n### Password\nPostgres password\n\n### Host\nPostgres host\nExample: `test-host.db.elephantsql.com`\n\n### Port\nPostgres port\nExample: `5432`\n\n### Database Name\nPostgres database name\n\n### Default Target Schema\nName of the schema where tables will be created (no database prefix)",
- "logo_url": "https://localhost:3443/assets/logos/loaders/postgres.png",
- "settings": [
- {
- "name": "user",
- "aliases": [
- "username"
- ],
- "kind": "string",
- "label": "User",
- "description": "PostgreSQL user",
- "protected": false,
- "required": "true"
- },
- {
- "name": "password",
- "kind": "password",
- "label": "Password",
- "description": "PostgreSQL password",
- "protected": false,
- "required": "true"
- },
- {
- "name": "host",
- "aliases": [
- "address"
- ],
- "kind": "string",
- "label": "Host",
- "description": "PostgreSQL host",
- "protected": false,
- "required": "true"
- },
- {
- "name": "port",
- "kind": "integer",
- "value": 5432,
- "label": "Port",
- "description": "PostgreSQL port",
- "protected": false,
- "required": "true"
- },
- {
- "name": "dbname",
- "aliases": [
- "database"
- ],
- "kind": "string",
- "label": "Database Name",
- "description": "PostgreSQL database name",
- "protected": false,
- "required": "true"
- },
- {
- "name": "default_target_schema",
- "kind": "string",
- "value": "analytics",
- "label": "Default Target Schema",
- "description": "Name of the schema where the tables will be created.",
- "protected": false,
- "required": "true"
- },
- {
- "name": "ssl",
- "kind": "hidden",
- "value": false,
- "label": "SSL",
- "protected": false,
- "value_post_processor": "stringify"
- },
- {
- "name": "batch_size_rows",
- "kind": "integer",
- "value": 100000,
- "label": "Batch Size Rows",
- "description": "Maximum number of rows in each batch. At the end of each batch, the rows in the batch are loaded into Postgres.",
- "protected": false
- },
- {
- "name": "underscore_camel_case_fields",
- "kind": "hidden",
- "value": true,
- "label": "Underscore Camel Case Fields",
- "description": "Enable underscoring camel case field names, make the resulting postgres column names the same as from the Meltano variant of target-postgres.",
- "protected": false
- },
- {
- "name": "flush_all_streams",
- "kind": "hidden",
- "value": false,
- "label": "Flush All Streams",
- "description": "Flush and load every stream into Postgres when one batch is full. Warning: This may trigger the COPY command to use files with low number of records.",
- "protected": false
- },
- {
- "name": "parallelism",
- "kind": "hidden",
- "value": 0,
- "label": "Parallelism",
- "description": "The number of threads used to flush tables. 0 will create a thread for each stream, up to parallelism_max. -1 will create a thread for each CPU core. Any other positive number will create that number of threads, up to parallelism_max.",
- "protected": false
- },
- {
- "name": "parallelism_max",
- "kind": "hidden",
- "value": 16,
- "label": "Max Parallelism",
- "description": "Max number of parallel threads to use when flushing tables.",
- "protected": false
- },
- {
- "name": "default_target_schema_select_permission",
- "kind": "hidden",
- "label": "Default Target Schema Select Permission",
- "description": "Grant USAGE privilege on newly created schemas and grant SELECT privilege on newly created tables to a specific role or a list of roles. If `schema_mapping` is not defined then every stream sent by the tap is granted accordingly.",
- "protected": false
- },
- {
- "name": "schema_mapping",
- "kind": "hidden",
- "label": "Schema Mapping",
- "description": "Useful if you want to load multiple streams from one tap to multiple Postgres schemas.\n\nIf the tap sends the `stream_id` in `-` format then this option overwrites the `default_target_schema` value.\nNote, that using `schema_mapping` you can overwrite the `default_target_schema_select_permission` value to grant SELECT permissions to different groups per schemas or optionally you can create indices automatically for the replicated tables.\n",
- "protected": false
- },
- {
- "name": "add_metadata_columns",
- "kind": "hidden",
- "value": true,
- "label": "Add Metadata Columns",
- "description": "Metadata columns add extra row level information about data ingestions, (i.e. when was the row read in source, when was inserted or deleted in postgres etc.) Metadata columns are creating automatically by adding extra columns to the tables with a column prefix `_SDC_`. The column names are following the stitch naming conventions documented at https://www.stitchdata.com/docs/data-structure/integration-schemas#sdc-columns. Enabling metadata columns will flag the deleted rows by setting the `_SDC_DELETED_AT` metadata column. Without the `add_metadata_columns` option the deleted rows from singer taps will not be recongisable in Postgres.",
- "protected": false
- },
- {
- "name": "hard_delete",
- "kind": "hidden",
- "value": false,
- "label": "Hard Delete",
- "description": "When `hard_delete` option is true then DELETE SQL commands will be performed in Postgres to delete rows in tables. It's achieved by continuously checking the `_SDC_DELETED_AT` metadata column sent by the singer tap. Due to deleting rows requires metadata columns, `hard_delete` option automatically enables the `add_metadata_columns` option as well.",
- "protected": false
- },
- {
- "name": "data_flattening_max_level",
- "kind": "hidden",
- "value": 10,
- "label": "Data Flattening Max Level",
- "description": "Object type RECORD items from taps can be transformed to flattened columns by creating columns automatically. When value is 0 (default) then flattening functionality is turned off.",
- "protected": false
- },
- {
- "name": "primary_key_required",
- "kind": "boolean",
- "value": false,
- "label": "Primary Key Required",
- "description": "Log based and Incremental replications on tables with no Primary Key cause duplicates when merging UPDATE events. When set to true, stop loading data if no Primary Key is defined.",
- "protected": false
- },
- {
- "name": "validate_records",
- "kind": "boolean",
- "value": false,
- "label": "Validate Records",
- "description": "Validate every single record message to the corresponding JSON schema. This option is disabled by default and invalid RECORD messages will fail only at load time by Postgres. Enabling this option will detect invalid records earlier but could cause performance degradation.",
- "protected": false
- },
- {
- "name": "temp_dir",
- "kind": "hidden",
- "label": "Temporary Directory",
- "description": "(Default: platform-dependent) Directory of temporary CSV files with RECORD messages.",
- "protected": false
- }
- ]
-}
\ No newline at end of file
diff --git a/workspaces/testrepo-joeiagq/plugins/transformers/dbt--dbt-labs.lock b/workspaces/testrepo-joeiagq/plugins/transformers/dbt--dbt-labs.lock
deleted file mode 100644
index 1daf261..0000000
--- a/workspaces/testrepo-joeiagq/plugins/transformers/dbt--dbt-labs.lock
+++ /dev/null
@@ -1,80 +0,0 @@
-{
- "plugin_type": "transformers",
- "name": "dbt",
- "namespace": "dbt",
- "variant": "dbt-labs",
- "label": "dbt",
- "docs": "https://meltano.com/docs/transforms.html",
- "repo": "https://github.com/dbt-labs/dbt-core",
- "pip_url": "dbt-core~=1.3.0 dbt-postgres~=1.3.0 dbt-snowflake~=1.3.0\n",
- "logo_url": "https://localhost:3443/assets/images/transformer/dbt.png",
- "settings": [
- {
- "name": "project_dir",
- "kind": "string",
- "value": "$MELTANO_PROJECT_ROOT/transform",
- "protected": false
- },
- {
- "name": "profiles_dir",
- "env": "DBT_PROFILES_DIR",
- "kind": "string",
- "value": "$MELTANO_PROJECT_ROOT/transform/profile",
- "protected": false
- },
- {
- "name": "target",
- "kind": "string",
- "value": "$MELTANO_LOAD__DIALECT",
- "protected": false
- },
- {
- "name": "source_schema",
- "kind": "string",
- "value": "$MELTANO_LOAD__TARGET_SCHEMA",
- "protected": false
- },
- {
- "name": "target_schema",
- "kind": "string",
- "value": "analytics",
- "protected": false
- },
- {
- "name": "models",
- "kind": "string",
- "value": "$MELTANO_TRANSFORM__PACKAGE_NAME $MELTANO_EXTRACTOR_NAMESPACE my_meltano_project",
- "protected": false
- }
- ],
- "commands": {
- "test": {
- "args": "test",
- "description": "Runs tests on data in deployed models."
- },
- "compile": {
- "args": "compile",
- "description": "Generates executable SQL from source model, test, and analysis files. Compiled SQL files are written to the target/ directory."
- },
- "seed": {
- "args": "seed",
- "description": "Load data from csv files into your data warehouse."
- },
- "run": {
- "args": "run",
- "description": "Compile SQL and execute against the current target database."
- },
- "deps": {
- "args": "deps",
- "description": "Pull the most recent version of the dependencies listed in packages.yml"
- },
- "clean": {
- "args": "clean",
- "description": "Delete all folders in the clean-targets list (usually the dbt_modules and target directories.)"
- },
- "snapshot": {
- "args": "snapshot",
- "description": "Execute snapshots defined in your project."
- }
- }
-}
\ No newline at end of file
diff --git a/workspaces/testrepo-joeiagq/requirements.txt b/workspaces/testrepo-joeiagq/requirements.txt
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/testrepo-joeiagq/transform/.gitignore b/workspaces/testrepo-joeiagq/transform/.gitignore
deleted file mode 100644
index 99de336..0000000
--- a/workspaces/testrepo-joeiagq/transform/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-target/
-dbt_modules/
-logs/
\ No newline at end of file
diff --git a/workspaces/testrepo-joeiagq/transform/.gitkeep b/workspaces/testrepo-joeiagq/transform/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/testrepo-joeiagq/transform/dbt_project.yml b/workspaces/testrepo-joeiagq/transform/dbt_project.yml
deleted file mode 100644
index 959a642..0000000
--- a/workspaces/testrepo-joeiagq/transform/dbt_project.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-name: my_meltano_project
-version: "1.0"
-require-dbt-version: [">=1.0.0", "<2.0.0"]
-profile: meltano
-config-version: 2
-model-paths:
- - models
-analysis-paths:
- - analysis
-test-paths:
- - tests
-seed-paths:
- - data
-macro-paths:
- - macros
-snapshot-paths:
- - snapshots
-target-path: ../.meltano/transformers/dbt/target
-log-path: logs
-packages-install-path: dbt_packages
-clean-targets:
- - ../.meltano/transformers/dbt/target
- - dbt_packages
- - logs
-models:
- my_meltano_project: null
diff --git a/workspaces/testrepo-joeiagq/transform/models/.gitkeep b/workspaces/testrepo-joeiagq/transform/models/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/workspaces/testrepo-joeiagq/transform/profile/profiles.yml b/workspaces/testrepo-joeiagq/transform/profile/profiles.yml
deleted file mode 100644
index b52532e..0000000
--- a/workspaces/testrepo-joeiagq/transform/profile/profiles.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-# This file is managed by the 'files-dbt' file bundle and updated automatically when `meltano upgrade` is run.
-# To prevent any manual changes from being overwritten, remove the file bundle from `meltano.yml` or disable automatic updates:
-# meltano config --plugin-type=files files-dbt set _update transform/profile/profiles.yml false
-
-config:
- send_anonymous_usage_stats: False
- use_colors: True
-meltano:
- target: "{{ env_var('DBT_TARGET') }}"
- outputs:
- snowflake:
- type: snowflake
- threads: 2
- account: "{{ env_var('TARGET_SNOWFLAKE_ACCOUNT') }}"
- user: "{{ env_var('TARGET_SNOWFLAKE_USERNAME') }}"
- password: "{{ env_var('TARGET_SNOWFLAKE_PASSWORD') }}"
- role: "{{ env_var('TARGET_SNOWFLAKE_ROLE') }}"
- database: "{{ env_var('TARGET_SNOWFLAKE_DATABASE') }}"
- warehouse: "{{ env_var('TARGET_SNOWFLAKE_WAREHOUSE') }}"
- schema: "{{ env_var('TARGET_SNOWFLAKE_SCHEMA') }}"
- postgres:
- type: postgres
- threads: 2
- host: "{{ env_var('TARGET_POSTGRES_HOST') }}"
- port: "{{ env_var('TARGET_POSTGRES_PORT') | int }}"
- user: "{{ env_var('TARGET_POSTGRES_USER') }}"
- pass: "{{ env_var('TARGET_POSTGRES_PASSWORD') }}"
- dbname: "{{ env_var('TARGET_POSTGRES_DBNAME') }}"
- schema: "{{ env_var('TARGET_POSTGRES_SCHEMA') }}"
- postgres_transferwise:
- type: postgres
- threads: 2
- host: "{{ env_var('TARGET_POSTGRES_HOST') }}"
- port: "{{ env_var('TARGET_POSTGRES_PORT') | int }}"
- user: "{{ env_var('TARGET_POSTGRES_USER') }}"
- pass: "{{ env_var('TARGET_POSTGRES_PASSWORD') }}"
- dbname: "{{ env_var('TARGET_POSTGRES_DBNAME') }}"
- schema: "{{ env_var('TARGET_POSTGRES_DEFAULT_TARGET_SCHEMA') }}"
- bigquery:
- type: bigquery
- threads: 2
- method: service-account
- project: "{{ env_var('TARGET_BIGQUERY_PROJECT_ID') }}"
- dataset: "{{ env_var('DBT_TARGET_SCHEMA') }}"
- keyfile: "{{ env_var('TARGET_BIGQUERY_CREDENTIALS_PATH') }}"
- location: "{{ env_var('TARGET_BIGQUERY_LOCATION') }}"
diff --git a/workspaces/testrepo-joeiagq/workspace.yml b/workspaces/testrepo-joeiagq/workspace.yml
deleted file mode 100644
index 66ebc53..0000000
--- a/workspaces/testrepo-joeiagq/workspace.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-version: workspaces/v0.1
-name: testrepo
-app_properties:
- WELCOME_DATASET_ALIAS: welcome
-dataset_paths:
-- analyze/datasets
-- .meltano/analyze/datasets
-channel_paths:
-- analyze/channels
-pipeline_paths:
-- pipelines
-plugin_paths:
-- plugins