feat: configure batch pipelines (#658)

Co-authored-by: Ken Payne <[email protected]>
meltano · Jun 29, 2023 · 668917e · 668917e
1 parent 9721dda
commit 668917e
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 3 deletions.
diff --git a/.gitignore b/.gitignore
@@ -163,3 +163,6 @@ cython_debug/
 
 # VS Code config
 .vscode/
+
+# Test data
+data/data/*
diff --git a/data/extract/extractors.meltano.yml b/data/extract/extractors.meltano.yml
@@ -30,17 +30,24 @@ plugins:
           __alias__: organizations_table
   - name: tap-cloudwatch
     variant: meltanolabs
-    pip_url: git+https://github.com/meltanolabs/tap-cloudwatch.git@0.3.0
+    pip_url: git+https://github.com/meltanolabs/tap-cloudwatch.git@0.4.0
     config:
       log_group_name: API-Gateway-Execution-Logs_i32s35df22/prod
       query: fields @timestamp, @message
       aws_region_name: us-west-2
       start_date: '2022-12-08'
       aws_access_key_id: ${AWS_ACCESS_KEY_ID}
       aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY}
+      batch_config:
+        encoding:
+          format: jsonl
+          compression: gzip
+        storage:
+          root: file://output/tap-cloudwatch/
   - name: tap-meltanohub
     variant: autoidm
-    pip_url: git+https://github.com/AutoIDM/tap-meltanohub.git@1b99b0ee7853b2d9db36de6afa16b15c1affce7b
+    pip_url: git+https://github.com/pnadolny13/tap-meltanohub.git@update_dependencies
+    # pip_url: git+https://github.com/AutoIDM/tap-meltanohub.git@1b99b0ee7853b2d9db36de6afa16b15c1affce7b
     select:
     - plugins.*
   - name: tap-spreadsheets-anywhere
@@ -71,14 +78,20 @@ plugins:
         json_path: values
   - name: tap-slack
     variant: meltanolabs
-    pip_url: git+https://github.com/MeltanoLabs/tap-slack.git@0.1.6
+    pip_url: git+https://github.com/MeltanoLabs/tap-slack.git@0.2.0
     config:
       start_date: '2021-01-01'
       auto_join_channels: false
       selected_channels:
       - C01SK13R9NJ
       channel_types:
       - private_channel
+      batch_config:
+        encoding:
+          format: jsonl
+          compression: gzip
+        storage:
+          root: file://output/tap-slack/
     select:
     - users.*
     - channels.*

diff --git a/data/output/README.md b/data/output/README.md
@@ -0,0 +1,2 @@
+Empty dirs for BATCH testing. Can Be deleted once this issue is resolved:
+https://github.com/meltano/sdk/issues/1014
diff --git a/data/output/tap-cloudwatch/.gitkeep b/data/output/tap-cloudwatch/.gitkeep
diff --git a/data/output/tap-slack/.gitkeep b/data/output/tap-slack/.gitkeep
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Empty dirs for BATCH testing. Can Be deleted once this issue is resolved:
		https://github.com/meltano/sdk/issues/1014