From 668917ed943c323edd84e511a966013034718049 Mon Sep 17 00:00:00 2001
From: Pat Nadolny <patnadolny@gmail.com>
Date: Thu, 29 Jun 2023 18:42:08 -0400
Subject: [PATCH] feat: configure batch pipelines (#658)

Co-authored-by: Ken Payne <ken@meltano.com>
---
 .gitignore                          |  3 +++
 data/extract/extractors.meltano.yml | 19 ++++++++++++++++---
 data/output/README.md               |  2 ++
 data/output/tap-cloudwatch/.gitkeep |  0
 data/output/tap-slack/.gitkeep      |  0
 5 files changed, 21 insertions(+), 3 deletions(-)
 create mode 100644 data/output/README.md
 create mode 100644 data/output/tap-cloudwatch/.gitkeep
 create mode 100644 data/output/tap-slack/.gitkeep

diff --git a/.gitignore b/.gitignore
index 8eb0730d..c4f6c49e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -163,3 +163,6 @@ cython_debug/
 
 # VS Code config
 .vscode/
+
+# Test data
+data/data/*
diff --git a/data/extract/extractors.meltano.yml b/data/extract/extractors.meltano.yml
index c927d94c..db95b876 100644
--- a/data/extract/extractors.meltano.yml
+++ b/data/extract/extractors.meltano.yml
@@ -30,7 +30,7 @@ plugins:
           __alias__: organizations_table
   - name: tap-cloudwatch
     variant: meltanolabs
-    pip_url: git+https://github.com/meltanolabs/tap-cloudwatch.git@0.3.0
+    pip_url: git+https://github.com/meltanolabs/tap-cloudwatch.git@0.4.0
     config:
       log_group_name: API-Gateway-Execution-Logs_i32s35df22/prod
       query: fields @timestamp, @message
@@ -38,9 +38,16 @@ plugins:
       start_date: '2022-12-08'
       aws_access_key_id: ${AWS_ACCESS_KEY_ID}
       aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY}
+      batch_config:
+        encoding:
+          format: jsonl
+          compression: gzip
+        storage:
+          root: file://output/tap-cloudwatch/
   - name: tap-meltanohub
     variant: autoidm
-    pip_url: git+https://github.com/AutoIDM/tap-meltanohub.git@1b99b0ee7853b2d9db36de6afa16b15c1affce7b
+    pip_url: git+https://github.com/pnadolny13/tap-meltanohub.git@update_dependencies
+    # pip_url: git+https://github.com/AutoIDM/tap-meltanohub.git@1b99b0ee7853b2d9db36de6afa16b15c1affce7b
     select:
     - plugins.*
   - name: tap-spreadsheets-anywhere
@@ -71,7 +78,7 @@ plugins:
         json_path: values
   - name: tap-slack
     variant: meltanolabs
-    pip_url: git+https://github.com/MeltanoLabs/tap-slack.git@0.1.6
+    pip_url: git+https://github.com/MeltanoLabs/tap-slack.git@0.2.0
     config:
       start_date: '2021-01-01'
       auto_join_channels: false
@@ -79,6 +86,12 @@ plugins:
       - C01SK13R9NJ
       channel_types:
       - private_channel
+      batch_config:
+        encoding:
+          format: jsonl
+          compression: gzip
+        storage:
+          root: file://output/tap-slack/
     select:
     - users.*
     - channels.*
diff --git a/data/output/README.md b/data/output/README.md
new file mode 100644
index 00000000..66355d19
--- /dev/null
+++ b/data/output/README.md
@@ -0,0 +1,2 @@
+Empty dirs for BATCH testing. Can Be deleted once this issue is resolved:
+https://github.com/meltano/sdk/issues/1014
diff --git a/data/output/tap-cloudwatch/.gitkeep b/data/output/tap-cloudwatch/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/data/output/tap-slack/.gitkeep b/data/output/tap-slack/.gitkeep
new file mode 100644
index 00000000..e69de29b