add include/exclude filter to S3CopyArc

ClusterlessHQ · Oct 3, 2023 · 896046f · 896046f
1 parent c9c5c19
commit 896046f
Show file tree

Hide file tree

Showing 10 changed files with 927 additions and 9 deletions.
diff --git a/clusterless-scenario/src/main/cls/scenarios/s3-copy-chain/s3-copy-arc-project-chain.jsonnet b/clusterless-scenario/src/main/cls/scenarios/s3-copy-chain/s3-copy-arc-project-chain.jsonnet
@@ -2,7 +2,7 @@ local stage = std.extVar('scenario.stage');
 local account = std.extVar('scenario.aws.account');
 local region = std.extVar('scenario.aws.region');
 local bucketName = 'clusterless-chain-test-' + account + '-' + region;
-local bucketPrefix = 's3://'+bucketName;
+local bucketPrefix = 's3://' + bucketName;
 local unit = 'Twelfths';
 
 {
@@ -30,7 +30,7 @@ local unit = 'Twelfths';
       dataset: {
         name: 'ingress-chain',
         version: '20220101',
-        pathURI: bucketPrefix+'/ingress/',
+        pathURI: bucketPrefix + '/ingress/',
       },
       lotUnit: unit,
     },
@@ -43,14 +43,14 @@ local unit = 'Twelfths';
         main: {
           name: 'ingress-chain',
           version: '20220101',
-          pathURI: bucketPrefix+'/ingress/',
+          pathURI: bucketPrefix + '/ingress/',
         },
       },
       sinks: {
         main: {
           name: 'copy-a-chain',
           version: '20230101',
-          pathURI: bucketPrefix+'/copy-a/',
+          pathURI: bucketPrefix + '/copy-a/',
         },
       },
     },
@@ -61,14 +61,14 @@ local unit = 'Twelfths';
         main: {
           name: 'copy-a-chain',
           version: '20230101',
-          pathURI: bucketPrefix+'/copy-a/',
+          pathURI: bucketPrefix + '/copy-a/',
         },
       },
       sinks: {
         main: {
           name: 'copy-b-chain',
           version: '20230101',
-          pathURI: bucketPrefix+'/copy-b/',
+          pathURI: bucketPrefix + '/copy-b/',
         },
       },
       workload: {
@@ -84,14 +84,14 @@ local unit = 'Twelfths';
         main: {
           name: 'copy-b-chain',
           version: '20230101',
-          pathURI: bucketPrefix+'/copy-b/',
+          pathURI: bucketPrefix + '/copy-b/',
         },
       },
       sinks: {
         main: {
           name: 'copy-c-chain',
           version: '20230101',
-          pathURI: bucketPrefix+'/copy-c/',
+          pathURI: bucketPrefix + '/copy-c/',
         },
       },
     },

diff --git a/clusterless-scenario/src/main/cls/scenarios/s3-frequent/s3-copy-arc.jsonnet b/clusterless-scenario/src/main/cls/scenarios/s3-frequent/s3-copy-arc.jsonnet
@@ -54,6 +54,13 @@ local unit = 'Twelfths';
           pathURI: bucketPrefix + '/copy-a/',
         },
       },
+      workload: {
+        workloadProps: {
+          filter: {
+            excludes: ['**/_*'],
+          },
+        },
+      },
     },
   ],
 }
diff --git a/clusterless-scenario/src/main/cls/scenarios/s3-frequent/scenario.jsonnet b/clusterless-scenario/src/main/cls/scenarios/s3-frequent/scenario.jsonnet
@@ -16,6 +16,13 @@ local project = import 's3-copy-arc.jsonnet';
       uploadDelaySec: 15,
       objectCount: 60/15 * 5 * 3,
     },
+    {
+      region: project.placement.region,
+      path: project.arcs[0].sources.main.pathURI,
+      uploadDelaySec: 15,
+      objectCount: 60/15 * 5 * 3,
+      objectName: '_SUCCESS-%04d-%d.txt',
+    },
   ],
   watchedStores: [
     {

diff --git a/...-construct-core/src/main/java/clusterless/substrate/aws/arc/s3copy/S3CopyArcProvider.java b/...-construct-core/src/main/java/clusterless/substrate/aws/arc/s3copy/S3CopyArcProvider.java
@@ -19,7 +19,16 @@
         type = "aws:core:s3CopyArc",
         synopsis = "The AWS S3 Copy Arc component copies data from one S3 bucket to another S3 bucket.",
         description = """
-                All data in the source manifest will be copied to the specified dataset.
+                All data in the source manifest will be copied to the specified dataset, except those paths that do
+                not pass the filter. The filter is a list of include and exclude patterns.
+                                
+                A common exclude pattern would be '**/_*'. This would exclude all files that start with an underscore,
+                like '_SUCCESS' or '_metadata'.
+                                
+                workloadProps.filter.includes: A list of include patterns.
+                workloadProps.filter.excludes: A list of exclude patterns.
+                workloadProps.filter.pathSeparator: The path separator to use when matching patterns. Default is '/'.
+                workloadProps.filter.ignoreCase: Whether to ignore case when matching patterns. Default is false.
                                 
                 workloadProps.failArcOnPartialPercent: The percentage of files that can fail before the Arc fails. Default is 0.0.
                 """