Enhance downloadFromS3 library (#135)

Signed-off-by: Sayali Gaikawad <[email protected]>

tests/jenkins/TestDownloadFromS3.groovy

@@ -10,20 +10,65 @@
 import jenkins.tests.BuildPipelineTest
 import org.junit.Before
 import org.junit.Test
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.MatcherAssert.assertThat
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
 
 
 class TestDownloadFromS3 extends BuildPipelineTest {
 
     @Before
     void setUp() {
 
-        this.registerLibTester(new DownloadFromS3LibTester('/tmp/src/path' , 'dummy_bucket', '/download/path', true))
+        this.registerLibTester(new DownloadFromS3LibTester(
+            'tmp-role',
+            'role-credential-id',
+            '/download/path',
+            'dummy_bucket',
+            "/tmp"
+            ))
+        this.registerLibTester(new DownloadFromS3LibTester(
+            'tmp-role',
+            'role-credential-id',
+            '/download/path',
+            'dummy_bucket',
+            "/tmp",
+            false,
+            'us-west-2'
+            ))
 
         super.setUp()
     }
 
     @Test
     public void testDownloadFromS3() {
-        super.testPipeline("tests/jenkins/jobs/DownloadFromS3_Jenkinsfile")
+        super.testPipeline('tests/jenkins/jobs/DownloadFromS3_Jenkinsfile')
+    }
+
+    @Test
+    void verify_default_args(){
+        runScript('tests/jenkins/jobs/DownloadFromS3_Jenkinsfile')
+        def aws = getMethodCall('withAWS')
+        assertThat(aws, hasItem('{role=tmp-role, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure'))
+        def download = getMethodCall('s3Download')
+        assertThat(download, hasItem('{file=/tmp, bucket=dummy_bucket, path=/download/path, force=false}'))
+    }
+
+    @Test
+    void verify_optional_args(){
+        runScript('tests/jenkins/jobs/DownloadFromS3_Jenkinsfile')
+        def aws = getMethodCall('withAWS')
+        assertThat(aws, hasItem('{role=tmp-role, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-west-2}, groovy.lang.Closure'))
+        def download = getMethodCall('s3Download')
+        assertThat(download, hasItem('{file=/tmp, bucket=dummy_bucket, path=/download/path, force=true}'))
+    }
+
+    def getMethodCall(methodName) {
+        def shCommands = helper.callStack.findAll { call ->
+            call.methodName == methodName
+        }.collect { call ->
+            callArgsToString(call)
+        }
+        return shCommands
     }
 }

tests/jenkins/jobs/DownloadFromS3_Jenkinsfile

@@ -14,10 +14,21 @@ pipeline {
             steps {
                 script {
                     downloadFromS3(
-                        destPath: '/tmp/src/path',
-                        bucket: 'dummy_bucket',
-                        path: '/download/path',
-                        force: true
+                        assumedRoleName: 'tmp-role',
+                        roleAccountNumberCred: 'role-credential-id',
+                        downloadPath: '/download/path',
+                        bucketName: 'dummy_bucket',
+                        localPath: "/tmp"
+                    )
+
+                    downloadFromS3(
+                        assumedRoleName: 'tmp-role',
+                        roleAccountNumberCred: 'role-credential-id',
+                        downloadPath: '/download/path',
+                        bucketName: 'dummy_bucket',
+                        localPath: "/tmp",
+                        force: true,
+                        region: 'us-west-2'
                     )
                 }
             }

tests/jenkins/jobs/DownloadFromS3_Jenkinsfile.txt

@@ -3,8 +3,13 @@
          DownloadFromS3_Jenkinsfile.echo(Executing on agent [label:none])
          DownloadFromS3_Jenkinsfile.stage(download, groovy.lang.Closure)
             DownloadFromS3_Jenkinsfile.script(groovy.lang.Closure)
-               DownloadFromS3_Jenkinsfile.downloadFromS3({destPath=/tmp/src/path, bucket=dummy_bucket, path=/download/path, force=true})
-                  downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
-                  downloadFromS3.withCredentials([AWS_ACCOUNT_PUBLIC], groovy.lang.Closure)
-                     downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
-                        downloadFromS3.s3Download({file=/tmp/src/path, bucket=dummy_bucket, path=/download/path, force=true})
+               DownloadFromS3_Jenkinsfile.downloadFromS3({assumedRoleName=tmp-role, roleAccountNumberCred=role-credential-id, downloadPath=/download/path, bucketName=dummy_bucket, localPath=/tmp})
+                  downloadFromS3.string({credentialsId=role-credential-id, variable=AWS_ACCOUNT_NUMBER})
+                  downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure)
+                     downloadFromS3.withAWS({role=tmp-role, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure)
+                        downloadFromS3.s3Download({file=/tmp, bucket=dummy_bucket, path=/download/path, force=false})
+               DownloadFromS3_Jenkinsfile.downloadFromS3({assumedRoleName=tmp-role, roleAccountNumberCred=role-credential-id, downloadPath=/download/path, bucketName=dummy_bucket, localPath=/tmp, force=true, region=us-west-2})
+                  downloadFromS3.string({credentialsId=role-credential-id, variable=AWS_ACCOUNT_NUMBER})
+                  downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure)
+                     downloadFromS3.withAWS({role=tmp-role, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-west-2}, groovy.lang.Closure)
+                        downloadFromS3.s3Download({file=/tmp, bucket=dummy_bucket, path=/download/path, force=true})

tests/jenkins/lib-testers/DownloadFromS3LibTester.groovy

@@ -13,41 +13,54 @@ import static org.hamcrest.CoreMatchers.equalTo
 
 class DownloadFromS3LibTester extends LibFunctionTester {
 
-    private String destPath
-    private String bucket
-    private String path
+    private String assumedRoleName
+    private String roleAccountNumberCred
+    private String downloadPath
+    private String bucketName
+    private String localPath
     private boolean force
+    private String region
 
-    public DownloadFromS3LibTester(destPath, bucket, path, force){
-        this.destPath = destPath
-        this.bucket = bucket
-        this.path = path
+    public DownloadFromS3LibTester(assumedRoleName, roleAccountNumberCred, downloadPath, bucketName, localPath){
+        this.assumedRoleName = assumedRoleName
+        this.roleAccountNumberCred = roleAccountNumberCred
+        this.downloadPath = downloadPath
+        this.bucketName = bucketName
+        this.localPath = localPath
+    }
+    public DownloadFromS3LibTester(assumedRoleName, roleAccountNumberCred, downloadPath, bucketName, localPath, force, region){
+        this.assumedRoleName = assumedRoleName
+        this.roleAccountNumberCred = roleAccountNumberCred
+        this.downloadPath = downloadPath
+        this.bucketName = bucketName
+        this.localPath = localPath
         this.force = force
+        this.region = region
     }
 
     void parameterInvariantsAssertions(call){
-        assertThat(call.args.destPath.first(), notNullValue())
-        assertThat(call.args.bucket.first(), notNullValue())
-        assertThat(call.args.path.first(), notNullValue())
-        assertThat(call.args.force.first(), notNullValue())
-        assertThat(call.args.force.first().toString(), anyOf(equalTo('true'), equalTo('false')))
+        assertThat(call.args.downloadPath.first(), notNullValue())
+        assertThat(call.args.assumedRoleName.first(), notNullValue())
+        assertThat(call.args.roleAccountNumberCred.first(), notNullValue())
+        assertThat(call.args.bucketName.first(), notNullValue())
+        assertThat(call.args.localPath.first(), notNullValue())
     }
 
     boolean expectedParametersMatcher(call) {
-        return call.args.destPath.first().toString().equals(this.destPath)
-                && call.args.bucket.first().toString().equals(this.bucket)
-                && call.args.path.first().toString().equals(this.path)
-                && call.args.force.first().toString().equals(this.force.toString())
+        return call.args.downloadPath.first().toString().equals(this.downloadPath)
+                && call.args.assumedRoleName.first().toString().equals(this.assumedRoleName)
+                && call.args.roleAccountNumberCred.first().toString().equals(this.roleAccountNumberCred)
+                && call.args.bucketName.first().toString().equals(this.bucketName)
+                && call.args.localPath.first().toString().equals(this.localPath)
     }
 
     String libFunctionName(){
         return 'downloadFromS3'
     }
 
     void configure(helper, binding){
-        binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role')
-        binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account')
         helper.registerAllowedMethod("s3Download", [Map])
+        helper.registerAllowedMethod("withCredentials", [Map])
         helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
             closure.delegate = delegate
             return helper.callClosure(closure)

vars/downloadFromS3.groovy

@@ -6,10 +6,23 @@
  * this file be licensed under the Apache-2.0 license or a
  * compatible open source license.
  */
+ /**
+Library to download the artifacts from S3 bucket to local
+@param Map[assumedRoleName] <Required> - Role to be assumed to download artifacts.
+@param Map[roleAccountNumberCred] <Required> - AWS account number to download artifacts from.
+@param Map[bucketName] <Required> -S3 bucket Name.
+@param Map[downloadPath] <Required> - This is the path inside the bucket to use.
+@param Map[localPath] <Required> - This is the local target file to download into.
+@param Map[region] <Optional> - AWS region of the S3 bucket. Defaults to us-east-1
+@param Map[force]<Optional> - Set this to true to overwrite local workspace files. Defaults to false
+*/
 void call(Map args = [:]) {
-    withCredentials([string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC')]) {
-            withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') {
-                s3Download(file: args.destPath, bucket: args.bucket, path: args.path, force: args.force)
+    boolean forceDownload = args.force ?: false
+    String region = args.region ?: 'us-east-1'
+
+    withCredentials([string(credentialsId: "${args.roleAccountNumberCred}", variable: 'AWS_ACCOUNT_NUMBER')]) {
+            withAWS(role: args.assumedRoleName, roleAccount: "${AWS_ACCOUNT_NUMBER}", duration: 900, roleSessionName: 'jenkins-session', region: "${region}") {
+                s3Download(file: args.localPath, bucket: args.bucketName, path: args.downloadPath, force: "${forceDownload}")
             }
     }
 }