sovrin-foundation · andkononykhin · Jun 28, 2019 · Jun 3, 2019 · Jun 6, 2019 · Jun 6, 2019
diff --git a/devops/Makefile b/devops/Makefile
@@ -53,7 +53,7 @@ ifeq ($(SRC_DIR_NAME),sovtoken)
 # 	pypi: indy-plenum
 # 	apt: indy-plenum (stable component)
 
-FPM_P_DEPENDS := indy-node(=1.8.1)
+FPM_P_DEPENDS := indy-node(=1.9.0~rc1)
 FPM_ARGS := --no-python-dependencies $(FPM_ARGS)
 endif
 

diff --git a/devops/aws-codebuild/Jenkinsfile.cd b/devops/aws-codebuild/Jenkinsfile.cd
@@ -1,5 +1,7 @@
 #!groovy
 
+String projectName = 'token-plugin'
+
 def sovLibrary = library(identifier: '[email protected]', retriever: modernSCM(
     github(credentialsId: 'sovbot-github', repoOwner: 'sovrin-foundation', repository: 'aws-codebuild-pipeline-plugin')
 )).com.sovrin.pipeline
@@ -18,12 +20,11 @@ def downloadPackagingUtils() {
 }
 
 def nodeLabels = [
-        codeBuild: env.LIBSOVTOKEN_CODEBUILD_NODE_LABEL ?: 'codebuild',
-        macos: env.LIBSOVTOKEN_MACOS_NODE_LABEL ?: 'macos',
+    codeBuild: env.LIBSOVTOKEN_CODEBUILD_NODE_LABEL ?: 'codebuild',
+    macos: env.LIBSOVTOKEN_MACOS_NODE_LABEL ?: 'macos',
 ]
 
 pipelineWrapper({
-
     //put code build containers inside a vpc under our dev account
     env.USE_VPC_CONFIG = true
     //this IP is outdated, should use the one from env
@@ -32,6 +33,11 @@ pipelineWrapper({
     nodeWrapper(nodeLabels.codeBuild) {
         def osname = 'xenial'
         List projects = ['sovtoken', 'sovtokenfees']
+        List pkgsList = []
+        String indyNodeVersion
+        Boolean isRC = false
+        Boolean isRelease = false
+        Boolean isDev = (env.BRANCH_NAME == 'master')
 
         stage('Checkout sources from SCM') {
             checkout scm
@@ -43,6 +49,18 @@ pipelineWrapper({
             ['devops', 'sovtoken/sovtoken/__metadata__.py', 'sovtokenfees/sovtokenfees/__metadata__.py'])
         def awsCBHelper = sovLibrary.AwsCodeBuildHelper.new(this, buildCtx)
 
+        stage('Get indy-node version') {
+            indyNodeVersion = utils.shStdout("""
+                sed -n "s/.*indy-node==\\([^\\"\\']\\+\\).*/\\1/p" sovtoken/setup.py
+            """)
+            if (!indyNodeVersion) {
+                throw new Exception('indy-node version is not found')
+            }
+            isRC = !!((env.BRANCH_NAME == 'stable') && indyNodeVersion.find(/[^0-9\.]/))
+            isRelease = ((env.BRANCH_NAME == 'stable') && !isRC)
+            logger.info("Detected indy-node version: '$indyNodeVersion', isRC: $isRC, isRelease: $isRelease")
+        }
+
         Map builds = [
             "s3-upload": {  // performs on the current jenkins node
                 stage('Upload source to S3') {
@@ -65,126 +83,142 @@ pipelineWrapper({
         }
 
         builds = projects.collectEntries { proj ->
-            ["$osname:$proj": [
-                 nodeLabel: "${nodeLabels.codeBuild}",
-                 build: {
-
-                     def packageName = proj
-                     def prTag = "ci-$osname-$proj" // use ci project
-                     def srcVersion
-                     def lastRevision
-                     def debPVersion
-                     def goals = ['package']
-
-                     def ciImageTag
-
-                     stage('Get source version') {
-                         srcVersion = utils.shStdout("SRC_DIR_NAME=$proj make -C devops src_version -s")
-                         logger.info("Current source version: $srcVersion")
-                     }
-
-//                stage('Get last revision') {
-//                    lastRevision = evernymRepo.getLastRevision {
-//                        delegate.packageName = packageName
-//                        packageSrcVersion = srcVersion
-//                        repoDistr = 'agency-dev'
-//                    }
+            String projLabel = "$osname-$proj"
+
+            ["$projLabel": [
+                nodeLabel: "${nodeLabels.codeBuild}",
+                build: {
+                    def packageName = proj
+                    def prTag = "ci-$osname-$proj" // use ci project
+                    def srcVersion
+                    def lastRevision
+                    def debPVersion
+                    def goals = ['package']
+
+                    def ciImageTag
+
+                    stage('Get source version') {
+                        srcVersion = utils.shStdout("SRC_DIR_NAME=$proj make -C devops src_version -s")
+                        logger.info("Current source version: $srcVersion")
+                    }
+
+//               stage('Get last revision') {
+//                   lastRevision = evernymRepo.getLastRevision {
+//                       delegate.packageName = packageName
+//                       packageSrcVersion = srcVersion
+//                       repoDistr = 'agency-dev'
+//                   }
 //
-//                    if (lastRevision) {
-//                        logger.info("Found last revision number: $lastRevision")
-//                    } else {
-//                        logger.info("No previous revision was found")
-//                    }
-//                }
-
-
-                     stage('Set release parameters') {
-                         def releaseVersion =  env.BRANCH_NAME == 'stable' ? '' : "$BUILD_NUMBER"
-
-                         debPVersion = "$srcVersion${releaseVersion ? '~' + releaseVersion : ''}"
-                         logger.info("Package version for sovrin repo: $debPVersion")
-                     }
-
-                     stage("$osname:$proj: Resolve image tag") {
-                         def _imgVersion = utils.shStdout("OSNAME=$osname make -C devops image_ci_version -s")
-                         ciImageTag = "$_imgVersion-$osname-$proj-ci"
-                         logger.info("CI docker image tag: $ciImageTag")
-                     }
-
-                     awsCBHelper.build() {
-                         projectTag = prTag
-
-                         // build spec for env image
-                         envBuildSrc = ['devops', "${proj}/${proj}/__metadata__.py"] // TODO make more accurate
-                         envBuildCmds = [
-                                 'export PROJECT_DIR=$PWD',
-                                 'make -C devops image_ci'
-                         ]
-                         envBuildEnvv = [
-                                 [name: 'OSNAME', value: osname],
-                                 [name: 'SRC_DIR_NAME', value: proj],
-                                 [name: 'DOCKER_NAME', value: awsCBHelper.buildCtx.projectName],
-                                 [name: 'DOCKER_TAG', value: ciImageTag],
-                         ]
-
-                         // env and build spec
-                         imageTag = ciImageTag
-                         buildspec = 'devops/aws-codebuild/buildspec.yml'
-                         envv = [
-                                 [name: 'OSNAME', value: osname],
-                                 [name: 'MAKE_GOALS', value: 'package'],
-                                 [name: 'SRC_DIR_NAME', value: proj],
-                                 [name: 'ARTIFACTS', value: "devops/_build/$proj/*$packageName*.*"], // TODO more accurate here
-                                 [name: 'FPM_P_VERSION', value: debPVersion],
-                         ]
-
-                         onArtifacts = {
-                             this.stage("$osname:$proj: Archive logs") {
-                                 utils.archiveArtifacts("logs/*.log*") {
-                                     truncate = true
-                                     allowEmptyArchive = true
-                                 }
-                             }
-                         }
-                     }
-
-
-                     stage('Upload debs to repos') {
-
-                         String debName
-
-                         dir("${awsCBHelper.buildCtx.projects[prTag].artifactsDir}") {
-                             dir("sovrin-packaging") {
-                                 downloadPackagingUtils()
-                             }
-
-                             debName = utils.shStdout("ls $packageName*$debPVersion*.deb")
-
-                             logger.info("Uploading debian package '$debName' to sovrin repo")
-                             sh "mkdir debs &&  mv $debName ./debs/"
-                             withCredentials([file(credentialsId: 'SovrinRepoSSHKey', variable: 'sovrin_key')]) {
-                                 sh "./sovrin-packaging/upload_debs.py ./debs $env.SOVRIN_CORE_REPO_NAME $env.BRANCH_NAME --host $env.SOVRIN_REPO_HOST --ssh-key $sovrin_key"
-                             }
-
-                         }
-
-                         notifier.email {
-                             to = '$DEFAULT_RECIPIENTS ' + (this.env.LIBSOVTOKEN_DEF_RECIPIENTS ?: '')
-                             subject = '$PROJECT_NAME - Build # $BUILD_NUMBER: ' + "new deb '$debName' was published"
-                             body = ("New debian package '$debName' was built and published" +
-                                     '\n\nCheck console output at $BUILD_URL to view the results.')
-                         }
-                     }
-
-                 }
-                ]
-            ]
+//                   if (lastRevision) {
+//                       logger.info("Found last revision number: $lastRevision")
+//                   } else {
+//                       logger.info("No previous revision was found")
+//                   }
+//               }
+
+
+                    stage('Set release parameters') {
+                        def releaseVersion =  isRelease ? '' : (isRC ? 'rc' : isDev ? 'dev' : '') + "$BUILD_NUMBER"
+                        debPVersion = "$srcVersion${releaseVersion ? '~' + releaseVersion : ''}"
+                        logger.info("Package version for sovrin repo: $debPVersion")
+                    }
+
+                    stage("$projLabel: Resolve image tag") {
+                        def _imgVersion = utils.shStdout("OSNAME=$osname make -C devops image_ci_version -s")
+                        ciImageTag = "$_imgVersion-$osname-$proj-ci"
+                        logger.info("CI docker image tag: $ciImageTag")
+                    }
+
+                    awsCBHelper.build() {
+                        projectTag = prTag
+
+                        // build spec for env image
+                        envBuildSrc = ['devops', "${proj}/${proj}/__metadata__.py"] // TODO make more accurate
+                        envBuildCmds = [
+                                'export PROJECT_DIR=$PWD',
+                                'make -C devops image_ci'
+                        ]
+                        envBuildEnvv = [
+                                [name: 'OSNAME', value: osname],
+                                [name: 'SRC_DIR_NAME', value: proj],
+                                [name: 'DOCKER_NAME', value: awsCBHelper.buildCtx.projectName],
+                                [name: 'DOCKER_TAG', value: ciImageTag],
+                        ]
+
+                        // env and build spec
+                        imageTag = ciImageTag
+                        buildspec = 'devops/aws-codebuild/buildspec.yml'
+                        envv = [
+                                [name: 'OSNAME', value: osname],
+                                [name: 'MAKE_GOALS', value: 'package'],
+                                [name: 'SRC_DIR_NAME', value: proj],
+                                [name: 'ARTIFACTS', value: "devops/_build/${proj}/*${packageName}*.*"], // TODO more accurate here
+                                [name: 'FPM_P_VERSION', value: debPVersion],
+                        ]
+
+                        onArtifacts = {
+                            this.stage("$projLabel: Archive logs") {
+                                utils.archiveArtifacts("logs/*.log*") {
+                                    truncate = true
+                                    allowEmptyArchive = true
+                                }
+
+                                String pkgBaseName = "${packageName}_${debPVersion}"
+
+                                this.stash includes: utils.shStdout("ls ${pkgBaseName}_*.deb"), name: pkgBaseName
+                                pkgsList += pkgBaseName
+                            }
+                        }
+                    }
+                }
+            ]]
         }
 
-        stage("Package and publish") {
+        stage("Build artifacts") {
             builds.failFast = false
             utils.parallel builds
         }
+
+        stage('Release artifacts') {
+            String repoName = env.SOVRIN_CORE_REPO_NAME
+            String repoComponent = env.BRANCH_NAME
+            String debsDir = 'debs'
+
+            if (isRC) {
+                repoComponent = 'rc'
+            }
+            logger.info("Debian repo component is set to: '$repoComponent'")
+
+            dir(debsDir) {
+                pkgsList.each {
+                    unstash name: it
+                }
+            }
+            logger.info("Uploading debian packages to '${repoName}/${repoComponent}' sovrin repo: $pkgsList")
+
+            dir("sovrin-packaging") {
+                downloadPackagingUtils()
+            }
+
+            withCredentials([file(credentialsId: 'SovrinRepoSSHKey', variable: 'sovrin_key')]) {
+                sh "./sovrin-packaging/upload_debs.py $debsDir $repoName $repoComponent --host $env.SOVRIN_REPO_HOST --ssh-key $sovrin_key --force-backup"
+            }
+
+            notifier.email {
+                to = '$DEFAULT_RECIPIENTS ' + (this.env.LIBSOVTOKEN_DEF_RECIPIENTS ?: '')
+                subject = "[$projectName] new packages ${pkgsList.join(' ')} published to '${repoName}/${repoComponent}' repo"
+                body = """
+                    |New debian packages ${pkgsList.join(' ')} were built and published to '${repoName}/${repoComponent}' repo
+                    |
+                    |Build:
+                    |   Project: \$PROJECT_NAME
+                    |   Url: ${this.env.BUILD_URL}
+                    |   Number: ${this.env.BUILD_NUMBER}
+                    |
+                    |Check console output at ${this.env.BUILD_URL}console to view the details.
+                """.stripMargin()
+            }
+        }
     }
 }, { err ->
     if (err) {
@@ -193,6 +227,7 @@ pipelineWrapper({
 
     stage("Build result notification") {
         notifier.email {
+            subject = "[$projectName] \$DEFAULT_SUBJECT"
             to = '$DEFAULT_RECIPIENTS ' + (this.env.LIBSOVTOKEN_DEF_RECIPIENTS ?: '')
         }
     }

diff --git a/devops/aws-codebuild/Jenkinsfile.ci b/devops/aws-codebuild/Jenkinsfile.ci
@@ -90,7 +90,7 @@ pipelineWrapper({
                                 [name: 'OSNAME', value: osname],
                                 [name: 'MAKE_GOALS', value: goals.join(' ')],
                                 [name: 'SRC_DIR_NAME', value: proj],
-                                [name: 'PYTEST_ARGS', value: "-l -v --junit-xml=/tmp/artifacts/logs/test.${proj}.xml"],
+                                [name: 'PYTEST_ARGS', value: "-l -vv --junit-xml=/tmp/artifacts/logs/test.${proj}.xml"],
                             ]
 
                             onArtifacts = {

diff --git a/devops/denv/Dockerfile b/devops/denv/Dockerfile
@@ -37,7 +37,7 @@ RUN apt-get update -y && apt-get install -y \
         zlib1g-dev \
         liblz4-dev \
         libsnappy-dev \
-    # libindy 
+    # libindy
         python3-nacl \
         libindy-crypto=0.4.5 \
         libindy=1.8.2

diff --git a/devops/docker/ci/xenial/Dockerfile b/devops/docker/ci/xenial/Dockerfile
@@ -25,7 +25,7 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 68DB5E88 \
 RUN add-apt-repository "deb https://repo.sovrin.org/sdk/deb xenial master" \
     && apt-get update && apt-get install -y \
         libindy-crypto=0.4.5 \
-        libindy=1.9.0~1122 \
+        libindy=1.9.0~1132 \
         libsovtoken=0.9.7~66 \
     && rm -rf /var/lib/apt/lists/*
 

diff --git a/docs/deploy_instructions.md b/docs/deploy_instructions.md
@@ -4,7 +4,7 @@
 
 1) POOL_UPGRADE with the `indy-node` package version 1.6.70 to support `sovrin` package in POOL_UPGRADE txn
 2) Upgrade the CLI (locally) to 1.6.6 libindy, indy-CLI
-3) POOL_UPGRADE with the `sovrin` package version `1.1.20` to have TDE-ready version of plugins installed. **TODO:  This version is still a stub. Define first version with plugins and with trust anchor restriction**
+3) POOL_UPGRADE with the `sovrin` package version `1.1.20` to have TDE-ready version of plugins installed. **TODO:  This version is still a stub. Define first version with plugins and with endorser restriction**
 It might be done with [POOL_UPGRADE](https://github.com/hyperledger/indy-node/blob/master/docs/pool-upgrade.md) transaction. You can do it from `indy-cli` with this command:
 ```
 Command:
@@ -131,8 +131,8 @@ Tokens can be distributed via `XFER_PUBLIC`. To make it we need to go through th
 
 Setting fees process is similar to minting -- you need the build transaction, get the approval (signature) of trustees and to send it to the ledger. Details of this process are described in [setting fees guide](https://github.com/sovrin-foundation/libsovtoken/blob/master/doc/set_fees_process.md)
 
-## Configure the ledger to remove the need for trust anchors
+## Configure the ledger to remove the need for endorsers
 
-When fees are set for the transactions, we can allow to write to the ledger from DIDs that are not a trust anchor (of course if the have tokens to pay the fee). For this we need to make a POOL_UPGRADE once again for a `sovrin` package of version `1.1.21`. **TODO:  This version is still a stub yoo. Define first version with plugins and without trust anchor restriction**
+When fees are set for the transactions, we can allow to write to the ledger from DIDs that are not a endorser (of course if the have tokens to pay the fee). For this we need to make a POOL_UPGRADE once again for a `sovrin` package of version `1.1.21`. **TODO:  This version is still a stub yoo. Define first version with plugins and without endorser restriction**