[FAB-6915] Fix chaincode startup issue in tests

This issue was caused by dangling chaincode images created with a different version of ccenv. This patch cleans up chaincode images and cleans up the docker-compose fixtures. Change-Id: I5caadf0bba6ba2bf120485030efef3098890b0b1 Signed-off-by: Troy Ronda <[email protected]>
hyperledger · Nov 8, 2017 · 562ea23 · 562ea23
1 parent 2443ac7
commit 562ea23
Show file tree

Hide file tree

Showing 6 changed files with 138 additions and 37 deletions.
diff --git a/Makefile b/Makefile
@@ -51,9 +51,13 @@ THIRDPARTY_FABRIC_CA_COMMIT ?= v1.1.0-preview
 THIRDPARTY_FABRIC_BRANCH    ?= master
 THIRDPARTY_FABRIC_COMMIT    ?= v1.1.0-preview
 
+# Force removal of images in cleanup
+FIXTURE_DOCKER_REMOVE_FORCE ?= false
+
 # Local variables used by makefile
-PACKAGE_NAME := github.com/hyperledger/fabric-sdk-go
-ARCH         := $(shell uname -m)
+PACKAGE_NAME         := github.com/hyperledger/fabric-sdk-go
+ARCH                 := $(shell uname -m)
+FIXTURE_PROJECT_NAME := fabsdkgo
 
 # The version of dep that will be installed by depend-install (or in the CI)
 GO_DEP_COMMIT := v0.3.1
@@ -77,6 +81,8 @@ export GO_LDFLAGS
 export GO_DEP_COMMIT
 export GO_TAGS
 export GO_TESTFLAGS
+export DOCKER_CMD
+export DOCKER_COMPOSE_CMD
 
 all: checks unit-test integration-test
 
@@ -148,8 +154,7 @@ populate-clean:
 	rm -Rf vendor
 
 clean:
-	$(GO_CMD) clean
-	rm -Rf /tmp/enroll_user /tmp/msp /tmp/keyvaluestore /tmp/hfc-kvs
-	rm -f integration-report.xml report.xml
-	rm -f test/fixtures/tls/fabricca/certs/server/ca.org*.example.com-cert.pem
-	cd test/fixtures && $(DOCKER_COMPOSE_CMD) -f docker-compose.yaml -f docker-compose-nopkcs11-test.yaml -f docker-compose-pkcs11-test.yaml down
+	-$(GO_CMD) clean
+	-rm -Rf /tmp/enroll_user /tmp/msp /tmp/keyvaluestore /tmp/hfc-kvs
+	-rm -f integration-report.xml report.xml
+	-FIXTURE_PROJECT_NAME=$(FIXTURE_PROJECT_NAME) DOCKER_REMOVE_FORCE=$(FIXTURE_DOCKER_REMOVE_FORCE) test/scripts/clean_integration.sh
diff --git a/test/fixtures/.env b/test/fixtures/.env
@@ -11,16 +11,21 @@
 # Default architecture
 ARCH=x86_64
 
+CORE_PEER_NETWORKID=fabsdkgo
+COMPOSE_PROJECT_NAME=fabsdkgo
+
 FABRIC_CA_FIXTURE_TAG=1.0.3
 FABRIC_ORDERER_FIXTURE_TAG=1.0.3
 FABRIC_PEER_FIXTURE_TAG=1.0.3
 FABRIC_COUCHDB_FIXTURE_TAG=1.0.3
 FABRIC_BUILDER_FIXTURE_TAG=1.0.3
 FABRIC_BASEOS_FIXTURE_TAG=0.4.2
+FABRIC_BASEIMAGE_FIXTURE_TAG=0.4.2
 
 FABRIC_CA_FIXTURE_IMAGE=hyperledger/fabric-ca
 FABRIC_ORDERER_FIXTURE_IMAGE=hyperledger/fabric-orderer
 FABRIC_PEER_FIXTURE_IMAGE=hyperledger/fabric-peer
 FABRIC_COUCHDB_FIXTURE_IMAGE=hyperledger/fabric-couchdb
 FABRIC_BUILDER_FIXTURE_IMAGE=hyperledger/fabric-ccenv
-FABRIC_BASEOS_FIXTURE_IMAGE=hyperledger/fabric-baseimage
+FABRIC_BASEOS_FIXTURE_IMAGE=hyperledger/fabric-baseos
+FABRIC_BASEIMAGE_FIXTURE_IMAGE=hyperledger/fabric-baseimage
diff --git a/test/fixtures/docker-compose-nopkcs11-test.yaml b/test/fixtures/docker-compose-nopkcs11-test.yaml
@@ -7,15 +7,18 @@ version: '2'
 
 services:
 
-  fabric-sdk-integration-tests:
-    container_name: fabric-sdk-integration-test
-    image: ${FABRIC_BASEOS_FIXTURE_IMAGE}:${ARCH}-${FABRIC_BASEOS_FIXTURE_TAG}
+  integration-tests:
+    image: ${FABRIC_BASEIMAGE_FIXTURE_IMAGE}:${ARCH}-${FABRIC_BASEIMAGE_FIXTURE_TAG}
     environment:
       - GO_TAGS
       - GO_TESTFLAGS
     volumes:
       - ../../:/opt/gopath/src/github.com/hyperledger/fabric-sdk-go
     command: /opt/gopath/src/github.com/hyperledger/fabric-sdk-go/test/scripts/integration.sh
     depends_on:
-      - orderer.example.com
+      - org1ca1
+      - org2ca1
+      - org1peer1
+      - org2peer1
+      - orderer1
       - builder
diff --git a/test/fixtures/docker-compose-pkcs11-test.yaml b/test/fixtures/docker-compose-pkcs11-test.yaml
@@ -7,8 +7,7 @@ version: '2'
 
 services:
 
-  fabric-sdk-pkcs11-integration-tests:
-    container_name: fabric-sdk-pkcs11-integration-test
+  pkcs11-integration-tests:
     image: softhsm2-image
     environment:
       - FABRIC_SDK_CLIENT_BCCSP_SECURITY_DEFAULT_PROVIDER=PKCS11
@@ -18,5 +17,9 @@ services:
       - ../../:/opt/gopath/src/github.com/hyperledger/fabric-sdk-go
     command: /opt/gopath/src/github.com/hyperledger/fabric-sdk-go/test/scripts/integration.sh
     depends_on:
-      - orderer.example.com
+      - org1ca1
+      - org2ca1
+      - org1peer1
+      - org2peer1
+      - orderer1
       - builder
diff --git a/test/fixtures/docker-compose.yaml b/test/fixtures/docker-compose.yaml
@@ -6,7 +6,7 @@
 version: '2'
 
 services:
-  ca0:
+  org1ca1:
     image: ${FABRIC_CA_FIXTURE_IMAGE}:${ARCH}-${FABRIC_CA_FIXTURE_TAG}
     environment:
       - FABRIC_CA_HOME=/etc/hyperledger/fabric-ca-server
@@ -25,9 +25,12 @@ services:
     volumes:
       - ./channel/crypto-config/peerOrganizations/org1.example.com/ca/:/etc/hyperledger/fabric-ca-server-config
       - ./tls/fabricca/certs/server:/etc/hyperledger/fabric-ca-server-config
-    container_name: ca_peerOrg1
+    networks:
+      default:
+        aliases:
+          - ca_peerOrg1
 
-  ca1:
+  org2ca1:
     image: ${FABRIC_CA_FIXTURE_IMAGE}:${ARCH}-${FABRIC_CA_FIXTURE_TAG}
     environment:
       - FABRIC_CA_HOME=/etc/hyperledger/fabric-ca-server
@@ -46,10 +49,12 @@ services:
     volumes:
       - ./channel/crypto-config/peerOrganizations/org2.example.com/ca/:/etc/hyperledger/fabric-ca-server-config
       - ./tls/fabricca/certs/server:/etc/hyperledger/fabric-ca-server-config
-    container_name: ca_peerOrg2
+    networks:
+      default:
+        aliases:
+          - ca_peerOrg2
 
-  orderer.example.com:
-    container_name: orderer.example.com
+  orderer1:
     image: ${FABRIC_ORDERER_FIXTURE_IMAGE}:${ARCH}-${FABRIC_ORDERER_FIXTURE_TAG}
     environment:
       - ORDERER_GENERAL_LOGLEVEL=debug
@@ -73,79 +78,119 @@ services:
         - ./channel:/etc/hyperledger/configtx
         - ./channel/crypto-config/ordererOrganizations/example.com/orderers/orderer.example.com/msp:/etc/hyperledger/msp/orderer
         - ./channel/crypto-config/ordererOrganizations/example.com/orderers/orderer.example.com/tls:/etc/hyperledger/tls/orderer
+    networks:
+      default:
+        aliases:
+          - orderer.example.com
 
-  peer0.org1.example.com:
-    container_name: peer0.org1.example.com
+  org1peer1:
     image: ${FABRIC_PEER_FIXTURE_IMAGE}:${ARCH}-${FABRIC_PEER_FIXTURE_TAG}
     environment:
       - CORE_VM_ENDPOINT=unix:///host/var/run/docker.sock
       - CORE_PEER_ID=peer0.org1.example.com
       - CORE_LOGGING_PEER=debug
+      - CORE_CHAINCODE_STARTUPTIMEOUT=30s
+      - CORE_CHAINCODE_LOGGING_SHIM=debug
+      - CORE_CHAINCODE_LOGGING_LEVEL=debug
+      - CORE_CHAINCODE_BUILDER=${FABRIC_BUILDER_FIXTURE_IMAGE}:${ARCH}-${FABRIC_BUILDER_FIXTURE_TAG}
+      - CORE_CHAINCODE_GOLANG_RUNTIME=${FABRIC_BASEOS_FIXTURE_IMAGE}:${ARCH}-${FABRIC_BASEOS_FIXTURE_TAG}      
       ## the following setting redirects chaincode container logs to the peer container logs
       - CORE_VM_DOCKER_ATTACHSTDOUT=true
       - CORE_PEER_LOCALMSPID=Org1MSP
       - CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/msp/peer/
-      - CORE_PEER_ADDRESS=peer0.org1.example.com:7051
+      - CORE_PEER_ADDRESS=0.0.0.0:7051
+      - CORE_PEER_CHAINCODELISTENADDRESS=peer0.org1.example.com:7052
+      - CORE_PEER_ADDRESSAUTODETECT=true
+      - CORE_PEER_GOSSIP_EXTERNALENDPOINT=peer0.org1.example.com:7051
       - CORE_PEER_TLS_ENABLED=true
       - CORE_PEER_TLS_KEY_FILE=/etc/hyperledger/tls/peer/server.key
       - CORE_PEER_TLS_CERT_FILE=/etc/hyperledger/tls/peer/server.crt
       - CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/tls/peer/ca.crt
       # # the following setting starts chaincode containers on the same
       # # bridge network as the peers
       # # https://docs.docker.com/compose/networking/
-      - CORE_VM_DOCKER_HOSTCONFIG_NETWORKMODE=fixtures_default
+      - CORE_PEER_NETWORKID=${CORE_PEER_NETWORKID}
+      - CORE_VM_DOCKER_HOSTCONFIG_NETWORKMODE=${CORE_PEER_NETWORKID}_default      
     #comment out logging.driver in order to render the debug logs 
     logging:
       driver: none 
     working_dir: /opt/gopath/src/github.com/hyperledger/fabric
     command: peer node start
     ports:
-      - 7051:7051
-      - 7053:7053
+      - "7051:7051"
+      - "7053:7053"
+    expose:
+      - "7051"
+      - "7052"
+      - "7053"
     volumes:
         - /var/run/:/host/var/run/
         - ./channel/crypto-config/peerOrganizations/org1.example.com/peers/peer0.org1.example.com/msp:/etc/hyperledger/msp/peer
         - ./channel/crypto-config/peerOrganizations/org1.example.com/peers/peer0.org1.example.com/tls:/etc/hyperledger/tls/peer
+    networks:
+      default:
+        aliases:
+          - peer0.org1.example.com
     depends_on:
-      - orderer.example.com
+      - orderer1
       - builder
 
-  peer0.org2.example.com:
-    container_name: peer0.org2.example.com
+  org2peer1:
     image: ${FABRIC_PEER_FIXTURE_IMAGE}:${ARCH}-${FABRIC_PEER_FIXTURE_TAG}
     environment:
       - CORE_VM_ENDPOINT=unix:///host/var/run/docker.sock
       - CORE_PEER_ID=peer0.org2.example.com
       - CORE_LOGGING_PEER=debug
+      - CORE_CHAINCODE_STARTUPTIMEOUT=30s
+      - CORE_CHAINCODE_LOGGING_SHIM=debug
+      - CORE_CHAINCODE_LOGGING_LEVEL=debug
+      - CORE_CHAINCODE_BUILDER=${FABRIC_BUILDER_FIXTURE_IMAGE}:${ARCH}-${FABRIC_BUILDER_FIXTURE_TAG}
+      - CORE_CHAINCODE_GOLANG_RUNTIME=${FABRIC_BASEOS_FIXTURE_IMAGE}:${ARCH}-${FABRIC_BASEOS_FIXTURE_TAG}
+      ## the following setting redirects chaincode container logs to the peer container logs
+      - CORE_VM_DOCKER_ATTACHSTDOUT=true      
       - CORE_PEER_LOCALMSPID=Org2MSP
       - CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/msp/peer/
-      - CORE_PEER_ADDRESS=peer0.org2.example.com:7051
+      - CORE_PEER_ADDRESS=0.0.0.0:7051
+      - CORE_PEER_CHAINCODELISTENADDRESS=peer0.org2.example.com:7052
+      - CORE_PEER_ADDRESSAUTODETECT=true
+      - CORE_PEER_GOSSIP_EXTERNALENDPOINT=peer0.org2.example.com:7051
       - CORE_PEER_TLS_ENABLED=true
       - CORE_PEER_TLS_KEY_FILE=/etc/hyperledger/tls/peer/server.key
       - CORE_PEER_TLS_CERT_FILE=/etc/hyperledger/tls/peer/server.crt
       - CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/tls/peer/ca.crt
       # # the following setting starts chaincode containers on the same
       # # bridge network as the peers
       # # https://docs.docker.com/compose/networking/
-      - CORE_VM_DOCKER_HOSTCONFIG_NETWORKMODE=fixtures_default
+      - CORE_PEER_NETWORKID=${CORE_PEER_NETWORKID}
+      - CORE_VM_DOCKER_HOSTCONFIG_NETWORKMODE=${CORE_PEER_NETWORKID}_default     
     #comment out logging.driver in order to render the debug logs
     logging:
       driver: none  
     working_dir: /opt/gopath/src/github.com/hyperledger/fabric
     command: peer node start
     ports:
-      - 8051:7051
-      - 8053:7053
+      - "8051:7051"
+      - "8053:7053"
+    expose:
+      - "7051"
+      - "7052"
+      - "7053"
     volumes:
         - /var/run/:/host/var/run/
         - ./channel/crypto-config/peerOrganizations/org2.example.com/peers/peer0.org2.example.com/msp:/etc/hyperledger/msp/peer
-        - ./channel/crypto-config/peerOrganizations/org2.example.com/peers/peer0.org2.example.com/tls:/etc/hyperledger/tls/peer
+        - ./channel/crypto-config/peerOrganizations/org2.example.com/peers/peer0.org2.example.com/tls:/etc/hyperledger/tls/peer 
+    networks:
+      default:
+        aliases:
+          - peer0.org2.example.com
     depends_on:
-      - orderer.example.com
+      - orderer1
       - builder
 
-
  # builder is only here to create a dependency on the image (not used as part of compose)
   builder:
     image: ${FABRIC_BUILDER_FIXTURE_IMAGE}:${ARCH}-${FABRIC_BUILDER_FIXTURE_TAG}
     command: tail -F anything
+
+networks:
+    default:
diff --git a/test/scripts/clean_integration.sh b/test/scripts/clean_integration.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright SecureKey Technologies Inc. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+DOCKER_CMD="${DOCKER_CMD:-docker}"
+DOCKER_COMPOSE_CMD="${DOCKER_COMPOSE_CMD:-docker-compose}"
+FIXTURE_PROJECT_NAME="${FIXTURE_PROJECT_NAME:-fabsdkgo}"
+DOCKER_REMOVE_FORCE="${DOCKER_REMOVE_FORCE:-false}"
+
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+rm -f ${SCRIPT_PATH}/../fixtures/tls/fabricca/certs/server/ca.org*.example.com-cert.pem
+echo "Removing docker-compose network created from fixtures ..."
+COMPOSE_PROJECT_NAME=FIXTURE_PROJECT_NAME cd ${SCRIPT_PATH}/../fixtures && $DOCKER_COMPOSE_CMD -f docker-compose.yaml -f docker-compose-nopkcs11-test.yaml -f docker-compose-pkcs11-test.yaml down
+
+DOCKER_REMOVE_ARGS=
+if [ "$DOCKER_REMOVE_FORCE" = "true" ]; then
+    DOCKER_REMOVE_ARGS="-f"
+fi
+
+CONTAINERS=$($DOCKER_CMD ps -a | grep "${FIXTURE_PROJECT_NAME}-peer.\.org.\.example\.com-" | awk '{print $1}')
+IMAGES=$($DOCKER_CMD images | grep "${FIXTURE_PROJECT_NAME}-peer.\.org.\.example\.com-" | awk '{print $1}')
+
+if [ ! -z "$CONTAINERS" ]; then
+    if [ "$DOCKER_REMOVE_FORCE" = "true" ]; then
+        echo "Stopping chaincode containers created from fixtures ..."
+        $DOCKER_CMD stop $CONTAINERS
+    fi
+
+    echo "Removing chaincode containers created from fixtures ..."
+    $DOCKER_CMD rm $DOCKER_REMOVE_ARGS $CONTAINERS
+fi
+
+if [ ! -z "$IMAGES" ]; then
+    echo "Removing chaincode images created from fixtures ..."
+    $DOCKER_CMD rmi $DOCKER_REMOVE_ARGS $IMAGES
+fi