Docker based reproducible builds

.gitignore

@@ -1 +1,2 @@
 *~
+build

.travis.yml

@@ -0,0 +1,9 @@
+sudo: required
+services:
+  - docker
+
+script:
+  - docker build -t decred/decred-binaries .
+  - docker run --rm -ti -e TAG="v1.2.0" -e PROD=1 -v $(pwd)/build:/build
+      decred/decred-binaries
+  - diff test-manifest-v1.2.0.txt ./build/decred-v1.2.0/manifest-v1.2.0.txt

Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.10.2
+
+WORKDIR /app
+
+RUN go get -u github.com/golang/dep/...
+
+RUN apt-get update && apt-get install -y zip && rm -rf /var/lib/apt/lists/*
+
+ADD decredbuild.sh .
+ADD fetch.sh .
+
+CMD ./fetch.sh && \
+		./decredbuild.sh $TAG

README.md

@@ -8,14 +8,16 @@ Decred.
 
 This repository contains releases for the following software:
 
-* [dcrd](https://github.com/decred/dcrd)/[dcrwallet](https://github.com/decred/dcrwallet)
+* [dcrd](https://github.com/decred/dcrd)/[dcrwallet](
+  https://github.com/decred/dcrwallet)
 * [Paymetheus](https://github.com/decred/Paymetheus)
 * [gominer](https://github.com/decred/gominer)
 * [decrediton](https://github.com/decred/decrediton)
 
 ## Release notes
 
-Notes for the current and previous releases can be found in the [Release Notes](./release-notes.md) document.
+Notes for the current and previous releases can be found in the
+[Release Notes](./release-notes.md) document.
 
 ## Verifying Binaries
 
@@ -30,8 +32,10 @@ binaries in that release.  To verify these, you will need:
 
 The steps to verify the binaries are as follows:
 
-1. Download the file manifest, the signature for the file manifest, and the zip/tarball for your OS from here.
-2. Obtain the SHA256 value for the zip/tarball for your OS and check that it matches the value in the file manifest, e.g. for 64-bit Linux
+1. Download the file manifest, the signature for the file manifest, and 
+the zip/tarball for your OS from here.
+2. Obtain the SHA256 value for the zip/tarball for your OS and check that 
+it matches the value in the file manifest, e.g. for 64-bit Linux
 
    ```
    $ sha256sum linux-amd64-20160127-02.tar.gz
@@ -70,3 +74,42 @@ This repo only contains build archives, build scripts, and similar
 content.  For the source code, please see the individual repositories
 at: https://github.com/decred/
 
+## Building Binaries
+
+To build the binaries, use the provided Docker image.  First, ensure it
+is built on your system:
+
+```
+docker build -t decred/decred-binaries .
+```
+
+With this image, you can build the production binaries by specifying the
+required tag.  This will automatically fetch the specified tag from
+Github, build it inside the docker container, and place those packages in
+a directory of your choosing.  Replace the TAG environment variable with
+the required tag, and specify the local folder you wish those packages to
+be placed in.  The below example builds version 1.2.0, and places it into
+./build on your local machine:
+
+```
+docker run --rm -ti -e TAG="v1.2.0" -e PROD=1 -v $(pwd)/build:/build \
+    decred/decred-binaries
+```
+
+If you have local folders cloned that you want to build, instead of
+fetching, then mount those as well.  E.g.:
+
+```
+docker run --rm -ti -e TAG="v1.2.0" -e PROD=1 -v $GOPATH/src:/go/src \
+    -v $(pwd)/build:/build decred/decred-binaries
+```
+
+Leave TAG unset if you wish to build the version you have already cloned
+without verifying it matches the specified tag.
+
+### Environment Variables
+
+* TAG: Specify the git tag to have it fetch these packages automatically.
+  Leave unset to use latest commit (if repo's do not exist locally) or
+  local version (if already cloned)
+* PROD: Set to 1 if you want to create a production build

decredbuild.sh

@@ -14,16 +14,42 @@ if [[ $1x = x ]]; then
     TAG=$DATE-$VERSION
 else
     TAG=$1
+fi
+
+PARENT=$(pwd)
+
+# Set the unix timestamp to match that of dcrd git commit, to have a 
+# stable anchor for setting file timestamps for reproducible builds
+cd $GOPATH/src/github.com/decred/dcrd
+# TZ is used by date command
+TZ='UTC'
+UNIXTIME=$(git log -1 --format=%ct)
+TOUCHTIME=$(date --date='@'$UNIXTIME +%Y%m%d%H%M.%S)
+TARTIME=$(date --date='@'$UNIXTIME "+%Y%m%d %H:%M:%S")
+
+if [[ $PROD = 1 ]]; then
+    echo "********************"
+    echo "* Production build *"
+    echo "********************"
     REL=(-ldflags "-X main.appBuild=release")
-    DCRWALLET_REL=(-ldflags "-X github.com/decred/dcrwallet/version.BuildMetadata=release")
+    DCRWALLET_REL=(-ldflags \
+      "-X github.com/decred/dcrwallet/version.BuildMetadata=release")
+else
+    echo "*********************"
+    echo "* Development build *"
+    echo "*********************"
 fi
 
+cd $PARENT
 PACKAGE=decred
-MAINDIR=$PACKAGE-$TAG
+MAINDIR=/build/$PACKAGE-$TAG
 mkdir -p $MAINDIR
 cd $MAINDIR
 
-SYS="windows-386 windows-amd64 openbsd-386 openbsd-amd64 linux-386 linux-amd64 linux-arm linux-arm64 darwin-amd64 dragonfly-amd64 freebsd-386 freebsd-amd64 freebsd-arm netbsd-386 netbsd-amd64 solaris-amd64"
+SYS="windows-386 windows-amd64 openbsd-386 openbsd-amd64 linux-386 \
+linux-amd64 linux-arm linux-arm64 darwin-amd64 dragonfly-amd64 \
+freebsd-386 freebsd-amd64 freebsd-arm netbsd-386 netbsd-amd64 \
+solaris-amd64"
 
 # Use the first element of $GOPATH in the case where GOPATH is a list
 # (something that is totally allowed).
@@ -35,19 +61,28 @@ for i in $SYS; do
     mkdir $PACKAGE-$i-$TAG
     cd $PACKAGE-$i-$TAG
     echo "Building:" $OS $ARCH
-    env GOOS=$OS GOARCH=$ARCH go build "${REL[@]}" github.com/decred/dcrd
-    env GOOS=$OS GOARCH=$ARCH go build "${REL[@]}" github.com/decred/dcrd/cmd/dcrctl
-    env GOOS=$OS GOARCH=$ARCH go build "${REL[@]}" github.com/decred/dcrd/cmd/promptsecret
-    env GOOS=$OS GOARCH=$ARCH go build "${DCRWALLET_REL[@]}" github.com/decred/dcrwallet
+    env GOOS=$OS GOARCH=$ARCH go build "${REL[@]}" \
+      github.com/decred/dcrd
+    env GOOS=$OS GOARCH=$ARCH go build "${REL[@]}" \
+      github.com/decred/dcrd/cmd/dcrctl
+    env GOOS=$OS GOARCH=$ARCH go build "${REL[@]}" \
+      github.com/decred/dcrd/cmd/promptsecret
+    env GOOS=$OS GOARCH=$ARCH go build "${DCRWALLET_REL[@]}"\
+      github.com/decred/dcrwallet
     cp $GPATH/src/github.com/decred/dcrd/cmd/dcrctl/sample-dcrctl.conf .
     cp $GPATH/src/github.com/decred/dcrwallet/sample-dcrwallet.conf .
     cd ..
     if [[ $OS = "windows" ]]; then
-	zip -r $PACKAGE-$i-$TAG.zip $PACKAGE-$i-$TAG
-	tar -cvzf $PACKAGE-$i-$TAG.tar.gz $PACKAGE-$i-$TAG
-    else
-	tar -cvzf $PACKAGE-$i-$TAG.tar.gz $PACKAGE-$i-$TAG
+        # To enable reproducible builds, we need to change the timestamp 
+        # of the files, pass those files in in a fixed order, and don't 
+        # pass in extra file attributes
+        find $PACKAGE-$i-$TAG -exec touch -t $TOUCHTIME {} \;
+        find $PACKAGE-$i-$TAG | sort | zip -X $PACKAGE-$i-$TAG.zip -@
     fi
+    # Strip out name and timestamp data so that builds can be reproducible
+    tar -c --mtime="$TARTIME" --sort=name --owner=0 --group=0 \
+      --numeric-owner $PACKAGE-$i-$TAG | gzip -9 -n > \
+      $PACKAGE-$i-$TAG.tar.gz
     rm -r $PACKAGE-$i-$TAG
 done
 

fetch.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Copyright (c) 2018 The Decred developers
+# Use of this source code is governed by the ISC
+# license.
+
+# Script to fetch either specified TAG for dcrd and dcrwallet, or else 
+# latest comment
+
+PROJECTS="dcrd dcrwallet"
+
+if [ -z "$TAG" ]; then
+    echo "No tag set"
+    BRANCH=""
+else
+    echo "Tag set to $TAG"
+    BRANCH="--branch $TAG"
+fi
+
+mkdir -p $GOPATH/src/github.com/decred
+
+for i in $PROJECTS; do
+    PROJECT=$(echo $i)
+    cd $GOPATH/src/github.com/decred
+
+    # Only fetch if project doesn't exist:
+    if [ ! -d "$PROJECT" ]; then
+        echo "No repo found, so fetching $PROJECT"
+        git clone --depth 1 $BRANCH https://github.com/decred/$PROJECT
+    fi
+
+    cd $PROJECT
+
+    if [ -n "$TAG" ]; then
+        # Check if specified tag is attached to current commit
+        CURRENT_TAG=$(git tag -l --points-at HEAD | grep "^$TAG$")
+
+        # Tag provided, so check that the tag of this branch matches the 
+        # provided tag:
+        echo "Verifying $PROJECT repo tag matches $TAG..."
+
+        if [[ "$CURRENT_TAG" != "$TAG" ]]; then
+            # Don't continue if tag of repo does not match
+            echo "Repo tag of '$CURRENT_TAG' does not match specified " \
+             "tag of '$TAG'"
+            exit 1
+        else
+            echo "...good."
+        fi
+    fi
+
+    echo "Running dep for $PROJECT..."
+    dep ensure
+    echo "...done."
+done

test-manifest-v1.2.0.txt

@@ -0,0 +1,18 @@
+2400328d286f481bc7e39dcbf072d66cf4a941fffd3b4cb399a3fbbe5c777f90  decred-darwin-amd64-v1.2.0.tar.gz
+bd918fd59efc331adac032da84df4c7232c9065ee30b30e0c4c6e676c7cfd24e  decred-dragonfly-amd64-v1.2.0.tar.gz
+eb7bc2d1c555996cef242cc4c08a5ca92d69c7ada9a35ef127ebdafdca34f80c  decred-freebsd-386-v1.2.0.tar.gz
+1ca95f5c60f12eee42ae4498ac3e2104e32aa8580263b7ab22efcbbba34c0d06  decred-freebsd-amd64-v1.2.0.tar.gz
+f0aae8e5ef4af677069bf32d2869668c1f88d73dbf5a37bcf3f41c220e4ee9c5  decred-freebsd-arm-v1.2.0.tar.gz
+e341b499b9624205e77a89254fb99759e8aac31cd69ba3dd44c9b0bf165fabfa  decred-linux-386-v1.2.0.tar.gz
+88c2ce7441dd9dd01c39fdb6d0ed0e7972179807d619d12863ad8eef975bd82d  decred-linux-amd64-v1.2.0.tar.gz
+18fa0b7a928604a4840847e583aec83dd1997fa56c2f15f17c4aa7d2215f3a5f  decred-linux-arm-v1.2.0.tar.gz
+d5ed86d123863e53a5e1c5c4848a7527088bd3822f620af130967ff3cd306f9e  decred-linux-arm64-v1.2.0.tar.gz
+ebd5633d6658dcfd498337ba38054ce0f5b6b21ad0ba6b203a7a25cf3666ce32  decred-netbsd-386-v1.2.0.tar.gz
+789a73e318af31b3916a61994e1ea0bf131b7b6c39cde3d993d5b03c85a79dd7  decred-netbsd-amd64-v1.2.0.tar.gz
+9c7095af8f2e846cbff8f005613ec675b415e896df7afd81a6f5389d8225606d  decred-openbsd-386-v1.2.0.tar.gz
+4c98b7b6261b8b2e01406cba1686d26514cdd35bda77f7043c0c4d3cc46dc5a8  decred-openbsd-amd64-v1.2.0.tar.gz
+88a1cf4e8b91235f4809b07bb7a6a9f4b6623c88798f151e700272411e6e5c57  decred-solaris-amd64-v1.2.0.tar.gz
+6952888716ab7729ac718059f3e0dc4db882a34ae3cd1db5d013599c676ad4c7  decred-windows-386-v1.2.0.tar.gz
+c94eb4fc8ea0c1a775e45a06cbf7f6c4ab687598f4911c82a7619dea515997c6  decred-windows-386-v1.2.0.zip
+0039081fc54c5a171370a01dc5c62b9abfa790cd2f1be43753e203cccd3566fe  decred-windows-amd64-v1.2.0.tar.gz
+a56c7eacff6dffe574fb14719fc1382304e3f8ea5773b60148086707da490ef6  decred-windows-amd64-v1.2.0.zip