Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Linux compile on Debian 6. #1

Merged
merged 4 commits into from
Sep 9, 2015
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 23 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,28 @@ all: packetbeat/deb packetbeat/rpm packetbeat/darwin packetbeat/win packetbeat/b

.PHONY: packetbeat topbeat
packetbeat topbeat: xgo-image build
cd build && xgo -image=tudorg/beats-builder -static \
cd build && xgo -image=tudorg/beats-builder \
-before-build=../xgo-scripts/$@_before_build.sh \
-branch $(RELEASE) \
github.com/elastic/$@

%/deb: % build/god-linux-386 build/god-linux-amd64 fpm-image
.PHONY: packetbeat-linux topbeat-linux
packetbeat-linux: xgo-image build
cd build && xgo -image=tudorg/beats-builder-deb6 \
-before-build=../xgo-scripts/packetbeat_before_build.sh \
-branch $(RELEASE) \
github.com/elastic/packetbeat
topbeat-linux: xgo-image build
cd build && xgo -image=tudorg/beats-builder-deb6 \
-before-build=../xgo-scripts/topbeat_before_build.sh \
-branch $(RELEASE) \
github.com/elastic/topbeat

%/deb: %-linux build/god-linux-386 build/god-linux-amd64 fpm-image
ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/debian/build.sh
ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/debian/build.sh

%/rpm: % build/god-linux-386 build/god-linux-amd64 fpm-image
%/rpm: %-linux build/god-linux-386 build/god-linux-amd64 fpm-image
ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/centos/build.sh
ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/centos/build.sh

Expand All @@ -28,7 +40,7 @@ packetbeat topbeat: xgo-image build
%/win: % fpm-image
ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/windows/build.sh

%/bin: % fpm-image
%/bin: %-linux fpm-image
ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/binary/build.sh
ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/binary/build.sh

Expand All @@ -39,7 +51,8 @@ deps:

.PHONY: xgo-image
xgo-image:
docker build -t tudorg/beats-builder docker/xgo-image/
cd docker/xgo-image/; ./build.sh
cd docker/xgo-image-deb6/; ./build.sh

.PHONY: fpm-image
fpm-image:
Expand All @@ -60,11 +73,15 @@ s3-nightlies-upload: all
echo $(BUILDID) > build/upload/build_id.txt
aws s3 cp --recursive --acl public-read build/upload s3://beats-nightlies

.PHONY: release-upload
release-upload:
aws s3 cp --recursive --acl public-read build/upload s3://download.elasticsearch.org/beats/

.PHONY: run-interactive
run-interactive:
docker run -t -i -v $(shell pwd)/build:/build \
-v $(shell pwd)/xgo-scripts/:/scripts \
--entrypoint=bash tudorg/beats-builder
--entrypoint=bash tudorg/beats-builder-deb6
.PHONY: clean
clean:
rm -rf build/ || true
44 changes: 44 additions & 0 deletions docker/xgo-image-deb6/base/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Go cross compiler (xgo): Base cross-compilation layer
# Copyright (c) 2014 Péter Szilágyi. All rights reserved.
#
# Released under the MIT license.

FROM debian:6

MAINTAINER Péter Szilágyi <[email protected]>
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

?


# Configure the Go environment, since it's not going to change
ENV PATH /usr/local/go/bin:$PATH
ENV GOPATH /go


# Inject the remote file fetcher and checksum verifier
ADD fetch.sh /fetch.sh
ENV FETCH /fetch.sh
RUN chmod +x $FETCH


# Make sure apt-get is up to date and dependent packages are installed
RUN \
apt-get update && \
apt-get install -y automake autogen build-essential ca-certificates \
gcc-multilib \
clang llvm-dev libtool libxml2-dev uuid-dev libssl-dev pkg-config \
patch make xz-utils cpio wget unzip git mercurial bzr --no-install-recommends

# Inject the Go package downloader and tool-chain bootstrapper
ADD bootstrap.sh /bootstrap.sh
ENV BOOTSTRAP /bootstrap.sh
RUN chmod +x $BOOTSTRAP

# Inject the C dependency cross compiler
ADD build_deps.sh /build_deps.sh
ENV BUILD_DEPS /build_deps.sh
RUN chmod +x $BUILD_DEPS

# Inject the container entry point, the build script
ADD build.sh /build.sh
ENV BUILD /build.sh
RUN chmod +x $BUILD

ENTRYPOINT ["/build.sh"]
31 changes: 31 additions & 0 deletions docker/xgo-image-deb6/base/bootstrap.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash
#
# Contains the Go tool-chain bootstrapper, that retrieves all the configured
# distribution packages, extracts the binaries and deletes anything not needed.
#
# Usage: bootstrap.sh
#
# Needed environment variables:
# FETCH - Remote file fetcher and checksum verifier (injected by image)
# DIST_LINUX_64, DIST_LINUX_64_SHA1 - 64 bit Linux Go binaries and checksum
# DIST_LINUX_32, DIST_LINUX_32_SHA1 - 32 bit Linux Go binaries and checksum
# DIST_LINUX_ARM, DIST_LINUX_ARM_SHA1 - ARM v5 Linux Go binaries and checksum
# DIST_OSX_64, DIST_OSX_64_SHA1 - 64 bit Mac OSX Go binaries and checksum
# DIST_OSX_32, DIST_OSX_32_SHA1 - 32 bit Mac OSX Go binaries and checksum
# DIST_WIN_64, DIST_WIN_64_SHA1 - 64 bit Windows Go binaries and checksum
# DIST_WIN_32, DIST_WIN_32_SHA1 - 32 bit Windows Go binaries and checksum
set -e

# Download and verify all the binary packages
$FETCH $DIST_LINUX_64 $DIST_LINUX_64_SHA1
$FETCH $DIST_LINUX_32 $DIST_LINUX_32_SHA1

# Extract the 64 bit Linux package as the primary Go SDK
tar -C /usr/local -xzf `basename $DIST_LINUX_64`

# Extract all other packages as secondary ones, keeping only the binaries
tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_32` go/pkg/linux_386*
GOOS=linux GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap

# Delete all the intermediate downloaded files
rm -f `basename $DIST_LINUX_64` `basename $DIST_LINUX_32`
114 changes: 114 additions & 0 deletions docker/xgo-image-deb6/base/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
#!/bin/bash
#
# Contains the main cross compiler, that individually sets up each target build
# platform, compiles all the C dependencies, then build the requested executable
# itself.
#
# Usage: build.sh <import path>
#
# Needed environment variables:
# REPO_REMOTE - Optional VCS remote if not the primary repository is needed
# REPO_BRANCH - Optional VCS branch to use, if not the master branch
# DEPS - Optional list of C dependency packages to build
# PACK - Optional sub-package, if not the import path is being built
# OUT - Optional output prefix to override the package name
# FLAG_V - Optional verbosity flag to set on the Go builder
# FLAG_RACE - Optional race flag to set on the Go builder

# Download the canonical import path (may fail, don't allow failures beyond)
echo "Fetching main repository $1..."
mkdir -p $GOPATH/src/`dirname $1`
cd $GOPATH/src/`dirname $1`
git clone https://$1.git

set -e

cd $GOPATH/src/$1
export GOPATH=$GOPATH:`pwd`/Godeps/_workspace

# Switch over the code-base to another checkout if requested
if [ "$REPO_REMOTE" != "" ]; then
echo "Switching over to remote $REPO_REMOTE..."
if [ -d ".git" ]; then
git remote set-url origin $REPO_REMOTE
git pull
elif [ -d ".hg" ]; then
echo -e "[paths]\ndefault = $REPO_REMOTE\n" >> .hg/hgrc
hg pull
fi
fi

if [ "$REPO_BRANCH" != "" ]; then
echo "Switching over to branch $REPO_BRANCH..."
if [ -d ".git" ]; then
git checkout $REPO_BRANCH
elif [ -d ".hg" ]; then
hg checkout $REPO_BRANCH
fi
fi

# Download all the C dependencies
echo "Fetching dependencies..."
mkdir -p /deps
DEPS=($DEPS) && for dep in "${DEPS[@]}"; do
echo Downloading $dep
if [ "${dep##*.}" == "tar" ]; then wget -q $dep -O - | tar -C /deps -x; fi
if [ "${dep##*.}" == "gz" ]; then wget -q $dep -O - | tar -C /deps -xz; fi
if [ "${dep##*.}" == "bz2" ]; then wget -q $dep -O - | tar -C /deps -xj; fi
done

# Configure some global build parameters
NAME=`basename $1/$PACK`
if [ "$OUT" != "" ]; then
NAME=$OUT
fi


if [ "$FLAG_V" == "true" ]; then V=-v; fi
if [ "$FLAG_RACE" == "true" ]; then R=-race; fi
if [ "$STATIC" == "true" ]; then LDARGS=--ldflags\ \'-extldflags\ \"-static\"\'; fi

if [ -n $BEFORE_BUILD ]; then
chmod +x /scripts/$BEFORE_BUILD
echo "Execute /scripts/$BEFORE_BUILD"
/scripts/$BEFORE_BUILD
fi

# Build for each platform individually
echo "Compiling for linux/amd64..."
HOST=x86_64-linux PREFIX=/usr/local $BUILD_DEPS /deps
GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
sh -c "GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build $V $R $LDARGS -o $NAME-linux-amd64$R ./$PACK"

echo "Compiling for linux/386..."
HOST=i686-linux PREFIX=/usr/local $BUILD_DEPS /deps
GOOS=linux GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
sh -c "GOOS=linux GOARCH=386 CGO_ENABLED=1 go build $V $LDARGS -o $NAME-linux-386 ./$PACK"

#echo "Compiling for linux/arm..."
#CC=arm-linux-gnueabi-gcc HOST=arm-linux PREFIX=/usr/local/arm $BUILD_DEPS /deps
#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go get -d ./$PACK
#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go build $V -o $NAME-linux-arm ./$PACK

#echo "Compiling for windows/amd64..."
#CC=x86_64-w64-mingw32-gcc HOST=x86_64-w64-mingw32 PREFIX=/usr/x86_64-w64-mingw32 $BUILD_DEPS /deps
#CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
#CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-windows-amd64$R.exe ./$PACK

#echo "Compiling for windows/386..."
#CC=i686-w64-mingw32-gcc HOST=i686-w64-mingw32 PREFIX=/usr/i686-w64-mingw32 $BUILD_DEPS /deps
#CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
#CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-windows-386.exe ./$PACK

#echo "Compiling for darwin/amd64..."
#CC=o64-clang HOST=x86_64-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps
#CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
#CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-darwin-amd64$R ./$PACK

#echo "Compiling for darwin/386..."
#CC=o32-clang HOST=i386-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps
#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-darwin-386 ./$PACK

echo "Moving binaries to host..."
cp `ls -t | head -n 2` /build
27 changes: 27 additions & 0 deletions docker/xgo-image-deb6/base/build_deps.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash
#
# Contains the a dependency builder to iterate over all installed dependencies
# and cross compile them to the requested target platform.
#
# Usage: build_deps.sh <dependency folder>
#
# Needed environment variables:
# CC - C cross compiler to use for the build
# HOST - Target platform to build (used to find the needed tool-chains)
# PREFIX - File-system path where to install the built binaries
set -e

# Remove any previous build leftovers, and copy a fresh working set (clean doesn't work for cross compiling)
rm -rf /deps-build && cp -r $1 /deps-build

# Build all the dependencies (no order for now)
for dep in `ls /deps-build`; do
echo "Configuring dependency $dep for $HOST..."
(cd /deps-build/$dep && ./configure --disable-shared --host=$HOST --prefix=$PREFIX --silent)

echo "Building dependency $dep for $HOST..."
(cd /deps-build/$dep && make --silent -j install)
done

# Remove any build artifacts
rm -rf /deps-build
17 changes: 17 additions & 0 deletions docker/xgo-image-deb6/base/fetch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash
#
# Contains a simple fetcher to download a file from a remote URL and verify its
# SHA1 checksum.
#
# Usage: fetch.sh <remote URL> <SHA1 checksum>
set -e

# Pull the file from the remote URL
file=`basename $1`
echo "Downloading $1..."
wget --no-check-certificate -q $1

# Generate a desired checksum report and check against it
echo "$2 $file" > $file.sum
sha1sum -c $file.sum
rm $file.sum
18 changes: 18 additions & 0 deletions docker/xgo-image-deb6/beats-builder/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
FROM tudorg/xgo-deb6-1.4.2
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we move this image to elastic/xog-deb6-1.4.2?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think the 'elastic' namespace on docker hub is ours :-(

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At least I don't know this guy: https://hub.docker.com/r/elastic/ I doesn't look like he needs it. Should we ask him?
Beat is taken: https://hub.docker.com/r/beat/
Beats is available: https://hub.docker.com/r/beats/

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@electrical How did you do it in the past, did you use your personal account for hub.docker.com images?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm currently using my personal account indeed.
I'm hoping we will have an internal docker registry at some point for these kind of things


MAINTAINER Tudor Golubenco <[email protected]>

# Get libpcap binaries for linux
RUN \
mkdir -p /libpcap && \
wget http://ftp.de.debian.org/debian/pool/main/libp/libpcap/libpcap0.8-dev_1.1.1-2+squeeze1_i386.deb && \
dpkg -x libpcap0.8-dev_*_i386.deb /libpcap/i386 && \
wget http://ftp.de.debian.org/debian/pool/main/libp/libpcap/libpcap0.8-dev_1.1.1-2+squeeze1_amd64.deb && \
dpkg -x libpcap0.8-dev_*_amd64.deb /libpcap/amd64 && \
rm libpcap0.8-dev*.deb
RUN \
apt-get update && \
apt-get install -y libpcap0.8-dev

# add patch for gopacket
ADD gopacket_pcap.patch /gopacket_pcap.patch
22 changes: 22 additions & 0 deletions docker/xgo-image-deb6/beats-builder/gopacket_pcap.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
diff --git a/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go b/Godeps/_workspace/src/github.
index d2465bb..7b23b84 100644
--- a/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go
+++ b/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go
@@ -8,12 +8,13 @@
package pcap

/*
-#cgo linux LDFLAGS: -lpcap
+#cgo linux,386 LDFLAGS: /libpcap/i386/usr/lib/libpcap.a
+#cgo linux,amd64 LDFLAGS: /libpcap/amd64/usr/lib/libpcap.a
#cgo freebsd LDFLAGS: -lpcap
#cgo darwin LDFLAGS: -lpcap
-#cgo windows CFLAGS: -I C:/WpdPack/Include
-#cgo windows,386 LDFLAGS: -L C:/WpdPack/Lib -lwpcap
-#cgo windows,amd64 LDFLAGS: -L C:/WpdPack/Lib/x64 -lwpcap
+#cgo windows CFLAGS: -I /libpcap/win/WpdPack/Include
+#cgo windows,386 LDFLAGS: -L /libpcap/win/WpdPack/Lib -lwpcap
+#cgo windows,amd64 LDFLAGS: -L /libpcap/win/WpdPack/Lib/x64 -lwpcap
#include <stdlib.h>
#include <pcap.h>

5 changes: 5 additions & 0 deletions docker/xgo-image-deb6/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/sh

docker build -t tudorg/xgo-deb6-base base/ && \

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add --rm=true to the build commands? this will remove the intermediate containers after a good build to save diskspace.

docker build -t tudorg/xgo-deb6-1.4.2 go-1.4.2/ &&
docker build -t tudorg/beats-builder-deb6 beats-builder
18 changes: 18 additions & 0 deletions docker/xgo-image-deb6/go-1.4.2/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Go cross compiler (xgo): Go 1.4.2 layer
# Copyright (c) 2014 Péter Szilágyi. All rights reserved.
#
# Released under the MIT license.

FROM tudorg/xgo-deb6-base

MAINTAINER Péter Szilágyi <[email protected]>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As this in the Elastic repo, I think you should put yourself here.


# Configure the Go packages and bootstrap them
RUN \
export DIST_LINUX_64=https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz && \
export DIST_LINUX_64_SHA1=5020af94b52b65cc9b6f11d50a67e4bae07b0aff && \
\
export DIST_LINUX_32=https://storage.googleapis.com/golang/go1.4.2.linux-386.tar.gz && \
export DIST_LINUX_32_SHA1=50557248e89b6e38d395fda93b2f96b2b860a26a && \
\
$BOOTSTRAP
Loading