master.cfg

# -*- python -*-
# ex: set syntax=python:

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

# load configuration secrets
from master_secret import *

####### BUILDSLAVES

# A generic latent slave for starting via ssh

import os, shlex
from buildbot.buildslave import AbstractBuildSlave, AbstractLatentBuildSlave
from twisted.internet import defer, utils, reactor, threads
from twisted.python import log, failure

class MyLatentBuildSlave(AbstractLatentBuildSlave):

    def __init__(self, name, password,
                 max_builds=None, notify_on_missing=[],
                 missing_timeout=10, build_wait_timeout=60, properties={}, locks=None):

        AbstractLatentBuildSlave.__init__(self, name, password, max_builds, notify_on_missing,
                                          missing_timeout, build_wait_timeout, properties, locks)

        self.name = name
        self.start_script = shlex.split("/home/checker/buildslave-start.sh '%s'" % name)
        self.stop_script = shlex.split("/home/checker/buildslave-stop.sh '%s'" % name)

    @defer.inlineCallbacks
    def start_instance(self, build):
        log.msg("Attempting to start '%s'" % self.name)
        retval = yield utils.getProcessValue(self.start_script[0], self.start_script[1:])
        defer.returnValue(retval == 0)

    @defer.inlineCallbacks
    def stop_instance(self, fast=False):
        log.msg("Attempting to stop '%s'" % self.name)
        retval = yield utils.getProcessValue(self.stop_script[0], self.stop_script[1:])

        log.msg("slave destroyed (%s): Forcing its connection closed." % self.name)
        yield AbstractBuildSlave.disconnect(self)

        log.msg("We forced disconnection (%s), cleaning up and triggering new build" % self.name)
        self.botmaster.maybeStartBuildsForSlave(self.name)

        defer.returnValue(retval == 0)

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
c['slaves'] = [
    MyLatentBuildSlave("i10pc150", i10secret, max_builds=4),
    MyLatentBuildSlave("i10pc151", i10secret, max_builds=4),
    MyLatentBuildSlave("i10pc152", i10secret, max_builds=4),
    MyLatentBuildSlave("i10pc153", i10secret, max_builds=4),
    MyLatentBuildSlave("i10pc154", i10secret, max_builds=4),
    MyLatentBuildSlave("i10pc155", i10secret, max_builds=4),
    MyLatentBuildSlave("i10pc156", i10secret, max_builds=4),
    MyLatentBuildSlave("i10pc157", i10secret, max_builds=4),
    BuildSlave("i10win1", i10secret, max_builds=4),
    BuildSlave("i10mac8", i10secret, max_builds=1),
    BuildSlave("i10vb-mac1", i10secret, max_builds=1),
    BuildSlave("i10vb-ubu8-32", i10secret, max_builds=1),
    BuildSlave("i10vb-ubu10-32", i10secret, max_builds=1),
    BuildSlave("i10vb-fbsd10", i10secret, max_builds=1),
]
lnxslaves = ["i10pc150","i10pc151"] #,"i10pc152","i10pc153","i10pc154","i10pc155","i10pc156","i10pc157"]
winslaves = ["i10win1"]
macslaves = ["i10vb-mac1"]

myrepourl = "git://github.com/bingmann/stxxl.git"
runtests = True

# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
c['slavePortnum'] = 9989

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "STXXL"
c['titleURL'] = "http://stxxl.sourceforge.net/"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://localhost:8010/"

####### DB URL

# This specifies what database buildbot uses to store change and scheduler
# state.  You can leave this at its default for all but the largest
# installations.
c['db_url'] = "sqlite:///state.sqlite"

####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

from buildbot.status.html import WebStatus
from buildbot.status.web.authz import Authz
from buildbot.status.web.auth import BasicAuth

authz = Authz(
    gracefulShutdown = False,
    forceBuild = True,
    forceAllBuilds = True,
    pingBuilder = True,
    stopBuild = True,
    stopAllBuilds = True,
    stopChange = True,
    cancelPendingBuild = True,
    cancelAllPendingBuilds = True,
)

c['status'].append(WebStatus(http_port=8010, authz=authz))

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of pyflakes.

from buildbot.changes.gitpoller import GitPoller
c['change_source'] = GitPoller(
    myrepourl,
    workdir='gitpoller-stxxl',
    branch='master',
    pollinterval=60)

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

# global CMake options
gcmakeopts = ["BUILD_TESTS=ON",
              "TRY_COMPILE_HEADERS=ON"]

c['builders'] = []
allbuilders = []

from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand,Compile,Test
from buildbot.steps.slave import MakeDirectory,RemoveDirectory

from buildbot.locks import SlaveLock
build_lock = SlaveLock('build', maxCount=1);

def unixbuilder(build_name, cmakeopts, build_slaves):
    f = BuildFactory()
    # check out the source
    f.addStep(Git(repourl=myrepourl, mode='full', method='fresh'))
    # create build directory
    f.addStep(MakeDirectory(dir="build/work"))
    # create TMPDIR
    tmpdir = "/tmp/stxxl-" + build_name
    f.addStep(RemoveDirectory(dir=tmpdir))
    f.addStep(MakeDirectory(dir=tmpdir))
    # run cmake with specified options
    cmakeopts.extend(gcmakeopts)
    cmakeopts.extend(["STXXL_TMPDIR=" + tmpdir])
    cmakeopts = map(lambda opt: "-D"+opt, cmakeopts)
    f.addStep(ShellCommand(name='cmakegen', workdir="build/work",
                                 command=["cmake", "../"] + cmakeopts))
    # compile
    f.addStep(Compile(workdir="build/work", command=["make"], warnOnWarnings=True))
    # run only simple tests
    if runtests:
        f.addStep(Test(workdir="build/work", command=["ctest","-V"]))
    # clean temp files
    f.addStep(RemoveDirectory(dir=tmpdir))

    c['builders'].append(
        BuilderConfig(name=build_name,
                      slavenames=build_slaves,
                      factory=f))

    allbuilders.append(build_name)
    return

for gccver in ["3.4","4.1"]:

    slaves = ["i10vb-ubu8-32"]

    opts = ["CMAKE_C_COMPILER=/usr/bin/gcc-" + gccver,
            "CMAKE_CXX_COMPILER=/usr/bin/g++-" + gccver]

    for mode in ["Release","Debug"]:

        unixbuilder(mode.lower() + "_gcc" + gccver + "_32",
                    ["CMAKE_BUILD_TYPE=" + mode,
                     "USE_GNU_PARALLEL=OFF"] + opts, slaves)

        unixbuilder(mode.lower() + "_gcc" + gccver + "_boost_32",
                    ["CMAKE_BUILD_TYPE=" + mode,
                     "USE_GNU_PARALLEL=OFF",
                     "USE_BOOST=ON"] + opts, slaves)

for gccver in ["4.3","4.4"]:

    slaves = ["i10vb-ubu10-32"]

    opts = ["CMAKE_C_COMPILER=/usr/bin/gcc-" + gccver,
            "CMAKE_CXX_COMPILER=/usr/bin/g++-" + gccver]

    for mode in ["Release","Debug"]:

        unixbuilder(mode.lower() + "_gcc" + gccver + "_32",
                    ["CMAKE_BUILD_TYPE=" + mode,
                     "USE_GNU_PARALLEL=OFF"] + opts, slaves)

        unixbuilder(mode.lower() + "_gcc" + gccver + "_boost_32",
                    ["CMAKE_BUILD_TYPE=" + mode,
                     "USE_GNU_PARALLEL=OFF",
                     "USE_BOOST=ON"] + opts, slaves)

for gccver in ["4.5","4.6","4.7","4.8"]:

    slaves = lnxslaves

    opts = ["CMAKE_C_COMPILER=/usr/bin/gcc-" + gccver,
            "CMAKE_CXX_COMPILER=/usr/bin/g++-" + gccver]

    for mode in ["Release","Debug"]:

        unixbuilder(mode.lower() + "_gcc" + gccver,
                    ["CMAKE_BUILD_TYPE=" + mode,
                     "USE_GNU_PARALLEL=OFF"] + opts, slaves)

        unixbuilder(mode.lower() + "_gcc" + gccver + "_boost",
                    ["CMAKE_BUILD_TYPE=" + mode,
                     "USE_GNU_PARALLEL=OFF",
                     "USE_BOOST=ON"] + opts, slaves)

        unixbuilder(mode.lower() + "_gcc" + gccver + "_para",
                    ["CMAKE_BUILD_TYPE=" + mode,
                     "USE_GNU_PARALLEL=ON"] + opts, slaves)

        unixbuilder(mode.lower() + "_gcc" + gccver + "_para_boost",
                    ["CMAKE_BUILD_TYPE=" + mode,
                     "USE_GNU_PARALLEL=ON",
                     "USE_BOOST=ON"] + opts, slaves)

for iccver in ["2011.13.367","2013.5.192"]:

    opts = ["CMAKE_C_COMPILER=/software/intel/" + iccver + "/bin/icc",
            "CMAKE_CXX_COMPILER=/software/intel/" + iccver + "/bin/icpc"]

    unixbuilder("release_i" + iccver,
                ["CMAKE_BUILD_TYPE=Release"] + opts, lnxslaves)

    unixbuilder("debug_i" + iccver,
                ["CMAKE_BUILD_TYPE=Debug"] + opts, lnxslaves)

for clangver in ["3.1","3.2","3.3"]:

    opts = ["CMAKE_C_COMPILER=/software/clang/" + clangver + "/bin/clang",
            "CMAKE_CXX_COMPILER=/software/clang/" + clangver + "/bin/clang++",
            "CMAKE_CXX_FLAGS=-I/usr/include/x86_64-linux-gnu/c++/4.7/"]

    unixbuilder("release_clang" + clangver,
                ["CMAKE_BUILD_TYPE=Release"] + opts, lnxslaves)

    unixbuilder("debug_clang" + clangver,
                ["CMAKE_BUILD_TYPE=Debug"] + opts, lnxslaves)

### Mac OS X BUILDERS

if True:

    opts = []

    unixbuilder("release_mac",
                ["CMAKE_BUILD_TYPE=Release"] + opts, macslaves)

    unixbuilder("debug_mac",
                ["CMAKE_BUILD_TYPE=Debug"] + opts, macslaves)

### FreeBSD BUILDERS

if True:

    opts = ["USE_GNU_PARALLEL=OFF"]

    unixbuilder("release_fbsd_gcc4.2",
                ["CMAKE_BUILD_TYPE=Release"] + opts, ["i10vb-fbsd10"])

    unixbuilder("debug_fbsd_gcc4.2",
                ["CMAKE_BUILD_TYPE=Debug"] + opts, ["i10vb-fbsd10"])

    opts = ["CMAKE_C_COMPILER=/usr/local/bin/gcc47",
            "CMAKE_CXX_COMPILER=/usr/local/bin/g++47"]

    unixbuilder("release_fbsd_gcc4.7",
                ["CMAKE_BUILD_TYPE=Release"] + opts, ["i10vb-fbsd10"])

    unixbuilder("debug_fbsd_gcc4.7",
                ["CMAKE_BUILD_TYPE=Debug"] + opts, ["i10vb-fbsd10"])

### WINDOWS BUILDERS

from buildbot.steps.vstudio import MsBuild,VisualStudio

class MyMsBuild(MsBuild):
    def __init__(self, **kwargs):
        MsBuild.__init__(self, **kwargs)
    def setupEnvironment(self, cmd):
        VisualStudio.setupEnvironment(self, cmd)
        cmd.args['env']['VCENV_BAT'] = "\"${VS120COMNTOOLS}..\\..\\VC\\vcvarsall.bat\""
    def start(self):
        command = ["%VCENV_BAT%","x86","&&",
                   "msbuild","/m", self.projectfile,
                   "/p:Configuration=%s" % (self.config),
                   "/p:Platform=%s" % (self.platform)]
        if self.project is not None:
            command.append("/t:%s" % (self.project))

        self.setCommand(command)
        return VisualStudio.start(self)

def winbuilder(build_name, cmakeopts, cmakegen, build_config, build_platform, build_slaves):
    f = BuildFactory()
    # check out the source
    f.addStep(Git(repourl=myrepourl, mode='full', method='fresh'))
    # create build directory
    f.addStep(MakeDirectory(dir="build/work"))
    # create TMPDIR
    tmpdir = "e:/stxxl-" + build_name
    f.addStep(RemoveDirectory(dir=tmpdir))
    f.addStep(MakeDirectory(dir=tmpdir))
    # run cmake with specified options
    cmakeopts.extend(gcmakeopts)
    cmakeopts.extend(["STXXL_TMPDIR=" + tmpdir])
    cmakeopts = map(lambda opt: "-D"+opt, cmakeopts) # map -Ddefines
    cmakeopts.append('-G' + cmakegen) # add generator
    f.addStep(ShellCommand(name='cmakegen', workdir="build/work",
                                 command=["cmake", "../"] + cmakeopts))
    # compile
    #f.addStep(Compile(workdir="build/work",
    #                  command=["cmake", "--build", ".", "--config", build_config],
    #                  warningPattern=" ?: warning [A-Z]+[0-9]+:"))

    f.addStep(MyMsBuild(projectfile="stxxl.sln", config=build_config,
                        platform=build_platform, workdir="build/work",
                        warnOnWarnings=True))

    # run only simple tests
    if build_config == "Release" and runtests:
        f.addStep(Test(workdir="build/work", command=["ctest", "-V"]))
    # clean temp files
    f.addStep(RemoveDirectory(dir=tmpdir))

    c['builders'].append(
        BuilderConfig(name=build_name,
                      slavenames=build_slaves,
                      factory=f))

    allbuilders.append(build_name)
    return

winbuilder("debug_msvc12",   [], "Visual Studio 12 Win64", "Debug", "x64", winslaves)
winbuilder("release_msvc12", [], "Visual Studio 12 Win64", "Release", "x64", winslaves)

winbuilder("debug_msvc11",   [], "Visual Studio 11 Win64", "Debug", "x64", winslaves)
winbuilder("release_msvc11", [], "Visual Studio 11 Win64", "Release", "x64", winslaves)

winbuilder("debug_msvc10",   ["BOOST_LIBRARYDIR=c:\\boost\\lib64-msvc-10.0"], "Visual Studio 10 Win64", "Debug", "x64", winslaves)
winbuilder("release_msvc10", ["BOOST_LIBRARYDIR=c:\\boost\\lib64-msvc-10.0"], "Visual Studio 10 Win64", "Release", "x64", winslaves)

### CYGWIN BUILDERS

def cygbuilder(build_name, cmakeopts, build_config, build_slaves):
    f = BuildFactory()
    xenv={'PATH': "C:\\Cygwin64\\bin;${PATH}",
          'CYGWIN': 'nodosfilewarning'}
    # check out the source
    f.addStep(Git(repourl=myrepourl, mode='full', method='fresh'))
    # create build directory
    f.addStep(MakeDirectory(dir="build/work"))
    # create TMPDIR
    tmpdir = "e:\\stxxl-" + build_name
    f.addStep(RemoveDirectory(dir=tmpdir))
    f.addStep(MakeDirectory(dir=tmpdir))
    # run cmake with specified options
    cmakeopts.extend(gcmakeopts)
    cmakeopts.extend(["CMAKE_BUILD_TYPE=" + build_config,
                      "STXXL_TMPDIR=/cygdrive/e/stxxl-" + build_name])
    cmakeopts = map(lambda opt: "-D"+opt, cmakeopts) # map -Ddefines
    f.addStep(ShellCommand(name='cmakegen', workdir="build/work", env=xenv,
                           command=["cmake", "../"] + cmakeopts))
    # compile
    f.addStep(ShellCommand(workdir="build/work",
                           command=["cmake", "--build", ".", "--", "-j4"],
                           warnOnWarnings=True, env=xenv))

    # run only simple tests
    if runtests:
        f.addStep(Test(workdir="build/work", command=["ctest", "-V"], env=xenv))

    # clean temp files
    f.addStep(RemoveDirectory(dir=tmpdir))

    c['builders'].append(
        BuilderConfig(name=build_name,
                      slavenames=build_slaves,
                      factory=f))

    allbuilders.append(build_name)
    return

cygbuilder("debug_cygwin",   [], "Debug", winslaves)
cygbuilder("release_cygwin", [], "Release", winslaves)

### MinGW BUILDERS

def mingwbuilder(build_name, cmakeopts, build_config, build_slaves):
    f = BuildFactory()
    xenv={'PATH': "C:\\mingw-w64\\mingw64\\bin;${PATH}"}
    # check out the source
    f.addStep(Git(repourl=myrepourl, mode='full', method='fresh'))
    # create build directory
    f.addStep(MakeDirectory(dir="build/work"))
    # create TMPDIR
    tmpdir = "e:/stxxl-" + build_name
    f.addStep(RemoveDirectory(dir=tmpdir))
    f.addStep(MakeDirectory(dir=tmpdir))
    # run cmake with specified options
    cmakeopts.extend(gcmakeopts)
    cmakeopts.extend(["CMAKE_BUILD_TYPE=" + build_config,
                      "STXXL_TMPDIR=" + tmpdir])
    cmakeopts = map(lambda opt: "-D"+opt, cmakeopts) # map -Ddefines
    cmakeopts.append('-G' + "MinGW Makefiles") # add generator
    f.addStep(ShellCommand(name='cmakegen', workdir="build/work", env=xenv,
                           command=["cmake", "../"] + cmakeopts))
    # compile
    f.addStep(ShellCommand(workdir="build/work",
                           command=["cmake", "--build", ".", "--", "-j4"],
                           warnOnWarnings=True, env=xenv))

    # run only simple tests
    if runtests:
        f.addStep(Test(workdir="build/work", command=["ctest", "-V"], env=xenv))

    # clean temp files
    f.addStep(RemoveDirectory(dir=tmpdir))

    c['builders'].append(
        BuilderConfig(name=build_name,
                      slavenames=build_slaves,
                      factory=f))

    allbuilders.append(build_name)
    return

if True:

    opts = ["CMAKE_MAKE_PROGRAM=mingw32-make",
            "CMAKE_C_COMPILER=x86_64-w64-mingw32-gcc",
            "CMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++"]

    mingwbuilder("debug_mingw",   opts + [], "Debug", winslaves)
    mingwbuilder("release_mingw", opts + [], "Release", winslaves)

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  In this
# case, just kick off a 'runtests' build

from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.changes import filter
c['schedulers'] = []

if False:
    c['schedulers'].append(SingleBranchScheduler(
        name="all",
        change_filter=filter.ChangeFilter(branch='master'),
        treeStableTimer=None,
        builderNames=allbuilders))

from buildbot.schedulers.forcesched import ForceScheduler
c['schedulers'].append(ForceScheduler(
    name="force",
    builderNames=allbuilders))