netci.groovy

// Import the utility functionality.

import jobs.generation.Utilities;
import jobs.generation.JobReport;

// The input project name (e.g. dotnet/corefx)
def project = GithubProject
// The input branch name (e.g. master)
def branch = GithubBranchName
// Folder that the project jobs reside in (project/branch)
def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)

// Globals

// Map of osName -> osGroup.
def osGroupMap = ['Windows 7':'Windows_NT',
                  'Windows_NT':'Windows_NT',
                  'Ubuntu14.04':'Linux',
                  'Ubuntu16.04':'Linux',
                  'Ubuntu16.10':'Linux',
                  'Debian8.4':'Linux',
                  'Fedora24':'Linux',
                  'OSX10.12':'OSX',
                  'CentOS7.1': 'Linux',
                  'RHEL7.2': 'Linux',
                  'PortableLinux': 'Linux']

def osShortName = ['Windows 7' : 'win7',
                   'Windows_NT' : 'windows_nt',
                   'Ubuntu14.04' : 'ubuntu14.04',
                   'Ubuntu16.04' : 'ubuntu16.04',
                   'Ubuntu16.10' : 'ubuntu16.10',
                   'Debian8.4' : 'debian8.4',
                   'Fedora24' : 'fedora24',
                   'OSX10.12' : 'osx',
                   'CentOS7.1' : 'centos7.1',
                   'RHEL7.2' : 'rhel7.2',
                   'PortableLinux' : 'portablelinux']

def buildArchConfiguration = ['Debug': 'x86',
                              'Release': 'x64']

def targetGroupOsMapOuterloop = ['netcoreapp': ['Windows 7', 'Windows_NT', 'Ubuntu14.04', 'Ubuntu16.04', 'Ubuntu16.10', 'CentOS7.1',
                                        'RHEL7.2', 'Fedora24', 'Debian8.4', 'OSX10.12', 'PortableLinux']]

def targetGroupOsMapInnerloop = ['netcoreapp': ['Windows_NT', 'Ubuntu14.04', 'Ubuntu16.04', 'Ubuntu16.10', 'CentOS7.1',
                                        'RHEL7.2', 'Fedora24', 'Debian8.4', 'OSX10.12', 'PortableLinux']]

// **************************
// Define code coverage build
// **************************

[true, false].each { isPR ->
    ['local', 'nonlocal'].each { localType ->
        def isLocal = (localType == 'local')

        def newJobName = 'code_coverage_windows'
        def batchCommand = 'call build.cmd && call build-tests.cmd -coverage -outerloop -- /p:IsCIBuild=true'
        if (isLocal) {
            newJobName = "${newJobName}_local"
            batchCommand = "${batchCommand}"
        }
        def newJob = job(Utilities.getFullJobName(project, newJobName, isPR)) {
            steps {
                batchFile(batchCommand)
            }
        }

        // Set up standard options
        Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
        // Set the machine affinity to windows machines
        Utilities.setMachineAffinity(newJob, 'Windows_NT', 'latest-or-auto')
        // Publish reports
        Utilities.addHtmlPublisher(newJob, 'bin/tests/coverage', 'Code Coverage Report', 'index.htm')
        // Archive results.
        Utilities.addArchival(newJob, '**/coverage/*,msbuild.log')
        // Timeout. Code coverage runs take longer, so we set the timeout to be longer.
        Utilities.setJobTimeout(newJob, 180)
        // Set triggers
        if (isPR) {
            if (!isLocal) {
                // Set PR trigger
                Utilities.addGithubPRTriggerForBranch(newJob, branch, 'Code Coverage Windows Debug', '(?i).*test\\W+code\\W+coverage.*')
            }
        }
        else {
            // Set a periodic trigger
            Utilities.addPeriodicTrigger(newJob, '@daily')
        }
    }
}

// **************************
// Define code formatter check build
// **************************

[true, false].each { isPR ->
    def newJob = job(Utilities.getFullJobName(project, 'native_code_format_check', isPR)) {
        steps {
            shell('python src/Native/Unix/format-code.py checkonly')
        }
    }

    // Set up standard options.
    Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
    // Set the machine affinity to Ubuntu14.04 machines
    Utilities.setMachineAffinity(newJob, 'Ubuntu14.04', 'latest-or-auto')
    if (isPR) {
        // Set PR trigger.  Only trigger when the phrase is said.
        Utilities.addGithubPRTriggerForBranch(newJob, branch, 'Code Formatter Check', '(?i).*test\\W+code\\W+formatter\\W+check.*', true)
    }
    else {
        // Set a push trigger
        Utilities.addGithubPushTrigger(newJob)
    }
}

// **************************
// Define outerloop testing for OSes that can build and run.  Run locally on each machine.
// **************************
[true, false].each { isPR ->
    ['netcoreapp'].each { targetGroup ->
        (targetGroupOsMapOuterloop[targetGroup]).each { osName ->
            ['Debug', 'Release'].each { configurationGroup ->

                def osForMachineAffinity = osName
                if (osForMachineAffinity == 'PortableLinux') {
                    // Portable Linux builds happen on RHEL7.2
                    osForMachineAffinity = "RHEL7.2"
                }

                def osGroup = osGroupMap[osName]
                def archGroup = "x64"
                def newJobName = "outerloop_${targetGroup}_${osShortName[osName]}_${configurationGroup.toLowerCase()}"

                def newJob = job(Utilities.getFullJobName(project, newJobName, isPR)) {
                    steps {
                        if (osName == 'Windows 7' || osName == 'Windows_NT') {
                            batchFile("call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && build.cmd -framework:${targetGroup} -${configurationGroup}")
                            batchFile("call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && build-tests.cmd -framework:${targetGroup} -${configurationGroup} -outerloop -- /p:IsCIBuild=true")
                            batchFile("C:\\Packer\\Packer.exe .\\bin\\build.pack .\\bin\\runtime\\${targetGroup}-${osGroup}-${configurationGroup}-${archGroup}")
                        }
                        else if (osName == 'OSX10.12') {
                            shell("HOME=\$WORKSPACE/tempHome ./build.sh -${configurationGroup.toLowerCase()}")
                            shell("HOME=\$WORKSPACE/tempHome ./build-tests.sh -${configurationGroup.toLowerCase()} -outerloop -- /p:IsCIBuild=true")
                            shell("tar -czf bin/build.tar.gz --directory=\"bin/runtime/${targetGroup}-${osGroup}-${configurationGroup}-${archGroup}\" .")
                        }
                        else if (osName == 'CentOS7.1') {
                            // On Centos7.1, the cmake toolset is currently installed in /usr/local/bin (it was built manually).  When
                            // running sudo, that will be typically eliminated from the PATH, so let's add it back in.
                            shell("sudo PATH=\$PATH:/usr/local/bin HOME=\$WORKSPACE/tempHome ./build.sh -${configurationGroup.toLowerCase()}")
                            shell("sudo PATH=\$PATH:/usr/local/bin HOME=\$WORKSPACE/tempHome ./build-tests.sh -${configurationGroup.toLowerCase()} -outerloop -- /p:IsCIBuild=true")
                            shell("sudo tar -czf bin/build.tar.gz --directory=\"bin/runtime/${targetGroup}-${osGroup}-${configurationGroup}-${archGroup}\" .")
                        }
                        else {
                            def portableLinux = (osName == 'PortableLinux') ? '-portable' : ''
                            shell("sudo HOME=\$WORKSPACE/tempHome ./build.sh -${configurationGroup.toLowerCase()} ${portableLinux}")
                            shell("sudo HOME=\$WORKSPACE/tempHome ./build-tests.sh -${configurationGroup.toLowerCase()} -outerloop -- /p:IsCIBuild=true")
                            // Tar up the appropriate bits.
                            shell("sudo tar -czf bin/build.tar.gz --directory=\"bin/runtime/${targetGroup}-${osGroup}-${configurationGroup}-${archGroup}\" .")

                        }
                    }
                }

                // Set the affinity.  OS name matches the machine affinity.
                if (osName == 'Windows_NT' || osName == 'OSX10.12') {
                    Utilities.setMachineAffinity(newJob, osForMachineAffinity, "latest-or-auto-elevated")
                }
                else if (osGroup == 'Linux') {
                    Utilities.setMachineAffinity(newJob, osForMachineAffinity, 'outer-latest-or-auto')
                } else {
                    Utilities.setMachineAffinity(newJob, osForMachineAffinity, 'latest-or-auto');
                }

                // Set up standard options.
                Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
                // Add the unit test results
                Utilities.addXUnitDotNETResults(newJob, 'bin/**/testResults.xml')
                def archiveContents = "msbuild.log"
                if (osName.contains('Windows')) {
                    // Packer.exe is a .NET Framework application. When we can use it from the tool-runtime, we can archive the ".pack" file here.
                    archiveContents += ",bin/build.pack"
                }
                else {
                    archiveContents += ",bin/build.tar.gz"
                }
                // Add archival for the built data.
                Utilities.addArchival(newJob, archiveContents, '', doNotFailIfNothingArchived=true, archiveOnlyIfSuccessful=false)
                // Set up appropriate triggers.  PR on demand, otherwise nightly
                if (isPR) {
                    // Set PR trigger.
                    // TODO: More elaborate regex trigger?
                    Utilities.addGithubPRTriggerForBranch(newJob, branch, "OuterLoop ${targetGroup} ${osName} ${configurationGroup} ${archGroup}", "(?i).*test\\W+outerloop\\W+${targetGroup} ${osName}\\W+${configurationGroup}.*")
                }
                else {
                    // Set a periodic trigger
                    Utilities.addPeriodicTrigger(newJob, '@daily')
                }
            }
        }
    }
}


// **************************
// Define innerloop testing.  These jobs run on every merge and a subset of them run on every PR, the ones
// that don't run per PR can be requested via a magic phrase.
// **************************
[true, false].each { isPR ->
    ['netcoreapp'].each { targetGroup ->
        (targetGroupOsMapInnerloop[targetGroup]).each { osName ->
            ['Debug', 'Release'].each { configurationGroup ->
                def osGroup = osGroupMap[osName]
                def osForMachineAffinity = osName

                if (osForMachineAffinity == 'PortableLinux') {
                    // Portable Linux builds happen on RHEL7.2
                    osForMachineAffinity = "RHEL7.2"
                }
                def archGroup = "x64"
                if (osName == 'Windows 7' || osName == 'Windows_NT') {
                    // On Windows, use different architectures for Debug and Release.
                    archGroup = buildArchConfiguration[configurationGroup]
                }
                def targetGroupString = targetGroup != 'netcoreapp' ? "${targetGroup}_" : '';
                def newJobName = "${targetGroupString}${osName.toLowerCase()}_${configurationGroup.toLowerCase()}"

                def newJob = job(Utilities.getFullJobName(project, newJobName, isPR)) {
                    // On Windows we use the packer to put together everything. On *nix we use tar
                    steps {
                        if (osName == 'Windows 7' || osName == 'Windows_NT') {
                            batchFile("call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && build.cmd -${configurationGroup} -os:${osGroup} -buildArch:${archGroup} -framework:${targetGroup}")
                            batchFile("call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && build-tests.cmd -${configurationGroup} -os:${osGroup} -buildArch:${archGroup} -framework:${targetGroup} -- /p:IsCIBuild=true")
                            batchFile("C:\\Packer\\Packer.exe .\\bin\\build.pack .\\bin")
                        }
                        else {
                            // Use Server GC for Ubuntu/OSX Debug PR build & test
                            def useServerGC = (configurationGroup == 'Release' && isPR) ? 'useServerGC' : ''
                            def portableLinux = (osName == 'PortableLinux') ? '-portable' : ''
                            shell("HOME=\$WORKSPACE/tempHome ./build.sh -${configurationGroup.toLowerCase()} -framework:${targetGroup} -os:${osGroup} ${portableLinux} -buildArch:${archGroup}")
                            shell("HOME=\$WORKSPACE/tempHome ./build-tests.sh -${configurationGroup.toLowerCase()} -framework:${targetGroup} -os:${osGroup} -buildArch:${archGroup} -- ${useServerGC} /p:IsCIBuild=true")
                            // Tar up the appropriate bits.
                            shell("tar -czf bin/build.tar.gz --directory=\"bin/runtime/${targetGroup}-${osGroup}-${configurationGroup}-x64\" .")
                        }
                    }
                }

                // Set the affinity.
                Utilities.setMachineAffinity(newJob, osForMachineAffinity, 'latest-or-auto')
                // Set up standard options.
                Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
                // Add the unit test results
                Utilities.addXUnitDotNETResults(newJob, 'bin/**/testResults.xml')
                def archiveContents = "msbuild.log"
                if (osName.contains('Windows')) {
                    // Packer.exe is a .NET Framework application. When we can use it from the tool-runtime, we can archive the ".pack" file here.
                    archiveContents += ",bin/build.pack"
                }
                else {
                    archiveContents += ",bin/build.tar.gz"
                }
                // Add archival for the built data.
                Utilities.addArchival(newJob, archiveContents, '', doNotFailIfNothingArchived=true, archiveOnlyIfSuccessful=false)
                // Set up triggers
                if (isPR) {
                    targetGroupString = targetGroupString.replaceAll('_', ' ');
                    Utilities.addGithubPRTriggerForBranch(newJob, branch, "Innerloop ${targetGroupString}${osName} ${configurationGroup} ${archGroup} Build and Test", "(?i).*test\\W+innerloop\\W+${targetGroupString}${osName}\\W+${configurationGroup}.*")
                }
                else {
                    // Set a push trigger
                    Utilities.addGithubPushTrigger(newJob)
                }
            }
        }
    }
}

// **************************
// Define Linux ARM builds. These jobs run on every merge.
// Some jobs run on every PR. The ones that don't run per PR can be requested via a phrase.
// **************************
[true, false].each { isPR ->
    ['netcoreapp'].each { targetGroup ->
        ['Debug', 'Release'].each { configurationGroup ->
            ['Linux', 'Tizen'].each { osName ->
                if (osName == "Linux") {
                    linuxCodeName="xenial"
                    abi = "arm"
                }
                else if (osName == "Tizen") {
                    linuxCodeName="tizen"
                    abi = "armel"
                }

                def osGroup = "Linux"
                def newJobName = "${osName.toLowerCase()}_${abi.toLowerCase()}_cross_${configurationGroup.toLowerCase()}"

                def newJob = job(Utilities.getFullJobName(project, newJobName, isPR)) {
                    steps {
                        // Call the arm32_ci_script.sh script to perform the cross build of native corefx
                        def script = "./cross/arm32_ci_script.sh --buildConfig=${configurationGroup.toLowerCase()} --${abi} --linuxCodeName=${linuxCodeName} --verbose"
                        shell(script)

                        // Tar up the appropriate bits.
                        shell("tar -czf bin/build.tar.gz --directory=\"bin/runtime/${targetGroup}-${osGroup}-${configurationGroup}-${abi}\" .")
                    }
                }

                // The cross build jobs run on Ubuntu. The arm-cross-latest version
                // contains the packages needed for cross building corefx
                Utilities.setMachineAffinity(newJob, 'Ubuntu14.04', 'arm-cross-latest')

                // Set up standard options.
                Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")

                // Add archival for the built binaries
                def archiveContents = "bin/build.tar.gz"
                Utilities.addArchival(newJob, archiveContents)
                
                newJob.with {
                    publishers {
                        azureVMAgentPostBuildAction {
                            agentPostBuildAction('Delete agent after build execution (when idle).')
                        }
                    }
                }

                // Set up triggers
                if (isPR) {
                    // We run Tizen Debug and Linux Release as default PR builds
                    if ((osName == "Tizen" && configurationGroup == "Debug") || (osName == "Linux" && configurationGroup == "Release")) {
                        Utilities.addGithubPRTriggerForBranch(newJob, branch, "${osName} ${abi} ${configurationGroup} Build")
                    }
                    else {
                        Utilities.addGithubPRTriggerForBranch(newJob, branch, "${osName} ${abi} ${configurationGroup} Build", "(?i).*test\\W+${osName}\\W+${abi}\\W+${configurationGroup}.*")
                    }
                }
                else {
                    // Set a push trigger
                    Utilities.addGithubPushTrigger(newJob)
                }
            } // osName
        } // configurationGroup
    } // targetGroup
} // isPR

// **************************
// Define Linux x86 builds. These jobs run daily and results will be used for CoreCLR test
// TODO: innerloop & outerloop testing & merge to general job generation routine
// **************************
['Debug', 'Release'].each { configurationGroup ->
    def osName = 'Ubuntu16.04'
    def archGroup = 'x86'
    def newJobName = "${osName.toLowerCase()}_${archGroup}_${configurationGroup.toLowerCase()}"

    def newJob = job(Utilities.getFullJobName(project, newJobName, false)) {
        steps {
            // Call x86_ci_script.sh script to perform the cross build of native corefx
            def script = "./cross/x86_ci_script.sh --buildConfig=${configurationGroup.toLowerCase()}"
            shell(script)

            // Tar up the appropriate bits
            shell("tar -czf bin/build.tar.gz --directory=\"bin/Linux.${archGroup}.${configurationGroup}/native\" .")
        }
    }

    // The cross build jobs run on Ubuntu 14.04 in spite of the target is Ubuntu 16.04.
    // The ubuntu 14.04 arm-cross-latest version contains the packages needed for cross building corefx
    Utilities.setMachineAffinity(newJob, 'Ubuntu14.04', 'arm-cross-latest')

    // Set up standard options.
    Utilities.standardJobSetup(newJob, project, false, "*/${branch}")

    // Add archival for the built binaries
    def archiveContents = "bin/build.tar.gz"
    Utilities.addArchival(newJob, archiveContents)

    // Set a push trigger as a daily work
    Utilities.addPeriodicTrigger(newJob, '@daily')
}

JobReport.Report.generateJobReport(out)

// Make the call to generate the help job
Utilities.createHelperJob(this, project, branch,
    "Welcome to the ${project} Repository",  // This is prepended to the help message
    "Have a nice day!")  // This is appended to the help message.  You might put known issues here.

Utilities.addCROSSCheck(this, project, branch)