Skip to content

Commit

Permalink
Merge pull request #277 from RobotLocomotion/cmake-packaging
Browse files Browse the repository at this point in the history
Start moving packaging to CMake builds (#277)
  • Loading branch information
mwoehlke-kitware authored Apr 4, 2024
2 parents f1e8f53 + f25b618 commit 360df1e
Show file tree
Hide file tree
Showing 8 changed files with 518 additions and 29 deletions.
21 changes: 1 addition & 20 deletions driver/configurations/bazel.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -43,26 +43,7 @@ set(CTEST_BINARY_DIRECTORY "${DASHBOARD_WORKSPACE}/_bazel_$ENV{USER}")
set(DASHBOARD_BAZEL_STARTUP_OPTIONS)
set(DASHBOARD_OUTPUT_USER_ROOT "${CTEST_BINARY_DIRECTORY}")

# Extract the version. Usually of the form x.y.z-*.
set(VERSION_ARGS "${DASHBOARD_BAZEL_STARTUP_OPTIONS} --output_user_root=${DASHBOARD_OUTPUT_USER_ROOT} version")
separate_arguments(VERSION_ARGS_LIST UNIX_COMMAND "${VERSION_ARGS}")
execute_process(COMMAND ${DASHBOARD_BAZEL_COMMAND} ${VERSION_ARGS_LIST}
WORKING_DIRECTORY "${DASHBOARD_SOURCE_DIRECTORY}"
RESULT_VARIABLE DASHBOARD_BAZEL_VERSION_RESULT_VARIABLE
OUTPUT_VARIABLE DASHBOARD_BAZEL_VERSION_OUTPUT_VARIABLE
ERROR_QUIET
OUTPUT_STRIP_TRAILING_WHITESPACE)

if(DASHBOARD_BAZEL_VERSION_RESULT_VARIABLE EQUAL 0)
string(REGEX MATCH "Build label: ([0-9a-zA-Z.\\-]+)"
DASHBOARD_BAZEL_REGEX_MATCH_OUTPUT_VARIABLE
"${DASHBOARD_BAZEL_VERSION_OUTPUT_VARIABLE}")
if(DASHBOARD_BAZEL_REGEX_MATCH_OUTPUT_VARIABLE)
set(DASHBOARD_BAZEL_VERSION "${CMAKE_MATCH_1}")
endif()
else()
fatal("could not determine bazel version")
endif()
execute_step(common get-bazel-version)

set(DASHBOARD_BUILD_EVENT_JSON_FILE "${CTEST_BINARY_DIRECTORY}/BUILD.JSON")

Expand Down
60 changes: 51 additions & 9 deletions driver/configurations/cmake.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,23 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Set build locations and ensure there are no leftover artifacts
set(CTEST_SOURCE_DIRECTORY "${DASHBOARD_SOURCE_DIRECTORY}")
set(CTEST_BINARY_DIRECTORY "${DASHBOARD_WORKSPACE}/_cmake_$ENV{USER}")
set(DASHBOARD_INSTALL_PREFIX "${CTEST_BINARY_DIRECTORY}/install")

file(REMOVE_RECURSE "${CTEST_BINARY_DIRECTORY}")
file(MAKE_DIRECTORY "${CTEST_BINARY_DIRECTORY}")
file(REMOVE_RECURSE "${DASHBOARD_INSTALL_PREFIX}")

# Jenkins passes down an incorrect value of JAVA_HOME from controller to agent
# for some inexplicable reason.
unset(ENV{JAVA_HOME})

# Pass along compiler.
# Pass along compiler
set(ENV{CC} "${DASHBOARD_CC_COMMAND}")
set(ENV{CXX} "${DASHBOARD_CXX_COMMAND}")

set(CTEST_SOURCE_DIRECTORY "${DASHBOARD_SOURCE_DIRECTORY}")
set(CTEST_BINARY_DIRECTORY "${DASHBOARD_WORKSPACE}/_cmake_$ENV{USER}")
set(DASHBOARD_INSTALL_PREFIX "${CTEST_BINARY_DIRECTORY}/install")

set(DASHBOARD_CXX_FLAGS)
if(DEFINED ENV{TERM})
if(COMPILER STREQUAL "clang")
Expand All @@ -56,13 +61,10 @@ else()
set(DASHBOARD_COLOR_MAKEFILE OFF)
endif()

# Set up build configuration
set(CTEST_CONFIGURATION_TYPE "${DASHBOARD_CONFIGURATION_TYPE}")
set(CTEST_TEST_TIMEOUT 300)

file(REMOVE_RECURSE "${CTEST_BINARY_DIRECTORY}")
file(MAKE_DIRECTORY "${CTEST_BINARY_DIRECTORY}")
file(REMOVE_RECURSE "${DASHBOARD_INSTALL_PREFIX}")

include(${DASHBOARD_DRIVER_DIR}/configurations/aws.cmake)
include(${DASHBOARD_DRIVER_DIR}/configurations/gurobi.cmake)
include(${DASHBOARD_DRIVER_DIR}/configurations/mosek.cmake)
Expand Down Expand Up @@ -113,6 +115,7 @@ file(COPY "${DASHBOARD_CI_DIR}/user.bazelrc"
file(APPEND "${DASHBOARD_SOURCE_DIRECTORY}/user.bazelrc"
"startup --output_user_root=${DASHBOARD_WORKSPACE}/_bazel_$ENV{USER}\n")

# Set up cache
include(${DASHBOARD_DRIVER_DIR}/configurations/cache.cmake)

if(REMOTE_CACHE)
Expand All @@ -128,6 +131,13 @@ if(REMOTE_CACHE)
"build --remote_upload_local_results=${DASHBOARD_REMOTE_UPLOAD_LOCAL_RESULTS}\n")
endif()

# Set package version
execute_step(common set-package-version)
cache_append(DRAKE_VERSION_OVERRIDE "${DASHBOARD_DRAKE_VERSION}")

# Report build configuration
execute_step(common get-bazel-version)

report_configuration("
==================================== ENV
CC
Expand Down Expand Up @@ -184,13 +194,18 @@ report_configuration("
OS_CACHE_VERSION
PYTHON_CACHE_VERSION
==================================== >DASHBOARD_
BAZEL_COMMAND
BAZEL_VERSION
==================================== >DASHBOARD_
REMOTE_CACHE_KEY_VERSION
REMOTE_CACHE_KEY
====================================
")

# Run the build
execute_step(cmake build)

# Determine build result
if(NOT DASHBOARD_FAILURE)
format_plural(DASHBOARD_MESSAGE
ZERO "SUCCESS"
Expand All @@ -199,3 +214,30 @@ if(NOT DASHBOARD_FAILURE)
${DASHBOARD_BUILD_NUMBER_WARNINGS})
endif()

# Create packages (if applicable)
if(PACKAGE)
execute_step(cmake create-package-archive)
if(NOT APPLE)
execute_step(cmake create-debian-archive)
endif()
if(PACKAGE STREQUAL "publish")
execute_step(cmake upload-package-archive)
if(NOT APPLE)
execute_step(cmake upload-debian-archive)
endif()
endif()
if(DOCKER)
# The default Ubuntu version for Docker should be the newest base OS.
# If this value changes, the Docker documentation in the drake repository
# (drake/doc/_pages/docker.md) also needs to be updated.
set(DEFAULT_DOCKER_DISTRIBUTION "jammy")

execute_step(cmake build-docker-image)
if(DOCKER STREQUAL "publish")
execute_step(cmake push-docker-image)
endif()
endif()
if(DISTRIBUTION STREQUAL "jammy" AND DASHBOARD_TRACK STREQUAL "Nightly")
execute_step(cmake push-nightly-release-branch)
endif()
endif()
126 changes: 126 additions & 0 deletions driver/configurations/cmake/step-build-docker-image.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
# -*- mode: cmake; -*-
# vi: set ft=cmake:

# BSD 3-Clause License
#
# Copyright (c) 2019, Massachusetts Institute of Technology.
# Copyright (c) 2019, Toyota Research Institute.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

if(DASHBOARD_FAILURE OR DASHBOARD_UNSTABLE)
notice("CTest Status: NOT BUILDING DOCKER IMAGE BECAUSE BAZEL BUILD WAS NOT SUCCESSFUL")
else()
notice("CTest Status: BUILDING DOCKER IMAGE")

set(DOCKER_BUILD_WORKING_DIRECTORY "${DASHBOARD_SOURCE_DIRECTORY}/tools/install/dockerhub")
if(EXISTS "${DASHBOARD_SOURCE_DIRECTORY}/tools/install/dockerhub/${DASHBOARD_UNIX_DISTRIBUTION_CODE_NAME}/Dockerfile")
set(DOCKER_BUILD_WORKING_DIRECTORY "${DASHBOARD_SOURCE_DIRECTORY}/tools/install/dockerhub/${DASHBOARD_UNIX_DISTRIBUTION_CODE_NAME}")
endif()

execute_process(COMMAND "${CMAKE_COMMAND}" -E copy "${DASHBOARD_WORKSPACE}/${DASHBOARD_PACKAGE_ARCHIVE_NAME}" "${DOCKER_BUILD_WORKING_DIRECTORY}/drake-latest-${DASHBOARD_PACKAGE_ARCHIVE_DISTRIBUTION}.tar.gz"
RESULT_VARIABLE COPY_PACKAGE_ARCHIVE_RESULT_VARIABLE
COMMAND_ECHO STDERR)
if(NOT COPY_PACKAGE_ARCHIVE_RESULT_VARIABLE EQUAL 0)
append_step_status("BAZEL BUILDING DOCKER IMAGE (COPY PACKAGE ARCHIVE)" UNSTABLE)
endif()

if(NOT DASHBOARD_UNSTABLE)
execute_process(COMMAND "sudo" "${DASHBOARD_SETUP_DIR}/docker/install_prereqs"
RESULT_VARIABLE DOCKER_INSTALL_PREREQS_RESULT_VARIABLE
COMMAND_ECHO STDERR)
if(NOT DOCKER_INSTALL_PREREQS_RESULT_VARIABLE EQUAL 0)
append_step_status("BAZEL BUILDING DOCKER IMAGE (INSTALL DOCKER PREREQUISITES)" UNSTABLE)
endif()
endif()

if(NOT DASHBOARD_UNSTABLE)
find_program(DASHBOARD_DOCKER_COMMAND NAMES "docker")
if(NOT DASHBOARD_DOCKER_COMMAND)
append_step_status("BAZEL BUILDING DOCKER IMAGE (FIND DOCKER)" UNSTABLE)
endif()
endif()

if(NOT DASHBOARD_UNSTABLE)
set(DOCKER_PULL_ARGS "pull ubuntu:${DASHBOARD_UNIX_DISTRIBUTION_CODE_NAME}")
separate_arguments(DOCKER_PULL_ARGS_LIST UNIX_COMMAND "${DOCKER_PULL_ARGS}")
foreach(RETRIES RANGE 3)
execute_process(COMMAND "sudo" "${DASHBOARD_DOCKER_COMMAND}" ${DOCKER_PULL_ARGS_LIST}
RESULT_VARIABLE DOCKER_PULL_RESULT_VARIABLE
COMMAND_ECHO STDERR)
if(DOCKER_PULL_RESULT_VARIABLE EQUAL 0)
break()
endif()
sleep(15)
endforeach()
if(NOT DOCKER_PULL_RESULT_VARIABLE EQUAL 0)
append_step_status("BAZEL PULLING DOCKER IMAGE (DOCKER PULL)" UNSTABLE)
endif()
endif()

if(DASHBOARD_TRACK STREQUAL "Staging")
set(DOCKER_TAG "${DASHBOARD_DRAKE_VERSION}-staging")
else()
set(DOCKER_TAG "${DATE}")
endif()

if(NOT DASHBOARD_UNSTABLE)
set(DOCKER_BUILD_ARGS "build -t robotlocomotion/drake:${DASHBOARD_UNIX_DISTRIBUTION_CODE_NAME} -t robotlocomotion/drake:${DASHBOARD_UNIX_DISTRIBUTION_CODE_NAME}-${DOCKER_TAG} .")
separate_arguments(DOCKER_BUILD_ARGS_LIST UNIX_COMMAND "${DOCKER_BUILD_ARGS}")
execute_process(COMMAND "sudo" "${DASHBOARD_DOCKER_COMMAND}" ${DOCKER_BUILD_ARGS_LIST}
WORKING_DIRECTORY "${DOCKER_BUILD_WORKING_DIRECTORY}"
RESULT_VARIABLE DOCKER_BUILD_RESULT_VARIABLE
COMMAND_ECHO STDERR)
if(NOT DOCKER_BUILD_RESULT_VARIABLE EQUAL 0)
append_step_status("BAZEL BUILDING DOCKER IMAGE (DOCKER BUILD)" UNSTABLE)
endif()
endif()

if(DASHBOARD_UNIX_DISTRIBUTION_CODE_NAME STREQUAL ${DEFAULT_DOCKER_DISTRIBUTION})
if(NOT DASHBOARD_UNSTABLE)
set(DOCKER_TAG_ARGS "tag robotlocomotion/drake:${DASHBOARD_UNIX_DISTRIBUTION_CODE_NAME} robotlocomotion/drake:${DOCKER_TAG}")
separate_arguments(DOCKER_TAG_ARGS_LIST UNIX_COMMAND "${DOCKER_TAG_ARGS}")
execute_process(COMMAND "sudo" "${DASHBOARD_DOCKER_COMMAND}" ${DOCKER_TAG_ARGS_LIST}
RESULT_VARIABLE DOCKER_TAG_RESULT_VARIABLE
COMMAND_ECHO STDERR)
if(NOT DOCKER_TAG_RESULT_VARIABLE EQUAL 0)
append_step_status("BAZEL BUILDING DOCKER IMAGE (DOCKER TAG ${DOCKER_TAG})" UNSTABLE)
endif()
endif()

if(DASHBOARD_TRACK STREQUAL "Nightly" AND NOT DASHBOARD_UNSTABLE)
set(DOCKER_TAG_LATEST_ARGS "tag robotlocomotion/drake:${DASHBOARD_UNIX_DISTRIBUTION_CODE_NAME} robotlocomotion/drake:latest")
separate_arguments(DOCKER_TAG_LATEST_ARGS_LIST UNIX_COMMAND "${DOCKER_TAG_LATEST_ARGS}")
execute_process(COMMAND "sudo" "${DASHBOARD_DOCKER_COMMAND}" ${DOCKER_TAG_LATEST_ARGS_LIST}
RESULT_VARIABLE DOCKER_TAG_LATEST_RESULT_VARIABLE
COMMAND_ECHO STDERR)
if(NOT DOCKER_TAG_LATEST_RESULT_VARIABLE EQUAL 0)
append_step_status("BAZEL BUILDING DOCKER IMAGE (DOCKER TAG LATEST)" UNSTABLE)
endif()
endif()
endif()
endif()
71 changes: 71 additions & 0 deletions driver/configurations/cmake/step-create-debian-archive.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# -*- mode: cmake; -*-
# vi: set ft=cmake:

# BSD 3-Clause License
#
# Copyright (c) 2022, Massachusetts Institute of Technology.
# Copyright (c) 2022, Toyota Research Institute.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

if(DASHBOARD_FAILURE OR DASHBOARD_UNSTABLE)
notice("CTest Status: NOT CREATING DEBIAN ARCHIVE BECAUSE BAZEL BUILD WAS NOT SUCCESSFUL")
else()
notice("CTest Status: CREATING DEBIAN ARCHIVE")
# NOTE: do not use DASHBOARD_BAZEL_*_OPTIONS with this script.
set(DEBIAN_ARGS
"run" "//tools/release_engineering:repack_deb" "--" "--tgz"
"${DASHBOARD_WORKSPACE}/${DASHBOARD_PACKAGE_ARCHIVE_NAME}"
"--output-dir" "${DASHBOARD_WORKSPACE}"
"--version" "${DASHBOARD_DRAKE_VERSION}")
execute_process(COMMAND ${DASHBOARD_BAZEL_COMMAND} ${DEBIAN_ARGS}
WORKING_DIRECTORY "${DASHBOARD_SOURCE_DIRECTORY}"
RESULT_VARIABLE DEBIAN_RESULT_VARIABLE)

if(NOT DEBIAN_RESULT_VARIABLE EQUAL 0)
append_step_status(
"BAZEL DEBIAN ARCHIVE CREATION (ERROR CODE=${DEBIAN_RESULT_VARIABLE})"
UNSTABLE)
else()
set(repack_deb_output "drake-dev_${DASHBOARD_DRAKE_VERSION}-1_amd64.deb")
set(repack_deb_path "${DASHBOARD_WORKSPACE}/${repack_deb_output}")
if(NOT EXISTS "${repack_deb_path}")
append_step_status("BAZEL PACKAGE DEBIAN CREATION COULD NOT FIND ${repack_deb_output} in ${DASHBOARD_WORKSPACE}" UNSTABLE)
else()
# For the uploaded package name, we want to structure it to include the
# ubuntu codename (e.g. jammy).
set(DASHBOARD_DEBIAN_ARCHIVE_NAME
"drake-dev_${DASHBOARD_DRAKE_VERSION}-1_amd64-${DASHBOARD_UNIX_DISTRIBUTION_CODE_NAME}.deb")
file(RENAME "${repack_deb_path}" "${DASHBOARD_WORKSPACE}/${DASHBOARD_DEBIAN_ARCHIVE_NAME}")
if(NOT EXISTS "${DASHBOARD_WORKSPACE}/${DASHBOARD_DEBIAN_ARCHIVE_NAME}")
append_step_status("BAZEL PACKAGE DEBIAN CREATION COULD NOT RENAME ${repack_deb_path} to ${DASHBOARD_DEBIAN_ARCHIVE_NAME} in ${DASHBOARD_WORKSPACE}" UNSTABLE)
else()
message(STATUS "Debian archive created: ${DASHBOARD_DEBIAN_ARCHIVE_NAME}")
endif()
endif()
endif()
endif()
Loading

0 comments on commit 360df1e

Please sign in to comment.