Add Ceph cluster

building/SHA512SUM.UPSTREAM

@@ -20,3 +20,4 @@ af7cdf5fec21f1a90ab2e608470f3bdd85fc03b57c99bf90dc86efcf3831cc9422604ea1acc26b23
 da301a8e48c09c7de2f3d166d4a5f71bad6e45dd441b5b8bf59c7ea5fd0973db7fe02a83d46e21f7f3c2e77512fe62cd2d1f1c7826ebca7782c1894ea5dae759  kube-state-metrics-1.2.0.tar.xz
 9b79b84e5a768d52f90f6335ccef2c404ecd7a13e78e49f4cd0755fffc6cf34d0dc96ad4c72cad1dab3c5743a8d0d789b3e9b6e625b03c5675bb898ca61a698b  dnsmasq-2.78.tar.xz
 3066c47182ac5d52c432b5f12f4039c4502c63f59653c3a126d3181db292fc7b6454c69b258ec23a3e2924401b11c1b3a8fb51f545076f1c664d7107240e3182  kubernetes-dns-1.14.8.tar.xz
+96e14e580696331669a2f2dc8fd28fb29217b345a6246d7cf315ad1d22871f78e0b43d65296f20b665e4e7c276694bc9a452102249e818c9d795cdb00e2109aa  ceph-12.2.3.tar.xz

building/build-acis/ceph-build/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e -u
+cd "$(dirname "$0")"
+source ../common/container-build-helpers.sh
+
+VERSION="stretch.20180111T215606Z"
+
+DEBVER="${VERSION}"
+UPDATE_TIMESTAMP="2017-11-05T19:27:00-0500"
+
+common_setup
+
+BUILD_DEPS="bc btrfs-tools cmake cpio cryptsetup-bin cython cython3 gdisk git gperf jq libaio-dev libbabeltrace-ctf-dev libbabeltrace-dev libblkid-dev libcurl4-gnutls-dev libexpat1-dev libgoogle-perftools-dev libibverbs-dev libkeyutils-dev libldap2-dev libleveldb-dev liblttng-ust-dev libleveldb-dev liblttng-ust-dev libnss3-dev libsnappy-dev libssl-dev libtool libudev-dev libxml2-dev lsb-release parted pkg-config python python-all-dev python-cherrypy3 python-nose python-pecan python-prettytable python-setuptools python-sphinx python-werkzeug python3-all-dev python3-setuptools socat uuid-runtime virtualenv xfslibs-dev xfsprogs xmlstarlet yasm zlib1g-dev"
+
+start_acbuild_from "debian-build" "${DEBVER}"
+add_packages_to_acbuild ${BUILD_DEPS}
+$ACBUILD set-exec -- /bin/bash
+finish_acbuild

building/build-acis/ceph-run/build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e -u
+cd "$(dirname "$0")"
+source ../common/container-build-helpers.sh
+
+CEPH_VER="12.2.3-1"
+REVISION="11"
+VERSION="${CEPH_VER}-${REVISION}"
+
+BUILDVER="stretch.20180111T215606Z"
+UPDATE_TIMESTAMP="2018-03-17T17:11:00-0400"
+
+common_setup
+
+# build container
+
+start_acbuild_from "ceph" "${CEPH_VER}"
+$ACBUILD copy-to-dir scripts/* /usr/bin/
+add_packages_to_acbuild curl uuid-runtime
+finish_acbuild

building/build-acis/ceph-run/scripts/homeworld-ceph-keygen.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e -u
+
+# this script is run exactly once, when the cluster is first deployed.
+
+# it has two responsibilities:
+#     generate keys
+#     update kubernetes secret with generated keys
+
+mkdir keyrings
+cd keyrings
+
+echo "generating ceph keys"
+
+ceph-authtool --create-keyring "./mon.keyring"                  --gen-key -n mon.                             --cap mon 'allow *'
+ceph-authtool --create-keyring "./client.admin.keyring"         --gen-key -n client.admin         --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+ceph-authtool --create-keyring "./client.bootstrap-osd.keyring" --gen-key -n client.bootstrap-osd             --cap mon 'profile bootstrap-osd'
+
+echo "merging ceph keys"
+
+ceph-authtool "./mon.keyring" --import-keyring "./client.admin.keyring"
+ceph-authtool "./mon.keyring" --import-keyring "./client.bootstrap-osd.keyring"
+
+echo "uploading ceph keys"
+
+echo "{\"data\": {\"mon.keyring\": \"$(base64 -w 0 <mon.keyring)\", \"client.admin.keyring\": \"$(base64 -w 0 <client.admin.keyring)\", \"client.bootstrap-osd.keyring\": \"$(base64 -w 0 <client.bootstrap-osd.keyring)\"}}" >secret.patch
+
+curl -v --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -X PATCH \
+     -H "Content-Type: application/strategic-merge-patch+json" "https://kubernetes.default.svc.hyades.local/api/v1/namespaces/${POD_NAMESPACE}/secrets/${SECRET_NAME}" \
+     -d "$(cat secret.patch)"
+
+echo "keys uploaded!"

building/build-acis/ceph-run/scripts/homeworld-ceph-mgr.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e -u
+
+# this script runs in each pod in the ceph manager daemonset
+
+if [ "${NODE_HOSTNAME:-}" = "" ]
+then
+	echo "NO NODE HOSTNAME" 1>&2
+	exit 1
+fi
+
+ceph-mgr --version  # to help with debugging
+
+if [ ! -e /etc/ceph-keyrings/client.admin.keyring ]
+then
+	echo "NO ADMIN KEYRING" 1>&2
+	exit 1
+fi
+
+mkdir -p "/var/lib/ceph/mgr/ceph-${NODE_HOSTNAME}"
+
+# TODO: don't have this container using the client admin keyring for this...
+ceph -k /etc/ceph-keyrings/client.admin.keyring auth get-or-create "mgr.${NODE_HOSTNAME}" mon 'allow profile mgr' osd 'allow *' mds 'allow *' -o "/var/lib/ceph/mgr/ceph-${NODE_HOSTNAME}/keyring"
+
+ceph-mgr -d -i "${NODE_HOSTNAME}"

building/build-acis/ceph-run/scripts/homeworld-ceph-monitor.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+set -e -u
+
+# this script runs in each pod in the ceph monitor daemonset
+
+# responsibilities:
+#     initialize monitor on first run
+#         collect keyring from kubernetes
+#         monmapgen
+#         ceph-mon mkfs
+#         signal ready to start
+#     launch ceph monitor
+
+if [ "${NODE_HOSTNAME:-}" = "" ]
+then
+	echo "NO NODE HOSTNAME" 1>&2
+	exit 1
+fi
+
+STORAGE_MOUNT="/var/lib/ceph/mon"
+
+ceph-mon --version  # to help with debugging
+
+MONDIR="${STORAGE_MOUNT}/ceph-${NODE_HOSTNAME}"
+
+if [ ! -e "${MONDIR}" ]
+then
+	if [ ! -e /etc/ceph-keyrings/mon.keyring ]
+	then
+		echo "NO MON KEYRING" 1>&2
+		exit 1
+	fi
+
+	rm -rf "${STORAGE_MOUNT}/ceph-mon-tmp-*"
+	WORKDIR="$(mktemp -d --suffix "-${NODE_HOSTNAME}" ceph-mon-tmp-XXXXXXXX -p "${STORAGE_MOUNT}")"
+	WORKTMP="$(mktemp -d)"
+
+	monmaptool --create "${WORKTMP}/monmap" --fsid "$(cat /etc/ceph/fs.uuid)" $(cat /etc/ceph/master.list | sed "s/^/--add /")
+
+	ceph-mon --mkfs -d -i "${NODE_HOSTNAME}" --monmap "${WORKTMP}/monmap" --keyring /etc/ceph-keyrings/mon.keyring --mon-data "${WORKDIR}"
+
+	touch "${WORKDIR}/done"
+
+	# atomically rename new configuration to destination
+	mv "${WORKDIR}" -T "${MONDIR}"
+fi
+
+ceph-mon -d -i "${NODE_HOSTNAME}" --mon-data "${MONDIR}"

building/build-acis/ceph-run/scripts/homeworld-ceph-osd.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -e -u
+
+# this script runs in each pod in the ceph OSD daemonset
+
+# responsibilities:
+#     initialize OSD on first run
+#         generate UUID for OSD
+#         generate key for OSD
+#         bootstrap OSD
+#         ceph mkfs
+#         ceph-osd mkfs
+#     mount device
+#     launch ceph OSD
+
+META_MOUNT="/etc/ceph-osd-meta/"  # preserved across container restarts
+DEVICE="/dev/osd-disk"
+OSDDIR="/var/lib/ceph-homeworld/osd/"  # **NOT** preserved across container restarts
+
+ceph-osd --version  # to help with debugging
+
+mkdir -p "${OSDDIR}"
+
+if [ ! -e "${META_MOUNT}/inited" ]
+then
+        if [ ! -e /etc/ceph-keyrings/client.bootstrap-osd.keyring ]
+        then
+                echo "NO BOOTSTRAP KEYRING" 1>&2
+                exit 1
+        fi
+        if [ -e /etc/ceph-keyrings/mon.keyring ]
+        then
+                echo "should not have access to monitor keyring!" 1>&2
+                exit 1
+        fi
+
+        UUID="$(uuidgen)"
+	OSD_SECRET="$(ceph-authtool --gen-print-key)"
+
+	# TODO: set up lockbox
+
+	# TODO: don't do this on the worker node
+	OSD_ID="$(echo '{"cephx_secret": "'"$OSD_SECRET"'"}' | ceph osd new "$UUID" -i - -n client.bootstrap-osd -k /etc/ceph-keyrings/client.bootstrap-osd.keyring)"
+
+	mkdir "${OSDDIR}/ceph-${OSD_ID}"
+
+	ceph mon getmap -n client.bootstrap-osd -k /etc/ceph-keyrings/client.bootstrap-osd.keyring -o "${OSDDIR}/ceph-${OSD_ID}/activate.monmap"
+	ceph-authtool "${OSDDIR}/ceph-${OSD_ID}/keyring" --create-keyring --name "osd.${OSD_ID}" --add-key "${OSD_SECRET}"
+
+	ln -snf "${DEVICE}" "${OSDDIR}/ceph-${OSD_ID}/block"
+	ceph-osd --osd-objectstore bluestore --mkfs -i "${OSD_ID}" --monmap "${OSDDIR}/ceph-${OSD_ID}/activate.monmap" --keyfile "${OSDDIR}/ceph-${OSD_ID}/keyring" --osd-data "${OSDDIR}/ceph-${OSD_ID}" --osd-uuid "${UUID}"
+
+	echo "${OSD_ID}" >"${META_MOUNT}/inited"
+	rm -rf "${OSDDIR}/ceph-${OSD_ID}"
+else
+	OSD_ID="$(cat "${META_MOUNT}/inited")"
+	if [ "${OSD_ID}" = "" ]
+	then
+		echo "no valid OSD_ID found" 1>&2
+		exit 1
+	fi
+fi
+
+mkdir "${OSDDIR}/ceph-${OSD_ID}"
+ceph-bluestore-tool prime-osd-dir --dev "${DEVICE}" --path "${OSDDIR}/ceph-${OSD_ID}"
+
+ceph-osd -d -i "${OSD_ID}" --osd-data "${OSDDIR}/ceph-${OSD_ID}"

building/build-acis/ceph/build.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+set -e -u
+cd "$(dirname "$0")"
+source ../common/container-build-helpers.sh
+
+CEPH_VER="12.2.3"
+REVISION="1"
+VERSION="${CEPH_VER}-${REVISION}"
+
+BUILDACI="ceph-build"
+DEBVER="stretch.20180111T215606Z"
+BUILDVER="stretch.20180111T215606Z"
+UPDATE_TIMESTAMP="2017-11-05T19:27:00-0500"
+
+common_setup
+
+if [ "${TMPDIR:-}" = "" -o "${TMPDIR:-}" = "/tmp" ]
+then
+	echo 'Warning: $TMPDIR is not set. This may lead to this build failing due to lack of disk space.'
+	echo 'Consider launching this script as:'
+	echo '    TMPDIR=/home/user/buildtmp/ ./build.sh'
+	echo
+fi
+
+# build ceph
+
+init_builder
+
+SRCDIR="${BUILDDIR}/src"
+DESTDIR="${BUILDDIR}/pkg"
+rm -rf "${SRCDIR}" "${DESTDIR}"
+mkdir -p "${SRCDIR}" "${DESTDIR}"
+
+tar -C "${BUILDDIR}" -xf "${UPSTREAM}/ceph-${CEPH_VER}.tar.xz" "ceph-${CEPH_VER}/"
+mv "${BUILDDIR}/ceph-${CEPH_VER}" -T "${SRCDIR}"
+for patch in ceph-fix-boost-detection.patch ceph-never-download-boost.patch
+do
+	patch -d "${SRCDIR}" -p1 -i "$(realpath "$patch")"
+done
+
+build_at_path "${SRCDIR}"
+
+JOBS="${JOBS:-4}"
+
+function gen_cmake() {
+	echo 'export CEPH_BUILD_VIRTUALENV="$(pwd)"'
+	echo "mkdir build"
+	echo "cd build"
+	echo "cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MANPAGE=OFF -DWITH_PYTHON3=OFF -DWITH_LTTNG=OFF -DWITH_EMBEDDED=OFF -DWITH_TESTS=OFF -DWITH_CEPHFS=OFF -DBOOST_J=${JOBS} -DWITH_RADOSGW_BEAST_FRONTEND=ON -DWITH_FUSE=OFF"
+	echo "make -j${JOBS}"
+	echo "make DESTDIR=\"$(path_to_buildpath "${DESTDIR}")\" install"
+}
+BUILDSCRIPT_GEN+=(gen_cmake)
+run_builder
+
+STRIPPABLE="ceph-authtool ceph-bluestore-tool ceph-conf ceph-dencoder ceph-mds ceph-mgr ceph-mon ceph-objectstore-tool ceph-osd ceph-syn rados radosgw radosgw-admin radosgw-es radosgw-object-expirer radosgw-token rbd rbd-mirror rbd-nbd rbd-replay rbd-replay-prep"
+for to_strip in ${STRIPPABLE}
+do
+	strip "${DESTDIR}/usr/bin/${to_strip}"
+done
+
+# build container
+
+start_acbuild_from "debian-mini" "${DEBVER}"
+$ACBUILD copy-to-dir "${DESTDIR}/usr/bin/"* /usr/bin/
+$ACBUILD copy-to-dir "${DESTDIR}/usr/lib/python2.7/dist-packages/"* /usr/lib/python2.7/dist-packages/
+$ACBUILD copy-to-dir "${DESTDIR}/usr/lib/x86_64-linux-gnu/"* /usr/lib/x86_64-linux-gnu/
+$ACBUILD copy-to-dir "${DESTDIR}/usr/libexec/"* /usr/libexec/
+$ACBUILD copy-to-dir "${DESTDIR}/usr/sbin/"* /usr/sbin/
+add_packages_to_acbuild cryptsetup-bin debianutils findutils gdisk grep logrotate psmisc xfsprogs btrfs-tools ntp python-cherrypy3 python-openssl python-pecan python-werkzeug python-flask parted python-prettytable python-requests mime-support libibverbs1 libnss3 libaio1 libleveldb1v5 libgoogle-perftools4 libcurl3-gnutls libbabeltrace1
+$ACBUILD set-exec -- /bin/bash
+finish_acbuild

building/build-acis/ceph/ceph-fix-boost-detection.patch

@@ -0,0 +1,26 @@
+commit e3b703eb12303fa365c7d9c1b11ba117cb3f7590
+Author: Kefu Chai <[email protected]>
+Date:   Fri Nov 24 10:35:00 2017 +0800
+
+    cmake: check bootstrap.sh instead before downloading boost
+
+    config.hpp does not exist if boost is not configured. so a boost source
+    directory unpacked from dist tarball does not have this file. the
+    bootstrap.sh instead, it is used to create the necessary config files
+    for jam.
+
+    Signed-off-by: Kefu Chai <[email protected]>
+
+diff --git a/cmake/modules/BuildBoost.cmake b/cmake/modules/BuildBoost.cmake
+index 1e5bcd5f86..d70828398e 100644
+--- a/cmake/modules/BuildBoost.cmake
++++ b/cmake/modules/BuildBoost.cmake
+@@ -75,7 +75,7 @@ function(do_build_boost version)
+   set(install_command
+     ${b2} install)
+   set(boost_root_dir "${CMAKE_BINARY_DIR}/boost")
+-  if(EXISTS "${PROJECT_SOURCE_DIR}/src/boost/libs/config/include/boost/config.hpp")
++  if(EXISTS "${PROJECT_SOURCE_DIR}/src/boost/bootstrap.sh")
+     message(STATUS "boost already in src")
+     set(source_dir
+       SOURCE_DIR "${PROJECT_SOURCE_DIR}/src/boost")

building/build-acis/ceph/ceph-never-download-boost.patch

@@ -0,0 +1,30 @@
+--- ceph-12.2.3/cmake/modules/BuildBoost.cmake
++++ ceph-12.2.3/cmake/modules/BuildBoost.cmake
+@@ -82,26 +82,7 @@
+   elseif(version VERSION_GREATER 1.66)
+     message(FATAL_ERROR "Unknown BOOST_REQUESTED_VERSION: ${version}")
+   else()
+-    message(STATUS "boost will be downloaded...")
+-    # NOTE: If you change this version number make sure the package is available
+-    # at the three URLs below (may involve uploading to download.ceph.com)
+-    set(boost_version 1.66.0)
+-    set(boost_md5 b2dfbd6c717be4a7bb2d88018eaccf75)
+-    string(REPLACE "." "_" boost_version_underscore ${boost_version} )
+-    set(boost_url 
+-      https://dl.bintray.com/boostorg/release/${boost_version}/source/boost_${boost_version_underscore}.tar.bz2)
+-    if(CMAKE_VERSION VERSION_GREATER 3.7)
+-      set(boost_url
+-        "${boost_url} http://downloads.sourceforge.net/project/boost/boost/${boost_version}/boost_${boost_version_underscore}.tar.bz2")
+-      set(boost_url
+-        "${boost_url} https://download.ceph.com/qa/boost_${boost_version_underscore}.tar.bz2")
+-    endif()
+-    set(source_dir
+-      URL ${boost_url}
+-      URL_MD5 ${boost_md5})
+-    if(CMAKE_VERSION VERSION_GREATER 3.1)
+-      list(APPEND source_dir DOWNLOAD_NO_PROGRESS 1)
+-    endif()
++    message(FATAL_ERROR "Not downloading boost")
+   endif()
+   # build all components in a single shot
+   include(ExternalProject)

building/build-debs/homeworld-admin-tools/debian/changelog

@@ -1,3 +1,27 @@
+homeworld-admin-tools (0.1.54) stretch; urgency=medium
+
+  * Updated debian release
+
+ -- Cel Skeggs <[email protected]>  Sat, 24 Mar 2018 20:43:25 -0400
+
+homeworld-admin-tools (0.1.53) stretch; urgency=medium
+
+  * Updated debian release
+
+ -- Cel Skeggs <[email protected]>  Sat, 24 Mar 2018 15:59:58 -0400
+
+homeworld-admin-tools (0.1.52) stretch; urgency=medium
+
+  * Updated debian release
+
+ -- Cel Skeggs <[email protected]>  Sat, 17 Mar 2018 19:20:23 -0400
+
+homeworld-admin-tools (0.1.51) stretch; urgency=medium
+
+  * Updated debian release
+
+ -- Cel Skeggs <[email protected]>  Sat, 17 Mar 2018 18:01:26 -0400
+
 homeworld-admin-tools (0.1.50) stretch; urgency=medium
 
   * Updated debian release