Skip to content

Commit

Permalink
Merge pull request #682 from cole-miller/musl-ci
Browse files Browse the repository at this point in the history
github: Test with musl
  • Loading branch information
cole-miller authored Sep 2, 2024
2 parents 12bfbd2 + 1fb3a1f commit a39b294
Show file tree
Hide file tree
Showing 12 changed files with 276 additions and 56 deletions.
16 changes: 4 additions & 12 deletions .github/workflows/build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,27 +38,19 @@ jobs:
autoreconf -i
./configure --enable-debug --enable-code-coverage --enable-sanitize \
--enable-build-raft --enable-dqlite-next=${{ matrix.dqlite-next }}
make -j4 unit-test integration-test \
raft-core-fuzzy-test \
raft-core-integration-test \
raft-core-unit-test \
raft-uv-integration-test \
raft-uv-unit-test
make -j$(nproc) check-norun
- name: Test
env:
CC: ${{ matrix.compiler }}
LIBDQLITE_TRACE: 1
run: |
# TODO: return to just `make check` once the mysterious hang is fixed
tests="integration-test \
raft-core-fuzzy-test \
raft-core-integration-test \
raft-core-unit-test \
raft-uv-integration-test \
raft-uv-unit-test"
tests="$(make print-test-programs | tr ' ' '\n' | grep -v '^unit-test$')"
make check TESTS="$tests"
# Grab backtraces when the unit-test binary hangs (this is a heisenbug
# that we've only been able to trigger in GHA jobs)
./unit-test --no-fork &
pid=$!
bash -c "sleep 10m; sudo gdb -p $pid -batch -ex 'thread apply all bt' -ex quit; false" &
Expand Down
22 changes: 22 additions & 0 deletions .github/workflows/static.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
name: CI Tests (musl build)

on:
- push
- pull_request

jobs:
build-and-test:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4

- name: Install dependencies
run: |
sudo apt install -y build-essential automake libtool gettext autopoint tclsh tcl libsqlite3-dev pkg-config git
- name: Build and test
env:
LIBDQLITE_TRACE: 1
run: |
contrib/build-static.sh || (cat ./test-suite.log && false)
29 changes: 23 additions & 6 deletions Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,16 @@ ACLOCAL_AMFLAGS = -I m4

AM_CFLAGS += $(CODE_COVERAGE_CFLAGS)
AM_CFLAGS += $(SQLITE_CFLAGS) $(UV_CFLAGS) $(PTHREAD_CFLAGS)
AM_LDFLAGS = $(UV_LIBS) $(PTHREAD_LIBS)

if WITH_STATIC_DEPS
AM_CFLAGS += -DDQLITE_STATIC_LIBC
static = -all-static
else
static =
endif

AM_LDFLAGS = $(static)
AM_LDFLAGS += $(UV_LIBS) $(PTHREAD_LIBS)

if DQLITE_NEXT_ENABLED
AM_CFLAGS += -DDQLITE_NEXT
Expand Down Expand Up @@ -216,7 +225,7 @@ libtest_la_SOURCES += \
test/raft/lib/loop.c

libraft_la_CFLAGS = $(AM_CFLAGS)
libraft_la_LDFLAGS = $(UV_LIBS)
libraft_la_LDFLAGS = $(static) $(UV_LIBS)

raft_core_unit_test_SOURCES = \
$(libraft_la_SOURCES) \
Expand Down Expand Up @@ -260,7 +269,7 @@ raft_core_integration_test_SOURCES = \
test/raft/integration/test_transfer.c \
test/raft/integration/test_voter_contacts.c
raft_core_integration_test_CFLAGS = $(AM_CFLAGS) -Wno-conversion
raft_core_integration_test_LDFLAGS = -no-install
raft_core_integration_test_LDFLAGS = $(static) -no-install
raft_core_integration_test_LDADD = libtest.la libraft.la

raft_core_fuzzy_test_SOURCES = \
Expand All @@ -273,7 +282,7 @@ raft_core_fuzzy_test_SOURCES = \
test/raft/fuzzy/test_membership.c \
test/raft/fuzzy/test_replication.c
raft_core_fuzzy_test_CFLAGS = $(AM_CFLAGS) -Wno-conversion
raft_core_fuzzy_test_LDFLAGS = -no-install
raft_core_fuzzy_test_LDFLAGS = $(static) -no-install
raft_core_fuzzy_test_LDADD = libtest.la libraft.la

raft_uv_unit_test_SOURCES = \
Expand Down Expand Up @@ -315,14 +324,14 @@ raft_uv_integration_test_SOURCES = \
test/raft/integration/test_uv_truncate_snapshot.c \
test/raft/integration/test_uv_work.c
raft_uv_integration_test_CFLAGS = $(AM_CFLAGS) -Wno-type-limits -Wno-conversion
raft_uv_integration_test_LDFLAGS = -no-install
raft_uv_integration_test_LDFLAGS = $(static) -no-install
raft_uv_integration_test_LDADD = libtest.la $(UV_LIBS)

if LZ4_AVAILABLE
libdqlite_la_CFLAGS += -DLZ4_AVAILABLE $(LZ4_CFLAGS)
libdqlite_la_LDFLAGS += $(LZ4_LIBS)
raft_core_unit_test_CFLAGS += -DLZ4_AVAILABLE $(LZ4_CFLAGS)
raft_core_unit_test_LDFLAGS = $(LZ4_LIBS)
raft_core_unit_test_LDFLAGS = $(static) $(LZ4_LIBS)
libraft_la_CFLAGS += -DLZ4_AVAILABLE $(LZ4_CFLAGS)
libraft_la_LDFLAGS += $(LZ4_LIBS)
raft_uv_integration_test_CFLAGS += -DLZ4_AVAILABLE
Expand Down Expand Up @@ -351,6 +360,14 @@ endif

TESTS = $(check_PROGRAMS)

check-norun: $(TESTS)

# Sometimes we want to run the test binaries manually. This target allows us to
# get the list of binaries out of the build system for that use-case,
# maintaining a single source of truth.
print-test-programs:
@echo '$(TESTS)'

if CODE_COVERAGE_ENABLED

include $(top_srcdir)/aminclude_static.am
Expand Down
10 changes: 6 additions & 4 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ AC_CONFIG_AUX_DIR([ac])
AM_INIT_AUTOMAKE([subdir-objects -Wall -Werror -Wno-portability foreign])
AM_SILENT_RULES([yes])

# Without this line, AC_PROG_CC boneheadedly adds `-g -O2` to our CFLAGS.
AC_SUBST(CFLAGS, "")
AC_SUBST(AM_CFLAGS)
AC_PROG_CC
AC_USE_SYSTEM_EXTENSIONS

Expand Down Expand Up @@ -43,6 +42,11 @@ AC_ARG_ENABLE(dqlite-next, AS_HELP_STRING([--enable-dqlite-next[=ARG]], [build w
AM_CONDITIONAL(DQLITE_NEXT_ENABLED, test "x$enable_dqlite_next" = "xyes")
AS_IF([test "x$enable_build_raft" != "xyes" -a "x$enable_dqlite_next" = "xyes"], [AC_MSG_ERROR([dqlite-next requires bundled raft])], [])

AC_ARG_WITH(static-deps,
AS_HELP_STRING([--with-static-deps[=ARG]],
[skip building a shared library and link test binaries statically]))
AM_CONDITIONAL(WITH_STATIC_DEPS, test "x$with_static_deps" = "xyes")

# Whether to enable code coverage.
AX_CODE_COVERAGE

Expand Down Expand Up @@ -96,10 +100,8 @@ CC_CHECK_FLAGS_APPEND([AM_CFLAGS],[CFLAGS],[ \
-pipe \
-fno-strict-aliasing \
-fdiagnostics-color \
-fexceptions \
-fstack-clash-protection \
-fstack-protector-strong \
-fasynchronous-unwind-tables \
-fdiagnostics-show-option \
-Wall \
-Wextra \
Expand Down
143 changes: 143 additions & 0 deletions contrib/build-static.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
#!/bin/bash -xeu

DIR="${DIR:=$(realpath `dirname "${0}"`)}"

REPO_MUSL="https://git.launchpad.net/musl"
REPO_LIBTIRPC="https://salsa.debian.org/debian/libtirpc.git"
REPO_LIBNSL="https://github.com/thkukuk/libnsl.git"
REPO_LIBUV="https://github.com/libuv/libuv.git"
REPO_LIBLZ4="https://github.com/lz4/lz4.git"
REPO_SQLITE="https://github.com/sqlite/sqlite.git"

TAG_MUSL="${TAG_MUSL:-v1.2.4}"
DQLITE_PATH="${DQLITE_PATH:-$DIR/..}"

BUILD_DIR="${DIR}/build"
INSTALL_DIR="${DIR}/prefix"
mkdir -p "${BUILD_DIR}" "${INSTALL_DIR}" "${INSTALL_DIR}/lib" "${INSTALL_DIR}/include"
BUILD_DIR="$(realpath "${BUILD_DIR}")"
INSTALL_DIR="$(realpath "${INSTALL_DIR}")"

export CFLAGS=""
MACHINE_TYPE="$(uname -m)"
if [ "${MACHINE_TYPE}" = "ppc64le" ]; then
MACHINE_TYPE="powerpc64le"
export CFLAGS="-mlong-double-64"
fi
export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig"

clone-latest-tag() {
name="$1"
repo="$2"
tagpattern="${3:-.*}"
mkdir "${name}"
pushd "${name}"
git init
git remote add upstream "${repo}"
git fetch upstream 'refs/tags/*:refs/tags/*'
tag="$(git tag | grep "${tagpattern}" | sort -V -r | head -n1)"
echo "Selected $name tag ${tag}"
git checkout "${tag}"
popd
}

# build musl
if [ ! -f "${INSTALL_DIR}/musl/bin/musl-gcc" ]; then
(
cd "${BUILD_DIR}"
rm -rf musl
git clone "${REPO_MUSL}" --depth 1 --branch "${TAG_MUSL}" musl
cd musl
./configure --prefix="${INSTALL_DIR}/musl"
make -j
make -j install

# missing musl header files
ln -s "/usr/include/${MACHINE_TYPE}-linux-gnu/sys/queue.h" "${INSTALL_DIR}/musl/include/sys/queue.h" || true
ln -s "/usr/include/${MACHINE_TYPE}-linux-gnu/asm" "${INSTALL_DIR}/musl/include/asm" || true
ln -s /usr/include/asm-generic "${INSTALL_DIR}/musl/include/asm-generic" || true
ln -s /usr/include/linux "${INSTALL_DIR}/musl/include/linux" || true
)
fi

export PATH="${PATH}:${INSTALL_DIR}/musl/bin"
export CFLAGS="${CFLAGS} -isystem ${INSTALL_DIR}/musl/include"
export CC=musl-gcc
export LDFLAGS=-static

# build libtirpc
if [ ! -f "${BUILD_DIR}/libtirpc/src/libtirpc.la" ]; then
(
cd "${BUILD_DIR}"
rm -rf libtirpc
clone-latest-tag libtirpc "${REPO_LIBTIRPC}" upstream
cd libtirpc
chmod +x autogen.sh
./autogen.sh
./configure --disable-shared --disable-gssapi --prefix="${INSTALL_DIR}"
make -j install
)
fi

# build libnsl
if [ ! -f "${BUILD_DIR}/libnsl/src/libnsl.la" ]; then
(
cd "${BUILD_DIR}"
rm -rf libnsl
clone-latest-tag libnsl "${REPO_LIBNSL}"
cd libnsl
./autogen.sh
autoreconf -i
autoconf
./configure --disable-shared --prefix="${INSTALL_DIR}"
make -j install
)
fi

# build libuv
if [ ! -f "${BUILD_DIR}/libuv/libuv.la" ]; then
(
cd "${BUILD_DIR}"
rm -rf libuv
clone-latest-tag libuv "${REPO_LIBUV}"
cd libuv
./autogen.sh
./configure --disable-shared --prefix="${INSTALL_DIR}"
make -j install
)
fi

# build liblz4
if [ ! -f "${BUILD_DIR}/lz4/lib/liblz4.a" ] || [ ! -f "${BUILD_DIR}/lz4/lib/liblz4.so" ]; then
(
cd "${BUILD_DIR}"
rm -rf lz4
clone-latest-tag lz4 "${REPO_LIBLZ4}"
cd lz4
make install -j PREFIX="${INSTALL_DIR}" BUILD_SHARED=no
)
fi

# build sqlite3
if [ ! -f "${BUILD_DIR}/sqlite/libsqlite3.la" ]; then
(
cd "${BUILD_DIR}"
rm -rf sqlite
clone-latest-tag sqlite "${REPO_SQLITE}"
cd sqlite
./configure --disable-shared --disable-readline --prefix="${INSTALL_DIR}" \
CFLAGS="${CFLAGS} -DSQLITE_ENABLE_DBSTAT_VTAB=1"
make install -j BCC="${CC} -g -O2 ${CFLAGS} ${LDFLAGS}"
)
fi

# build dqlite
if [ ! -f "${BUILD_DIR}/dqlite/libdqlite.la" ]; then
(
cd "${DQLITE_PATH}"
autoreconf -i
./configure --enable-build-raft --with-static-deps --prefix="${INSTALL_DIR}"
make -j check-norun
make check
)
fi
14 changes: 7 additions & 7 deletions test/integration/test_vfs.c
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ static void tearDownRestorePendingByte(void *data)
tearDown(data);
}

#define PAGE_SIZE 512
#define DB_PAGE_SIZE 512

#define PRAGMA(DB, COMMAND) \
_rv = sqlite3_exec(DB, "PRAGMA " COMMAND, NULL, NULL, NULL); \
Expand Down Expand Up @@ -216,12 +216,12 @@ struct tx
if (_frames != NULL) { \
TX.page_numbers = \
munit_malloc(sizeof *TX.page_numbers * TX.n); \
TX.frames = munit_malloc(PAGE_SIZE * TX.n); \
TX.frames = munit_malloc(DB_PAGE_SIZE * TX.n); \
for (_i = 0; _i < TX.n; _i++) { \
dqlite_vfs_frame *_frame = &_frames[_i]; \
TX.page_numbers[_i] = _frame->page_number; \
memcpy(TX.frames + _i * PAGE_SIZE, \
_frame->data, PAGE_SIZE); \
memcpy(TX.frames + _i * DB_PAGE_SIZE, \
_frame->data, DB_PAGE_SIZE); \
sqlite3_free(_frame->data); \
} \
sqlite3_free(_frames); \
Expand Down Expand Up @@ -1502,7 +1502,7 @@ TEST(vfs, snapshotInitialDatabase, setUp, tearDown, 0, vfs_params)

SNAPSHOT("1", snapshot);

munit_assert_int(snapshot.n, ==, PAGE_SIZE);
munit_assert_int(snapshot.n, ==, DB_PAGE_SIZE);
page = snapshot.data;

munit_assert_int(memcmp(&page[16], page_size, 2), ==, 0);
Expand Down Expand Up @@ -1536,7 +1536,7 @@ TEST(vfs, snapshotAfterFirstTransaction, setUp, tearDown, 0, vfs_params)

SNAPSHOT("1", snapshot);

munit_assert_int(snapshot.n, ==, PAGE_SIZE + 32 + (24 + PAGE_SIZE) * 2);
munit_assert_int(snapshot.n, ==, DB_PAGE_SIZE + 32 + (24 + DB_PAGE_SIZE) * 2);
page = snapshot.data;

munit_assert_int(memcmp(&page[16], page_size, 2), ==, 0);
Expand Down Expand Up @@ -1571,7 +1571,7 @@ TEST(vfs, snapshotAfterCheckpoint, setUp, tearDown, 0, vfs_params)

SNAPSHOT("1", snapshot);

munit_assert_int(snapshot.n, ==, PAGE_SIZE * 2);
munit_assert_int(snapshot.n, ==, DB_PAGE_SIZE * 2);
page = snapshot.data;

munit_assert_int(memcmp(&page[16], page_size, 2), ==, 0);
Expand Down
1 change: 0 additions & 1 deletion test/lib/runner.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
#define TEST_RUNNER_H

#include <signal.h>
#include <execinfo.h>
#include <stdio.h>
#include <stdlib.h>

Expand Down
6 changes: 3 additions & 3 deletions test/raft/integration/test_uv_tcp_connect.c
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ TEST(tcp_connect, connectByName, setUp, tearDown, 0, NULL)
}

/* Successfully connect to the peer by first IP */
TEST(tcp_connect, firstIP, setUp, tearDown, 0, NULL)
ADDRINFO_TEST(tcp_connect, firstIP, setUp, tearDown, 0, NULL)
{
struct fixture *f = data;
const struct AddrinfoResult results[] = {{"127.0.0.1", TCP_SERVER_PORT},
Expand All @@ -203,7 +203,7 @@ TEST(tcp_connect, firstIP, setUp, tearDown, 0, NULL)
}

/* Successfully connect to the peer by second IP */
TEST(tcp_connect, secondIP, setUp, tearDown, 0, NULL)
ADDRINFO_TEST(tcp_connect, secondIP, setUp, tearDown, 0, NULL)
{
struct fixture *f = data;
const struct AddrinfoResult results[] = {{"127.0.0.1", .6666},
Expand Down Expand Up @@ -332,7 +332,7 @@ TEST(tcp_connect, closeDuringConnectAbort, setUp, tearDownDeps, 0, NULL)

/* The transport gets closed right after the first connection attempt failed,
* while doing a second connection attempt. */
TEST(tcp_connect, closeDuringSecondConnect, setUp, tearDownDeps, 0, NULL)
ADDRINFO_TEST(tcp_connect, closeDuringSecondConnect, setUp, tearDownDeps, 0, NULL)
{
struct fixture *f = data;
struct uv_check_s check;
Expand Down
Loading

0 comments on commit a39b294

Please sign in to comment.