From e23882c85b9fa3bce15ba1fed03e894ff990d099 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 19:27:24 -0700 Subject: [PATCH 01/37] rpc: colocate service code generator with rpc This fits with meson's default behavior of finding programs below the current directory. Otherwise we have to find the program at the very top level of the build. Signed-off-by: Noah Watkins --- cmake/rpcgen.cmake | 2 +- tools/rpcgen.py => src/v/rpc/rpc_compiler.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tools/rpcgen.py => src/v/rpc/rpc_compiler.py (100%) diff --git a/cmake/rpcgen.cmake b/cmake/rpcgen.cmake index ba219a50d0e1..9910cdaee6c1 100644 --- a/cmake/rpcgen.cmake +++ b/cmake/rpcgen.cmake @@ -3,7 +3,7 @@ function(rpcgen) set(multi_value_args INCLUDES LIBRARIES DEFINITIONS COMPILE_OPTIONS) cmake_parse_arguments(args "" "${one_value_args}" "${multi_value_args}" ${ARGN}) get_filename_component(out_dir ${args_OUT_FILE} DIRECTORY) - set(generator "${PROJECT_SOURCE_DIR}/tools/rpcgen.py") + set(generator "${PROJECT_SOURCE_DIR}/src/v/rpc/rpc_compiler.py") add_custom_command( DEPENDS ${args_IN_FILE} diff --git a/tools/rpcgen.py b/src/v/rpc/rpc_compiler.py similarity index 100% rename from tools/rpcgen.py rename to src/v/rpc/rpc_compiler.py From a79858a423dc4993cf6b2aabf7f9bacf422c36df Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sun, 15 May 2022 08:16:04 -0700 Subject: [PATCH 02/37] kafka: protocol code gen should take output paths This changes makes the code generator easier to integrate into meson which wants to have control over output file paths. Signed-off-by: Noah Watkins --- src/v/kafka/protocol/schemata/CMakeLists.txt | 2 +- src/v/kafka/protocol/schemata/generator.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) mode change 100644 => 100755 src/v/kafka/protocol/schemata/generator.py diff --git a/src/v/kafka/protocol/schemata/CMakeLists.txt b/src/v/kafka/protocol/schemata/CMakeLists.txt index f0781e2ce72a..2875de3aad3e 100644 --- a/src/v/kafka/protocol/schemata/CMakeLists.txt +++ b/src/v/kafka/protocol/schemata/CMakeLists.txt @@ -78,7 +78,7 @@ foreach(schema ${schemata}) add_custom_command( OUTPUT ${hdr} ${src} COMMAND ${KAFKA_CODEGEN_VENV} ${CMAKE_CURRENT_SOURCE_DIR}/generator.py - ARGS ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/${schema} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}/${schema} ${hdr} ${src} DEPENDS ${schema} ${CMAKE_CURRENT_SOURCE_DIR}/generator.py ${KAFKA_CODEGEN_VENV} COMMENT "Running kafka request codegen on ${schema}" VERBATIM) diff --git a/src/v/kafka/protocol/schemata/generator.py b/src/v/kafka/protocol/schemata/generator.py old mode 100644 new mode 100755 index 01a5b1a78fcd..617b2bcc922b --- a/src/v/kafka/protocol/schemata/generator.py +++ b/src/v/kafka/protocol/schemata/generator.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# # Copyright 2020 Redpanda Data, Inc. # # Use of this software is governed by the Business Source License @@ -1318,11 +1320,10 @@ def parse_flexible_versions(flex_version): if __name__ == "__main__": - assert len(sys.argv) == 3 - outdir = pathlib.Path(sys.argv[1]) - schema_path = pathlib.Path(sys.argv[2]) - src = (outdir / schema_path.name).with_suffix(".cc") - hdr = (outdir / schema_path.name).with_suffix(".h") + assert len(sys.argv) == 4 + schema_path = pathlib.Path(sys.argv[1]) + hdr = pathlib.Path(sys.argv[2]) + src = pathlib.Path(sys.argv[3]) # remove comments from the json file. comments are a non-standard json # extension that is not supported by the python json parser. From a982ae1640bd18b2c3921b722b3e47a32c633c89 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 11:46:20 -0700 Subject: [PATCH 03/37] build: setup and project and dependencies Signed-off-by: Noah Watkins --- meson.build | 7 +++++ src/v/meson.build | 55 ++++++++++++++++++++++++++++++++++ subprojects/avro.wrap | 7 +++++ subprojects/base64.wrap | 7 +++++ subprojects/crc32c.wrap | 7 +++++ subprojects/ctre.wrap | 7 +++++ subprojects/hdr_histogram.wrap | 7 +++++ subprojects/rapidjson.wrap | 7 +++++ subprojects/seastar.wrap | 10 +++++++ 9 files changed, 114 insertions(+) create mode 100644 meson.build create mode 100644 src/v/meson.build create mode 100644 subprojects/avro.wrap create mode 100644 subprojects/base64.wrap create mode 100644 subprojects/crc32c.wrap create mode 100644 subprojects/ctre.wrap create mode 100644 subprojects/hdr_histogram.wrap create mode 100644 subprojects/rapidjson.wrap create mode 100644 subprojects/seastar.wrap diff --git a/meson.build b/meson.build new file mode 100644 index 000000000000..8d25dd032454 --- /dev/null +++ b/meson.build @@ -0,0 +1,7 @@ +project('redpanda', 'cpp', + default_options : [ + 'cpp_std=c++20', + 'default_library=static', + ]) + +subdir('src/v') diff --git a/src/v/meson.build b/src/v/meson.build new file mode 100644 index 000000000000..bd89a537c12c --- /dev/null +++ b/src/v/meson.build @@ -0,0 +1,55 @@ +# +# Dependencies that are not satisifed by system packages. These are compiled +# from source by both the public and production release builds. +# +seastar = dependency('seastar', include_type: 'system') +seastar_testing = dependency('seastar_testing', include_type: 'system') +seastar_perf_testing = dependency('seastar_perf_testing', include_type: 'system') +seastar_json2code = find_program('seastar-json2code') +crc32c = dependency('crc32c') +rapidjson = dependency('rapidjson') +ctre = dependency('ctre') +hdr_histogram = dependency('hdr_histogram') +base64 = dependency('base64') +avro = dependency('avro') + +# +# Dependencies that are satisified by system packages for the public build, but +# are compiled from source for production release builds. +# +gnutls = dependency('gnutls') +xxhash = dependency('libxxhash') +absl = dependency('absl', + modules: [ + 'absl::flat_hash_map', + 'absl::btree', + 'absl::flat_hash_set', + 'absl::node_hash_set', + 'absl::node_hash_map', + 'absl::random_seed_sequences']) +yaml = dependency('yaml-cpp') +zlib = dependency('zlib') +zstd = dependency('libzstd') +snappy = dependency('snappy') +boost = dependency('boost', + modules: [ + 'program_options', + 'thread', + 'filesystem', + 'unit_test_framework']) +protobuf = dependency('protobuf', + modules: ['protobuf::libprotobuf', 'protobuf::libprotoc']) +roaring = dependency('roaring') + +# this enables our flat includes like `#include "bytes/bytes.h"` +common = declare_dependency( + include_directories: include_directories('.')) + +# ignore warnings that are common in tests. these should be fixed +# so that we can eliminate the disabling of these warnings. +add_project_arguments( + '-Wno-deprecated-experimental-coroutine', + '-Wno-unused-lambda-capture', + '-Wno-unused-function', + '-Wno-unused-variable', + language: 'cpp') diff --git a/subprojects/avro.wrap b/subprojects/avro.wrap new file mode 100644 index 000000000000..568b8028b48d --- /dev/null +++ b/subprojects/avro.wrap @@ -0,0 +1,7 @@ +[wrap-git] +url = https://github.com/dotnwat/avro.git +revision = meson +depth = 1 + +[provide] +avro = avro_dep diff --git a/subprojects/base64.wrap b/subprojects/base64.wrap new file mode 100644 index 000000000000..c721e0bd3985 --- /dev/null +++ b/subprojects/base64.wrap @@ -0,0 +1,7 @@ +[wrap-git] +url = https://github.com/dotnwat/base64.git +revision = meson +depth = 1 + +[provide] +base64 = base64_dep diff --git a/subprojects/crc32c.wrap b/subprojects/crc32c.wrap new file mode 100644 index 000000000000..31f9a2720b3e --- /dev/null +++ b/subprojects/crc32c.wrap @@ -0,0 +1,7 @@ +[wrap-git] +url = https://github.com/dotnwat/crc32c +revision = meson +depth = 1 + +[provide] +crc32c = crc32c_dep diff --git a/subprojects/ctre.wrap b/subprojects/ctre.wrap new file mode 100644 index 000000000000..cdbd82858351 --- /dev/null +++ b/subprojects/ctre.wrap @@ -0,0 +1,7 @@ +[wrap-git] +url = https://github.com/dotnwat/compile-time-regular-expressions.git +revision = meson +depth = 1 + +[provide] +ctre = ctre_dep diff --git a/subprojects/hdr_histogram.wrap b/subprojects/hdr_histogram.wrap new file mode 100644 index 000000000000..4a1ce881bd7d --- /dev/null +++ b/subprojects/hdr_histogram.wrap @@ -0,0 +1,7 @@ +[wrap-git] +url = https://github.com/dotnwat/HdrHistogram_c.git +revision = meson +depth = 1 + +[provide] +hdr_histogram = hdr_histogram_dep diff --git a/subprojects/rapidjson.wrap b/subprojects/rapidjson.wrap new file mode 100644 index 000000000000..3647c45f7392 --- /dev/null +++ b/subprojects/rapidjson.wrap @@ -0,0 +1,7 @@ +[wrap-git] +url = https://github.com/dotnwat/rapidjson.git +revision = meson +depth = 1 + +[provide] +rapidjson = rapidjson_dep diff --git a/subprojects/seastar.wrap b/subprojects/seastar.wrap new file mode 100644 index 000000000000..2a0699be3773 --- /dev/null +++ b/subprojects/seastar.wrap @@ -0,0 +1,10 @@ +[wrap-git] +url = https://github.com/dotnwat/seastar.git +revision = meson +depth = 1 + +[provide] +seastar = seastar_dep +seastar_testing = seastar_testing_dep +seastar_perf_testing = seastar_perf_testing_dep +program_names = seastar-json2code From 5a9a9955e221fe2ac91ead521594f06f92d81f2c Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Wed, 1 Jun 2022 17:58:57 -0700 Subject: [PATCH 04/37] build: add test utilities Signed-off-by: Noah Watkins --- src/v/meson.build | 2 ++ src/v/test_utils/meson.build | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 src/v/test_utils/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index bd89a537c12c..aa3b3cc82990 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -53,3 +53,5 @@ add_project_arguments( '-Wno-unused-function', '-Wno-unused-variable', language: 'cpp') + +subdir('test_utils') diff --git a/src/v/test_utils/meson.build b/src/v/test_utils/meson.build new file mode 100644 index 000000000000..89933c23dafd --- /dev/null +++ b/src/v/test_utils/meson.build @@ -0,0 +1,8 @@ +libseastar_testing_main = library('seastar_testing_main', + 'seastar_testing_main.cc', + dependencies: [seastar_testing], + implicit_include_directories: false) + +seastar_testing_main = declare_dependency( + dependencies: [seastar_testing], + link_with: libseastar_testing_main) From 9810e13e3905a93e471a669cbfacc903d449f6e5 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 27 May 2022 07:34:44 -0700 Subject: [PATCH 05/37] build: add ssx Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/ssx/meson.build | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 src/v/ssx/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index aa3b3cc82990..ace39990e7a2 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -55,3 +55,4 @@ add_project_arguments( language: 'cpp') subdir('test_utils') +subdir('ssx') diff --git a/src/v/ssx/meson.build b/src/v/ssx/meson.build new file mode 100644 index 000000000000..75fa792eb9b1 --- /dev/null +++ b/src/v/ssx/meson.build @@ -0,0 +1,11 @@ +t = executable('test_ssx', + 'tests/async_transforms.cc', + 'tests/sformat.cc', + 'tests/future_util.cc', + dependencies: [common, seastar_testing_main]) +test('ssx', t) + +t = executable('bench_ssx', + 'tests/sformat_bench.cc', + dependencies: [common, seastar_perf_testing]) +benchmark('ssx', t) From 072a93e10a709a701a9101609438f2ac803510fe Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 12:37:11 -0700 Subject: [PATCH 06/37] build: add bytes Signed-off-by: Noah Watkins --- src/v/bytes/meson.build | 17 +++++++++++++++++ src/v/meson.build | 1 + 2 files changed, 18 insertions(+) create mode 100644 src/v/bytes/meson.build diff --git a/src/v/bytes/meson.build b/src/v/bytes/meson.build new file mode 100644 index 000000000000..feee9c1249a0 --- /dev/null +++ b/src/v/bytes/meson.build @@ -0,0 +1,17 @@ +libbytes = library('bytes', + 'bytes.cc', + 'iobuf.cc', + dependencies: [common, seastar], + implicit_include_directories: false) + +bytes = declare_dependency( + link_with: libbytes, + dependencies: [common, seastar, absl]) + +t = executable('test_bytes', + 'tests/iobuf_tests.cc', + 'tests/iobuf_utils_tests.cc', + 'tests/bytes_tests.cc', + dependencies: [seastar_testing_main, bytes]) + +test('bytes test', t) diff --git a/src/v/meson.build b/src/v/meson.build index ace39990e7a2..9ee90607fc8d 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -56,3 +56,4 @@ add_project_arguments( subdir('test_utils') subdir('ssx') +subdir('bytes') From c456a8e43129347e62d5768b8a666a776425636c Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Wed, 1 Jun 2022 07:53:09 -0700 Subject: [PATCH 07/37] build: add hashing Signed-off-by: Noah Watkins --- src/v/hashing/meson.build | 24 ++++++++++++++++++++++++ src/v/meson.build | 1 + 2 files changed, 25 insertions(+) create mode 100644 src/v/hashing/meson.build diff --git a/src/v/hashing/meson.build b/src/v/hashing/meson.build new file mode 100644 index 000000000000..59e2bad1b803 --- /dev/null +++ b/src/v/hashing/meson.build @@ -0,0 +1,24 @@ +libhashing = library('hashing', + 'murmur.cc', + dependencies: [common], + implicit_include_directories: false) + +hashing = declare_dependency( + dependencies: [common, crc32c, bytes], + compile_args: '-DXXH_PRIVATE_API', + link_with: libhashing) + +t = executable('test_xxhash', + 'tests/xx_tests.cc', + dependencies: [hashing, boost]) +test('xxhash tests', t) + +t = executable('test_secure_hashing', + 'tests/secure_tests.cc', + dependencies: [hashing, boost]) +test('secure hashing tests', t) + +t = executable('bench_hashing', + 'tests/hash_bench.cc', + dependencies: [hashing, seastar_perf_testing]) +benchmark('hashing benchmark', t) diff --git a/src/v/meson.build b/src/v/meson.build index 9ee90607fc8d..cb1a88450576 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -57,3 +57,4 @@ add_project_arguments( subdir('test_utils') subdir('ssx') subdir('bytes') +subdir('hashing') From 8b55b82bd297fe53b70e00de4831046c989779ce Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Wed, 1 Jun 2022 07:53:23 -0700 Subject: [PATCH 08/37] build: add serde Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/serde/meson.build | 4 ++++ src/v/serde/test/meson.build | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 src/v/serde/meson.build create mode 100644 src/v/serde/test/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index cb1a88450576..47f715091107 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -58,3 +58,4 @@ subdir('test_utils') subdir('ssx') subdir('bytes') subdir('hashing') +subdir('serde') diff --git a/src/v/serde/meson.build b/src/v/serde/meson.build new file mode 100644 index 000000000000..db0f54a18511 --- /dev/null +++ b/src/v/serde/meson.build @@ -0,0 +1,4 @@ +serde = declare_dependency( + dependencies: [hashing]) + +subdir('test') diff --git a/src/v/serde/test/meson.build b/src/v/serde/test/meson.build new file mode 100644 index 000000000000..0f0772d84e94 --- /dev/null +++ b/src/v/serde/test/meson.build @@ -0,0 +1,33 @@ +t = executable('test_serde', + 'serde_test.cc', + dependencies: [seastar_testing_main, serde]) +test('serde', t) + +t = executable('bench_serde', + 'bench.cc', + dependencies: [seastar_perf_testing, serde]) +benchmark('serde bench', t) + +serde_test_codegen = find_program('struct_gen.py') + +serde_test_structs_h = custom_target( + output: 'generated_structs.h', + command: [serde_test_codegen, '@OUTPUT@']) + +# debug only +tt = executable('serde_fuzz', + 'fuzz.cc', + serde_test_structs_h, + cpp_args: ['-g', '-O0', '-fsanitize=address,fuzzer,undefined'], + link_args: ['-fsanitize=address,fuzzer,undefined'], + dependencies: [serde]) +test('serde fuzz', tt, + args: ['-max_total_time=60', '-rss_limit_mb=8192']) + +# all builds +t = executable('serde_fuzz_main', + 'fuzz.cc', + serde_test_structs_h, + cpp_args: ['-DMAIN=1'], + dependencies: [serde]) +test('serde fuzz main', t) From 96910f504b4d979891f8cf4b2c7644210c0d6cf6 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 14:10:20 -0700 Subject: [PATCH 09/37] build: add failure injector Signed-off-by: Noah Watkins --- src/v/finjector/meson.build | 8 ++++++++ src/v/meson.build | 1 + 2 files changed, 9 insertions(+) create mode 100644 src/v/finjector/meson.build diff --git a/src/v/finjector/meson.build b/src/v/finjector/meson.build new file mode 100644 index 000000000000..4b5fd0c405ef --- /dev/null +++ b/src/v/finjector/meson.build @@ -0,0 +1,8 @@ +libfinjector = library('finjector', + 'hbadger.cc', + dependencies: [common, seastar, absl], + implicit_include_directories: false) + +finjector = declare_dependency( + dependencies: [common, seastar, absl], + link_with: libfinjector) diff --git a/src/v/meson.build b/src/v/meson.build index 47f715091107..4a2b7f168d29 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -59,3 +59,4 @@ subdir('ssx') subdir('bytes') subdir('hashing') subdir('serde') +subdir('finjector') From e8be0c7ebf6e766c4e85f3f912ee5411f2009df1 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 15:03:48 -0700 Subject: [PATCH 10/37] build: add random Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/random/meson.build | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 src/v/random/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 4a2b7f168d29..60588e395277 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -60,3 +60,4 @@ subdir('bytes') subdir('hashing') subdir('serde') subdir('finjector') +subdir('random') diff --git a/src/v/random/meson.build b/src/v/random/meson.build new file mode 100644 index 000000000000..9469913bfb3b --- /dev/null +++ b/src/v/random/meson.build @@ -0,0 +1,8 @@ +random = declare_dependency( + dependencies: [common, seastar, absl]) + +t = executable('test_random', + 'random_test.cc', + dependencies: [random, boost]) + +test('random tests', t) From b404b7eae7986dbc8e43b0c86c40eb7e54622f19 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 19:26:46 -0700 Subject: [PATCH 11/37] build: add compression Signed-off-by: Noah Watkins --- src/v/compression/meson.build | 25 +++++++++++++++++++++++++ src/v/meson.build | 1 + 2 files changed, 26 insertions(+) create mode 100644 src/v/compression/meson.build diff --git a/src/v/compression/meson.build b/src/v/compression/meson.build new file mode 100644 index 000000000000..e233519f7361 --- /dev/null +++ b/src/v/compression/meson.build @@ -0,0 +1,25 @@ +libcompression = library('compression', + 'compression.cc', + 'stream_zstd.cc', + 'logger.cc', + 'snappy_standard_compressor.cc', + 'internal/snappy_java_compressor.cc', + 'internal/lz4_frame_compressor.cc', + 'internal/gzip_compressor.cc', + cpp_args: ['-DZSTD_STATIC_LINKING_ONLY'], + implicit_include_directories: false, + dependencies: [common, bytes, zlib, zstd, snappy]) + +compression = declare_dependency( + dependencies: [common, bytes, zlib, zstd, snappy], + link_with: libcompression) + +t = executable('bench_zstd_stream', + 'tests/zstd_stream_bench.cc', + dependencies: [compression, seastar_perf_testing]) +benchmark('zstd stream', t) + +t = executable('test_zstd', + 'tests/zstd_tests.cc', + dependencies: [compression, seastar_testing_main]) +test('zstd testing', t) diff --git a/src/v/meson.build b/src/v/meson.build index 60588e395277..493cccb7ae12 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -61,3 +61,4 @@ subdir('hashing') subdir('serde') subdir('finjector') subdir('random') +subdir('compression') From f4097ca6f8c6afe8ca6e019d9df94944639002da Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 18:55:57 -0700 Subject: [PATCH 12/37] build: add model Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/model/meson.build | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 src/v/model/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 493cccb7ae12..ef2156087a52 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -62,3 +62,4 @@ subdir('serde') subdir('finjector') subdir('random') subdir('compression') +subdir('model') diff --git a/src/v/model/meson.build b/src/v/model/meson.build new file mode 100644 index 000000000000..a89e0a3154a7 --- /dev/null +++ b/src/v/model/meson.build @@ -0,0 +1,39 @@ +libmodel = library('model', + 'model.cc', + 'record_batch_reader.cc', + 'record_utils.cc', + 'async_adl_serde.cc', + 'adl_serde.cc', + 'validation.cc', + implicit_include_directories: false, + dependencies: [seastar, common, hashing, serde]) + +model = declare_dependency( + dependencies: [seastar, common, hashing, serde], + link_with: libmodel) + +libmodel_test_utils = library('model_test_utils', + 'tests/random_batch.cc', + implicit_include_directories: false, + dependencies: [seastar, common, compression, model]) + +model_test_utils = declare_dependency( + dependencies: [seastar, common, compression, model], + link_with: libmodel_test_utils) + +t = executable('test_model', + 'tests/lexical_cast_tests.cc', + 'tests/ntp_path_test.cc', + 'tests/topic_view_tests.cc', + implicit_include_directories: false, + dependencies: [model, boost, model_test_utils]) +test('model testing', t) + +# ARGS "-- -c 1" +t = executable('test_model_thread', + 'tests/record_batch_test.cc', + 'tests/record_batch_reader_test.cc', + 'tests/model_serialization_test.cc', + implicit_include_directories: false, + dependencies: [model, seastar_testing_main, model_test_utils]) +test('model thread tests', t) From 2dac67c9d4b383f88b302b55e76768f0b903f050 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 27 May 2022 07:25:05 -0700 Subject: [PATCH 13/37] build: add reflection Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/reflection/meson.build | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 src/v/reflection/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index ef2156087a52..b27cd10fa8af 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -63,3 +63,4 @@ subdir('finjector') subdir('random') subdir('compression') subdir('model') +subdir('reflection') diff --git a/src/v/reflection/meson.build b/src/v/reflection/meson.build new file mode 100644 index 000000000000..0b2eedd7c0f1 --- /dev/null +++ b/src/v/reflection/meson.build @@ -0,0 +1,9 @@ +t = executable('test_reflection_arity', + 'test/arity_test.cc', + dependencies: [common, boost, bytes]) +test('reflection arity', t) + +t = executable('test_async_adl', + 'test/async_adl_test.cc', + dependencies: [seastar_testing_main, common, bytes, model]) +test('reflection async adl', t) From 3cc0cedaa1e9486bfdd89f3eade04f5226743916 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 15:18:45 -0700 Subject: [PATCH 14/37] build: add json Signed-off-by: Noah Watkins --- src/v/json/meson.build | 13 +++++++++++++ src/v/meson.build | 1 + 2 files changed, 14 insertions(+) create mode 100644 src/v/json/meson.build diff --git a/src/v/json/meson.build b/src/v/json/meson.build new file mode 100644 index 000000000000..88d7a891d3ce --- /dev/null +++ b/src/v/json/meson.build @@ -0,0 +1,13 @@ +libjson = library('json', + 'json.cc', + dependencies: [common, seastar, rapidjson, model], + implicit_include_directories: false) + +json = declare_dependency( + dependencies: [common, seastar, rapidjson, model], + link_with: libjson) + +t = executable('test_json', + 'tests/json_serialization_test.cc', + dependencies: [json, seastar_testing_main]) +test('json serialization tests', t) diff --git a/src/v/meson.build b/src/v/meson.build index b27cd10fa8af..c8db85c96130 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -64,3 +64,4 @@ subdir('random') subdir('compression') subdir('model') subdir('reflection') +subdir('json') From bb4c6073deda9462c630a58f5035326470aca529 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 19:40:06 -0700 Subject: [PATCH 15/37] build: add utils Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/utils/meson.build | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 src/v/utils/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index c8db85c96130..63fa26133c62 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -65,3 +65,4 @@ subdir('compression') subdir('model') subdir('reflection') subdir('json') +subdir('utils') diff --git a/src/v/utils/meson.build b/src/v/utils/meson.build new file mode 100644 index 000000000000..78259b3e4113 --- /dev/null +++ b/src/v/utils/meson.build @@ -0,0 +1,45 @@ +libutils = library('utils', + 'hdr_hist.cc', + 'human.cc', + 'file_io.cc', + 'base64.cc', + 'retry_chain_node.cc', + 'vint.cc', + dependencies: [hdr_histogram, base64, common, bytes, model]) + +utils = declare_dependency( + dependencies: [hdr_histogram, base64, common, bytes, model], + link_with: libutils) + +# ARGS "-- -c 1" +t = executable('test_utils', + 'tests/vint_test.cc', + 'tests/directory_walker_test.cc', + 'tests/outcome_utils_test.cc', + 'tests/base64_test.cc', + 'tests/timed_mutex_test.cc', + 'tests/expiring_promise_test.cc', + 'tests/retry_chain_node_test.cc', + 'tests/input_stream_fanout_test.cc', + 'tests/waiter_queue_test.cc', + 'tests/delta_for_test.cc', + dependencies: [utils, seastar_testing_main]) +test('utils', t) + +# ARGS "-- -c 2" +t = executable('test_utils_threaded', + 'tests/remote_test.cc', + 'tests/retry_test.cc', + dependencies: [utils, seastar_testing_main]) +test('utils threaded', t) + +t = executable('test_utils_unit', + 'tests/string_switch_test.cc', + 'tests/constexpr_string_switch.cc', + 'tests/named_type_tests.cc', + 'tests/tristate_test.cc', + 'tests/moving_average_test.cc', + 'tests/human_test.cc', + 'tests/fragmented_vector_test.cc', + dependencies: [utils, boost]) +test('utils unit', t) From 99f5d0e6a4ab1b4eea72b387b30188011c89144f Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 19:13:33 -0700 Subject: [PATCH 16/37] build: add security security/tests is added separately to avoid circular or forward dependencies. Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/security/meson.build | 12 ++++++++++++ src/v/security/tests/meson.build | 7 +++++++ 3 files changed, 20 insertions(+) create mode 100644 src/v/security/meson.build create mode 100644 src/v/security/tests/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 63fa26133c62..031b5e2ea7a6 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -66,3 +66,4 @@ subdir('model') subdir('reflection') subdir('json') subdir('utils') +subdir('security') diff --git a/src/v/security/meson.build b/src/v/security/meson.build new file mode 100644 index 000000000000..3fb5f1e32b22 --- /dev/null +++ b/src/v/security/meson.build @@ -0,0 +1,12 @@ +libsecurity = library('security', + 'scram_algorithm.cc', + 'scram_credential.cc', + 'scram_authenticator.cc', + 'acl_store.cc', + 'mtls.cc', + implicit_include_directories: false, + dependencies: [seastar, ctre, common, bytes, utils, model]) + +security = declare_dependency( + dependencies: [seastar, ctre, common, bytes, utils, model], + link_with: libsecurity) diff --git a/src/v/security/tests/meson.build b/src/v/security/tests/meson.build new file mode 100644 index 000000000000..edb06b0cb4be --- /dev/null +++ b/src/v/security/tests/meson.build @@ -0,0 +1,7 @@ +t = executable('test_security', + 'scram_algorithm_test.cc', + 'credential_store_test.cc', + 'authorizer_test.cc', + 'mtls_test.cc', + dependencies: [security, boost, config]) +test('security', t) From 2b100d2a855a93fa8e7eb16323e48c2415baaaf4 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 16:15:17 -0700 Subject: [PATCH 17/37] build: add config Signed-off-by: Noah Watkins --- src/v/config/meson.build | 23 +++++++++++++++++++++++ src/v/meson.build | 1 + 2 files changed, 24 insertions(+) create mode 100644 src/v/config/meson.build diff --git a/src/v/config/meson.build b/src/v/config/meson.build new file mode 100644 index 000000000000..29b31a5c7528 --- /dev/null +++ b/src/v/config/meson.build @@ -0,0 +1,23 @@ +libconfig = library('config', + 'configuration.cc', + 'node_config.cc', + 'base_property.cc', + 'rjson_serialization.cc', + 'validators.cc', + dependencies: [common, seastar, json, absl, model, security, yaml, boost]) + +config = declare_dependency( + dependencies: [common, seastar, json, absl, model, security, yaml, boost], + link_with: libconfig) + +t = executable('test_configuration', + 'tests/bounded_property_test.cc', + 'tests/enum_property_test.cc', + 'tests/retention_property_test.cc', + 'tests/config_store_test.cc', + 'tests/socket_address_convert_test.cc', + 'tests/tls_config_convert_test.cc', + 'tests/advertised_kafka_api_test.cc', + 'tests/seed_server_property_test.cc', + dependencies: [config, seastar_testing_main]) +test('config testing', t) diff --git a/src/v/meson.build b/src/v/meson.build index 031b5e2ea7a6..37339bded99e 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -67,3 +67,4 @@ subdir('reflection') subdir('json') subdir('utils') subdir('security') +subdir('config') From 145621058a95e8e97f2b8243d1bbca35ff908db4 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 19:59:49 -0700 Subject: [PATCH 18/37] build: generate version tag from git Signed-off-by: Noah Watkins --- src/v/meson.build | 5 +++++ src/v/version.h.in | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/v/meson.build b/src/v/meson.build index 37339bded99e..2b714f6c97f3 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -54,6 +54,11 @@ add_project_arguments( '-Wno-unused-variable', language: 'cpp') +version_h = vcs_tag( + input: 'version.h.in', + output: 'version.h', +) + subdir('test_utils') subdir('ssx') subdir('bytes') diff --git a/src/v/version.h.in b/src/v/version.h.in index ec1c8223e540..5c026f485a97 100644 --- a/src/v/version.h.in +++ b/src/v/version.h.in @@ -10,5 +10,5 @@ static constexpr inline std::string_view redpanda_git_revision() { } static constexpr inline std::string_view redpanda_version() { - return std::string_view("@GIT_VER@ - @GIT_SHA1@@GIT_CLEAN_DIRTY@"); + return std::string_view("@VCS_TAG@"); } From 670451b1bc88958f831aef202c522d9fecc65b3c Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 13 May 2022 19:54:18 -0700 Subject: [PATCH 19/37] build: add syschecks Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/syschecks/meson.build | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 src/v/syschecks/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 2b714f6c97f3..c311768d92a5 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -73,3 +73,4 @@ subdir('json') subdir('utils') subdir('security') subdir('config') +subdir('syschecks') diff --git a/src/v/syschecks/meson.build b/src/v/syschecks/meson.build new file mode 100644 index 000000000000..e3934ef40aef --- /dev/null +++ b/src/v/syschecks/meson.build @@ -0,0 +1,10 @@ +libsyschecks = library('syschecks', + 'syschecks.cc', + 'pidfile.cc', + version_h, + implicit_include_directories: false, + dependencies: [seastar, common]) + +syschecks = declare_dependency( + dependencies: [seastar, common], + link_with: libsyschecks) From e69b3e2e0e540a1601026b5fef6950cb855691f4 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 07:05:21 -0700 Subject: [PATCH 20/37] build: add networking Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/net/meson.build | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 src/v/net/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index c311768d92a5..cdca7d615d3f 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -74,3 +74,4 @@ subdir('utils') subdir('security') subdir('config') subdir('syschecks') +subdir('net') diff --git a/src/v/net/meson.build b/src/v/net/meson.build new file mode 100644 index 000000000000..e5a9e18bcf07 --- /dev/null +++ b/src/v/net/meson.build @@ -0,0 +1,27 @@ +libnet = library('net', + 'batched_output_stream.cc', + 'dns.cc', + 'transport.cc', + 'connection.cc', + 'conn_quota.cc', + 'server.cc', + 'probes.cc', + 'tls.cc', + implicit_include_directories: false, + dependencies: [seastar, common, config, hashing, utils]) + +net = declare_dependency( + dependencies: [seastar, common, config, hashing, utils], + link_with: libnet) + +# ARGS "-- -c 1" +t = executable('test_net_conn_rate', + 'tests/connection_rate_test.cc', + dependencies: [net, seastar_testing_main]) +test('test conn rate', t) + +# ARGS "-- -c 8" +t = executable('test_conn_quota', + 'tests/conn_quota_test.cc', + dependencies: [net, seastar_testing_main]) +test('test conn quota', t) From e89c66bcb80b2d0f177958b733c2a18dc56c14e7 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 11:47:42 -0700 Subject: [PATCH 21/37] build: add http Signed-off-by: Noah Watkins --- src/v/http/meson.build | 23 +++++++++++++++++++++++ src/v/meson.build | 1 + 2 files changed, 24 insertions(+) create mode 100644 src/v/http/meson.build diff --git a/src/v/http/meson.build b/src/v/http/meson.build new file mode 100644 index 000000000000..f5b716a3197a --- /dev/null +++ b/src/v/http/meson.build @@ -0,0 +1,23 @@ +# -DBOOST_ASIO_HAS_STD_INVOKE_RESULT +libhttp = library('http', + 'iobuf_body.cc', + 'chunk_encoding.cc', + 'client.cc', + 'logger.cc', + implicit_include_directories: false, + dependencies: [boost, common, bytes, net]) + +http = declare_dependency( + dependencies: [boost, common, bytes, net], + link_with: libhttp) + +# ARGS "-- -c 1" +t = executable('test_http_client', + 'tests/http_client_test.cc', + 'tests/framing_test.cc', + dependencies: [http, seastar_testing_main]) +test('http client', t) + +executable('demo_http_client', + 'demo/client.cc', + dependencies: [http]) diff --git a/src/v/meson.build b/src/v/meson.build index cdca7d615d3f..4d7647c9efaa 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -75,3 +75,4 @@ subdir('security') subdir('config') subdir('syschecks') subdir('net') +subdir('http') From b24378673770dfd4bbc05af2422e3b03a831fab4 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 18:59:54 -0700 Subject: [PATCH 22/37] build: add s3 Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/s3/meson.build | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 src/v/s3/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 4d7647c9efaa..100955557c43 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -76,3 +76,4 @@ subdir('config') subdir('syschecks') subdir('net') subdir('http') +subdir('s3') diff --git a/src/v/s3/meson.build b/src/v/s3/meson.build new file mode 100644 index 000000000000..0c51de228e79 --- /dev/null +++ b/src/v/s3/meson.build @@ -0,0 +1,23 @@ +# -DBOOST_ASIO_HAS_STD_INVOKE_RESULT +libs3 = library('s3', + 'client.cc', + 'signature.cc', + 'error.cc', + 'client_probe.cc', + implicit_include_directories: false, + dependencies: [seastar, common, bytes, http]) + +s3 = declare_dependency( + dependencies: [seastar, common, bytes, http], + link_with: libs3) + +# ARGS "-- -c 1" +t = executable('test_s3', + 'tests/signature_test.cc', + 'tests/s3_client_test.cc', + dependencies: [s3, seastar_testing_main]) +test('http client', t) + +executable('demo_s3_test_client', + 'test_client/s3_test_client_main.cc', + dependencies: [s3]) From 543c36bf68014b5c088a9f247d09aeab18c0ecb5 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 19:04:19 -0700 Subject: [PATCH 23/37] build: add storage Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/storage/meson.build | 123 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/v/storage/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 100955557c43..1be7454e4df6 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -77,3 +77,4 @@ subdir('syschecks') subdir('net') subdir('http') subdir('s3') +subdir('storage') diff --git a/src/v/storage/meson.build b/src/v/storage/meson.build new file mode 100644 index 000000000000..78b01b9427b0 --- /dev/null +++ b/src/v/storage/meson.build @@ -0,0 +1,123 @@ +libstorage = library('storage', + 'segment_reader.cc', + 'log_manager.cc', + 'mem_log_impl.cc', + 'disk_log_impl.cc', + 'disk_log_appender.cc', + 'parser.cc', + 'log_reader.cc', + 'log_replayer.cc', + 'offset_translator_state.cc', + 'probe.cc', + 'record_batch_builder.cc', + 'logger.cc', + 'segment_appender.cc', + 'segment_set.cc', + 'segment.cc', + 'segment_index.cc', + 'segment_appender_utils.cc', + 'batch_cache.cc', + 'index_state.cc', + 'lock_manager.cc', + 'types.cc', + 'spill_key_index.cc', + 'compacted_index_chunk_reader.cc', + 'snapshot.cc', + 'kvstore.cc', + 'segment_utils.cc', + 'compaction_reducers.cc', + 'parser_utils.cc', + 'readers_cache.cc', + 'backlog_controller.cc', + 'compaction_controller.cc', + implicit_include_directories: false, + dependencies: [seastar, common, model, config, compression, roaring, + syschecks, utils]) + +storage = declare_dependency( + dependencies: [seastar, common, model, config, compression, roaring, + syschecks, utils], + link_with: libstorage) + +libstorage_test_utils = library('storage_test_utils', + 'tests/utils/disk_log_builder.cc', + dependencies: [storage, model_test_utils]) + +storage_test_utils = declare_dependency( + dependencies: [storage], + link_with: libstorage_test_utils) + +# ARGS "-- -c 1" +t = executable('test_storage', + 'tests/log_segment_appender_test.cc', + 'tests/segment_size_jitter_test.cc', + 'tests/log_segment_reader_test.cc', + 'tests/log_manager_test.cc', + 'tests/offset_assignment_test.cc', + 'tests/storage_e2e_test.cc', + 'tests/log_truncate_test.cc', + 'tests/offset_index_utils_tests.cc', + 'tests/compaction_index_format_tests.cc', + 'tests/appender_chunk_manipulations.cc', + 'tests/disk_log_builder_test.cc', + 'tests/log_retention_tests.cc', + 'tests/produce_consume_test.cc', + 'tests/half_page_concurrent_dispatch.cc', + 'tests/timequery_test.cc', + 'tests/kvstore_test.cc', + 'tests/backlog_controller_test.cc', + dependencies: [storage, seastar_testing_main, model_test_utils, + storage_test_utils]) +test('storage basic', t) + +# Putting this first, last, or in the middle of the other single thread tests +# results in: +# runtime error: member access within null pointer of type +# 'shared_ptr_no_esft' +# ARGS "-- -c 1" +t = executable('test_log_replayer_single_thread', + 'tests/log_replayer_test.cc', + dependencies: [storage, seastar_testing_main, model_test_utils, + storage_test_utils]) +test('storage log replayer', t) + +t = executable('test_segment_index_state', + 'tests/index_state_test.cc', + dependencies: [storage, boost]) +test('segment index state', t) + +t = executable('test_storage_threaded', + 'tests/batch_cache_test.cc', + 'tests/record_batch_builder_test.cc', + 'tests/snapshot_test.cc', + dependencies: [storage, seastar_testing_main, model_test_utils, + storage_test_utils]) +test('storage threaded', t) + +#rp_test( +# UNIT_TEST +# BINARY_NAME batch_cache_reclaim_test +# SOURCES batch_cache_reclaim_test.cc +# LIBRARIES v::seastar_testing_main v::storage_test_utils +# ARGS "-- -c1 -m100" +# LABELS storage +# SKIP_BUILD_TYPES "Debug" +#) +t = executable('test_batch_reclaim', + 'tests/batch_cache_reclaim_test.cc', + dependencies: [storage, boost, seastar_testing_main]) +# this test is only run on release builds because non-release builds use the +# default allocator which doesn't support reclaim +# test('batch cache reclaim', t) + +t = executable('bench_compaction_index', + 'tests/compaction_idx_bench.cc', + dependencies: [storage, seastar_perf_testing]) +benchmark('compaction index', t) + +# ARGS "-- -c 1" +t = executable('test_storage_opfuzz', + 'opfuzz/opfuzz.cc', + 'opfuzz/opfuzz_test.cc', + dependencies: [storage, seastar_testing_main, model_test_utils]) +test('storage opfuzz', t) From 06bd5da9530735ad94fedff6ce51e1c3c6abd058 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 19:11:44 -0700 Subject: [PATCH 24/37] build: add rpc Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/rpc/demo/meson.build | 22 ++++++++++++++++++ src/v/rpc/meson.build | 18 +++++++++++++++ src/v/rpc/test/meson.build | 47 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) create mode 100644 src/v/rpc/demo/meson.build create mode 100644 src/v/rpc/meson.build create mode 100644 src/v/rpc/test/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 1be7454e4df6..014fed0907bb 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -78,3 +78,4 @@ subdir('net') subdir('http') subdir('s3') subdir('storage') +subdir('rpc') diff --git a/src/v/rpc/demo/meson.build b/src/v/rpc/demo/meson.build new file mode 100644 index 000000000000..5689def46935 --- /dev/null +++ b/src/v/rpc/demo/meson.build @@ -0,0 +1,22 @@ +rpc_demo_service_h = custom_target( + output: 'simple_service.h', + input: 'gen.json', + command: [rpc_compiler, '--service_file', '@INPUT@', + '--output_file', '@OUTPUT@']) + +executable('demo_rpc_server', + 'server.cc', + rpc_demo_service_h, + dependencies: [rpc]) + +executable('demo_rpc_client', + 'client.cc', + rpc_demo_service_h, + dependencies: [rpc]) + +# ARGS "-- -c 1" +t = executable('test_rpc_roundtrip_types', + 'type_tests.cc', + rpc_demo_service_h, + dependencies: [seastar_testing_main, rpc]) +test('rpc roundtrip types', t) diff --git a/src/v/rpc/meson.build b/src/v/rpc/meson.build new file mode 100644 index 000000000000..d7549f2f8897 --- /dev/null +++ b/src/v/rpc/meson.build @@ -0,0 +1,18 @@ +librpc = library('rpc', + 'types.cc', + 'netbuf.cc', + 'transport.cc', + 'reconnect_transport.cc', + 'connection_cache.cc', + 'simple_protocol.cc', + implicit_include_directories: false, + dependencies: [common, net, utils, compression]) + +rpc = declare_dependency( + dependencies: [common, net, utils, compression], + link_with: librpc) + +rpc_compiler = find_program('rpc_compiler.py') + +subdir('test') +subdir('demo') diff --git a/src/v/rpc/test/meson.build b/src/v/rpc/test/meson.build new file mode 100644 index 000000000000..17d167a809fa --- /dev/null +++ b/src/v/rpc/test/meson.build @@ -0,0 +1,47 @@ +cycling_service_h = custom_target( + output: 'cycling_service.h', + input: 'cycling_service.json', + command: [rpc_compiler, '--service_file', '@INPUT@', + '--output_file', '@OUTPUT@']) + +echo_service_h = custom_target( + output: 'echo_service.h', + input: 'echo_service.json', + command: [rpc_compiler, '--service_file', '@INPUT@', + '--output_file', '@OUTPUT@']) + +# ARGS "-- -c 1" +t = executable('test_rpc', + 'netbuf_tests.cc', + 'roundtrip_tests.cc', + 'response_handler_tests.cc', + 'serialization_test.cc', + dependencies: [rpc, seastar_testing_main]) +test('rpc', t) + +# +# TODO looks like this needs to have the tls inputs setup +# +# INPUT_FILES ${CMAKE_CURRENT_SOURCE_DIR}/redpanda.crt +# ${CMAKE_CURRENT_SOURCE_DIR}/redpanda.key +# ${CMAKE_CURRENT_SOURCE_DIR}/root_certificate_authority.chain_cert +# ${CMAKE_CURRENT_SOURCE_DIR}/redpanda.other.crt +# ${CMAKE_CURRENT_SOURCE_DIR}/redpanda.other.key +# ${CMAKE_CURRENT_SOURCE_DIR}/root_certificate_authority.other.chain_cert +# ARGS "-- -c 1" +t = executable('test_rpc_generator', + 'rpc_gen_cycling_test.cc', + cycling_service_h, + echo_service_h, + dependencies: [rpc, seastar_testing_main]) +test('rpc generator', t) + +t = executable('test_exponential_backoff', + 'exponential_backoff.cc', + dependencies: [boost, rpc]) +test('exponential backoff', t) + +t = executable('bench_rpc_serialization', + 'rpc_bench.cc', + dependencies: [rpc, seastar_perf_testing]) +benchmark('rpc serialization', t) From b00f7be7da87e9fe29e9eff3d8b0867a83138917 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 19:21:07 -0700 Subject: [PATCH 25/37] build: add raft Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/raft/meson.build | 84 +++++++++++++++++++++++++++++++++++++ src/v/raft/tron/meson.build | 17 ++++++++ 3 files changed, 102 insertions(+) create mode 100644 src/v/raft/meson.build create mode 100644 src/v/raft/tron/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 014fed0907bb..1e136de1df86 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -79,3 +79,4 @@ subdir('http') subdir('s3') subdir('storage') subdir('rpc') +subdir('raft') diff --git a/src/v/raft/meson.build b/src/v/raft/meson.build new file mode 100644 index 000000000000..d85882fbfb7a --- /dev/null +++ b/src/v/raft/meson.build @@ -0,0 +1,84 @@ +raft_service_h = custom_target( + output: 'raftgen_service.h', + input: 'raftgen.json', + command: [rpc_compiler, '--service_file', '@INPUT@', + '--output_file', '@OUTPUT@']) + +libraft = library('raft', + 'consensus.cc', + 'consensus_utils.cc', + 'heartbeat_manager.cc', + 'configuration_bootstrap_state.cc', + 'logger.cc', + 'types.cc', + 'replicate_entries_stm.cc', + 'vote_stm.cc', + 'prevote_stm.cc', + 'recovery_stm.cc', + 'follower_stats.cc', + 'replicate_batcher.cc', + 'rpc_client_protocol.cc', + 'group_manager.cc', + 'probe.cc', + 'offset_monitor.cc', + 'event_manager.cc', + 'state_machine.cc', + 'log_eviction_stm.cc', + 'configuration_manager.cc', + 'group_configuration.cc', + 'append_entries_buffer.cc', + 'follower_queue.cc', + 'offset_translator.cc', + 'recovery_memory_quota.cc', + raft_service_h, + implicit_include_directories: false, + dependencies: [common, model, utils, storage, rpc, finjector]) + +raft = declare_dependency( + sources: raft_service_h, + dependencies: [common, model, utils, storage, rpc, finjector], + link_with: libraft) + +t = executable('test_consensus_utils', + 'tests/consensus_utils_test.cc', + dependencies: [boost, raft, model_test_utils]) +test('consensus utils', t) + +t = executable('test_group_configuration', + 'tests/group_configuration_tests.cc', + dependencies: [boost, raft]) +test('group configuration', t) + +t = executable('test_raft', + 'tests/jitter_tests.cc', + 'tests/bootstrap_configuration_test.cc', + 'tests/foreign_entry_test.cc', + 'tests/configuration_serialization_test.cc', + 'tests/type_serialization_tests.cc', + 'tests/term_assigning_reader_test.cc', + 'tests/membership_test.cc', + 'tests/leadership_test.cc', + 'tests/append_entries_test.cc', + 'tests/offset_monitor_test.cc', + 'tests/mux_state_machine_test.cc', + 'tests/mutex_buffer_test.cc', + 'tests/manual_log_deletion_test.cc', + 'tests/state_removal_test.cc', + 'tests/configuration_manager_test.cc', + dependencies: [raft, seastar_testing_main, model_test_utils, + storage_test_utils]) +test('raft', t) + +t = executable('test_offset_translator', + 'tests/offset_translator_tests.cc', + dependencies: [raft, seastar_testing_main]) +test('offset translator', t) + +subdir('tron') + +executable('demo_kvelldb', + 'kvelldb/kvserver.cc', + 'kvelldb/logger.cc', + 'kvelldb/kvrsm.cc', + 'kvelldb/httpkvrsm.cc', + dependencies: [raft]) diff --git a/src/v/raft/tron/meson.build b/src/v/raft/tron/meson.build new file mode 100644 index 000000000000..6dfe41d4141f --- /dev/null +++ b/src/v/raft/tron/meson.build @@ -0,0 +1,17 @@ +tron_service_h = custom_target( + output: 'trongen_service.h', + input: 'gen.json', + command: [rpc_compiler, '--service_file', '@INPUT@', + '--output_file', '@OUTPUT@']) + +executable('demo_tron_server', + 'server.cc', + 'logger.cc', + tron_service_h, + dependencies: [raft]) + +executable('demo_tron_client', + 'client.cc', + 'logger.cc', + tron_service_h, + dependencies: [raft]) From 6bd771a70da2ffa63319616a0ecb644a134c858f Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sun, 15 May 2022 08:06:24 -0700 Subject: [PATCH 26/37] build: add v8 engine Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/v8_engine/meson.build | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 src/v/v8_engine/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 1e136de1df86..8b0884e65223 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -80,3 +80,4 @@ subdir('s3') subdir('storage') subdir('rpc') subdir('raft') +subdir('v8_engine') diff --git a/src/v/v8_engine/meson.build b/src/v/v8_engine/meson.build new file mode 100644 index 000000000000..b7a42ced9c0b --- /dev/null +++ b/src/v/v8_engine/meson.build @@ -0,0 +1,9 @@ +libv8_engine = library('v8_engine', + 'data_policy.cc', + 'data_policy_table.cc', + implicit_include_directories: false, + dependencies: [common, model]) + +v8_engine = declare_dependency( + dependencies: [common, model], + link_with: libv8_engine) From b1ece79743b2d0e00c3e3258e46c5e791d8f8d0a Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 21:49:39 -0700 Subject: [PATCH 27/37] build: add cloud storage cloud_storage/tests is added separately to avoid circular or forward dependencies. Signed-off-by: Noah Watkins --- src/v/cloud_storage/meson.build | 21 +++++++++++++++++++++ src/v/cloud_storage/tests/meson.build | 15 +++++++++++++++ src/v/meson.build | 1 + 3 files changed, 37 insertions(+) create mode 100644 src/v/cloud_storage/meson.build create mode 100644 src/v/cloud_storage/tests/meson.build diff --git a/src/v/cloud_storage/meson.build b/src/v/cloud_storage/meson.build new file mode 100644 index 000000000000..d3df45cd3ff5 --- /dev/null +++ b/src/v/cloud_storage/meson.build @@ -0,0 +1,21 @@ +libcloud_storage = library('cloud_storage', + 'cache_service.cc', + 'cache_probe.cc', + 'topic_manifest.cc', + 'partition_manifest.cc', + 'recursive_directory_walker.cc', + 'remote.cc', + 'offset_translation_layer.cc', + 'probe.cc', + 'partition_probe.cc', + 'partition_recovery_manager.cc', + 'types.cc', + 'remote_segment.cc', + 'remote_partition.cc', + 'remote_segment_index.cc', + implicit_include_directories: false, + dependencies: [seastar, common, config, model, s3, storage]) + +cloud_storage = declare_dependency( + dependencies: [seastar, common, config, model, s3, storage], + link_with: libcloud_storage) diff --git a/src/v/cloud_storage/tests/meson.build b/src/v/cloud_storage/tests/meson.build new file mode 100644 index 000000000000..e17abde87c7b --- /dev/null +++ b/src/v/cloud_storage/tests/meson.build @@ -0,0 +1,15 @@ +# ARGS "-- -c 1" +t = executable('test_cloud_storage', + 'directory_walker_test.cc', + 'partition_manifest_test.cc', + 'topic_manifest_test.cc', + 's3_imposter.cc', + 'remote_test.cc', + 'cache_test.cc', + 'offset_translation_layer_test.cc', + 'remote_segment_test.cc', + 'remote_partition_test.cc', + 'remote_segment_index_test.cc', + dependencies: [seastar_testing_main, cloud_storage, storage_test_utils, + model_test_utils, cluster]) +test('cloud storage', t) diff --git a/src/v/meson.build b/src/v/meson.build index 8b0884e65223..3de35f41cc8d 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -81,3 +81,4 @@ subdir('storage') subdir('rpc') subdir('raft') subdir('v8_engine') +subdir('cloud_storage') From 43762c74f6b641f541c0a9feeabadf8fef617fbf Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 19:50:32 -0700 Subject: [PATCH 28/37] build: add cluster cluster/tests is added separately to avoid circular or forward dependencies. Signed-off-by: Noah Watkins --- src/v/cluster/meson.build | 88 +++++++++++++++++++++++++++++++++ src/v/cluster/tests/meson.build | 62 +++++++++++++++++++++++ src/v/meson.build | 1 + 3 files changed, 151 insertions(+) create mode 100644 src/v/cluster/meson.build create mode 100644 src/v/cluster/tests/meson.build diff --git a/src/v/cluster/meson.build b/src/v/cluster/meson.build new file mode 100644 index 000000000000..ea6ea2644ee3 --- /dev/null +++ b/src/v/cluster/meson.build @@ -0,0 +1,88 @@ +cluster_services = [ + 'controller', + 'id_allocator', + 'metadata_dissemination_rpc', + 'tx_gateway', +] + +cluster_services_h = [] +foreach service : cluster_services + t = custom_target( + input: f'@service@.json', + output: f'@service@_service.h', + command: [rpc_compiler, '--service_file', '@INPUT@', + '--output_file', '@OUTPUT@']) + cluster_services_h += [t] +endforeach + +libcluster = library('cluster', + 'archival_metadata_stm.cc', + 'metadata_cache.cc', + 'partition_manager.cc', + 'scheduling/partition_allocator.cc', + 'logger.cc', + 'config_frontend.cc', + 'config_manager.cc', + 'cluster_utils.cc', + 'id_allocator.cc', + 'id_allocator_frontend.cc', + 'rm_partition_frontend.cc', + 'tx_gateway_frontend.cc', + 'tx_gateway.cc', + 'service.cc', + 'metadata_dissemination_handler.cc', + 'metadata_dissemination_service.cc', + 'metadata_dissemination_utils.cc', + 'types.cc', + 'notification_latch.cc', + 'topic_table.cc', + 'topic_updates_dispatcher.cc', + 'members_table.cc', + 'members_manager.cc', + 'partition_leaders_table.cc', + 'topics_frontend.cc', + 'controller_backend.cc', + 'controller.cc', + 'partition.cc', + 'partition_probe.cc', + 'id_allocator_stm.cc', + 'persisted_stm.cc', + 'tm_stm.cc', + 'rm_stm.cc', + 'tx_helpers.cc', + 'security_manager.cc', + 'security_frontend.cc', + 'data_policy_manager.cc', + 'data_policy_frontend.cc', + 'controller_api.cc', + 'members_frontend.cc', + 'members_backend.cc', + 'health_manager.cc', + 'non_replicable_topics_frontend.cc', + 'scheduling/allocation_node.cc', + 'scheduling/types.cc', + 'scheduling/allocation_state.cc', + 'scheduling/allocation_strategy.cc', + 'scheduling/constraints.cc', + 'scheduling/leader_balancer.cc', + 'scheduling/leader_balancer_probe.cc', + 'health_monitor_types.cc', + 'health_monitor_backend.cc', + 'health_monitor_frontend.cc', + 'metrics_reporter.cc', + 'node/types.cc', + 'node/local_monitor.cc', + 'feature_backend.cc', + 'feature_manager.cc', + 'feature_barrier.cc', + 'feature_table.cc', + 'drain_manager.cc', + cluster_services_h, + version_h, + implicit_include_directories: false, + dependencies: [common, config, model, raft, http, v8_engine, cloud_storage]) + +cluster = declare_dependency( + sources: [cluster_services_h, version_h], + dependencies: [common, config, model, raft, http, v8_engine, cloud_storage], + link_with: libcluster) diff --git a/src/v/cluster/tests/meson.build b/src/v/cluster/tests/meson.build new file mode 100644 index 000000000000..f13cd0d1898b --- /dev/null +++ b/src/v/cluster/tests/meson.build @@ -0,0 +1,62 @@ +t = executable('bench_partition_allocator', + 'allocation_bench.cc', + dependencies: [cluster, seastar_perf_testing]) +benchmark('partition allocator', t) + +t = executable('test_metadata_dissemination_utils', + 'metadata_dissemination_utils_test.cc', + dependencies: [cluster, boost]) +test('metadata dissemination', t) + +t = executable('test_cluster', + 'partition_allocator_tests.cc', + 'simple_batch_builder_test.cc', + 'serialization_rt_test.cc', + 'cluster_utils_tests.cc', + 'cluster_tests.cc', + 'metadata_dissemination_test.cc', + 'notification_latch_test.cc', + 'autocreate_test.cc', + 'controller_state_test.cc', + 'commands_serialization_test.cc', + 'topic_table_test.cc', + 'topic_updates_dispatcher_test.cc', + 'controller_backend_test.cc', + 'configuration_change_test.cc', + 'idempotency_tests.cc', + 'feature_barrier_test.cc', + 'feature_table_test.cc', + 'tm_stm_tests.cc', + 'rm_stm_tests.cc', + 'controller_api_tests.cc', + 'decommissioning_tests.cc', + 'replicas_rebalancing_tests.cc', + 'create_partitions_test.cc', + 'id_allocator_stm_test.cc', + 'data_policy_controller_test.cc', + 'health_monitor_test.cc', + 'topic_configuration_compat_test.cc', + 'local_monitor_test.cc', + dependencies: [cluster, seastar_testing_main, model_test_utils, redpanda_app]) +test('cluster', t) + +t = executable('test_partition_movement', + 'partition_moving_test.cc', + dependencies: [common, cluster, seastar_testing_main, model_test_utils, + redpanda_app]) +test('partition movement', t) + +t = executable('test_leader_balancer', + 'leader_balancer_test.cc', + dependencies: [boost, cluster]) +test('leader balancer', t) + +t = executable('bench_leader_balancer', + 'leader_balancer_bench.cc', + dependencies: [seastar_perf_testing, cluster]) +benchmark('leader balancer', t) + +t = executable('test_metrics_reporter', + 'metrics_reporter_test.cc', + dependencies: [boost, cluster]) +test('metrics reporter', t) diff --git a/src/v/meson.build b/src/v/meson.build index 3de35f41cc8d..701ae8006d6c 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -82,3 +82,4 @@ subdir('rpc') subdir('raft') subdir('v8_engine') subdir('cloud_storage') +subdir('cluster') From 7fc147d02982c04c0adef82f07195cc4b95e357a Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 14 May 2022 22:44:15 -0700 Subject: [PATCH 29/37] build: add archival archival/tests is added separately to avoid circular or forward dependencies. Signed-off-by: Noah Watkins --- src/v/archival/meson.build | 13 +++++++++++++ src/v/archival/tests/meson.build | 8 ++++++++ src/v/meson.build | 1 + 3 files changed, 22 insertions(+) create mode 100644 src/v/archival/meson.build create mode 100644 src/v/archival/tests/meson.build diff --git a/src/v/archival/meson.build b/src/v/archival/meson.build new file mode 100644 index 000000000000..dbe6d5839eda --- /dev/null +++ b/src/v/archival/meson.build @@ -0,0 +1,13 @@ +libarchival = library('archival', + 'archival_policy.cc', + 'service.cc', + 'ntp_archiver_service.cc', + 'probe.cc', + 'types.cc', + 'upload_controller.cc', + implicit_include_directories: false, + dependencies: [common, s3, storage, cluster]) + +archival = declare_dependency( + dependencies: [common, s3, storage, cluster], + link_with: libarchival) diff --git a/src/v/archival/tests/meson.build b/src/v/archival/tests/meson.build new file mode 100644 index 000000000000..214d8bf0e9bf --- /dev/null +++ b/src/v/archival/tests/meson.build @@ -0,0 +1,8 @@ +# ARGS "-- -c 1" +t = executable('test_archival', + 'service_fixture.cc', + 'ntp_archiver_test.cc', + 'service_test.cc', + dependencies: [seastar_testing_main, archival, storage_test_utils, + redpanda_app, model_test_utils]) +test('archival', t) diff --git a/src/v/meson.build b/src/v/meson.build index 701ae8006d6c..177b3e093839 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -83,3 +83,4 @@ subdir('raft') subdir('v8_engine') subdir('cloud_storage') subdir('cluster') +subdir('archival') From dc007cc6888c0c7ae10be13b697f0cc00e232691 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sun, 15 May 2022 08:17:04 -0700 Subject: [PATCH 30/37] build: add kafka kafka/client/test and kafka/server/tests are added separately to avoid circular or forward dependencies. Signed-off-by: Noah Watkins --- src/v/kafka/client/meson.build | 20 +++++++ src/v/kafka/client/test/meson.build | 23 ++++++++ src/v/kafka/meson.build | 67 +++++++++++++++++++++++ src/v/kafka/protocol/meson.build | 13 +++++ src/v/kafka/protocol/schemata/meson.build | 50 +++++++++++++++++ src/v/kafka/protocol/tests/meson.build | 7 +++ src/v/kafka/server/tests/meson.build | 51 +++++++++++++++++ src/v/meson.build | 1 + 8 files changed, 232 insertions(+) create mode 100644 src/v/kafka/client/meson.build create mode 100644 src/v/kafka/client/test/meson.build create mode 100644 src/v/kafka/meson.build create mode 100644 src/v/kafka/protocol/meson.build create mode 100644 src/v/kafka/protocol/schemata/meson.build create mode 100644 src/v/kafka/protocol/tests/meson.build create mode 100644 src/v/kafka/server/tests/meson.build diff --git a/src/v/kafka/client/meson.build b/src/v/kafka/client/meson.build new file mode 100644 index 000000000000..82210fb18139 --- /dev/null +++ b/src/v/kafka/client/meson.build @@ -0,0 +1,20 @@ +libkafka_client = library('kafka_client', + 'assignment_plans.cc', + 'broker.cc', + 'brokers.cc', + 'client.cc', + 'client_fetch_batch_reader.cc', + 'configuration.cc', + 'consumer.cc', + 'fetcher.cc', + 'fetch_session.cc', + 'partitioners.cc', + 'producer.cc', + 'topic_cache.cc', + 'sasl_client.cc', + implicit_include_directories: false, + dependencies: [common, kafka_protocol, config, cluster]) + +kafka_client = declare_dependency( + dependencies: [common, kafka_protocol, config, cluster], + link_with: libkafka_client) diff --git a/src/v/kafka/client/test/meson.build b/src/v/kafka/client/test/meson.build new file mode 100644 index 000000000000..9ecbc64ad4ff --- /dev/null +++ b/src/v/kafka/client/test/meson.build @@ -0,0 +1,23 @@ +t = executable('test_kafka_client_unit', + 'partitioners.cc', + dependencies: [kafka_client, boost]) +test('kafka client unit', t) + +# ARGS "-- -c 1" +t = executable('test_kafka_client_threaded', + 'fetch_session.cc', + 'produce_batcher.cc', + 'produce_partition.cc', + 'retry_with_mitigation.cc', + dependencies: [kafka_client, seastar_testing_main, kafka_server]) +test('kafka client threaded', t) + +t = executable('test_kafka_client_fixture', + 'consumer_group.cc', + 'fetch.cc', + 'produce.cc', + 'reconnect.cc', + 'retry.cc', + dependencies: [kafka_client, seastar_testing_main, redpanda_app, + storage_test_utils, model_test_utils]) +test('kafka client fixture', t) diff --git a/src/v/kafka/meson.build b/src/v/kafka/meson.build new file mode 100644 index 000000000000..64af4b956142 --- /dev/null +++ b/src/v/kafka/meson.build @@ -0,0 +1,67 @@ +subdir('protocol') +subdir('client') + +libkafka_server = library('kafka_server', + 'server/handlers/api_versions.cc', + 'server/handlers/metadata.cc', + 'server/handlers/add_partitions_to_txn.cc', + 'server/handlers/txn_offset_commit.cc', + 'server/handlers/add_offsets_to_txn.cc', + 'server/handlers/end_txn.cc', + 'server/handlers/list_groups.cc', + 'server/handlers/find_coordinator.cc', + 'server/handlers/describe_configs.cc', + 'server/handlers/offset_fetch.cc', + 'server/handlers/produce.cc', + 'server/handlers/list_offsets.cc', + 'server/handlers/fetch.cc', + 'server/handlers/join_group.cc', + 'server/handlers/heartbeat.cc', + 'server/handlers/leave_group.cc', + 'server/handlers/sync_group.cc', + 'server/handlers/init_producer_id.cc', + 'server/handlers/create_topics.cc', + 'server/handlers/offset_commit.cc', + 'server/handlers/delete_topics.cc', + 'server/handlers/alter_configs.cc', + 'server/handlers/describe_groups.cc', + 'server/handlers/sasl_handshake.cc', + 'server/handlers/sasl_authenticate.cc', + 'server/handlers/incremental_alter_configs.cc', + 'server/handlers/delete_groups.cc', + 'server/handlers/describe_acls.cc', + 'server/handlers/describe_log_dirs.cc', + 'server/handlers/create_acls.cc', + 'server/handlers/delete_acls.cc', + 'server/handlers/create_partitions.cc', + 'server/handlers/offset_for_leader_epoch.cc', + 'server/handlers/topics/types.cc', + 'server/handlers/topics/topic_utils.cc', + 'protocol/batch_reader.cc', + 'protocol/kafka_batch_adapter.cc', + 'server/requests.cc', + 'server/member.cc', + 'server/group_stm.cc', + 'server/group.cc', + 'server/group_router.cc', + 'server/group_manager.cc', + 'server/rm_group_frontend.cc', + 'server/connection_context.cc', + 'server/protocol.cc', + 'server/protocol_utils.cc', + 'server/logger.cc', + 'server/quota_manager.cc', + 'server/fetch_session_cache.cc', + 'server/replicated_partition.cc', + 'server/partition_proxy.cc', + 'server/group_recovery_consumer.cc', + 'server/group_metadata.cc', + 'server/group_metadata_migration.cc', + implicit_include_directories: false, + dependencies: [common, kafka_protocol, config, cluster, security]) + +kafka_server = declare_dependency( + dependencies: [common, kafka_protocol, config, cluster, security], + link_with: libkafka_server) + +subdir('protocol/tests') diff --git a/src/v/kafka/protocol/meson.build b/src/v/kafka/protocol/meson.build new file mode 100644 index 000000000000..18dd0dfe082b --- /dev/null +++ b/src/v/kafka/protocol/meson.build @@ -0,0 +1,13 @@ +subdir('schemata') + +libkafka_protocol = library('kafka_protocol', + 'errors.cc', + kafka_protocol_h, + kafka_protocol_cc, + dependencies: [common, bytes, model], + implicit_include_directories: false) + +kafka_protocol = declare_dependency( + sources: kafka_protocol_h, + dependencies: [common, bytes, model], + link_with: libkafka_protocol) diff --git a/src/v/kafka/protocol/schemata/meson.build b/src/v/kafka/protocol/schemata/meson.build new file mode 100644 index 000000000000..dcf9ed2d4fff --- /dev/null +++ b/src/v/kafka/protocol/schemata/meson.build @@ -0,0 +1,50 @@ +kafka_protocol_schemas = [ + 'api_versions', + 'offset_fetch', + 'offset_commit', + 'join_group', + 'sync_group', + 'heartbeat', + 'leave_group', + 'delete_topics', + 'describe_configs', + 'find_coordinator', + 'list_offset', + 'alter_configs', + 'list_groups', + 'describe_groups', + 'create_topics', + 'sasl_handshake', + 'sasl_authenticate', + 'init_producer_id', + 'incremental_alter_configs', + 'delete_groups', + 'describe_acls', + 'describe_log_dirs', + 'create_acls', + 'delete_acls', + 'produce', + 'metadata', + 'add_partitions_to_txn', + 'txn_offset_commit', + 'fetch', + 'end_txn', + 'create_partitions', + 'add_offsets_to_txn', + 'offset_for_leader_epoch'] + +kafka_protocol_codegen = find_program('generator.py') + +kafka_protocol_h = [] +kafka_protocol_cc = [] +foreach schema : kafka_protocol_schemas + foreach flavor : ['request', 'response'] + name = f'@schema@_@flavor@' + t = custom_target( + input: f'@name@.json', + output: [f'@name@.h', f'@name@.cc'], + command: [kafka_protocol_codegen, '@INPUT@', '@OUTPUT0@', '@OUTPUT1@']) + kafka_protocol_h += [t[0]] + kafka_protocol_cc += [t[1]] + endforeach +endforeach diff --git a/src/v/kafka/protocol/tests/meson.build b/src/v/kafka/protocol/tests/meson.build new file mode 100644 index 000000000000..d1ec96474fa0 --- /dev/null +++ b/src/v/kafka/protocol/tests/meson.build @@ -0,0 +1,7 @@ +# batch_reader_test was run with ARGS "-- -c 1" +t = executable('test_kafka_protocol', + 'batch_reader_test.cc', + 'security_test.cc', + dependencies: [kafka_server, storage_test_utils, seastar_testing_main, + model_test_utils]) +test('kafka protocol', t) diff --git a/src/v/kafka/server/tests/meson.build b/src/v/kafka/server/tests/meson.build new file mode 100644 index 000000000000..87b817f7f1b4 --- /dev/null +++ b/src/v/kafka/server/tests/meson.build @@ -0,0 +1,51 @@ +t = executable('test_kafka_server', + 'error_mapping_test.cc', + 'timeouts_conversion_test.cc', + 'types_conversion_tests.cc', + 'topic_utils_test.cc', + dependencies: [kafka_server, boost]) +test('kafka server', t) + +# ARGS "-- -c 1" +t = executable('test_kafka_server_fixture', + 'consumer_groups_test.cc', + 'member_test.cc', + 'group_test.cc', + 'read_write_roundtrip_test.cc', + 'fetch_test.cc', + 'delete_topics_test.cc', + 'offset_fetch_test.cc', + 'api_versions_test.cc', + 'create_topics_test.cc', + 'find_coordinator_test.cc', + 'list_offsets_test.cc', + 'topic_recreate_test.cc', + 'fetch_session_test.cc', + 'alter_config_test.cc', + 'produce_consume_test.cc', + 'group_metadata_serialization_test.cc', + dependencies: [kafka_server, seastar_testing_main, redpanda_app, + storage_test_utils, model_test_utils]) +test('kafka server', t) + +# +# 1. propose delete for limited value, librdkakfa dependency +# +# 'metadata_test.cc', librdkafka dep + +# +# 2. propose delete for very limited value nowdays +# +#find_program(KAFKA_PYTHON_ENV "kafka-python-env") +#rp_test( +# UNIT_TEST +# BINARY_NAME test_kafka_request_parser +# SOURCES request_parser_test.cc +# DEFINITIONS BOOST_TEST_DYN_LINK +# LIBRARIES v::seastar_testing_main v::application v::raft v::kafka +# # generate request data as a prepare step. the test and prepare command run in +# # the same scratch directory so the generated output file writes to pwd. +# PREPARE_COMMAND "${KAFKA_PYTHON_ENV} ${PROJECT_SOURCE_DIR}/tools/kafka-python-api-serde.py 1000 > requests.bin" +# ARGS "-- -c 1" +# LABELS kafka +#) diff --git a/src/v/meson.build b/src/v/meson.build index 177b3e093839..a3ad57c29fb1 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -84,3 +84,4 @@ subdir('v8_engine') subdir('cloud_storage') subdir('cluster') subdir('archival') +subdir('kafka') From 70d190caf2b544f1fc3dc1ef7839476e50965a1c Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sun, 15 May 2022 16:00:48 -0700 Subject: [PATCH 31/37] build: add coproc coproc/tests is added separately to avoid circular or forward dependencies. Signed-off-by: Noah Watkins --- src/v/coproc/meson.build | 31 +++++++++++++++++++++++++++++++ src/v/coproc/tests/meson.build | 27 +++++++++++++++++++++++++++ src/v/meson.build | 1 + 3 files changed, 59 insertions(+) create mode 100644 src/v/coproc/meson.build create mode 100644 src/v/coproc/tests/meson.build diff --git a/src/v/coproc/meson.build b/src/v/coproc/meson.build new file mode 100644 index 000000000000..0dad1a8d21f5 --- /dev/null +++ b/src/v/coproc/meson.build @@ -0,0 +1,31 @@ +coproc_supervisor_service_h = custom_target( + output: 'supervisor.h', + input: 'gen.json', + command: [rpc_compiler, '--service_file', '@INPUT@', + '--output_file', '@OUTPUT@']) + +libcoproc = library('coproc', + 'api.cc', + 'types.cc', + 'logger.cc', + 'script_context.cc', + 'script_context_frontend.cc', + 'script_context_backend.cc', + 'script_context_router.cc', + 'pacemaker.cc', + 'offset_storage_utils.cc', + 'wasm_event.cc', + 'event_listener.cc', + 'script_dispatcher.cc', + 'event_handler.cc', + 'partition.cc', + 'partition_manager.cc', + 'reconciliation_backend.cc', + coproc_supervisor_service_h, + implicit_include_directories: false, + dependencies: [common, cluster, kafka_client]) + +coproc = declare_dependency( + sources: coproc_supervisor_service_h, + dependencies: [common, cluster, kafka_client], + link_with: libcoproc) diff --git a/src/v/coproc/tests/meson.build b/src/v/coproc/tests/meson.build new file mode 100644 index 000000000000..ba43134de288 --- /dev/null +++ b/src/v/coproc/tests/meson.build @@ -0,0 +1,27 @@ +f = library('coproc_test_fixture', + 'utils/event_publisher_utils.cc', + 'utils/supervisor.cc', + 'utils/wasm_event_generator.cc', + 'utils/batch_utils.cc', + 'fixtures/coproc_cluster_fixture.cc', + 'fixtures/coproc_test_fixture.cc', + 'fixtures/coproc_bench_fixture.cc', + 'fixtures/fiber_mock_fixture.cc', + dependencies: [common, coproc, model_test_utils]) + +t = executable('test_coproc_fixture', + 'retry_logic_tests.cc', + 'partition_movement_tests.cc', + 'topic_ingestion_policy_tests.cc', + 'failure_recovery_tests.cc', + 'pacemaker_tests.cc', + 'autocreate_topic_tests.cc', + 'coproc_bench_tests.cc', + 'offset_storage_utils_tests.cc', + 'wasm_event_tests.cc', + 'event_handler_tests.cc', + 'event_listener_tests.cc', + 'kafka_api_materialized_tests.cc', + dependencies: [seastar_testing_main, coproc, redpanda_app], + link_with: f) +test('coproc fixture', t) diff --git a/src/v/meson.build b/src/v/meson.build index a3ad57c29fb1..5368e1b98489 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -85,3 +85,4 @@ subdir('cloud_storage') subdir('cluster') subdir('archival') subdir('kafka') +subdir('coproc') From 8f96bcb9d07b5a9f1dcb19bd8dec58d7c7c1d7cc Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sun, 15 May 2022 16:14:20 -0700 Subject: [PATCH 32/37] build: add pandaproxy pandaproxy/rest/test and pandaproxy/schema_registry/test are added separately to avoid circular or forward dependencies. Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/pandaproxy/api/api-doc/meson.build | 8 ++++++ src/v/pandaproxy/json/meson.build | 19 +++++++++++++ src/v/pandaproxy/meson.build | 22 +++++++++++++++ src/v/pandaproxy/parsing/meson.build | 14 ++++++++++ src/v/pandaproxy/rest/meson.build | 12 ++++++++ src/v/pandaproxy/rest/test/meson.build | 7 +++++ src/v/pandaproxy/schema_registry/meson.build | 27 ++++++++++++++++++ .../schema_registry/test/meson.build | 28 +++++++++++++++++++ 9 files changed, 138 insertions(+) create mode 100644 src/v/pandaproxy/api/api-doc/meson.build create mode 100644 src/v/pandaproxy/json/meson.build create mode 100644 src/v/pandaproxy/meson.build create mode 100644 src/v/pandaproxy/parsing/meson.build create mode 100644 src/v/pandaproxy/rest/meson.build create mode 100644 src/v/pandaproxy/rest/test/meson.build create mode 100644 src/v/pandaproxy/schema_registry/meson.build create mode 100644 src/v/pandaproxy/schema_registry/test/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index 5368e1b98489..e18f4c2871de 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -86,3 +86,4 @@ subdir('cluster') subdir('archival') subdir('kafka') subdir('coproc') +subdir('pandaproxy') diff --git a/src/v/pandaproxy/api/api-doc/meson.build b/src/v/pandaproxy/api/api-doc/meson.build new file mode 100644 index 000000000000..e30ca45bf86f --- /dev/null +++ b/src/v/pandaproxy/api/api-doc/meson.build @@ -0,0 +1,8 @@ +pandaproxy_api_h = [] +foreach swag : ['rest', 'schema_registry'] + t = custom_target( + output: f'@swag@.json.h', + input: f'@swag@.json', + command: [seastar_json2code, '-f', '@INPUT@', '-o', '@OUTPUT@']) + pandaproxy_api_h += [t] +endforeach diff --git a/src/v/pandaproxy/json/meson.build b/src/v/pandaproxy/json/meson.build new file mode 100644 index 000000000000..6e90e77a3442 --- /dev/null +++ b/src/v/pandaproxy/json/meson.build @@ -0,0 +1,19 @@ +libpandaproxy_json = library('pandaproxy_json', + 'error.cc', + implicit_include_directories: false, + dependencies: [common, json, model, kafka_protocol, pandaproxy_common]) + +pandaproxy_json = declare_dependency( + dependencies: [common, json, model, kafka_protocol, pandaproxy_common], + link_with: libpandaproxy_json) + +t = executable('test_pp_json_requests', + 'requests/test/fetch.cc', + 'requests/test/produce.cc', + dependencies: [seastar_testing_main, pandaproxy_json, utils, kafka_server, kafka_client]) +test('panda proxy json requests', t) + +t = executable('test_pp_json_types', + 'test/iobuf.cc', + dependencies: [seastar_testing_main, pandaproxy_json, utils]) +test('panda proxy json types', t) diff --git a/src/v/pandaproxy/meson.build b/src/v/pandaproxy/meson.build new file mode 100644 index 000000000000..1bde3c79f2eb --- /dev/null +++ b/src/v/pandaproxy/meson.build @@ -0,0 +1,22 @@ +libpandaproxy_common = library('pandaproxy_common', + 'error.cc', + 'logger.cc', + 'probe.cc', + 'server.cc', + implicit_include_directories: false, + dependencies: [common, config, kafka_client, utils]) + +pandaproxy_common = declare_dependency( + dependencies: [common, config, kafka_client, utils], + link_with: libpandaproxy_common) + +subdir('api/api-doc') +subdir('parsing') +subdir('json') +subdir('schema_registry') +subdir('rest') + +t = executable('test_pandaproxy_unit', + 'test/errors.cc', + dependencies: [pandaproxy_common, pandaproxy_parsing, pandaproxy_json, kafka_protocol]) +test('pandaproxy unit', t) diff --git a/src/v/pandaproxy/parsing/meson.build b/src/v/pandaproxy/parsing/meson.build new file mode 100644 index 000000000000..c5ca5e8032dd --- /dev/null +++ b/src/v/pandaproxy/parsing/meson.build @@ -0,0 +1,14 @@ +libpandaproxy_parsing = library('pandaproxy_parsing', + 'error.cc', + implicit_include_directories: false, + dependencies: [common, utils]) + +pandaproxy_parsing = declare_dependency( + dependencies: [common, utils], + link_with: libpandaproxy_parsing) + +t = executable('test_pp_parsing', + 'test/from_chars.cc', + 'test/httpd.cc', + dependencies: [boost, pandaproxy_parsing]) +test('panda proxy parsing', t) diff --git a/src/v/pandaproxy/rest/meson.build b/src/v/pandaproxy/rest/meson.build new file mode 100644 index 000000000000..0a6bdf89b582 --- /dev/null +++ b/src/v/pandaproxy/rest/meson.build @@ -0,0 +1,12 @@ +libpandaproxy_rest = library('pandaproxy_rest', + 'configuration.cc', + 'handlers.cc', + 'proxy.cc', + pandaproxy_api_h, + implicit_include_directories: false, + dependencies: [common, config, kafka_client]) + +pandaproxy_rest = declare_dependency( + sources: pandaproxy_api_h, + dependencies: [common, config, kafka_client], + link_with: libpandaproxy_rest) diff --git a/src/v/pandaproxy/rest/test/meson.build b/src/v/pandaproxy/rest/test/meson.build new file mode 100644 index 000000000000..ceda2599b356 --- /dev/null +++ b/src/v/pandaproxy/rest/test/meson.build @@ -0,0 +1,7 @@ +t = executable('test_pp_rest', + 'fetch.cc', + 'list_topics.cc', + 'produce.cc', + 'consumer_group.cc', + dependencies: [seastar_testing_main, redpanda_app, http]) +test('pp rest', t) diff --git a/src/v/pandaproxy/schema_registry/meson.build b/src/v/pandaproxy/schema_registry/meson.build new file mode 100644 index 000000000000..bf7629b91c29 --- /dev/null +++ b/src/v/pandaproxy/schema_registry/meson.build @@ -0,0 +1,27 @@ +libpandaproxy_schema_registry = library('pandaproxy_schema_registry', + 'api.cc', + 'configuration.cc', + 'handlers.cc', + 'error.cc', + 'service.cc', + 'seq_writer.cc', + 'sharded_store.cc', + 'types.cc', + 'avro.cc', + 'protobuf.cc', + pandaproxy_api_h, + implicit_include_directories: false, + dependencies: [avro, common, kafka_client, kafka_server, protobuf, pandaproxy_common, + pandaproxy_json, pandaproxy_parsing]) + +pandaproxy_schema_registry = declare_dependency( + sources: pandaproxy_api_h, + dependencies: [avro, common, kafka_client, kafka_server, protobuf, pandaproxy_common, + pandaproxy_json, pandaproxy_parsing], + link_with: libpandaproxy_schema_registry) + +t = executable('test_schema_reg_requests', + 'requests/test/get_subject_versions_version.cc', + 'requests/test/post_subject_versions.cc', + dependencies: [seastar_testing_main, pandaproxy_schema_registry, utils]) +test('schema registry requests', t) diff --git a/src/v/pandaproxy/schema_registry/test/meson.build b/src/v/pandaproxy/schema_registry/test/meson.build new file mode 100644 index 000000000000..e20ee978dcc6 --- /dev/null +++ b/src/v/pandaproxy/schema_registry/test/meson.build @@ -0,0 +1,28 @@ +t = executable('test_scheam_reg_unit', + 'sanitize_avro.cc', + 'util.cc', + 'storage.cc', + 'store.cc', + dependencies: [boost, pandaproxy_schema_registry]) +test('schema registry unit', t) + +# ARGS "-- -c 1" +t = executable('test_scheam_reg_single_thread', + 'one_shot.cc', + 'sharded_store.cc', + 'consume_to_store.cc', + 'compatibility_store.cc', + 'compatibility_3rdparty.cc', + 'compatibility_avro.cc', + 'compatibility_protobuf.cc', + dependencies: [seastar_testing_main, pandaproxy_schema_registry]) +test('schema registry single thread', t) + +# TODO(Ben): Make schema_registry properly sharded +# ARGS "-- -c 1" +t = executable('test_scheam_reg_fixture', + 'get_schema_types.cc', + 'post_subjects_subject_version.cc', + 'delete_subject_endpoints.cc', + dependencies: [seastar_testing_main, pandaproxy_schema_registry, redpanda_app]) +test('schema registry fixture', t) From ee265fb138e062bbcfa2276dd34b6ce96de07ab0 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sun, 15 May 2022 17:32:01 -0700 Subject: [PATCH 33/37] build: add redpanda Signed-off-by: Noah Watkins --- src/v/meson.build | 1 + src/v/redpanda/admin/api-doc/meson.build | 22 +++++++++++++++++++++ src/v/redpanda/meson.build | 25 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 src/v/redpanda/admin/api-doc/meson.build create mode 100644 src/v/redpanda/meson.build diff --git a/src/v/meson.build b/src/v/meson.build index e18f4c2871de..a623c2c77b2e 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -87,3 +87,4 @@ subdir('archival') subdir('kafka') subdir('coproc') subdir('pandaproxy') +subdir('redpanda') diff --git a/src/v/redpanda/admin/api-doc/meson.build b/src/v/redpanda/admin/api-doc/meson.build new file mode 100644 index 000000000000..2aba97665ed2 --- /dev/null +++ b/src/v/redpanda/admin/api-doc/meson.build @@ -0,0 +1,22 @@ +admin_api_swags = [ + 'config', + 'cluster_config', + 'raft', + 'security', + 'status', + 'features', + 'broker', + 'partition', + 'hbadger', + 'transaction', + 'cluster', + 'debug'] + +admin_api_h = [] +foreach swag : admin_api_swags + t = custom_target( + output: f'@swag@.json.h', + input: f'@swag@.json', + command: [seastar_json2code, '-f', '@INPUT@', '-o', '@OUTPUT@']) + admin_api_h += [t] +endforeach diff --git a/src/v/redpanda/meson.build b/src/v/redpanda/meson.build new file mode 100644 index 000000000000..94bec9e09f9e --- /dev/null +++ b/src/v/redpanda/meson.build @@ -0,0 +1,25 @@ +subdir('admin/api-doc') + +libredpanda_app = library('redpanda_app', + 'admin_server.cc', + 'request_auth.cc', + 'cli_parser.cc', + 'application.cc', + admin_api_h, + implicit_include_directories: false, + dependencies: [common, cluster, config, kafka_server, coproc, + pandaproxy_schema_registry, pandaproxy_rest, archival]) + +redpanda_app = declare_dependency( + dependencies: [common, cluster, config, kafka_server, coproc, + pandaproxy_schema_registry, pandaproxy_rest, archival], + link_with: libredpanda_app) + +executable('redpanda', + 'main.cc', + dependencies: [common, redpanda_app]) + +t = executable('test_cli_parser', + 'tests/cli_parser_tests.cc', + dependencies: [boost, redpanda_app]) +test('cli parser', t) From 821d5d51cae5bd2fd7a585d3876af9a315d1d5f7 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Wed, 1 Jun 2022 17:57:38 -0700 Subject: [PATCH 34/37] build: add tests with forward dependencies Signed-off-by: Noah Watkins --- src/v/meson.build | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/v/meson.build b/src/v/meson.build index a623c2c77b2e..dde93b24d5d5 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -88,3 +88,16 @@ subdir('kafka') subdir('coproc') subdir('pandaproxy') subdir('redpanda') + +# some tests have circular or forward dependencies if added along with their +# normal containing directory. a common example are tests that depend on the +# application fixture that isn't added until the end with subdir('redpanda'). +subdir('cluster/tests') +subdir('cloud_storage/tests') +subdir('archival/tests') +subdir('kafka/client/test') +subdir('kafka/server/tests') +subdir('coproc/tests') +subdir('pandaproxy/rest/test') +subdir('pandaproxy/schema_registry/test') +subdir('security/tests') From c27bb03c15b707ec2b27aadb3afdd6f22c1889cd Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Wed, 1 Jun 2022 17:57:47 -0700 Subject: [PATCH 35/37] build: add outcome test Signed-off-by: Noah Watkins --- src/v/meson.build | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/v/meson.build b/src/v/meson.build index dde93b24d5d5..3393798055ce 100644 --- a/src/v/meson.build +++ b/src/v/meson.build @@ -101,3 +101,8 @@ subdir('coproc/tests') subdir('pandaproxy/rest/test') subdir('pandaproxy/schema_registry/test') subdir('security/tests') + +t = executable('test_outcome', + 'outcome_tests.cc', + dependencies: [boost]) +test('outcome', t) From 0e019acbd5adca3adfa81cb36fd7f275aa53eb5e Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Mon, 16 May 2022 19:43:29 -0700 Subject: [PATCH 36/37] build: add script for testing build in docker Signed-off-by: Noah Watkins --- .dockerignore | 1 + Dockerfile | 16 ++++++++++++++++ build-meson.sh | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 build-meson.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..72e8ffc0db8a --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000000..626bd9ca2570 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +from ubuntu:jammy + +run apt-get update + +run DEBIAN_FRONTEND=noninteractive apt-get install -qq \ + python3 python3-pip python3-setuptools python3-wheel ninja-build \ + libboost1.74-all-dev libprotobuf-dev libprotoc-dev libcrypto++-dev \ + pkg-config libfmt-dev liblz4-dev libgnutls28-dev libc-ares-dev \ + libyaml-cpp-dev ragel clang libabsl-dev libsnappy-dev libxxhash-dev \ + libzstd-dev git python3-jsonschema xfslibs-dev valgrind systemtap-sdt-dev \ + libsctp-dev ccache python3-jinja2 libroaring-dev cmake lld + +run pip3 install meson + +workdir /src/redpanda +cmd ["bash", "-c", "meson builddir && cd builddir && meson compile"] diff --git a/build-meson.sh b/build-meson.sh new file mode 100755 index 000000000000..2894c1e12c1b --- /dev/null +++ b/build-meson.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e +set -x + +CCACHE_DIR=$HOME/redpanda-meson-ccache +CC="ccache clang" +CXX="ccache clang++" +CC_LD="lld" +CXX_LD="lld" +NAME=redpanda-meson + +mkdir -p $CCACHE_DIR +docker build -t $NAME . + +function run { + local args=$* + docker run -v $PWD:/src/redpanda \ + -v $CCACHE_DIR:/mnt/ccache \ + -e CCACHE_DIR=/mnt/ccache -e CC="$CC" \ + -e CC_LD="$CC_LD" -e CXX_LD="$CXX_LD" \ + -e CXX="$CXX" -u $(id -u):$(id -g) -it $NAME $args +} + +function reset_build { + rm -rf builddir + run meson builddir +} + +function purge_subprojects { + run meson subprojects purge --confirm +} + +function build { + reset_build + run meson compile -C builddir +} + +function build_targets_isolated { + reset_build + targets=$(run meson introspect builddir --targets | jq -r "reverse | .[] | select(.subproject == null).name") + for target in ${targets}; do + reset_build + run meson compile -C builddir $target + done +} + +function run_tests { + run meson test -C builddir +} + +build From 122fb7b6db35237d61b4f2f4783a70f01cf0b95a Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sun, 5 Jun 2022 14:21:38 -0700 Subject: [PATCH 37/37] meson github action build Signed-off-by: Noah Watkins --- .github/workflows/redpanda-meson-build.yml | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/redpanda-meson-build.yml diff --git a/.github/workflows/redpanda-meson-build.yml b/.github/workflows/redpanda-meson-build.yml new file mode 100644 index 000000000000..b3dcec793b72 --- /dev/null +++ b/.github/workflows/redpanda-meson-build.yml @@ -0,0 +1,29 @@ +# Copyright 2022 Redpanda Data, Inc. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.md +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0 + +name: build-meson +on: push + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: install dependencies + run: | + sudo apt-get update + sudo apt-get install -qq \ + python3 python3-pip python3-setuptools python3-wheel ninja-build \ + libboost1.74-all-dev libprotobuf-dev libprotoc-dev libcrypto++-dev \ + pkg-config libfmt-dev liblz4-dev libgnutls28-dev libc-ares-dev \ + libyaml-cpp-dev ragel clang libabsl-dev libsnappy-dev libxxhash-dev \ + libzstd-dev git python3-jsonschema xfslibs-dev valgrind systemtap-sdt-dev \ + libsctp-dev ccache python3-jinja2 libroaring-dev cmake lld + sudo pip3 install meson + - name: checkout + uses: actions/checkout@v3