calyptia: switch to YAML for Fleet config

.github/workflows/call-build-linux-packages.yaml

@@ -54,7 +54,7 @@ jobs:
   call-build-capture-source:
     # Capture source tarball and generate checksum for it
     name: Extract any supporting metadata
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: ${{ inputs.environment }}
     permissions:
       contents: read
@@ -105,8 +105,8 @@ jobs:
   call-build-linux-packages:
     name: ${{ matrix.distro }} package build and stage to S3
     environment: ${{ inputs.environment }}
-    # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
-    runs-on: ${{ (contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
+    # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-22.04 usage.
+    runs-on: ${{ (contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-22.04' }}
     permissions:
       contents: read
     strategy:
@@ -214,7 +214,7 @@ jobs:
     name: Create repo metadata in S3
     # Only upload for staging
     if: inputs.environment == 'staging'
-    # Need to use 18.04 as 20.04 has no createrepo available
+    # Need to use 22.04 as 20.04 has no createrepo available
     runs-on: ubuntu-22.04
     environment: ${{ inputs.environment }}
     needs:

.github/workflows/call-build-macos.yaml

@@ -74,8 +74,6 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - name: "Normal macOS-latest runner (Intel)"
-            runner: macos-12
           - name: "Apple Silicon macOS runner"
             runner: macos-14
 
@@ -121,8 +119,6 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - name: "Normal macOS-latest package (Intel)"
-            os: macos-12
           - name: "Apple Silicon macOS package"
             os: macos-14
 

.github/workflows/call-run-integration-test.yaml

@@ -192,7 +192,7 @@ jobs:
           bats-version: 1.9.0
 
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.10.0
+        uses: helm/kind-action@v1.11.0
         with:
           node_image: kindest/node:${{ matrix.k8s-release }}
           cluster_name: kind

.github/workflows/call-test-images.yaml

@@ -184,7 +184,7 @@ jobs:
           ref: ${{ inputs.ref }}
 
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.10.0
+        uses: helm/kind-action@v1.11.0
 
       - name: Set up Helm
         uses: azure/setup-helm@v4

.github/workflows/staging-release.yaml

@@ -172,7 +172,7 @@ jobs:
 
   staging-release-apt-packages:
     name: S3 - update APT packages bucket
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: release
     needs:
       - staging-release-version-check
@@ -327,7 +327,7 @@ jobs:
 
   staging-release-source-s3:
     name: S3 - update source bucket
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: release
     needs:
       - staging-release-version-check

.github/workflows/unit-tests.yaml

@@ -78,6 +78,7 @@ jobs:
           CC: ${{ matrix.compiler }}
           CXX: ${{ matrix.compiler }}
           FLB_OPT: ${{ matrix.flb_option }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-macos-unit-tests:
     # We chain this after Linux one as there are costs and restrictions associated
@@ -111,6 +112,7 @@ jobs:
           CC: gcc
           CXX: g++
           FLB_OPT: ${{ matrix.flb_option }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-aarch64-unit-tests:
     # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
@@ -167,6 +169,7 @@ jobs:
         env:
           CC: ${{ matrix.config.compiler }}
           CXX: ${{ matrix.config.compiler }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-qemu-ubuntu-unit-tests:
     # We chain this after Linux one as there are CPU time costs for QEMU emulation
@@ -179,6 +182,7 @@ jobs:
       matrix:
         arch:
           - s390x
+          - riscv64
     steps:
       - name: Checkout Fluent Bit code
         uses: actions/checkout@v4
@@ -188,19 +192,17 @@ jobs:
         id: build-and-test-on-qemu
         with:
           arch: ${{ matrix.arch }}
-          distro: ubuntu20.04
+          distro: ubuntu22.04
           shell: /bin/bash
           dockerRunArgs: |
             --volume "/var/lib/dbus/machine-id:/var/lib/dbus/machine-id"
             --volume "/etc/machine-id:/etc/machine-id"
           install: |
             apt-get update
-            apt-get install -y gcc-7 g++-7 clang-6.0 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common#libsystemd-dev
-            ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
+            apt-get install -y gcc-12 g++-12 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common
 
-            update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
-            update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
-            update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90
+            update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90
+            update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90
           run: |
             cd build
             export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
@@ -218,7 +220,6 @@ jobs:
             make -j $nparallel
             ctest -j $nparallel --build-run-dir . --output-on-failure
 
-
   # Required check looks at this so do not remove
   run-all-unit-tests:
     if: always()

.gitignore

@@ -17,6 +17,7 @@ init/fluent-bit.service
 lib/chunkio/include/chunkio/cio_version.h
 lib/monkey/monkey.service
 lib/monkey/include/monkey/mk_core/mk_core_info.h
+lib/cprofiles/include/cprofiles/cprof_info.h
 
 packaging/.env
 packaging/packages

CMakeLists.txt

@@ -321,6 +321,11 @@ if (FLB_SYSTEM_LINUX)
   include(cmake/s390x.cmake)
 endif ()
 
+# Build for Linux - riscv64 arch
+if (FLB_SYSTEM_LINUX)
+  include(cmake/riscv64.cmake)
+endif ()
+
 # Enable signed char support on Linux AARCH64 if specified
 if (FLB_LINUX_ON_AARCH64)
   set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsigned-char")

README.md

@@ -13,16 +13,14 @@
 
 ![](documentation/fluentbit_ecosystem.png)
 
-[Fluent Bit](http://fluentbit.io) is a fast Log Processor and Forwarder for Linux, Windows, Embedded Linux, MacOS and BSD family operating systems. It's part of the Graduated [Fluentd](http://fluentd.org) Ecosystem and a [CNCF](https://cncf.io) sub-project.
+[Fluent Bit](http://fluentbit.io) is a fast Log, Metrics and Traces Processor and Forwarder for Linux, Windows, Embedded Linux, MacOS and BSD family operating systems. It's part of the Graduated [Fluentd](http://fluentd.org) Ecosystem and a [CNCF](https://cncf.io) sub-project.
 
-Fluent Bit allows to collect log events or metrics from different sources, process them and deliver them to different backends such as [Fluentd](http://fluentd.org), Elasticsearch, Splunk, DataDog, Kafka, New Relic, Azure services, AWS services, Google services, NATS, InfluxDB or any custom HTTP end-point.
+Fluent Bit allows to collect different signal types such as logs, metrics and traces from different sources, process them and deliver them to different backends such as [Fluentd](http://fluentd.org), Elasticsearch, Splunk, DataDog, Kafka, New Relic, Azure services, AWS services, Google services, NATS, InfluxDB or any custom HTTP end-point.
 
 Fluent Bit comes with full SQL [Stream Processing](https://docs.fluentbit.io/manual/stream-processing/introduction) capabilities: data manipulation and analytics using SQL queries.
 
 Fluent Bit runs on x86_64, x86, arm32v7, and arm64v8 architectures.
 
-
-
 ## Features
 
 - High Performance at low CPU and Memory footprint
@@ -48,7 +46,7 @@ Fluent Bit runs on x86_64, x86, arm32v7, and arm64v8 architectures.
 
 ## Fluent Bit in Production
 
-[Fluent Bit](https://fluentbit.io) is used widely in production environments. As of 2022, [Fluent Bit surpasses 3 Billion downloads](https://www.cncf.io/blog/2022/10/13/fluent-bit-surpasses-three-billion-downloads/) and continues to be deployed over **10 million times a day**. The following is a preview of who uses Fluent Bit heavily in production:
+Fluent Bit is a widely adopted solution in production environments. As of 2024, Fluent Bit has surpassed 15 billion downloads and continues to be deployed over 10 million times daily. Below is a preview of some of the organizations that rely heavily on Fluent Bit in their production systems:
 
 > If your company uses Fluent Bit and is not listed, feel free to open a GitHub issue and we will add the logo.
 

cmake/headers.cmake

@@ -37,6 +37,7 @@ include_directories(
 
   ${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_JANSSON}/include
   ${CMAKE_CURRENT_BINARY_DIR}/lib/cmetrics
+  ${CMAKE_CURRENT_BINARY_DIR}/lib/cprofiles/include
   ${CMAKE_CURRENT_BINARY_DIR}/include
 
   ${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/

cmake/libraries.cmake

@@ -20,7 +20,7 @@ set(FLB_PATH_LIB_ONIGMO      "lib/onigmo")
 set(FLB_PATH_LIB_MPACK       "lib/mpack-amalgamation-1.1.1")
 set(FLB_PATH_LIB_MINIZ       "lib/miniz")
 set(FLB_PATH_LIB_TUTF8E      "lib/tutf8e")
-set(FLB_PATH_LIB_CARES       "lib/c-ares-1.33.1")
+set(FLB_PATH_LIB_CARES       "lib/c-ares-1.34.4")
 set(FLB_PATH_LIB_SNAPPY      "lib/snappy-fef67ac")
 set(FLB_PATH_LIB_RDKAFKA     "lib/librdkafka-2.4.0")
 set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb")

cmake/riscv64.cmake

@@ -0,0 +1,8 @@
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv64)")
+  message(STATUS "Forcing characters to be signed, as on x86_64.")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsigned-char")
+  if(FLB_LUAJIT)
+    message(WARNING "LuaJIT is disabled, this platform does not support built-in LuaJIT and system provided one neither.")
+    set(FLB_LUAJIT OFF)
+  endif()
+endif ()

include/fluent-bit/calyptia/calyptia_constants.h

@@ -36,8 +36,10 @@
 #define CALYPTIA_ENDPOINT_TRACE   "/v1/traces/%s"
 
 #define CALYPTIA_ENDPOINT_FLEETS "/v1/fleets"
-#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini"
+#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini&config_format=ini"
+#define CALYPTIA_ENDPOINT_FLEET_CONFIG_YAML "/v1/fleets/%s/config?format=yaml&config_format=yaml"
 #define CALYPTIA_ENDPOINT_FLEET_FILES "/v1/fleets/%s/files"
+#define CALYPTIA_ENDPOINT_FLEET_BY_NAME "/v1/search?project_id=%s&resource=fleet&term=%s&exact=true"
 
 /* Storage */
 #define CALYPTIA_SESSION_FILE     "session.CALYPTIA"

include/fluent-bit/flb_utf8.h

@@ -20,84 +20,16 @@
 #ifndef FLB_UTF8_H
 #define FLB_UTF8_H
 
+#define FLB_UTF8_ACCEPT   0
+#define FLB_UTF8_REJECT   1
+#define FLB_UTF8_CONTINUE 2
+
 #include <fluent-bit/flb_info.h>
 #include <inttypes.h>
 
-/* is the start of a UTF-8 string ? */
-#define flb_utf8_check(c) (((c) & 0xC0) != 0x80)
-
-static const char trailingBytesForUTF8[256] = {
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
-};
-
 /* returns length of next utf-8 sequence */
-static inline int flb_utf8_len(const char *s)
-{
-    return trailingBytesForUTF8[(unsigned int)(unsigned char)s[0]] + 1;
-}
-
-/*
- * UTF-8 Decoding routines are originally written by Bjoern Hoehrmann
- * <[email protected]> and taken from the following web site:
- *
- *   http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
- *
- * They have been siglhy renamed to follow Fluent Bit naming requirements.
- */
-
-#define FLB_UTF8_ACCEPT 0
-#define FLB_UTF8_REJECT 1
-
-static const uint8_t utf8d[] = {
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
-    7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
-    8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
-    0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
-    0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
-    0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2
-    1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4
-    1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6
-    1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8
-};
-
-static inline uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep,
-                                       uint32_t byte)
-{
-    uint32_t type = utf8d[byte];
-
-    *codep = (*state != FLB_UTF8_ACCEPT) ?
-        (byte & 0x3fu) | (*codep << 6) :
-        (0xff >> type) & (byte);
-
-    *state = utf8d[256 + *state*16 + type];
-    return *state;
-}
-
-
-static inline void flb_utf8_print(const uint8_t *s) {
-    uint32_t codepoint;
-    uint32_t state = 0;
-
-    for (; *s; ++s)
-        if (!flb_utf8_decode(&state, &codepoint, *s)) {
-            printf("\\u%04x\n", codepoint);
-        }
-
-    if (state != FLB_UTF8_ACCEPT) {
-        printf("The string is not well-formed\n");
-    }
-}
+int flb_utf8_len(const char *s);
+uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep, uint8_t byte);
+void flb_utf8_print(char *input);
 
 #endif