Merge

CMakeLists.txt

@@ -18,22 +18,23 @@ endif()
 # Update CFLAGS
 if (MSVC)
   add_definitions(-D_CRT_SECURE_NO_WARNINGS)
+  add_definitions(-D_CRT_NONSTDC_NO_WARNINGS)
 
   # Use custom CFLAGS for MSVC
   #
-  #   /W3 ...... Show up to 'production quality' msgs.
   #   /Zi ...... Generate pdb files.
   #   /MT ...... Static link C runtimes.
+  #   /wd4711 .. C4711 (function selected for inline expansion)
+  #   /wd4100 .. C4100 (unreferenced formal parameter)
+  #   /wd5045 .. C5045 (Spectre mitigation)
   #
-  set(CMAKE_C_FLAGS "/DWIN32 /D_WINDOWS /DNDEBUG /W3 /O2 /Zi")
+  set(CMAKE_C_FLAGS "/DWIN32 /D_WINDOWS /DNDEBUG /O2 /Zi /wd4100 /wd4711 /wd5045")
   set(CMAKE_EXE_LINKER_FLAGS "/Debug /INCREMENTAL:NO")
   set(CMAKE_BUILD_TYPE None)
 
   # Use add_compile_options() to set /MT since Visual Studio
   # Generator does not notice /MT in CMAKE_C_FLAGS.
   add_compile_options(/MT)
-elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -fno-common")
 else()
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
 endif()
@@ -60,14 +61,15 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/library")
 
 # Build Options
 option(FLB_ALL                 "Enable all features"           No)
-option(FLB_DEBUG               "Build with debug symbols"      No)
+option(FLB_DEBUG               "Build with debug symbols"     Yes)
+option(FLB_RELEASE             "Build with debug symbols"      No)
 option(FLB_SMALL               "Optimise for small size"       No)
 option(FLB_COVERAGE            "Build with code-coverage"      No)
 option(FLB_JEMALLOC            "Build with Jemalloc support"   No)
 option(FLB_REGEX               "Build with Regex support"     Yes)
 option(FLB_UTF8_ENCODER        "Build with UTF8 encoding support" Yes)
 option(FLB_PARSER              "Build with Parser support"    Yes)
-option(FLB_TLS                 "Build with SSL/TLS support"    No)
+option(FLB_TLS                 "Build with SSL/TLS support"   Yes)
 option(FLB_BINARY              "Build executable binary"      Yes)
 option(FLB_EXAMPLES            "Build examples"               Yes)
 option(FLB_SHARED_LIB          "Build shared library"         Yes)
@@ -81,7 +83,7 @@ option(FLB_MTRACE              "Enable mtrace support"         No)
 option(FLB_POSIX_TLS           "Force POSIX thread storage"    No)
 option(FLB_INOTIFY             "Enable inotify support"       Yes)
 option(FLB_SQLDB               "Enable SQL embedded DB"       Yes)
-option(FLB_HTTP_SERVER         "Enable HTTP Server"            No)
+option(FLB_HTTP_SERVER         "Enable HTTP Server"           Yes)
 option(FLB_BACKTRACE           "Enable stacktrace support"    Yes)
 option(FLB_LUAJIT              "Enable Lua Scripting support" Yes)
 option(FLB_RECORD_ACCESSOR     "Enable record accessor"       Yes)
@@ -100,20 +102,6 @@ option(FLB_METRICS            "Enable metrics support"       No)
 # Proxy Plugins
 option(FLB_PROXY_GO           "Enable Go plugins support"   Yes)
 
-# Advanced options for Flushing methods
-# =====================================
-# Flush methods defines under what kind of context/environment the plugins
-# flush callback is invoked. Since this is done through co-routines
-# there are two options available:
-#
-# FLB_FLUSH_LIBCO: set by default, use lib/flb_libco implementation which
-# supports amd64, arm, x86 and ppc.
-#
-# FLB_FLUSH_UCONTEXT (DEPRECATED): It uses POSIX ucontext for co-routines
-# implementation.
-# Despites this is POSIX deprecated, it's supported on Linux and BSD
-# systems, but specific old-toolchains do not implement it.
-
 # Build-in Plugins
 option(FLB_IN_CPU             "Enable CPU input plugin"             Yes)
 option(FLB_IN_THERMAL         "Enable Thermal plugin"               Yes)
@@ -123,7 +111,7 @@ option(FLB_IN_DOCKER_EVENTS   "Enable Docker events input plugin"   Yes)
 option(FLB_IN_EXEC            "Enable Exec input plugin"            Yes)
 option(FLB_IN_FORWARD         "Enable Forward input plugin"         Yes)
 option(FLB_IN_HEALTH          "Enable Health input plugin"          Yes)
-option(FLB_IN_HTTP            "Enable HTTP input plugin"             No)
+option(FLB_IN_HTTP            "Enable HTTP input plugin"            Yes)
 option(FLB_IN_MEM             "Enable Memory input plugin"          Yes)
 option(FLB_IN_KMSG            "Enable Kernel log input plugin"      Yes)
 option(FLB_IN_LIB             "Enable library mode input plugin"    Yes)
@@ -178,6 +166,7 @@ option(FLB_OUT_KAFKA_REST     "Enable Kafka Rest output plugin"     Yes)
 option(FLB_OUT_CLOUDWATCH_LOGS "Enable AWS CloudWatch output plugin" Yes)
 option(FLB_OUT_KINESIS_FIREHOSE "Enable AWS Firehose output plugin" Yes)
 option(FLB_OUT_S3             "Enable AWS S3 output plugin"         Yes)
+option(FLB_OUT_WEBSOCKET      "Enable Websocket output plugin"      Yes)
 option(FLB_FILTER_ALTER_SIZE  "Enable alter_size filter"            Yes)
 option(FLB_FILTER_AWS         "Enable aws filter"                   Yes)
 option(FLB_FILTER_EXPECT      "Enable expect filter"                Yes)
@@ -193,6 +182,7 @@ option(FLB_FILTER_NEST        "Enable nest filter"                  Yes)
 option(FLB_FILTER_LUA         "Enable Lua scripting filter"         Yes)
 option(FLB_FILTER_RECORD_MODIFIER "Enable record_modifier filter"   Yes)
 option(FLB_FILTER_TENSORFLOW  "Enable tensorflow filter"             No)
+option(FLB_FILTER_GEOIP2      "Enable geoip2 filter"                Yes)
 
 # Debug callbacks
 option(FLB_HTTP_CLIENT_DEBUG  "Enable HTTP Client debug callbacks"   No)
@@ -233,6 +223,7 @@ if(FLB_ALL)
   set(FLB_OUT_SYSLOG   1)
   set(FLB_OUT_LIB      1)
   set(FLB_OUT_FLOWCOUNTER 1)
+  set(FLB_OUT_WEBSOCKET   1)
 endif()
 
 if(FLB_DEV)
@@ -308,6 +299,8 @@ endif()
 # Enable Debug symbols if specified
 if(FLB_DEBUG)
   set(CMAKE_BUILD_TYPE "Debug")
+elseif(FLB_RELEASE)
+  set(CMAKE_BUILD_TYPE "RelWithDebInfo")
 endif()
 
 if(FLB_PARSER)
@@ -421,10 +414,18 @@ endif()
 
 if(FLB_TLS)
   FLB_DEFINITION(FLB_HAVE_TLS)
+
+  # Build mbedtls: yes, in this transition period we always enable it
   option(ENABLE_TESTING  OFF)
   option(ENABLE_PROGRAMS OFF)
   option(INSTALL_MBEDTLS_HEADERS OFF)
   add_subdirectory(${FLB_PATH_LIB_MBEDTLS} EXCLUDE_FROM_ALL)
+
+  # find OpenSSL (our preferred choice now)
+  find_package(OpenSSL)
+  if(OPENSSL_FOUND)
+    FLB_DEFINITION(FLB_HAVE_OPENSSL)
+  endif()
 endif()
 
 # Metrics

ci/do-ut.ps1

@@ -1,8 +1,17 @@
 cd build
 
+if ( "x64" -eq $env:PLATFORM ) {
+    $OPENSSL_DIR = "C:\OpenSSL-v11-Win64"
+}
+else {
+    $OPENSSL_DIR = "C:\OpenSSL-v11-Win32"
+}
+
+
 # CACHE GENERATION
 cmake -G "NMake Makefiles" `
                      -D FLB_TESTS_INTERNAL=On `
+                     -D OPENSSL_ROOT_DIR=$OPENSSL_DIR `
                      -D FLB_WITHOUT_flb-rt-out_elasticsearch=On `
                      -D FLB_WITHOUT_flb-rt-out_td=On `
                      -D FLB_WITHOUT_flb-rt-out_forward=On `

cmake/libraries.cmake

@@ -4,7 +4,7 @@ set(FLB_PATH_LIB_RBTREE  "lib/rbtree")
 set(FLB_PATH_LIB_MSGPACK "lib/msgpack-c")
 set(FLB_PATH_LIB_AVRO    "lib/avro")
 set(FLB_PATH_LIB_CHUNKIO "lib/chunkio")
-set(FLB_PATH_LIB_LUAJIT  "lib/luajit-2.1.0-dd5032e")
+set(FLB_PATH_LIB_LUAJIT  "lib/luajit-2.1.0-1e66d0f")
 set(FLB_PATH_LIB_MONKEY  "lib/monkey")
 set(FLB_PATH_LIB_JSMN    "lib/jsmn")
 set(FLB_PATH_LIB_MBEDTLS "lib/mbedtls-2.24.0")

cmake/windows-setup.cmake

@@ -83,6 +83,7 @@ set(FLB_FILTER_NEST           Yes)
 set(FLB_FILTER_LUA            Yes)
 set(FLB_FILTER_RECORD_MODIFIER Yes)
 set(FLB_FILTER_REWRITE_TAG    Yes)
+set(FLB_FILTER_GEOIP2         Yes)
 
 # Search bison and flex executables
 if(CMAKE_SYSTEM_NAME MATCHES "Windows")

include/fluent-bit/flb_config.h

@@ -31,10 +31,6 @@
 
 #include <monkey/mk_core.h>
 
-#ifdef FLB_HAVE_TLS
-#include <fluent-bit/flb_io_tls.h>
-#endif
-
 #define FLB_CONFIG_FLUSH_SECS   5
 #define FLB_CONFIG_HTTP_LISTEN  "0.0.0.0"
 #define FLB_CONFIG_HTTP_PORT    "2020"

include/fluent-bit/flb_coro.h

@@ -0,0 +1,122 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2020 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_CORO_H
+#define FLB_CORO_H
+
+/* Required by OSX */
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE
+#endif
+
+#ifndef _DEFAULT_SOURCE
+#define _DEFAULT_SOURCE
+#endif
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_macros.h>
+#include <fluent-bit/flb_log.h>
+#include <fluent-bit/flb_mem.h>
+
+#include <monkey/mk_core.h>
+
+#include <stdlib.h>
+#include <limits.h>
+#include <libco.h>
+
+#ifdef FLB_HAVE_VALGRIND
+#include <valgrind/valgrind.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct flb_coro {
+
+#ifdef FLB_HAVE_VALGRIND
+    unsigned int valgrind_stack_id;
+#endif
+
+    /* libco 'contexts' */
+    cothread_t caller;
+    cothread_t callee;
+
+    void *data;
+};
+
+#ifdef FLB_CORO_STACK_SIZE
+#define FLB_CORO_STACK_SIZE      FLB_CORO_STACK_SIZE
+#else
+#define FLB_CORO_STACK_SIZE      ((3 * PTHREAD_STACK_MIN) / 2)
+#endif
+
+#define FLB_CORO_DATA(coro)      (((char *) coro) + sizeof(struct flb_coro))
+
+static FLB_INLINE void flb_coro_yield(struct flb_coro *coro, int ended)
+{
+    co_switch(coro->caller);
+}
+
+static FLB_INLINE void flb_coro_destroy(struct flb_coro *coro)
+{
+    flb_trace("[coro] destroy coroutine=%p data=%p", coro,
+              FLB_CORO_DATA(coro));
+
+#ifdef FLB_HAVE_VALGRIND
+    VALGRIND_STACK_DEREGISTER(coro->valgrind_stack_id);
+#endif
+
+    co_delete(coro->callee);
+    flb_free(coro);
+}
+
+#define flb_coro_return(th) co_switch(th->caller)
+
+void flb_coro_init();
+struct flb_coro *flb_coro_get();
+void flb_coro_set(struct flb_coro *coro);
+
+static FLB_INLINE void flb_coro_resume(struct flb_coro *coro)
+{
+    flb_coro_set(coro);
+    coro->caller = co_active();
+    co_switch(coro->callee);
+}
+
+static FLB_INLINE struct flb_coro *flb_coro_create(void *data)
+{
+    struct flb_coro *coro;
+
+    /* Create a thread context and initialize */
+    coro = (struct flb_coro *) flb_calloc(1, sizeof(struct flb_coro));
+    if (!coro) {
+        flb_errno();
+        return NULL;
+    }
+    coro->data = data;
+    return coro;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !FLB_CORO_H */

include/fluent-bit/flb_engine.h

@@ -26,6 +26,7 @@
 #include <fluent-bit/flb_config.h>
 #include <fluent-bit/flb_input.h>
 #include <fluent-bit/flb_output.h>
+#include <fluent-bit/flb_thread_storage.h>
 
 /* Types of events handled by the Server engine */
 #define FLB_ENGINE_EV_CORE          MK_EVENT_NOTIFICATION
@@ -34,6 +35,7 @@
 #define FLB_ENGINE_EV_SCHED         2048
 #define FLB_ENGINE_EV_SCHED_FRAME   (FLB_ENGINE_EV_SCHED + 4096)
 #define FLB_ENGINE_EV_OUTPUT        8192
+#define FLB_ENGINE_EV_THREAD_OUTPUT 16384
 
 /* Engine events: all engine events set the left 32 bits to '1' */
 #define FLB_ENGINE_EV_STARTED   FLB_BITS_U64_SET(1, 1) /* Engine started    */
@@ -60,4 +62,9 @@ int flb_engine_exit_status(struct flb_config *config, int status);
 int flb_engine_shutdown(struct flb_config *config);
 int flb_engine_destroy_tasks(struct mk_list *tasks);
 
+/* Engine event loop */
+void flb_engine_evl_init();
+struct mk_event_loop *flb_engine_evl_get();
+void flb_engine_evl_set(struct mk_event_loop *evl);
+
 #endif

include/fluent-bit/flb_fstore.h

@@ -38,7 +38,7 @@ struct flb_fstore_file {
     size_t meta_size;               /* metadata size */
     void *data;                     /* opaque data type for user/caller context */
     struct cio_chunk *chunk;        /* chunk context */
-    struct cio_chunk *stream;       /* parent stream that owns this file */
+    struct cio_stream *stream;      /* parent stream that owns this file */
     struct mk_list _head;           /* link to parent flb_fstore->files */
 };
 

include/fluent-bit/flb_thread.h → include/fluent-bit/flb_help.h

@@ -18,32 +18,16 @@
  *  limitations under the License.
  */
 
-#ifndef FLB_THREAD_H
-#define FLB_THREAD_H
-
-/* Required by OSX */
-#ifndef _XOPEN_SOURCE
-#define _XOPEN_SOURCE
-#endif
-
-#ifndef _DEFAULT_SOURCE
-#define _DEFAULT_SOURCE
-#endif
+#ifndef FLB_HELP_H
+#define FLB_HELP_H
 
 #include <fluent-bit/flb_info.h>
-#include <fluent-bit/flb_macros.h>
+#include <fluent-bit/flb_input.h>
+#include <fluent-bit/flb_filter.h>
+#include <fluent-bit/flb_output.h>
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern FLB_EXPORT pthread_key_t flb_thread_key;
+int flb_help_input(struct flb_input_instance *ins, void **out_buf, size_t *out_size);
+int flb_help_filter(struct flb_filter_instance *ins, void **out_buf, size_t *out_size);
+int flb_help_output(struct flb_output_instance *ins, void **out_buf, size_t *out_size);
 
-/* Use 'libco' backend for co-routines handling */
-#include <fluent-bit/flb_thread_libco.h>
-
-#ifdef __cplusplus
-}
 #endif
-
-#endif /* !FLB_THREAD_H */

include/fluent-bit/flb_http_client.h

@@ -136,6 +136,9 @@ struct flb_http_client {
     void *cb_ctx;
 };
 
+void flb_http_client_debug(struct flb_http_client *c,
+                           struct flb_callback *cb_ctx);
+
 struct flb_http_client *flb_http_client(struct flb_upstream_conn *u_conn,
                                         int method, const char *uri,
                                         const char *body, size_t body_len,