Merge remote-tracking branch 'origin/topic/robin/zeek6-docs'

* origin/topic/robin/zeek6-docs:
  Update `scripts/autogen-zeek-docs` and corresponding content.
  Update Zeek docs for Zeek 6.0.

CHANGES

@@ -1,3 +1,7 @@
+1.8.0-dev.176 | 2023-06-14 08:52:14 +0200
+
+  * Update Zeek docs for Zeek 6.0. (Robin Sommer, Corelight)
+
 1.8.0-dev.173 | 2023-06-09 10:01:28 +0200
 
   * GH-1447: Do not forcibly make `strong_ref` `in` function parameters immutable. (Benjamin Bannier, Corelight)

VERSION

@@ -1 +1 @@
-1.8.0-dev.173
+1.8.0-dev.176

doc/Makefile

@@ -58,12 +58,12 @@ check:
 
 spicy-%:
 	git clone https://github.com/zeek/$@
-check-zeek-docs: spicy-plugin spicy-tftp
+check-zeek-docs: spicy-tftp
 	@echo Refreshing checkouts
 	@for REPO in $^; do (cd $$REPO && git pull && git reset HEAD --hard)>/dev/null; done
 	@
 	@echo Checking whether docs for Zeek integration are up-to-date
-	@./scripts/autogen-zeek-docs spicy-plugin spicy-tftp
+	@./scripts/autogen-zeek-docs spicy-tftp
 	@
 	@git diff --quiet autogen/zeek \
 		|| (echo "Zeek docs are not up-to-date, rerun './scripts/autogen-zeek-docs'." && exit 1)

doc/autogen/zeek/__preload__.zeek → doc/autogen/zeek/init-bare.zeek

@@ -1,5 +1,4 @@
 
-# doc-common-start
 module Spicy;
 
 export {
@@ -24,7 +23,8 @@ export {
 # doc-options-end
 
 # doc-types-start
-    ## Result type for `Spicy::resource_usage()`.
+    ## Result type for `Spicy::resource_usage()`. The values reflect resource
+    ## usage as reported by the Spicy runtime system.
     type ResourceUsage: record {
         user_time : interval;           ##< user CPU time of the Zeek process
         system_time :interval;          ##< system CPU time of the Zeek process

doc/autogen/zeek/record-spicy-batch.zeek

@@ -1,4 +1,4 @@
-# Saves all input traffic in Spicy's batch format.
+##! Saves all input traffic in Spicy's batch format.
 
 module SpicyBatch;
 

doc/autogen/zeek/tftp-no-accept.spicy

@@ -1,13 +1,11 @@
-# Copyright (c) 2021 by the Zeek Project. See LICENSE for details.
 #
 # Trivial File Transfer Protocol
 #
 # Specs from https://tools.ietf.org/html/rfc1350
+#
 
 module TFTP;
 
-import spicy;
-
 # Common header for all messages:
 #
 #      2 bytes
@@ -24,6 +22,8 @@ public type Packet = unit {    # public top-level entry point for parsing
         Opcode::ACK   -> ack:   Acknowledgement;
         Opcode::ERROR -> error: Error;
         };
+
+    on %done { print self; }
 };
 
 # TFTP supports five types of packets [...]:
@@ -52,7 +52,6 @@ type Opcode = enum {
 type Request = unit(is_read: bool) {
     filename: bytes &until=b"\x00";
     mode:     bytes &until=b"\x00";
-
 };
 
 # Figure 5-2: DATA packet
@@ -67,6 +66,7 @@ type Data = unit {
     data: bytes &eod;
 };
 
+
 # Figure 5-3: ACK packet
 #
 #  2 bytes     2 bytes

doc/autogen/zeek/tftp.evt

@@ -1,6 +1,4 @@
-# Copyright (c) 2021 by the Zeek Project. See LICENSE for details.
-#
-# Note: When line numbers change in this file, update the documentation that pulls it in.
+# Note: When line number changes in this file, update the documentation that pulls it in.
 
 protocol analyzer spicy::TFTP over UDP:
     parse with TFTP::Packet,

doc/autogen/zeek/tftp.spicy

@@ -1,13 +1,11 @@
-# Copyright (c) 2021 by the Zeek Project. See LICENSE for details.
 #
 # Trivial File Transfer Protocol
 #
 # Specs from https://tools.ietf.org/html/rfc1350
+#
 
 module TFTP;
 
-import spicy;
-
 # Common header for all messages:
 #
 #      2 bytes
@@ -24,6 +22,8 @@ public type Packet = unit {    # public top-level entry point for parsing
         Opcode::ACK   -> ack:   Acknowledgement;
         Opcode::ERROR -> error: Error;
         };
+
+    on %done { print self; }
 };
 
 # TFTP supports five types of packets [...]:
@@ -52,8 +52,6 @@ type Opcode = enum {
 type Request = unit(is_read: bool) {
     filename: bytes &until=b"\x00";
     mode:     bytes &until=b"\x00";
-
-    on %done { spicy::accept_input(); }
 };
 
 # Figure 5-2: DATA packet
@@ -68,6 +66,7 @@ type Data = unit {
     data: bytes &eod;
 };
 
+
 # Figure 5-3: ACK packet
 #
 #  2 bytes     2 bytes

doc/autogen/zeek/tftp.zeek

@@ -1,4 +1,3 @@
-# Copyright (c) 2021 by the Zeek Project. See LICENSE for details.
 
 module TFTP;
 
@@ -78,8 +77,8 @@ function init_request(c: connection, is_orig: bool, fname: string, mode: string,
 
 	# The data will come in from a different source port.
         #
-	# TODO: Starting with Zeek 5.2, we will be able to use
-	# Analyzer::ANALYZER_SPICY_TFTP instead of get_tag(),
+        # TODO: We should be able to use Analyzer::ANALYZER_SPICY_TFTP instead of get_tag(),
+        # but seems the ID is not yet available at parse time.
 	Analyzer::schedule_analyzer(c$id$resp_h, c$id$orig_h, c$id$orig_p, Analyzer::get_tag("Spicy_TFTP"), 1min);
 	expected_data_conns[c$id$resp_h, c$id$orig_p, c$id$orig_h] = info;
 	}

doc/autogen/zeek/zeek-functions.spicy

@@ -18,7 +18,7 @@ instead, which will have the same effect with Zeek.
 
 .. _spicy_weird:
 
-.. rubric:: ``function zeek::weird(id: string, addl: string = "") : &cxxname="spicy::zeek::rt::weird";``
+.. rubric:: ``function zeek::weird(id: string, addl: string = "") : &cxxname="zeek::spicy::rt::weird";``
 
 Reports a "weird" to Zeek. This should be used with similar semantics as in
 Zeek: something quite unexpected happening at the protocol level, which however

doc/development/testing.rst

@@ -67,7 +67,7 @@ libraries aren't in a standard runtime library path, you'll also need
 to set ``LD_LIBRARY_PATH`` (Linux) or ``DYLD_LIBRARY_PATH`` (macOS) to
 point there (e.g., ``LD_LIBRARY_PATH=/opt/clang9/lib/clang/9.0.1/lib/linux``).
 
-When using the Spicy plugin for Zeek and Zeek hasn't been compiled
+When using Zeek's spicy support, and Zeek hasn't been compiled
 with sanitizer support, you'll also need to set ``LD_PRELOAD`` (Linux)
 or ``DYLD_INSERT_LIBRARIES`` (macOS) to the shared ``asan`` library to
 use (e.g.,

doc/examples/_usage-spicy-config.output

@@ -4,32 +4,28 @@ Usage: spicy-config [options]
 
 Available options:
 
-    --bindir                Prints the path to the directory where binaries are installed.
-    --build                 Prints "debug" or "release", depending on the build configuration.
-    --cmake-path            Prints the path to Spicy-provided CMake modules
-    --cxx                   Print the path to the C++ compiler used to build Spicy
-    --cxxflags              Print flags for C++ compiler when compiling generated code statically
-    --cxxflags-hlto         Print flags for C++ compiler when building precompiled HLTO libraries
-    --debug                 Output flags for working with debugging versions.
-    --distbase              Print path of the Spicy source distribution.
-    --dynamic-loading       Adjust --ldflags for host applications that dynamically load precompiled modules
-    --have-toolchain        Prints 'yes' if the Spicy toolchain was built, 'no' otherwise.
-    --have-zeek             Prints 'yes' if the Spicy was compiled with Zeek support, 'no' otherwise.
-    --help                  Print this usage summary
-    --include-dirs          Prints the Spicy runtime's C++ include directories
-    --ldflags               Print flags for linker when compiling generated code statically
-    --ldflags-hlto          Print flags for linker linker when building precompiled HLTO libraries
-    --libdirs               Print standard Spicy library directories.
-    --prefix                Print path of installation
-    --spicy-build           Print the path to the spicy-build script.
-    --spicyc                Print the path to the spicyc binary.
-    --version               Print the Spicy version as a string.
-    --version-number        Print the Spicy version as a numerical value.
-    --zeek                  Print the path to the Zeek executable
-    --zeek-include-dirs     Print the Spicy runtime's C++ include directories
-    --zeek-module-path      Print the path of the directory the Zeek plugin searches for *.hlto modules
-    --zeek-plugin-path      Print the path to go into ZEEK_PLUGIN_PATH for enabling the Zeek Spicy plugin
-    --zeek-prefix           Print the path to the Zeek installation prefix
-    --zeek-version          Print the Zeek version (empty if no Zeek available)
-    --zeek-version-number   Print the Zeek version as a numerical value (zero if no Zeek available)
+    --bindir                 Prints the path to the directory where binaries are installed.
+    --build                  Prints "debug" or "release", depending on the build configuration.
+    --cmake-path             Prints the path to Spicy-provided CMake modules
+    --cxx                    Print the path to the C++ compiler used to build Spicy
+    --cxx-launcher           Print the full path to the compiler launcher used to compile HILTI.
+    --cxxflags               Print flags for C++ compiler when compiling generated code statically
+    --cxxflags-hlto          Print flags for C++ compiler when building precompiled HLTO libraries
+    --debug                  Output flags for working with debugging versions.
+    --distbase               Print path of the Spicy source distribution.
+    --dynamic-loading        Adjust --ldflags for host applications that dynamically load precompiled modules
+    --have-toolchain         Prints 'yes' if the Spicy toolchain was built, 'no' otherwise.
+    --help                   Print this usage summary
+    --include-dirs           Prints the Spicy runtime's C++ include directories
+    --include-dirs-toolchain Prints the Spicy compiler's C++ include directories
+    --ldflags                Print flags for linker when compiling generated code statically
+    --ldflags-hlto           Print flags for linker linker when building precompiled HLTO libraries
+    --libdirs                Print standard Spicy library directories.
+    --libdirs-cxx-runtime    Print C++ library directories for runtime.
+    --libdirs-cxx-toolchain  Print C++ library directories for toolchain.
+    --prefix                 Print path of installation
+    --spicy-build            Print the path to the spicy-build script.
+    --spicyc                 Print the path to the spicyc binary.
+    --version                Print the Spicy version as a string.
+    --version-number         Print the Spicy version as a numerical value.
 

doc/faq.rst

@@ -59,33 +59,31 @@ Zeek
 
 .. _faq_zeek_install_spicy_and_plugin_to_use_parsers:
 
-.. rubric:: Do I need to install Spicy and its Zeek plugin to use Spicy parsers in Zeek?
+.. rubric:: Do I need to install Spicy and/or a Zeek plugin to use Spicy parsers in Zeek?
 
-As of version 5.0 Zeek by default bundles Spicy and its Zeek plugin. If that is
-the case the folder containing the ``zeek`` binary should also contain e.g.,
-``spicyc`` (provided by Spicy) and ``spicyz`` (provided by Spicy plugin). To
-check that the Spicy plugin is active look for ``Zeek::Spicy`` in the output
-of ``zeek -N``::
+If you're using Zeek >= 5.0 with a default build configuration,
+there's nothing else you need to install. After installing Zeek, the
+same folder containing the ``zeek`` binary will also have the relevant
+Spicy tools, such as  ``spicyc`` (provided by Spicy) and ``spicyz``
+(provided by Zeek). To double check that the Spicy support is indeed
+available, look for ``Zeek::Spicy`` in the output of ``zeek -N``::
 
     # zeek -N
     <...>
     Zeek::Spicy - Support for Spicy parsers (``*.spicy``, ``*.evt``, ``*.hlto``) (built-in)
 
-If ``spicyc`` is missing, you need to :ref:`install Spicy <installation>`; if
-``spicyz`` is missing or ``Zeek::Spicy`` is not listed you need to :ref:`install
-Spicy plugin <zeek_spicy_plugin_installation>`.
+Note that it remains possible to build Zeek against an external Spicy
+installation, or even without any Spicy support at all. Look at Zeek's
+``configure`` for corresponding options.
 
-.. _faq_zeek_install_spicy_to_use_plugin:
+.. note::
 
-.. rubric:: Do I need a Spicy installation for using the Zeek plugin?
+    For some historic background: Zeek 5.0 started bundling Spicy, as well
+    as the former Zeek plugin for Spicy, so that now nothing else needs to
+    be installed separately anymore to use Spicy parsers. Since Zeek 6.0,
+    the code for that former plugin has further moved into Zeek itself,
+    and is now maintained directly by the Zeek developers.
 
-No, if the Zeek plugin was compiled with ``--build-toolchain=no``,
-it will not require Spicy to be installed on the system. It will only
-be able to load pre-compiled analyzers then (i.e., ``*.hlto`` files),
-which you can create on a similar system that has Spicy installed
-through :ref:`spicyz <spicyz>`. The build process will leave a binary
-distribution inside your build directory at
-``zeek/plugin/Zeek_Spicy.tgz``.
 
 .. _faq_zeek_spicy_dpd_support:
 
@@ -106,7 +104,7 @@ to declare such an analyzer.
 
 .. rubric:: I have ``print`` statements in my Spicy grammar, why do I not see any output when running Zeek?
 
-The Zeek plugin by default disables the output of Spicy-side ``print``
+Zeek by default disables the output of Spicy-side ``print``
 statements. To enable them, add ``Spicy::enable_print=T`` to the Zeek
 command line (or ``redef Spicy::enable_print=T;`` to a Zeek script
 that you are loading).
@@ -115,7 +113,7 @@ that you are loading).
 
 .. rubric:: My analyzer recognizes only one or two TCP packets even though there are more in the input.
 
-The Zeek Spicy plugin parses the sending and receiving sides of a TCP
+In Zeek, a Spicy analyzer parses the sending and receiving sides of a TCP
 connection each according to the given Spicy grammar. This means that
 if more than one message can be sent per side the grammar needs to
 allow for that. For example, if the grammar parses messages of the

doc/getting-started.rst

@@ -200,9 +200,26 @@ and ``spicyc -p my-http.spicy`` will show the intermediary HILTI code.
 Zeek Integration
 ----------------
 
-Now let's use our ``RequestLine`` parser with Zeek. For that we need to prepare
-some input and then we can use the grammar that we already got to add a new
-protocol analyzer to Zeek.
+Now let's use our ``RequestLine`` parser with Zeek.
+
+Since version 5.0, Zeek comes with Spicy support built-in by default,
+so normally you won't need to install anything further to make use of
+Spicy parsers. To double check that your Zeek indeed supports Spicy,
+confirm that it shows up in the output of ``zeek -N``:
+
+.. code::
+
+    # zeek -N
+    <...>
+    Zeek::Spicy - Support for Spicy parsers (``*.spicy``, ``*.evt``, ``*.hlto``) (built-in)
+
+If you do not see the Spicy listed, it must have been disabled at Zeek
+build time. Assuming Spicy is indeed available, you will also find the
+Zeek-side compiler tool ``spicyz`` in the same location as the ``zeek``
+binary.
+
+Next, we prepare some input traffic for testing our ``RequestLine``
+parser with Zeek.
 
 .. rubric:: Preparations
 
@@ -221,26 +238,6 @@ payload::
 
 This gets us :download:`this trace file <examples/request-line.pcap>`.
 
-Next we need to make sure that Zeek can load Spicy parsers. As of version 5.0
-Zeek by default bundles Spicy and spicy-plugin which makes Spicy parsers
-available to Zeek.
-
-.. code::
-
-    # zeek -N
-    <...>
-    Zeek::Spicy - Support for Spicy parsers (``*.spicy``, ``*.evt``, ``*.hlto``) (built-in)
-
-Depending on the Zeek build configuration this could also report
-
-.. code::
-
-    Zeek::Spicy - Support for Spicy parsers (``*.spicy``, ``*.evt``, ``*.hlto``) (dynamic, version 1.3.13)
-
-If you do not see the Spicy plugin listed you need to install the Zeek plugin
-spicy-plugin yourself, see :ref:`the Zeek package installation guide
-<zeek_spicy_plugin_installation>`.
-
 .. _example_zeek_my_http_adding_analyzer:
 
 .. rubric:: Adding a Protocol Analyzer
@@ -270,7 +267,7 @@ type that our Spicy grammar defines (line 2); and we want Zeek to
 activate our analyzer for all connections with a responder port of
 12345 (which, of course, matches the packet trace we created).
 
-The second block (line 5) tells the Spicy plugin that we want to
+The second block (line 5) tells Zeek that we want to
 define one event. On the left-hand side of that line we give the unit
 that is to trigger the event. The right-hand side defines its name and
 arguments. What we are saying here is that every time a ``RequestLine``
@@ -280,7 +277,7 @@ Three of them are the values of corresponding unit fields, accessed
 just through normal Spicy expressions (inside an event argument
 expression, ``self`` refers to the unit instance that has led to the
 generation of the current event). The first parameter, ``$conn``, is a
-"magic" keyword that lets the Spicy plugin pass the Zeek-side
+"magic" keyword that passes the Zeek-side
 connection ID (``conn_id``) to the event.
 
 Now we got everything in place that we need for our new protocol
@@ -316,7 +313,7 @@ output we would expect.
 
 .. note::
 
-    By default, the Zeek plugin suppresses any output from Spicy-side
+    By default, Zeek suppresses any output from Spicy-side
     ``print`` statements. You can add ``Spicy::enable_print=T`` to the
     command line to see it. In the example above, you would then get
     an additional line of output: ``GET, /index.html, 1.0``.

doc/host-applications.rst

@@ -155,7 +155,7 @@ If we want that, we can use ``parse2()`` instead and provide it with a
     version: 1.0
 
 Another approach to retrieving field values goes through Spicy hooks
-calling back into the host application. That's how the Zeek plugin
+calling back into the host application. That's how the Zeek's Spicy support
 operates. Let's say we want to execute a custom C++ function every
 time a ``RequestList`` has been parsed. By adding the following code
 to ``my-http.spicy``, we (1) declare that function on the Spicy-side,
@@ -223,7 +223,7 @@ Supporting Arbitrary Parsers
 This approach is more complex, and we'll just briefly describe the
 main pieces here. All of the tools coming with Spicy support arbitrary
 parsers and can serve as further examples (e.g., :ref:`spicy-driver`,
-:ref:`spicy-dump`, the :ref:`Zeek plugin <zeek_plugin>`). Indeed, they all
+:ref:`spicy-dump`, :ref:`zeek_plugin`). Indeed, they all
 build on the same C++ library class ``spicy::rt::Driver`` that
 provides a higher-level API to working with Spicy's parsers in a
 generic fashion. We'll do the same in the following.