Update libmseed to 3.1.1

ChangeLog

@@ -1,3 +1,6 @@
+2024.024: 1.0.1
+	- Update libmseed to 3.1.1.
+
 2023.206: 1.0.0
 	- Update libmseed to 3.0.17.
 

libmseed/ChangeLog

@@ -1,3 +1,42 @@
+2024.024: 3.1.1
+	- Change library compatibility version in Makefile to MAJOR.1.0, as this is now
+	incompatible with the x.0.0 releases.
+
+2024.024: 3.1.0
+	BREAKING CHANGES, data structure and an API changes:
+	- `MS3Record.encoding` now int16_t instead of int8_t.
+	- `MS3Record.datalength` now uint32_t instead of uint16_t (change in 3.0.18).
+	- `MS3Record.datasize` and `MS3TraceSeg.datasize` now uint64_t instead of size_t.
+	- `ms3_detect()` now returns int64_t instead of int.
+	- `mstl3_unpack_recordlist()`'s `outputsize` argument is now uint64_t instead of size_t.
+	- `mseh_get_ptr_r()`'s `maxlength` is now uint32_t instead of size_t.
+
+	- Fix to handle full range of allowed encoding values from 0-255.
+	- Fix to handle detection of huge record lengths beyond MAXRECLEN gracefully.
+	- Add more checks to avoid writing impossible values to miniSEED v2 blockette fields.
+	- Fix all compiler warnings at level -Wextra for clang, gcc, and \W3 for MSVC.
+
+2024.015:
+	- Fix build of DLL target on Windows by defining exports in libmseed.def.
+
+2024.007:
+	- Improve formatting of gap list produced by mstl3_printgaplist().
+
+2024.006: 3.0.18
+	- ms_nslc2sid() requires all codes except location to be set to reduce misuse.
+	- Fix raw, diagnostic generation of SIDs from miniSEED v2 when full codes are used.
+	- Fix handling of data length field in miniSEED v3 as 32-bits instead of 16-bits.
+	- Increase MAXRECLEN from 131172 to 10485760 (10 MiB) for v3 records.
+	- Define MAXRECLENv2 as 131172 (131+ KiB) to limit miniSEED v2 records and add
+	test for sample counts beyond the 16-bit field limit in v2 records when packing.
+	- Update yyjson to release 0.8.0.
+
+2023.335:
+	- Add LMIO_FD type of reading IO via arbitrary file descriptor and hooks for setting.
+	- Add mseh_replace() to replace extra headers.
+	- Consistently initialize header values to 0, fixing edge cases like header-only records.
+	- Update link in documentation of ms_readleapsecondfile() for sources of leap-seconds.list.
+
 2023.206: 3.0.17
 	- Add tests for default record length and encoding.
 	- Document versioning schema as semantic versioning.

libmseed/Makefile

@@ -9,7 +9,7 @@
 # Extract version from libmseed.h, expected line should include LIBMSEED_VERSION "#.#.#"
 MAJOR_VER = $(shell grep LIBMSEED_VERSION libmseed.h | grep -Eo '[0-9]+.[0-9]+.[0-9]+' | cut -d . -f 1)
 FULL_VER = $(shell grep LIBMSEED_VERSION libmseed.h | grep -Eo '[0-9]+.[0-9]+.[0-9]+')
-COMPAT_VER = $(MAJOR_VER).0.0
+COMPAT_VER = $(MAJOR_VER).1.0
 
 # Default settings for install target
 PREFIX ?= /usr/local
@@ -75,7 +75,7 @@ $(LIB_SO): $(LIB_LOBJS)
 	ln -s $(LIB_SO) $(LIB_SO_BASE)
 	ln -s $(LIB_SO) $(LIB_SO_MAJOR)
 
-test check: static FORCE
+test tests check: static FORCE
 	@$(MAKE) -C test test
 
 example: static FORCE

libmseed/Makefile.win

@@ -34,7 +34,7 @@ lib: $(OBJS)
 	link.exe /lib /nologo /OUT:$(LM_LIB) $(OBJS)
 
 dll: $(OBJS)
-	link.exe /dll /nologo /OUT:$(LM_DLL) $(OBJS)
+	link.exe /dll /nologo /DEF:$(LM_DEF) /OUT:$(LM_DLL) $(OBJS)
 
 .c.obj:
 	$(CC) /nologo $(CFLAGS) $(INCS) $(OPTS) /c $<

libmseed/README.md

@@ -29,4 +29,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Copyright (C) 2023 Chad Trabant, EarthScope Data Services
+Copyright (C) 2024 Chad Trabant, EarthScope Data Services

libmseed/doc/tutorial.dox

@@ -160,12 +160,12 @@ int main()
   msr = msr3_init (NULL);
 
   /* Populate MS3Record values */
-  strcpy (msr->sid, "XX_TEXT__B_H_E");
-  msr->formatversion = 3;
+  strcpy (msr->sid, "FDSN:XX_TEXT__B_H_E");
   msr->reclen = 512;
-  msr->encoding = DE_STEIM2;
+  msr->pubversion = 1;
   msr->starttime = ms_timestr2nstime ("2018-12-01T00:00:00.000000000");
   msr->samprate = 20.0;
+  msr->encoding = DE_STEIM2;
 
   msr->datasamples = datasamples;  /* pointer to 32-bit integer data samples */
   msr->numsamples = 11;

libmseed/example/Makefile.win

@@ -9,7 +9,7 @@ OPTS = /O2 /D_CRT_SECURE_NO_WARNINGS
 SRCS = lm_pack.c \
        lm_pack_rollingbuffer.c \
        lm_parse.c \
-	   lm_read_buffer.c \
+       lm_read_buffer.c \
        lm_read_recordlist.c \
        lm_read_selection.c \
        lm_sids.c \
@@ -25,4 +25,5 @@ all: $(BINS)
 
 # Clean-up directives
 clean:
-	-del *.obj *.exe *% *~
+	-del *.obj *.exe *% *~
+

libmseed/example/lm_pack_rollingbuffer.c

@@ -37,6 +37,7 @@
 void
 record_handler (char *record, int reclen, void *handlerdata)
 {
+  (void)handlerdata;
   MS3Record *msr = NULL;
 
   if (!msr3_parse (record, reclen, &msr, 0, 0))

libmseed/example/lm_read_recordlist.c

@@ -41,15 +41,15 @@ main (int argc, char **argv)
   char fileptrstr[30];
   uint32_t flags = 0;
   int8_t verbose = 0;
-  size_t idx;
+  uint64_t idx;
   int rv;
 
   char printdata = 0;
   int64_t unpacked;
   uint8_t samplesize;
   char sampletype;
-  size_t lineidx;
-  size_t lines;
+  uint64_t lineidx;
+  uint64_t lines;
   int col;
   void *sptr;
 
@@ -62,7 +62,7 @@ main (int argc, char **argv)
   mseedfile = argv[1];
 
   /* Simplistic argument parsing */
-  for (idx = 2; idx < argc; idx++)
+  for (idx = 2; (int)idx < argc; idx++)
   {
     if (strncmp (argv[idx], "-v", 2) == 0)
       verbose += strspn (&argv[idx][1], "v");
@@ -166,7 +166,7 @@ main (int argc, char **argv)
 
             for (idx = 0, lineidx = 0; lineidx < lines; lineidx++)
             {
-              for (col = 0; col < 6 && idx < seg->numsamples; col++)
+              for (col = 0; col < 6 && (int64_t)idx < seg->numsamples; col++)
               {
                 sptr = (char *)seg->datasamples + (idx * samplesize);
 

libmseed/extraheaders.c

@@ -3,7 +3,7 @@
  *
  * This file is part of the miniSEED Library.
  *
- * Copyright (c) 2023 Chad Trabant, EarthScope Data Services
+ * Copyright (c) 2024 Chad Trabant, EarthScope Data Services
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -148,7 +148,7 @@ parse_json (char *jsonstring, size_t length, LM_PARSED_JSON *parsed)
  ***************************************************************************/
 int
 mseh_get_ptr_r (const MS3Record *msr, const char *ptr,
-                 void *value, char type, size_t maxlength,
+                 void *value, char type, uint32_t maxlength,
                  LM_PARSED_JSON **parsestate)
 {
   LM_PARSED_JSON *parsed  = (parsestate) ? *parsestate : NULL;
@@ -161,7 +161,7 @@ mseh_get_ptr_r (const MS3Record *msr, const char *ptr,
 
   if (!msr || !ptr)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'ptr'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'ptr'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -324,7 +324,7 @@ mseh_set_ptr_r (MS3Record *msr, const char *ptr,
 
   if (!msr || !ptr || !value)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr', 'ptr', or 'value'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr', 'ptr', or 'value'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -513,7 +513,7 @@ mseh_add_event_detection_r (MS3Record *msr, const char *ptr,
 
   if (!msr || !eventdetection)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'eventdetection'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'eventdetection'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -656,7 +656,7 @@ mseh_add_calibration_r (MS3Record *msr, const char *ptr,
 
   if (!msr || !calibration)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'calibration'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'calibration'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -834,7 +834,7 @@ mseh_add_timing_exception_r (MS3Record *msr, const char *ptr,
 
   if (!msr || !exception)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'exception'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'exception'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -923,7 +923,7 @@ mseh_add_recenter_r (MS3Record *msr, const char *ptr, MSEHRecenter *recenter,
 
   if (!msr || !recenter)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'recenter'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'recenter'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -1036,7 +1036,7 @@ mseh_serialize (MS3Record *msr, LM_PARSED_JSON **parsestate)
   msr->extra       = serialized;
   msr->extralength = (uint16_t)serialsize;
 
-  return (int)serialsize;
+  return msr->extralength;
 }
 
 /**********************************************************************/ /**
@@ -1071,6 +1071,76 @@ mseh_free_parsestate (LM_PARSED_JSON **parsestate)
   *parsestate = NULL;
 }
 
+/**********************************************************************/ /**
+ * @brief Replace extra headers with supplied JSON
+ *
+ * Parse the supplied JSON string, re-serialize into compact form, and replace
+ * the extra headers of \a msr with the result.
+ *
+ * To _remove_ all of the extra headers, set \a jsonstring to NULL.
+ *
+ * This function cannot be used in combination with the routines that use
+ * a parsed state, i.e. mseh_get_ptr_r() and mseh_set_ptr_r().
+ *
+ * @param[in] msr ::MS3Record to generate extra headers for
+ * @param[in] jsonstring JSON replacment for extra headers of \a msr
+ *
+ * @returns Length of extra headers on success, otherwise a (negative) libmseed error code
+ ***************************************************************************/
+int
+mseh_replace (MS3Record *msr, char *jsonstring)
+{
+  yyjson_read_flag read_flg = YYJSON_READ_NOFLAG;
+  yyjson_write_flag write_flg = YYJSON_WRITE_NOFLAG;
+  yyjson_read_err read_err;
+  yyjson_write_err write_err;
+  yyjson_alc alc = {_priv_malloc, _priv_realloc, _priv_free, NULL};
+  yyjson_doc *doc = NULL;
+
+  char *serialized  = NULL;
+  size_t serialsize = 0;
+
+  if (!msr)
+    return MS_GENERROR;
+
+  if (jsonstring != NULL)
+  {
+    /* Parse JSON into immutable form */
+    if ((doc = yyjson_read_opts (jsonstring, strlen (jsonstring), read_flg, &alc, &read_err)) == NULL)
+    {
+      ms_log (2, "%s() Cannot parse extra header JSON: %s\n",
+              __func__, (read_err.msg) ? read_err.msg : "Unknown error");
+      return MS_GENERROR;
+    }
+
+    /* Serialize new JSON string */
+    serialized = yyjson_write_opts (doc, write_flg, &alc, &serialsize, &write_err);
+
+    if (serialized == NULL)
+    {
+      ms_log (2, "%s() Cannot write extra header JSON: %s\n",
+              __func__, (write_err.msg) ? write_err.msg : "Unknown error");
+      return MS_GENERROR;
+    }
+
+    if (serialsize > UINT16_MAX)
+    {
+      ms_log (2, "%s() New serialization size exceeds limit of %d bytes: %" PRIu64 "\n",
+              __func__, UINT16_MAX, (uint64_t)serialsize);
+      libmseed_memory.free (serialized);
+      return MS_GENERROR;
+    }
+  }
+
+  /* Set new extra headers, replacing existing headers */
+  if (msr->extra)
+    libmseed_memory.free (msr->extra);
+  msr->extra       = serialized;
+  msr->extralength = (uint16_t)serialsize;
+
+  return msr->extralength;
+}
+
 /**********************************************************************/ /**
  * @brief Print the extra header structure for the specified MS3Record.
  *

libmseed/extraheaders.h

@@ -3,7 +3,7 @@
  *
  * This file is part of the miniSEED Library.
  *
- * Copyright (c) 2023 Chad Trabant, EarthScope Data Services
+ * Copyright (c) 2024 Chad Trabant, EarthScope Data Services
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.