Update libmseed to v3.0.18

ChangeLog

@@ -1,3 +1,6 @@
+2024.006: 4.1
+	- Update libmseed to v3.0.18.
+
 2023.204: 4.0
 	- Update to libmseed 3, with support for miniSEED 3, URLs, etc.
 	- Remove the -M and -R options.

libmseed/ChangeLog

@@ -1,3 +1,24 @@
+2024.006: 3.0.18
+	- ms_nslc2sid() requires all codes except location to be set to reduce misuse.
+	- Fix raw, diagnostic generation of SIDs from miniSEED v2 when full codes are used.
+	- Fix handling of data length field in miniSEED v3 as 32-bits instead of 16-bits.
+	- Increase MAXRECLEN from 131172 to 10485760 (10 MiB) for v3 records.
+	- Define MAXRECLENv2 as 131172 (131+ KiB) to limit miniSEED v2 records and add
+	test for sample counts beyond the 16-bit field limit in v2 records when packing.
+	- Update yyjson to release 0.8.0.
+
+2023.335:
+	- Add LMIO_FD type of reading IO via arbitrary file descriptor and hooks for setting.
+	- Add mseh_replace() to replace extra headers.
+	- Consistently initialize header values to 0, fixing edge cases like header-only records.
+	- Update link in documentation of ms_readleapsecondfile() for sources of leap-seconds.list.
+
+2023.206: 3.0.17
+	- Add tests for default record length and encoding.
+	- Document versioning schema as semantic versioning.
+	- Add const qualifiers to packing routines that do not modify the
+	passed MS3Record structure.
+
 2023.204: 3.0.16
 	- Additional Steim 1 & 2 decoder optimization.
 	- Add more const qualifiers to pointer values.  Thanks @damb.

libmseed/Makefile

@@ -75,7 +75,7 @@ $(LIB_SO): $(LIB_LOBJS)
 	ln -s $(LIB_SO) $(LIB_SO_BASE)
 	ln -s $(LIB_SO) $(LIB_SO_MAJOR)
 
-test check: static FORCE
+test tests check: static FORCE
 	@$(MAKE) -C test test
 
 example: static FORCE

libmseed/README.md

@@ -29,4 +29,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Copyright (C) 2023 Chad Trabant, EarthScope Data Services
+Copyright (C) 2024 Chad Trabant, EarthScope Data Services

libmseed/doc/faq.dox

@@ -70,7 +70,7 @@ You can use the @ref ms_sid2nslc() and @ref ms_nslc2sid() routines to convert be
 Except for potentially larger individual codes, the major difference in the two representations is the
 \a channel code.  In the SEED specification this is a fixed 3-character code, while in the FDSN Source Identifier
 specification the channel is 3 codes separated by underscores, with each code potentially being larger than a
-single character.  The library is agnostic to form of a channel reprsentation, accepting either and converting
+single character.  The library is agnostic to form of a channel representation, accepting either and converting
 between them whenever possible and appropriate.
 
 @subsection faq-7 The library API changed! Why? How do I port my libmseed 2.x program to the 3.x API?
@@ -93,7 +93,7 @@ Thank you!
 
 It won't be everyone's cup of tea.  Some alternative libraries are:
 
-P. Crotwell's <a class="el" href="https://www.seis.sc.edu/seisFile.html">seisFile</a> - Java library with miniSEED support.
+P. Crotwell's <a class="el" href="https://www.seis.sc.edu/seisFile.html">SeisFile</a> - Java library with miniSEED support.
 
 D. Neuhauser's <a class="el" href="https://www.ncedc.org/qug/software/ucb/">Qlib2</a> - C library with miniSEED support.
 

libmseed/doc/mainpage.dox

@@ -43,11 +43,15 @@ libmseed project</a>.
 A @ref porting-guide is available for developer porting software that
 previously used version 2 of the library.
 
+@subsection versioning-main Versioning
+
+Releases are versioned using <a class="el" href="http://semver.org/">Semantic Versioning</a>.
+
 @subsection main-acknowledgements Acknowlegements
 
 Numerous improvements have been incorporated based on feedback and
 patches submitted by others.  Individual acknowlegements are included
-in the ChangeLog.  Contributions are welcome and greatly appreciated.
+in the ChangeLog and/or commit messages.  Contributions are welcome and greatly appreciated.
 
 This library also uses code bits published in the public domain and
 acknowledgement is included in the code whenever possible.

libmseed/doc/tutorial.dox

@@ -160,12 +160,12 @@ int main()
   msr = msr3_init (NULL);
 
   /* Populate MS3Record values */
-  strcpy (msr->sid, "XX_TEXT__B_H_E");
-  msr->formatversion = 3;
+  strcpy (msr->sid, "FDSN:XX_TEXT__B_H_E");
   msr->reclen = 512;
-  msr->encoding = DE_STEIM2;
+  msr->pubversion = 1;
   msr->starttime = ms_timestr2nstime ("2018-12-01T00:00:00.000000000");
   msr->samprate = 20.0;
+  msr->encoding = DE_STEIM2;
 
   msr->datasamples = datasamples;  /* pointer to 32-bit integer data samples */
   msr->numsamples = 11;

libmseed/extraheaders.c

@@ -3,7 +3,7 @@
  *
  * This file is part of the miniSEED Library.
  *
- * Copyright (c) 2023 Chad Trabant, EarthScope Data Services
+ * Copyright (c) 2024 Chad Trabant, EarthScope Data Services
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -161,7 +161,7 @@ mseh_get_ptr_r (const MS3Record *msr, const char *ptr,
 
   if (!msr || !ptr)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'ptr'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'ptr'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -324,7 +324,7 @@ mseh_set_ptr_r (MS3Record *msr, const char *ptr,
 
   if (!msr || !ptr || !value)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr', 'ptr', or 'value'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr', 'ptr', or 'value'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -513,7 +513,7 @@ mseh_add_event_detection_r (MS3Record *msr, const char *ptr,
 
   if (!msr || !eventdetection)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'eventdetection'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'eventdetection'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -656,7 +656,7 @@ mseh_add_calibration_r (MS3Record *msr, const char *ptr,
 
   if (!msr || !calibration)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'calibration'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'calibration'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -834,7 +834,7 @@ mseh_add_timing_exception_r (MS3Record *msr, const char *ptr,
 
   if (!msr || !exception)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'exception'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'exception'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -923,7 +923,7 @@ mseh_add_recenter_r (MS3Record *msr, const char *ptr, MSEHRecenter *recenter,
 
   if (!msr || !recenter)
   {
-    ms_log (2, "%s() Required argument not defined: 'msr' or 'recenter'\n", __func__);
+    ms_log (2, "%s() Required input not defined: 'msr' or 'recenter'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -1036,7 +1036,7 @@ mseh_serialize (MS3Record *msr, LM_PARSED_JSON **parsestate)
   msr->extra       = serialized;
   msr->extralength = (uint16_t)serialsize;
 
-  return (int)serialsize;
+  return msr->extralength;
 }
 
 /**********************************************************************/ /**
@@ -1071,6 +1071,76 @@ mseh_free_parsestate (LM_PARSED_JSON **parsestate)
   *parsestate = NULL;
 }
 
+/**********************************************************************/ /**
+ * @brief Replace extra headers with supplied JSON
+ *
+ * Parse the supplied JSON string, re-serialize into compact form, and replace
+ * the extra headers of \a msr with the result.
+ *
+ * To _remove_ all of the extra headers, set \a jsonstring to NULL.
+ *
+ * This function cannot be used in combination with the routines that use
+ * a parsed state, i.e. mseh_get_ptr_r() and mseh_set_ptr_r().
+ *
+ * @param[in] msr ::MS3Record to generate extra headers for
+ * @param[in] jsonstring JSON replacment for extra headers of \a msr
+ *
+ * @returns Length of extra headers on success, otherwise a (negative) libmseed error code
+ ***************************************************************************/
+int
+mseh_replace (MS3Record *msr, char *jsonstring)
+{
+  yyjson_read_flag read_flg = YYJSON_READ_NOFLAG;
+  yyjson_write_flag write_flg = YYJSON_WRITE_NOFLAG;
+  yyjson_read_err read_err;
+  yyjson_write_err write_err;
+  yyjson_alc alc = {_priv_malloc, _priv_realloc, _priv_free, NULL};
+  yyjson_doc *doc = NULL;
+
+  char *serialized  = NULL;
+  size_t serialsize = 0;
+
+  if (!msr)
+    return MS_GENERROR;
+
+  if (jsonstring != NULL)
+  {
+    /* Parse JSON into immutable form */
+    if ((doc = yyjson_read_opts (jsonstring, strlen (jsonstring), read_flg, &alc, &read_err)) == NULL)
+    {
+      ms_log (2, "%s() Cannot parse extra header JSON: %s\n",
+              __func__, (read_err.msg) ? read_err.msg : "Unknown error");
+      return MS_GENERROR;
+    }
+
+    /* Serialize new JSON string */
+    serialized = yyjson_write_opts (doc, write_flg, &alc, &serialsize, &write_err);
+
+    if (serialized == NULL)
+    {
+      ms_log (2, "%s() Cannot write extra header JSON: %s\n",
+              __func__, (write_err.msg) ? write_err.msg : "Unknown error");
+      return MS_GENERROR;
+    }
+
+    if (serialsize > UINT16_MAX)
+    {
+      ms_log (2, "%s() New serialization size exceeds limit of %d bytes: %" PRIu64 "\n",
+              __func__, UINT16_MAX, (uint64_t)serialsize);
+      libmseed_memory.free (serialized);
+      return MS_GENERROR;
+    }
+  }
+
+  /* Set new extra headers, replacing existing headers */
+  if (msr->extra)
+    libmseed_memory.free (msr->extra);
+  msr->extra       = serialized;
+  msr->extralength = (uint16_t)serialsize;
+
+  return msr->extralength;
+}
+
 /**********************************************************************/ /**
  * @brief Print the extra header structure for the specified MS3Record.
  *

libmseed/extraheaders.h

@@ -3,7 +3,7 @@
  *
  * This file is part of the miniSEED Library.
  *
- * Copyright (c) 2023 Chad Trabant, EarthScope Data Services
+ * Copyright (c) 2024 Chad Trabant, EarthScope Data Services
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

libmseed/fileutils.c

@@ -3,7 +3,7 @@
  *
  * This file is part of the miniSEED Library.
  *
- * Copyright (c) 2023 Chad Trabant, EarthScope Data Services
+ * Copyright (c) 2024 Chad Trabant, EarthScope Data Services
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -55,6 +55,51 @@ libmseed_url_support (void)
 #endif
 } /* End of libmseed_url_support() */
 
+/*****************************************************************/ /**
+ * @brief Initialize ::MS3FileParam parameters for a file descriptor
+ *
+ * Initialize a ::MS3FileParam for reading from a specified \a fd
+ * (file descriptor).
+ *
+ * The ::MS3FileParam should be used with ms3_readmsr_r() or
+ * ms3_readmsr_selection().  Once all data has been read from the
+ * stream, it will be closed during the cleanup call of those routines.
+ *
+ * @param[in] fd File descriptor for input reading
+ *
+ * @returns Allocated ::MS3FileParam on success and NULL on error.
+ *
+ * \ref MessageOnError - this function logs a message on error
+ *********************************************************************/
+MS3FileParam *
+ms3_mstl_init_fd (int fd)
+{
+  MS3FileParam *msfp;
+
+  /* Initialize the read parameters if needed */
+  msfp = (MS3FileParam *)libmseed_memory.malloc (sizeof (MS3FileParam));
+
+  if (msfp == NULL)
+  {
+    ms_log (2, "%s(): Cannot allocate memory for MS3FileParam\n", __func__);
+    return NULL;
+  }
+
+  *msfp = (MS3FileParam)MS3FileParam_INITIALIZER;
+
+  msfp->input.type = LMIO_FD;
+  msfp->input.handle = fdopen (fd, "rb");
+
+  if (msfp->input.handle == NULL)
+  {
+    ms_log (2, "%s(): Cannot open file descriptor %d\n", __func__, fd);
+    libmseed_memory.free (msfp);
+    return NULL;
+  }
+
+  return msfp;
+}
+
 /*****************************************************************/ /**
  * @brief Read miniSEED records from a file or URL
  *
@@ -116,7 +161,7 @@ ms3_shift_msfp (MS3FileParam *msfp, int shift)
 {
   if (!msfp)
   {
-    ms_log (2, "Required argument not defined: 'msfp'\n");
+    ms_log (2, "%s(): Required input not defined: 'msfp'\n", __func__);
     return;
   }
 
@@ -223,7 +268,7 @@ ms3_readmsr_selection (MS3FileParam **ppmsfp, MS3Record **ppmsr, const char *msp
 
   if (!ppmsr || !ppmsfp)
   {
-    ms_log (2, "Required argument not defined: 'ppmsr' or 'ppmsfp'\n");
+    ms_log (2, "%s(): Required input not defined: 'ppmsr' or 'ppmsfp'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -236,7 +281,7 @@ ms3_readmsr_selection (MS3FileParam **ppmsfp, MS3Record **ppmsr, const char *msp
 
     if (msfp == NULL)
     {
-      ms_log (2, "Cannot allocate memory for MSFP\n");
+      ms_log (2, "Cannot allocate memory for MS3FileParam\n");
       return MS_GENERROR;
     }
 
@@ -306,7 +351,7 @@ ms3_readmsr_selection (MS3FileParam **ppmsfp, MS3Record **ppmsr, const char *msp
 
     if (strcmp (mspath, "-") == 0)
     {
-      msfp->input.type = LMIO_FILE;
+      msfp->input.type = LMIO_FD;
       msfp->input.handle = stdin;
     }
     else
@@ -642,7 +687,7 @@ ms3_readtracelist_selection (MS3TraceList **ppmstl, const char *mspath,
 
   if (!ppmstl)
   {
-    ms_log (2, "Required argument not defined: 'ppmstl'\n");
+    ms_log (2, "%s(): Required input not defined: 'ppmstl'\n", __func__);
     return MS_GENERROR;
   }
 
@@ -856,7 +901,7 @@ msr3_writemseed (MS3Record *msr, const char *mspath, int8_t overwrite,
 
   if (!msr || !mspath)
   {
-    ms_log (2, "Required argument not defined: 'msr' or 'mspath'\n");
+    ms_log (2, "%s(): Required input not defined: 'msr' or 'mspath'\n", __func__);
     return -1;
   }
 
@@ -917,7 +962,7 @@ mstl3_writemseed (MS3TraceList *mstl, const char *mspath, int8_t overwrite,
 
   if (!mstl || !mspath)
   {
-    ms_log (2, "Required argument not defined: 'msr' or 'mspath'\n");
+    ms_log (2, "%s(): Required input not defined: 'msr' or 'mspath'\n", __func__);
     return -1;
   }