Draft PR for tracking experiment branch

.env.sample

@@ -1,2 +1,9 @@
 MINISCOPE_IO_BASE_DIR="~/.config/miniscope_io"
-MINISCOPE_IO_LOGS__LEVEL="INFO"
+MINISCOPE_IO_LOGS__LEVEL="INFO"
+MINISCOPE_IO_CSVWRITER_BUFFER=1000
+MINISCOPE_IO_SERIAL_BUFFER=100
+MINISCOPE_IO_FRAME_BUFFER=100
+MINISCOPE_IO_IMAGE_BUFFER=100
+MINISCOPE_IO_STREAM_HEADER_PLOT_HISTORY=1000
+MINISCOPE_IO_STREAM_HEADER_PLOT_UPDATE_MS=1000
+MINISCOPE_IO_STREAM_HEADER_PLOT_KEY="linked_list,frame_num,buffer_count,frame_buffer_count,timestamp"

.github/workflows/docs-test.yml

@@ -0,0 +1,33 @@
+name: Build and test documentation
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  test_docs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: pip install -e .[docs] pytest-md
+
+      - name: Build docs
+        working-directory: docs
+        env:
+          SPHINXOPTS: "-W --keep-going"
+        run: make html

docs/conf.py

@@ -41,7 +41,6 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
 html_theme = "furo"
-html_static_path = ["_static"]
 
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),

miniscope_io/data/config/WLMS_v02_160px.yml

@@ -0,0 +1,52 @@
+# capture device. "OK" (Opal Kelly) or "UART"
+device: "OK"
+
+# bitstream file to upload to Opal Kelly board
+bitstream: "XEM7310-A75/USBInterface-6_67mhz-J2_2-3v3-IEEE.bit"
+
+# COM port and baud rate is only required for UART mode
+port: null
+baudrate: null
+
+# Preamble for each data buffer.
+preamble: 0x12345678
+
+# Image format. StreamDaq will calculate buffer size, etc. based on these parameters
+frame_width: 160
+frame_height: 160
+pix_depth: 8
+
+# Buffer data format. These have to match the firmware value
+header_len: 384 # 12 * 32 (in bits)
+buffer_block_length: 10
+block_size: 512
+num_buffers: 32
+dummy_words: 10
+
+# Flags to flip bit/byte order when recovering headers and data. See model document for details.
+reverse_header_bits: True
+reverse_header_bytes: True
+reverse_payload_bits: True
+reverse_payload_bytes: True
+
+adc_scale:
+  ref_voltage: 1.1
+  bitdepth: 8
+  battery_div_factor: 5
+  vin_div_factor: 11.3
+
+runtime:
+  serial_buffer_queue_size: 10
+  frame_buffer_queue_size: 5
+  image_buffer_queue_size: 5
+  csv:
+    buffer: 100
+  plot:
+    keys:
+      - timestamp
+      - buffer_count
+      - frame_buffer_count
+      - battery_voltage
+      - input_voltage
+    update_ms: 1000
+    history: 500

miniscope_io/data/config/WLMS_v02_200px.yml

@@ -34,6 +34,8 @@ adc_scale:
   bitdepth: 8
   battery_div_factor: 5
   vin_div_factor: 11.3
+  battery_max_voltage: 10.0
+  vin_max_voltage: 20.0
 
 runtime:
   serial_buffer_queue_size: 10

miniscope_io/io.py

@@ -57,7 +57,10 @@ def append(self, data: List[Any]) -> None:
         data : List[Any]
             The data to be appended.
         """
-        data = [int(value) if isinstance(value, np.generic) else value for value in data]
+        data = [
+            float(value) if isinstance(value, (np.integer, np.floating)) else value
+            for value in data
+        ]
         self.buffer.append(data)
         if len(self.buffer) >= self.buffer_size:
             self.flush_buffer()

miniscope_io/models/stream.py

@@ -35,6 +35,16 @@ class ADCScaling(MiniscopeConfig):
         11.3,
         description="Voltage divider factor for the Vin voltage",
     )
+    battery_max_voltage: float = Field(
+        10.0,
+        description="Maximum voltage of the battery."
+        "Scaled battery voltage will be 0 if it is greater than this value",
+    )
+    vin_max_voltage: float = Field(
+        20.0,
+        description="Maximum voltage of the Vin"
+        "Scaled Vin voltage will be 0 if it is greater than this value",
+    )
 
     def scale_battery_voltage(self, voltage_raw: float) -> float:
         """
@@ -113,7 +123,8 @@ def battery_voltage(self) -> float:
         if self._adc_scaling is None:
             return self.battery_voltage_raw
         else:
-            return self._adc_scaling.scale_battery_voltage(self.battery_voltage_raw)
+            battery_voltage = self._adc_scaling.scale_battery_voltage(self.battery_voltage_raw)
+            return battery_voltage if battery_voltage < self._adc_scaling.battery_max_voltage else 0
 
     @computed_field
     def input_voltage(self) -> float:
@@ -123,7 +134,8 @@ def input_voltage(self) -> float:
         if self._adc_scaling is None:
             return self.input_voltage_raw
         else:
-            return self._adc_scaling.scale_input_voltage(self.input_voltage_raw)
+            vin_voltage = self._adc_scaling.scale_input_voltage(self.input_voltage_raw)
+            return vin_voltage if vin_voltage < self._adc_scaling.vin_max_voltage else 0
 
 
 class StreamDevRuntime(MiniscopeConfig):

miniscope_io/plots/headers.py

@@ -14,14 +14,11 @@
 
 try:
     import matplotlib.pyplot as plt
-except ImportError as e:
-    raise ImportError(
-        "matplotlib is not a required dependency of miniscope-io, "
-        "install it with the miniscope-io[plot] extra or manually in your environment :)"
-    ) from e
+except ImportError:
+    plt = None
 
 
-def buffer_count(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes:
+def buffer_count(headers: pd.DataFrame, ax: "plt.Axes") -> "plt.Axes":
     """
     Plot number of buffers by time
     """
@@ -35,7 +32,7 @@ def buffer_count(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes:
     return ax
 
 
-def dropped_buffers(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes:
+def dropped_buffers(headers: pd.DataFrame, ax: "plt.Axes") -> "plt.Axes":
     """
     Plot number of buffers by time
     """
@@ -45,7 +42,7 @@ def dropped_buffers(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes:
     return ax
 
 
-def timestamps(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes:
+def timestamps(headers: pd.DataFrame, ax: "plt.Axes") -> "plt.Axes":
     """
     Plot frame number against time
     """
@@ -60,7 +57,7 @@ def timestamps(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes:
     return ax
 
 
-def battery_voltage(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes:
+def battery_voltage(headers: pd.DataFrame, ax: "plt.Axes") -> "plt.Axes":
     """
     Plot battery voltage against time
     """
@@ -73,7 +70,7 @@ def battery_voltage(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes:
 
 def plot_headers(
     headers: pd.DataFrame, size: Optional[Tuple[int, int]] = None
-) -> (plt.Figure, plt.Axes):
+) -> ("plt.Figure", "plt.Axes"):
     """
     Plot the headers (generated from :meth:`.Frame.to_df` )
 
@@ -132,6 +129,13 @@ def __init__(
             history_length (int): Number of headers to plot
             update_ms (int): milliseconds between each plot update
         """
+        global plt
+        if plt is None:
+            raise ModuleNotFoundError(
+                "matplotlib is not a required dependency of miniscope-io, to use it, "
+                "install it manually or install miniscope-io with `pip install miniscope-io[plot]`"
+            )
+
         # If a single string is provided, convert it to a list with one element
         if isinstance(header_keys, str):
             header_keys = [header_keys]
@@ -147,7 +151,7 @@ def __init__(
 
     def _init_plot(
         self,
-    ) -> tuple[plt.Figure, dict[str, plt.Axes], dict[str, plt.Line2D]]:
+    ) -> tuple["plt.Figure", dict[str, "plt.Axes"], dict[str, "plt.Line2D"]]:
 
         # initialize matplotlib
         plt.ion()

miniscope_io/stream_daq.py

@@ -710,14 +710,6 @@ def capture(
                 update_ms=self.config.runtime.plot.update_ms,
             )
 
-        if metadata:
-            self._buffered_writer = BufferedCSVWriter(
-                metadata, buffer_size=self.config.runtime.csvwriter.buffer
-            )
-            self._buffered_writer.append(
-                list(StreamBufferHeader.model_fields.keys()) + ["unix_time"]
-            )
-
         try:
             for image, header_list in exact_iter(imagearray.get, None):
                 self._handle_frame(
@@ -797,6 +789,13 @@ def _handle_frame(
                     except Exception as e:
                         self.logger.exception(f"Exception plotting headers: \n{e}")
                 if metadata:
+                    if self._buffered_writer is None:
+                        self._buffered_writer = BufferedCSVWriter(
+                            metadata, buffer_size=self.config.runtime.csvwriter.buffer
+                        )
+                        self._buffered_writer.append(
+                            list(header.model_dump(warnings=False).keys()) + ["unix_time"]
+                        )
                     self.logger.debug("Saving header metadata")
                     try:
                         self._buffered_writer.append(