Parse recordings once and reuse numeric parser state

.github/workflows/run_unix.yml

@@ -395,7 +395,13 @@ jobs:
       env:
         LD_LIBRARY_PATH: ${{ github.workspace }}/installed/lib
     - name: Signal Processing Regression Csharp
-      run: dotnet run --project $GITHUB_WORKSPACE/csharp_package/brainflow/examples/signal_processing_regression
+      run: |
+        project=$GITHUB_WORKSPACE/csharp_package/brainflow/examples/signal_processing_regression
+        dotnet build "$project" --configuration Release
+        if [[ "$RUNNER_OS" == "macOS" ]]; then
+          cp "$GITHUB_WORKSPACE/installed/lib/"*.dylib "$project/bin/Release/net8.0/"
+        fi
+        dotnet run --project "$project" --configuration Release --no-build
       env:
         LD_LIBRARY_PATH: ${{ github.workspace }}/installed/lib
     - name: Signal Processing Regression Rust

cpp_package/examples/signal_processing/src/signal_processing_regression.cpp

@@ -20,6 +20,7 @@ static void reject (const std::function<void ()> &action)
 }
 
 int main ()
+try
 {
     BrainFlowArray<double, 2> data (2, 256);
     reject ([&] { data.at (-1, 0); });
@@ -72,4 +73,10 @@ int main ()
     reject (
         [&] { DataFilter::perform_inverse_wavelet_transform (coefficients, lengths, 256, 3, 3); });
     std::cout << "Signal-processing binding validation passed." << std::endl;
+    return 0;
+}
+catch (const std::exception &error)
+{
+    std::cerr << "Signal-processing binding validation failed: " << error.what () << std::endl;
+    return 1;
 }

csharp_package/brainflow/brainflow/data_filter.cs

@@ -605,23 +605,24 @@ public static void write_file (double[,] data, string file_name, string file_mod
         /// <returns></returns>
         public static double[,] read_file (string file_name)
         {
-            int[] num_elements = new int[1];
-            int res = DataHandlerLibrary.get_num_elements_in_file (file_name, num_elements);
+            int[] handle = new int[1];
+            int[] rows = new int[1];
+            int[] cols = new int[1];
+            int res = DataHandlerLibrary.prepare_file_read (file_name, handle, rows, cols);
             if (res != (int)BrainFlowExitCodes.STATUS_OK)
-            {
                 throw new BrainFlowError (res);
+            try
+            {
+                double[,] data = new double[rows[0], cols[0]];
+                res = DataHandlerLibrary.copy_file_read (handle[0], data, data.Length);
+                if (res != (int)BrainFlowExitCodes.STATUS_OK)
+                    throw new BrainFlowError (res);
+                return data;
             }
-            double[] data_arr = new double[num_elements[0]];
-            int[] num_rows = new int[1];
-            int[] num_cols = new int[1];
-            res = DataHandlerLibrary.read_file (data_arr, num_rows, num_cols, file_name, num_elements[0]);
-            if (res != (int)BrainFlowExitCodes.STATUS_OK)
+            finally
             {
-                throw new BrainFlowError (res);
+                DataHandlerLibrary.release_file_read (handle[0]);
             }
-
-            double[,] result = data_arr.Reshape(num_rows[0], num_cols[0]);
-            return result;
         }
 
         /// <summary>

csharp_package/brainflow/brainflow/data_handler_library.cs

@@ -140,6 +140,12 @@ class DataHandlerLibrary64
         [DllImport ("DataHandler", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
         public static extern int read_file (double[] data, int[] num_rows, int[] num_cols, string file_name, int max_elements);
         [DllImport ("DataHandler", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
+        public static extern int prepare_file_read (string file_name, int[] handle, int[] rows, int[] cols);
+        [DllImport ("DataHandler", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
+        public static extern int copy_file_read (int handle, double[,] data, int capacity);
+        [DllImport ("DataHandler", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
+        public static extern int release_file_read (int handle);
+        [DllImport ("DataHandler", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
         public static extern int get_num_elements_in_file (string file_name, int[] num_elements);
         [DllImport ("DataHandler", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
         public static extern int perform_wavelet_transform (double[] data, int data_len, int wavelet, int decomposition_level, int extension, double[] output_data, int[] decomposition_lengths);
@@ -254,6 +260,12 @@ class DataHandlerLibrary32
         [DllImport ("DataHandler32", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
         public static extern int read_file (double[] data, int[] num_rows, int[] num_cols, string file_name, int max_elements);
         [DllImport ("DataHandler32", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
+        public static extern int prepare_file_read (string file_name, int[] handle, int[] rows, int[] cols);
+        [DllImport ("DataHandler32", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
+        public static extern int copy_file_read (int handle, double[,] data, int capacity);
+        [DllImport ("DataHandler32", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
+        public static extern int release_file_read (int handle);
+        [DllImport ("DataHandler32", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
         public static extern int get_num_elements_in_file (string file_name, int[] num_elements);
         [DllImport ("DataHandler32", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
         public static extern int perform_wavelet_transform (double[] data, int data_len, int wavelet, int decomposition_level, int extension, double[] output_data, int[] decomposition_lengths);
@@ -528,6 +540,42 @@ public static int perform_downsampling (double[] data, int len, int period, int
             return (int)BrainFlowExitCodes.GENERAL_ERROR;
         }
 
+        public static int prepare_file_read (string file_name, int[] handle, int[] rows, int[] cols)
+        {
+            switch (PlatformHelper.get_library_environment ())
+            {
+                case LibraryEnvironment.x64:
+                    return DataHandlerLibrary64.prepare_file_read (file_name, handle, rows, cols);
+                case LibraryEnvironment.x86:
+                    return DataHandlerLibrary32.prepare_file_read (file_name, handle, rows, cols);
+            }
+            return (int)BrainFlowExitCodes.GENERAL_ERROR;
+        }
+
+        public static int copy_file_read (int handle, double[,] data, int capacity)
+        {
+            switch (PlatformHelper.get_library_environment ())
+            {
+                case LibraryEnvironment.x64:
+                    return DataHandlerLibrary64.copy_file_read (handle, data, capacity);
+                case LibraryEnvironment.x86:
+                    return DataHandlerLibrary32.copy_file_read (handle, data, capacity);
+            }
+            return (int)BrainFlowExitCodes.GENERAL_ERROR;
+        }
+
+        public static int release_file_read (int handle)
+        {
+            switch (PlatformHelper.get_library_environment ())
+            {
+                case LibraryEnvironment.x64:
+                    return DataHandlerLibrary64.release_file_read (handle);
+                case LibraryEnvironment.x86:
+                    return DataHandlerLibrary32.release_file_read (handle);
+            }
+            return (int)BrainFlowExitCodes.GENERAL_ERROR;
+        }
+
         public static int read_file (double[] data, int[] num_rows, int[] num_cols, string file_name, int max_elements)
         {
             switch (PlatformHelper.get_library_environment ())

csharp_package/brainflow/examples/signal_processing_regression/Program.cs

@@ -32,6 +32,18 @@ static void CheckMLSnapshot(string modelPath)
 
     static void Main(string[] args)
     {
+        string path = System.IO.Path.GetTempFileName();
+        try
+        {
+            System.IO.File.WriteAllText(path, "1,2\n3,4");
+            var loaded = DataFilter.read_file(path);
+            if (loaded.GetLength(0) != 2 || loaded.GetLength(1) != 2 ||
+                loaded[0, 1] != 3 || loaded[1, 0] != 2)
+                throw new Exception("File read layout changed");
+            System.IO.File.WriteAllText(path, "1,,2");
+            Reject(() => DataFilter.read_file(path));
+        }
+        finally { System.IO.File.Delete(path); }
         var data = new double[2, 256];
         Reject(() => DataFilter.perform_fft(data, 1, 0, 512, 0));
         Reject(() => DataFilter.get_psd(data, 1, 0, 512, 256, 0));

python_package/brainflow/data_filter.py

@@ -319,6 +319,18 @@ def __init__(self):
             ctypes.c_char_p
         ]
 
+        self.prepare_file_read = self.lib.prepare_file_read
+        self.prepare_file_read.restype = ctypes.c_int
+        self.prepare_file_read.argtypes = [ctypes.c_char_p, ndpointer(ctypes.c_int32),
+                                          ndpointer(ctypes.c_int32), ndpointer(ctypes.c_int32)]
+        self.copy_file_read = self.lib.copy_file_read
+        self.copy_file_read.restype = ctypes.c_int
+        self.copy_file_read.argtypes = [ctypes.c_int,
+            ndpointer(ctypes.c_double, ndim=1, flags=('C_CONTIGUOUS', 'WRITEABLE')), ctypes.c_int]
+        self.release_file_read = self.lib.release_file_read
+        self.release_file_read.restype = ctypes.c_int
+        self.release_file_read.argtypes = [ctypes.c_int]
+
         self.get_num_elements_in_file = self.lib.get_num_elements_in_file
         self.get_num_elements_in_file.restype = ctypes.c_int
         self.get_num_elements_in_file.argtypes = [
@@ -1726,24 +1738,21 @@ def read_file(cls, file_name: str):
         except BaseException:
             file = file_name
 
-        num_elements = numpy.zeros(1).astype(numpy.int32)
-        res = DataHandlerDLL.get_instance().get_num_elements_in_file(file, num_elements)
-        if res != BrainFlowExitCodes.STATUS_OK.value:
-            raise BrainFlowError('unable to determine number of elements in file', res)
-
-        data_arr = numpy.zeros(num_elements[0]).astype(numpy.float64)
-        num_rows = numpy.zeros(1).astype(numpy.int32)
-        num_cols = numpy.zeros(1).astype(numpy.int32)
-
-        res = DataHandlerDLL.get_instance().read_file(data_arr, num_rows, num_cols, file, num_elements[0])
+        native = DataHandlerDLL.get_instance()
+        handle = numpy.zeros(1, dtype=numpy.int32)
+        rows = numpy.zeros(1, dtype=numpy.int32)
+        cols = numpy.zeros(1, dtype=numpy.int32)
+        res = native.prepare_file_read(file, handle, rows, cols)
         if res != BrainFlowExitCodes.STATUS_OK.value:
             raise BrainFlowError('unable to read file', res)
-
-        if len(num_rows) == 0 or len(num_cols) == 0:
-            return None
-
-        data_arr = data_arr[0:num_rows[0] * num_cols[0]].reshape(num_rows[0], num_cols[0])
-        return data_arr
+        try:
+            data = numpy.empty(int(rows[0]) * int(cols[0]), dtype=numpy.float64)
+            res = native.copy_file_read(handle[0], data, data.size)
+            if res != BrainFlowExitCodes.STATUS_OK.value:
+                raise BrainFlowError('unable to read file', res)
+            return data.reshape(rows[0], cols[0])
+        finally:
+            native.release_file_read(handle[0])
 
     @classmethod
     def get_version(cls) -> str:

python_package/examples/tests/signal_processing_regression.py

@@ -179,12 +179,34 @@ def test_file_round_trip_preserves_full_precision_and_wide_rows(self):
     def test_file_parser_rejects_partial_tokens_and_ragged_rows(self):
         with tempfile.TemporaryDirectory() as directory:
             path = Path(directory) / 'invalid.tsv'
-            for content in ('1x\t2\n', '1\t2\n3\n', '1\t2\t\n', '1\tNaN\n'):
+            for content in ('1x\t2\n', '1\t2\n3\n', '1\t2\t\n', '1\tNaN\n', '', ' \n', '1,,2', '1\t\t2',
+                            ',1', '1,', '1,2,  ', '1 2', '1e999,2', '1,inf', '1e,2'):
                 with self.subTest(content=content):
                     path.write_text(content)
                     self.assert_invalid(DataFilter.read_file, str(path))
 
+    def test_file_snapshot_survives_file_changes_and_checks_capacity(self):
+        native = DataHandlerDLL.get_instance()
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'data.csv'
+            path.write_text(' +1.25, -2e-3 \r\n3,4')
+            handle, rows, cols = (np.zeros(1, dtype=np.int32) for _ in range(3))
+            self.assertEqual(native.prepare_file_read(str(path).encode(), handle, rows, cols), 0)
+            try:
+                path.unlink()
+                self.assertEqual((rows[0], cols[0]), (2, 2))
+                output = np.full(4, 99.0)
+                self.assertEqual(native.copy_file_read(handle[0], output, 3), 9)
+                np.testing.assert_array_equal(output, np.full(4, 99.0))
+                self.assertEqual(native.copy_file_read(handle[0], output, 4), 0)
+                np.testing.assert_array_equal(output, [1.25, 3, -0.002, 4])
+            finally:
+                self.assertEqual(native.release_file_read(handle[0]), 0)
+            self.assertEqual(native.copy_file_read(handle[0], output, 4), 13)
+            self.assertEqual(native.release_file_read(handle[0]), 13)
+
     def test_native_file_read_enforces_actual_capacity_without_writing(self):
+        # The legacy API remains available alongside snapshots.
         with tempfile.TemporaryDirectory() as directory:
             path = Path(directory) / 'data.tsv'
             path.write_text('1\t2\n3\t4')
@@ -195,6 +217,41 @@ def test_native_file_read_enforces_actual_capacity_without_writing(self):
             self.assertEqual(result, BrainFlowExitCodes.INVALID_BUFFER_SIZE_ERROR.value)
             np.testing.assert_array_equal(output, np.full(3, 99.0))
 
+    def test_snapshot_copy_can_race_release_without_partial_output(self):
+        from concurrent.futures import ThreadPoolExecutor
+        from threading import Barrier
+
+        native = DataHandlerDLL.get_instance()
+        with tempfile.TemporaryDirectory() as directory, ThreadPoolExecutor(3) as executor:
+            path = Path(directory) / 'snapshot.tsv'
+            data = np.arange(8192, dtype=np.float64).reshape(4, 2048)
+            DataFilter.write_file(data, str(path), 'w')
+            for _ in range(20):
+                handle, rows, cols = (np.zeros(1, dtype=np.int32) for _ in range(3))
+                self.assertEqual(native.prepare_file_read(str(path).encode(), handle, rows, cols), 0)
+                start = Barrier(3)
+
+                def copy():
+                    output = np.full(data.size, -99.0)
+                    start.wait(timeout=5)
+                    result = native.copy_file_read(handle[0], output, output.size)
+                    if result == 0:
+                        np.testing.assert_array_equal(output, data.ravel())
+                    else:
+                        self.assertEqual(result, BrainFlowExitCodes.INVALID_ARGUMENTS_ERROR.value)
+                        np.testing.assert_array_equal(output, np.full(data.size, -99.0))
+
+                def release():
+                    start.wait(timeout=5)
+                    self.assertEqual(native.release_file_read(handle[0]), 0)
+
+                futures = [executor.submit(copy), executor.submit(copy), executor.submit(release)]
+                try:
+                    for future in futures:
+                        future.result(timeout=10)
+                finally:
+                    native.release_file_read(handle[0])
+
 
 if __name__ == '__main__':
     unittest.main()

src/data_handler/inc/data_handler.h

@@ -120,6 +120,14 @@ extern "C"
         const char *file_name, int *num_elements); // its an internal method for bindings its not
                                                    // available via high level api
 
+    // Parse once into an immutable snapshot. Dimensions use BrainFlow's channel-major layout.
+    // copy_file_read writes into caller-owned memory; capacity is measured in doubles.
+    // Always release a successful handle, including when allocation or copying fails.
+    SHARED_EXPORT int CALLING_CONVENTION prepare_file_read (
+        const char *file_name, int *handle, int *num_rows, int *num_cols);
+    SHARED_EXPORT int CALLING_CONVENTION copy_file_read (int handle, double *data, int capacity);
+    SHARED_EXPORT int CALLING_CONVENTION release_file_read (int handle);
+
     // platform types and methods
     SHARED_EXPORT int CALLING_CONVENTION get_version_data_handler (
         char *version, int *num_chars, int max_chars);