v.0.5.0 added USV summary function, added sorted everywhere for Linux…

…, removed quantumrandom forever, changed appearance of GUI to simplify processing

README.md

@@ -1,4 +1,4 @@
-# usv-playpen v0.4.2
+# usv-playpen v0.5.0
 
 <br>
 <div align="center">
@@ -87,4 +87,4 @@ Run the GUI.
 python usv_playpen_gui.py
 ```
 
-Developed and tested in PyCharm Pro 2024.1.1, on macOS Sonoma 14.4 / Ubuntu 22.04 LTS/ Windows 10.
+Developed and tested in PyCharm Pro 2024.2.3, on macOS Sonoma 14.4 / Pop!_OS 22.04 / Windows 11.

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='usv-playpen',
-    version='0.4.2',
+    version='0.5.0',
     author='@bartulem',
     author_email='mimica.bartul@gmail.com',
     classifiers=[
@@ -41,7 +41,6 @@
                       'PIMS==0.6.1',
                       'PyQt6==6.7.0',
                       'requests==2.32.3',
-                      'quantumrandom==1.9.0',
                       'scipy==1.10.0',
                       'sleap-anipose==0.1.7',
                       'scikit-learn==1.5.0',

src/anipose_operations.py

@@ -314,11 +314,11 @@ def conduct_anipose_triangulation(self):
                                       disp_progress=self.input_parameter_dict['conduct_anipose_triangulation']['display_progress_bool'],
                                       constraints=self.input_parameter_dict['conduct_anipose_triangulation']['rigid_body_constraints'],
                                       constraints_weak=self.input_parameter_dict['conduct_anipose_triangulation']['weak_body_constraints'],
-                                      scale_smooth=self.input_parameter_dict['conduct_anipose_triangulation']['scale_smooth_len_weak'][0],
-                                      scale_length=self.input_parameter_dict['conduct_anipose_triangulation']['scale_smooth_len_weak'][1],
-                                      scale_length_weak=self.input_parameter_dict['conduct_anipose_triangulation']['scale_smooth_len_weak'][2],
-                                      reproj_error_threshold=self.input_parameter_dict['conduct_anipose_triangulation']['reprojection_error_loss'][0],
-                                      reproj_loss=self.input_parameter_dict['conduct_anipose_triangulation']['reprojection_error_loss'][1],
+                                      scale_smooth=self.input_parameter_dict['conduct_anipose_triangulation']['scale_smooth'],
+                                      scale_length=self.input_parameter_dict['conduct_anipose_triangulation']['weight_rigid'],
+                                      scale_length_weak=self.input_parameter_dict['conduct_anipose_triangulation']['weight_weak'],
+                                      reproj_error_threshold=self.input_parameter_dict['conduct_anipose_triangulation']['reprojection_error_threshold'],
+                                      reproj_loss=self.input_parameter_dict['conduct_anipose_triangulation']['regularization_function'],
                                       n_deriv_smooth=self.input_parameter_dict['conduct_anipose_triangulation']['n_deriv_smooth'])
         else:
 

src/behavioral_experiments.py

@@ -342,7 +342,7 @@ def conduct_behavioral_recording(self):
             recording_duration (int/float)
                 Recording duration; defaults to 20 (minutes).
             recording_codec (str)
-                Video codec to use; defaults to "lq".
+                Video codec to use; defaults to "hq".
             monitor_recording (bool)
                 To monitor or not to monitor the recording; defaults to True.
             monitor_specific_camera (bool)

src/extract_phidget_data.py

@@ -60,7 +60,7 @@ def prepare_data_for_analyses(self):
                 sub_directory = one_dir
                 break
 
-        phidget_file_list = glob.glob(f"{self.root_directory}{os.sep}video{os.sep}{sub_directory}{os.sep}*.json")
+        phidget_file_list = sorted(glob.glob(f"{self.root_directory}{os.sep}video{os.sep}{sub_directory}{os.sep}*.json"))
 
         # load raw phidget data
         phidget_data = []

src/file_loader.py

@@ -89,7 +89,7 @@ def load_wavefile_data(self):
 
         wave_data_dict = {}
         for one_dir in self.input_parameter_dict['wave_data_loc']:
-            for one_file in os.listdir(one_dir):
+            for one_file in sorted(os.listdir(one_dir)):
 
                 # additional conditional argument to reduce numbers of files loaded
                 if len(self.input_parameter_dict['load_wavefile_data']['conditional_arg']) == 0:

src/file_manipulation.py

@@ -103,14 +103,14 @@ def split_clusters_to_sessions(self):
         calibrated_sr_config.read(f"{self.exp_settings_dict['config_settings_directory']}{os.sep}calibrated_sample_rates_imec.ini")
 
         for one_root_dir in self.root_directory:
-            for ephys_dir in glob.glob(pathname=f"{one_root_dir.replace('Data', 'EPHYS')[:-7]}_imec*", recursive=True):
+            for ephys_dir in sorted(glob.glob(pathname=f"{one_root_dir.replace('Data', 'EPHYS')[:-7]}_imec*", recursive=True)):
 
                 probe_id = ephys_dir.split('_')[-1]
 
                 self.message_output(f"Working on getting spike times from clusters in: {ephys_dir}, started at {datetime.now()}.")
 
                 # load the changepoint .json file
-                with open(glob.glob(pathname=f'{ephys_dir}{os.sep}changepoints_info_*.json', recursive=True)[0], 'r') as binary_info_input_file:
+                with open(sorted(glob.glob(pathname=f'{ephys_dir}{os.sep}changepoints_info_*.json', recursive=True))[0], 'r') as binary_info_input_file:
                     binary_files_info = json.load(binary_info_input_file)
 
                     for session_key in binary_files_info.keys():
@@ -126,7 +126,7 @@ def split_clusters_to_sessions(self):
                     unit_count_dict[session_key] = {'good': 0, 'mua': 0}
 
                     # load info from camera_frame_count_dict
-                    with open(glob.glob(f"{binary_files_info[session_key]['root_directory']}{os.sep}video{os.sep}*_camera_frame_count_dict.json")[0], 'r') as frame_count_infile:
+                    with open(sorted(glob.glob(f"{binary_files_info[session_key]['root_directory']}{os.sep}video{os.sep}*_camera_frame_count_dict.json"))[0], 'r') as frame_count_infile:
                         esr_dict[session_key] = json.load(frame_count_infile)['median_empirical_camera_sr']
                         root_dict[session_key] = binary_files_info[session_key]['root_directory']
 
@@ -244,8 +244,8 @@ def concatenate_binary_files(self):
             concatenation_command = 'copy /b '
             for ord_idx, one_root_dir in enumerate(self.root_directory):
                 if os.path.isdir(f'{one_root_dir}{os.sep}ephys{os.sep}{probe_id}'):
-                    for one_file, one_meta in zip(list(pathlib.Path(f'{one_root_dir}{os.sep}ephys{os.sep}{probe_id}').glob(f"*{npx_file_type}.bin*")),
-                                                  list(pathlib.Path(f'{one_root_dir}{os.sep}ephys{os.sep}{probe_id}').glob(f"*{npx_file_type}.meta*"))):
+                    for one_file, one_meta in zip(sorted(list(pathlib.Path(f'{one_root_dir}{os.sep}ephys{os.sep}{probe_id}').glob(f"*{npx_file_type}.bin*"))),
+                                                  sorted(list(pathlib.Path(f'{one_root_dir}{os.sep}ephys{os.sep}{probe_id}').glob(f"*{npx_file_type}.meta*")))):
                         if one_file.is_file() and one_meta.is_file():
 
                             # parse metadata file for channel and headstage information
@@ -363,7 +363,7 @@ def multichannel_to_channel_audio(self):
         mc_audio_files = 0
 
         # find name origin for file naming purposes
-        name_origin = glob.glob(f"{self.root_directory}{os.sep}audio{os.sep}temp{os.sep}m_*_ch*.wav")[0].split('_')[2]
+        name_origin = sorted(glob.glob(f"{self.root_directory}{os.sep}audio{os.sep}temp{os.sep}m_*_ch*.wav"))[0].split('_')[2]
 
         # concatenate single channel files for master/slave
         separation_subprocesses = []
@@ -414,7 +414,7 @@ def hpss_audio(self):
         self.message_output(f"Harmonic-percussive source separation started at: {datetime.now().hour:02d}:{datetime.now().minute:02d}.{datetime.now().second:02d}")
         QTest.qWait(1000)
 
-        wav_file_lst = glob.glob(f'{self.root_directory}{os.sep}audio{os.sep}cropped_to_video{os.sep}*.wav')
+        wav_file_lst = sorted(glob.glob(f'{self.root_directory}{os.sep}audio{os.sep}cropped_to_video{os.sep}*.wav'))
 
         for one_wav_file in wav_file_lst:
             self.message_output(f"Working on file: {one_wav_file}")
@@ -510,8 +510,8 @@ def filter_audio_files(self):
             pathlib.Path(f"{self.root_directory}{os.sep}audio{os.sep}{one_dir}_filtered").mkdir(parents=True, exist_ok=True)
 
             filter_subprocesses = []
-            all_audio_files = glob.glob(f"{self.root_directory}{os.sep}audio{os.sep}{one_dir}{os.sep}"
-                                        f"*.{self.input_parameter_dict['filter_audio_files']['audio_format']}")
+            all_audio_files = sorted(glob.glob(f"{self.root_directory}{os.sep}audio{os.sep}{one_dir}{os.sep}"
+                                               f"*.{self.input_parameter_dict['filter_audio_files']['audio_format']}"))
 
             if len(all_audio_files) > 0:
                 for one_file in all_audio_files:
@@ -565,13 +565,12 @@ def concatenate_audio_files(self):
                 name_origin = list(data_dict.keys())[0].split('_')[1]
                 dim_1 = data_dict[list(data_dict.keys())[0]]['wav_data'].shape[0]
                 dim_2 = len(data_dict.keys())
-                data_type = data_dict[list(data_dict.keys())[0]]['dtype']
                 sr = data_dict[list(data_dict.keys())[0]]['sampling_rate']
                 complete_mm_file_name = (f"{self.root_directory}{os.sep}audio{os.sep}{audio_file_type}{os.sep}{name_origin}"
-                                         f"_concatenated_audio_{audio_file_type}_{sr}_{dim_1}_{dim_2}_{str(data_type).split('.')[-1][:-2]}.mmap")
+                                         f"_concatenated_audio_{audio_file_type}_{sr}_{dim_1}_{dim_2}_int16.mmap")
 
                 audio_mm_arr = np.memmap(filename=complete_mm_file_name,
-                                         dtype=data_type,
+                                         dtype='int16',
                                          mode='w+',
                                          shape=(dim_1, dim_2))
 

src/preprocess_data.py

@@ -8,13 +8,13 @@
 import traceback
 from datetime import datetime
 from anipose_operations import ConvertTo3D
+from das_inference import FindMouseVocalizations
 from extract_phidget_data import Gatherer
 from file_manipulation import Operator
 from prepare_cluster_job import PrepareClusterJob
 from preprocessing_plot import SummaryPlotter
 from send_email import Messenger
 from synchronize_files import Synchronizer
-from usv_inference import FindMouseVocalizations
 
 
 class Stylist:
@@ -212,6 +212,12 @@ def prepare_data_for_analyses(self):
                                                input_parameter_dict=self.input_parameter_dict,
                                                message_output=self.message_output).das_command_line_inference()
 
+                    # # # conduct DAS summary
+                    if self.input_parameter_dict['processing_booleans']['das_summarize']:
+                        FindMouseVocalizations(root_directory=one_directory,
+                                               input_parameter_dict=self.input_parameter_dict,
+                                               message_output=self.message_output).summarize_das_findings()
+
                     self.message_output(f"Preprocessing data in {one_directory} finished at: "
                                         f"{datetime.now().hour:02d}:{datetime.now().minute:02d}.{datetime.now().second:02d}")
 

src/preprocessing_plot.py

@@ -71,7 +71,7 @@ def preprocessing_summary(self, prediction_error_dict, phidget_data_dictionary):
         """
 
         # get the total number of frames in the video
-        json_loc = glob.glob(f"{self.root_directory}{os.sep}video{os.sep}*_camera_frame_count_dict.json")[0]
+        json_loc = sorted(glob.glob(f"{self.root_directory}{os.sep}video{os.sep}*_camera_frame_count_dict.json"))[0]
         with open(json_loc, 'r') as camera_count_json_file:
             duration_min = json.load(camera_count_json_file)['total_video_time_least']
 
@@ -97,7 +97,7 @@ def preprocessing_summary(self, prediction_error_dict, phidget_data_dictionary):
         med_temp = np.round(np.nanmedian(phidget_data_dictionary['temperature']), 2)
 
         # get audio information
-        wav_audio_files = glob.glob(f"{self.root_directory}{os.sep}audio{os.sep}cropped_to_video{os.sep}*.wav")
+        wav_audio_files = sorted(glob.glob(f"{self.root_directory}{os.sep}audio{os.sep}cropped_to_video{os.sep}*.wav"))
         with wave.open(wav_audio_files[0], 'rb') as example_audio_file:
             audio_sampling_rate = example_audio_file.getframerate()
             audio_sample_number = example_audio_file.getnframes()