diff --git a/CFF_coffee_version.m b/CFF_coffee_version.m new file mode 100644 index 0000000..1240e47 --- /dev/null +++ b/CFF_coffee_version.m @@ -0,0 +1,23 @@ +function ver = CFF_coffee_version() +%CFF_COFFEE_VERSION Get version of CoFFee +% +% Get version of coffee. Using standard semantic versioning rules. +% See info on: +% https://semver.org/ +% https://interrupt.memfault.com/blog/release-versioning +% +% IMPORTANT NOTE FOR DEVELOPERS: Whenever you develop CoFFee and intend +% to release a new tag on git, please update this function appropriately +% before. Keep the existing version as a comment and add the new one as a +% new line above. Add the date. +% +% See also CFF_GET_CURRENT_FDATA_VERSION. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2022-2022; Last revision: 11-08-2022 + +ver = '2.0.0-alpha.1'; % 11/08/2022 + +end + diff --git a/custom_validation_functions/CFF_mustBeTwoNonnegativeUnsignedIntegers.m b/custom_validation_functions/CFF_mustBeTwoNonnegativeUnsignedIntegers.m new file mode 100644 index 0000000..822f7e0 --- /dev/null +++ b/custom_validation_functions/CFF_mustBeTwoNonnegativeUnsignedIntegers.m @@ -0,0 +1,8 @@ +function CFF_mustBeTwoNonnegativeUnsignedIntegers(a) +condition = isvector(a) && numel(a)==2 && isreal(a) && isnumeric(a) && all(ge(a,0)) && all(a==floor(a)); +if ~condition + eidType = 'mustBeTwoPositiveUnsignedIntegers:notTwoPositiveUnsignedIntegers'; + msgType = 'Input must be two positive unsigned integers.'; + throwAsCaller(MException(eidType,msgType)) +end +end \ No newline at end of file diff --git a/custom_validation_functions/CFF_mustBeVector.m b/custom_validation_functions/CFF_mustBeVector.m new file mode 100644 index 0000000..4d488e2 --- /dev/null +++ b/custom_validation_functions/CFF_mustBeVector.m @@ -0,0 +1,8 @@ +function CFF_mustBeVector(a) +condition = isreal(a) && isnumeric(a) && sum(size(a)==1)>=ndims(a)-1; +if ~condition + eidType = 'mustBeVector:notVector'; + msgType = 'Input must be vector.'; + throwAsCaller(MException(eidType,msgType)) +end +end \ No newline at end of file diff --git a/data/EM2040C/0001_20140213_052736_Yolla.all b/data/EM2040C/0001_20140213_052736_Yolla.all deleted file mode 100644 index a748df5..0000000 Binary files a/data/EM2040C/0001_20140213_052736_Yolla.all and /dev/null differ diff --git a/data/EM2040C/0002_20140213_052836_Yolla.all b/data/EM2040C/0002_20140213_052836_Yolla.all deleted file mode 100644 index eead01d..0000000 Binary files a/data/EM2040C/0002_20140213_052836_Yolla.all and /dev/null differ diff --git a/data/EM2040C/0003_20140213_052947_Yolla.all b/data/EM2040C/0003_20140213_052947_Yolla.all deleted file mode 100644 index 96a8274..0000000 Binary files a/data/EM2040C/0003_20140213_052947_Yolla.all and /dev/null differ diff --git a/docstring_template.m b/docstring_template.m new file mode 100644 index 0000000..b978abd --- /dev/null +++ b/docstring_template.m @@ -0,0 +1,40 @@ +function [output1,output2] = CFF_function_name(input1,input2,input3) +%CFF_FUNCTION_NAME One-line description +% +% See also CFF_OTHER_FUNCTION_NAME. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% XXXX-XXXX; Last revision: XX-XX-XXXX + + + +%CFF_FUNCTION_NAME One-line description of what the function performs +% +% Optional multiple lines of information giving more details about the +% function. The first line above (so-called H1 line) has no space before +% the function name and two spaces after. CFF_FUNCTION_NAME is written in +% upper-case throughout this docstring. One or multiple examples syntaxes +% follow. The docstring is completed by a "See also" section that allows +% help functions such as "help" or "doc" to automatically create +% hyperlinks. Separated from the docstring by a single empty line are the +% author(s) and information on last versions. +% +% [X,Y] = CFF_FUNCTION_NAME(A,B,C) returns the sum of A+B as X, and C as +% Y. Note the input and output variables are also written in upper case. +% This first syntax should show the basic use +% +% X = CFF_FUNCTION_NAME(A,B) returns the sum of A+B as X, since the +% additional input and outputs in this example are unecessary. This other +% syntax shows alterative functioning. +% +% CFF_FUNCTION_NAME(...,'parameter',VAL) is another syntax to introduce +% optional or paramter inputs. Since the basic inputs and outputs have +% already been discussed, they can be ommitted, so the text focuses +% exclusively on what the option does compared to the basic syntax. +% +% See also CFF_FUNCTION_NAME_1, CFF_OTHER_FUNCTION_NAME_2. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% XXXX-XXXX; Last revision: XX-XX-XXXX \ No newline at end of file diff --git a/fData_tools/CFF_fix_fData_paths.m b/fData_tools/CFF_fix_fData_paths.m new file mode 100644 index 0000000..d3f9718 --- /dev/null +++ b/fData_tools/CFF_fix_fData_paths.m @@ -0,0 +1,81 @@ +function [fData, flagPathsFixed] = CFF_fix_fData_paths(fData,rawFile) +%CFF_FIX_FDATA_PATHS Fix paths in converted data if files were moved +% +% When a raw data file is converted to a fData.mat file, the path of the +% source file (field ALLfilename) is saved, as well as the paths to the +% binary files containing water-column data (if that data type was +% converted). If you then move the data to another folder (even if moving +% appropriately both the source data, and the converted data), the +% absolute paths in fData are no longer correct. This function fixes it. +% +% FDATA = CFF_FIX_FDATA_PATHS(FDATA,RAWFILE) checks if +% the absolute paths in some fields of FDATA are correct, given the +% absolute path of the raw data file RAWFILE, and corrects them if +% necessary. The fixed FDATA is saved back on the drive. +% +% [FDATA, FLAGPATHSFIXED] = CFF_FIX_FDATA_PATHS(...) returns +% FLAGPATHSFIXED = 1 if a correction was performed, 0 otherwise. +% +% See also CFF_LOAD_CONVERTED_FILES. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 25-07-2022 + + +% path to converted data on disk +fDataFolder = CFF_converted_data_folder(rawFile); +fDataFile = fullfile(fDataFolder,'fData.mat'); + +% init flag indicating change +flagPathsFixed = 0; + +% grab source file names in fData +fDataSourceFile = fData.ALLfilename; +if ischar(fDataSourceFile) + fDataSourceFile = {fDataSourceFile}; +end + +% let's only deal with cell arrays, wether single or paired files +if ischar(rawFile) + rawFile = {rawFile}; +end + +% check that input raw file(s) match fData source file(s) +if ~isequal(sort(CFF_file_name(rawFile,1)),sort(CFF_file_name(fDataSourceFile,1))) + error('Names of source file(s) do not match those saved in fData. Please reconvert file.'); +end + +% check paths of source file(s) and fix if necessary +if ~isequal(sort(rawFile),sort(fDataSourceFile)) + fData.ALLfilename = sort(rawFile); + flagPathsFixed = 1; +end + +% WCD binary files fields +fields = fieldnames(fData); +fields = fields(startsWith(fields,{'WC_SBP' 'AP_SBP' 'X_SBP'})); + +% Check path of WCD binary file(s) and fix if necessary +for ii = 1:numel(fields) + field = fields{ii}; + for jj = 1:numel(fData.(field)) + if ~isempty(fData.(field){jj}) + [filepathSaved,name,ext] = fileparts(fData.(field){jj}.Filename); % path in fData + if ~strcmp(filepathSaved,fDataFolder) % compare with expected folder + fData.(field){jj}.Filename = fullfile(fDataFolder,[name ext]); % rename + flagPathsFixed = 1; + end + end + end +end + +% If anything was fixed +if flagPathsFixed + % update on disk + try + save(fDataFile,'-struct','fData','-v7.3'); + catch + warning('Wrong paths in fData were found and modified, but it was not possible to save the corrected fData back on the disk.'); + end +end diff --git a/fData_tools/CFF_get_datagramSource.m b/fData_tools/CFF_get_datagramSource.m new file mode 100644 index 0000000..3e9b840 --- /dev/null +++ b/fData_tools/CFF_get_datagramSource.m @@ -0,0 +1,60 @@ +function datagramSource = CFF_get_datagramSource(fData,varargin) +%CFF_GET_DATAGRAMSOURCE Get or set a datagramSource for a fData +% +% fData may have multiple datagram types that can be used for further +% processing (typically, seabed data and water-column data) but we can +% only process one at a time. datagramSource is the two-letter code +% defining which datagram type are being (or, to be) processed. This +% function allows deciding an appropriate datagramSource for unprocessed +% data, or returns the datagramSource for processed data. +% +% DS = CFF_GET_DATAGRAMSOURCE(FDATA) checks if FDATA is processed. If +% yes, it returns its datagramSource as DS. If not, it returns a +% suitable datagramSource DS for processing. +% +% CFF_GET_DATAGRAMSOURCE(FDATA, DS) checks if DS is a suitable +% datagramSource for (presumably unprocessed) FDATA. If yes, it returns +% DS. If not, it returns another datagramSource that is suitable. + +% See also CFF_COMPUTE_PING_NAVIGATION_V2. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 22-07-2022 + +% init output +datagramSource = []; + +if ~isempty(varargin) && ~isempty(varargin{1}) + % datagramSource requested in input + datagramSource = varargin{1}; +elseif isfield(fData,'MET_datagramSource') + % datagramSource not requested in input, but fData appears processed + % with a set datagramSource already + datagramSource = fData.MET_datagramSource; +end + +if ~isempty(datagramSource) + % check that there are indeed fields for that datagramSource + if ~isfield(fData,sprintf('%s_1P_Date',datagramSource)) + % if not, that datagramSource is unsuitable so reset it to empty + datagramSource = []; + end +end + +if isempty(datagramSource) + % if datagramSource still empty at this point, it means we need to find + % a suitable one based on fields available in fData. Test all suitable + % fields, by order of priority + if isfield(fData, 'AP_1P_Date') + datagramSource = 'AP'; + elseif isfield(fData, 'WC_1P_Date') + datagramSource = 'WC'; + elseif isfield(fData, 'X8_1P_Date') + datagramSource = 'X8'; + elseif isfield(fData, 'De_1P_Date') + datagramSource = 'De'; + else + error('can''t find a suitable datagramSource') + end +end \ No newline at end of file diff --git a/fData_tools/CFF_set_bottom_sample.m b/fData_tools/CFF_set_bottom_sample.m new file mode 100644 index 0000000..6a87e5c --- /dev/null +++ b/fData_tools/CFF_set_bottom_sample.m @@ -0,0 +1,10 @@ +function fData = CFF_set_bottom_sample(fData,bot) +%CFF_SET_BOTTOM_SAMPLE One-line description +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +datagramSource = CFF_get_datagramSource(fData); +fData.(sprintf('X_BP_bottomSample_%s',datagramSource)) = bot; %in sample number diff --git a/fData_tools/fData_version/CFF_get_current_fData_version.m b/fData_tools/fData_version/CFF_get_current_fData_version.m new file mode 100644 index 0000000..53a6f51 --- /dev/null +++ b/fData_tools/fData_version/CFF_get_current_fData_version.m @@ -0,0 +1,28 @@ +function ver = CFF_get_current_fData_version() +%CFF_GET_CURRENT_FDATA_VERSION Current version of fData. +% +% The fData format sometimes requires updating to implement novel +% features. Such changes imply that data that have been previously +% converted may not be compatible with newer versions of processing code. +% As a result, it is necessary to version the fData format. This function +% returns the current version of the fData format in order for code to +% test whether a converted data is up-to-date or if re-converting is +% advised. +% +% IMPORTANT NOTE FOR DEVELOPERS: Whenever you change the fData format, +% please update this function appropriately. Keep the existing version +% number as a comment and add the new one as a new line above. Add the +% date, and ideally a quick summary of changes. +% +% See also CFF_GET_FDATA_VERSION, CFF_coffee_version. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 11-08-2022 + +ver = '0.3'; % DD-MM-YYYY. Changes? +% ver = '0.2'; % DD-MM-YYYY. Changes? +% ver = '0.1'; % DD-MM-YYYY. Changes? +% ver = '0.0'; % DD-MM-YYYY. Changes? + +end \ No newline at end of file diff --git a/fData_tools/fData_version/CFF_get_fData_version.m b/fData_tools/fData_version/CFF_get_fData_version.m new file mode 100644 index 0000000..3a0be6a --- /dev/null +++ b/fData_tools/fData_version/CFF_get_fData_version.m @@ -0,0 +1,83 @@ +function fDataVersionList = CFF_get_fData_version(fDataInputList) +%CFF_GET_FDATA_VERSION Get the fData version of input fData +% +% fDataVersion = CFF_GET_FDATA_VERSION(fDataFilepath) returns the +% (string) version of the fData mat file whose (string) filepath is +% specified in input. +% +% fDataVersion = CFF_GET_FDATA_VERSION(fData) returns the (string) +% version of the input fData structure or matfile object. +% +% fDataVersionList = CFF_GET_FDATA_VERSION(fDataInputList) returns a cell +% array where each element is the (string) version of the corresponding +% element in the input cell array. Each element of the input cell array +% can be either a (string) filepath to a fData mat file, or a fData +% structure or matfile object. +% +% Note that oldest versions of fData dit not have a version stored in it, +% so if the input (or an input element) is 1) a filepath to a mat file +% with no version field, or 2) a struct (or matfile object) with no +% version field, the function will assume this is an old fData struct and +% returns the corresponding old version, i.e. '0.0'. +% If the input (or an input element) is not an existing mat file, struct, +% or matfile object, then the function returns empty. +% +% WARNING: do not confuse this function with +% CFF_GET_CURRENT_FDATA_VERSION, which returns the latest version of +% fData used by the converting code. +% +% See also CFF_GET_CURRENT_FDATA_VERSION. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 19-07-2022 + +% single input +if ~iscell(fDataInputList) + fDataInputList = {fDataInputList}; +end + +% init output +sz = size(fDataInputList); +fDataVersionList = cell(sz); + +% process by input +for iF = 1:numel(fDataInputList) + + % get that item + fDataInput = fDataInputList{iF}; + + if ischar(fDataInput) + % input is filepath + if strcmp(CFF_file_extension(fDataInput), '.mat') && isfile(fDataInput) + % file exists + matObj = matfile(fDataInput); + if isprop(matObj,'MET_Fmt_version') + fDataVersionList{iF} = matObj.MET_Fmt_version; + else + fDataVersionList{iF} = '0.0'; + end + end + elseif isstruct(fDataInput) + % input is fData structure + if isfield(fDataInput,'MET_Fmt_version') + fDataVersionList{iF} = fDataInput.MET_Fmt_version; + else + fDataVersionList{iF} = '0.0'; + end + elseif isa(fDataInput,'matlab.io.MatFile') + % input is matfile to the fData file + if isprop(fDataInput,'MET_Fmt_version') + fDataVersionList{iF} = fDataInput.MET_Fmt_version; + else + fDataVersionList{iF} = '0.0'; + end + + end + +end + +% return in case of single input +if numel(fDataVersionList) == 1 + fDataVersionList = fDataVersionList{1}; +end \ No newline at end of file diff --git a/fData_tools/fData_version/CFF_is_fData_version_current.m b/fData_tools/fData_version/CFF_is_fData_version_current.m new file mode 100644 index 0000000..d261dbf --- /dev/null +++ b/fData_tools/fData_version/CFF_is_fData_version_current.m @@ -0,0 +1,19 @@ +function bool = CFF_is_fData_version_current(fdata_input) +%CFF_IS_FDATA_VERSION_CURRENT Check if fData version of input is current +% +% Input can be either the filepath to a fData.mat file, OR a fData +% structure. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 24-05-2021 + +% version if the fData file +fdata_ver = CFF_get_fData_version(fdata_input); + +% current version for the conversion code +curr_ver = CFF_get_current_fData_version(); + +% match? +bool = strcmpi(fdata_ver,curr_ver); \ No newline at end of file diff --git a/functions/CFF_new_function_head_template.m b/functions/CFF_new_function_head_template.m deleted file mode 100644 index 4121afe..0000000 --- a/functions/CFF_new_function_head_template.m +++ /dev/null @@ -1,131 +0,0 @@ -%% this_function_name.m -% -% TODO: write short description of function -% -%% Help -% -% *USE* -% -% TODO: write longer description of function -% -% *INPUT VARIABLES* -% -% * |input_variable_1|: TODO: write description and info on variable -% -% *OUTPUT VARIABLES* -% -% * |output_variable_1|: TODO: write description and info on variable -% -% *RESEARCH NOTES* -% -% TODO: write research notes -% -% *NEW FEATURES* -% -% * YYYY-MM-DD: first version (Author). TODO: complete date and comment -% -% *EXAMPLE* -% -% TODO: write examples -% -% *AUTHOR, AFFILIATION & COPYRIGHT* -% -% Alexandre Schimel, NIWA. - -%% Function - - - - - - - - - - DESCRIPTION OF FIELDS BELOW: - - - - - - - - -%% this_function_name.m -% -% _This section contains a very short description of the function, for the -% user to know this is part of ESP3 and what it is. Example below to -% replace. Delete these lines._ -% -% Template of ESP3 function header. -% -%% Help -% -% *USE* -% -% _This section contains a more detailed description of what the function -% does and how to use it, for the interested user to have an overall -% understanding of its function. Example below to replace. Delete these -% lines._ -% -% This is a text file containing the basic comment template to add at the -% start of any new ESP3 function to serve as function help. -% -% *INPUT VARIABLES* -% -% _This section contains bullet points of input variables with description -% and information. Put input variable and other valid entries or defaults -% between | symbols so it shows as monospace. Information section to -% contain, in order: requirement (i.e. Required/Optional), valid type (e.g. -% Num, Positive num, char, 1xN cell array, etc.) and default value if -% there is one (e.g. Default: '10'). Example below to replace. Delete these -% lines._ -% -% * |input_variable_1|: Description (Information). -% * |input_variable_2|: Description (Information). -% * |input_variable_3|: Description (Information). -% -% *OUTPUT VARIABLES* -% -% _This section contains bullet points of output variables with description -% and information. See input variables for template. Example below to -% replace. Delete these lines._ -% -% * |output_variable_1|: Description (Information). -% * |output_variable_2|: Description (Information). -% -% *RESEARCH NOTES* -% -% _This section describes what features are temporary, needed future -% developments and paper references. Example below to replace. Delete these -% lines._ -% -% * research point 1 -% * research point 2 -% -% *NEW FEATURES* -% -% _This section contains dates and descriptions of major updates. Example -% below to replace. Delete these lines._ -% -% * YYYY-MM-DD: second version. Describes the update. -% * YYYY-MM-DD: first version. -% -% *EXAMPLE* -% -% _This section contains examples of valid function calls. Note that -% example lines start with 3 white spaces so that the publish function -% shows them correctly as matlab code. Example below to replace. Delete -% these lines._ -% -% example_use_1; % comment on what this does. -% example_use_2: % comment on what this line does. -% -% *AUTHOR, AFFILIATION & COPYRIGHT* -% -% _This last section contains at least author name and affiliation. Delete -% these lines._ -% -% Author, affiliation. - -%% Function \ No newline at end of file diff --git a/functions/files_and_folders/CFF_file_name.m b/functions/files_and_folders/CFF_file_name.m deleted file mode 100644 index af993ef..0000000 --- a/functions/files_and_folders/CFF_file_name.m +++ /dev/null @@ -1,37 +0,0 @@ -%% CFF_file_name.m -% -% Get name of file (without path or extension -% -%% Help -% -% *USE* -% -% CFF_file_extension(filename) returns the extension of filename. -% -% *INPUT VARIABLES* -% -% * |filename|: Required. String filename. -% -% *OUTPUT VARIABLES* -% -% * |ext|: String filename extension -% -% *DEVELOPMENT NOTES* -% -% *NEW FEATURES* -% -% * 2018-10-11: added header -% * YYYY-MM-DD: first version. XXX -% -% *EXAMPLE* -% -% ext = CFF_file_extension('test.mat'); % returns 'mat' -% -% *AUTHOR, AFFILIATION & COPYRIGHT* -% -% Alexandre Schimel, Waikato University, Deakin University, NIWA. - -%% Function -function name = CFF_file_name(filename) - -[~,name,~] = fileparts(filename); \ No newline at end of file diff --git a/functions/mbes_rawfiles_kongsberg/CFF_converted_data_folder.m b/functions/mbes_rawfiles_kongsberg/CFF_converted_data_folder.m deleted file mode 100644 index 208afb1..0000000 --- a/functions/mbes_rawfiles_kongsberg/CFF_converted_data_folder.m +++ /dev/null @@ -1,58 +0,0 @@ -%% CFF_converted_data_folder.m -% -% Gets the path to the folder for data converted by CoFFee from a raw -% filename. -% -%% Help -% -% *USE* -% -% wc_dir = CFF_converted_data_folder(files_full) returns the folder path -% X/Coffee_files/filename/ for filename X/filename.ext. ALso works with -% cell arrays of string filenames. -% -% *INPUT VARIABLES* -% -% * |files_full|: Required. Description (Information). XXX -% -% *OUTPUT VARIABLES* -% -% * |wc_dir|: Description (Information). XXX -% -% *DEVELOPMENT NOTES* -% -% * Formerly "get_wc_dir.m" then revamped -% -% *NEW FEATURES* -% -% * 2018-10-11: first version. -% -% *EXAMPLE* -% -% example_use_1; % comment on what this does. XXX -% example_use_2: % comment on what this line does. XXX -% -% *AUTHOR, AFFILIATION & COPYRIGHT* -% -% Yoann Ladroit, Alexandre Schimel, NIWA. - -%% Function -function wc_dir = CFF_converted_data_folder(files_full) - -if ischar(files_full) - files_full = {files_full}; -end - -% get file's path and filename -[filepath,name,~] = cellfun(@fileparts,files_full,'UniformOutput',0); - -% coffee folder -coffee_dir = 'Coffee_files'; -coffee_dir = repmat({coffee_dir},size(files_full)); - -% putting everything together -wc_dir = cellfun(@fullfile,filepath,coffee_dir,name,'UniformOutput',0); - -if numel(wc_dir) == 1 - wc_dir = cell2mat(wc_dir); -end \ No newline at end of file diff --git a/functions/mbes_rawfiles_kongsberg/CFF_read_all.m b/functions/mbes_rawfiles_kongsberg/CFF_read_all.m deleted file mode 100644 index 80fe770..0000000 --- a/functions/mbes_rawfiles_kongsberg/CFF_read_all.m +++ /dev/null @@ -1,319 +0,0 @@ -%% CFF_read_all.m -% -% Reads contents of one Kongsberg EM series binary .all or .wcd data file, -% or a pair of .all/.wcd files, allowing choice on which type of datagrams -% to parse. -% -%% Help -% -% *USE* -% -% Considering ALLfilename is a Kongsberg file (extension .all or .wcd) that -% exists, ALLdata = CFF_read_all(ALLfilename) reads all datagrams in -% ALLfilename and store them in ALLdata. -% -% ALLdata = CFF_read_all(ALLfilename,datagrams) reads only those -% datagrams in ALLfilename that are specified by datagrams, and store them -% in ALLdata. -% -% ALLdata = CFF_read_all(ALLfilename,'datagrams',datagrams) does the same. -% -% Considering ALLfilename is the common root filename of a .all/.wcd pair -% (that is, ALLfilename = 'myfile' for a myfile.all and myfile.wcd pair), -% then the above commands will extract the requested datagrams from the -% .wcd file and the remaining in the .all file. -% -% Note this function will extract all datagram types of interest. For more -% control (say you only want the first ten depth datagrams and the last -% position datagram), use CFF_read_all_from_fileinfo. -% -% *INPUT VARIABLES* -% -% * |ALLfilename|: Required. String filename to parse (extension in .all or -% .wcd and existing file) OR the common root filename of a .all/.wcd pair. -% * |datagrams|: Optional. character string, or cell array of character -% string, or numeric values designating the types of datagrams to be -% parsed. If character string or cell array of character string, the string -% must match the datagTypeText of the datagram. If numeric, it must matches -% the datagTypeNumber. The possible values are: -% datagTypeNumber = 49. datagTypeText = 'PU STATUS OUTPUT (31H)'; -% datagTypeNumber = 65. datagTypeText = 'ATTITUDE (41H)'; -% datagTypeNumber = 67. datagTypeText = 'CLOCK (43H)'; -% datagTypeNumber = 68. datagTypeText = 'DEPTH DATAGRAM (44H)'; -% datagTypeNumber = 72. datagTypeText = 'HEADING (48H)'; -% datagTypeNumber = 73. datagTypeText = 'INSTALLATION PARAMETERS - START (49H)'; -% datagTypeNumber = 78. datagTypeText = 'RAW RANGE AND ANGLE 78 (4EH)'; -% datagTypeNumber = 79. datagTypeText = 'QUALITY FACTOR DATAGRAM 79 (4FH)'; -% datagTypeNumber = 80. datagTypeText = 'POSITION (50H)'; -% datagTypeNumber = 82. datagTypeText = 'RUNTIME PARAMETERS (52H)'; -% datagTypeNumber = 83. datagTypeText = 'SEABED IMAGE DATAGRAM (53H)'; -% datagTypeNumber = 85. datagTypeText = 'SOUND SPEED PROFILE (55H)'; -% datagTypeNumber = 88. datagTypeText = 'XYZ 88 (58H)'; -% datagTypeNumber = 89. datagTypeText = 'SEABED IMAGE DATA 89 (59H)'; -% datagTypeNumber = 102. datagTypeText = 'RAW RANGE AND BEAM ANGLE (f) (66H)'; -% datagTypeNumber = 104. datagTypeText = 'DEPTH (PRESSURE) OR HEIGHT DATAGRAM (68H)'; -% datagTypeNumber = 105. datagTypeText = 'INSTALLATION PARAMETERS - STOP (69H)'; -% datagTypeNumber = 107. datagTypeText = 'WATER COLUMN DATAGRAM (6BH)'; -% datagTypeNumber = 110. datagTypeText = 'NETWORK ATTITUDE VELOCITY DATAGRAM 110 (6EH)'; -% datagTypeNumber = 114. datagTypeText = 'AMPLITUDE AND PHASE WC DATAGRAM 114 (72H)'; -% -% *OUTPUT VARIABLES* -% -% * |ALLdata|: structure containing the data. Each field corresponds a -% different type of datagram. The field |ALLfileinfo| is a structure -% containing information about datagrams in ALLfilename, with fields: -% * |ALLfilename|: input file name -% * |filesize|: file size in bytes -% * |datagsizeformat|: endianness of the datagram size field 'b' or 'l' -% * |datagramsformat|: endianness of the datagrams 'b' or 'l' -% * |datagNumberInFile|: number of datagram in file -% * |datagPositionInFile|: position of beginning of datagram in file -% * |datagTypeNumber|: for each datagram, SIMRAD datagram type in -% decimal -% * |datagTypeText|: for each datagram, SIMRAD datagram type -% description -% * |parsed|: 0 for each datagram at this stage. To be later turned to -% 1 for parsing -% * |counter|: the counter of this type of datagram in the file (ie -% first datagram of that type is 1 and last datagram is the total -% number of datagrams of that type) -% * |number|: the number/counter found in the datagram (usually -% different to counter) -% * |size|: for each datagram, datagram size in bytes -% * |syncCounter|: for each datagram, the number of bytes founds -% between this datagram and the previous one (any number different than -% zero indicates a sync error) -% * |emNumber|: EM Model number (eg 2045 for EM2040c) -% * |date|: datagram date in YYYMMDD -% * |timeSinceMidnightInMilliseconds|: time since midnight in msecs -% * |datagrams_parsed_idx|: array of logical values of the same dimension -% as input |datagrams| indicating which of these datagrams have been parsed -% (1) or not (0). If no datagrams were specified in input, this output is -% empty. -% -% *DEVELOPMENT NOTES* -% -% * Research notes for CFF_all_file_info.m and CFF_read_all_from_fileinfo.m -% apply. -% -% *NEW FEATURES* -% -% * 2018-10-31: updated to read pair of .all/.wcd files. -% * 2018-10-11: updated header before adding to Coffee v3 -% * 2017-06-28: first version. Adapated from CFF_convert_all_to_mat_v2.m -% -% *EXAMPLE* -% -% ALLfilename = '.\data\EM2040c\0001_20140213_052736_Yolla.all'; -% ALLdata = CFF_read_all(ALLfilename); % read all datagrams -% ALLdata = CFF_read_all(ALLfilename, 'ATTITUDE (41H)'); % read only attitude datagrams -% ALLdata = CFF_read_all(ALLfilename, {'ATTITUDE (41H)','POSITION (50H)'}); % read attitude and position datagrams -% ALLdata = CFF_read_all(ALLfilename, [65,80]); % same, but using datagram type numbers intead of names -% ALLdata = CFF_read_all(ALLfilename,'datagrams',[65,80]); % same, but using proper input variable name -% -% *AUTHOR, AFFILIATION & COPYRIGHT* -% -% Alexandre Schimel, Waikato University, Deakin University, NIWA. -% Yoann Ladroit, NIWA. - -%% Function -function [ALLdata,datagrams_parsed_idx] = CFF_read_all(ALLfilename, varargin) - - -%% Input arguments management using inputParser - -p = inputParser; - -% ALLfilename to parse as required argument. -% Check file existence -argName = 'ALLfilename'; -argCheck = @(x) CFF_check_ALLfilename(x); -addRequired(p,argName,argCheck); - -% datagrams as optional argument. -% Check that cell array -argName = 'datagrams'; -argDefault = []; -argCheck = @(x) isnumeric(x)||iscell(x)||(ischar(x)&&~strcmp(x,'datagrams')); % that last part allows the use of the couple name,param -addOptional(p,argName,argDefault,argCheck); - -% now parse inputs -parse(p,ALLfilename,varargin{:}); - -% and get input variables from parser -ALLfilename = p.Results.ALLfilename; -datagrams_to_parse = p.Results.datagrams; - - -%% PREP - -% if filename has no extension, build the full all/wcd filenames -if ~isempty(CFF_file_extension(ALLfilename)) - - ALLfilename = {ALLfilename}; - -else - - ALLfilename = CFF_get_Kongsberg_files(ALLfilename); - - % in case of a pair of all/wcd files, the order is important as this - % function only reads in the 2nd file what it could not find in the - % 1st. By default, we want the wcd file to be read first, and only grab - % from the all file what is needed and couldn't be found in the wcd - % file. Flipping it here rather than in CFF_get_Kongsberg_files because - % I want this function to output all/wcd by default and not wcd/all - if ~strcmp(CFF_file_extension(ALLfilename{1}),'.wcd') - ALLfilename = fliplr(ALLfilename); - end - -end - - -%% FIRST FILE - -% Get info in first (or only) file -info = CFF_all_file_info(ALLfilename{1}); - -if isempty(datagrams_to_parse) - % parse all datagrams in firt file - - info.parsed(:) = 1; - datagrams_parsed_in_first_file = unique(info.datagTypeNumber); - - datagrams_parsed_idx = []; - -else - % datagrams to parse are listed - - if isnumeric(datagrams_to_parse) - - % datagrams available - datagrams_available = unique(info.datagTypeNumber); - - % find which datagrams can be read here - datagrams_parsable_idx = ismember(datagrams_to_parse,datagrams_available); - - % if any, read those datagrams - if any(datagrams_parsable_idx) - idx = ismember(info.datagTypeNumber,datagrams_to_parse(datagrams_parsable_idx)); - info.parsed(idx) = 1; - datagrams_parsed_idx = datagrams_parsable_idx; - end - - elseif ischar(datagrams_to_parse) || iscell(datagrams_to_parse) - % datagrams is one or several datagTypeText - - if ischar(datagrams_to_parse) - datagrams_to_parse = {datagrams_to_parse}; - end - - % datagrams available - datagrams_available = unique(info.datagTypeText); - - % find which datagrams can be read here - datagrams_parsable_idx = ismember(datagrams_to_parse,datagrams_available); - - % if any, read those datagrams - if any(datagrams_parsable_idx) - idx = ismember(info.datagTypeText,datagrams_to_parse(datagrams_parsable_idx)); - info.parsed(idx) = 1; - datagrams_parsed_idx = datagrams_parsable_idx; - end - - end - -end - -% read data -ALLdata = CFF_read_all_from_fileinfo(ALLfilename{1}, info); - - - -%% SECOND FILE -% do only if we have a second file, and either we requested to read all -% datagrams (in which case, the second file might have datagrams not read -% in the first and we need to grab those) OR we requested a specific set of -% datagrams and didn't get them all from the first file. -if numel(ALLfilename)>1 && (isempty(datagrams_to_parse) || ~all(datagrams_parsed_idx)) - - % Get info in second file - info = CFF_all_file_info(ALLfilename{2}); - - if isempty(datagrams_to_parse) - % parse all datagrams in second file which we didn't get in the - % first one. - - % datagrams in second file - datagrams_available_in_second_file = unique(info.datagTypeNumber); - - % those in second file that were not in first - datagrams_to_parse_in_second_file = setdiff(datagrams_available_in_second_file,datagrams_parsed_in_first_file); - - % parse those - idx = ismember(info.datagTypeNumber,datagrams_to_parse_in_second_file); - info.parsed(idx) = 1; - - % for output - datagrams_parsed_idx = []; - - else - % datagrams to parse are listed - - if isnumeric(datagrams_to_parse) - - datagrams_available_in_second_file = unique(info.datagTypeNumber); - - % find which remaining datagram types can be read here - datagrams_to_parse_in_second_file_idx = ismember(datagrams_to_parse,datagrams_available_in_second_file) & ~datagrams_parsed_idx; - - % if any, read those datagrams - if any(datagrams_to_parse_in_second_file_idx) - - idx = ismember(info.datagTypeNumber,datagrams_to_parse(datagrams_to_parse_in_second_file_idx)); - info.parsed(idx) = 1; - datagrams_parsed_idx = datagrams_parsed_idx | datagrams_to_parse_in_second_file_idx; - - end - - elseif ischar(datagrams_to_parse) || iscell(datagrams_to_parse) - % datagrams is one or several datagTypeText - - datagrams_available_in_second_file = unique(info.datagTypeText); - - % find which remaining datagram types can be read here - datagrams_to_parse_in_second_file_idx = ismember(datagrams_to_parse,datagrams_available_in_second_file) & ~datagrams_parsed_idx; - - % if any, read those datagrams - if any(datagrams_to_parse_in_second_file_idx) - - idx = ismember(info.datagTypeText,datagrams_to_parse(datagrams_to_parse_in_second_file_idx)); - info.parsed(idx) = 1; - datagrams_parsed_idx = datagrams_parsed_idx | datagrams_to_parse_in_second_file_idx; - - end - - end - - end - - % read data in second file - ALLdata2 = CFF_read_all_from_fileinfo(ALLfilename{2}, info); - - % combine to data from first file - ALLdata = {ALLdata ALLdata2}; - - -end - - - - - - - - - - - - - diff --git a/general/CFF_Comms.m b/general/CFF_Comms.m new file mode 100644 index 0000000..97139cb --- /dev/null +++ b/general/CFF_Comms.m @@ -0,0 +1,281 @@ +classdef CFF_Comms < handle + %CFF_COMMS Information communication object. + % + % A CFF_COMMS object specifies if and how a function communicates on + % its internal state (progress, info, warnings, errors). It is + % initalized with a string specifying the type of communication: + % 'disp' will display communication as its own line in the command + % window. + % 'textprogressbar' will display communication in a text progress bar + % in the command window. + % 'waitbar' will display communication in a Matlab waitbar figure. + % 'oneline' will display communication as a dynamic (changing) + % single line. + % 'multilines' will display communication as dynamic (changing) + % multiple lines. + % '' (default) will not display any communication. + + % Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann + % Ladroit (NIWA, yoann.ladroit@niwa.co.nz) + % 2021-2022; Last revision: 22-07-2022 + + properties + Type (1,:) char {mustBeMember(Type,{'', 'disp','textprogressbar','waitbar','oneline','multilines'})} = '' + FigObj = [] + Msgs = {} % messages received (includes error) + Prog = {} % progress values received + end + + methods + function obj = CFF_Comms(inputArg) + %CFF_COMMS Construct an instance of this class + % Detailed explanation goes here + if nargin == 0 + obj.Type = ''; + else + obj.Type = inputArg; + end + end + + function start(obj,str) + %START Summary of this method goes here + % Detailed explanation goes here + + % record start message + obj.Msgs(end+1,:) = {datetime('now'), 'Start', str}; + + % display + switch obj.Type + case 'disp' + dispstr = [char(string(obj.Msgs{end,1},'HH:mm:ss')) ' - ' obj.Msgs{end,2} ' - ' obj.Msgs{end,3}]; + disp(dispstr); + case 'textprogressbar' + % init textprogressbar with a string + textprogressbar([obj.Msgs{end,3} ': ']); + case 'waitbar' + % init waitbar with title + obj.FigObj = waitbar(0,''); + obj.FigObj.Name = obj.Msgs{end,3}; + % init interpreter for future info messages + obj.FigObj.Children.Title.Interpreter = 'None'; + % init message of two line + set(obj.FigObj.Children.Title,'String',newline); + drawnow + case {'oneline','multilines'} + % init dipstat + dispstat('','init'); + % top line to print and keep + dispstr = [obj.Msgs{end,3} ' (started ' char(string(obj.Msgs{end,1})) '):']; + dispstat(dispstr,'keepthis'); + end + end + + function step(obj,str) + %STEP Summary of this method goes here + % Detailed explanation goes here + + % record step message + obj.Msgs(end+1,:) = {datetime('now'), 'Step', str}; + + switch obj.Type + case 'disp' + dispstr = [char(string(obj.Msgs{end,1},'HH:mm:ss')) ' - ' obj.Msgs{end,2} ' - ' obj.Msgs{end,3}]; + disp(dispstr); + case 'waitbar' + set(obj.FigObj.Children.Title,'String',sprintf('%s\n',obj.Msgs{end,3})); + drawnow; + case 'oneline' + % step message + dispstr = [obj.Msgs{end,3} '.']; + if obj.Prog{end,2}>0 + % estimated time to complete, from prior progress + durationPerStep = cellfun(@minus, obj.Prog(2:end,1), obj.Prog(1:end-1,1)); + nRemainingSteps = obj.Prog{end,3} - obj.Prog{end,2}; + ETC = median(durationPerStep).*nRemainingSteps; + dispstr = sprintf([dispstr newline 'Estimated time to complete: ' char(string(ETC))]); + end + % print with dispstat, to be overwritten + dispstat(dispstr); + case 'multilines' + % if there was a previous step, reprint and keep it + idxPrevStep = find(matches(string(obj.Msgs(1:end-1,2)),'Step'),1,'last'); + if ~isempty(idxPrevStep) + % first, get that last step message + dispstr = [obj.Msgs{idxPrevStep,3}]; + if matches(obj.Msgs(end-1,2),["Info","Error"]) + % next, add last info message if it exists + dispstr = [dispstr '. ' obj.Msgs{end-1,3}]; + end + % finally, add completion + dispstr = [dispstr ' (completed ' char(string(obj.Msgs{end,1})) ')']; + dispstat(dispstr,'keepthis'); + end + % step message + dispstr = [obj.Msgs{end,3} '.']; + if obj.Prog{end,2}>0 + % estimated time to complete, from prior progress + durationPerStep = cellfun(@minus, obj.Prog(2:end,1), obj.Prog(1:end-1,1)); + nRemainingSteps = obj.Prog{end,3} - obj.Prog{end,2}; + ETC = median(durationPerStep).*nRemainingSteps; + dispstr = sprintf([dispstr newline 'Estimated time to complete: ' char(string(ETC))]); + end + % print with dispstat, to be overwritten + dispstat(dispstr); + end + end + + function info(obj,str) + %INFO Summary of this method goes here + % Detailed explanation goes here + + % record info message + obj.Msgs(end+1,:) = {datetime('now'), 'Info', str}; + + switch obj.Type + case 'disp' + dispstr = [char(string(obj.Msgs{end,1},'HH:mm:ss')) ' - ' obj.Msgs{end,2} ' - ' obj.Msgs{end,3}]; + disp(dispstr); + case 'waitbar' + % get last step message + idx = find(strcmp(obj.Msgs(:,2),'Step'),1,'last'); + if isempty(idx) + stepStr = ''; + else + stepStr = obj.Msgs{idx,3}; + end + % set waitbar title + set(obj.FigObj.Children.Title,'String',sprintf('%s\n%s',stepStr,obj.Msgs{end,3})); + drawnow; + case {'oneline','multilines'} + % last step, and new info message + idxLastStep = find(matches(string(obj.Msgs(:,2)),'Step'),1,'last'); + dispstr = [obj.Msgs{idxLastStep,3} '. ' obj.Msgs{end,3}]; + if obj.Prog{end,2}>0 + % estimated time to complete, from prior progress + durationPerStep = cellfun(@minus, obj.Prog(2:end,1), obj.Prog(1:end-1,1)); + nRemainingSteps = obj.Prog{end,3} - obj.Prog{end,2}; + ETC = median(durationPerStep).*nRemainingSteps; + dispstr = sprintf([dispstr newline 'Estimated time to complete: ' char(string(ETC))]); + end + % print with dispstat, to be overwritten + dispstat(dispstr); + end + end + + function error(obj,str) + %ERROR Summary of this method goes here + % Detailed explanation goes here + + % record info message + obj.Msgs(end+1,:) = {datetime('now'), 'Error', str}; + + switch obj.Type + case 'disp' + dispstr = [char(string(obj.Msgs{end,1},'HH:mm:ss')) ' - ' obj.Msgs{end,2} ' - ' obj.Msgs{end,3}]; + disp(dispstr); + case 'waitbar' + % get last step message + idx = find(strcmp(obj.Msgs(:,2),'Step'),1,'last'); + if isempty(idx) + stepStr = ''; + else + stepStr = obj.Msgs{idx,3}; + end + % set waitbar title + set(obj.FigObj.Children.Title,'String',sprintf('%s\n%s',stepStr,obj.Msgs{end,3})); + drawnow; + case {'oneline','multilines'} + % last step, and new info message + idxLastStep = find(matches(string(obj.Msgs(:,2)),'Step'),1,'last'); + dispstr = [obj.Msgs{idxLastStep,3} '. ' obj.Msgs{end,3}]; + if obj.Prog{end,2}>0 + % estimated time to complete, from prior progress + durationPerStep = cellfun(@minus, obj.Prog(2:end,1), obj.Prog(1:end-1,1)); + nRemainingSteps = obj.Prog{end,3} - obj.Prog{end,2}; + ETC = median(durationPerStep).*nRemainingSteps; + dispstr = sprintf([dispstr newline 'Estimated time to complete: ' char(string(ETC))]); + end + % print with dispstat, to be overwritten + dispstat(dispstr); + end + end + + + function finish(obj,str) + %FINISH Summary of this method goes here + % Detailed explanation goes here + + % record finish message + obj.Msgs(end+1,:) = {datetime('now'), 'Finish', str}; + + % complete + switch obj.Type + case 'disp' + dispstr = [char(string(obj.Msgs{end,1},'HH:mm:ss')) ' - ' obj.Msgs{end,2} ' - ' obj.Msgs{end,3}]; + disp(dispstr); + case 'textprogressbar' + textprogressbar(100); + textprogressbar([' ' obj.Msgs{end,3}]); + case 'waitbar' + waitbar(1,obj.FigObj,obj.Msgs{end,3}); + pause(0.1); + close(obj.FigObj); + case 'oneline' + dispstr = [obj.Msgs{end,3} ' (completed ' char(string(obj.Msgs{end,1})) '). Total processing time: ' char(string(obj.Msgs{end,1}-obj.Msgs{1,1}))]; + dispstat(dispstr,'keepthis'); + case 'multilines' + % if there was a previous step, reprint and keep it + idxLastStep = find(matches(string(obj.Msgs(:,2)),'Step'),1,'last'); + dispstr = [obj.Msgs{idxLastStep,3}]; + if matches(obj.Msgs(end-1,2),["Info","Error"]) + % next, add last info message if it exists + dispstr = [dispstr '. ' obj.Msgs{end-1,3}]; + end + dispstat(dispstr,'keepthis'); + % complete progress + dispstr = [obj.Msgs{end,3} ' (completed ' char(string(obj.Msgs{end,1})) '). Total processing time: ' char(string(obj.Msgs{end,1}-obj.Msgs{1,1}))]; + dispstat(dispstr,'keepthis'); + end + + % show received error messages, if any, and + % corresponding steps + if ~isempty(obj.Type) && any(strcmp(obj.Msgs(:,2),'Error')) + idxStp = cellfun(@(x) strcmp(x,'Step'), obj.Msgs(:,2)); + idxErr = cellfun(@(x) strcmp(x,'Error'), obj.Msgs(:,2)); + idxStp = ismember((1:numel(idxStp))', unique(arrayfun(@(idx) find(idxStp(1:idx-1),1,'last') ,find(idxErr)))); + idxFinal = num2cell(find(idxStp|idxErr)); + errlog = cellfun(@(idx) [char(string(obj.Msgs{idx,1},'HH:mm:ss')) ' - ' obj.Msgs{idx,2} ' - ' obj.Msgs{idx,3}], idxFinal,'UniformOutput', false); + errlog = strjoin(errlog,newline); + switch obj.Type + case 'waitbar' + wardlgTxt = sprintf('Error messages were received:\n'); + warndlg([wardlgTxt, newline, errlog],'Warning'); + otherwise + % print to screen + fprintf('Error messages were received:\n'); + disp(errlog); + end + end + + end + + function progress(obj,ii,N) + %PROGRESS Summary of this method goes here + % Detailed explanation goes here + + % record progress values + obj.Prog(end+1,:) = {datetime('now'), ii, N}; + + switch obj.Type + case 'disp' + fprintf('#%.10g/%.10g\n',ii,N); + case 'textprogressbar' + textprogressbar(100.*ii./N); + case 'waitbar' + waitbar(ii./N,obj.FigObj); + end + end + + end +end + diff --git a/general/CFF_get_bottom_sample.m b/general/CFF_get_bottom_sample.m new file mode 100644 index 0000000..75cc77b --- /dev/null +++ b/general/CFF_get_bottom_sample.m @@ -0,0 +1,46 @@ +function BP_bottomSample = CFF_get_bottom_sample(fData,varargin) +%CFF_GET_BOTTOM_SAMPLE Get the bottom sample (per ping and beam) in fData +% +% Gets the number of the sample corresponding to the bottom detect for +% each ping and beam. +% +% BP_bottomSample = CFF_GET_BOTTOM_SAMPLE(fData) returns the processed +% bottom sample (i.e. possibly filtered) if it exists, otherwise the raw +% bottom sample, from the datagram source in fData. +% +% BP_bottomSample = +% CFF_GET_BOTTOM_SAMPLE(fData,'datagramSource',datagramSource) forces the +% use of the input datagramSource, with datagramSource being one of +% either 'WC', 'AP', 'De', or 'X8'. +% +% BP_bottomSample = +% CFF_GET_BOTTOM_SAMPLE(fData,'which','raw') forces the return of the raw +% bottom sample. + + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 11-11-2021 + + +%% INPUT PARSING +p = inputParser; +addRequired(p,'fData',@isstruct); +addOptional(p,'datagramSource', CFF_get_datagramSource(fData),@(x) ismember(x,{'WC','AP','De','X8'})); +addOptional(p,'which','processed',@(x) ismember(x,{'raw','processed'})); +parse(p,fData,varargin{:}); +datagramSource = p.Results.datagramSource; +which = p.Results.which; +clear p + + +%% EXTRACT +if strcmp(which,'processed') && isfield(fData,sprintf('X_BP_bottomSample_%s',datagramSource)) + % A processed bottom sample already exists (possibly filtered). + % Extract that. + BP_bottomSample = fData.(sprintf('X_BP_bottomSample_%s',datagramSource)); % in sample number +else + % Extracting raw bottom sample + BP_bottomSample = fData.(sprintf('%s_BP_DetectedRangeInSamples',datagramSource)); % in sample number + BP_bottomSample(BP_bottomSample==0) = NaN; +end \ No newline at end of file diff --git a/general/CFF_is_field_or_prop.m b/general/CFF_is_field_or_prop.m new file mode 100644 index 0000000..9040fee --- /dev/null +++ b/general/CFF_is_field_or_prop.m @@ -0,0 +1,27 @@ +function out = CFF_is_field_or_prop(structOrObj,fieldOrPropName) +%CFF_IS_FIELD_OR_PROP Test if field or property is in struct or object +% +% CFF_IS_FIELD_OR_PROP(structOrObj,fieldOrPropName) returns 1 if +% "structOrObj" is a structure and "fieldOrPropName" is one of its +% fields, OR if "structOrObj" is an object and "fieldOrPropName" is one +% of its fields properties. +% +% Created so as to test for the existence of a field in a structure even +% when that structure was not loaded in memory but mapped as a MapFile +% object. + +% See also ISSTRUCT, ISPROP. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2021-2021; Last revision: 01-12-2021 + + +if isstruct(structOrObj) + out = isfield(structOrObj,fieldOrPropName); +elseif isobject(structOrObj) + out = isprop(structOrObj,fieldOrPropName); +else + out = NaN; +end + diff --git a/general/CFF_is_parallel_computing_available.m b/general/CFF_is_parallel_computing_available.m new file mode 100644 index 0000000..5a56386 --- /dev/null +++ b/general/CFF_is_parallel_computing_available.m @@ -0,0 +1,148 @@ +function [gpuAvail,info] = CFF_is_parallel_computing_available() +%CFF_IS_PARALLEL_COMPUTING_AVAILABLE Suitability of GPU for parallel computing +% +% This function checks 1) if you have the appropriate licence for +% parallel computing, 2) if you have a GPU, 3) if that GPU is suitable +% for parall computing, and 4) if its drivers are up to date given your +% MATLAB release. +% +% [GPUAVAIL, INFO] = CFF_IS_PARALLEL_COMPUTING_AVAILABLE() returns +% GPUAVAIL = 1 if you can use parallel computing, 0 if not. The second +% output INFO is a text string providing information. +% +% NOTES: +% * modified from function get_gpu_comp_stat.m. This function does not +% return memory available on the GPU anymore. Use CFF_MEMORY_AVAILABLE +% for this. +% * the look-up table for MATLAB Release vs CUDA Toolkit driver version +% must be edited manually when new MATLAB are released. See +% https://mathworks.com/help/releases/R2021b/parallel-computing/gpu-support-by-release.html. +% +% See also CFF_MEMORY_AVAILABLE. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 03-08-2022 + + +% Each new release of MATLAB requires updated CUDA toolkit, with a driver +% version at least as recent as indicated on the table at: +% https://mathworks.com/help/releases/R2021b/parallel-computing/gpu-support-by-release.html. +% Reproducing the table here, but it must be updated for later MATLAB +% releases +lookUpTable = {... + 'R2021b - 11.0',... + 'R2021a - 11.0',... + 'R2020b - 10.2',... + 'R2020a - 10.1',... + 'R2019b - 10.1',... + 'R2019a - 10.0',... + 'R2018b - 9.1',... + 'R2018a - 9.0',... + 'R2017b - 8.0',... + 'R2017a - 8.0',... + 'R2016b - 7.5',... + 'R2016a - 7.5',... + 'R2015b - 7.0',... + 'R2015a - 6.5',... + 'R2014b - 6.0',... + 'R2014a - 5.5',... + 'R2013b - 5.0',... + 'R2013a - 5.0',... + 'R2012b - 4.2',... + 'R2012a - 4.0',... + 'R2011b - 4.0'}; + +% 1) check if there is a licence for parallel computing +if license('checkout','Distrib_Computing_Toolbox') + + % 2) check if there is a GPU + try + D = gpuDevice; % get the default GPU device + catch err + % cannot detect a GPU. Could be a CUDA driver error + if contains((err.message),'CUDA') || contains((err.message),'graphics driver') + gpuAvail = 0; + info = ['No GPU detected, but it could just be a driver issue.',... + ' If you do have a GPU, update its driver and try this function again.']; + else + gpuAvail = 0; + info = 'No GPU detected.'; + end + end + + if exist('D','var') + % 3) check GPU suitability for parallel computing + if str2double(D.ComputeCapability) >= 3 && ... % Computational capability of the CUDA device. Must meet required specification. + D.SupportsDouble && ... % Indicates if this device can support double precision operations. + D.DeviceSupported > 0 % Indicates if toolbox can use this device. Not all devices are supported; for example, if their ComputeCapability is insufficient, the toolbox cannot use them. + + % 4) check that GPU driver is up to date for current MATLAB + % release + curMatRel = version('-release'); + k = contains(lookUpTable,curMatRel); + if any(k) + % we have a match in the lookup table + minToolkitVersion = str2double(extractAfter(lookUpTable(k),'-')); + if D.DriverVersion > 10 && ... % The CUDA device driver version currently in use. Must meet required specification. + D.ToolkitVersion >= minToolkitVersion % Version of the CUDA toolkit used by the current release of MATLAB. + % all good, proceed + gpuAvail = 1; + info = sprintf('Your GPU is supported (%s).',D.Name); + else + gpuAvail = 0; + info = sprintf(['Your GPU is supported (%s), but you must',... + ' update its driver to be able to use it.'],D.Name); + end + elseif str2double(curMatRel(1:4)) >= str2double(lookUpTable{1}(2:5)) + % no match because MATLAB is too recent + minToolkitVersion = str2double(extractAfter(lookUpTable(1),'-')); + if D.DriverVersion > 10 && ... % The CUDA device driver version currently in use. Must meet required specification. + D.ToolkitVersion >= minToolkitVersion % Version of the CUDA toolkit used by the current release of MATLAB. + % all good, proceed + gpuAvail = 1; + info = sprintf(['Your GPU is supported (%s), and its',... + ' driver version is suitable for the latest MATLAB release (%s)',... + ' for which we have information available in this function (%s).',... + ' However, your MATLAB release is MORE RECENT (%s), so we cannot',... + ' tell if your driver is sufficiently up-to-date to actually allow GPU computing.',... + ' If you experience issues, try updating your driver. For more certainty about',... + ' GPU support, update the look-up table in this function with the most recent',... + ' information online.'],D.Name,lookUpTable{1}(2:6),upper(mfilename),curMatRel); + else + gpuAvail = 0; + info = sprintf(['Your GPU is supported (%s), but you must',... + ' update its driver to be able to use it. Note: your MATLAB release (%s)',... + ' is more recent than the latest one (%s) for which we have',... + ' information available in this function (%s). Update the',... + ' look-up table in this function with the most recent',... + ' information online.'],D.Name,curMatRel,lookUpTable{1}(2:6),upper(mfilename)); + end + else + % if here, must be an antique MATLAB + gpuAvail = 0; + info = sprintf(['Your GPU is supported (%s), but your MATLAB release (%s)',... + ' is OLDER than the earliest release (%s) for which',... + ' parallel computing is available.'],D.Name,curMatRel,lookUpTable{end}(2:6)); + end + + else + gpuAvail = 0; + info = 'Your GPU is not supported.'; + end + end + +else + gpuAvail = 0; + info = 'You do not seem to have a MATLAB Parallel Computing Toolbox installed.'; +end + +% adding availability +if gpuAvail + info = sprintf('Parallel computing AVAILABLE. %s',info); +else + info = sprintf('Parallel computing UNAVAILABLE. %s',info); +end + + +end \ No newline at end of file diff --git a/general/CFF_memory_available.m b/general/CFF_memory_available.m new file mode 100644 index 0000000..d99ac34 --- /dev/null +++ b/general/CFF_memory_available.m @@ -0,0 +1,73 @@ +function mem = CFF_memory_available(varargin) +%CFF_MEMORY_AVAILABLE Memory available guaranteed to hold data, in bytes +% +% Returns the memory available to hold data on CPU or GPU. +% +% MEM = CFF_MEMORY_AVAILABLE() returns the memory available on the CPU to +% hold data. +% +% MEM = CFF_MEMORY_AVAILABLE('GPU') returns the memory available on the +% GPU (if any) to hold data. If no GPU is detected, returns MEM = 0. +% +% NOTE: this function will return the memory available if there is a GPU, +% but this does not mean parallel computing is possible, because there +% are other necessary criteria to meet than just having a GPU (e.g. +% appropriate toolbox license, GPU drivers version, etc.). Use +% CFF_IS_PARALLEL_COMPUTING_AVAILABLE to test if parallel computing is +% available. +% +% See also CFF_IS_PARALLEL_COMPUTING_AVAILABLE. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 03-08-2022 + +% input parser +p = inputParser; +addOptional(p,'processingUnit','CPU',@(x) ismember(upper(x),{'CPU','GPU'})); +parse(p,varargin{:}); +processingUnit = p.Results.processingUnit; +clear p + +switch processingUnit + case 'GPU' + % for now only considering the active gpuDevice + n = gpuDeviceCount; + if n + D = gpuDevice; + mem = D.AvailableMemory; + % "Total memory (in bytes) available for data, specified as a + % scalar value. This property is available only for the + % currently selected device. This value can differ from the + % value reported by the NVIDIA® System Management Interface due + % to memory caching." + else + mem = 0; + end + + case 'CPU' + if ispc + % on pc, just use the MATLAB function 'memory' + mem_struct = memory; + mem = mem_struct.MemAvailableAllArrays; + % "Total memory available to hold data. The amount of memory + % available is guaranteed to be at least as large as this + % value. This field's value is the smaller of these two values: + % 1) The total available MATLAB virtual address space. 2) The + % total available system memory." + + elseif ismac + % on mac, get memory information from terminal + [~,txt] = system('top -l 1 | grep PhysMem: | awk ''{print $6}'''); + if strcmp(txt(end-1),'M') + mem = str2num(txt(1:end-2)).*1024^2; + elseif strcmp(txt(end),'G') + mem = str2num(txt(1:end-2)).*1024^3; + else + mem = str2num(txt(1:end-1)); + end + + end +end + +end \ No newline at end of file diff --git a/geometry/CFF_2Drotmat.m b/geometry/CFF_2Drotmat.m new file mode 100644 index 0000000..84976bf --- /dev/null +++ b/geometry/CFF_2Drotmat.m @@ -0,0 +1,9 @@ +function R = CFF_2Drotmat(rotAngle) +% +% angle in radians +% +% + + +R = CFF_3Drotmat(rotAngle,'z','rad'); +R = R(1:2,1:2,:); \ No newline at end of file diff --git a/geometry/CFF_3Drotmat.m b/geometry/CFF_3Drotmat.m new file mode 100644 index 0000000..20d3977 --- /dev/null +++ b/geometry/CFF_3Drotmat.m @@ -0,0 +1,83 @@ +function R = CFF_3Drotmat(rotAngle,varargin) +% Rotation matrix for a given angle about a given axis. +% By default angle in radians, but can be specified in degrees with +% argument 'unit'. +% A set of angles can be specified to output a set of rotation matrices. In +% that case, the angle input must be a vector (row or column). The rotation +% matrices in output will be 3x3xN matrices with N being the number of +% angles requested in input. +% +% IMPORTANT NOTE: HOW TO USE A ROTATION MATRIX DEPENDS ON WHETHER YOU WANT +% TO 1) ROTATE A VECTOR IN A COORDINATE SYSTEM ("VECTOR ROTATION") OR 2) +% GET THE COORDINATES OF A VECTOR IN AN ALTERNATIVE, ROTATED COORDINATE +% SYSTEM ("AXES ROTATION") +% +% Vector rotation: In a UNIQUE coordinate system, a 3D column vector v = +% [x;y;z] rotated about an axis will result in a new 3D column vector v' +% with coordinates [x';y';z'] = R*v. +% +% For example, if you have a vector on the x axis [n;0;0] and you want to +% rotate it +90 degrees around the z-axis (positive means +% counter-clockwise), then the rotated vector will be lying on the y axis +% with coordinates [0;n;0]. +% +% Axes rotation: Considering a NEW coordinate system (x',y',z') created +% from an EARLIER coordinate system (x,y,z) by rotation about an axis, a +% vector v with coordinates [x;y;z] in the EARLIER system will have +% coordinates [x';y';z'] = R'*v in the NEW system. +% +% For example, if you have a vector on the x axis [n;0;0] in a given +% coordinate system, that SAME vector in a NEW system +% rotated +90 degrees around the z-axis from the original one (positive +% means counter-clockwise) will be negative along the y-axis, with +% coordaintes [0;-n;0]. +% +% Note in both cases, the rotation follows the standard counter-clockwise +% rule, e.g. a positive rotation about the x axis is a rotation from y +% towards z (for y: from z towards x. For z; from x towards y). +% +% https://en.wikipedia.org/wiki/Rotation_matrix + +% parsing inputs +p = inputParser; +addRequired(p,'rotAngle',@CFF_mustBeVector); +addOptional(p,'rotAxis','z',@(x) ismember(lower(x),{'x','y','z'})); +addOptional(p,'angleUnit','rad',@(x) ismember(lower(x),{'r','rad','radian','radians','d','deg','degree','degrees'})); +parse(p,rotAngle,varargin{:}); +rotAxis = lower(p.Results.rotAxis); +angleUnit = lower(p.Results.angleUnit); +clear p + +% turn degrees to radians if necessary +switch angleUnit + case {'d','deg','degree','degrees'} + rotAngle = deg2rad(rotAngle); +end + +% make rotAngle a 3rd-dimension vector +rotAngle = permute(reshape(rotAngle,1,[]),[1,3,2]); +n = numel(rotAngle); + +% rotation matrix elements as 3rd-dimension vectors +O = zeros(1,1,n); +I = ones(1,1,n); +C = cos(rotAngle); +S = sin(rotAngle); + +% create rotation matrices +switch rotAxis + case 'x' + R = [[ I, O O ];... + [ O, C, -S ];... + [ O, S, C ]]; + + case 'y' + R = [[ C, O, S ];... + [ O, I, O ];... + [ -S, O, C ]]; + + case 'z' + R = [[ C, -S, O ];... + [ S, C, O ];... + [ O, O, I ]]; +end \ No newline at end of file diff --git a/geoprocessing/CFF_compute_ping_navigation_v2.m b/geoprocessing/CFF_compute_ping_navigation_v2.m new file mode 100644 index 0000000..959a727 --- /dev/null +++ b/geoprocessing/CFF_compute_ping_navigation_v2.m @@ -0,0 +1,378 @@ +function [fData,params] = CFF_compute_ping_navigation_v2(fData,varargin) +%CFF_COMPUTE_PING_NAVIGATION_V2 Computes navigation for each ping +% +% Computes navigation data for each ping (easting, northing, height, grid +% convergence, heading, speed) by matching/interpolating navigation data +% from ancillary sensors to ping time. +% +% FDATA = CFF_COMPUTE_PING_NAVIGATION_V2(FDATA) computes ping navigation +% for each ping in FDATA using default processing parameters, and returns +% FDATA with additional fields from the processing. +% +% CFF_COMPUTE_PING_NAVIGATION_V2(FDATA,PARAMS) uses processing parameters +% defined as the fields in the PARAMS structure. Possible parameters are: +% 'datagramSource': two-letters code string for the datagram type to use +% as a source for the date, time, and counter of pings: 'WC', 'AP', 'X8', +% etc. By default, a datagramSource is defined automatically from the +% datagrams available in FDATA. +% 'navLat': navigation latency in milliseconds. By default, using 0. +% 'ellips': code string for the coordinates' ellipsoid used for +% projection. See possible codes in CFF_LL2TM: e.g. 'wgs84', 'grs80', +% etc. By default, using 'wgs84'. +% 'tmproj': code string for the Transverse Mercator projection. See +% possible codes in CFF_LL2TM: e.g. 'utm54s', 'nztm2000', etc. By +% default, using the UTM projection corresponding to the location of the +% first ping. +% +% CFF_COMPUTE_PING_NAVIGATION_V2(...,'comms',COMMS) specifies if and how +% this function communicates on its internal state (progress, info, +% errors). COMMS can be either a CFF_COMMS object, or a text string to +% initiate a new CFF_COMMS object. Options are 'disp', 'textprogressbar', +% 'waitbar', 'oneline', 'multilines'. By default, using an empty +% CFF_COMMS object (i.e. no communication). See CFF_COMMS for more +% information. +% +% [FDATA,PARAMS] = CFF_COMPUTE_PING_NAVIGATION_V2(...) also outputs the +% parameters used in processing. Useful if parameters not set in input +% but determined from the data are to be reused for another file. +% +% NOTES: improvements to code in the future: +% * obtain best info on sonar location (E,N,H) and orientation (azimuth, +% depression, heading) at time of ping. +% * accept improved/modified/processed navigation (e.g. SBET). +% * add input parameter "skipIfAlreadyDone" that compare the params to +% the fData.MET fields and skip all the processing if they match. +% +% See also CFF_LL2TM, CFF_LOAD_CONVERTED_FILES, +% CFF_GEOREFERENCE_BOTTOM_DETECT, CFF_GROUP_PROCESSING. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 27-07-2022 + + +%% Input arguments management +p = inputParser; +addRequired(p,'fData',@(x) CFF_is_fData_version_current(x)); % line fData to process +addOptional(p,'params',struct(),@(x) isstruct(x)); % processing parameters +addParameter(p,'comms',CFF_Comms()); % information communication (none by default) +parse(p,fData,varargin{:}); +params = p.Results.params; +comms = p.Results.comms; +clear p +if ischar(comms) + comms = CFF_Comms(comms); +end + + +%% Prep + +% start message +comms.start('Processing navigation and heading'); + +% start progress +comms.progress(0,6); + + +%% EXTRACT PING DATA +% create ping time vectors in serial date number (SDN, Matlab, the whole +% and fractional number of days from January 0, 0000) and Time Since +% Midnight In Milliseconds (TSMIM, Kongsberg). + +comms.step('Extract ping data'); + +% get datagramSource parameter +if ~isfield(params,'datagramSource'), params.datagramSource = CFF_get_datagramSource(fData); end % default +mustBeMember(params.datagramSource,{'AP','WC','X8','De'}); % validate +datagramSource = params.datagramSource; + +% get navLat parameter +if ~isfield(params,'navLat'), params.navLat = 0; end % default +mustBeNumeric(params.navLat); % validate +navLat = params.navLat; + +% get data +pingTSMIM = fData.([datagramSource '_1P_TimeSinceMidnightInMilliseconds']); +pingDate = fData.([datagramSource '_1P_Date']); +pingCounter = fData.([datagramSource '_1P_PingCounter']); +pingDate = datenum(cellfun(@num2str,num2cell(pingDate),'un',0),'yyyymmdd'); +pingSDN = pingDate(:)'+ pingTSMIM/(24*60*60*1000) + navLat./(1000.*60.*60.*24); % apply navigation latency here + +comms.progress(1,6); + + +%% EXTRACT NAVIGATION DATA +% same for navigation. In the future, offer possibility to import +% position/orientation from other files, say SBET + +comms.step('Extract navigation data'); + +% test if there are several sources of GPS data +if isfield(fData,'Po_1D_PositionSystemDescriptor') + ID = unique(fData.Po_1D_PositionSystemDescriptor); + if numel(ID) > 1 + % several sources available, we will need to choose one + % start by eliminating those that are obviously bad. + % I have found data where one source had lat/long values that were + % both constant and outside of normal values. You may want to + % devise more tests if you ever come across different examples of + % bad position data + isSingleEntry = arrayfun(@(x) sum(fData.Po_1D_PositionSystemDescriptor==x)==1, ID); % check if single entry (then they will be constant) + isLatAllConst = arrayfun(@(x) all(diff(fData.Po_1D_Latitude(fData.Po_1D_PositionSystemDescriptor==x))==0), ID); % check if all constant values + isLonAllConst = arrayfun(@(x) all(diff(fData.Po_1D_Longitude(fData.Po_1D_PositionSystemDescriptor==x))==0), ID); % check if all constant values + isLatAllBad = arrayfun(@(x) all(abs(fData.Po_1D_Latitude(fData.Po_1D_PositionSystemDescriptor==x))>90), ID); % check if all outside [-90:90] + isLonAllBad = arrayfun(@(x) all(abs(fData.Po_1D_Longitude(fData.Po_1D_PositionSystemDescriptor==x))>180), ID); % check if all outside [-180:180] + idxBadPos = (~isSingleEntry & (isLatAllConst|isLonAllConst)) | isLatAllBad | isLonAllBad; + % removing those bad sources + ID = ID(~idxBadPos); + if numel(ID)==1 + % only one good source left, just use that one + pos_idx = fData.Po_1D_PositionSystemDescriptor==ID; + else + % still several sources available + % find the one with the best fix quality + meanFixQuality = arrayfun(@(x) nanmean(fData.Po_1D_MeasureOfPositionFixQuality(fData.Po_1D_PositionSystemDescriptor==x)), ID); + [~,idx_keep] = min(meanFixQuality); + pos_idx = fData.Po_1D_PositionSystemDescriptor==ID(idx_keep); + comms.info(sprintf('Several sources of GPS data available. Using source with ID: %d',ID(idx_keep))); + end + else + % single source. Use all datagrams. + pos_idx = 1:numel(fData.Po_1D_Latitude); + end +else + % using older version of converted data, throw warning and continue + comms.info('Navigation information in your converted data indicates it is not up to date with fData version. Consider reconverting this file, particularly if you see strange patterns in the navigation, or if two GPS sources have been logged in the file.'); + pos_idx = 1:numel(fData.Po_1D_Latitude); +end + +% get data +posLatitude = fData.Po_1D_Latitude(pos_idx); +posLongitude = fData.Po_1D_Longitude(pos_idx); +posHeading = fData.Po_1D_HeadingOfVessel(pos_idx); +posSpeed = fData.Po_1D_SpeedOfVesselOverGround(pos_idx); +posTSMIM = fData.Po_1D_TimeSinceMidnightInMilliseconds(pos_idx); % time since midnight in milliseconds +posDate = datenum(cellfun(@num2str,num2cell(fData.Po_1D_Date(pos_idx)),'un',0),'yyyymmdd'); +posSDN = posDate(:)'+ posTSMIM/(24*60*60*1000); % serial date number + +comms.progress(2,6); + +%% EXTRACT HEIGHT DATA + +comms.step('Extract height data'); + +if isfield(fData,'He_1D_Height') + heiHeight = fData.He_1D_Height; % now m + heiDate = datenum(cellfun(@num2str,num2cell(fData.He_1D_Date),'un',0),'yyyymmdd'); + heiSDN = heiDate(:)' + fData.He_1D_TimeSinceMidnightInMilliseconds/(24*60*60*1000); +else + % no height datagrams, create fake variables + heiHeight = zeros(size(pingTSMIM)); + heiSDN = pingSDN; +end + +comms.progress(3,6); + + +%% PROCESS NAVIGATION AND HEADING +% Get position and heading for each ping. Position and heading were +% recorded at the sensor's time so we need to interpolate them at the same +% time to match ping time. + +comms.step('Processing navigation and heading'); + +% get ellips parameter +if ~isfield(params,'ellips'), params.ellips = 'wgs84'; end % default +mustBeMember(params.ellips,{'wgs84','grs80'}); % validate +ellips = params.ellips; + +% get tmproj parameter, or use default UTM zone from first ping +if ~isfield(params,'tmproj') + % default + [~,~,~,~,params.tmproj] = CFF_ll2tm(posLongitude(1),posLatitude(1),ellips,'utm'); + params.tmproj = ['utm' params.tmproj]; + comms.info(['tmproj not specified in input. Defining it from first position fix: ''' params.tmproj '''']); +end +mustBeA(params.tmproj,'char'); % validate (too many cases to validate better) +tmproj = params.tmproj; + +% convert posLatitude/posLongitude to easting/northing/grid convergence: +[posE, posN, posGridConv] = CFF_ll2tm(posLongitude, posLatitude, ellips, tmproj); + +% we need at least two position samples to process the navigation. If there +% is only one, make up another one using dead reckoning +if numel(posE)==1 + posE = [posE, posE + posSpeed.*cosd(posHeading)]; + posN = [posN, posN + posSpeed.*sind(posHeading)]; + posGridConv = [posGridConv, posGridConv]; + posHeading = [posHeading, posHeading]; + posSpeed = [posSpeed, posSpeed]; + posTSMIM = [posTSMIM, posTSMIM + 1000]; % + 1 sec + posSDN = [posSDN, posSDN + 1/(24*60*60)]; % + 1 sec +end + +% convert heading to degrees and allow heading values superior to +% 360 or inferior to 0 (because every time the vessel crossed the NS +% line, the heading jumps from 0 to 360 (or from 360 to 0) and this +% causes a problem for following interpolation): + +posJump = find(diff(posHeading)>300); +negJump = find(diff(posHeading)<-300); +jumps = zeros(1,length(posHeading)); + +if ~isempty(posJump) + for jj = 1:length(posJump) + jumps(posJump(jj)+1:end) = jumps(posJump(jj)+1:end) - 1; + end +end + +if ~isempty(negJump) + for jj = 1:length(negJump) + jumps(negJump(jj)+1:end) = jumps(negJump(jj)+1:end) + 1; + end +end + +posHeading = posHeading + jumps.*360; + +% dirty heading fix if first value is null +if posHeading(1)==0 && abs(posHeading(2))>5 + posLatitude(1) = []; + posLongitude(1) = []; + posHeading(1) = []; + posSpeed(1) = []; + posTSMIM(1) = []; + posDate(1) = []; + posSDN(1) = []; + jumps(1) = []; +end + +% initialize new vectors +pingE = nan(size(pingTSMIM)); +pingN = nan(size(pingTSMIM)); +pingGridConv = nan(size(pingTSMIM)); +pingHeading = nan(size(pingTSMIM)); +pingSpeed = nan(size(pingTSMIM)); + +% interpolate Easting, Northing, Grid Convergence and Heading at ping times +for jj = 1:length(pingTSMIM) + A = posSDN-pingSDN(jj); + iA = find (A == 0); + if A > 0 + % the ping time is older than any navigation time, extrapolate from the first items in navigation array. + pingE(jj) = posE(2) + (posE(2)-posE(1)).*(pingSDN(jj)-posSDN(2))./(posSDN(2)-posSDN(1)); + pingN(jj) = posN(2) + (posN(2)-posN(1)).*(pingSDN(jj)-posSDN(2))./(posSDN(2)-posSDN(1)); + pingGridConv(jj) = posGridConv(2) + (posGridConv(2)-posGridConv(1)).*(pingSDN(jj)-posSDN(2))./(posSDN(2)-posSDN(1)); + pingHeading(jj) = posHeading(2) + (posHeading(2)-posHeading(1)).*(pingSDN(jj)-posSDN(2))./(posSDN(2)-posSDN(1)); + pingSpeed(jj) = posSpeed(2) + (posSpeed(2)-posSpeed(1)).*(pingTSMIM(jj)-posTSMIM(2))./(posTSMIM(2)-posTSMIM(1)); + elseif A < 0 + % the ping time is more recent than any navigation time, extrapolate from the last items in navigation array. + pingE(jj) = posE(end) + (posE(end)-posE(end-1)).*(pingSDN(jj)-posSDN(end))./(posSDN(end)-posSDN(end-1)); + pingN(jj) = posN(end) + (posN(end)-posN(end-1)).*(pingSDN(jj)-posSDN(end))./(posSDN(end)-posSDN(end-1)); + pingGridConv(jj) = posGridConv(end) + (posGridConv(end)-posGridConv(end-1)).*(pingSDN(jj)-posSDN(end))./(posSDN(end)-posSDN(end-1)); + pingHeading(jj) = posHeading(end) + (posHeading(end)-posHeading(end-1)).*(pingSDN(jj)-posSDN(end))./(posSDN(end)-posSDN(end-1)); + pingSpeed(jj) = posSpeed(end) + (posSpeed(end)-posSpeed(end-1)).*(pingTSMIM(jj)-posTSMIM(end))./(posTSMIM(end)-posTSMIM(end-1)); + elseif ~isempty(iA) + % the ping time corresponds to an existing navigation time, get easting and northing from it. + pingE(jj) = posE(iA); + pingN(jj) = posN(iA); + pingGridConv(jj) = posGridConv(iA); + pingHeading(jj) = posHeading(iA); + pingSpeed(jj) = posSpeed(iA); + else + % the ping time is within the limits of the navigation time array but doesn't correspond to any value in it, interpolate from nearest values + iNegA = find(A<0); + [~,iMax] = max(A(iNegA)); + iA(1) = iNegA(iMax); % index of navigation time just older than ping time + iPosA = find(A>0); + [~,iMin] = min(A(iPosA)); + iA(2) = iPosA(iMin); % index of navigation time just more recent ping time + % now extrapolate easting, northing, grid convergence and heading + pingE(jj) = posE(iA(2)) + (posE(iA(2))-posE(iA(1))).*(pingSDN(jj)-posSDN(iA(2)))./(posSDN(iA(2))-posSDN(iA(1))); + pingN(jj) = posN(iA(2)) + (posN(iA(2))-posN(iA(1))).*(pingSDN(jj)-posSDN(iA(2)))./(posSDN(iA(2))-posSDN(iA(1))); + pingGridConv(jj) = posGridConv(iA(2)) + (posGridConv(iA(2))-posGridConv(iA(1))).*(pingSDN(jj)-posSDN(iA(2)))./(posSDN(iA(2))-posSDN(iA(1))); + pingHeading(jj) = posHeading(iA(2)) + (posHeading(iA(2))-posHeading(iA(1))).*(pingSDN(jj)-posSDN(iA(2)))./(posSDN(iA(2))-posSDN(iA(1))); + pingSpeed(jj) = posSpeed(iA(2)) + (posSpeed(iA(2))-posSpeed(iA(1))).*(pingTSMIM(jj)-posTSMIM(iA(2)))./(posTSMIM(iA(2))-posTSMIM(iA(1))); + end +end + +% bring heading back into the interval [0 360] +posHeading = posHeading - jumps.*360; +pingHeading = mod(pingHeading,360); + +comms.progress(4,6); + + +%% PROCESS HEIGHT +% Get height for each ping. Height were recorded at the sensor's time so we +% need to interpolate them at the same time to match ping time. + +comms.step('Processing height'); + +% initialize new vectors +pingH = nan(size(pingSDN)); + +% interpolate Height at ping times +for jj = 1:length(pingSDN) + A = heiSDN-pingSDN(jj); + iA = find (A == 0,1); + if A > 0 + % the ping time is older than any height time, extrapolate from the first items in height array. + pingH(jj) = heiHeight(2) + (heiHeight(2)-heiHeight(1)).*(pingSDN(jj)-heiSDN(2))./(heiSDN(2)-heiSDN(1)); + elseif A < 0 + % the ping time is more recent than any height time, extrapolate from the last items in height array. + pingH(jj) = heiHeight(end) + (heiHeight(end)-heiHeight(end-1)).*(pingSDN(jj)-heiSDN(end))./(heiSDN(end)-heiSDN(end-1)); + elseif ~isempty(iA) + % the ping time corresponds to an existing height time, get height + % from it + pingH(jj) = heiHeight(iA); + else + % the ping time is within the limits of the height time array but doesn't correspond to any value in it, interpolate from nearest values + iNegA = find(A<0); + [~,iMax] = max(A(iNegA)); + iA(1) = iNegA(iMax); % index of height time just older than ping time + iPosA = find(A>0); + [~,iMin] = min(A(iPosA)); + iA(2) = iPosA(iMin); % index of height time just more recent ping time + % now extrapolate height + pingH(jj) = heiHeight(iA(2)) + (heiHeight(iA(2))-heiHeight(iA(1))).*(pingSDN(jj)-heiSDN(iA(2)))./(heiSDN(iA(2))-heiSDN(iA(1))); + end +end + +comms.progress(5,6); + + +%% SAVE RESULTS + +comms.step('Save results'); + +% save processed results +fData.X_1P_pingCounter = pingCounter; +fData.X_1P_pingTSMIM = pingTSMIM; +fData.X_1P_pingSDN = pingSDN; +fData.X_1P_pingE = pingE; +fData.X_1P_pingN = pingN; +fData.X_1P_pingH = pingH; +fData.X_1P_pingGridConv = pingGridConv; +fData.X_1P_pingHeading = pingHeading; +fData.X_1P_pingSpeed = pingSpeed; + +% save processing parameters +fData.MET_datagramSource = datagramSource; +fData.MET_navigationLatencyInMilliseconds = navLat; +fData.MET_ellips = ellips; +fData.MET_tmproj = tmproj; + +% sort fields by name +fData = orderfields(fData); + +% also save parameters back in params, for possible reuse for another file +params.datagramSource = datagramSource; +params.navLat = navLat; +params.ellips = ellips; +params.tmproj = tmproj; + +comms.progress(6,6); + + +%% end message +comms.finish('Done'); diff --git a/geoprocessing/CFF_georeference_bottom_detect.m b/geoprocessing/CFF_georeference_bottom_detect.m new file mode 100644 index 0000000..59c915c --- /dev/null +++ b/geoprocessing/CFF_georeference_bottom_detect.m @@ -0,0 +1,138 @@ +function fData = CFF_georeference_bottom_detect(fData,varargin) +%CFF_GEOREFERENCE_BOTTOM_DETECT Geo-reference bottom detect +% +% Get range, swathe coordinates (across and upwards distance from sonar), +% and projected coordinates (easting, northing, height) of the bottom +% detect samples +% +% FDATA = CFF_GEOREFERENCE_BOTTOM_DETECT(FDATA) range, swathe +% coordinates (across and upwards distance from sonar), and projected +% coordinates (easting, northing, height) of the bottom detect samples in +% FDATA, then saves them as new fields in FDATA. +% +% CFF_GEOREFERENCE_BOTTOM_DETECT(...,'comms',COMMS) specifies if and how +% this function communicates on its internal state (progress, info, +% errors). COMMS can be either a CFF_COMMS object, or a text string to +% initiate a new CFF_COMMS object. Options are 'disp', 'textprogressbar', +% 'waitbar', 'oneline', 'multilines'. By default, using an empty +% CFF_COMMS object (i.e. no communication). See CFF_COMMS for more +% information. +% +% See also CFF_COMPUTE_PING_NAVIGATION_V2, CFF_GROUP_PROCESSING. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 27-07-2022 + + +%% Input arguments management +p = inputParser; +addRequired(p,'fData',@(x) CFF_is_fData_version_current(x)); % line fData to process +addParameter(p,'comms',CFF_Comms()); % information communication (none by default) +parse(p,fData,varargin{:}); +comms = p.Results.comms; +clear p +if ischar(comms) + comms = CFF_Comms(comms); +end + + +%% Prep + +% start message +comms.start('Georeferencing the bottom detections'); + +% start progress +comms.progress(0,1); + +% extract needed ping info +X_1P_sonarHeight = fData.X_1P_pingH; %m +X_1P_sonarEasting = fData.X_1P_pingE; %m +X_1P_sonarNorthing = fData.X_1P_pingN; %m +X_1P_gridConvergenceDeg = fData.X_1P_pingGridConv; %deg +X_1P_vesselHeadingDeg = fData.X_1P_pingHeading; %deg +X_1_sonarHeadingOffsetDeg = fData.IP_ASCIIparameters.S1H; %deg + +% get datagramSource +datagramSource = CFF_get_datagramSource(fData); + + +%% Process +switch datagramSource + + case {'WC' 'AP'} + + % For water-column data, we start from the number of the sample + % corresponding to the bottom detect in each beam. + + % Get it. Not precising wether raw or processed here, as this code + % is used both when loading the data and after filtering, and + % permute to get it as a 1BP array + X_BP_bottomSample = CFF_get_bottom_sample(fData); + X_1BP_bottomSample = permute(X_BP_bottomSample,[3,1,2]); + + % To get from sample number to range in meters, we need the OWTT + % distance traveled in one sample + X_1P_oneSampleDistance = CFF_inter_sample_distance(fData); + + % To complete our coordinates, we need the beam pointing angle + X_BP_beamPointingAngleDeg = fData.(sprintf('%s_BP_BeamPointingAngle',datagramSource)); + X_BP_beamPointingAngleRad = deg2rad(X_BP_beamPointingAngleDeg); + + % Range and angle will get us the coordinates of the bottom sample + % in the swath frame. To get in a geographical frame, we also need + % the heading angle, made up of the sonar heading offset, the + % vessel heading, and the grid convergence + X_1P_thetaDeg = - mod( X_1P_gridConvergenceDeg + X_1P_vesselHeadingDeg + X_1_sonarHeadingOffsetDeg, 360 ); + X_1P_thetaRad = deg2rad(X_1P_thetaDeg); + + % Use all of this to georeference those bottom samples + [X_1BP_bottomEasting, X_1BP_bottomNorthing, X_1BP_bottomHeight, X_1BP_bottomAcrossDist, X_1BP_bottomUpDist, X_1BP_bottomRange] = CFF_georeference_sample(X_1BP_bottomSample, 0, X_1P_oneSampleDistance, X_BP_beamPointingAngleRad, X_1P_sonarEasting, X_1P_sonarNorthing, X_1P_sonarHeight, X_1P_thetaRad); + + % save info + fData = CFF_set_bottom_sample(fData,X_BP_bottomSample); + fData.X_BP_beamPointingAngleRad = X_BP_beamPointingAngleRad; + fData.X_BP_bottomRange = permute(X_1BP_bottomRange,[2,3,1]); + + % save data in the swath frame + fData.X_BP_bottomUpDist = permute(X_1BP_bottomUpDist,[2,3,1]); + fData.X_BP_bottomAcrossDist = permute(X_1BP_bottomAcrossDist,[2,3,1]); + + % save data in the projected frame + fData.X_BP_bottomEasting = permute(X_1BP_bottomEasting,[2,3,1]); + fData.X_BP_bottomNorthing = permute(X_1BP_bottomNorthing,[2,3,1]); + fData.X_BP_bottomHeight = permute(X_1BP_bottomHeight,[2,3,1]); + + case 'X8' + + % For normal seafloor data, it's a bit simpler, as we start from + % the (x:forward,y:starboard,z:depth) coordinates of each sounding. + % We only need to rotate around the z/depth axis. + + % This time however, the data are already corrected for the sonar + % heading offset, so the vessel azimuth is only the vessel heading + % and the grid convergence + X_1P_thetaDeg = mod(X_1P_gridConvergenceDeg+X_1P_vesselHeadingDeg,360); + + % apply rotation for the (x,y) -> (E,N) coordinates. (x,y) are + % referenced to the horizontal position of the position system so + % just add the values obtained before. + fData.X_BP_bottomEasting = fData.X_1P_pingE + fData.X8_BP_AcrosstrackDistanceY.*cosd(X_1P_thetaDeg) + fData.X8_BP_AlongtrackDistanceX.*sind(X_1P_thetaDeg); + fData.X_BP_bottomNorthing = fData.X_1P_pingN - fData.X8_BP_AcrosstrackDistanceY.*sind(X_1P_thetaDeg) + fData.X8_BP_AlongtrackDistanceX.*cosd(X_1P_thetaDeg); + + % z values are referenced to the sonar head depth, so we need to + % add the heave + fData.X_BP_bottomHeight = - (fData.X8_1P_TransmitTransducerDepth + fData.X8_BP_DepthZ); + + % also save the across-track and depth in the swath frame + fData.X_BP_bottomUpDist = -fData.X8_BP_DepthZ; + fData.X_BP_bottomAcrossDist = fData.X8_BP_AcrosstrackDistanceY; + +end + +% sort fields by name +fData = orderfields(fData); + + +%% end message +comms.finish('Done'); diff --git a/geoprocessing/CFF_get_WC_coordinates.m b/geoprocessing/CFF_get_WC_coordinates.m new file mode 100644 index 0000000..49fb48a --- /dev/null +++ b/geoprocessing/CFF_get_WC_coordinates.m @@ -0,0 +1,41 @@ +function [sampleAcrossDist,sampleUpDist] = CFF_get_WC_coordinates(fData,iPing,nSamples) +%CFF_GET_WC_COORDINATES WCD samples coordinates in swath frame from fData +% +% Computes the cartesian coordinates in the swath frame (i.e. the +% distances across and upwards from the sonar) of all samples of +% water-column data in one ping. Use this function along with +% CFF_GET_WC_DATA to get the coordinates of the data for wedge display. +% +% The swath frame is defined as: +% - origin: sonar face +% - Xs: across distance (positive ~starboard) +% - Ys: along distance (positive ~forward) = 0 +% - Zs: up distance (positive up) +% +% [ACD,UPD] = CFF_GET_WC_COORDINATES(FDATA,IPING,NSAMPLES) returns the +% across distance ACD and the upward distance UPD (both in m) in the +% swath frame for all WCD samples with index 1 to NSAMPLES in ping of +% index IPING in FDATA. +% +% See also CFF_GET_WC_DATA, CFF_INTER_SAMPLE_DISTANCE, +% CFF_GET_SAMPLES_RANGE, CFF_GET_SAMPLES_DIST, CFF_GEOREFERENCE_SAMPLE + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2022-2022; Last revision: 20-07-2022 + +% get source datagram +datagramSource = CFF_get_datagramSource(fData); + +% get the range from sonar (in m) for each sample index +idxSamples = (1:nSamples)'; +startSampleNumber = fData.(sprintf('%s_BP_StartRangeSampleNumber',datagramSource))(:,iPing); +interSamplesDistance = CFF_inter_sample_distance(fData,iPing); +sampleRange = CFF_get_samples_range(idxSamples,startSampleNumber,interSamplesDistance); + +% get beam pointing angle (in rad) +sampleAngle = deg2rad(fData.(sprintf('%s_BP_BeamPointingAngle',datagramSource))(:,iPing)); + +% calculate and return swath coordinates +[sampleAcrossDist,sampleUpDist] = CFF_get_samples_dist(sampleRange,sampleAngle); + diff --git a/functions/geodesy/CFF_ll2tm.m b/geoprocessing/CFF_ll2tm.m similarity index 78% rename from functions/geodesy/CFF_ll2tm.m rename to geoprocessing/CFF_ll2tm.m index 6ba5e02..573b7da 100644 --- a/functions/geodesy/CFF_ll2tm.m +++ b/geoprocessing/CFF_ll2tm.m @@ -1,31 +1,19 @@ -%%% CFF_ll2tm.m -% -% Converts geographic coordinates (Latitude, Longitude) to Transverse -% Mercator Projections coordinates (Northing, Easting). Also provides grid -% convergence (angle between true north and grid north), point scale factor -% and UTM zone if not specified in input. Different ellipsoids or -% projections can be specified. -% -%% Help -% -% *USE* -% -% _This section contains a more detailed description of what the function -% does and how to use it, for the interested user to have an overall -% understanding of its function. Example below to replace. Delete these -% lines XXX._ -% -% This is a text file containing the basic comment template to add at the -% start of any new ESP3 function to serve as function help. XXX +function [E, N, gridConv, pointScaleFactor, utmzone] = CFF_ll2tm(lon, lat, ellips, tmproj) +%CFF_LL2TM One-line description % -% *INPUT VARIABLES* +% Converts geographic coordinates (Latitude, Longitude) to Transverse +% Mercator Projections coordinates (Northing, Easting). Also provides +% grid convergence (angle between true north and grid north), point scale +% factor and UTM zone if not specified in input. Different ellipsoids or +% projections can be specified. % -% * |lon: Required. Longitude scalar or vector in decimal degrees. -% * |lat|: Required. Latitude scalar or vector in decimal degrees. -% * |ellips|: Required. code string for the input coordinates' ellipsoid. -% Supported codes: 'wgs84', 'grs80' -% * |tmproj|: Required. Code string for the ouptut transverse mercator -% projection. Supported codes: +% *INPUT VARIABLES* +% * |lon: Required. Longitude scalar or vector in decimal degrees. +% * |lat|: Required. Latitude scalar or vector in decimal degrees. +% * |ellips|: Required. code string for the input coordinates' ellipsoid. +% Supported codes: 'wgs84', 'grs80' +% * |tmproj|: Required. Code string for the ouptut transverse mercator +% projection. Supported codes: % 'utm' -> Universal Transvere Mercator projection without zone % specified. The function computes the longitudinal zone for input % coordinates and returns the result in variable utmzone. Note: UTM @@ -73,51 +61,34 @@ % 'taietm2000' -> "North Taieri 2000". % 'bluftm2000' -> "Bluff 2000". % -% *OUTPUT VARIABLES* -% -% * |E|: Projection easting scalar or vector -% * |N|: Projection northing scalar or vector -% * |gridConv|: Grid convergence in degrees. Grid convergence is the angle -% at a point between true and grid North. It is positive when grid north -% lies to the West of the true North. -% * |pointScaleFactor|: Point scale factor. The scale factor at a point -% away from the central meridian. -% * |utmzone|: UTM longitudinal zone (ouput if zone was not specified in -% input, i.e. ellips = 'utm'). -% -% *DEVELOPMENT NOTES* +% *OUTPUT VARIABLES* +% * |E|: Projection easting scalar or vector +% * |N|: Projection northing scalar or vector +% * |gridConv|: Grid convergence in degrees. Grid convergence is the +% angle at a point between true and grid North. It is positive when grid +% north lies to the West of the true North. +% * |pointScaleFactor|: Point scale factor. The scale factor at a point +% away from the central meridian. +% * |utmzone|: UTM longitudinal zone (ouput if zone was not specified in +% input, i.e. ellips = 'utm'). % -% * The GRS80 and WGS84 ellipsoids are so close that the differences in -% Lat/Long are usually considered insignificant for most applications. This -% function makes this assumption too, even though a fair conversion from -% say, WGS84 lat/long to a NZ projection based on GRS80 would require a -% datum transformation. If this function is ever extended to different -% ellipsoids, a datum transformation will be required. -% * This function is based on "LINZS25002. Standard for New Zealand -% Geodetic Datum 2000 Projections: version 2." and original code from -% "ll2tm.m" by David Johnson and Brett Beamsley (Metocean Solutions LTD). -% -% *NEW FEATURES* -% -% * 2018-10-11: new header -% * 2010-06: first version. -% -% *EXAMPLE* -% -% _This section contains examples of valid function calls. Note that -% example lines start with 3 white spaces so that the publish function -% shows them correctly as matlab code. Example below to replace. Delete -% these lines XXX._ -% -% example_use_1; % comment on what this does. XXX -% example_use_2: % comment on what this line does. XXX -% -% *AUTHOR, AFFILIATION & COPYRIGHT* -% -% Alexandre Schimel, University of Waikato +% *DEVELOPMENT NOTES* +% * The GRS80 and WGS84 ellipsoids are so close that the differences in +% Lat/Long are usually considered insignificant for most applications. +% This function makes this assumption too, even though a fair conversion +% from say, WGS84 lat/long to a NZ projection based on GRS80 would +% require a datum transformation. If this function is ever extended to +% different ellipsoids, a datum transformation will be required. +% * This function is based on "LINZS25002. Standard for New Zealand +% Geodetic Datum 2000 Projections: version 2." and original code from +% "ll2tm.m" by David Johnson and Brett Beamsley (Metocean Solutions LTD). +% * Special UTM zone definitions in Norway finally done correctly in +% later version. +% * UTM is only defined between 80°S and 84°N but this function ignores +% these limits. -%% Function -function [E, N, gridConv, pointScaleFactor, utmzone] = CFF_ll2tm(lon, lat, ellips, tmproj) +% Authors: Alex Schimel (NGU) +% 2010-2022; Last revision: 29-04-2022 %% ellipsoid parameters: @@ -206,7 +177,12 @@ %% changing parameters if 'utm' input if strcmp(tmproj,'utm') origLat = 0; - origLon = floor(lon./6).*6+3; + origLon = floor(lon./6).*6+3; % normal case, central meridian of the zone. + origLon(lat>=56&lat<64&lon>=3&lon<6) = 9; % west coast Norway (32N) + origLon(lat>=72&lon>=6&lon<9) = 3; % Svalbard #1 (31N) + origLon(lat>=72&lon>=9&lon<21) = 15; % Svalbard #2 (33N) + origLon(lat>=72&lon>=21&lon<33) = 27; % Svalbard #3 (35N) + origLon(lat>=72&lon>=33&lon<42) = 39; % Svalbard #4 (37N) FN = (lat < 0).*10000000; FE = 500000; k0 = 0.9996; diff --git a/geoprocessing/samples_location/CFF_georeference_sample.m b/geoprocessing/samples_location/CFF_georeference_sample.m new file mode 100644 index 0000000..0a33b96 --- /dev/null +++ b/geoprocessing/samples_location/CFF_georeference_sample.m @@ -0,0 +1,27 @@ +function [sampleEasting, sampleNorthing, sampleHeight, sampleAcrossDistance, sampleUpwardsDistance, sampleRange] = CFF_georeference_sample(idxSamples, startRangeSampleNumber, interSamplesDistance, beamPointingAngle, sonarEasting, sonarNorthing, sonarHeight, sonarHeading) +%CFF_GEOREFERENCE_SAMPLE Georeference samples +% +% Get range, swathe coordinates (across and upwards distance from sonar), +% and projected coordinates (easting, northing, height) of any sample +% based on the sample index, sample start index offset, inter-samples +% distance, beam pointing angle and the sonar's projected coordinates +% and orientation (heading). +% +% See also CFF_GET_SAMPLES_RANGE, CFF_GET_SAMPLES_DIST, +% CFF_GET_SAMPLES_ENH + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 19-07-2022 + +% Calculate range +sampleRange = CFF_get_samples_range(idxSamples,startRangeSampleNumber,interSamplesDistance); + +% Calculate cartesian coordinates in the swath frame +[sampleAcrossDistance,sampleUpwardsDistance] = CFF_get_samples_dist(sampleRange,beamPointingAngle); + +% Calculate projected coordinates in the geographical frame +[sampleEasting, sampleNorthing, sampleHeight] = CFF_get_samples_ENH(sonarEasting,sonarNorthing,sonarHeight,sonarHeading,sampleAcrossDistance,sampleUpwardsDistance); + + + diff --git a/geoprocessing/samples_location/CFF_get_samples_ENH.m b/geoprocessing/samples_location/CFF_get_samples_ENH.m new file mode 100644 index 0000000..6d6170b --- /dev/null +++ b/geoprocessing/samples_location/CFF_get_samples_ENH.m @@ -0,0 +1,58 @@ +function [sampleEasting, sampleNorthing, sampleHeight] = CFF_get_samples_ENH(sonarEasting,sonarNorthing,sonarHeight,sonarHeading,sampleAcrossDistance,sampleUpwardsDistance) +%CFF_GET_SAMPLES_ENH Compute samples coordinates in the geographical frame +% +% Compute coordinates in the geographical frame (Easting, Northing, +% Height) of samples using their coordinates in the swath frame +% (Distances across and upwards). +% +% The geographical frame is defined as: +% - origin: the (0,0) Easting/Northing projection/datum reference +% - Xp: Easting (positive grid East) +% - Yp: Northing (grid North, positive grid North) +% - Zp: Elevation/Height (positive up) + +% The swath frame is defined as: +% - origin: sonar face +% - Xs: across distance (positive ~starboard) +% - Ys: along distance (positive ~forward) +% - Zs: up distance (positive up) +% +% [E,N,H] = CFF_GET_SAMPLES_ENH(SE,SN,SH,SHEADING,SAMPACD,SAMPUPD) +% returns the easting E, northing N, and height H in the geographical +% frame of samples, from the sonar easting SE (in m), sonar northing SN +% (in m), sonar height SH (in m), sonar heading SHEADING (in radians +% relative to North?), and the samples' across distance SAMPACD (in m) +% and upward distance SAMPUPD (in m) in the swath frame. SE, SN, SH, and +% SHEADING must be matching 1P matrices or compatible (e.g. scalars for +% one ping). SAMPACD and SAMPUPD must be SBP tensors or compatible (e.g. +% SB matrices for 1 ping). +% +% See also CFF_GET_SAMPLES_DIST, CFF_GEOREFERENCE_SAMPLE + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 19-07-2022 + +% permute dimensions of input to get everything as SBP matrices +sonarEasting = permute(sonarEasting,[3,1,2]); +sonarNorthing = permute(sonarNorthing,[3,1,2]); +sonarHeight = permute(sonarHeight,[3,1,2]); +sonarHeading = permute(sonarHeading,[3,1,2]); + +% compute outputs +sampleEasting = sonarEasting + sampleAcrossDistance.*cos(sonarHeading); +sampleNorthing = sonarNorthing + sampleAcrossDistance.*sin(sonarHeading); +sampleHeight = sonarHeight + sampleUpwardsDistance; + +% NOTE: We make the STRONG ASSUMPTION here that the geographical and swath +% frames have THE SAME Z axis, so that going from one to the other only +% requires considering the rotation about the Z axis (aka heading/yaw), +% which includes the vessel heading, the sonar head heading offset, and the +% grid convergence (aka angle between true north and grid north). +% +% This is only valid if the system was corrected in real time for pitch +% and roll, and the single swath is pointed directly downwards. Aka, this +% is most likely wrong for multi-sectors, and systems using +% yaw-compensation. + +end diff --git a/geoprocessing/samples_location/CFF_get_samples_ENH_v2.m b/geoprocessing/samples_location/CFF_get_samples_ENH_v2.m new file mode 100644 index 0000000..f358faf --- /dev/null +++ b/geoprocessing/samples_location/CFF_get_samples_ENH_v2.m @@ -0,0 +1,72 @@ +function [sampleEasting, sampleNorthing, sampleHeight] = CFF_get_samples_ENH_v2(acDist,alDist,upDist,sonHead,sonE,sonN,AxRotAngle) +%CFF_GET_SAMPLES_ENH_V2 Compute sample coordinates in projected frame +% +% Given coordinates of samples in the swath frame (distances across, +% along and upwards), calculate coordinates in the projected frame +% (Easting, Northing, Height). +% +% The swath frame SF is defined as: +% - origin: sonar face +% - x_s: distance across-track (positive towards starboard) +% - y_s: distance along-track (positive forward) +% - z_s: distance upwards (positive up) +% +% The projected frame PF is defined as: +% - origin: the (0,0) Easting/Northing projection/datum reference +% - x_p: Easting (positive towards grid East) +% - y_p: Northing (positive towards grid North) +% - z_p: Elevation/Height (positive up) +% +% NOTE: This function works on the STRONG ASSUMPTION that the +% swath frame and projected frames have THE SAME Z axis, so that going +% from one to the other only requires considering the rotation about that +% Z axis. This assumption is only valid if the system was corrected in +% real time for pitch and roll, so that swath is pointed directly +% downwards. Aka, this is most likely wrong for multi-sectors, and +% systems using yaw-compensation. +% +% % The two frames share their z axis so the transformation is a simple +% rotation around the z axis. We need the coordinates of the swath frame +% origin (i.e. sonar) in the projected frame, as well as the axes +% rotation (i.e. angle between y_s and y_p, positive counter-clockwise. + +% [E,N,H] = CFF_GET_SAMPLES_ENH(SE,SN,SH,SHEADING,SAMPACD,SAMPUPD) +% returns the easting E, northing N, and height H in the geographical +% frame of samples, from the sonar easting SE (in m), sonar northing SN +% (in m), sonar height SH (in m), sonar heading SHEADING (in radians +% relative to North?), and the samples' across distance SAMPACD (in m) +% and upward distance SAMPUPD (in m) in the swath frame. SE, SN, SH, and +% SHEADING must be matching 1P matrices or compatible (e.g. scalars for +% one ping). SAMPACD and SAMPUPD must be SBP tensors or compatible (e.g. +% SB matrices for 1 ping). +% +% See also CFF_GET_SAMPLES_DIST, CFF_GEOREFERENCE_SAMPLE + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 19-07-2022 + +% permute dimensions of input to get everything as SBP matrices +sonE = permute(sonE,[3,1,2]); +sonN = permute(sonN,[3,1,2]); +sonH = permute(sonH,[3,1,2]); +sonHead = permute(sonHead,[3,1,2]); + + +% get axes rotation matrix +R = CFF_2Drotmat(-sonHead); +% normally with column vectors X' = R'*X +xp = sonE + acDist.*R(1,1,:) + y.*R(2,1,:); +yp = sonN + acDist.*R(1,2,:) + y.*R(2,2,:); +zp = sonH + upDist; + +% % compute outputs +% sampleEasting = sonE + acDist.*cos(sonHead); +% sampleNorthing = sonN + acDist.*sin(sonHead); +% sampleHeight = sonH + upDist; + + + + + +end diff --git a/geoprocessing/samples_location/CFF_get_samples_dist.m b/geoprocessing/samples_location/CFF_get_samples_dist.m new file mode 100644 index 0000000..8b9e76a --- /dev/null +++ b/geoprocessing/samples_location/CFF_get_samples_dist.m @@ -0,0 +1,37 @@ +function [SBP_sampleAcrossDistance,SBP_sampleUpwardsDistance] = CFF_get_samples_dist(SBP_sampleRange,BP_beamPointingAngle) +%CFF_GET_SAMPLES_DIST Sample coordinates in swath frame from range & angle +% +% Computes the cartesian coordinates in the swath frame (i.e. the +% distances across and upwards from the sonar) of samples using their +% range and beam pointing angle. +% +% The swath frame is defined as: +% - origin: sonar face +% - Xs: across distance (positive ~starboard) +% - Ys: along distance (positive ~forward) +% - Zs: up distance (positive up) +% +% [ACD,UPD] = CFF_GET_SAMPLES_DIST(R,THETA) returns the across distance +% ACD and the upward distance UPD (both in m) in the swath frame of +% samples, from their range R from the sonar (in m) and the beam pointing +% angle THETA (in rad). R must be a SBP tensor or compatible (e.g. a SB +% matrix for 1 ping, or a 1BP matrix for a common sample across all beams +% and pings, etc.). THETA must be a BP matrix or compatible (e.g. B1 or +% 1P vector) of beam pointing angle in each ping/beam (in radians). The +% returned ACD and UPD are SBP matrices (or SB for 1 ping). +% +% See also CFF_GET_SAMPLES_RANGE, CFF_GET_SAMPLES_ENH, +% CFF_GEOREFERENCE_SAMPLE + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 19-07-2022 + +% permute dimension of angle input to get SBP matrix-compatible +SBP_beamPointingAngle = permute(BP_beamPointingAngle,[3,1,2]); + +% compute outputs +SBP_sampleAcrossDistance = -SBP_sampleRange.*sin(SBP_beamPointingAngle); +SBP_sampleUpwardsDistance = -SBP_sampleRange.*cos(SBP_beamPointingAngle); + +end \ No newline at end of file diff --git a/geoprocessing/samples_location/CFF_get_samples_range.m b/geoprocessing/samples_location/CFF_get_samples_range.m new file mode 100644 index 0000000..616de06 --- /dev/null +++ b/geoprocessing/samples_location/CFF_get_samples_range.m @@ -0,0 +1,40 @@ +function SBP_sampleRange = CFF_get_samples_range(SBP_idxSamples,BP_startSampleNumber,interSamplesDistance) +%CFF_GET_SAMPLES_RANGE Range of samples from indices +% +% Computes the range (in m from the sonar head) of samples, based on +% their index in a beam and the distance between two samples (in m). For +% water column data, the sample number must be corrected by a fixed index +% offset (startSampleNumber). +% +% SAMPR = CFF_GET_SAMPLES_RANGE(INDSAMP,STARTSAMPNUM,INTERSAMPDIST) +% returns the range SAMPR (in m) of samples based on their index INDSAMP, +% the corresponding beams' start index offset STARTSAMPNUM and the +% corresponding pings' inter-sample distance INTERSAMPDIST. The index of +% samples must be a SBP tensor or compatible (e.g. a 1BP matrix of one +% sample per ping and beam, or a S1 column vector of indices). The index +% offset STARTSAMPNUM must be a BP matrix or compatible (e.g. a B1 or 1P +% vector). The inter-sample distance INTERSAMPDIST must be a scalar or a +% ping vector (1P or P1) and is in m, as obtained from +% CFF_INTER_SAMPLE_DISTANCE. +% +% See also CFF_INTER_SAMPLE_DISTANCE, CFF_GET_SAMPLES_DIST, +% CFF_GEOREFERENCE_SAMPLE + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 19-07-2022 + +% permute dimensions of input to get everything as SBP matrix-compatible +SBP_startSampleNumber = permute(BP_startSampleNumber,[3,1,2]); % 1BP matrix +interSamplesDistance = reshape(interSamplesDistance,[1,numel(interSamplesDistance)]); % ensure 1P vector +SBP_interSamplesDistance = permute(interSamplesDistance,[3,1,2]); % 11P vector + +% compute sample range +SBP_sampleRange = (SBP_idxSamples+SBP_startSampleNumber).*SBP_interSamplesDistance; + +end + + + + + diff --git a/geoprocessing/samples_location/CFF_inter_sample_distance.m b/geoprocessing/samples_location/CFF_inter_sample_distance.m new file mode 100644 index 0000000..224ba07 --- /dev/null +++ b/geoprocessing/samples_location/CFF_inter_sample_distance.m @@ -0,0 +1,49 @@ +function interSamplesDistance = CFF_inter_sample_distance(fData, varargin) +%CFF_INTER_SAMPLE_DISTANCE Distance (in m) between two data samples +% +% Returns the distance in meters between two samples of a beam +% time-series in fData, based on the sound speed and sample frequency as +% recorded in the data. Note that sampling frequency may have been +% modified to account for the decimation in samples when reading the +% data. +% +% INTERSAMPLESDISTANCE = CFF_INTER_SAMPLE_DISTANCE(FDATA) returns the +% inter-sample distance for all pings in FDATA. +% +% INTERSAMPLESDISTANCE = CFF_INTER_SAMPLE_DISTANCE(FDATA,IPINGS) returns +% the inter-sample distance only for specified ping indices IPINGS in +% FDATA. +% +% See also CFF_GET_SAMPLES_RANGE + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 28-07-2022 + +% input parsing +p = inputParser; +addRequired(p,'fData',@(x) CFF_is_fData_version_current(x)); +addOptional(p,'iPings',[]); +parse(p,fData,varargin{:}); +iPings = p.Results.iPings; +clear p; + +% get sound speed and sampling frequency +datagramSource = CFF_get_datagramSource(fData); +switch datagramSource + case {'WC','AP'} + sound_speed = fData.(sprintf('%s_1P_SoundSpeed',datagramSource)); % m/s + sampling_freq = fData.(sprintf('%s_1P_SamplingFrequencyHz',datagramSource)); % Hz + case 'X8' + sound_speed = fData.X8_1P_SoundSpeedAtTransducer; % m/s + sampling_freq = fData.X8_1P_SamplingFrequencyInHz; % Hz +end + +% calculate +interSamplesDistance = sound_speed./(sampling_freq.*2); % in m + +% return only for desired pings +if ~isempty(iPings) + interSamplesDistance = interSamplesDistance(iPings); +end + diff --git a/functions/gis_raster/CFF_clip_raster.m b/gis_raster/CFF_clip_raster.m similarity index 100% rename from functions/gis_raster/CFF_clip_raster.m rename to gis_raster/CFF_clip_raster.m diff --git a/functions/gis_raster/CFF_coregister_rasters.m b/gis_raster/CFF_coregister_rasters.m similarity index 100% rename from functions/gis_raster/CFF_coregister_rasters.m rename to gis_raster/CFF_coregister_rasters.m diff --git a/functions/gis_raster/CFF_grid.m b/gis_raster/CFF_grid.m similarity index 100% rename from functions/gis_raster/CFF_grid.m rename to gis_raster/CFF_grid.m diff --git a/functions/gis_raster/CFF_interpolate_grid.m b/gis_raster/CFF_interpolate_grid.m similarity index 100% rename from functions/gis_raster/CFF_interpolate_grid.m rename to gis_raster/CFF_interpolate_grid.m diff --git a/mosaic/CFF_add_to_mosaic.m b/mosaic/CFF_add_to_mosaic.m new file mode 100644 index 0000000..77d63a2 --- /dev/null +++ b/mosaic/CFF_add_to_mosaic.m @@ -0,0 +1,190 @@ +function mosaic = CFF_add_to_mosaic(mosaic,x,y,v,varargin) +%CFF_ADD_TO_MOSAIC Update mosaic with new data +% +% MOSAIC = CFF_ADD_TO_MOSAIC(MOSAIC,X,Y,V) adds data with value V at +% coordinates X and Y to an existing MOSAIC, and outputs the updated +% MOSAIC. MOSAIC is a mosaic structure such as initialized with +% CFF_INIT_MOSAIC_V2 or resulting from a prior use of CFF_ADD_TO_MOSAIC. +% X,Y, and V can be all vectors, or all 2D arrays, in which case they +% need to have matching size. Or V can be a matrix and X and Y its vector +% coordinates with X as a row vector and Y a column vector. Note you need +% to run CFF_FINALIZE_MOSAIC when you have finished adding all desired +% data to the mosaic. +% +% CFF_ADD_TO_MOSAIC(...,W) also uses the weights W corresponding to the +% values V. W must be same size as V, or a single value that applies to +% all data. By default, W = 1. The MOSAIC field MOSAIC.mode governs how +% weight is used to update the mosaic. With 'blend', the new and existing +% data get (possibly weighted) averaged. Actual weights can be used to +% privilege some data, but by default, the weight of a cell is the number +% of data points that contributed to a cell value, so the iterative +% weighted averaging is equivalent to a normal averaging. With 'stitch', +% we retain for each cell whichever data has largest weight. Actual +% weights can be used to privilege some data, but by default, the new +% data takes precedence over the old. +% +% Note that the averaging in 'blend' mode is performed on input values V +% "as is". If V is backscatter data and you don't want to avearge in dB, +% you need to transform the values before using this function, and apply +% the reverse transformation when the mosaicking is complete. See +% CFF_MOSAIC_LINES. +% +% See also CFF_MOSAIC_LINES, CFF_INIT_MOSAIC_V2, CFF_FINALIZE_MOSAIC + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) +% 2017-2022; Last revision: 06-04-2022 + +% input parser +p = inputParser; +addRequired(p,'mosaic',@(x) isstruct(x)); +addRequired(p,'x',@(u) validateattributes(u,{'numeric'},{'2d'})); +addRequired(p,'y',@(u) validateattributes(u,{'numeric'},{'2d'})); +addRequired(p,'v',@(u) validateattributes(u,{'numeric'},{'2d'})); +addOptional(p,'w',1, @(u) isnumeric(u)); +parse(p,mosaic,x,y,v,varargin{:}); +w = p.Results.w; +clear p; + +% input additional checks and data preparation +if ~isempty(setxor(fieldnames(mosaic),fieldnames(CFF_init_mosaic_v2([0,1,0,1])))) + error("'mosaic' must be a mosaic struct as produced by CFF_INIT_MOSAIC_V2."); +end +sx = size(x); +sy = size(y); +sv = size(v); +check_xyv_same_size = all(sx==sy)&&all(sx==sv); +check_xyv_grid = sx(1)==1&&sx(2)==sv(2)&&sy(1)==sv(1)&&sy(2)==1; +if ~check_xyv_same_size && ~check_xyv_grid + error("The formats of 'x', 'y' and 'v' are not correct. They must either have the same size, or 'v' must be a 2D array with 'x' and 'y' as its vector coordinates."); +end +if numel(w)==1 + w = w.*ones(sv); +else + sw = size(w); + if ~all(sw==sv) + error("'w' must be same size as v."); + end +end + +% meshgrid x,y +if check_xyv_grid + x = repmat(x,sv(1),1); + y = repmat(y,1,sv(2)); +end + +% vectorize everything +if size(x,2)>1 + x = x(:); + y = y(:); + v = v(:); + w = w(:); +end + +% indices of new data in the mosaic +ix = round((x-mosaic.xg(1))/mosaic.res+1); +iy = round((y-mosaic.yg(1))/mosaic.res+1); + +% remove data 1) outside of mosaic boundaries, 2) with a nan value, 3) +% with zero weight +iKeep = ix>=1 & ix<=numel(mosaic.xg) ... + & iy>=1 & iy<=numel(mosaic.yg) ... + & ~isnan(x) & ~isnan(y) & ~isnan(v) & ~isnan(w) & w~=0; +if any(iKeep==0) + ix = ix(iKeep); + iy = iy(iKeep); + x = x(iKeep); + y = y(iKeep); + v = v(iKeep); + w = w(iKeep); +end + +% if ROI was a polygon, we also remove all data outside it +if ~isempty(mosaic.x_pol) + iKeep = inpolygon(x,y,mosaic.x_pol,mosaic.y_pol); + if any(iKeep==0) + ix = ix(iKeep); + iy = iy(iKeep); + v = v(iKeep); + w = w(iKeep); + end +end + +% indices of block (extract of mosaic where new data will contribute) in +% mosaic +iR_block = min(iy):max(iy); +iC_block = min(ix):max(ix); + +% indices of new data in block +iy_in_blc = iy - min(iy) + 1; +ix_in_blc = ix - min(ix) + 1; + +% prepare for accumaray the new data +subs = [iy_in_blc ix_in_blc]; % indices of new data in block +sz = [numel(iR_block) numel(iC_block)]; % size of output array + +% regrid the new data in a new block. Since the data were originally +% gridded at the same resolution, there should only be one point per block +% cell, so we can use any function, both for value and weight. +new_v_blc = accumarray(subs,v,sz,@min,0); % gridded value +new_w_blc = accumarray(subs,w,sz,@min,0); % gridded weight + +% get the current values and weights in the block +cur_v_blc = mosaic.value(iR_block,iC_block); +cur_w_blc = mosaic.weight(iR_block,iC_block); + +% next we merge the new block data into the current mosaic +switch mosaic.mode + + case 'blend' + % In this mode, for each grid cell in the block, the current value + % and new value are averaged using their respective weights. This + % has for effect of "blending" overlapping lines together. + + % we sum the current block and the new block weight to get the + % updated weight + upd_w_blc = cur_w_blc + new_w_blc; + + % and the updated value is the average of current and new values, + % weighted by their respective weights. + upd_v_blc = ((cur_v_blc.*cur_w_blc)+(new_v_blc.*new_w_blc))./upd_w_blc; + + % where the updated weight is still zero (no current nor new data), + % we get nan updated values, which we don't want to keep. Reset + % those updated values to zero + upd_v_blc(upd_w_blc==0) = 0; + + case 'stitch' + % In this mode, for each grid cell in the block, we retain either + % the current value or the new value, whichever has the highest + % weight. This mode is typically used with the inverse of the + % distance to nadir as the weight, which effectively results in + % "stitching" lines data together with stitches occuring at + % equidistance from the vessel tracks. + + % get indice of data to retain (1=new, 2=current) + [~,ind] = max([new_w_blc(:),cur_w_blc(:)],[],2,'omitnan'); + ind = reshape(ind,size(cur_w_blc)); + + % updated data is new data where ind=1 and current data where ind=2 + upd_v_blc = new_v_blc.*(ind==1) + cur_v_blc.*(ind==2); + upd_w_blc = new_w_blc.*(ind==1) + cur_w_blc.*(ind==2); + + case 'min' + % In this mode, for each grid cell in the block, we retain + % whichever value is the smallest. + % We only use weights to check where there is data + new_v_blc(new_w_blc==0) = NaN; + cur_v_blc(cur_w_blc==0) = NaN; + upd_v_blc = min(new_v_blc,cur_v_blc,'omitnan'); + upd_v_blc(isnan(upd_v_blc)) = 0; + + % the udpated weight is where there is data + upd_w_blc = double(cur_w_blc+new_w_blc>0); + +end + +% update mosaic with updated block value and weight +mosaic.value(iR_block,iC_block) = upd_v_blc; +mosaic.weight(iR_block,iC_block) = upd_w_blc; + +end diff --git a/mosaic/CFF_finalize_mosaic.m b/mosaic/CFF_finalize_mosaic.m new file mode 100644 index 0000000..317855c --- /dev/null +++ b/mosaic/CFF_finalize_mosaic.m @@ -0,0 +1,17 @@ +function mosaic = CFF_finalize_mosaic(mosaic) +%CFF_FINALIZE_MOSAIC Finalize a mosaic +% +% MOSAIC = CFF_FINALIZE_MOSAIC(MOSAIC) finalizes MOSAIC after all +% accumulating with CFF_ADD_TO_MOSAIC is finished. The only thing it does +% is set to NaN the grid cells that did not get filled during mosaicking. +% See CFF_INIT_MOSAIC_V2 for explanations why it is needed. +% +% See also CFF_MOSAIC_LINES, CFF_INIT_MOSAIC_V2, CFF_ADD_TO_MOSAIC + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) +% 2017-2022; Last revision: 06-04-2022 + +mosaic.value(mosaic.weight == 0) = NaN; + +end + diff --git a/mosaic/CFF_init_mosaic.m b/mosaic/CFF_init_mosaic.m new file mode 100644 index 0000000..79f2b9c --- /dev/null +++ b/mosaic/CFF_init_mosaic.m @@ -0,0 +1,44 @@ +function mosaic = CFF_init_mosaic(E_lim,N_lim,res,varargin) +%CFF_INIT_MOSAIC Initialize a new mosaic +% +% CFF_INIT_MOSAIC(E_lim,N_lim,res,mode) initializes a new mosaic object +% to be later filled with data. Required input are the Easting and +% Northing boundaries (resp E_lim and N_lim as two elements vectors) and +% the resolution, all of them in m. The optional input mode, is either +% "blend" (default) or "stitch". + +% Authors: Yoann Ladroit (NIWA, yoann.ladroit@niwa.co.nz) and Alex +% Schimel (NGU, alexandre.schimel@ngu.no) +% 2017-2021; Last revision: 25-02-2022 + +% input parser +p = inputParser; +addRequired(p,'E_lim',@(x) isnumeric(x) && all(size(x)==[1,2]) && x(1) 0 + numElemMosaicE = ceil((E_lim(2)-E_lim(1))./res)+1; + numElemMosaicN = ceil((N_lim(2)-N_lim(1))./res)+1; + mosaic.mosaic_level = zeros(numElemMosaicN,numElemMosaicE,'single'); +else + mosaic.mosaic_level = single([]); +end diff --git a/mosaic/CFF_init_mosaic_v2.m b/mosaic/CFF_init_mosaic_v2.m new file mode 100644 index 0000000..815e7ce --- /dev/null +++ b/mosaic/CFF_init_mosaic_v2.m @@ -0,0 +1,87 @@ +function mosaic = CFF_init_mosaic_v2(xy_roi,varargin) +%CFF_INIT_MOSAIC_V2 Initialize a new mosaic +% +% MOSAIC = CFF_INIT_MOSAIC_V2(XY_ROI) initializes a new mosaic to be +% later iteratively filled with data using CFF_ADD_TO_MOSAIC and +% finalized with CFF_FINALIZE_MOSAIC. XY_ROI are the x,y coordinates of +% the ROI to be mosaicked and can be of two types: either a 4-elements +% vector containing the desired min and max limits in x and y of a box +% [x_min x_max y_min y_max], OR a Nx2 array (with N>=3) where each row is +% the x,y coordinates of a vertex of a polygon inside which the mosaic is +% to be calculated. MOSAIC is a mosaic structure whose fields include the +% two grids 'value' (containing the mosaicked value) and 'weight' +% (containing the accumulated weight, see option 'mode' below), and other +% metadata. +% +% CFF_INIT_MOSAIC_V2(...,'res',VALUE) specifies the desired grid size +% (resolution). Use the same unit as XY_ROI. VALUE is 1 by default. +% +% CFF_INIT_MOSAIC_V2(...,'mode',VALUE) specifies the mosaicking mode, +% i.e. the rules of how new data gets merged with existing data when +% adding to the mosaic. Options are 'blend' (default) or 'stitch'. With +% 'blend', the new and existing data get (possibly weighted) averaged. +% Actual weights can be used to privilege some data, but by default, the +% weight of a cell is the number of data points that contributed to a +% cell value, so the iterative weighted averaging is equivalent to a +% normal averaging. With 'stitch', we retain for each cell whichever data +% has largest weight. Actual weights can be used to privilege some data, +% but by default, the new data takes precedence over the old. See +% CFF_ADD_TO_MOSAIC for detail on accumulating algorithms. +% +% See also CFF_MOSAIC_LINES, CFF_ADD_TO_MOSAIC, CFF_FINALIZE_MOSAIC + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) +% 2017-2022; Last revision: 06-04-2022 + +% input parser +p = inputParser; +addRequired(p,'xy_roi',@(u) validateattributes(u,{'numeric'},{'2d'})); +addParameter(p,'res',1,@(u) validateattributes(u,{'numeric'},{'scalar','positive'})); +addParameter(p,'mode','blend',@(u) ismember(u,{'blend','stitch','min'})); +parse(p,xy_roi,varargin{:}); +res = p.Results.res; +mode = p.Results.mode; +clear p; + +% reformat ROI x,y +if isvector(xy_roi) && numel(xy_roi)==4 + % x,y input was box limits + x_lim = reshape(sort(xy_roi(1:2)),1,[]); + y_lim = reshape(sort(xy_roi(3:4)),1,[]); + + % no polygon + x_pol = []; + y_pol = []; +elseif size(xy_roi,1)>2 && size(xy_roi,2)==2 + % x,y input was polygon vertices + x_pol = xy_roi(:,1); + y_pol = xy_roi(:,2); + + % define box limits from them + x_lim = [min(x_pol), max(x_pol)]; + y_lim = [min(y_pol), max(y_pol)]; +else + error("The format of 'xy_roi' is not correct. It must be either a 4-elements vector or a N*2 array with N>2."); +end + +% create x,y grids +[xg,yg] = CFF_init_grid(x_lim,y_lim,res); + +% initialize value. These cannot be NaN because this would break the +% accumulation algorithm. We initialize with zero instead and we can refer +% the where weight = 0 at the end of the accumlation to identify where no +% data were contributed. +value = zeros(numel(yg),numel(xg)); + +% initialize weight. +weight = zeros(numel(yg),numel(xg)); + +% save in mosaic structure +mosaic.xg = xg; +mosaic.yg = yg; +mosaic.x_pol = x_pol; +mosaic.y_pol = y_pol; +mosaic.res = res; +mosaic.mode = mode; +mosaic.value = value; +mosaic.weight = weight; diff --git a/mosaic/CFF_mosaic_lines.m b/mosaic/CFF_mosaic_lines.m new file mode 100644 index 0000000..f36a741 --- /dev/null +++ b/mosaic/CFF_mosaic_lines.m @@ -0,0 +1,193 @@ +function mosaic = CFF_mosaic_lines(fDataGroup,fieldname,varargin) +%CFF_MOSAIC_LINES Mosaic a set of gridded lines +% +% MOSAIC = CFF_MOSAIC_LINES(FDATAGROUP,FIELDNAME), where FDATAGROUP is a +% cell array of fData structures and FIELDNAME is the (string) name of a +% gridded data field common to these structures, creates a mosaic of that +% data. MOSAIC is a mosaic structure whose fields include the two grids +% 'value' (containing the mosaicked value) and 'weight' (containing the +% accumulated weight, see option 'mode' below), and other metadata. +% +% CFF_MOSAIC_LINES(...,'xy_roi',VALUE) specifies the x,y coordinates of +% the ROI to be mosaicked and can be of two types: either a 4-elements +% vector containing the desired min and max limits in x and y of a box +% [x_min x_max y_min y_max], OR a Nx2 array (with N>=3) where each row is +% the x,y coordinates of a vertex of a polygon inside which the mosaic is +% to be calculated. +% +% CFF_MOSAIC_LINES(...,'res',VALUE) with VALUE a positive scalar +% specifies the desired mosaic grid size (resolution) (use the same unit +% as 'xy_roi', i.e. usually, meters). With VALUE empty (default), the +% grid size will be selected as the coarsest grid size of the data +% gridded in FDATAGROUP. Note that while you can set here a finer +% resolution than that of the gridded data, the end-product will not +% appear as more detailed than the componenent grids. To get a better +% resolution, you would need to re-grid the files at a finer grid size. +% +% CFF_MOSAIC_LINES(...,'mode',VALUE) specifies the mosaicking mode, i.e. +% the rules of how new data gets merged with existing data when adding to +% the mosaic. Options are 'blend' (default) or 'stitch'. With 'blend', +% the new and existing data get (possibly weighted) averaged. Actual +% weights can be used to privilege some data, but by default, the weight +% of a cell is the number of data points that contributed to a cell +% value, so the iterative weighted averaging is equivalent to a normal +% averaging. With 'stitch', we retain for each cell whichever data has +% largest weight. Actual weights can be used to privilege some data, but +% by default, the new data takes precedence over the old. See +% CFF_ADD_TO_MOSAIC for detail on accumulating algorithms. +% +% CFF_MOSAIC_LINES(...,'comms',VALUE) specifies the logging and display +% method. See CFF_COMMS. +% +% See also CFF_INIT_MOSAIC_V2, CFF_ADD_TO_MOSAIC, CFF_FINALIZE_MOSAIC + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) +% 2017-2022; Last revision: 06-04-2022 + +% input parser +p = inputParser; +addRequired(p,'fDataGroup',@(x) all(CFF_is_fData_version_current(x))); +addRequired(p,'fieldname',@(x) ischar(x)); +addParameter(p,'xy_roi',@(u) validateattributes(u,{'numeric'},{'2d'})); +addParameter(p,'res',[],@(x) isempty(x) || x>0); +addParameter(p,'mode','blend', @(u) ischar(u)&&ismember(u,{'blend','stitch','min'})); +addParameter(p,'comms',CFF_Comms()); +parse(p,fDataGroup,fieldname,varargin{:}); +xy_roi = p.Results.xy_roi; +res = p.Results.res; +mode = p.Results.mode; +comms = p.Results.comms; +clear p; +if ischar(comms) + comms = CFF_Comms(comms); +end + +% first, check that fieldname exist in every fData +if ~all(cellfun(@(s) isfield(s,fieldname),fDataGroup)) + error('Not all fData structures in "fDataGroup" have the required field "%s"',fieldname); +end + +% start message +comms.start(sprintf('Mosaicking "%s" data from line(s)',fieldname)); + +% number of files +nLines = numel(fDataGroup); + +% start progress +comms.progress(0,nLines); + +% for mosaic grid size, we use by default the coarsest resolution of +% component grids +if isempty(res) + res = max(cellfun(@(x) x.X_1_2DgridHorizontalResolution, fDataGroup)); +end + +% if no ROI in input, do the entire dataset +if isempty(xy_roi) + xy_roi = [NaN NaN NaN NaN]; + for iF = 1:numel(fDataGroup) + xy_roi(1) = min( xy_roi(1), min(fDataGroup{iF}.X_1E_gridEasting) ); + xy_roi(2) = max( xy_roi(2), max(fDataGroup{iF}.X_1E_gridEasting) ); + xy_roi(3) = min( xy_roi(3), min(fDataGroup{iF}.X_N1_gridNorthing) ); + xy_roi(4) = max( xy_roi(4), max(fDataGroup{iF}.X_N1_gridNorthing) ); + end +end + +% initialize mosaic +mosaic = CFF_init_mosaic_v2(xy_roi,'res',res,'mode',mode); + +% flag for special processing if we average backscatter values +flagAverageBackscatter = 0; +if strcmp(fieldname,'X_NE_bs') && strcmp(mosaic.mode,'blend') + flagAverageBackscatter = 1; +end + +if flagAverageBackscatter + % for backscatter and mosaicking modes involving averaging (i.e. + % 'blend') one needs to decide whether to average the dB values, or the + % equivalent amplitude or power values. The "mathematically" correct + % one is power, but is strongly affected by outliers. The best choice + % "aesthetically" is to use dB. We set here the transformation + % necessary before data is averaged, and the reverse transformation to + % get back in dB. For now the choice is hard-code, but perhaps + % eventually turn it as an input parameter + bs_averaging_mode = 'power'; + switch bs_averaging_mode + case 'dB' + % no transformation as data is natively in dB. + bs_trsfm = @(x) x; + inv_trsfm = @(x) x; + case 'amplitude' + bs_trsfm = @(x) 10.^(x./20); % dB to amplitude + inv_trsfm = @(x) 20.*log10(x); % amplitude to dB + case 'power' + bs_trsfm = @(x) 10.^(x./10); % dB to power + inv_trsfm = @(x) 10.*log10(x); % power to dB + end +end + +% loop through fData +for ii = 1:nLines + + % get fData for this line + if iscell(fDataGroup) + fData = fDataGroup{ii}; + else + fData = fDataGroup; + end + + % display for this line + lineName = CFF_file_name(char(CFF_onerawfileonly(fData.ALLfilename)),1); + comms.step(sprintf('%i/%i: line %s',ii,nLines,lineName)); + + % get x,y,v data + x = fData.X_1E_2DgridEasting; + y = fData.X_N1_2DgridNorthing; + v = fData.(fieldname); + + if flagAverageBackscatter + % transform backscatter now. Mosaic will remain in transformed unit + % until finalization. + v = bs_trsfm(v); + end + + % set weight + switch mosaic.mode + case 'blend' + % weight here is the number of points per gridded cell + w = fData.X_NE_weight; + case 'stitch' + % weight here is the inverse of the distance of the grid cell + % to nadir + tempIndBP = fData.X_NE_indexBP; + indNan = isnan(tempIndBP); + tempIndBP(indNan) = 1; + w = 1./abs(fData.X8_BP_AcrosstrackDistanceY(tempIndBP)); + w(indNan) = 0; + case 'min' + % no need for weight in this mode, use dummy value + w = 1; + end + % add to mosaic + mosaic = CFF_add_to_mosaic(mosaic,x,y,v,w); + + % successful end of this iteration + comms.info('Done'); + + % communicate progress + comms.progress(ii,nLines); + +end + +% finalize mosaic +mosaic = CFF_finalize_mosaic(mosaic); + +if flagAverageBackscatter + % apply reverse transform to backscatter + mosaic.value = inv_trsfm(mosaic.value); +end + +% end message +comms.finish('Done'); + +end \ No newline at end of file diff --git a/processing/CFF_analyze_backscatter.m b/processing/CFF_analyze_backscatter.m new file mode 100644 index 0000000..fbf4095 --- /dev/null +++ b/processing/CFF_analyze_backscatter.m @@ -0,0 +1,143 @@ +function fDataGroup = CFF_analyze_backscatter(fDataGroup,varargin) + +global DEBUG + +% input parser +p = inputParser; +addRequired(p,'fDataGroup',@(x) all(CFF_is_fData_version_current(x))); +addParameter(p,'comms',CFF_Comms()); +parse(p,fDataGroup,varargin{:}); +comms = p.Results.comms; +clear p; +if ischar(comms) + comms = CFF_Comms(comms); +end + +% start message +comms.start('Analyzing backscatter in line(s)'); + +% number of lines +nLines = numel(fDataGroup); + +% start progress +comms.progress(0,nLines); + +% display to check +if DEBUG + figure(999); + tiledlayout(2,1); +end + +% process per file +for ii = 1:nLines + + % get fData for this line + fData = fDataGroup{ii}; + + % display for this line + lineName = CFF_file_name(char(CFF_onerawfileonly(fData.ALLfilename)),1); + comms.step(sprintf('%i/%i: line %s',ii,nLines,lineName)); + + % data dimension + [nBeams, nPings] = size(fData.X8_BP_ReflectivityBS); + + if DEBUG + % display BS + ax1 = nexttile; + imagesc(fData.X8_BP_ReflectivityBS); + colormap gray; caxis([-40 0]); grid on + hold on + title(lineName,'Interpreter','none'); + end + + %% step 1. find bad soundings + % criteria 1: identifying bad beams from detection info + [DetectInfo, ~] = CFF_decode_X8_DetectionInfo(fData.X8_BP_DetectionInformation); + badSoundings = DetectInfo>1; + + if DEBUG + % display bad soundings + [row,col] = ind2sub([nBeams, nPings],find(badSoundings)); + plot(col,row,'y*'); + end + + %% step 2. find bad pings + + % criteria 1: a ping with a percentage of bad soundings that exceeds a + % given percentage threshold is a bad ping + thr = 0.1; % between 0 and 1 + badPings1 = (sum(badSoundings)./nBeams) > thr; + + if DEBUG + % display bad pings due to too many bad soundings + [row,col] = ind2sub([nBeams, nPings],find(badSoundings.*badPings1)); + plot(col,row,'r*'); + end + + % criteria 2: a ping whose average backscatter level is abnormally low + % is a bad ping + avgBS = median(fData.X8_BP_ReflectivityBS); + + iKeep = zeros(1,nPings); + lowth = nan(1,nPings); + + trailNum = 20; + iKeep(1:trailNum) = 1; + + iKeep(1:trailNum) = 1; + + for jj = trailNum+1:nPings + idx = find(iKeep,trailNum,'last'); + dat = avgBS(idx); + lowth(jj) = median(dat) - 2.*range(dat); + if (avgBS(jj)=nThr; + + %% step 4. save results + fData.X_BP_goodData = ~badSoundings; + fData.X_1P_badPing = badPings; + fData.X_1P_toResurvey = badSections; + fDataGroup{ii} = fData; + + % successful end of this iteration + comms.info('Done'); + + % communicate progress + comms.progress(ii,nLines); + +end + +% end message +comms.finish('Done'); + +end \ No newline at end of file diff --git a/processing/CFF_filter_bottom_detect_v2.m b/processing/CFF_filter_bottom_detect_v2.m new file mode 100644 index 0000000..72b4257 --- /dev/null +++ b/processing/CFF_filter_bottom_detect_v2.m @@ -0,0 +1,312 @@ +function [fData,params] = CFF_filter_bottom_detect_v2(fData,varargin) +%CFF_FILTER_BOTTOM_DETECT_V2 Filter and interpolate the bottom detections +% +% Bottom detects are the sample number in each beam corresponding to the +% bottom, according to the bottom detect algorithms. This +% filtering/interpolating algorithm can be applied to data either in +% bathymetry, or water-column datagrams. The bottom samples must have +% been previously geoprocessed. +% +% FDATA = CFF_FILTER_BOTTOM_DETECT_V2(FDATA) filters the bottom detect in +% FDATA using default processing parameters, and returns +% FDATA with additional fields from the processing. +% +% CFF_FILTER_BOTTOM_DETECT_V2(FDATA,PARAMS) uses processing parameters +% defined as the fields in the PARAMS structure. Possible parameters are: +% 'sourceBottom': string for the bottom detects to start from, either +% 'raw' (default) or 'processed'. 'raw' starts from the raw detects as +% recorded in the raw data. 'processed' starts from the +% previously-processed detects, in order to allow for extra-processing. +% 'method': string for the filtering method to be applied, either +% 'filter' (default) or 'flag'. With 'filter' the bottom detects are +% filtered using a 1D or 2D median filter (see other parameters). With +% 'flag' the bottom detects are either conserved or removed, based on +% whether they fit a criteria (see other parameters). +% 'pingBeamWindowSize': a two-elements vector (first for pings, second +% for beams) of non-negative integers defining the window size around a +% detect to consider for the filtering/flagging. By default using [3,3] +% that is, we consider windows of 7 beams (f3 starboard, 3 port) and 7 +% pings (3 prior, 3 after) around the detect of interest. +% 'maxHorizDist': nonnegative scalar defining the maximum horizontal +% distance (in m) to keep windows elements. Use maxHorizDist = inf to +% indicate NOT using a max horizontal distance (default). +% 'flagVariable': string code defining the variable to be calculated for +% each element of the window, when using the 'flag' method. Can be 'vert' +% (vertical distance, default), 'eucl' (euclidian distance), or 'slope' +% (slope in degrees). All are relative to the bottom detect. +% 'flagThreshold': scalar value defining the threshold for the variable +% above, i.e. distance (in m) or slope (in degrees). +% 'flagType': string code defining the function applied to return a +% single value from the window, when using the flag method. Can be +% 'median' (i.e. only the median value needs to meet the criteria, +% default), 'all' (i.e. ALL the values in the window must meet the +% criteria), or 'any' (i.e. AT LEAST ONE value in the window must meet +% the criteria). +% 'interpolateFlag': boolean indicating if remaining NaNs after filtering +% or flagging must be interpolated. Default is 1, indicating +% interpolating is requested. +% +% CFF_FILTER_BOTTOM_DETECT_V2(...,'comms',COMMS) specifies if and how +% this function communicates on its internal state (progress, info, +% errors). COMMS can be either a CFF_COMMS object, or a text string to +% initiate a new CFF_COMMS object. Options are 'disp', 'textprogressbar', +% 'waitbar', 'oneline', 'multilines'. By default, using an empty +% CFF_COMMS object (i.e. no communication). See CFF_COMMS for more +% information. +% +% [FDATA,PARAMS] = CFF_FILTER_BOTTOM_DETECT_V2(...) also outputs the +% parameters used in processing. +% +% See also CFF_GEOREFERENCE_BOTTOM_DETECT, CFF_GROUP_PROCESSING. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 27-07-2022 + +global DEBUG +% DEBUG = 1; + +%% Input arguments management +p = inputParser; +addRequired(p,'fData',@(x) CFF_is_fData_version_current(x)); % line fData to process +addOptional(p,'params',struct(),@(x) isstruct(x)); % processing parameters +addParameter(p,'comms',CFF_Comms()); % information communication (none by default) +parse(p,fData,varargin{:}); +params = p.Results.params; +comms = p.Results.comms; +clear p +if ischar(comms) + comms = CFF_Comms(comms); +end + + +%% Prep + +% start message +comms.start('Filtering the bottom detections'); + +% get sourceBottom parameter +if ~isfield(params,'sourceBottom'), params.sourceBottom = 'raw'; end % default +mustBeMember(params.sourceBottom,{'raw','processed'}); % validate +sourceBottom = params.sourceBottom; + +% get bottom data to process +b0 = CFF_get_bottom_sample(fData,'which',sourceBottom); +b0(b0==0) = NaN; % replace no detects by NaNs + +% get bottom easting, northing and height +bE = fData.X_BP_bottomEasting; +bN = fData.X_BP_bottomNorthing; +bH = fData.X_BP_bottomHeight; + +% dimensions +[nBeams, nPings] = size(b0); + +% initialize results +b1 = b0; + +comms.progress(0,4); + + +%% Processing parameters common to all methods + +% get method +if ~isfield(params,'method'), params.method = 'filter'; end % default +mustBeMember(params.method,{'filter','flag'}); % validate +method = params.method; + +% get maxHorizDist +if ~isfield(params,'maxHorizDist'), params.maxHorizDist = inf; end % default +mustBePositive(params.maxHorizDist); % validate +maxHorizDist = params.maxHorizDist; + +% get pingBeamWindowSize +if ~isfield(params,'pingBeamWindowSize'), params.pingBeamWindowSize = [3,3]; end % default +CFF_mustBeTwoNonnegativeUnsignedIntegers(params.pingBeamWindowSize); % validate +pingBeamWindowSize = params.pingBeamWindowSize; + + +%% Processing +switch method + + case 'filter' + + comms.step('Filtering'); + + if isinf(maxHorizDist) + + % filter method, with no limit on horiz distance + for pp = 1:nPings + for bb = 1:nBeams + % find the subset of all bottom detects within set + % interval in pings and beams + pmin = max(1,pp-pingBeamWindowSize(1)); + pmax = min(nPings,pp+pingBeamWindowSize(1)); + bmin = max(1,bb-pingBeamWindowSize(2)); + bmax = min(nBeams,bb+pingBeamWindowSize(2)); + % get bottom for subset + subBottom = b0(bmin:bmax,pmin:pmax); + % compute median value + b1(bb,pp) = median(subBottom(:),'omitnan'); + end + end + + else + + % filter method, with limit on horiz distance + for pp = 1:nPings + for bb = 1:nBeams + % find the subset of all bottom detects within set + % interval in pings and beams + pmin = max(1,pp-pingBeamWindowSize(1)); + pmax = min(nPings,pp+pingBeamWindowSize(1)); + bmin = max(1,bb-pingBeamWindowSize(2)); + bmax = min(nBeams,bb+pingBeamWindowSize(2)); + % get bottom for subset + subBottom = b0(bmin:bmax,pmin:pmax); + % get easting and northing + subEasting = bE(bmin:bmax,pmin:pmax); + subNorthing = bN(bmin:bmax,pmin:pmax); + % compute horizontal distance in m + subHzDist = sqrt( (bE(bb,pp)-subEasting).^2 + (bN(bb,pp)-subNorthing).^2 ); + % keep only subset within desired horizontal distance + subBottom(subHzDist>maxHorizDist) = NaN; + % compute median value + b1(bb,pp) = median(subBottom(:),'omitnan'); + end + end + + end + + case 'flag' + + comms.step('Flagging'); + + % flag method additional parameters + + % get flagType + if ~isfield(params,'flagType'), params.flagType = 'median'; end % default + mustBeMember(params.flagType,{'median','all','any'}); % validate + flagType = params.flagType; + + % get flagVariable + if ~isfield(params,'flagVariable'), params.flagVariable = 'vert'; end % default + mustBeMember(params.flagVariable,{'vert','eucl','slope'}); % validate + flagVariable = params.flagVariable; + + % get flagThreshold + if ~isfield(params,'flagThreshold'), params.flagThreshold = 1; end % default + mustBeNumeric(params.flagThreshold); % validate + flagThreshold = params.flagThreshold; + + % get/set flagging type first + switch flagType + case 'median' + f = @(x)median(x); + case 'all' + f = @(x)all(x); + case 'any' + f = @(x)any(x); + end + + % next, processing for each bottom detect (BT) + for pp = 1:nPings + for bb = 1:nBeams + % first off, flag method is inapplicable if BT doesn't + % exist + if isnan(b0(bb,pp)) + % keep b1(bb,pp) as Nan. + continue + end + % find the subset of all bottom detects within set interval + % in pings and beams + pmin = max(1,pp-pingBeamWindowSize(1)); + pmax = min(nPings,pp+pingBeamWindowSize(1)); + bmin = max(1,bb-pingBeamWindowSize(2)); + bmax = min(nBeams,bb+pingBeamWindowSize(2)); + + subHzDist = sqrt( (bE(bb,pp)-bE(bmin:bmax,pmin:pmax)).^2 + (bN(bb,pp)-bN(bmin:bmax,pmin:pmax)).^2 ); + subHzDist(subHzDist>maxHorizDist) = NaN; + + % if there are no subset left, flag that bottom anyway + if all(isnan(subHzDist(:))) + b1(bb,pp) = NaN; + continue + end + % compute vertical distance in m + subVertDist = bH(bmin:bmax,pmin:pmax)-bH(bb,pp); + % now switch on the flagging variable + switch flagVariable + case 'vert' + v = subVertDist(:); + case 'eucl' + % compute euclidian distance in m + subEucliDist = sqrt( subHzDist.^2 + subVertDist.^2); + v = subEucliDist(:); + case 'slope' + % compute slope in degrees + subSlope = abs(atan2d(subVertDist,subHzDist)); + v = subSlope(:); + end + % finally, apply flagging decision + if f(abs(v) > flagThreshold) + b1(bb,pp) = NaN; + end + end + end + +end + +comms.progress(1,4); + + +%% Interpolate + +% get interpolateFlag +if ~isfield(params,'interpolateFlag'), params.interpolateFlag = 1; end % default +mustBeNumericOrLogical(params.interpolateFlag); % validate +interpolateFlag = params.interpolateFlag; + +if interpolateFlag + comms.step('Interpolating'); + b1 = round(inpaint_nans(b1)); + % safeguard against inpaint_nans occasionally yielding numbers below + % zeros in areas where there are a lot of NaNs + b1(b1<1)=2; +end + +comms.progress(2,4); + + +%% Test display +if DEBUG + figure; tiledlayout(3,1); + minb = min([b0(:);b1(:)]); maxb=max([b0(:);b1(:)]); + ax1 = nexttile; imagesc(b0); colormap(jet); colorbar; title('range of raw bottom'); grid on; caxis([minb maxb]) + ax2 = nexttile; imagesc(b1); colormap(jet); colorbar; title('range of filtered bottom'); grid on; caxis([minb maxb]) + ax3 = nexttile; imagesc(b1-b0); colormap(jet); colorbar; title('filtered minus raw'); + linkaxes([ax1 ax2 ax3],'xy') +end + + +%% Saving results + +comms.step('Saving'); + +fData = CFF_set_bottom_sample(fData,b1); +fData.X_MET_bottomFilterParams = params; + +comms.progress(3,4); + + +%% Re-georeference bottom after filtering + +comms.step('Re-georeferencing'); + +fData = CFF_georeference_bottom_detect(fData); + +comms.progress(4,4); + + +%% End message +comms.finish('Done'); \ No newline at end of file diff --git a/processing/CFF_group_processing.m b/processing/CFF_group_processing.m new file mode 100644 index 0000000..976df54 --- /dev/null +++ b/processing/CFF_group_processing.m @@ -0,0 +1,270 @@ +function [fDataGroup,params] = CFF_group_processing(procFun,fDataGroup,varargin) +%CFF_GROUP_PROCESSING Apply fData processing function(s) to array of fData +% +% CoFFee processing functions are normally operating with a single fData +% structure in input as in: CFF_my_function_name(fData). +% CFF_GROUP_PROCESSING allows applying one or several such processing +% functions to cell arrays of fData, i.e. fDataGroup. For now, this +% function only works for processing functions that output an updated +% fData, as in: fData=CFF_my_function_name(fData). It also works with +% processing functions that take a parameter, as in: +% [fData,params]=CFF_my_function_name(fData,params). See +% CFF_COMPUTE_PING_NAVIGATION_V2 for example of a suitable processing +% function. +% +% FDATAGROUP = CFF_GROUP_PROCESSING(PROCFUN,FDATAGROUP), with PROCFUN +% being a suitable CFF processing function handle (e.g. +% @CFF_name_of_my_function), applies the function to the cell array of +% fData structures FDATAGROUP, and returns the cell array of processed +% fData structures. PROCFUN can also be a cell array of suitable function +% handles, in which case all functions are applied in chain to each +% element of FDATAGROUP before moving to the next element, with the fData +% output by a processing function used in input for the next processing +% function. No other input arguments are passed to PROCFUN, so if +% the PROCFUN function(s) require(s) parameters, then it/they will use +% default parameters. +% +% CFF_GROUP_PROCESSING(PROCFUN,FDATAGROUP,PARAMS) also passes the +% structure of parameters PARAMS in input to PROCFUN. If PROCFUN is a +% cell array of functions, CFF_GROUP_PROCESSING expects a matching array +% of cell parameter structures. Note that PARAMS gets updated with each +% application of PROCFUN to a fData, as in +% [FDATA,PARAMS]=PROCFUN(FDATA,PARAMS), implying that subsequent +% iterations may use a different PARAMS than earlier ones. For example, +% it is typical to pass an empty PARAMS to the function +% CFF_COMPUTE_PING_NAVIGATION_V2 in order to request default parameters, +% which are then output by that function. As a result, for the next +% fData, the function will use those defined parameters. +% +% [FDATAGROUP,PARAMS] = CFF_GROUP_PROCESSING(...) also outputs the +% parameters PARAMS in output of the last iteration of PROCFUN. +% +% CFF_GROUP_PROCESSING(...,'procMsg',MSG) uses the string of characters +% MSG as a communication message. If a single processing function is +% used, this message will be the start message. If several processing +% functions are in input, this message will be info messages for each +% step of the processing. Note that if there is no communication +% requested (see parameter 'comms'), then this parameter is ignored. By +% default (or specifying MSG = 'default'), the message is 'Applying +% PROCFUN'. +% +% CFF_GROUP_PROCESSING(...,'saveFDataToDrive',FLAG) with FLAG = 1 will +% force a re-write of fData on the disk after the data are processed. By +% default (FLAG = 0), fData on the disk are not re-written. +% +% CFF_GROUP_PROCESSING(...,'abortOnError',FLAG) with FLAG = 1 will +% interrupt processing if an error is encountered. By default (FLAG = 0), +% the error is logged and processing continues to the next fData. +% +% CFF_GROUP_PROCESSING(...,'comms',COMMS) specifies if and how this +% function communicates on its internal state (progress, info, errors). +% COMMS can be either a CFF_COMMS object, or a text string to initiate a +% new CFF_COMMS object. Options are 'disp', 'textprogressbar', 'waitbar', +% 'oneline', 'multilines'. By default, using an empty CFF_COMMS object +% (i.e. no communication). See CFF_COMMS for more information. +% +% See also CFF_COMPUTE_PING_NAVIGATION_V2, +% CFF_GEOREFERENCE_BOTTOM_DETECT_V2, CFF_FILTER_BOTTOM_DETECT. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2022-2022; Last revision: 27-07-2022 + + +%% Input arguments management +p = inputParser; + +% fData processing function (or cell array of functions) +OneProcFunCheck = @(x) isa(x,'function_handle'); +MultProcFunCheck = @(x) iscell(x) && all(cellfun(OneProcFunCheck,x)); +addRequired(p,'procFun',@(x) OneProcFunCheck(x) || MultProcFunCheck(x)); + +% array of fData structures +addRequired(p,'fDataGroup',@(x) iscell(x) && all(cellfun(@CFF_is_fData_version_current,x))); + +% function parameters (or cell array of parameters) +OneParamsCheck = @(x) isstruct(x); +MultParamsCheck = @(x) iscell(x) && all(cellfun(OneParamsCheck,x)); +addOptional(p,'params',struct(),@(x) OneParamsCheck(x)||MultParamsCheck(x)); + +% string message for each processing function ('default' for default +% message) +OneProcMsgCheck = @(x) ischar(x); +MultProcMsgCheck = @(x) iscell(x) && all(cellfun(OneProcMsgCheck,x)); +addParameter(p,'procMsg','default',@(x) OneProcMsgCheck(x) || MultProcMsgCheck(x)); + +% save fData to hard-drive? 0: no (default), 1: yes +addParameter(p,'saveFDataToDrive',0,@(x) mustBeMember(x,[0,1])); + +% what if error during conversion? 0: to next file (default), 1: abort +addParameter(p,'abortOnError',0,@(x) mustBeMember(x,[0,1])); + +% information communication (none by default) +addParameter(p,'comms',CFF_Comms()); + +% parse and clean up +parse(p,procFun,fDataGroup,varargin{:}); +params = p.Results.params; +procMsg = p.Results.procMsg; +saveFDataToDrive = p.Results.saveFDataToDrive; +abortOnError = p.Results.abortOnError; +comms = p.Results.comms; +clear p +if ischar(comms) + comms = CFF_Comms(comms); +end + + +%% Prep + +% if single procFun, params, and procMsg in input, turn to cell +if ~iscell(procFun), procFun = {procFun}; end +if ~iscell(params), params = {params}; end +if ~iscell(procMsg), procMsg = {procMsg}; end + +% number of processing functions +nFun = numel(procFun); + +% check that number of inputs match +if nFun == 1 + % one function + if numel(params)>1 || numel(procMsg)>1 + error('Too many params or procMsg in input'); + end +else + % more than one function + if numel(params)==1 + % but only 1 param, assumed to be used for all functions + % repeat input + params = repmat(params,size(procFun)); + end + if numel(procMsg)==1 + % but only 1 procMsg, assumed to be used for all functions + % repeat input + procMsg = repmat(procMsg,size(procFun)); + end + if numel(params)~=nFun || numel(procMsg)~=nFun + error('Number of procFun, params, and procMsg in input must match'); + end +end + +% start message +if nFun == 1 + % with a single processing function, the start message is that function + % message + startMsg = procMsg{1}; + if strcmp(startMsg,'default') + % default start message is using the name of the function + procFunInfo = functions(procFun{1}); + startMsg = sprintf('Applying %s',procFunInfo.function); + end +else + % with multiple processing functions, the start message is a generic + % message + startMsg = 'Applying multiple processing'; +end +comms.start(startMsg); + +% number of fData to process +nFData = numel(fDataGroup); + +% start progress +comms.progress(0,nFData); + + +%% Processing +for iFD = 1:nFData + + % try-catch sequence to allow continuing to next file if one fails + try + + % get fData + fData = fDataGroup{iFD}; + + % display for this line + filename = CFF_file_name(fData.ALLfilename{1}); + comms.step(sprintf('%i/%i: fData line %s',iFD,nFData,filename)); + + % apply each processing function in turn + for iFun = 1:nFun + + % if applying multiple processing functions, use each + % function's procMsg as an info message. If only one single + % function, this was done at the start. + if nFun>1 + infoMsg = procMsg{iFun}; + if strcmp(infoMsg,'default') + % default start message is using the name of the function + procFunInfo = functions(procFun{iFun}); + infoMsg = sprintf('Applying %s',procFunInfo.function); + end + comms.info(infoMsg) + end + + % function application is dependent on whether function takes + % parameters or not + if nargin(procFun{iFun}) == 1 + % processing function takes only one input (fData) + fData = feval(procFun{iFun},fData); + else + % processing function takes at least one argument + % (nargin>=2) or a varargin (nargin<0). In any case, give + % it parameters in input + if nargout(procFun{iFun}) == 1 + % processing function has only one output (fData) + fData = feval(procFun{iFun},fData,params{iFun}); + else + % processing function has at least one extra output + % argument (nargout>=2) or a varargout (nargout<0). In + % any case, assume the first is output parameters + [fData,params{iFun}] = feval(procFun{iFun},fData,params{iFun}); + end + + end + end + + % save fData to drive + if saveFDataToDrive + % get output folder and create it if necessary + wc_dir = CFF_converted_data_folder(fData.ALLfilename); + mat_fdata_file = fullfile(wc_dir, 'fData.mat'); + comms.info('Saving'); + save(mat_fdata_file,'-struct','fData','-v7.3'); + end + + % save fData + fDataGroup{iFD} = fData; + + % successful end of this iteration + comms.info('Done'); + + catch err + if abortOnError + % just rethrow error to terminate execution + rethrow(err); + else + % log the error and continue + errorFile = CFF_file_name(err.stack(1).file,1); + errorLine = err.stack(1).line; + errrorFullMsg = sprintf('%s (error in %s, line %i)',err.message,errorFile,errorLine); + comms.error(errrorFullMsg); + end + end + + % communicate progress + comms.progress(iFD,nFData); + +end + + +%% finalise + +% if we had a single function, output params as struct, not cell array of +% one struct +if nFun == 1 + params = params{1}; +end + + +%% end message +comms.finish('Done'); \ No newline at end of file diff --git a/processing/CFF_setup_block_processing.m b/processing/CFF_setup_block_processing.m new file mode 100644 index 0000000..3e44f09 --- /dev/null +++ b/processing/CFF_setup_block_processing.m @@ -0,0 +1,32 @@ +function blocks = CFF_setup_block_processing(nUnits,maxNumUnitsPerBlock) +%CFF_SETUP_BLOCK_PROCESSING Setup block processing manually +% +% Setup block processing manually +% +% BLOCKS = CFF_SETUP_BLOCK_PROCESSING(NUNITS,MAXNUMUNITSPERBLOCK) returns an +% Nx2 array BLOCKS where each row is the start and end indices for +% a block of units to process at a time (typically, pings), using the +% total number of units NUNITS, and a maximum number of units per block +% MAXNUMUNITSPERBLOCK. If MAXNUMUNITSPERBLOCK is superior or equal to +% NUNITS, then the function returns a single block. If +% MAXNUMUNITSPERBLOCK is inferior, then the function returns several +% blocks of the desired size, and a last, smaller one. +% +% See also CFF_SETUP_OPTIMIZED_BLOCK_PROCESSING. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 05-08-2022 + +% input arguments management +p = inputParser; +addRequired(p,'nUnits',@(x)isscalar(x)>(x,0)&(round(x)==x)); % total number of units +addRequired(p,'maxNumUnitsPerBlock',@(x)isscalar(x)>(x,0)&(round(x)==x)); % max number of units per block +parse(p,nUnits,maxNumUnitsPerBlock); +clear p + +nBlocks = ceil(nUnits/maxNumUnitsPerBlock); +blocks = [ 1+(0:nBlocks-1)'.*maxNumUnitsPerBlock , (1:nBlocks)'.*maxNumUnitsPerBlock ]; +blocks(end,2) = nUnits; + +end \ No newline at end of file diff --git a/processing/CFF_setup_optimized_block_processing.m b/processing/CFF_setup_optimized_block_processing.m new file mode 100644 index 0000000..8d05416 --- /dev/null +++ b/processing/CFF_setup_optimized_block_processing.m @@ -0,0 +1,136 @@ +function [blocks, info] = CFF_setup_optimized_block_processing(nUnits,unitSizeInBytes,varargin) +%CFF_SETUP_OPTIMIZED_BLOCK_PROCESSING Setup block processing from memory +% +% Setup block processing based on memory +% +% BLOCKS = CFF_SETUP_OPTIMIZED_BLOCK_PROCESSING(NUNITS,UNITSIZEINBYTES) +% returns an Nx2 array BLOCKS where each row is the start and end indices +% for a block of units to process at a time (typically, pings), using the +% total number of units NUNITS, and the size of one unit in bytes +% UNITSIZEINBYTES (typically for SBP processing, equal to the product of +% the number of samples, the number of beams, and the size of a sample in +% bytes - 4 bytes for class 'single'). The blocks are defined based on +% the memory available for CPU processing in order to avoid Out-Of-memory +% error. +% +% CFF_SETUP_OPTIMIZED_BLOCK_PROCESSING(NUNITS,UNITSIZEINBYTES,PU) +% where PU = 'GPU' does the same but based on the memory available for +% GPU processing (parallel computing). By default, PU = 'CPU'. +% +% CFF_SETUP_OPTIMIZED_BLOCK_PROCESSING(...,'minMemFracToKeepFree',VAL) +% uses VAL as the desired fraction of total device memory to keep free to +% safeguard against out-of-memory errors. By default, VAL = 0.15. If you +% experience Out-Of-Memory errors, you can increase this fraction. Tests +% have found 0.1-0.4 to be reasonable values. +% +% CFF_SETUP_OPTIMIZED_BLOCK_PROCESSING(...,'desiredMaxMemFracToUse',VAL) +% uses VAL as the desired fraction of available memory to use for +% calculations. By default, VAL = 0.1. If you experience Out-Of-Memory +% errors, you can decrease this fraction. Tests have found 0.01-0.4 to be +% reasonable values. +% +% CFF_SETUP_OPTIMIZED_BLOCK_PROCESSING(...,'maxNumBlockVar',VAL) uses VAL +% as the maximum number of "big" block variables in memory at one time in +% the upcoming processing, in order to estimate maximum memory usage. +% Ideally, check your code to find the true number and inform it here in +% order to get more accurate estimate. If you don't do it, this +% function uses a guess VAL = 5 by default. If the true number is lower, +% you will miss on performance. If the true number is higher, you risk an +% Out-Of-Memory error. If you are unsure, set a higher number (i.e. 10) +% to get conservative estimates, at the cost of lower performance. Note: +% write your code to limit that maximum number, by clearing "big" block +% variables as soon as they are not necessary anymore. +% +% [BLOCKS,INFO] = CFF_SETUP_OPTIMIZED_BLOCK_PROCESSING(...) returns the +% additional INFO structure with metadata on the internal calculations. +% +% See also CFF_MEMORY_AVAILABLE, CFF_IS_PARALLEL_COMPUTING_AVAILABLE, +% CFF_SETUP_BLOCK_PROCESSING. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 05-08-2022 + + +% input arguments management +p = inputParser; +addRequired(p,'nUnits',@(x)isnumeric(x)&isscalar(x)); % total number of units +addRequired(p,'unitSizeInBytes',@(x)isnumeric(x)&isscalar(x)); % size in bytes for one unit of data +addOptional(p,'processingUnit','CPU',@(x)ismember(upper(x),{'CPU','GPU'})); % device +addParameter(p,'minMemFracToKeepFree',0.15,@(x)isscalar(x)&ge(x,0)<(1)); % min fraction of device memory to keep free +addParameter(p,'desiredMaxMemFracToUse',0.1,@(x)isscalar(x)>(x,0)&le(x,1)); % desired max fraction of memory available to use +addParameter(p,'maxNumBlockVar',5,@(x)isscalar(x)&(x==round(x))>(x,0)); % max number of block variables in memory at a time +parse(p,nUnits,unitSizeInBytes,varargin{:}); +processingUnit = p.Results.processingUnit; +minMemFracToKeepFree = p.Results.minMemFracToKeepFree; +desiredMaxMemFracToUse = p.Results.desiredMaxMemFracToUse; +maxNumBlockVar = p.Results.maxNumBlockVar; +clear p + + +%% Calculate desired maximum memory to use +% all sizes in MB for ease of reading/debugging + +% total memory available on device +totDeviceMemInMB = CFF_memory_available(processingUnit)./(1024.^2); + +% memory available accounting for safety buffer +memAvailInMB = totDeviceMemInMB.*(1-minMemFracToKeepFree); + +% desired max amount of memory to use +desiredMaxMemToUseInMB = desiredMaxMemFracToUse.*memAvailInMB; + + +%% Check against one unit limit + +% at minimum, we need to be able to process one unit at a time +memNeededForOneUnitInMB = (unitSizeInBytes./(1024.^2)).*maxNumBlockVar; + +% if that amount of memory is higher than the desired max amount we want to +% use, we have no choice but up the max amount to use +maxMemToUseInMB = max(desiredMaxMemToUseInMB,memNeededForOneUnitInMB); + +% so the actual fraction of memory to use is: +maxMemFracToUse = maxMemToUseInMB./memAvailInMB; + + +%% Calculate blocks + +% maximum number of such units that can be processed at a time (block) +maxNumUnitsPerBlock = floor(maxMemToUseInMB./memNeededForOneUnitInMB); + +% get blocks +blocks = CFF_setup_block_processing(nUnits,maxNumUnitsPerBlock); + + +%% Extra info + +% number of blocks +nBlocks = size(blocks,1); + +% number of units per block (except, possibly the last one, smaller) +nUnitsPerBlock = blocks(1,2); + +% predicted memory used per block +predMemUsedPerBlockInMB = nUnitsPerBlock.*memNeededForOneUnitInMB; + +% actual memory on device to stay free +freeMemInMB = totDeviceMemInMB - predMemUsedPerBlockInMB; + +% saving internal calculations as info for check/debugging +info.processingUnit = processingUnit; +info.totDeviceMemInMB = totDeviceMemInMB; +info.minMemFracToKeepFree = minMemFracToKeepFree; +info.memAvailInMB = memAvailInMB; +info.desiredMaxMemFracToUse = desiredMaxMemFracToUse; +info.desiredMaxMemToUseInMB = desiredMaxMemToUseInMB; +info.memNeededForOneUnitInMB = memNeededForOneUnitInMB; +info.maxMemToUseInMB = maxMemToUseInMB; +info.maxMemFracToUse = maxMemFracToUse; +info.nUnits = nUnits; +info.nBlocks = nBlocks; +info.nUnitsPerBlock = nUnitsPerBlock; +info.predMemUsedPerBlockInMB = predMemUsedPerBlockInMB; +info.freeMemInMB = freeMemInMB; + +end \ No newline at end of file diff --git a/processing/gridding/CFF_create_blank_grid.m b/processing/gridding/CFF_create_blank_grid.m new file mode 100644 index 0000000..a35065c --- /dev/null +++ b/processing/gridding/CFF_create_blank_grid.m @@ -0,0 +1,30 @@ +function [gridX,gridY,gridNaN] = CFF_create_blank_grid(x,y,res) +%CFF_CREATE_BLANK_GRID Prepare blank grid for 2D data +% +% Takes data coordinates (x,y) (vector or matrices of any dimensions) and +% generate grid coordinate vectors (gridX,gridY) at desired resolution +% (res), along with a 2D grid indicating cells where there are no data +% (gridNaN) to use as mask. +% +% See also CFF_GRID_2D_DATA. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2021-2021; Last revision: 15-11-2021 + +% calculate grid coordinates +idxKeep = ~isnan(x(:)) & ~isnan(y(:)); +gridX = min(x(idxKeep)):res:max(x(idxKeep)); +gridY = (min(y(idxKeep)):res:max(y(idxKeep)))'; + +% indices of data points in grid +iX = floor((x(idxKeep)-gridX(1))/res)+1; +iY = floor((y(idxKeep)-gridY(1))/res)+1; +subs = single([iY iX]); + +% size of output grid +sz = single([numel(gridY) numel(gridX)]); + +% use accumarray to generate grid +gridNaN = accumarray(subs,ones(numel(x(idxKeep)),1),sz,@(x) sum(x),0); +gridNaN = gridNaN==0; \ No newline at end of file diff --git a/processing/gridding/CFF_grid_2D_data.m b/processing/gridding/CFF_grid_2D_data.m new file mode 100644 index 0000000..eb93079 --- /dev/null +++ b/processing/gridding/CFF_grid_2D_data.m @@ -0,0 +1,125 @@ +function [gridV, gridX, gridY, interpolant] = CFF_grid_2D_data(x,y,v,res,interpolant) +%CFF_GRID_2D_DATA Grid 2D data +% +% Grid 2D data values 'v' at resolution 'res' given the data's +% coordinates 'x' (columns) and 'y' (rows). Optionally, use in input an +% existing 'interpolant' (class scatteredInterpolant) already fitted to +% the data to speed up the gridding process (see below information of +% this variable). +% +% Gridding resolution is the same in x and y dimensions. +% +% Dimensions of x, y and v must all match. Can be vectors or arrays. +% +% Gridding creates a scatteredInterpolant from the x,y,v data. This +% operation is what takes the longest time. The function returns this +% scatteredInterpolant as 'interpolant', so it can be re-used in input of +% this function for a subsequent re-gridding of the same data, which will +% then be significantly faster. +% +% It is possible to grid multiple datasets v1,v2,... sharing the same +% coordinates x,y, using a cell array for 'v'. Each element of that cell +% array mut obey the dimensions rules above, and the optional +% 'interpolant' in input must also match the number of datasets. The +% outputs 'gridV' and 'interpolant' are then also cell arrays of +% corresponding size. +% +% See also CFF_CREATE_BLANK_GRID. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2021-2021; Last revision: 24-11-2021 + + +%% Input management + +% number of datasets to grid +if isnumeric(v) + nV = 1; +elseif iscell(v) && all(cellfun(@(x) isnumeric(x),v)) + nV = numel(v); +else + error('v must be numeric or cell array of numeric'); +end + +% check that x,y,v dimensions match +if ~isnumeric(x) || ~isnumeric(y) + error('x and y must be numeric'); +end +refDims = size(x); +if ~all(size(y)==refDims) + error('dimensions of x and y must match'); +end +if ( nV==1 && ~all(size(v)==refDims) ) || ... + ( nV>1 && any(cellfun(@(ii) ~all(size(ii)==refDims),v)) ) + error('dimensions of v numeric array(s) must match that of x and y'); +end + +% interpolant(s) +if ~exist('interpolant','var') || isempty(interpolant) + if nV == 1 + interpolant = []; + else + interpolant = cell(1,nV); + end +end +interpolantValFun = @(x) (isnumeric(x)&&isempty(x)) || isa(x,'scatteredInterpolant'); +if nV==1 + if ~interpolantValFun(interpolant) + error('interpolant must be empty or of type ''scatteredInterpolant'''); + end +else + if ~iscell(interpolant) || numel(interpolant)~=nV + error('interpolant must be cell array of size matching that of v cell array'); + end + if ~all(cellfun(interpolantValFun,interpolant)) + error('interpolant elements must be either empty or of type ''scatteredInterpolant'''); + end +end + + +%% Prep + +% prepare grid coordinates, and the mask grid +[gridX,gridY,gridNaN] = CFF_create_blank_grid(x,y,res); + +% indices of data to keep, based on x and y only +idxXYKeep = ~isnan(x) & ~isinf(x) & ~isnan(y) & ~isinf(y); + +% gridding v. Keeping single and multiple cases separate to keep code fast +if nV == 1 + % single v array to grid + % indices of data to keep + idxKeep = idxXYKeep & ~isnan(v) & ~isinf(v); + % prepare interpolant if it was not provided in input + if isempty(interpolant) + warning('off'); % disables scatteredInterpolant complaining about duplicate points. + interpolant = scatteredInterpolant(y(idxKeep),x(idxKeep),v(idxKeep),'natural','none'); + warning('on'); + end + % apply interpolant + gridV = interpolant({gridY,gridX}); + % mask data + gridV(gridNaN) = NaN; +else + % multiple v arrays to grid + gridV = cell(size(v)); + for ii = 1:nV + v_temp = v{ii}; + interpolant_temp = interpolant{ii}; + % indices of data to keep + idxKeep = idxXYKeep & ~isnan(v_temp) & ~isinf(v_temp); + % prepare interpolant if it was not provided in input + if isempty(interpolant_temp) + warning('off'); % disables scatteredInterpolant complaining about duplicate points. + interpolant_temp = scatteredInterpolant(y(idxKeep),x(idxKeep),v_temp(idxKeep),'natural','none'); + warning('on'); + end + gridV_temp = interpolant_temp({gridY,gridX}); + % mask data + gridV_temp(gridNaN) = NaN; + % save in output cell array + gridV{ii} = gridV_temp; + interpolant{ii} = interpolant_temp; + end +end \ No newline at end of file diff --git a/processing/gridding/CFF_grid_data.m b/processing/gridding/CFF_grid_data.m new file mode 100644 index 0000000..e70846d --- /dev/null +++ b/processing/gridding/CFF_grid_data.m @@ -0,0 +1,213 @@ +function [vg,wg] = CFF_grid_data(x,y,v,xg,yg,varargin) +%CFF_GRID_DATA Grid 2D data +% +% [VG,WG] = CFF_GRID_DATA(X,Y,V,XG,YG) grids data with value V at +% coordinates X,Y into the grid defined by vectors of coordinates XG,YG. +% The function returns the gridded value VG and the number of points WG +% (density) that contributed to each grid cell. XG and YG must have a +% constant and equal grid size. X,Y,V must be vectors or arrays with +% matching dimensions. +% +% CFF_GRID_DATA(...,W) where W is the weights associated with the data, +% performs the weighted averaging of data points into the grid. In this +% case, the output WG becomes the cumulative weight of points that +% contributed to each grid cell. +% +% CFF_GRID_DATA(...,MODE), where MODE can be 'blend' (default) or +% 'stitch', informs the gridding strategy. With 'blend', the strategy is +% the averaging or weighted-averaging explained above. With 'stitch', the +% value of a single data point is retained for each grid cell, that which +% had the maximum weight. This option can only be used with weight in +% input. +% +% Note: Use CFF_INIT_GRID to create xg,yg from data +% +% Note: Strategy of gridding, using the x dimension as example: +% 1. the function calculates the grid size res from diff(xg) +% 2. x points falling between xg(i)-res/2 (included) and xg(i)+res/2 +% (excluded) will be gridded in that i-th grid cell. +% +% See also CFF_GRID_LINES, CFF_INIT_GRID + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2021-2021; Last revision: 06-04-2022 + +% TODOs: +% 1. check that this function also works with gpuArrays +% 2. the stitch mode does not work. To fix, or remove. +% 3. when necessary changes are done, check the test code at the end + +% input parser +p = inputParser; +addRequired(p,'x', @(u) validateattributes(u,{'numeric'},{'2d'})); +addRequired(p,'y', @(u) validateattributes(u,{'numeric'},{'2d'})); +addRequired(p,'v', @(u) validateattributes(u,{'numeric'},{'2d'})); +addRequired(p,'xg', @(u) validateattributes(u,{'numeric'},{'size',[1,NaN],'increasing'})); +addRequired(p,'yg', @(u) validateattributes(u,{'numeric'},{'size',[NaN,1],'increasing'})); +addOptional(p,'w', 1, @(u) isnumeric(u)); +addOptional(p,'mode','blend',@(u) ismember(u,{'blend','stitch','min'})); +parse(p,x,y,v,xg,yg,varargin{:}); +w = p.Results.w; +mode = p.Results.mode; +clear p; + +% input additional checks and data preparation +sx = size(x); +sy = size(y); +sv = size(v); +if ~(all(sx==sy)&&all(sx==sv)) + error("'x', 'y' and 'v' must have same size."); +end +if numel(w) == 1 + w = w.*ones(sv); +end +sw = size(w); +if ~all(sw==sv) + error("'w' must either be a scalar, or have same size as other inputs."); +end +xg_res = unique(diff(xg)); +if numel(xg_res)>1 + if numel(xg_res)==2 + % check for a floating point error + tol = 10.^-8; + if xg_res(2)-xg_res(1)1 + if numel(yg_res)==2 + % check for a floating point error + tol = 10.^-8; + if yg_res(2)-yg_res(1)1 + error("'xg' and 'yg' must have the same grid size.") +end + +% vectorize everything +if sx(2)>1 + x = x(:); + y = y(:); + v = v(:); + w = w(:); +end + +% indices of data in grid +ix = round((x-xg(1))/res+1); +iy = round((y-yg(1))/res+1); + +% remove data 1) outside of grid boundaries, 2) with a nan value, 3) +% with zero weight +iKeep = ix>=1 & ix<=numel(xg) ... + & iy>=1 & iy<=numel(yg) ... + & ~isnan(x) & ~isnan(y) & ~isnan(v) & ~isnan(w) & w~=0; +if any(iKeep==0) + ix = ix(iKeep); + iy = iy(iKeep); + v = v(iKeep); + w = w(iKeep); +end + +% prepare for accumarray +subs = [iy ix]; +sz = [numel(yg) numel(xg)]; + +switch mode + case 'blend' + % Here we calculate the weighted average of v,w. + % NOTE: I originally wrote a special case when output weights are + % not requested so a single accumarray using @mean could be used, + % but it turns out it takes more time than using the weighted + % average below with unit weights. + wg = accumarray(subs,w,sz,@sum,0); % gridded weight + wvg = accumarray(subs,v.*w,sz,@sum,NaN); % gridded weighted sum + vg = wvg./wg; % gridded weighted average + case 'stitch' + wg = accumarray(subs,w,sz,@max,NaN); % gridded maximum weight + [~,iw] = ismember(wg,w); % indices of data point to retain + iNaN = iw==0; % NaN indices are grid cells with no data + iw(iNaN) = 1; % temporarily give them a dummy index + vg = v(iw); % get corresponding values + vg(iNaN) = NaN; % put the NaNs back in + wg(iNaN) = 0; % in weight grid, no data should be 0 + case 'min' + vg = accumarray(subs,v,sz,@min,NaN); % gridded weight + wg = []; +end + +end + +% TEST CODE +% +% % define grid +% [xg,yg] = CFF_init_grid([0 100], [0 100]); +% +% % data points +% N = 10000000; +% x = [randn(N,1).*10+33;randn(N,1).*10+66]; +% y = [randn(N,1).*10+33;randn(N,1).*10+66]; +% v = [2.*ones(N,1);1.*ones(N,1)]; +% +% % weight options +% w1 = [10.*ones(N,1);10.*ones(N,1)]; +% w2 = [1.*ones(N,1);100.*ones(N,1)]; +% +% +% exp = 1; +% tic +% [vg,wg] = CFF_grid_data(x,y,v,xg,yg); +% t= toc; +% figure; +% subplot(121); imagesc(vg,'AlphaData',~isnan(vg)); +% set(gca,'YDir','default'); axis equal; grid on; colorbar; +% title(sprintf('exp #%i: %f s',exp,t)) +% subplot(122); imagesc(wg,'AlphaData',wg~=0); +% set(gca,'YDir','default'); axis equal; grid on; colorbar; +% title('blend, no weights') +% +% exp = 2; +% tic +% [vg,wg] = CFF_grid_data(x,y,v,xg,yg,w1); +% t= toc; +% figure; +% subplot(121); imagesc(vg,'AlphaData',~isnan(vg)); +% set(gca,'YDir','default'); axis equal; grid on; colorbar; +% title(sprintf('exp #%i: %f s',exp,t)) +% subplot(122); imagesc(wg,'AlphaData',wg~=0); +% set(gca,'YDir','default'); axis equal; grid on; colorbar; +% title('blend, same weights of 10') +% +% exp = 3; +% tic +% [vg,wg] = CFF_grid_data(x,y,v,xg,yg,w2); +% t= toc; +% figure; +% subplot(121); imagesc(vg,'AlphaData',~isnan(vg)); +% set(gca,'YDir','default'); axis equal; grid on; colorbar; +% title(sprintf('exp #%i: %f s',exp,t)) +% subplot(122); imagesc(wg,'AlphaData',wg~=0); +% set(gca,'YDir','default'); axis equal; grid on; colorbar; +% title('blend, different weights') +% +% exp = 4; +% tic +% [vg,wg] = CFF_grid_data(x,y,v,xg,yg,w2,'stitch'); +% t= toc; +% figure; +% subplot(121); imagesc(vg,'AlphaData',~isnan(vg)); +% set(gca,'YDir','default'); axis equal; grid on; colorbar; +% title(sprintf('exp #%i: %f s',exp,t)) +% subplot(122); imagesc(wg,'AlphaData',wg~=0); +% set(gca,'YDir','default'); axis equal; grid on; colorbar; +% title('stitch, different weights') diff --git a/processing/gridding/CFF_grid_lines.m b/processing/gridding/CFF_grid_lines.m new file mode 100644 index 0000000..f5cc937 --- /dev/null +++ b/processing/gridding/CFF_grid_lines.m @@ -0,0 +1,229 @@ +function fDataGroup = CFF_grid_lines(fDataGroup,varargin) +%CFF_GRID_LINES Create Easting-Northing grids of data in a set of lines +% +% FDATAGROUP = CFF_GRID_LINES(FDATAGROUP) creates Easting-Northing grids +% of bathymetry and bacskcatter data from a set of fData structures. The +% grid size is 1m. The gridded data and the metadata are saved back into +% the corresponding fData structure. +% +% CFF_GRID_LINES(...,'res',VALUE) to specify the grid size +% +% CFF_GRID_LINES(...,'saveFDataToDrive',VALUE) to specify whether to save +% the fData on the hard-drive (VALUE=1) or not (VALUE=0, default). +% +% CFF_GRID_LINES(...,'abortOnError',VALUE) to specify whether the process +% is to be aborted upon encountering an error in a file (VALUE=1) or +% simply continuing to the next file (VALUE=0, default). +% +% CFF_GRID_LINES(...,'comms',VALUE) to provide a CFF_Comms() object, +% initiate a new one (possible values 'disp', 'textprogressbar', +% 'waitbar', 'oneline', 'multilines') or leave empty for no comms +% (default). +% +% See also CFF_INIT_GRID, CFF_GRID_DATA + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2021-2021; Last revision: 06-04-2022 + +% input parser +p = inputParser; +addRequired(p, 'fDataGroup', @(x) all(CFF_is_fData_version_current(x))); +addParameter(p,'res', 1, @mustBePositive); +addParameter(p,'saveFDataToDrive',0, @(x) mustBeMember(x,[0,1])); +addParameter(p,'abortOnError', 0, @(x) mustBeMember(x,[0,1])); +addParameter(p,'comms', CFF_Comms()); +parse(p,fDataGroup,varargin{:}); +res = p.Results.res; +saveFDataToDrive = p.Results.saveFDataToDrive; +abortOnError = p.Results.abortOnError; +comms = p.Results.comms; +clear p; +if ischar(comms) + comms = CFF_Comms(comms); +end + +% start message +comms.start('Gridding data in line(s)'); + +% number of files +nLines = numel(fDataGroup); + +% start progress +comms.progress(0,nLines); + +% Process files one by one +for ii = 1:nLines + + % try-catch sequence to allow continuing to next file if one fails + try + + % get fData for this line + if iscell(fDataGroup) + fData = fDataGroup{ii}; + else + fData = fDataGroup; + end + + % display for this line + lineName = CFF_file_name(char(CFF_onerawfileonly(fData.ALLfilename)),1); + comms.step(sprintf('%i/%i: line %s',ii,nLines,lineName)); + + % get x,y data + x = fData.X_BP_bottomEasting(:); + y = fData.X_BP_bottomNorthing(:); + + % initialize grid. Make it neat + x_lim(1) = floor((min(x)+res./2)./res).*res; + x_lim(2) = max(x); + y_lim(1) = floor((min(y)+res./2)./res).*res; + y_lim(2) = max(y); + [gridE,gridN] = CFF_init_grid(x_lim,y_lim,res); + + % grid bathymetry + comms.info('Gridding bathymetry') + [gridBathy,gridWeight] = CFF_grid_data(x,y, ... + fData.X_BP_bottomHeight(:), ... + gridE,gridN); + + % calculate slope + comms.info('Calculating gridded slope'); + [gridSlopeX,gridSlopeY] = gradient(gridBathy); + gridSlope = sqrt(gridSlopeX.^2 + gridSlopeY.^2); + + % grid backscatter + % DEV NOTE: For backscatter we can only do it if + % X_BP_bottomEasting/Northing match X8_BP_ReflectivityBS + flagGridBS = CFF_is_field_or_prop(fData,'X8_BP_ReflectivityBS') && all(size(fData.X8_BP_ReflectivityBS)==size(fData.X_BP_bottomEasting)); + if flagGridBS + comms.info('Gridding backscatter'); + gridBS = CFF_grid_data(x,y,... + fData.X8_BP_ReflectivityBS(:),... + gridE,gridN); + end + + % grid ping and beam number. + % Here we keep only the minimum linear index per grid cell. Since + % linear indices increase by ping THEN beam, it means that the + % smallest ping takes precedence. For example, if a grid cell has + % only two points being at beam #256 in ping #1 and beam #1 in ping + % #2, then the grid cell will store the information of the former, + % because it has lower ping index. Use [iB,iP] = + % ind2sub([nB,nP],linearIndices); to retrieve indices in beam and + % ping from the linear indices. + comms.info('Gridding ping and beam indices'); + [nB,nP] = size(fData.X8_BP_DepthZ); + linIndexBP = 1:nB*nP; + gridMinIndexBP = CFF_grid_data(x,y,... + linIndexBP(:),... + gridE,gridN,... + 1,... % weight, unecessary here + 'min'); % mode + + % [nB,nP] = size(fData.X8_BP_DepthZ); + % gridIndexBP = fData.X_NE_indexBP; + % [gridBind,gridPind] = ind2sub([nB,nP],gridIndexBP); + % indNan = isnan(gridIndexBP); + % gridIndexBP(indNan) = 1; + % A = abs(fData.X8_BP_AcrosstrackDistanceY(gridIndexBP)); + % A(indNan) = NaN; + + % grid beam angle (away from nadir) + % for now doing an approximative calculation from the distance + % along/across-track since they are available in the X8 datagrams, + % but when it's working and proving useful, do switch to using the + % actual angles in the RRA78 datagrams + % + % Note we do not average but keep the minimum angle + comms.info('Gridding beam angle'); + BP_beamAngle = abs(atand(fData.X_BP_bottomAcrossDist./(-fData.X_BP_bottomUpDist))); + gridMinBeamAngle = CFF_grid_data(x,y,... + BP_beamAngle(:),... + gridE,gridN,... + 1,... % weight, unecessary here + 'min'); % mode + + + % % gridding pingcounter as BP to display ping data as grids + % comms.info('Gridding ping indices'); + % nB = size(fData.X8_B1_BeamNumber,1); + % v = reshape(ones(nB,1)*fData.X8_1P_PingCounter,[],1); + % [gridPing,gridPingWeight] = CFF_grid_data(x,y,v,gridE,gridN,... + % 1,'stitch'); + % % why stitch does not work here? + % + % %indexing in Ru_1D + % % do column per column + % X_NE_indexInRu_1D = nan(size(gridPing)); + % nC = size(X_NE_indexInRu_1D,2); + % for iC = 1:nC + % A = gridPing(:,iC) - fData.Ru_1D_PingCounter; + % A(A<0) = max(A(:)); + % [M,I] = min(A,[],2); + % I(isnan(M)) = NaN; + % X_NE_indexInRu_1D(:,iC) = I; + % end + + % save everything + fData.X_1_2DgridHorizontalResolution = res; + fData.X_1E_2DgridEasting = gridE; + fData.X_N1_2DgridNorthing = gridN; + fData.X_NE_bathy = gridBathy; + fData.X_NE_weight = gridWeight; + fData.X_NE_slope = gridSlope; + fData.X_NE_bs = gridBS; + fData.X_NE_minIndexBP = gridMinIndexBP; + fData.X_NE_minBeamAngle = gridMinBeamAngle; + %fData.X_NE_indexInRu_1D = X_NE_indexInRu_1D; + + % save fData to drive + if saveFDataToDrive + % get output folder and create it if necessary + rawFile = fData.ALLfilename; + wc_dir = CFF_converted_data_folder(rawFile); + if ~isfolder(wc_dir) + mkdir(wc_dir); + end + mat_fdata_file = fullfile(wc_dir, 'fData.mat'); + comms.info('Saving'); + save(mat_fdata_file,'-struct','fData','-v7.3'); + end + + % save fData back into group + if iscell(fDataGroup) + fDataGroup{ii} = fData; + else + fDataGroup = fData; + end + + % successful end of this iteration + comms.info('Done'); + + catch err + if abortOnError + % just rethrow error to terminate execution + rethrow(err); + else + % log the error and continue + errorFile = CFF_file_name(err.stack(1).file,1); + errorLine = err.stack(1).line; + errrorFullMsg = sprintf('%s (error in %s, line %i)',err.message,errorFile,errorLine); + comms.error(errrorFullMsg); + end + end + + % communicate progress + comms.progress(ii,nLines); + +end + +% output fDataGroup as single struct if that was the input +if isstruct(fDataGroup) + fDataGroup = fDataGroup{1}; +end + + +%% end message +comms.finish('Done'); + +end \ No newline at end of file diff --git a/processing/gridding/CFF_init_grid.m b/processing/gridding/CFF_init_grid.m new file mode 100644 index 0000000..8bfe9b8 --- /dev/null +++ b/processing/gridding/CFF_init_grid.m @@ -0,0 +1,67 @@ +function [xg,yg,zg] = CFF_init_grid(x_lim,y_lim,varargin) +%CFF_INIT_GRID Define grid vectors from input x,y or x,y,z data +% +% [XG,YG] = CFF_INIT_GRID(X_LIM,Y_LIM), where X_LIM and Y_LIM are two +% elements vectors containing the data min and max limits in the x,y +% dimensions, returns vectors containing the x,y coordinates of the +% middle of grid cells with grid size equal to 1. +% +% CFF_INIT_GRID(...,RES) uses RES for grid size instead of the default +% value of 1. +% +% [XG,YG,ZG] = CFF_INIT_GRID(X_LIM,Y_LIM,RES,Z_LIM) does the same thing +% but with an added third dimension z. The grid size in z is also RES. +% +% CFF_INIT_GRID(X_LIM,Y_LIM,RES,Z_LIM,Z_RES) does the same thing but +% where Z_RES defines the grid size in z, so potentially different than +% the grid size RES in x,y. +% +% Note: Strategy of grid definition from input, using the x dimension as +% example: +% 1. x_lim(1) is the center of the first grid cell +% 2. points falling between x_lim(1)-res/2 (included) and x_lim(1)+res/2 +% (excluded) will be gridded in that first grid cell. +% 3. There are as many grid cells as necessary to include a point at +% x_lim(2). +% For example, with x_lim = [2 5] and res = 2, this function will return +% xg = [2,4,6], as the middle coordinates of the columns with bounds: +% [1-3[, [3-5[, and [5-7[. +% +% See also CFF_GRID_LINES, CFF_GRID_DATA + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) +% 2017-2022; Last revision: 06-04-2022 + +% input parser +p = inputParser; +addRequired(p,'x_lim', @(u) validateattributes(u,{'numeric'},{'numel',2,'increasing'})); +addRequired(p,'y_lim', @(u) validateattributes(u,{'numeric'},{'numel',2,'increasing'})); +addOptional(p,'res', 1, @(u) validateattributes(u,{'numeric'},{'scalar','positive'})); +addOptional(p,'z_lim', [], @(u) validateattributes(u,{'numeric'},{'numel',2,'increasing'})); +addOptional(p,'z_res', [], @(u) validateattributes(u,{'numeric'},{'scalar','positive'})); +parse(p,x_lim,y_lim,varargin{:}); +res = p.Results.res; +z_lim = p.Results.z_lim; +z_res = p.Results.z_res; +clear p; + +% total number of columns (x) and rows (y) +n_cols = round((x_lim(2)-x_lim(1))./res+1); +n_rows = round((y_lim(2)-y_lim(1))./res+1); + +% coordinates of middle of cells +xg = (0:1:n_cols-1).*res + x_lim(1); +yg = (0:1:n_rows-1)'.*res + y_lim(1); + +% same thing in z dimension if requested +if ~isempty(z_lim) + % if no input z_res, use res + if isempty(z_res) + z_res = res; + end + % total number of layers (z) + n_lays = round((z_lim(2)-z_lim(1))./z_res+1); + % middle coordinates + zg = (0:1:n_lays-1).*z_res + z_lim(1); +end + diff --git a/read_data_files/CFF_are_raw_files_converted.m b/read_data_files/CFF_are_raw_files_converted.m new file mode 100644 index 0000000..9bb9924 --- /dev/null +++ b/read_data_files/CFF_are_raw_files_converted.m @@ -0,0 +1,27 @@ +function idxConverted = CFF_are_raw_files_converted(rawFilesList) +%CFF_ARE_RAW_FILES_CONVERTED Check if raw files are already converted. +% +% A = CFF_ARE_RAW_FILES_CONVERTED(F) tests if each input file in F +% is converted to the fData format (A=1) or not (A=0). +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 03-12-2021 + +% exit if no input +if isempty(rawFilesList) + idxConverted = []; + return +end + +% list of names of converted files, if input were converted +fDataFolders = CFF_converted_data_folder(rawFilesList); +fDataFiles = fullfile(fDataFolders,'fData.mat'); +if ischar(fDataFiles) + fDataFiles = {fDataFiles}; +end + +% check if files exist +idxConverted = isfile(fDataFiles); + diff --git a/read_data_files/CFF_are_raw_files_loaded.m b/read_data_files/CFF_are_raw_files_loaded.m new file mode 100644 index 0000000..c8cacab --- /dev/null +++ b/read_data_files/CFF_are_raw_files_loaded.m @@ -0,0 +1,99 @@ +function idxLoaded = CFF_are_raw_files_loaded(rawFilesList, fData) +%CFF_ARE_RAW_FILES_LOADED One-line description +% +% List the files available for the app in input folder. Files are +% available only if the pair .all/.wcd exists. Also returns whether these +% pairs have been converted to .mat format. + + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +% exit if no input +if isempty(rawFilesList) + idxLoaded = []; + return +end + +% test if input is just ONE file (or ONE pair of files), and turn to cell +if ischar(rawFilesList) + % single file + rawFilesList = {rawFilesList}; +elseif is_pair_of_files(rawFilesList) + % pair of files + rawFilesList = {rawFilesList}; +end +n_rawfiles = size(rawFilesList,1); + +% if no files are loaded, simply output false here, and exit +if isempty(fData) + idxLoaded = false([n_rawfiles,1]); + return +end + +% otherwise, first get list of loaded files +loaded_files = cell(numel(fData),1); +for nF = 1:numel(fData) + ALLfilename = fData{nF}.ALLfilename; + if numel(ALLfilename) == 1 + % single file. Save the string + loaded_files{nF} = ALLfilename{1}; + else + % pair of files. Save the cell array + loaded_files{nF} = ALLfilename; + end +end + +% next, split single files from pairs +idx_single = cell2mat(cellfun(@ischar,loaded_files,'UniformOutput',0)); +loaded_files_single = loaded_files(idx_single); +loaded_files_pair = loaded_files(~idx_single); + +% and extract the filenames +[~,loaded_files_single_rootfilename,~] = fileparts(loaded_files_single); +if ischar(loaded_files_single_rootfilename) + loaded_files_single_rootfilename = {loaded_files_single_rootfilename}; +end +[~,loaded_files_pair_rootfilename,~] = fileparts(CFF_onerawfileonly(loaded_files_pair)); +if ischar(loaded_files_pair_rootfilename) + loaded_files_pair_rootfilename = {loaded_files_pair_rootfilename}; +end + +% init output +idxLoaded = false([n_rawfiles,1]); + +% fill in for each rawfile +for nF = 1:n_rawfiles + + rawfile = rawFilesList{nF}; + + if ischar(rawfile) + % single file + + % extract the filename + [~,rawfilename,~] = fileparts(rawfile); + rawfilename = {rawfilename}; + + % and compare to those loaded + idxLoaded(nF) = ismember(rawfilename, loaded_files_single_rootfilename); + + elseif is_pair_of_files(rawfile) + % pair of files + + % extract the common filename + [~,onerawfilename,~] = fileparts(CFF_onerawfileonly(rawfile)); + onerawfilename = {onerawfilename}; + + % and compare to those loaded + idxLoaded(nF) = ismember(onerawfilename, loaded_files_pair_rootfilename); + + else + % not recognized + idxLoaded(nF) = false; + end + +end + +function bool = is_pair_of_files(input) +bool = iscell(input) && all(size(input)==[1,2]) && all(cellfun(@ischar,input)); diff --git a/read_data_files/CFF_check_filename.m b/read_data_files/CFF_check_filename.m new file mode 100644 index 0000000..0f919b2 --- /dev/null +++ b/read_data_files/CFF_check_filename.m @@ -0,0 +1,42 @@ +function out = CFF_check_filename(rawFileName,extension) +%CFF_CHECK_FILENAME Check existence and extension of file(s) +% +% out = CFF_CHECK_FILENAME(rawFile,ext) checks if single file +% rawFile (char) exists and has extension ext (char). +% +% out = CFF_CHECK_FILENAME(rawFilesPair,exts) checks if pair of files +% rawFilesPair (2x1 cell array of chars) exist, match (same name), and +% have extensions as specified in exts (2x1 cell array of chars). +% +% See also CFF_CHECK_ALLFILENAME, CFF_CHECK_KMALLFILENAME. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +if ischar(rawFileName) + % single file. + % Check extension is valid, and that file exists. + + isExtValid = any(strcmpi(CFF_file_extension(rawFileName),extension)); + out = isfile(rawFileName) && isExtValid; + +elseif iscell(rawFileName) && numel(rawFileName)==2 + % pair of files. + % Check that extension is a valid pair, that filenames match, and that + % files exist. + + exts = CFF_file_extension(rawFileName); + areExtValidPair = all(strcmp(sort(lower(exts)),sort(lower(extension)))); + + [~,filenames,~] = fileparts(rawFileName); + doFilenamesMatch = strcmp(filenames{1}, filenames{2}); + + doBothFilesExist = all(isfile(rawFileName)); + + out = areExtValidPair && doFilenamesMatch && doBothFilesExist; + +else + out = false; +end + diff --git a/read_data_files/CFF_clean_delete_fdata.m b/read_data_files/CFF_clean_delete_fdata.m new file mode 100644 index 0000000..c821b68 --- /dev/null +++ b/read_data_files/CFF_clean_delete_fdata.m @@ -0,0 +1,128 @@ +function CFF_clean_delete_fdata(wc_dir) +%CFF_CLEAN_DELETE_FDATA Delete a fData including all memmapped files +% + +% Authors: Yoann Ladroit (NIWA, yoann.ladroit@niwa.co.nz) and Alex +% Schimel (NGU, alexandre.schimel@ngu.no) +% 2017-2021; Last revision: 27-07-2021 + +% if wc_dir does not exist, exit here +if ~isfolder(wc_dir) + return +end + +% if wc_dir exists but is empty, delete it and exit +flag_wc_dir_empty = CFF_is_folder_empty(wc_dir); +if flag_wc_dir_empty + rmdir(wc_dir); + return +end + +% init flag indicating failure to delete a file +failFlag = 0; + +% clean delete the fData if it exists +mat_fdata_file = fullfile(wc_dir,'fdata.mat'); +if isfile(mat_fdata_file) + + % load fData + fData = load(mat_fdata_file); + + % find all memmaped files linked in fData, save their location on the + % drive, and remove the field to clear the link + j = 0; + dname = {}; + fields = fieldnames(fData); + for ifi = 1:numel(fields) + + % init + fieldname = fields{ifi}; + rmb = 0; + + if isa(fData.(fieldname),'memmapfile') + % field is a memory-mapped file + j = j+1; + rmb = 1; + dname{j} = fData.(fieldname).Filename; + fData.(fieldname) = []; + + elseif iscell(fData.(fieldname)) + % field is a cell array, test each cell + for ic = 1:numel(fData.(fieldname)) + if isa(fData.(fieldname){ic},'memmapfile') + % field is a memory-mapped file + j = j+1; + rmb = 1; + dname{j} = fData.(fieldname){ic}.Filename; + fData.(fieldname){ic} = []; + end + end + end + + % remove memory-mapped fields + if rmb > 0 + fData = rmfield(fData,fieldname); + end + end + + % next delete all memmaped files + fclose('all'); + dname = unique(dname); + for id = 1:numel(dname) + if isfile(dname{id}) + try + delete(dname{id}); + catch + % if delete returns an error, it means the file is open + % (should not happen with the fclose. Investigate). + failFlag = 1; + end + if isfile(dname{id}) + % if delete didn't return an error but file still exists, + % it means there is still a fData somwhere in memory with + % the link active. + failFlag = 2; + end + end + end + + % delete fData + try + delete(mat_fdata_file); + catch + failFlag = 3; + end + +end + +% There may be old memmaped files (.dat) left over in the folder, due to a +% prior failure to delete them. As commented above this happens when there +% is a fData somewhere in memory with the link active. We may try to delete +% them here +datFiles = dir([wc_dir filesep '**' filesep '*.dat']); +for ii = 1:numel(datFiles) + filename = fullfile(datFiles(ii).folder, datFiles(ii).name); + try + delete(filename); + catch + failFlag = 4; + end +end + +% finally, delete the folder. +if ~CFF_is_folder_empty(wc_dir) + failFlag = 5; +else + try + rmdir(wc_dir); + catch + failFlag = 6; + end +end + +% final message if failure, investigate using number +if failFlag + fprintf('Warning - Could not completely delete contents of folder %s during fData clean-up.\n',wc_dir); +end + +end \ No newline at end of file diff --git a/read_data_files/CFF_convert_raw_files.m b/read_data_files/CFF_convert_raw_files.m new file mode 100644 index 0000000..7391262 --- /dev/null +++ b/read_data_files/CFF_convert_raw_files.m @@ -0,0 +1,586 @@ +function fDataGroup = CFF_convert_raw_files(rawFilesList,varargin) +%CFF_CONVERT_RAW_FILES Read raw data file(s) and convert to fData format +% +% Reads contents of one or several multibeam raw data files and convert +% each of them to the CoFFee fData format used for data processing on +% Matlab. Data supported are Kongsberg EM series binary data file in +% .all format (.all or .wcd, or pair of .all/.wcd) or .kmall format +% (.kmall or .kmwcd, or pair of .kmall/.kmwcd) and Reson-Teledyne .s7k +% format. +% +% fDataGroup = CFF_CONVERT_RAW_FILES(rawFile) converts a single, +% non-paired file rawFile specified with full path either as a character +% string (e.g. rawFilesList='D:\Data\myfile.all') or a 1x1 cell +% containing the character string (e.g. +% rawFilesList={'D:\Data\myfile.all'}). +% +% fDataGroup = CFF_CONVERT_RAW_FILES(pairedRawFiles) converts a pair of +% files specified as a 1x1 cell containing a 2x1 cell where each cell +% contains the full path as a character string (e.g. +% rawFilesList={{'D:\Data\myfile.all','D:\Data\myfile.wcd'}}). Note: If +% you omit the double cell (i.e. +% rawFilesList={'D:\Data\myfile.all','D:\Data\myfile.wcd'}), the two +% files will be converted separately. +% +% fDataGroup = CFF_CONVERT_RAW_FILES(rawFilesList) converts a cell vector +% where each cell corresponds to a file or pair of files to convert, +% specified as above either a character string, or 2x1 cells of paired +% files (e.g. rawFilesList = {'D:\Data\mySingleFile.all', +% {'D:\Data\myPairedFile.all','D:\Data\myPairedFile.wcd'}}). +% Note: Use CFF_LIST_RAW_FILES_IN_DIR to generate rawFilesList from a +% folder containing raw data files. +% +% By default, CFF_CONVERT_RAW_FILES converts every datagram supported. It +% does not reconvert a file that has already been converted if it's found +% on the disk (fData.mat) with the suitable version. In this case, the +% data are simply loaded. If an error is encountered, the error message +% is logged and processing moves onto the next file. After conversion, +% the converted data are NOT saved on the hard-drive. +% Use the format fDataGroup = CFF_CONVERT_RAW_FILES(...,Name,Parameter) +% to modify this default behaviour. Options below: +% +% 'conversionType' informs the datagrams to be read and converted, for +% different purposes. +% 'conversionType': 'everything' (default) will convert every datagram +% supported. +% 'conversionType': 'seafloor' will only convert datagrams necessary for +% bathy and BS processing. Water-column data are ignored.. +% 'conversionType': 'WCD' will only convert datagrams necessary for +% water-column data processing. Seafloor data (bathymetry and +% backscatter) are ignored (although in some formats, they are necessary +% for water-column data processing and in this case are also converted). +% 'conversionType': 'seafloorOrWCD' will convert datagrams necessary for +% seafloor OR water-column data processing, and complete successfully if +% either are found. +% +% 'saveFDataToDrive': 1 will save the converted fData to the hard-drive. +% 'saveFDataToDrive': 0 (default) will NOT save the data to hard-drive. +% Note that if water-column datagrams are present and to be converted, +% then this parameter is overriden and fData is saved to the hard-drive +% anyway. Converted data are in the 'Coffee_files' folder created in the +% same folder as the raw data files. +% +% 'forceReconvert': 1 will force the conversion of a raw data file, even +% if a suitable converted file is found on the hard-drive. +% 'forceReconvert': 0 (default) will skip conversion if a converted +% version is found on the hard-drive and can be loaded. +% +% 'outputFData': 0 will clear fData after conversion of each file so that +% the function returns empty. This avoids memory errors when converting +% many files. Use this in combination with 'saveFDataToDrive': 1 as a +% routine to convert fData for the purpose of saving it to the +% hard-drive. +% 'outputFData': 1 (default) will conserve fData and return it. +% +% 'abortOnError': 1 will interrupt processing if an error is encountered. +% 'abortOnError': 0 (default) will log the error message and move onto +% the next file. +% +% 'convertEvenIfDtgrmsMissing': 1 will continue the conversion of a file +% even in the event that one or more datagram types required by +% 'conversionType' are not found in a file. +% 'convertEvenIfDtgrmsMissing': 0 (default) will stop conversion instead. +% +% 'dr_sub': N where N is an integer will decimate water-column data in +% range by a factor of N. By default, 'dr_sub': 1 so that all data are +% read and converted. +% +% 'db_sub': N where N is an integer will decimate water-column data in +% beam by a factor of N. By default, 'db_sub': 1 so that all data are +% read and converted. +% +% 'comms': 'disp' will display text and progress information in the +% command window. +% 'comms': 'textprogressbar': will display text and progress information +% in a text progress bar in the command window. +% 'comms': 'waitbar': will display text and progress information +% in a Matlab waitbar figure. +% 'comms': '' (default) will not display any text and progress +% information. +% +% See also CFF_CONVERTED_DATA_FOLDER, +% CFF_ARE_RAW_FILES_CONVERTED, CFF_READ_ALL, CFF_READ_S7K, +% CFF_READ_KMALL, CFF_CONVERT_ALLDATA_TO_FDATA, +% CFF_CONVERT_S7KDATA_TO_FDATA, CFF_CONVERT_KMALLDATA_TO_FDATA, CFF_COMMS + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2021; Last revision: 11-11-2021 + + +%% Input arguments management +p = inputParser; + +% list of files (or pairs of files) to convert +argName = 'rawFilesList'; +argCheck = @(x) ~isempty(x) && (ischar(x) || iscell(x)); +addRequired(p,argName,argCheck); + +% 'conversionType' informs the datagrams to be read and converted, for +% different purposes. +% 'conversionType': 'everything' (default) will convert every datagram +% supported. +% 'conversionType': 'seafloor' will only convert datagrams necessary for +% bathy and BS processing. Water-column data are ignored.. +% 'conversionType': 'WCD' will only convert datagrams necessary for +% water-column data processing. Seafloor data (bathymetry and backscatter) +% are ignored (although in some formats, they are necessary for +% water-column data processing and in this case are also converted). +% 'conversionType': 'seafloorOrWCD' will convert datagrams necessary for +% seafloor OR water-column data processing, and complete successfully if +% either are found. +addParameter(p,'conversionType','everything',@(x) mustBeMember(x,{'everything','seafloor','WCD','seafloorOrWCD'})); + +% save fData to hard-drive? 0: no (default), 1: yes +% Note that if we convert for WCD processing, we will disregard that info +% and save fData to drive anyway +addParameter(p,'saveFDataToDrive',0,@(x) mustBeMember(x,[0,1])); + +% what if file already converted? 0: to next file (default), 1: reconvert +addParameter(p,'forceReconvert',0,@(x) mustBeMember(x,[0,1])); + +% output fData? 0: no, 1: yes (default) +% Unecessary in apps, but useful in scripts +addParameter(p,'outputFData',1,@(x) mustBeMember(x,[0,1])); + +% what if error during conversion? 0: to next file (default), 1: abort +addParameter(p,'abortOnError',0,@(x) mustBeMember(x,[0,1])); + +% what if missing required dtgrms? 0: to next file (def), 1: convert anyway +addParameter(p,'convertEvenIfDtgrmsMissing',0,@(x) mustBeMember(x,[0,1])); + +% decimation factor in range and beam (def 1, aka no decimation) +addParameter(p,'dr_sub',1,@(x) isnumeric(x)&&x>0&&mod(x,1)==0); +addParameter(p,'db_sub',1,@(x) isnumeric(x)&&x>0&&mod(x,1)==0); + +% information communication (none by default) +addParameter(p,'comms',CFF_Comms()); + +% parse inputs +parse(p,rawFilesList,varargin{:}); + +% and get results +rawFilesList = p.Results.rawFilesList; +forceReconvert = p.Results.forceReconvert; +abortOnError = p.Results.abortOnError; +convertEvenIfDtgrmsMissing = p.Results.convertEvenIfDtgrmsMissing; +conversionType = p.Results.conversionType; +dr_sub = p.Results.dr_sub; +db_sub = p.Results.db_sub; +saveFDataToDrive = p.Results.saveFDataToDrive; +outputFData = p.Results.outputFData; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end +clear p + + +%% Prep + +% start message +comms.start('Reading and converting file(s)'); + +% single filename in input +if ischar(rawFilesList) + rawFilesList = {rawFilesList}; +end + +% number of files +nFiles = numel(rawFilesList); + +% init output +if outputFData + fDataGroup = cell(1,nFiles); +else + fDataGroup = []; +end + +% start progress +comms.progress(0,nFiles); + + +%% Read and convert files +for iF = 1:nFiles + + % try-catch sequence to allow continuing to next file if one fails + try + + % get the file (or pair of files) to convert + rawFile = rawFilesList{iF}; + + % display for this file + if ischar(rawFile) + filename = CFF_file_name(rawFile,1); + comms.step(sprintf('%i/%i: file %s',iF,nFiles,filename)); + else + % paired files + filename_1 = CFF_file_name(rawFile{1},1); + filename_2_ext = CFF_file_extension(rawFile{2}); + comms.step(sprintf('%i/%i: pair of files %s and %s',iF,nFiles,filename_1,filename_2_ext)); + end + + % file format + [~,~,f_ext] = fileparts(CFF_onerawfileonly(rawFile)); + if strcmpi(f_ext,'.all') || strcmpi(f_ext,'.wcd') + file_format = 'Kongsberg_all'; + elseif strcmpi(f_ext,'.kmall') || strcmpi(f_ext,'.kmwcd') + file_format = 'Kongsberg_kmall'; + elseif strcmpi(f_ext,'.s7k') + file_format = 'Reson_s7k'; + else + error('Cannot be converted. Format ("%s") not supported',f_ext); + end + + % convert, reconvert, update, or ignore based on file status + idxConverted = CFF_are_raw_files_converted(rawFile); + if ~idxConverted + % File is not converted yet: proceed with conversion. + comms.info('Never converted. Try to convert'); + else + % File has already been converted... + if forceReconvert + % ...but asking for reconversion: proceed with + % reconversion. + comms.info('Already converted. Try to re-convert'); + else + % ...and not asking for reconversion. Load data for checks + wc_dir = CFF_converted_data_folder(rawFile); + mat_fdata_file = fullfile(wc_dir, 'fData.mat'); + fData = load(mat_fdata_file); + % test if version correct, and if has WC data) + idxFDataUpToDate = strcmp(CFF_get_fData_version(fData),CFF_get_current_fData_version()); + idxHasWCD = any(startsWith(fieldnames(fData),{'WC_','AP_'})); + if ~idxFDataUpToDate || (strcmp(conversionType,'WCD') && ~idxHasWCD) + comms.info('Already converted but unsuitable. Try to update conversion'); + else + % Converted file is suitable and doesn't need to be + % reconverted. + comms.info('Already converted and suitable. Ignore'); + if outputFData + fDataGroup{iF} = fData; + end + % communicate progress and move on to next file + comms.progress(iF,nFiles); + continue + end + end + end + + % reading and converting depending on file format + switch file_format + case 'Kongsberg_all' + + % datagram types to read + switch conversionType + case 'everything' + % convert every datagrams supported + dtgs = []; + case 'seafloor' + dtgsAllRequired = [73, ... % installation parameters (73) + 80, ... % position (80) + 82, ... % runtime parameters (82) + 88]; % X8 depth (88) + dtgs = sort(unique(dtgsAllRequired)); + case 'WCD' + dtgsAllRequired = [73, ... % installation parameters (73) + 80, ... % position (80) + 82]; % runtime parameters (82) + % dtgsOptional = 88; % X8 depth (88) + dtgsAtLeastOneOf = [107, ... % water-column (107) + 114]; % Amplitude and Phase (114) + % dtgs = sort(unique([dtgsAllRequired, dtgsOptional, dtgsAtLeastOneOf])); + dtgs = sort(unique([dtgsAllRequired, dtgsAtLeastOneOf])); + case 'seafloorOrWCD' + dtgsAllRequired = [73, ... % installation parameters (73) + 80, ... % position (80) + 82]; % runtime parameters (82) + dtgsAtLeastOneOf = [88, ... % X8 depth (88) + 107, ... % water-column (107) + 114]; % Amplitude and Phase (114) + dtgs = sort(unique([dtgsAllRequired, dtgsAtLeastOneOf])); + end + + % conversion step 1: read what we can + if ischar(rawFile) + comms.info('Reading data in file'); + else + comms.info('Reading data in pair of files'); + end + [EMdata,iDtgsParsed] = CFF_read_all(rawFile, dtgs); + + if ~strcmp(conversionType,'everything') + % if requesting specific conversion, a couple of checks + % are necessary + + % check if all required datagrams have been found + iDtgsRequired = ismember(dtgsAllRequired,dtgs(iDtgsParsed)); + if ~all(iDtgsRequired) + strdisp = sprintf('File is missing required datagram type(s) %s.',strjoin(string(dtgsAllRequired(~iDtgsRequired)),', ')); + if convertEvenIfDtgrmsMissing + % log message and resume conversion + comms.info([strdisp ' Converting anyway']); + else + % abort conversion by throwing error + error([strdisp ' Conversion aborted']); + end + end + + % check if at least one of the desired datagrams have + % been found + if exist('dtgsAtLeastOneOf','var') && ~any(ismember(dtgsAtLeastOneOf,dtgs(iDtgsParsed))) + iDtgsAtLeastOne = ismember(dtgsAtLeastOneOf,dtgs(iDtgsParsed)); + strdisp = sprintf('File has none of desired datagram type(s) %s.',strjoin(string(dtgsAtLeastOneOf(~iDtgsAtLeastOne)),', ')); + if convertEvenIfDtgrmsMissing + % log message and resume conversion + comms.info([strdisp ' Converting anyway']) + else + % abort conversion by throwing error + error([strdisp ' Conversion aborted']); + end + end + + end + + % conversion step 2: convert + comms.info('Converting to fData format'); + fData = CFF_convert_ALLdata_to_fData(EMdata,dr_sub,db_sub); + + % sort fields by name + fData = orderfields(fData); + + case 'Kongsberg_kmall' + + % datagram types to read + switch conversionType + case 'everything' + % convert every datagrams supported + dtgs = []; + case 'seafloor' + dtgsAllRequired = {'#IIP',... % Installation Parameters + '#SPO',... % Position + '#MRZ'}; % Bathy and BS + dtgs = sort(unique(dtgsAllRequired)); + case 'WCD' + dtgsAllRequired = {'#IIP',... % Installation Parameters + '#SPO',... % Position + '#MWC'}; % Water-column Data + % USED TO ALSO HAVE '#MRZ',... % Bathy and BS + dtgs = sort(unique(dtgsAllRequired)); + case 'seafloorOrWCD' + dtgsAllRequired = {'#IIP',... % Installation Parameters + '#SPO'}; % Position + dtgsAtLeastOneOf = {'#MRZ',... % Bathy and BS + '#MWC'}; % Water-column Data + dtgs = sort(unique([dtgsAllRequired, dtgsAtLeastOneOf])); + end + + % conversion step 1: read what we can + if ischar(rawFile) + comms.info('Reading data in file'); + else + comms.info('Reading data in pair of files'); + end + [EMdata,iDtgsParsed] = CFF_read_kmall(rawFile, dtgs); + + if ~strcmp(conversionType,'everything') + % if requesting specific conversion, a couple of checks + % are necessary + + % check if all required datagrams have been found + iDtgsRequired = ismember(dtgsAllRequired,dtgs(iDtgsParsed)); + if ~all(iDtgsRequired) + strdisp = sprintf('File is missing required datagram type(s) %s.',strjoin(string(dtgsAllRequired(~iDtgsRequired)),', ')); + if convertEvenIfDtgrmsMissing + % log message and resume conversion + comms.info([strdisp ' Converting anyway']); + else + % abort conversion by throwing error + error([strdisp ' Conversion aborted']); + end + end + + % check if at least one of the desired datagrams have + % been found + if exist('dtgsAtLeastOneOf','var') && ~any(ismember(dtgsAtLeastOneOf,dtgs(iDtgsParsed))) + iDtgsAtLeastOne = ismember(dtgsAtLeastOneOf,dtgs(iDtgsParsed)); + strdisp = sprintf('File has none of desired datagram type(s) %s.',strjoin(string(dtgsAtLeastOneOf(~iDtgsAtLeastOne)),', ')); + if convertEvenIfDtgrmsMissing + % log message and resume conversion + comms.info([strdisp ' Converting anyway']) + else + % abort conversion by throwing error + error([strdisp ' Conversion aborted']); + end + end + + end + + % conversion step 2: convert + comms.info('Converting to fData format'); + fData = CFF_convert_KMALLdata_to_fData(EMdata,dr_sub,db_sub); + + % sort fields by name + fData = orderfields(fData); + + case 'Reson_s7k' + + % datagram types to read + switch conversionType + case 'everything' + % convert every datagrams supported + dtgs = []; + case 'seafloor' + dtgsAllRequired = [7000, ... % R7000_SonarSettings + 7027]; % R7027_RawDetectionData + dtgsAtLeastOneOfNav = [1015, ... % R1015_Navigation + 1003]; % R1003_Position + dtgs = sort(unique([dtgsAllRequired, dtgsAtLeastOneOfNav])); + case 'WCD' + dtgsAllRequired = [7000, ... % R7000_SonarSettings + 7004, ... % R7004_BeamGeometry + 7027]; % R7027_RawDetectionData, NOTE: contains X8 data but needed for WCD + dtgsAtLeastOneOfNav = [1015, ... % R1015_Navigation + 1003]; % R1003_Position + dtgsAtLeastOneOfWCD = [7018, ... % R7018_BeamformedData + 7042]; % R7042_CompressedWaterColumnData + dtgs = sort(unique([dtgsAllRequired, dtgsAtLeastOneOfNav, dtgsAtLeastOneOfWCD])); + case 'seafloorOrWCD' + dtgsAllRequired = [7000, ... % R7000_SonarSettings + 7027]; % R7027_RawDetectionData + dtgsAtLeastOneOfNav = [1015, ... % R1015_Navigation + 1003]; % R1003_Position + dtgsOptionalWCD = [7018, ... % R7018_BeamformedData + 7042]; % R7042_CompressedWaterColumnData + dtgRequiredForWCD = 7004; % R7004_BeamGeometry + dtgs = sort(unique([dtgsAllRequired, dtgsAtLeastOneOfNav, dtgsOptionalWCD, dtgRequiredForWCD])); + end + + % conversion step 1: read what we can + comms.info('Reading data in file'); + [S7Kdata,iDtgsParsed] = CFF_read_s7k(rawFile, dtgs); + + if ~strcmp(conversionType,'everything') + % if requesting specific conversion, a couple of checks + % are necessary + + % check if all required datagrams have been found + iDtgsRequired = ismember(dtgsAllRequired,dtgs(iDtgsParsed)); + if ~all(iDtgsRequired) + strdisp = sprintf('File is missing required datagram type(s) %s.',strjoin(string(dtgsAllRequired(~iDtgsRequired)),', ')); + if convertEvenIfDtgrmsMissing + % log message and resume conversion + comms.info([strdisp ' Converting anyway']); + else + % abort conversion by throwing error + error([strdisp ' Conversion aborted']); + end + end + + % check if at least one type of navigation datagram has + % been found + if ~any(ismember(dtgsAtLeastOneOfNav,dtgs(iDtgsParsed))) + strdisp = 'File does not contain navigation datagrams.'; + if convertEvenIfDtgrmsMissing + % log message and resume conversion + comms.info([strdisp ' Converting anyway']) + else + % abort conversion by throwing error + error([strdisp ' Conversion aborted']); + end + end + + % and special cases for 'WCD' and 'seafloorOrWCD' + if strcmp(conversionType,'WCD') + % if requesting conversion for WCD, check if at + % least one type of water-column datagram has been + % found + if ~any(ismember(dtgsAtLeastOneOfWCD,dtgs(iDtgsParsed))) + strdisp = 'File does not contain water-column datagrams.'; + if convertEvenIfDtgrmsMissing + % log message and resume conversion + comms.info([strdisp ' Converting anyway']) + else + % abort conversion by throwing error + error([strdisp ' Conversion aborted']); + end + end + elseif strcmp(conversionType,'seafloorOrWCD') + % if requesting conversion for seafloorOrWCD, check + % that if we have WCD (either of dtgsOptionalWCD), + % then we also have dtgRequiredForWCD. + if any(ismember(dtgsOptionalWCD,dtgs(iDtgsParsed))) && ~all(ismember(dtgRequiredForWCD,dtgs(iDtgsParsed))) + strdisp = 'File has water-column datagrams, but not the necessary ancillary datagrams.'; + if convertEvenIfDtgrmsMissing + % log message and resume conversion + comms.info([strdisp ' Converting anyway']) + else + % abort conversion by throwing error + error([strdisp ' Conversion aborted']); + end + end + end + end + + % conversion step 2: convert + comms.info('Converting to fData format'); + fData = CFF_convert_S7Kdata_to_fData(S7Kdata,dr_sub,db_sub); + + % sort fields by name + fData = orderfields(fData); + + end + + % save fData to drive + if saveFDataToDrive || any(startsWith(fieldnames(fData),{'WC_','AP_'})) + % get output folder and create it if necessary + wc_dir = CFF_converted_data_folder(rawFile); + if ~isfolder(wc_dir) + mkdir(wc_dir); + end + mat_fdata_file = fullfile(wc_dir, 'fData.mat'); + comms.info('Saving'); + save(mat_fdata_file,'-struct','fData','-v7.3'); + end + + % add to group for output + if outputFData + fDataGroup{iF} = fData; + end + clear fData + + % successful end of this iteration + comms.info('Done'); + + catch err + if abortOnError + % just rethrow error to terminate execution + rethrow(err); + else + % log the error and continue + errorFile = CFF_file_name(err.stack(1).file,1); + errorLine = err.stack(1).line; + errrorFullMsg = sprintf('%s (error in %s, line %i)',err.message,errorFile,errorLine); + comms.error(errrorFullMsg); + end + end + + % communicate progress + comms.progress(iF,nFiles); + +end + +if outputFData + % output struct directly if only one element + if numel(fDataGroup)==1 + fDataGroup = fDataGroup{1}; + end +end + +%% end message +comms.finish('Done'); + +end + + diff --git a/read_data_files/CFF_converted_data_folder.m b/read_data_files/CFF_converted_data_folder.m new file mode 100644 index 0000000..bf7d55f --- /dev/null +++ b/read_data_files/CFF_converted_data_folder.m @@ -0,0 +1,42 @@ +function wc_dir = CFF_converted_data_folder(rawfileslist) +%CFF_CONVERTED_DATA_FOLDER Returns converted data folder for file(s) +% +% Gets the path to the folder for data converted by CoFFee from a raw +% filename. +% +% WC_DIR = CFF_CONVERTED_DATA_FOLDER(RAWFILESLIST) returns the folder +% path 'X/Coffee_files/filename/' for input filename 'X/filename.ext'. +% Also works with cell arrays of string filenames. +% +% See also CFF_CONVERT_RAW_FILES, CFF_LOAD_CONVERTED_FILES. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2022-2022; Last revision: 25-07-2022 + +if isempty(rawfileslist) + wc_dir = []; + return +end + +if ischar(rawfileslist) + rawfileslist = {rawfileslist}; +end + +% simplify rawfileslist when pairs occur +rawfileslist = CFF_onerawfileonly(rawfileslist); +n_files = numel(rawfileslist); + +% get path and filename (without extension) for each file +[filepath,filename,~] = cellfun(@fileparts,rawfileslist,'UniformOutput',0); + +% define coffee folder +coffee_dir_name = 'Coffee_files'; +coffee_dir_name = repmat({coffee_dir_name},[n_files, 1]); + +% putting everything together +wc_dir = cellfun(@fullfile,filepath,coffee_dir_name,filename,'UniformOutput',0); + +if numel(wc_dir) == 1 + wc_dir = cell2mat(wc_dir); +end \ No newline at end of file diff --git a/read_data_files/CFF_file_extension.m b/read_data_files/CFF_file_extension.m new file mode 100644 index 0000000..cea5cf9 --- /dev/null +++ b/read_data_files/CFF_file_extension.m @@ -0,0 +1,29 @@ +function ext = CFF_file_extension(filename) +%CFF_FILE_EXTENSION Get extension of file(s) +% +% ext = CFF_FILE_EXTENSION(filename) returns the STRING extension of +% input STRING filename. +% +% ext = CFF_FILE_EXTENSION(filename) returns a cell array of STRING +% extensions of input cell array of STRING filenames. +% +% *EXAMPLE* +% ext = CFF_file_extension('f.mat'); % returns 'mat' +% ext = CFF_file_extension({'f.mat', 'g.bin'}); % returns {'mat','bin'} +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +if ischar(filename) + [~,~,ext] = fileparts(filename); + return +elseif iscell(filename) + ext = cell(size(filename)); + for ii = 1:numel(filename) + [~,~,ext{ii}] = fileparts(filename{ii}); + end +end + + diff --git a/read_data_files/CFF_file_name.m b/read_data_files/CFF_file_name.m new file mode 100644 index 0000000..63db35a --- /dev/null +++ b/read_data_files/CFF_file_name.m @@ -0,0 +1,55 @@ +function name = CFF_file_name(filename, varargin) +%CFF_FILE_NAME Get name of file(s) +% +% Optional argument allows returning the extension as well. See syntax +% examples. +% +% NAME = CFF_FILE_NAME(FILENAME) returns the string name of the input +% string filename, with no folder, and no extension. For example, +% CFF_FILE_NAME('C:\my_folder\my_file.bin') returns 'my_file'. +% +% CFF_FILE_NAME(FILENAME,FLAG) with FLAG = 1 returns the string name, +% with extension, of the input string filename, with no folder. For +% example, CFF_FILE_NAME('C:\my_folder\my_file.bin',1) returns +% 'my_file.bin'. +% +% NAMES = CFF_FILE_NAME(FILENAMES) returns the cell arrray of string +% names of the input cell array of string filenames, with no folder. For +% example, CFF_file_extension({'C:\my_file.bin','C:\my_other_file.jpg'}) +% returns {'my_file','my_other_file'}. +% CFF_file_extension({'C:\my_file.bin','C:\my_other_file.jpg'},1) returns +% {'my_file.bin','my_other_file.jpg'} + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2021-2022; Last revision: 25-07-2022 + +% input parser +p = inputParser; +addRequired(p,'filename',@(x) ischar(x) || iscell(x)); +addOptional(p,'with_extension',0,@(x) isnumeric(x) && numel(x)==1 && (x==0|x==1) ); +parse(p,filename,varargin{:}); +filename = p.Results.filename; +with_extension = p.Results.with_extension; +clear p + +if ischar(filename) + % single file + [~,name,ext] = fileparts(filename); + % add extension if requested + if with_extension + name = [name, ext]; + end +elseif iscell(filename) + % cell array of files + name = cell(size(filename)); + for ii = 1:numel(filename) + [~,name{ii},ext] = fileparts(filename{ii}); + % add extension if requested + if with_extension + name{ii} = [name{ii}, ext]; + end + end +end + + diff --git a/read_data_files/CFF_fileparts_as_cell.m b/read_data_files/CFF_fileparts_as_cell.m new file mode 100644 index 0000000..28d6a05 --- /dev/null +++ b/read_data_files/CFF_fileparts_as_cell.m @@ -0,0 +1,29 @@ +function [filepath,name,ext] = CFF_fileparts_as_cell(file_list) +%CFF_FILEPARTS_AS_CELL Like FILEPARTS, but always returning cell arrays +% +% The MATLAB function FILEPARTS can take a cell array of file names as +% input. If this array has zero or N>2 elements, the parts are cell +% arrays. But if it has only one element, the returned parts are strings. +% CFF_FILEPARTS_AS_CELL corrects this silly behaviour by always +% outputting cells. +% +% +% See also FILEPARTS. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 15-06-2021 + +[filepath,name,ext] = fileparts(file_list); + +if ischar(filepath) + filepath = {filepath}; +end +if ischar(name) + name = {name}; +end +if ischar(ext) + ext = {ext}; +end + +end diff --git a/read_data_files/CFF_is_folder_empty.m b/read_data_files/CFF_is_folder_empty.m new file mode 100644 index 0000000..9494ef0 --- /dev/null +++ b/read_data_files/CFF_is_folder_empty.m @@ -0,0 +1,22 @@ +function bools = CFF_is_folder_empty(folders) +%CFF_IS_FOLDER_EMPTY Test if input folder(s) is empty. Return 1 if so. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) +% 2017-2021; Last revision: 21-05-2021 + +if ischar(folders) + folders = {folders}; +end + +bools = false(size(folders)); + +for ii = 1:numel(folders) + folder = folders{ii}; + dcont = dir(folder); + if numel(dcont)==2 && strcmp(dcont(1).name,'.') && strcmp(dcont(2).name,'..') + bools(ii) = 1; + end +end + + diff --git a/read_data_files/CFF_list_raw_files_in_dir.m b/read_data_files/CFF_list_raw_files_in_dir.m new file mode 100644 index 0000000..8710fef --- /dev/null +++ b/read_data_files/CFF_list_raw_files_in_dir.m @@ -0,0 +1,119 @@ +function rawfileslist = CFF_list_raw_files_in_dir(folder_init) +%CFF_LIST_RAW_FILES_IN_DIR List raw multibeam files available in folder +% +% Returns in this order: pairs of .all/.wcd, unpaired .all files, +% unpaired .wcd files, pairs of .kmall/.kmwcd, unpaired .kmall files, +% unpaired .kmwcd files, .s7k files. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +% for now, list all raw files whatever the sonar type, but maybe at some +% point add some control in input to sspecify what sonar types we want + +if isempty(folder_init) + rawfileslist = {}; + return +end + + +%% Kongsberg all format + +% list files +all_files = list_files_with_ext(folder_init,'*.all'); +wcd_files = list_files_with_ext(folder_init,'*.wcd'); + +% finding pairs +[paired_all_wcd_files, all_only_files, wcd_only_files] = pair_files(all_files, wcd_files); + + +%% Kongsberg kmall format + +% list files +kmall_files = list_files_with_ext(folder_init,'*.kmall'); +kmwcd_files = list_files_with_ext(folder_init,'*.kmwcd'); + +% finding pairs +[paired_kmall_kmwcd_files, kmall_only_files, kmwcd_only_files] = pair_files(kmall_files, kmwcd_files); + + +%% Reson s7k format + +% list files +s7k_files = list_files_with_ext(folder_init,'*.s7k'); + + +%% Compiling full list of files +rawfileslist = cat(1, ... + paired_all_wcd_files, all_only_files, wcd_only_files, ... + paired_kmall_kmwcd_files, kmall_only_files, kmwcd_only_files, ... + s7k_files); + + +end + +%% subfunctions %% + +%% +function files = list_files_with_ext(folder,extension) +% returns full filepath (with folder, name, and extension) of all files in +% "folder" with matching "extension". Note this works whatever the case of +% extension, e.g. if looking for ".all", it will also find ".ALL" + +files_list = dir(fullfile(folder,extension)); + +if ~isempty(files_list) + idx = [files_list(:).isdir]==0; + filenames = {files_list(idx).name}'; + folders = {files_list(idx).folder}'; + files = fullfile(folders,filenames); +else + files = {}; +end + +end + + +%% +function [paired_files_list, A_only_list, B_only_list] = pair_files(file_list_A, file_list_B) +% compares the filenames (i.e. without folder or extension) of two list of +% files and returns when they match as "pairs". Also returns those that +% don't match + +% extract parts of files in each list +[filepath_A,name_A,ext_A] = CFF_fileparts_as_cell(file_list_A); +[filepath_B,name_B,ext_B] = CFF_fileparts_as_cell(file_list_B); + +% pairs +[C,ia,ib] = intersect(name_A, name_B); +if ~isempty(C) + paired_files_list = cell(length(C),1); + for ii = 1:length(C) + match_A_file = fullfile(filepath_A{ia(ii)},strcat(name_A{ia(ii)},ext_A{ia(ii)})); + match_B_file = fullfile(filepath_B{ib(ii)},strcat(name_B{ib(ii)},ext_B{ib(ii)})); + paired_files_list{ii,1} = {match_A_file, match_B_file}; + end +else + paired_files_list = {}; +end + +% A only +[C,ia] = setdiff(name_A,name_B); +if ~isempty(C) + A_only_list = fullfile(filepath_A(ia),strcat(name_A(ia),ext_A(ia))); +else + A_only_list = {}; +end + +% B only +[C,ib] = setdiff(name_B,name_A); +if ~isempty(C) + B_only_list = fullfile(filepath_B(ib),strcat(name_B(ib),ext_B(ib))); +else + B_only_list = {}; +end + +end + diff --git a/read_data_files/CFF_load_converted_files.m b/read_data_files/CFF_load_converted_files.m new file mode 100644 index 0000000..14a9724 --- /dev/null +++ b/read_data_files/CFF_load_converted_files.m @@ -0,0 +1,135 @@ +function fDataGroup = CFF_load_converted_files(rawFilesList, varargin) +%CFF_LOAD_CONVERTED_FILES Load converted data (fData) from raw file list +% +% FDATAGROUP = CFF_LOAD_CONVERTED_FILES(RAWFILESLIST) looks for the +% converted versions (fData.mat) of the list of raw files RAWFILESLIST in +% input, and load them. If an error is encountered trying to locate and +% load a file, the error message will be logged and the function moves +% onto the next file. +% +% CFF_LOAD_CONVERTED_FILES(...,'fixPaths',FLAG) with FLAG = 1 will check +% if the paths in fData (for ALLFILENAME, and memmaped files) match that +% of the rawFile in input, fix them if they are incorrect, and re-write +% fData on the disk (recommended, unless you have specific reasons to not +% want to update fData on the disk). +% +% CFF_LOAD_CONVERTED_FILES(...,'abortOnError',FLAG) with FLAG = 1 will +% interrupt processing if an error is encountered. By default (FLAG = 0), +% the error is logged and processing continues to the next file. +% +% CFF_LOAD_CONVERTED_FILES(...,'comms',COMMS) specifies if and how this +% function communicates on its internal state (progress, info, errors). +% COMMS can be either a CFF_COMMS object, or a text string to initiate a +% new CFF_COMMS object. Options are 'disp', 'textprogressbar', 'waitbar', +% 'oneline', 'multilines'. By default, using an empty CFF_COMMS object +% (i.e. no communication). See CFF_COMMS for more information. +% +% See also CFF_CONVERT_RAW_FILES, CFF_COMPUTE_PING_NAVIGATION_V2 + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 25-07-2022 + + +%% Input arguments management +p = inputParser; +addRequired(p,'rawFilesList',@(x) ~isempty(x)&&(ischar(x)||iscell(x))); % list of files (or pairs of files) to load +addParameter(p,'fixPaths',0,@(x) mustBeMember(x,[0,1])); % 1: check paths are correct, fix them, and re-write fData on the disk +addParameter(p,'abortOnError',0,@(x) mustBeMember(x,[0,1])); % what if error during conversion? 0: to next file (default), 1: abort +addParameter(p,'comms',CFF_Comms()); % information communication (none by default) +parse(p,rawFilesList,varargin{:}); +fixPaths = p.Results.fixPaths; +abortOnError = p.Results.abortOnError; +comms = p.Results.comms; +clear p +if ischar(comms) + comms = CFF_Comms(comms); +end + + +%% Prep + +% start message +comms.start('Loading converted file(s)'); + +% single filename in input +if ischar(rawFilesList) + rawFilesList = {rawFilesList}; +end + +% number of files +nFiles = numel(rawFilesList); + +% init output +fDataGroup = cell(1,nFiles); + +% start progress +comms.progress(0,nFiles); + + +%% Load files +for iF = 1:nFiles + + % try-catch sequence to allow continuing to next file if one fails + try + + % get the file (or pair of files) to process + rawFile = rawFilesList{iF}; + + % display for this file + if ischar(rawFile) + filename = CFF_file_name(rawFile,1); + comms.step(sprintf('%i/%i: file %s',iF,nFiles,filename)); + else + % paired files + filename_1 = CFF_file_name(rawFile{1},1); + filename_2_ext = CFF_file_extension(rawFile{2}); + comms.step(sprintf('%i/%i: pair of files %s and %s',iF,nFiles,filename_1,filename_2_ext)); + end + + % load converted data + fDataFolder = CFF_converted_data_folder(rawFile); + fDataGroup{iF} = load(fullfile(fDataFolder,'fData.mat')); + + % check paths and fix them, if necessary + if fixPaths + [fDataGroup{iF}, flagPathsFixed] = CFF_fix_fData_paths(fDataGroup{iF},rawFile); + if flagPathsFixed + comms.info('Paths in fData were fixed') + end + end + + % time-tag that fData + fDataGroup{iF}.ID = str2double(datestr(now,'yyyymmddHHMMSSFFF')); + pause(1e-3); % pause to ensure unique time-tags + + % sort fields by name + fDataGroup{iF} = orderfields(fDataGroup{iF}); + + % successful end of this iteration + comms.info('Done'); + + catch err + if abortOnError + % just rethrow error to terminate execution + rethrow(err); + else + % log the error and continue + errorFile = CFF_file_name(err.stack(1).file,1); + errorLine = err.stack(1).line; + errrorFullMsg = sprintf('%s (error in %s, line %i)',err.message,errorFile,errorLine); + comms.error(errrorFullMsg); + end + end + + % communicate progress + comms.progress(iF,nFiles); + +end + + +%% end message +comms.finish('Done'); + + +end \ No newline at end of file diff --git a/read_data_files/CFF_onerawfileonly.m b/read_data_files/CFF_onerawfileonly.m new file mode 100644 index 0000000..3f4e41c --- /dev/null +++ b/read_data_files/CFF_onerawfileonly.m @@ -0,0 +1,34 @@ +function rawfileslist_out = CFF_onerawfileonly(rawfileslist_in) +%CFF_ONERAWFILEONLY Simplify a raw files list to a single file per pair +% +% Simplify a raw files list to a single file per pair, for filenames +% manipulation purposes. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 11-11-2021 + +if ischar(rawfileslist_in) + % single file + rawfileslist_out = rawfileslist_in; +else + % cell array of files + + % number of files, counting pairs as one + n_files = size(rawfileslist_in,1); + + % initialize output + rawfileslist_out = cell(n_files,1); + + % fill in output + for ii = 1:n_files + if iscell(rawfileslist_in{ii}) && numel(rawfileslist_in{ii})==2 + % for pairs, select the second file + rawfileslist_out{ii} = rawfileslist_in{ii}{2}; + else + % for single files, simply copy + rawfileslist_out{ii} = rawfileslist_in{ii}; + end + end + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_all/CFF_all_file_info.m b/read_data_files/Kongsberg/format_all/CFF_all_file_info.m new file mode 100644 index 0000000..232d2da --- /dev/null +++ b/read_data_files/Kongsberg/format_all/CFF_all_file_info.m @@ -0,0 +1,457 @@ +function ALLfileinfo = CFF_all_file_info(ALLfilename, varargin) +%CFF_ALL_FILE_INFO Records basic info about contents of .all file +% +% Records basic info about the datagrams contained in one Kongsberg EM +% series binary data file in .all format (.all or .wcd) +% +% ALLfileinfo = CFF_ALL_FILE_INFO(ALLfilename) opens file ALLfilename and +% reads through the start of each datagram to get basic information about +% it, and store it all in ALLfileinfo. +% +% *INPUT VARIABLES* +% * |ALLfilename|: Required. String filename to parse (extension in .all +% or .wcd) +% +% *OUTPUT VARIABLES* +% * |ALLfileinfo|: structure containing information about datagrams in +% ALLfilename, with fields: +% * |ALLfilename|: input file name +% * |filesize|: file size in bytes +% * |datagsizeformat|: endianness of the datagram size field 'b' or 'l' +% * |datagramsformat|: endianness of the datagrams 'b' or 'l' +% * |datagNumberInFile|: number of datagram in file +% * |datagPositionInFile|: position of beginning of datagram in file +% * |datagTypeNumber|: datagram type in decimal (Kongsberg .all format) +% * |datagTypeText|: datagram type description (Kongsberg .all format) +% * |parsed|: flag for whether the datagram has been parsed. Initiated +% at 0 at this stage. To be later turned to 1 for parsing. +% * |counter|: counter of this type of datagram in the file (ie +% first datagram of that type is 1 and last datagram is the total +% number of datagrams of that type) +% * |number|: the number/counter found in the datagram (usually +% different to counter) +% * |size|: datagram size in bytes +% * |syncCounter|: number of bytes found between this datagram and the +% previous one (any number different than zero indicates a sync error) +% * |emNumber|: EM Model number (eg 2045 for EM2040c) +% * |systemSerialNumber|: System serial number +% * |date|: datagram date in YYYMMDD +% * |timeSinceMidnightInMilliseconds|: time since midnight in +% milliseconds +% +% *DEVELOPMENT NOTES* +% * The code currently lists the EM model numbers supported as a test for +% sync. Add your model number in the list if it is not currently there +% (and if the parsing works). It would be better to remove this test and +% try to sync on ETX and Checksum instead. +% * Check regularly with Kongsberg doc to keep updated with new +% datagrams. + + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + + + +%% HARD-CODED PARAMETERS + +% supported systems: +emNumberList = [122; 300; 302; 304; 710; 712; 2040; 2045; 3000; 3002; 3020; 12040]; %2045 is 2040c + + +%% Input arguments management +p = inputParser; + +% name of the .all or .wcd file +argName = 'ALLfilename'; +argCheck = @(x) CFF_check_ALLfilename(x); +addRequired(p,argName,argCheck); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,ALLfilename,varargin{:}); + +% and get results +ALLfilename = p.Results.ALLfilename; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end + + +%% Start message +filename = CFF_file_name(ALLfilename,1); +comms.start(sprintf('Listing datagrams in file %s',filename)); + + +%% Checking byte ordering +% - Luciano's all files are in 'b' +% - Erik's all file is in 'l' +% - my converted files are in 'b' +% - DataDistrib files are in 'b' but datagram size in 'l'! We need to +% separate the byte ordering tests for these two types. + +% opening file +[fid,~] = fopen(ALLfilename, 'r'); + +% go to end of file +fseek(fid,0,1); + +% number of bytes in file +filesize = ftell(fid); + +% rewind to start +fseek(fid,0,-1); + +% counter for resynchornization attempts +syncCounter = 0; + +% reading data from first datagram +while 1 + + % read in little endian + dgm_start_pif = ftell(fid); + nbDatagL = fread(fid,1,'uint32','l'); % number of bytes in datagram + + if isempty(nbDatagL) + %file finished + error('.all file parsing synchronization failed'); + end + + fseek(fid,dgm_start_pif,-1); % come back to re-read in b + nbDatagB = fread(fid,1,'uint32','b'); % number of bytes in datagram + stxDatag = fread(fid,1,'uint8'); % STX (always H02) + datagTypeNumber = fread(fid,1,'uint8'); % SIMRAD type of datagram + emNumberL = fread(fid,1,'uint16','l'); % EM Model Number + fseek(fid,-2,0); % come back to re-read in b + emNumberB = fread(fid,1,'uint16','b'); % EM Model Number + + % trying to read ETX + if fseek(fid,dgm_start_pif+4+nbDatagL-3,-1) + 1 + etxDatagL = fread(fid,1,'uint8'); % ETX (always H03) + else + etxDatagL = NaN; + end + + if fseek(fid,dgm_start_pif+4+nbDatagB-3,-1) + 1 + etxDatagB = fread(fid,1,'uint8'); % ETX (always H03) + else + etxDatagB = NaN; + end + + % testing need for synchronization + synchronized = (sum(emNumberL==emNumberList) || sum(emNumberB==emNumberList)) ... + & (etxDatagB==3 || etxDatagL==3) ... + & stxDatag==2; + if synchronized + break + else + % trying to re-synchronize: fwd one byte and repeat the above + syncCounter = syncCounter+1; + if syncCounter == 10000 + error('Struggling to recognize start of file. Ensure your EM model is in the list of this function.'); + end + fseek(fid,dgm_start_pif+1,-1); + continue + end +end + +% test for the byte ordering of the datagram size field +if etxDatagL == 3 + datagsizeformat = 'l'; +elseif etxDatagB == 3 + datagsizeformat = 'b'; +end + +% test for byte ordering of datagrams +if sum(emNumberL==emNumberList) + datagramsformat = 'l'; +elseif sum(emNumberB==emNumberList) + datagramsformat = 'b'; +end + +fclose(fid); + +clear emNumberL emNumberB fid nbDatagL nbDatagB stxDatag datagTypeNumber dgm_start_pif etxDatagL etxDatagB synchronized + +% init output info +ALLfileinfo.ALLfilename = ALLfilename; +ALLfileinfo.filesize = filesize; +ALLfileinfo.datagsizeformat = datagsizeformat; +ALLfileinfo.datagramsformat = datagramsformat; + + +%% Reopening file with the good byte ordering + +[fid,~] = fopen(ALLfilename, 'r',datagramsformat); + +% intitializing the counter of datagrams in this file +kk = 0; + +% initializing synchronization counter: the number of bytes that needed to +% be passed before this datagram appeared +syncCounter = 0; + + +%% Start progress +comms.progress(0,filesize); + + +%% Reading datagrams +next_dgm_start_pif = 0; +while next_dgm_start_pif < filesize + + %% new datagram begins + dgm_start_pif = ftell(fid); + + % .all datagrams are composed of + % - nbDatag (4 bytes): total size of datagram (excluding this field) + % - STX = 2 (1 byte) + % ... + % - ETX = 3 (1 byte) + % - checksum (2 bytes) + + % read start of datagram + nbDatag = fread(fid,1,'uint32',datagsizeformat); % number of bytes in datagram + stxDatag = fread(fid,1,'uint8'); % STX (always H02) + + % pif of presumed end of datagram + next_dgm_start_pif = dgm_start_pif + 4 + nbDatag; + + % read STX at end of datagram + if next_dgm_start_pif <= filesize + fseek(fid,next_dgm_start_pif-3,-1); + etxDatag = fread(fid,1,'uint8'); % ETX (always H03) + % checkSum = fread(fid,1,'uint16'); % Check sum of data between STX and ETX + fseek(fid,dgm_start_pif+5, -1); % rewind to where we left reading + else + % would be here if overshooting the end of the file, aka datagram + % is incomplete, or nbDatag is wrong. Set a wrong etxDatag to + % trigger resync + etxDatag = 0; + end + + %% test for synchronization + flag_inSync = ~isempty(stxDatag) && stxDatag==2 && etxDatag==3; + if ~flag_inSync + % NOT SYNCHRONIZED + % trying to re-synchronize: fwd one byte and repeat the above + next_dgm_start_pif = dgm_start_pif+1; + fseek(fid,next_dgm_start_pif,-1); + syncCounter = syncCounter+1; % update sync counter + if syncCounter == 1 + % just lost sync, throw a message just now + comms.error('Lost sync while reading datagrams. A datagram may be corrupted. Trying to resync...'); + end + continue + else + % SYNCHRONIZED + if syncCounter + % if we had lost sync, warn here we're back + comms.info(sprintf('Back in sync (%i bytes later). Resume process.',syncCounter)); + % reinitialize sync counter + syncCounter = 0; + end + end + + % read rest of start of datagram + datagTypeNumber = fread(fid,1,'uint8'); % SIMRAD type of datagram + emNumber = fread(fid,1,'uint16'); % EM Model Number + date = fread(fid,1,'uint32'); % date + timeSinceMidnightInMilliseconds = fread(fid,1,'uint32'); % time since midnight in milliseconds + number = fread(fid,1,'uint16'); % datagram or ping number + systemSerialNumber = fread(fid,1,'uint16'); % EM system serial number + + % reset the datagram counter and parsed switch + counter = NaN; + + switch datagTypeNumber + + case 48 + datagTypeText = 'PU ID OUTPUT (30H)'; + try i48=i48+1; catch, i48=1; end + counter = i48; + case 49 + datagTypeText = 'PU STATUS OUTPUT (31H)'; + try i49=i49+1; catch, i49=1; end + counter = i49; + case 51 + datagTypeText = 'EXTRAPARAMETERS DATAGRAM (33H)'; + try i51=i51+1; catch, i51=1; end + counter = i51; + case 65 + datagTypeText = 'ATTITUDE (41H)'; + try i65=i65+1; catch, i65=1; end + counter = i65; + case 66 + datagTypeText = 'PU BIST RESULT OUTPUT (42H)'; + try i66=i66+1; catch, i66=1; end + counter = i66; + case 67 + datagTypeText = 'CLOCK (43H)'; + try i67=i67+1; catch, i67=1; end + counter = i67; + case 68 + datagTypeText = 'DEPTH DATAGRAM (44H)'; + try i68 = i68+1; catch, i68=1; end + counter = i68; + case 69 + datagTypeText = 'SINGLE BEAM ECHO SOUNDER DEPTH (45H)'; + try i69=i69+1; catch, i69=1; end + counter = i69; + case 70 + datagTypeText = 'RAW RANGE AND BEAM ANGLE (F) (46H)'; + try i70=i70+1; catch, i70=1; end + counter = i70; + case 71 + datagTypeText = 'SURFACE SOUND SPEED (47H)'; + try i71=i71+1; catch, i71=1; end + counter = i71; + case 72 + datagTypeText = 'HEADING (48H)'; + try i72=i72+1; catch, i72=1; end + counter = i72; + case 73 + datagTypeText = 'INSTALLATION PARAMETERS - START (49H)'; + try i73=i73+1; catch, i73=1; end + counter = i73; + case 74 + datagTypeText = 'MECHANICAL TRANSDUCER TILT (4AH)'; + try i74=i74+1; catch, i74=1; end + counter = i74; + case 75 + datagTypeText = 'CENTRAL BEAMS ECHOGRAM (4BH)'; + try i75=i75+1; catch, i75=1; end + counter = i75; + case 78 + datagTypeText = 'RAW RANGE AND ANGLE 78 (4EH)'; + try i78=i78+1; catch, i78=1; end + counter = i78; + case 79 + datagTypeText = 'QUALITY FACTOR DATAGRAM 79 (4FH)'; + try i79=i79+1; catch, i79=1; end + counter = i79; + case 80 + datagTypeText = 'POSITION (50H)'; + try i80=i80+1; catch, i80=1; end + counter = i80; + case 82 + datagTypeText = 'RUNTIME PARAMETERS (52H)'; + try i82=i82+1; catch, i82=1; end + counter = i82; + case 83 + datagTypeText = 'SEABED IMAGE DATAGRAM (53H)'; + try i83=i83+1; catch, i83=1; end + counter = i83; + case 84 + datagTypeText = 'TIDE DATAGRAM (54H)'; + try i84=i84+1; catch, i84=1; end + counter = i84; + case 85 + datagTypeText = 'SOUND SPEED PROFILE (55H)'; + try i85=i85+1; catch, i85=1; end + counter = i85; + case 87 + datagTypeText = 'KONGSBERG MARITIME SSP OUTPUT DATAGRAM (057H)'; + try i87=i87+1; catch, i87=1; end + counter = i87; + case 88 + datagTypeText = 'XYZ 88 (58H)'; + try i88=i88+1; catch, i88=1; end + counter = i88; + case 89 + datagTypeText = 'SEABED IMAGE DATA 89 (59H)'; + try i89=i89+1; catch, i89=1; end + counter = i89; + case 102 + datagTypeText = 'RAW RANGE AND BEAM ANGLE (f) (66H)'; + try i102=i102+1; catch, i102=1; end + counter = i102; + case 104 + datagTypeText = 'DEPTH (PRESSURE) OR HEIGHT DATAGRAM (68H)'; + try i104=i104+1; catch, i104=1; end + counter = i104; + case 105 + datagTypeText = 'INSTALLATION PARAMETERS - STOP (69H)'; + try i105=i105+1; catch, i105=1; end + counter = i105; + case 107 + datagTypeText = 'WATER COLUMN DATAGRAM (6BH)'; + try i107=i107+1; catch, i107=1; end + counter = i107; + case 108 + datagTypeText = 'EXTRA DETECTIONS (6CH)'; + try i108=i108+1; catch, i108=1; end + counter = i108; + case 110 + datagTypeText = 'NETWORK ATTITUDE VELOCITY DATAGRAM 110 (6EH)'; + try i110=i110+1; catch, i110=1; end + counter = i110; + case 112 + datagTypeText = 'INSTALLATION PARAMETERS - REMOTE INFO (70H)'; + try i112=i112+1; catch, i112=1; end + counter = i112; + case 114 + datagTypeText = 'AMPLITUDE AND PHASE WC DATAGRAM 114 (72H)'; + try i114=i114+1; catch, i114=1; end + counter = i114; + otherwise + % this datagTypeNumber is not recognized yet + datagTypeText = sprintf('UNKNOWN DATAGRAM (%sH)',dec2hex(datagTypeNumber)); + + end + + %% write output ALLfileinfo + + % Datagram complete + kk = kk+1; + + % Datagram number in file + ALLfileinfo.datagNumberInFile(kk,1) = kk; + + % Datagram info + ALLfileinfo.datagTypeNumber(kk,1) = datagTypeNumber; + ALLfileinfo.datagTypeText{kk,1} = datagTypeText; + ALLfileinfo.counter(kk,1) = counter; + ALLfileinfo.datagPositionInFile(kk,1) = dgm_start_pif; + ALLfileinfo.size(kk,1) = nbDatag; + ALLfileinfo.number(kk,1) = number; + ALLfileinfo.parsed(kk,1) = 0; + + % System info + ALLfileinfo.emNumber(kk,1) = emNumber; + ALLfileinfo.systemSerialNumber(kk,1)=systemSerialNumber; + + % Time info + ALLfileinfo.date(kk,1) = date; + ALLfileinfo.timeSinceMidnightInMilliseconds(kk,1) = timeSinceMidnightInMilliseconds; + + % Report any sync issue in reading + ALLfileinfo.syncCounter(kk,1) = syncCounter; + + + %% Prepare for reloop + + % go to end of datagram + fseek(fid,next_dgm_start_pif,-1); + + % communicate progress + comms.progress(next_dgm_start_pif,filesize); +end + + +%% finalizing + +% closing file +fclose(fid); + +% end message +comms.finish('Done'); + + + + diff --git a/read_data_files/Kongsberg/format_all/CFF_check_ALLfilename.m b/read_data_files/Kongsberg/format_all/CFF_check_ALLfilename.m new file mode 100644 index 0000000..d566c6c --- /dev/null +++ b/read_data_files/Kongsberg/format_all/CFF_check_ALLfilename.m @@ -0,0 +1,17 @@ +function out = CFF_check_ALLfilename(rawfilename) +%CFF_CHECK_ALLFILENAME Check file exists and has all extension +% +% out = CFF_CHECK_ALLFILENAME(rawFile) checks if single file +% rawFile (char) exists and has '.all' or ',wcd' extension. +% +% out = CFF_CHECK_ALLFILENAME(rawFilesPair) checks if pair of files +% rawFilesPair (2x1 cell array of chars) exist, match (same name), and +% have '.all' and '.wcd' extensions. +% +% See also CFF_CHECK_FILENAME. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +out = CFF_check_filename(rawfilename,{'.all','.wcd'}); \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_all/CFF_convert_ALLdata_to_fData.m b/read_data_files/Kongsberg/format_all/CFF_convert_ALLdata_to_fData.m new file mode 100644 index 0000000..678d86f --- /dev/null +++ b/read_data_files/Kongsberg/format_all/CFF_convert_ALLdata_to_fData.m @@ -0,0 +1,1217 @@ +function fData = CFF_convert_ALLdata_to_fData(ALLdataGroup,varargin) +%CFF_CONVERT_ALLDATA_TO_FDATA Convert all data to the CoFFee format +% +% Converts Kongsberg EM series data FROM the ALLdata format (read by +% CFF_READ_ALL) TO the CoFFee fData format used in processing. +% +% fData = CFF_CONVERT_ALLDATA_TO_FDATA(ALLdata) converts the contents of +% one ALLdata structure to a structure in the fData format. +% +% fData = CFF_CONVERT_ALLDATA_TO_FDATA(ALLdataGroup) converts an array of +% two ALLdata structures into one fData sructure. The pair of structure +% must correspond to an .all/.wcd pair of files. Do not try to use this +% feature to convert ALLdata structures from different acquisition files. +% It will not work. Convert each into its own fData structure. +% +% Note that the ALLdata structures are converted to fData in the order +% they are in input, and that the first ones take precedence. Aka in the +% example above, if the second structure contains a type of datagram that +% is already in the first, they will NOT be converted. This is to avoid +% doubling up the data that may exist in duplicate in the pair of raw +% files. You need to order the ALLdata structures in input in order of +% desired precedence. +% +% fData = CFF_CONVERT_ALLDATA_TO_FDATA(ALLdata,dr_sub,db_sub) operates +% the conversion with a sub-sampling of the water-column data (either WC +% or AP datagrams) in range and in beams. For example, to sub-sample +% range by a factor of 10 and beams by a factor of 2, use: +% fData = CFF_CONVERT_ALLDATA_TO_FDATA(ALLdata,10,2). +% +% *INPUT VARIABLES* +% * |ALLdataGroup|: Required. ALLdata structure or cells of ALLdata +% structures. +% * |dr_sub|: Optional. Scalar for decimation in range. Default: 1 (no +% decimation). +% * |db_sub|: Optional. Scalar for decimation in beams. Default: 1 (no +% decimation). +% * |fData|: Optional. Existing fData structure to add to. +% +% *OUTPUT VARIABLES* +% * |fData|: structure for the storage of kongsberg EM series multibeam +% data in a format more convenient for processing. The data is recorded +% as fields coded "a_b_c" where "a" is a code indicating data origing, +% "b" is a code indicating data dimensions, and "c" is the data name. +% * a: code indicating data origin: +% * IP: installation parameters +% * Ru: Runtime Parameters +% * De: depth datagram +% * He: height datagram +% * X8: XYZ88 datagram +% * SI: seabed image datagram +% * S8: seabed image data 89 +% * WC: watercolumn data +% * Po: position datagram +% * At: attitude datagram +% * SS: sound speed profile datagram +% * AP: "Amplitude and phase" water-column data +% More codes for the 'a' part will be created if more datagrams are +% parsed. Data derived from computations can be recorded back into +% fData using 'X' for the "a" code. +% * b: code indicating data dimensions (rows/columns) +% * 1P: ping-like single-row-vector +% * B1: beam-like single-column-vector +% * BP: beam/ping array +% * TP: transmit-sector/ping array +% * SP: samples/ping array (note: samples are not sorted, this is +% not equivalent to range!) +% * 1D: datagram-like single-row-vector (for attitude or +% position data) +% * ED: entries-per-datagram/datagram array (for attitude or +% position data) +% * SBP: sample/beam/ping array (water-column data) +% More codes for the 'b' part will be created if the storage of other +% datagrams needs them. Data derived from computations can be recorded +% back into fData using appropriate "b" codes such as: +% * RP: range (choose distance, time or sample) / ping +% * SP: swathe (meters) / ping +% * LL: lat/long (WGS84) +% * N1: northing-like single-column-vector +% * 1E: easting-like single-row-vector +% * NE: northing/easting +% * NEH: northing/easting/height +% * c: data type, obtained from the original variable name in the +% Kongsberg datagram, or from the user's imagination for derived data +% obtained from subsequent functions. +% +% *DEVELOPMENT NOTES* +% * only water column data can be subsampled, all other datagrams are +% converted in full. To be consistent, develop code to subsample all +% datagrams as desired in parameters. Add a subsampling in pings while +% you're at it. +% * Have not tested the loading of data from 'EM_Depth' and +% 'EM_SeabedImage' in the new format version (v2). Might need debugging. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 07-04-2022 + + +%% Input arguments management +p = inputParser; + +% array of ALLdata structures +addRequired(p,'ALLdataGroup',@(x) isstruct(x) || iscell(x)); + +% decimation factor in range and beam +addOptional(p,'dr_sub',1,@(x) isnumeric(x)&&x>0); +addOptional(p,'db_sub',1,@(x) isnumeric(x)&&x>0); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,ALLdataGroup,varargin{:}) + +% and get results +ALLdataGroup = p.Results.ALLdataGroup; +dr_sub = p.Results.dr_sub; +db_sub = p.Results.db_sub; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end +clear p; + +if isstruct(ALLdataGroup) + % just one structure + ALLdataGroup = {ALLdataGroup}; +end + +% check input +if numel(ALLdataGroup)==1 + % single ALLdata structure + + % check it's from Kongsberg and that source file exist + has_ALLfilename = isfield(ALLdataGroup{1}, 'ALLfilename'); + if ~has_ALLfilename || ~CFF_check_ALLfilename(ALLdataGroup{1}.ALLfilename) + error('Invalid input'); + end + +elseif numel(ALLdataGroup)==2 + % pair of ALLdata structures + + % check it's from a pair of Kongsberg all/wcd files and that source + % files exist + has_ALLfilename = cell2mat(cellfun(@(x) isfield(x, 'ALLfilename'), ALLdataGroup, 'UniformOutput', false)); + rawfilenames = cellfun(@(x) x.ALLfilename, ALLdataGroup, 'UniformOutput', false); + if ~all(has_ALLfilename) || ~CFF_check_ALLfilename(rawfilenames) + error('Invalid input'); + end + +else + error('Invalid input'); +end + + +%% Prep + +% start message +comms.start('Converting to fData format'); + +% number of individual ALLdata structures in input ALLdataGroup +nStruct = length(ALLdataGroup); + +% initialize fData, with current version number +fData.MET_Fmt_version = CFF_get_current_fData_version(); + +% initialize source filenames +fData.ALLfilename = cell(1,nStruct); + +% start progress +comms.progress(0,nStruct); + + +%% take one ALLdata structure at a time and add its contents to fData +for iF = 1:nStruct + + % get current structure + ALLdata = ALLdataGroup{iF}; + + % add source filename + fData.ALLfilename{iF} = ALLdata.ALLfilename; + + % now reading each type of datagram. + % Note we only convert the datagrams if fData does not already contain + % any. + + + %% EM_InstallationStart + if isfield(ALLdata,'EM_InstallationStart') && ~isfield(fData,'IP_ASCIIparameters') + + comms.step('Converting EM_InstallationStart'); + + % initialize struct + IP_ASCIIparameters = struct; + + % read ASCIIdata + ASCIIdata = char(ALLdata.EM_InstallationStart.ASCIIData(1)); + + % remove carriage returns, tabs and linefeed + ASCIIdata = regexprep(ASCIIdata,char(9),''); + ASCIIdata = regexprep(ASCIIdata,newline,''); + ASCIIdata = regexprep(ASCIIdata,char(13),''); + + % read individual fields + if ~isempty(ASCIIdata) + + yo = strfind(ASCIIdata,',')'; + yo(:,1) = [1; yo(1:end-1)+1]; % beginning of ASCII field name + yo(:,2) = strfind(ASCIIdata,'=')'-1; % end of ASCII field name + yo(:,3) = strfind(ASCIIdata,'=')'+1; % beginning of ASCII field value + yo(:,4) = strfind(ASCIIdata,',')'-1; % end of ASCII field value + + for ii = 1:size(yo,1) + + % get field string + field = ASCIIdata(yo(ii,1):yo(ii,2)); + + % try turn value into numeric + value = str2double(ASCIIdata(yo(ii,3):yo(ii,4))); + if length(value)~=1 + % looks like it cant. Keep as string + value = ASCIIdata(yo(ii,3):yo(ii,4)); + end + + % store field/value + IP_ASCIIparameters.(field) = value; + + end + + end + + % finally store in fData + fData.IP_ASCIIparameters = IP_ASCIIparameters; + + end + + + %% EM_Runtime + if isfield(ALLdata,'EM_Runtime') && ~isfield(fData,'Ru_1D_Date') + + comms.step('Converting EM_Runtime'); + + % Here we have experienced redundant datagrams so remove them + N = numel(ALLdata.EM_Runtime.Date); + allFieldnames = fieldnames(ALLdata.EM_Runtime); + iRed = nan(1,N); % index of redundant datagrams + iRed(1) = 0; + for ii = 2:N + % check if all fields has same value as previous datagram + iRed(ii) = all(cellfun(@(f) ALLdata.EM_Runtime.(f)(ii)==ALLdata.EM_Runtime.(f)(ii-1) ,allFieldnames)); + end + iRed = logical(iRed); + % remove redundancy + for ii = 1:numel(allFieldnames) + ALLdata.EM_Runtime.(allFieldnames{ii})(iRed) = []; + end + + % For now we only record the fields we need later. More fields are + % available than those below. + fData.Ru_1D_Date = ALLdata.EM_Runtime.Date; + fData.Ru_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_Runtime.TimeSinceMidnightInMilliseconds; + fData.Ru_1D_PingCounter = ALLdata.EM_Runtime.PingCounter; + fData.Ru_1D_TransmitPowerReMaximum = ALLdata.EM_Runtime.TransmitPowerReMaximum; + fData.Ru_1D_ReceiveBeamwidth = ALLdata.EM_Runtime.ReceiveBeamwidth./10; % now in degrees + + % decoding encoded fields + decodedParameters = CFF_decode_RuntimeParameters(ALLdata.EM_Runtime); + for fn = fieldnames(decodedParameters)' + fn2 = ['Ru_1D_' fn{1}]; + fData.(fn2) = decodedParameters.(fn{1}); + end + end + + + %% EM_SoundSpeedProfile + if isfield(ALLdata,'EM_SoundSpeedProfile') && ~isfield(fData,'SS_1D_Date') + + comms.step('Converting EM_SoundSpeedProfile'); + + nDatagrams = length(ALLdata.EM_SoundSpeedProfile.TypeOfDatagram); + maxnEntries = max(ALLdata.EM_SoundSpeedProfile.NumberOfEntries); + + fData.SS_1D_Date = ALLdata.EM_SoundSpeedProfile.Date; + fData.SS_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_SoundSpeedProfile.TimeSinceMidnightInMilliseconds; + fData.SS_1D_ProfileCounter = ALLdata.EM_SoundSpeedProfile.ProfileCounter; + fData.SS_1D_DateWhenProfileWasMade = ALLdata.EM_SoundSpeedProfile.DateWhenProfileWasMade; + fData.SS_1D_TimeSinceMidnightInMillisecondsWhenProfileWasMade = ALLdata.EM_SoundSpeedProfile.TimeSinceMidnightInMillisecondsWhenProfileWasMade; + fData.SS_1D_NumberOfEntries = ALLdata.EM_SoundSpeedProfile.NumberOfEntries; + fData.SS_1D_DepthResolution = ALLdata.EM_SoundSpeedProfile.DepthResolution; + + fData.SS_ED_Depth = nan(maxnEntries,nDatagrams); + fData.SS_ED_SoundSpeed = nan(maxnEntries,nDatagrams); + + for iD = 1:nDatagrams + + nEntries = ALLdata.EM_SoundSpeedProfile.NumberOfEntries(iD); + + fData.SS_ED_Depth(1:nEntries,iD) = cell2mat(ALLdata.EM_SoundSpeedProfile.Depth(iD)); + fData.SS_ED_SoundSpeed(1:nEntries,iD) = cell2mat(ALLdata.EM_SoundSpeedProfile.SoundSpeed(iD)); + + end + + end + + %% EM_Attitude + if isfield(ALLdata,'EM_Attitude') && ~isfield(fData,'At_1D_Date') + + comms.step('Converting EM_Attitude'); + + nDatagrams = length(ALLdata.EM_Attitude.TypeOfDatagram); + maxnEntries = max(ALLdata.EM_Attitude.NumberOfEntries); + + fData.At_1D_Date = ALLdata.EM_Attitude.Date; + fData.At_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_Attitude.TimeSinceMidnightInMilliseconds; + fData.At_1D_AttitudeCounter = ALLdata.EM_Attitude.AttitudeCounter; + fData.At_1D_NumberOfEntries = ALLdata.EM_Attitude.NumberOfEntries; + + fData.At_ED_TimeInMillisecondsSinceRecordStart = nan(maxnEntries, nDatagrams); + fData.At_ED_SensorStatus = nan(maxnEntries, nDatagrams); + fData.At_ED_Roll = nan(maxnEntries, nDatagrams); + fData.At_ED_Pitch = nan(maxnEntries, nDatagrams); + fData.At_ED_Heave = nan(maxnEntries, nDatagrams); + fData.At_ED_Heading = nan(maxnEntries, nDatagrams); + + for iD = 1:nDatagrams + + nEntries = ALLdata.EM_Attitude.NumberOfEntries(iD); + + fData.At_ED_TimeInMillisecondsSinceRecordStart(1:nEntries, iD) = cell2mat(ALLdata.EM_Attitude.TimeInMillisecondsSinceRecordStart(iD)); + fData.At_ED_SensorStatus(1:nEntries, iD) = cell2mat(ALLdata.EM_Attitude.SensorStatus(iD)); + fData.At_ED_Roll(1:nEntries, iD) = cell2mat(ALLdata.EM_Attitude.Roll(iD)); + fData.At_ED_Pitch(1:nEntries, iD) = cell2mat(ALLdata.EM_Attitude.Pitch(iD)); + fData.At_ED_Heave(1:nEntries, iD) = cell2mat(ALLdata.EM_Attitude.Heave(iD)); + fData.At_ED_Heading(1:nEntries, iD) = cell2mat(ALLdata.EM_Attitude.Heading(iD)); + + end + + end + + + %% EM_Height + if isfield(ALLdata,'EM_Height') && ~isfield(fData,'He_1D_Date') + + comms.step('Converting EM_Height'); + + fData.He_1D_Date = ALLdata.EM_Height.Date; + fData.He_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_Height.TimeSinceMidnightInMilliseconds; + fData.He_1D_HeightCounter = ALLdata.EM_Height.HeightCounter; + fData.He_1D_Height = ALLdata.EM_Height.Height/100; + + end + + + %% EM_Position + if isfield(ALLdata,'EM_Position') && ~isfield(fData,'Po_1D_Date') + + comms.step('Converting EM_Position'); + + fData.Po_1D_Date = ALLdata.EM_Position.Date; + fData.Po_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_Position.TimeSinceMidnightInMilliseconds; % in ms + fData.Po_1D_PositionCounter = ALLdata.EM_Position.PositionCounter; + fData.Po_1D_Latitude = ALLdata.EM_Position.Latitude./20000000; % now in decimal degrees + fData.Po_1D_Longitude = ALLdata.EM_Position.Longitude./10000000; % now in decimal degrees + fData.Po_1D_SpeedOfVesselOverGround = ALLdata.EM_Position.SpeedOfVesselOverGround./100; % now in m/s + fData.Po_1D_HeadingOfVessel = ALLdata.EM_Position.HeadingOfVessel./100; % now in degrees relative to north + fData.Po_1D_MeasureOfPositionFixQuality = ALLdata.EM_Position.MeasureOfPositionFixQuality./100; % in meters + fData.Po_1D_PositionSystemDescriptor = ALLdata.EM_Position.PositionSystemDescriptor; % indicator if there are several GPS sources + + end + + + %% EM_Depth XXX1 to update for dual head support + if isfield(ALLdata,'EM_Depth') && ~isfield(fData,'De_1P_Date') + + comms.step('Converting EM_Depth'); + + nPings = length(ALLdata.EM_Depth.TypeOfDatagram); % total number of pings in file + maxnBeams = max(cellfun(@(x) max(x),ALLdata.EM_Depth.BeamNumber)); % maximum beam number in file + + fData.De_1P_Date = ALLdata.EM_Depth.Date; + fData.De_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_Depth.TimeSinceMidnightInMilliseconds; + fData.De_1P_PingCounter = ALLdata.EM_Depth.PingCounter; + fData.De_1P_HeadingOfVessel = ALLdata.EM_Depth.HeadingOfVessel; + fData.De_1P_SoundSpeedAtTransducer = ALLdata.EM_Depth.SoundSpeedAtTransducer*0.1; + fData.De_1P_TransmitTransducerDepth = ALLdata.EM_Depth.TransmitTransducerDepth + 65536.*ALLdata.EM_Depth.TransducerDepthOffsetMultiplier; + fData.De_1P_MaximumNumberOfBeamsPossible = ALLdata.EM_Depth.MaximumNumberOfBeamsPossible; + fData.De_1P_NumberOfValidBeams = ALLdata.EM_Depth.NumberOfValidBeams; + fData.De_1P_ZResolution = ALLdata.EM_Depth.ZResolution; + fData.De_1P_XAndYResolution = ALLdata.EM_Depth.XAndYResolution; + fData.De_1P_SamplingRate = ALLdata.EM_Depth.SamplingRate; + + % initialize + fData.De_BP_DepthZ = nan(maxnBeams,nPings); + fData.De_BP_AcrosstrackDistanceY = nan(maxnBeams,nPings); + fData.De_BP_AlongtrackDistanceX = nan(maxnBeams,nPings); + fData.De_BP_BeamDepressionAngle = nan(maxnBeams,nPings); + fData.De_BP_BeamAzimuthAngle = nan(maxnBeams,nPings); + fData.De_BP_Range = nan(maxnBeams,nPings); + fData.De_BP_QualityFactor = nan(maxnBeams,nPings); + fData.De_BP_LengthOfDetectionWindow = nan(maxnBeams,nPings); + fData.De_BP_ReflectivityBS = nan(maxnBeams,nPings); + fData.De_B1_BeamNumber = (1:maxnBeams)'; + + for iP = 1:nPings + + N = cell2mat(ALLdata.EM_Depth.BeamNumber(iP)); + + fData.De_BP_DepthZ(N,iP) = cell2mat(ALLdata.EM_Depth.DepthZ(iP)); + fData.De_BP_AcrosstrackDistanceY(N,iP) = cell2mat(ALLdata.EM_Depth.AcrosstrackDistanceY(iP)); + fData.De_BP_AlongtrackDistanceX(N,iP) = cell2mat(ALLdata.EM_Depth.AlongtrackDistanceX(iP)); + fData.De_BP_BeamDepressionAngle(N,iP) = cell2mat(ALLdata.EM_Depth.BeamDepressionAngle(iP)); + fData.De_BP_BeamAzimuthAngle(N,iP) = cell2mat(ALLdata.EM_Depth.BeamAzimuthAngle(iP)); + fData.De_BP_Range(N,iP) = cell2mat(ALLdata.EM_Depth.Range(iP)); + fData.De_BP_QualityFactor(N,iP) = cell2mat(ALLdata.EM_Depth.QualityFactor(iP)); + fData.De_BP_LengthOfDetectionWindow(N,iP) = cell2mat(ALLdata.EM_Depth.LengthOfDetectionWindow(iP)); + fData.De_BP_ReflectivityBS(N,iP) = cell2mat(ALLdata.EM_Depth.ReflectivityBS(iP)); + + end + + end + + + %% EM_XYZ88 + if isfield(ALLdata,'EM_XYZ88') && ~isfield(fData,'X8_1P_Date') + + comms.step('Converting EM_XYZ88'); + + % get the number of heads + headNumber = unique(ALLdata.EM_XYZ88.SystemSerialNumber,'stable'); + + % get ping numbers and datagrams indices + if numel(headNumber) == 1 + % there should not be multiple datagrams per ping in + % single-head data, but taking unique here just in case there + % are duplicates datagrams + [pingCounters, idxDtg] = unique(ALLdata.EM_XYZ88.PingCounter,'stable'); + idxDtg = idxDtg'; + else + % in case there's more than one head, we're going to only keep + % pings for which we have data for all heads + + % pings in first head + idxFirstHead = ALLdata.EM_XYZ88.SystemSerialNumber==headNumber(1); + pingCounters = unique(ALLdata.EM_XYZ88.PingCounter(idxFirstHead),'stable'); + + % update by keeping only common values to other heads in turn + for iH = 2:numel(headNumber) + idxThisOtherHead = ALLdata.EM_XYZ88.SystemSerialNumber==headNumber(iH); + pingCountersThisOhterHead = unique(ALLdata.EM_XYZ88.PingCounter(idxThisOtherHead),'stable'); + pingCounters = intersect(pingCounters, pingCountersThisOhterHead); + end + + % get the index of first datagram per head for a given ping + % number + idxDtg = nan(numel(headNumber), numel(pingCounters)); + for iH = 1:numel(headNumber) + idxDtg(iH,:) = arrayfun(@(x) find(ALLdata.EM_XYZ88.SystemSerialNumber==headNumber(iH) & ALLdata.EM_XYZ88.PingCounter==x, 1), pingCounters); + end + + % There is no index on head order, so sort them from portmost + % to starboardmost + [~,I] = sort(cellfun(@(x) x(1), ALLdata.EM_XYZ88.AcrosstrackDistanceY(idxDtg(:,1)))); + idxDtg = idxDtg(I,:); + headNumber = headNumber(I); + end + + % save ping numbers + fData.X8_1P_PingCounter = pingCounters; + + % for those fields, we only retain the value from the first head, + % although in practice for some fields, the values may be different + % across heads. + fData.X8_1P_Date = ALLdata.EM_XYZ88.Date(idxDtg(1,:)); + fData.X8_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_XYZ88.TimeSinceMidnightInMilliseconds(idxDtg(1,:)); + fData.X8_1P_HeadingOfVessel = ALLdata.EM_XYZ88.HeadingOfVessel(idxDtg(1,:)); + fData.X8_1P_SoundSpeedAtTransducer = ALLdata.EM_XYZ88.SoundSpeedAtTransducer(idxDtg(1,:))*0.1; + fData.X8_1P_TransmitTransducerDepth = ALLdata.EM_XYZ88.TransmitTransducerDepth(idxDtg(1,:)); + fData.X8_1P_SamplingFrequencyInHz = ALLdata.EM_XYZ88.SamplingFrequencyInHz(idxDtg(1,:)); + + % for those fields, we sum the values from + fData.X8_1P_NumberOfBeamsInDatagram = sum(ALLdata.EM_XYZ88.NumberOfBeamsInDatagram(idxDtg),1); + fData.X8_1P_NumberOfValidDetections = sum(ALLdata.EM_XYZ88.NumberOfValidDetections(idxDtg),1); + + % save dimensions + nPings = numel(pingCounters); % total number of pings in file + maxnBeams = max(fData.X8_1P_NumberOfBeamsInDatagram); % maximum beam number in file + + % initialize BP fields + fData.X8_B1_BeamNumber = (1:maxnBeams)'; + fData.X8_BP_DepthZ = nan(maxnBeams,nPings); + fData.X8_BP_AcrosstrackDistanceY = nan(maxnBeams,nPings); + fData.X8_BP_AlongtrackDistanceX = nan(maxnBeams,nPings); + fData.X8_BP_DetectionWindowLength = nan(maxnBeams,nPings); + fData.X8_BP_QualityFactor = nan(maxnBeams,nPings); + fData.X8_BP_BeamIncidenceAngleAdjustment = nan(maxnBeams,nPings); + fData.X8_BP_DetectionInformation = nan(maxnBeams,nPings); + fData.X8_BP_RealTimeCleaningInformation = nan(maxnBeams,nPings); + fData.X8_BP_ReflectivityBS = nan(maxnBeams,nPings); + fData.X8_BP_HeadSystemSerialNumber = nan(maxnBeams,nPings); + + % and fill that data ping per ping + for iPOut = 1:nPings + + % init number of beams recorded so far + nBeamTot = 0; + + % parse data per head + for iH = 1:numel(headNumber) + + % index of ping for this head in ALLdata + iPIn = find( ALLdata.EM_XYZ88.PingCounter==pingCounters(iPOut) & ... + ALLdata.EM_XYZ88.SystemSerialNumber==headNumber(iH)); + + % index of beams in output array + nBeamsIn = ALLdata.EM_XYZ88.NumberOfBeamsInDatagram(iPIn); + iBOut = nBeamTot + (1:nBeamsIn); + + fData.X8_BP_DepthZ(iBOut,iPOut) = ALLdata.EM_XYZ88.DepthZ{iPIn}; + fData.X8_BP_AcrosstrackDistanceY(iBOut,iPOut) = ALLdata.EM_XYZ88.AcrosstrackDistanceY{iPIn}; + fData.X8_BP_AlongtrackDistanceX(iBOut,iPOut) = ALLdata.EM_XYZ88.AlongtrackDistanceX{iPIn}; + fData.X8_BP_DetectionWindowLength(iBOut,iPOut) = ALLdata.EM_XYZ88.DetectionWindowLength{iPIn}; + fData.X8_BP_QualityFactor(iBOut,iPOut) = ALLdata.EM_XYZ88.QualityFactor{iPIn}; + fData.X8_BP_BeamIncidenceAngleAdjustment(iBOut,iPOut) = ALLdata.EM_XYZ88.BeamIncidenceAngleAdjustment{iPIn}*0.1; % now in deg + fData.X8_BP_DetectionInformation(iBOut,iPOut) = ALLdata.EM_XYZ88.DetectionInformation{iPIn}; + fData.X8_BP_RealTimeCleaningInformation(iBOut,iPOut) = ALLdata.EM_XYZ88.RealTimeCleaningInformation{iPIn}; + fData.X8_BP_ReflectivityBS(iBOut,iPOut) = ALLdata.EM_XYZ88.ReflectivityBS{iPIn}*0.1; % now in dB + + % add head number + fData.X8_BP_HeadSystemSerialNumber(iBOut,iPOut) = headNumber(iH); + + % update + nBeamTot = iBOut(end); + end + end + + end + + + %% EM_SeabedImage XXX1 to update for dual head support + if isfield(ALLdata,'EM_SeabedImage') && ~isfield(fData,'SI_1P_Date') + + comms.step('Converting EM_SeabedImage'); + + nPings = length(ALLdata.EM_SeabedImage.TypeOfDatagram); % total number of pings in file + maxnBeams = max(cellfun(@(x) max(x),ALLdata.EM_SeabedImage.BeamIndexNumber))+1; % maximum beam number (beam index number +1), in file + maxnSamples = max(cellfun(@(x) max(x),ALLdata.EM_SeabedImage.NumberOfSamplesPerBeam)); % maximum number of samples for a beam, in file + + fData.SI_1P_Date = ALLdata.EM_SeabedImage.Date; + fData.SI_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_SeabedImage.TimeSinceMidnightInMilliseconds; + fData.SI_1P_PingCounter = ALLdata.EM_SeabedImage.PingCounter; + fData.SI_1P_MeanAbsorptionCoefficient = ALLdata.EM_SeabedImage.MeanAbsorptionCoefficient; + fData.SI_1P_PulseLength = ALLdata.EM_SeabedImage.PulseLength; + fData.SI_1P_RangeToNormalIncidence = ALLdata.EM_SeabedImage.RangeToNormalIncidence; + fData.SI_1P_StartRangeSampleOfTVGRamp = ALLdata.EM_SeabedImage.StartRangeSampleOfTVGRamp; + fData.SI_1P_StopRangeSampleOfTVGRamp = ALLdata.EM_SeabedImage.StopRangeSampleOfTVGRamp; + fData.SI_1P_NormalIncidenceBS = ALLdata.EM_SeabedImage.NormalIncidenceBS; + fData.SI_1P_ObliqueBS = ALLdata.EM_SeabedImage.ObliqueBS; + fData.SI_1P_TxBeamwidth = ALLdata.EM_SeabedImage.TxBeamwidth; + fData.SI_1P_TVGLawCrossoverAngle = ALLdata.EM_SeabedImage.TVGLawCrossoverAngle; + fData.SI_1P_NumberOfValidBeams = ALLdata.EM_SeabedImage.NumberOfValidBeams; + + % initialize + fData.SI_BP_SortingDirection = nan(maxnBeams,nPings); + fData.SI_BP_NumberOfSamplesPerBeam = nan(maxnBeams,nPings); + fData.SI_BP_CentreSampleNumber = nan(maxnBeams,nPings); + fData.SI_B1_BeamNumber = (1:maxnBeams)'; + fData.SI_SBP_SampleAmplitudes = cell(nPings,1); % saving as a cell vector of sparse matrices, per ping + + for iP = 1:nPings + + % Get data from datagram + BeamNumber = cell2mat(ALLdata.EM_SeabedImage.BeamIndexNumber(iP))+1; + NumberOfSamplesPerBeam = cell2mat(ALLdata.EM_SeabedImage.NumberOfSamplesPerBeam(iP)); + Samples = cell2mat(ALLdata.EM_SeabedImage.SampleAmplitudes(iP).beam(:)); + + % from number of samples per beam, get indices of first and last + % sample for each beam in the Samples data vector + iFirst = [1;cumsum(NumberOfSamplesPerBeam(1:end-1))+1]; + iLast = iFirst+NumberOfSamplesPerBeam-1; + + % store + fData.SI_BP_SortingDirection(BeamNumber,iP) = cell2mat(ALLdata.EM_SeabedImage.SortingDirection(iP)); + fData.SI_BP_NumberOfSamplesPerBeam(BeamNumber,iP) = NumberOfSamplesPerBeam; + fData.SI_BP_CentreSampleNumber(BeamNumber,iP) = cell2mat(ALLdata.EM_SeabedImage.CentreSampleNumber(iP)); + + % initialize the beams/sample array (use zero instead of NaN to + % allow turning it to sparse + temp = zeros(maxnSamples,length(BeamNumber)); + + % fill in + for iB = 1:length(BeamNumber) + temp(1:NumberOfSamplesPerBeam(iB),BeamNumber(iB)) = Samples(iFirst(iB):iLast(iB)); + end + + % and save as sparse matrix, as sparse version + % to use full matrices, fData.SI_SBP_SampleAmplitudes(:,:,iP) = temp; + fData.SI_SBP_SampleAmplitudes(iP,1) = {sparse(temp)}; + + end + + end + + + %% EM_SeabedImage89 XXX1 to update for dual head support + if isfield(ALLdata,'EM_SeabedImage89') && ~isfield(fData,'S8_1P_Date') + + comms.step('Converting EM_SeabedImage89'); + + nPings = length(ALLdata.EM_SeabedImage89.TypeOfDatagram); % total number of pings in file + maxnBeams = max(ALLdata.EM_SeabedImage89.NumberOfValidBeams); % maximum beam number (beam index number +1), in file + maxnSamples = max(cellfun(@(x) max(x),ALLdata.EM_SeabedImage89.NumberOfSamplesPerBeam)); % maximum number of samples for a beam, in file + + fData.S8_1P_Date = ALLdata.EM_SeabedImage89.Date; + fData.S8_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_SeabedImage89.TimeSinceMidnightInMilliseconds; + fData.S8_1P_PingCounter = ALLdata.EM_SeabedImage89.PingCounter; + fData.S8_1P_SamplingFrequencyInHz = ALLdata.EM_SeabedImage89.SamplingFrequencyInHz; + fData.S8_1P_RangeToNormalIncidence = ALLdata.EM_SeabedImage89.RangeToNormalIncidence; + fData.S8_1P_NormalIncidenceBS = ALLdata.EM_SeabedImage89.NormalIncidenceBS; + fData.S8_1P_ObliqueBS = ALLdata.EM_SeabedImage89.ObliqueBS; + fData.S8_1P_TxBeamwidthAlong = ALLdata.EM_SeabedImage89.TxBeamwidthAlong; + fData.S8_1P_TVGLawCrossoverAngle = ALLdata.EM_SeabedImage89.TVGLawCrossoverAngle; + fData.S8_1P_NumberOfValidBeams = ALLdata.EM_SeabedImage89.NumberOfValidBeams; + + % initialize + fData.S8_BP_SortingDirection = nan(maxnBeams,nPings); + fData.S8_BP_DetectionInfo = nan(maxnBeams,nPings); + fData.S8_BP_NumberOfSamplesPerBeam = nan(maxnBeams,nPings); + fData.S8_BP_CentreSampleNumber = nan(maxnBeams,nPings); + fData.S8_B1_BeamNumber = (1:maxnBeams)'; + fData.S8_SBP_SampleAmplitudes = cell(nPings,1); % saving as a cell vector of sparse matrices, per ping + + % in this more recent datagram, all beams are in. No beamnumber anymore + BeamNumber = fData.S8_B1_BeamNumber; + + for iP = 1:nPings + + % Get data from datagram + NumberOfSamplesPerBeam = cell2mat(ALLdata.EM_SeabedImage89.NumberOfSamplesPerBeam(iP)); + Samples = cell2mat(ALLdata.EM_SeabedImage89.SampleAmplitudes(iP).beam(:)); + + % from number of samples per beam, get indices of first and last + % sample for each beam in the Samples data vector + iFirst = [1;cumsum(NumberOfSamplesPerBeam(1:end-1))+1]; + iLast = iFirst+NumberOfSamplesPerBeam-1; + + % store + fData.S8_BP_SortingDirection(BeamNumber,iP) = cell2mat(ALLdata.EM_SeabedImage89.SortingDirection(iP)); + fData.S8_BP_NumberOfSamplesPerBeam(BeamNumber,iP) = NumberOfSamplesPerBeam; + fData.S8_BP_CentreSampleNumber(BeamNumber,iP) = cell2mat(ALLdata.EM_SeabedImage89.CentreSampleNumber(iP)); + + % initialize the beams/sample array (use zero instead of NaN to + % allow turning it to sparse + temp = zeros(maxnSamples,length(BeamNumber)); + + % and fill in + for iB = 1:length(BeamNumber) + temp(1:NumberOfSamplesPerBeam(iB),BeamNumber(iB)) = Samples(iFirst(iB):iLast(iB)); + end + + % and save as sparse matrix, as sparse version + % to use full matrices, fData.S8_SBP_SampleAmplitudes(:,:,iP) = temp; + fData.S8_SBP_SampleAmplitudes(iP,1) = {sparse(temp)}; + + end + + end + + + %% EM_WaterColumn + if isfield(ALLdata,'EM_WaterColumn') && ~isfield(fData,'WC_1P_Date') + + comms.step('Converting EM_WaterColumn'); + + % get the number of heads + headNumber = unique(ALLdata.EM_WaterColumn.SystemSerialNumber,'stable'); + + % There are multiple datagrams per ping. Get the list of pings, and + % the index of the first datagram for each ping + if numel(headNumber) == 1 + % if only one head, it's simple + [pingCounters, iFirstDatagram] = unique(ALLdata.EM_WaterColumn.PingCounter,'stable'); + else + % in case there's more than one head, we're going to only keep + % pings for which we have data for all heads + + % pings for first head + pingCounters = unique(ALLdata.EM_WaterColumn.PingCounter(ALLdata.EM_WaterColumn.SystemSerialNumber==headNumber(1)),'stable'); + + % for each other head, get ping numbers and only keep + % intersection + for iH = 2:length(headNumber) + pingCountersOtherHead = unique(ALLdata.EM_WaterColumn.PingCounter(ALLdata.EM_WaterColumn.SystemSerialNumber==headNumber(iH)),'stable'); + pingCounters = intersect(pingCounters, pingCountersOtherHead); + end + + % get the index of first datagram for each ping and each head + for iH = 1:length(headNumber) + + iFirstDatagram(:,iH) = arrayfun(@(x) find(ALLdata.EM_WaterColumn.SystemSerialNumber==headNumber(iH) & ALLdata.EM_WaterColumn.PingCounter==x, 1),pingCounters); + + % Originally we would require datagram number 1, but it + % turns out it doesn't always exist. Keep this code here + % for now, but the replacement above to just find the first + % datagram for each ping seems to work fine. + % + % iFirstDatagram(:,iH) = find( ALLdata.EM_WaterColumn.SystemSerialNumber == headNumber(iH) & ... + % ismember(ALLdata.EM_WaterColumn.PingCounter,pingCounters) & ... + % ALLdata.EM_WaterColumn.DatagramNumbers == 1); + end + end + + % add the WCD decimation factors given here in input + fData.dr_sub = dr_sub; + fData.db_sub = db_sub; + + % save ping numbers + fData.WC_1P_PingCounter = pingCounters; + + % values for the following fields are constant for a ping, over all + % datagrams and all heads. Simply take value from first datagram + fData.WC_1P_Date = ALLdata.EM_WaterColumn.Date(iFirstDatagram(:,1)); + fData.WC_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_WaterColumn.TimeSinceMidnightInMilliseconds(iFirstDatagram(:,1)); + fData.WC_1P_SoundSpeed = ALLdata.EM_WaterColumn.SoundSpeed(iFirstDatagram(:,1))*0.1; + fData.WC_1P_OriginalSamplingFrequencyHz = ALLdata.EM_WaterColumn.SamplingFrequency(iFirstDatagram(:,1)).*0.01; % in Hz + fData.WC_1P_SamplingFrequencyHz = (ALLdata.EM_WaterColumn.SamplingFrequency(iFirstDatagram(:,1)).*0.01)./dr_sub; % in Hz + fData.WC_1P_TXTimeHeave = ALLdata.EM_WaterColumn.TXTimeHeave(iFirstDatagram(:,1)); + fData.WC_1P_TVGFunctionApplied = ALLdata.EM_WaterColumn.TVGFunctionApplied(iFirstDatagram(:,1)); + fData.WC_1P_TVGOffset = ALLdata.EM_WaterColumn.TVGOffset(iFirstDatagram(:,1)); + fData.WC_1P_ScanningInfo = ALLdata.EM_WaterColumn.ScanningInfo(iFirstDatagram(:,1)); + + % Still, test for inconsistencies between heads and raise a warning + % if we detect one. To be investigated if it every happens + if length(headNumber) > 1 + fields = {'SoundSpeed','SamplingFrequency','TXTimeHeave','TVGFunctionApplied','TVGOffset','ScanningInfo'}; + for iFi = 1:length(fields) + if any(any(ALLdata.EM_WaterColumn.(fields{iFi})(iFirstDatagram(:,1))'.*ones(1,length(headNumber))~=ALLdata.EM_WaterColumn.(fields{iFi})(iFirstDatagram))) + warning('System has more than one head and "%s" data are inconsistent between heads for at least one ping. Using information from first head anyway.',fields{iFi}); + end + end + end + + % for the following fields, values need need to be summed over all + % heads + if length(headNumber) > 1 + fData.WC_1P_NumberOfDatagrams = sum(ALLdata.EM_WaterColumn.NumberOfDatagrams(iFirstDatagram),2)'; + fData.WC_1P_NumberOfTransmitSectors = sum(ALLdata.EM_WaterColumn.NumberOfTransmitSectors(iFirstDatagram),2)'; + fData.WC_1P_TotalNumberOfReceiveBeams = sum(ALLdata.EM_WaterColumn.TotalNumberOfReceiveBeams(iFirstDatagram),2)'; + fData.WC_1P_NumberOfBeamsToRead = sum(ceil(ALLdata.EM_WaterColumn.TotalNumberOfReceiveBeams(iFirstDatagram)/db_sub),2)'; % each head is decimated in beam individually + else + fData.WC_1P_NumberOfDatagrams = ALLdata.EM_WaterColumn.NumberOfDatagrams(iFirstDatagram); + fData.WC_1P_NumberOfTransmitSectors = ALLdata.EM_WaterColumn.NumberOfTransmitSectors(iFirstDatagram); + fData.WC_1P_TotalNumberOfReceiveBeams = ALLdata.EM_WaterColumn.TotalNumberOfReceiveBeams(iFirstDatagram); + fData.WC_1P_NumberOfBeamsToRead = ceil(ALLdata.EM_WaterColumn.TotalNumberOfReceiveBeams(iFirstDatagram)/db_sub); + end + + % number of pings + nPings = length(pingCounters); % total number of pings in file + + % number of Tx sectors + maxnTxSectors = max(fData.WC_1P_NumberOfTransmitSectors); % maximum number of transmit sectors in a ping + + % number of beams + % maxnBeams = max(fData.WC_1P_TotalNumberOfReceiveBeams); % maximum number of receive beams in a ping (not using it) + maxnBeams_sub = max(fData.WC_1P_NumberOfBeamsToRead); % maximum number of receive beams TO READ + + % number of samples + % maxnSamples = max(cellfun(@(x) max(x), ALLdata.EM_WaterColumn.NumberOfSamples(ismember(ALLdata.EM_WaterColumn.PingCounter,pingCounters)))); % maximum number of samples in a ping (not using it) + % maxnSamples_sub = ceil(maxnSamples/dr_sub); + [maxnSamples_groups, ping_group_start, ping_group_end] = CFF_group_pings(ALLdata.EM_WaterColumn.NumberOfSamples, pingCounters, ALLdata.EM_WaterColumn.PingCounter); % making groups of pings to limit size of memmaped files + maxnSamples_groups = ceil(maxnSamples_groups/dr_sub); % maximum number of samples TO READ, per group. + + % initialize data per transmit sector and ping + fData.WC_TP_TiltAngle = nan(maxnTxSectors,nPings); + fData.WC_TP_CenterFrequency = nan(maxnTxSectors,nPings); + fData.WC_TP_TransmitSectorNumber = nan(maxnTxSectors,nPings); + fData.WC_TP_SystemSerialNumber = nan(maxnTxSectors,nPings); + + % initialize data per decimated beam and ping + fData.WC_BP_BeamPointingAngle = nan(maxnBeams_sub,nPings); + fData.WC_BP_StartRangeSampleNumber = nan(maxnBeams_sub,nPings); + fData.WC_BP_NumberOfSamples = nan(maxnBeams_sub,nPings); + fData.WC_BP_DetectedRangeInSamples = zeros(maxnBeams_sub,nPings); + fData.WC_BP_TransmitSectorNumber = nan(maxnBeams_sub,nPings); + fData.WC_BP_BeamNumber = nan(maxnBeams_sub,nPings); + fData.WC_BP_SystemSerialNumber = nan(maxnBeams_sub,nPings); + + % The actual water-column data will not be saved in fData but in + % binary files. Get the output directory to store those files + wc_dir = CFF_converted_data_folder(fData.ALLfilename{iF}); + + % Clean up that folder first before adding anything to it + CFF_clean_delete_fdata(wc_dir); + + % DEV NOTE: Info format for raw WC data and storage + % In the raw datagrams, there is only amplitude and no phase. + % Samples are recorded in "int8" (signed integers from + % -128 to 127) with -128 being the NaN value. Raw values needs to + % be multiplied by a factor of 1/2 to retrieve the true value, aka + % real values go from -127/2 = -63.5 dB to 127/2 = 63.5 dB in + % increments of 0.5 dB + % For storage, we keep the same format in order to save disk space. + + % initialize data-holding binary files + fData = CFF_init_memmapfiles(fData, ... + 'field', 'WC_SBP_SampleAmplitudes', ... + 'wc_dir', wc_dir, ... + 'Class', 'int8', ... + 'Factor', 1./2, ... + 'Nanval', intmin('int8'), ... + 'Offset', 0, ... + 'MaxSamples', maxnSamples_groups, ... + 'MaxBeams', maxnBeams_sub, ... + 'ping_group_start', ping_group_start, ... + 'ping_group_end', ping_group_end); + + % Also the samples data were not recorded in ALLdata, only their + % location in the source file, so we need to fopen the source file + % to grab the data. + fid_all = fopen(fData.ALLfilename{iF},'r',ALLdata.datagramsformat); + + % initialize ping group counter, to use to specify which memmapfile + % to fill. We start in the first. + iG = 1; + + % now get data for each ping + for iP = 1:nPings + + % ping number (ex: 50455) + pingCounter = fData.WC_1P_PingCounter(1,iP); + + % update ping group counter if needed + if pingCounter > fData.WC_1P_PingCounter(ping_group_end(iG)) + iG = iG+1; + end + + % initialize amplitude matrix for that ping + SB_temp = intmin('int8').*ones(maxnSamples_groups(iG),maxnBeams_sub,'int8'); + + % initialize number of sectors and beams recorded so far for + % that ping (needed for multiple heads) + nTxSectTot = 0; + nBeamTot = 0; + + for iH = 1:length(headNumber) + + headSSN = headNumber(iH); + + % index of the datagrams making up this ping/head in ALLdata.EM_Watercolumn (ex: 58-59-61-64) + iDatagrams = find( ALLdata.EM_WaterColumn.PingCounter == pingCounter & ... + ALLdata.EM_WaterColumn.SystemSerialNumber == headSSN); + + % actual number of datagrams available (ex: 4) + nDatagrams = length(iDatagrams); + + % some datagrams may be missing. Need to detect and adjust. + datagramOrder = ALLdata.EM_WaterColumn.DatagramNumbers(iDatagrams); % order of the datagrams (ex: 4-3-6-2, the missing one is 1st, 5th and 7th) + [~,IX] = sort(datagramOrder); + iDatagrams = iDatagrams(IX); % index of the datagrams making up this ping in ALLdata.EM_Watercolumn, but in the right order (ex: 64-59-58-61, missing datagrams are still missing) + nBeamsPerDatagram = ALLdata.EM_WaterColumn.NumberOfBeamsInThisDatagram(iDatagrams); % number of beams in each datagram making up this ping (ex: 56-61-53-28) + + % number of transmit sectors to record + nTxSect = ALLdata.EM_WaterColumn.NumberOfTransmitSectors(iDatagrams(1)); + + % indices of those sectors in output structure + iTxSectDest = nTxSectTot + (1:nTxSect); + + % data per Tx sector + fData.WC_TP_TiltAngle(iTxSectDest,iP) = ALLdata.EM_WaterColumn.TiltAngle{iDatagrams(1)}; + fData.WC_TP_CenterFrequency(iTxSectDest,iP) = ALLdata.EM_WaterColumn.CenterFrequency{iDatagrams(1)}; + fData.WC_TP_TransmitSectorNumber(iTxSectDest,iP) = ALLdata.EM_WaterColumn.TransmitSectorNumber{iDatagrams(1)}; + fData.WC_TP_SystemSerialNumber(iTxSectDest,iP) = headSSN; + + % updating total number of sectors recorded so far + nTxSectTot = nTxSectTot + nTxSect; + + % and then read the data in each datagram + for iD = 1:nDatagrams + + % indices of desired beams in this head/datagram + if iD == 1 + % if first datagram, start with first beam + iBeamStart = 1; + else + % if not first datagram, continue the decimation where we left it + nBeamsLastDatag = nBeamsPerDatagram(iD-1); + lastRecBeam = iBeamSource(end); + iBeamStart = db_sub - (nBeamsLastDatag-lastRecBeam); + end + + % select beams with decimation + iBeamSource = iBeamStart:db_sub:nBeamsPerDatagram(iD); + + % number of beams to record + nBeam = length(iBeamSource); + + % indices of those beams in output structure + iBeamDest = nBeamTot + (1:nBeam); + + % record those beams' data, applying decimation in range to the data that are samples numbers. + fData.WC_BP_BeamPointingAngle(iBeamDest,iP) = ALLdata.EM_WaterColumn.BeamPointingAngle{iDatagrams(iD)}(iBeamSource)/100; + fData.WC_BP_StartRangeSampleNumber(iBeamDest,iP) = round(ALLdata.EM_WaterColumn.StartRangeSampleNumber{iDatagrams(iD)}(iBeamSource)./dr_sub); + fData.WC_BP_NumberOfSamples(iBeamDest,iP) = round(ALLdata.EM_WaterColumn.NumberOfSamples{iDatagrams(iD)}(iBeamSource)./dr_sub); + fData.WC_BP_DetectedRangeInSamples(iBeamDest,iP) = round(ALLdata.EM_WaterColumn.DetectedRangeInSamples{iDatagrams(iD)}(iBeamSource)./dr_sub); + fData.WC_BP_TransmitSectorNumber(iBeamDest,iP) = ALLdata.EM_WaterColumn.TransmitSectorNumber2{iDatagrams(iD)}(iBeamSource); + fData.WC_BP_BeamNumber(iBeamDest,iP) = ALLdata.EM_WaterColumn.BeamNumber{iDatagrams(iD)}(iBeamSource); + fData.WC_BP_SystemSerialNumber(iBeamDest,iP) = headSSN; + + % and then, in each beam... + for iB = 1:nBeam + + % get actual number of samples in that beam + nSamp = ALLdata.EM_WaterColumn.NumberOfSamples{iDatagrams(iD)}(iBeamSource(iB)); + + % number of samples we're going to record + nSamp_sub = ceil(nSamp/dr_sub); + + % get to the start of the data in original file + pos = ALLdata.EM_WaterColumn.SampleAmplitudePosition{iDatagrams(iD)}(iBeamSource(iB)); + curr_pos = ftell(fid_all); + fseek(fid_all,pos-curr_pos,'cof'); + + % read raw data, with decimation in range + SB_temp(1:nSamp_sub,iBeamDest(iB)) = fread(fid_all, nSamp_sub, 'int8=>int8', dr_sub-1); + + end + + % updating total number of beams recorded so far + nBeamTot = nBeamTot + nBeam; + + end + + end + + % finished reading this ping's WC data. Store the data in the + % appropriate binary file, at the appropriate ping, through the + % memory mapping + fData.WC_SBP_SampleAmplitudes{iG}.Data.val(:,:,iP-ping_group_start(iG)+1) = SB_temp; + + end + + % close the original raw file + fclose(fid_all); + + end + + + %% EM_AmpPhase + if isfield(ALLdata,'EM_AmpPhase') && ~isfield(fData,'AP_1P_Date') + + comms.step('Converting EM_AmpPhase'); + + % get the number of heads + headNumber = unique(ALLdata.EM_AmpPhase.SystemSerialNumber,'stable'); + % note we don't support multiple-head data for AP as we do for WC. + % To be coded if we ever come across it... XXX1 + + % There are multiple datagrams per ping. Get the list of pings, and + % the index of the first datagram for each ping + [pingCounters,iFirstDatagram] = unique(ALLdata.EM_AmpPhase.PingCounter,'stable'); + + % add the WCD decimation factors given here in input + fData.dr_sub = dr_sub; + fData.db_sub = db_sub; + + % save ping numbers + fData.AP_1P_PingCounter = ALLdata.EM_AmpPhase.PingCounter(iFirstDatagram); + + % values for the following fields are constant over all datagrams + % making up a ping. Take value from first datagram + fData.AP_1P_Date = ALLdata.EM_AmpPhase.Date(iFirstDatagram); + fData.AP_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_AmpPhase.TimeSinceMidnightInMilliseconds(iFirstDatagram); + fData.AP_1P_SoundSpeed = ALLdata.EM_AmpPhase.SoundSpeed(iFirstDatagram)*0.1; + fData.AP_1P_OriginalSamplingFrequencyHz = ALLdata.EM_AmpPhase.SamplingFrequency(iFirstDatagram).*0.01; % in Hz + fData.AP_1P_SamplingFrequencyHz = (ALLdata.EM_AmpPhase.SamplingFrequency(iFirstDatagram).*0.01)./dr_sub; % in Hz + fData.AP_1P_TXTimeHeave = ALLdata.EM_AmpPhase.TXTimeHeave(iFirstDatagram); + fData.AP_1P_TVGFunctionApplied = ALLdata.EM_AmpPhase.TVGFunctionApplied(iFirstDatagram); + fData.AP_1P_TVGOffset = ALLdata.EM_AmpPhase.TVGOffset(iFirstDatagram); + fData.AP_1P_ScanningInfo = ALLdata.EM_AmpPhase.ScanningInfo(iFirstDatagram); + fData.AP_1P_NumberOfDatagrams = ALLdata.EM_AmpPhase.NumberOfDatagrams(iFirstDatagram); + fData.AP_1P_NumberOfTransmitSectors = ALLdata.EM_AmpPhase.NumberOfTransmitSectors(iFirstDatagram); + fData.AP_1P_TotalNumberOfReceiveBeams = ALLdata.EM_AmpPhase.TotalNumberOfReceiveBeams(iFirstDatagram); + fData.AP_1P_NumberOfBeamsToRead = ceil(ALLdata.EM_AmpPhase.TotalNumberOfReceiveBeams(iFirstDatagram)/db_sub); + + % ----- get original data dimensions ------------------------------ + % total number of pings in file + nPings = length(pingCounters); + + % maximum number of transmit sectors in a ping + maxnTxSectors = max(fData.AP_1P_NumberOfTransmitSectors); + + % maximum number of receive beams in a ping (not using it) + % maxnBeams = max(fData.AP_1P_TotalNumberOfReceiveBeams); + + % max number of samples for a beam in file (not using it) + % maxnSamples = max(cellfun(@(x) max(x), ALLdata.EM_AmpPhase.NumberOfSamples)); + % ----------------------------------------------------------------- + + % ----- get dimensions of data to read after decimation ----------- + % maximum number of receive beams TO READ + maxnBeams_sub = max(fData.AP_1P_NumberOfBeamsToRead); + + % maximum number of samples TO READ + % maxnSamples_sub = ceil(maxnSamples/dr_sub); + + % make groups of pings, so that indiviudal binary files are not too + % big + [maxnSamples_groups, ping_group_start, ping_group_end] = CFF_group_pings(ALLdata.EM_AmpPhase.NumberOfSamples, pingCounters, ALLdata.EM_AmpPhase.PingCounter); + + % maximum number of samples TO READ, per group. + maxnSamples_groups = ceil(maxnSamples_groups/dr_sub); + % ----------------------------------------------------------------- + + % initialize data per transmit sector and ping + fData.AP_TP_TiltAngle = nan(maxnTxSectors,nPings); + fData.AP_TP_CenterFrequency = nan(maxnTxSectors,nPings); + fData.AP_TP_TransmitSectorNumber = nan(maxnTxSectors,nPings); + % fData.AP_TP_SystemSerialNumber = nan(maxnTxSectors,nPings); + + % initialize data per decimated beam and ping + fData.AP_BP_BeamPointingAngle = nan(maxnBeams_sub,nPings); + fData.AP_BP_StartRangeSampleNumber = nan(maxnBeams_sub,nPings); + fData.AP_BP_NumberOfSamples = nan(maxnBeams_sub,nPings); + fData.AP_BP_DetectedRangeInSamples = zeros(maxnBeams_sub,nPings); + fData.AP_BP_TransmitSectorNumber = nan(maxnBeams_sub,nPings); + fData.AP_BP_BeamNumber = nan(maxnBeams_sub,nPings); + % fData.AC_BP_SystemSerialNumber = nan(maxnBeams_sub,nPings); + + % The actual water-column data will not be saved in fData but in + % binary files. Get the output directory to store those files + wc_dir = CFF_converted_data_folder(fData.ALLfilename{iF}); + + % Clean up that folder first before adding anything to it, BUT NOT + % IF WE JUST RECORDED DATA WITH THE WC DATAGRAM + if ~isfield(fData,'WC_1P_Date') + CFF_clean_delete_fdata(wc_dir); + end + + % DEV NOTE: Info format for raw WC data and storage + % This was originally coded by Yoann and I don't have documentation + % about the raw data format for these datagrams. Inferring from the + % code: + % * The amplitude appears to be stored as uint16 in natural values + % with a factor of 1/10000. No idea about the NaN value but + % assuming it's intmin. We need to convert those to dB so we can't + % reuse that format like we did in the WC datagram. Here we will + % store the data as int16 with a factor of 1/200. + % * The phase appears to be stored as int16 in radians, with a + % factor of 1/10000. No idea about the NaN value but assuming it's + % intmin. Here we will store the data as int16, as degrees, with a + % factor of 1/30 and 200 as the NaN value. + + % initialize data-holding binary files + fData = CFF_init_memmapfiles(fData, ... + 'field', 'AP_SBP_SampleAmplitudes', ... + 'wc_dir', wc_dir, ... + 'Class', 'int16', ... + 'Factor', 1/200, ... + 'Nanval', intmin('int16'), ... + 'Offset', 0, ... + 'MaxSamples', maxnSamples_groups, ... + 'MaxBeams', maxnBeams_sub, ... + 'ping_group_start', ping_group_start, ... + 'ping_group_end', ping_group_end); + + fData = CFF_init_memmapfiles(fData, ... + 'field', 'AP_SBP_SamplePhase', ... + 'wc_dir', wc_dir, ... + 'Class', 'int16', ... + 'Factor', 1/30, ... + 'Nanval', 200, ... + 'Offset', 0, ... + 'MaxSamples', maxnSamples_groups, ... + 'MaxBeams', maxnBeams_sub, ... + 'ping_group_start', ping_group_start, ... + 'ping_group_end', ping_group_end); + + % Also the samples data were not recorded in ALLdata, only their + % location in the source file, so we need to fopen the source file + % to grab the data. + fid_all = fopen(fData.ALLfilename{iF},'r',ALLdata.datagramsformat); + + % initialize ping group counter, to use to specify which memmapfile + % to fill. We start in the first. + iG = 1; + + % debug graph + disp_wc = 0; + if disp_wc + f = figure(); + ax_mag = axes(f,'outerposition',[0 0.5 1 0.5]); + ax_phase = axes(f,'outerposition',[0 0 1 0.5]); + end + + % now get data for each ping + for iP = 1:nPings + + % ping number (ex: 50455) + pingCounter = fData.AP_1P_PingCounter(1,iP); + + % update ping group counter if needed + if pingCounter > fData.AP_1P_PingCounter(ping_group_end(iG)) + iG = iG+1; + end + + % initialize the water column data matrix for that ping. + SB2_temp = intmin('int16').*ones(maxnSamples_groups(iG),maxnBeams_sub,'int16'); + Ph_temp = 200.*ones(maxnSamples_groups(iG),maxnBeams_sub,'int16'); + + % index of the datagrams making up this ping/head in ALLdata.EM_Watercolumn (ex: 58-59-61-64) + iDatagrams = find(ALLdata.EM_AmpPhase.PingCounter==pingCounter); + + % actual number of datagrams available (ex: 4) + nDatagrams = length(iDatagrams); + + % some datagrams may be missing. Need to detect and adjust. + datagramOrder = ALLdata.EM_AmpPhase.DatagramNumbers(iDatagrams); % order of the datagrams (ex: 4-3-6-2, the missing one is 1st, 5th and 7th) + [~,IX] = sort(datagramOrder); + iDatagrams = iDatagrams(IX); % index of the datagrams making up this ping in ALLdata.EM_AmpPhase, but in the right order (ex: 64-59-58-61, missing datagrams are still missing) + nBeamsPerDatagram = ALLdata.EM_AmpPhase.NumberOfBeamsInThisDatagram(iDatagrams); % number of beams in each datagram making up this ping (ex: 56-61-53-28) + + % number of transmit sectors in this ping + nTxSect = fData.AP_1P_NumberOfTransmitSectors(1,iP); + + % recording data per transmit sector + fData.AP_TP_TiltAngle(1:nTxSect,iP) = ALLdata.EM_AmpPhase.TiltAngle{iDatagrams(1)}; + fData.AP_TP_CenterFrequency(1:nTxSect,iP) = ALLdata.EM_AmpPhase.CenterFrequency{iDatagrams(1)}; + fData.AP_TP_TransmitSectorNumber(1:nTxSect,iP) = ALLdata.EM_AmpPhase.TransmitSectorNumber{iDatagrams(1)}; + + % and then read the data in each datagram + for iD = 1:nDatagrams + + % index of beams in output structure for this datagram + [iBeams,idx_beams] = unique(ceil((sum(nBeamsPerDatagram(1:iD-1)) + (1:nBeamsPerDatagram(iD)))/db_sub)); + % old approach + % iBeams = sum(nBeamsPerDatagram(1:iD-1)) + (1:nBeamsPerDatagram(iD)); + % idx_beams = (1:numel(iBeams)); + + % ping x beam data + fData.AP_BP_BeamPointingAngle(iBeams,iP) = ALLdata.EM_AmpPhase.BeamPointingAngle{iDatagrams(iD)}(idx_beams)/100; + fData.AP_BP_StartRangeSampleNumber(iBeams,iP) = round(ALLdata.EM_AmpPhase.StartRangeSampleNumber{iDatagrams(iD)}(idx_beams)./dr_sub); + fData.AP_BP_NumberOfSamples(iBeams,iP) = round(ALLdata.EM_AmpPhase.NumberOfSamples{iDatagrams(iD)}(idx_beams)./dr_sub); + fData.AP_BP_DetectedRangeInSamples(iBeams,iP) = round(ALLdata.EM_AmpPhase.DetectedRangeInSamples{iDatagrams(iD)}(idx_beams)./dr_sub); + fData.AP_BP_TransmitSectorNumber(iBeams,iP) = ALLdata.EM_AmpPhase.TransmitSectorNumber2{iDatagrams(iD)}(idx_beams); + fData.AP_BP_BeamNumber(iBeams,iP) = ALLdata.EM_AmpPhase.BeamNumber{iDatagrams(iD)}(idx_beams); + + % and then, in each beam... + for iB = 1:numel(iBeams) + + % get actual number of samples in that beam + nSamp = ALLdata.EM_AmpPhase.NumberOfSamples{iDatagrams(iD)}(idx_beams(iB)); + + % number of samples we're going to record: + nSamp_sub = ceil(nSamp/dr_sub); + + % get the data: + if nSamp_sub > 0 + + % get to the start of amplitude data + pos = ALLdata.EM_AmpPhase.SamplePhaseAmplitudePosition{iDatagrams(iD)}(idx_beams(iB)); + fseek(fid_all,pos,'bof'); + + % read and decode amplitude data + tmp = fread(fid_all,nSamp_sub,'uint16',2); % XXX1 it's missing dr_sub + tmp = 20*log10(single(tmp)*0.0001); + + % rencode amplitude data in storage format + SB2_temp((1:(nSamp_sub)),iBeams(iB)) = int16(tmp*200); + + % get to the start of phase data + pos = pos+2; + fseek(fid_all,pos,'bof'); + + % read and decode phase data + tmp = fread(fid_all,nSamp_sub,'int16',2); % XXX1 it's missing dr_sub + tmp = -0.0001*single(tmp)/pi*180; + + % transform phase data to storage format + Ph_temp((1:(nSamp_sub)),iBeams(iB)) = int16(tmp*30); + + end + end + end + + % finished reading this ping's WC data. Store the data in the + % appropriate binary file, at the appropriate ping, through the + % memory mapping + fData.AP_SBP_SampleAmplitudes{iG}.Data.val(:,:,iP-ping_group_start(iG)+1) = SB2_temp; + fData.AP_SBP_SamplePhase{iG}.Data.val(:,:,iP-ping_group_start(iG)+1) = Ph_temp; + + % debug graph + if disp_wc + imagesc(ax_mag,single(SB2_temp)/200); + hold(ax_mag,'on');plot(ax_mag,fData.AP_BP_DetectedRangeInSamples(:,iP)); + hold(ax_mag,'off'); + caxis(ax_mag,[-100 -20]); + imagesc(ax_phase,single(Ph_temp)/30); + hold(ax_phase,'on');plot(ax_phase,fData.AP_BP_DetectedRangeInSamples(:,iP)); + hold(ax_phase,'off'); + drawnow; + end + + end + + % close the original raw file + fclose(fid_all); + + end + + % communicate progress + comms.progress(iF,nStruct); + +end + +%% end message +comms.finish('Done'); diff --git a/read_data_files/Kongsberg/format_all/CFF_convert_ALLdata_to_fData_old.m b/read_data_files/Kongsberg/format_all/CFF_convert_ALLdata_to_fData_old.m new file mode 100644 index 0000000..aee4953 --- /dev/null +++ b/read_data_files/Kongsberg/format_all/CFF_convert_ALLdata_to_fData_old.m @@ -0,0 +1,1185 @@ +function [fData,update_flag] = CFF_convert_ALLdata_to_fData(ALLdataGroup,varargin) +%CFF_CONVERT_ALLDATA_TO_FDATA One-line description +% +% Converts the Kongsberg EM series data files in ALLdata format +% (containing the KONGSBERG datagrams) to the fData format used in +% processing. +% +% fData = CFF_CONVERT_ALLDATA_TO_FDATA(ALLdata) converts the contents of +% one ALLdata structure to a structure in the fData format. +% +% fData = CFF_CONVERT_ALLDATA_TO_FDATA({ALLdata;WCDdata}) converts two +% ALLdata structures into one fData sructure. While many more structures +% can thus be loaded, this is typically done because ALLdata structures +% exist on a per-file basis and Kongsberg raw data often come in pairs of +% files: one .all and one .wcd. Note that the ALLdata structures are +% converted to fData in the order they are in input, and that the first +% ones take precedence, aka in the example above, if WCDdata contains a +% type of datagram that is already in ALLdata, they will NOT be +% converted. +% This is to avoid doubling up. Order the ALLdata structures in input in +% order of desired precedence. DO NOT use this feature to convert ALLdata +% structures from different acquisition files. It will not work. Convert +% each into its own fData structure. +% +% fData = CFF_CONVERT_ALLDATA_TO_FDATA(ALLdata,10,2) operates the +% conversion with sub-sampling in range and in beams. For example, to +% sub-sample range by a factor of 10 and beams by a factor of 2, use +% fData = CFF_CONVERT_ALLDATA_TO_FDATA(ALLdata,10,2). +% +% [fData,update_flag] = CFF_CONVERT_ALLDATA_TO_FDATA(ALLdata,1,1,fData) +% takes the result of a precedent conversion in input to allow +% potentially saving time. The function will start by loading the result +% of the precedent conversion, check that what you're trying to add to it +% comes from the same raw data source, and add to fData only those types +% of datagrams that may be missing. If fData has been modified, it will +% return a update_flag=1. If the output is the same and no modification +% occured, then it will return a update_flag=0. NOTE: If the decimation +% factors in input are different to those used in the input fData, then +% the data WILL be updated. This is actually the main use of this +% feature... +% +% *INPUT VARIABLES* +% * |ALLdataGroup|: Required. ALLdata structure or cells of ALLdata +% structures. +% * |dr_sub|: Optional. Scalar for decimation in range. Default: 1 (no +% decimation). +% * |db_sub|: Optional. Scalar for decimation in beams. Default: 1 (no +% decimation). +% * |fData|: Optional. Existing fData structure to add to. +% +% *OUTPUT VARIABLES* +% * |fData|: structure for the storage of kongsberg EM series multibeam +% data in a format more convenient for processing. The data is recorded +% as fields coded "a_b_c" where "a" is a code indicating data origing, +% "b" is a code indicating data dimensions, and "c" is the data name. +% * a: code indicating data origin: +% * IP: installation parameters +% * Ru: Runtime Parameters +% * De: depth datagram +% * He: height datagram +% * X8: XYZ88 datagram +% * SI: seabed image datagram +% * S8: seabed image data 89 +% * WC: watercolumn data +% * Po: position datagram +% * At: attitude datagram +% * SS: sound speed profile datagram +% * AP: "Amplitude and phase" water-column data +% More codes for the 'a' part will be created if more datagrams are +% parsed. Data derived from computations can be recorded back into +% fData using 'X' for the "a" code. +% * b: code indicating data dimensions (rows/columns) +% * 1P: ping-like single-row-vector +% * B1: beam-like single-column-vector +% * BP: beam/ping array +% * TP: transmit-sector/ping array +% * SP: samples/ping array (note: samples are not sorted, this is +% not equivalent to range!) +% * 1D: datagram-like single-row-vector (for attitude or +% position data) +% * ED: entries-per-datagram/datagram array (for attitude or +% position data) +% * SBP: sample/beam/ping array (water-column data) +% More codes for the 'b' part will be created if the storage of other +% datagrams needs them. Data derived from computations can be recorded +% back into fData using appropriate "b" codes such as: +% * RP: range (choose distance, time or sample) / ping +% * SP: swathe (meters) / ping +% * LL: lat/long (WGS84) +% * N1: northing-like single-column-vector +% * 1E: easting-like single-row-vector +% * NE: northing/easting +% * NEH: northing/easting/height +% * c: data type, obtained from the original variable name in the +% Kongsberg datagram, or from the user's imagination for derived data +% obtained from subsequent functions. +% * |update_flag|: 1 if a fData was given in input and was modified with +% this function +% +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +%% input parsing + +% init +p = inputParser; + +% required +addRequired(p,'ALLdataGroup',@(x) isstruct(x) || iscell(x)); + +% optional +addOptional(p,'dr_sub',1,@(x) isnumeric(x)&&x>0); +addOptional(p,'db_sub',1,@(x) isnumeric(x)&&x>0); +addOptional(p,'fData',{},@(x) isstruct(x) || iscell(x)); + +% parse +parse(p,ALLdataGroup,varargin{:}) + +% get results +ALLdataGroup = p.Results.ALLdataGroup; +dr_sub = p.Results.dr_sub; +db_sub = p.Results.db_sub; +fData = p.Results.fData; +clear p; + + +%% pre-processing + +if ~iscell(ALLdataGroup) + ALLdataGroup = {ALLdataGroup}; +end + +% number of individual ALLdata structures in input ALLdataGroup +nStruct = length(ALLdataGroup); + +% initialize fData if one not given in input +if isempty(fData) + + + % initialize FABC structure by writing in the raw data filenames to be + % added here + fData.ALLfilename = cell(1,nStruct); + for iF = 1:nStruct + fData.ALLfilename{iF} = ALLdataGroup{iF}.ALLfilename; + end + + % add the decimation factors given here in input + fData.dr_sub = dr_sub; + fData.db_sub = db_sub; + +end + + +if ~isfield(fData,'MET_Fmt_version')&&~isempty(fData) + %added a version for fData + fData.MET_Fmt_version='0.0'; +end + +if ~strcmpi(ver,CFF_get_current_fData_version) + f_reconvert = 1; + update_mode = 0; +else + f_reconvert = 0; + update_mode = 1; +end + +% initialize update_flag +update_flag = 0; + + + +%% take one ALLdata structure at a time and add its contents to fData + +for iF = 1:nStruct + + + %% pre processing + + % get current structure + ALLdata = ALLdataGroup{iF}; + + % Make sure we don't update fData with datagrams from different + % sources + % XXX2 clean up that display later + if ~ismember(ALLdata.ALLfilename,fData.ALLfilename) + fprintf('Cannot add different files to this structure.\n') + continue; + end + + % open the original raw file in case we need to grab WC data from it + fid_all = fopen(fData.ALLfilename{iF},'r',ALLdata.datagramsformat); + + % get folder for converted data + wc_dir = CFF_converted_data_folder(fData.ALLfilename{iF}); + + % now reading each type of datagram... + + %% EM_InstallationStart (v2 VERIFIED) + + if isfield(ALLdata,'EM_InstallationStart') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'IP_ASCIIparameters') + + if update_mode + update_flag = 1; + end + + % initialize struct + IP_ASCIIparameters = struct; + + % read ASCIIdata + ASCIIdata = char(ALLdata.EM_InstallationStart.ASCIIData(1)); + + % remove carriage returns, tabs and linefeed + ASCIIdata = regexprep(ASCIIdata,char(9),''); + ASCIIdata = regexprep(ASCIIdata,newline,''); + ASCIIdata = regexprep(ASCIIdata,char(13),''); + + % read individual fields + if ~isempty(ASCIIdata) + + yo = strfind(ASCIIdata,',')'; + yo(:,1) = [1; yo(1:end-1)+1]; % beginning of ASCII field name + yo(:,2) = strfind(ASCIIdata,'=')'-1; % end of ASCII field name + yo(:,3) = strfind(ASCIIdata,'=')'+1; % beginning of ASCII field value + yo(:,4) = strfind(ASCIIdata,',')'-1; % end of ASCII field value + + for ii = 1:size(yo,1) + + % get field string + field = ASCIIdata(yo(ii,1):yo(ii,2)); + + % try turn value into numeric + value = str2double(ASCIIdata(yo(ii,3):yo(ii,4))); + if length(value)~=1 + % looks like it cant. Keep as string + value = ASCIIdata(yo(ii,3):yo(ii,4)); + end + + % store field/value + IP_ASCIIparameters.(field) = value; + + end + + end + + % finally store in fData + fData.IP_ASCIIparameters = IP_ASCIIparameters; + + end + + end + + + %% EM_Runtime (v2 VERIFIED) + + if isfield(ALLdata,'EM_Runtime') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'Ru_1D_Date') + + if update_mode + update_flag = 1; + end + + % NumberOfDatagrams = length(ALLdata.EM_Runtime.TypeOfDatagram); + % MaxNumberOfEntries = max(ALLdata.EM_Runtime.NumberOfEntries); + + fData.Ru_1D_Date = ALLdata.EM_Runtime.Date; + fData.Ru_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_Runtime.TimeSinceMidnightInMilliseconds; + fData.Ru_1D_PingCounter = ALLdata.EM_Runtime.PingCounter; + % the rest to code... XXX3 + fData.Ru_1D_TransmitPowerReMaximum = ALLdata.EM_Runtime.TransmitPowerReMaximum; + fData.Ru_1D_ReceiveBeamwidth = ALLdata.EM_Runtime.ReceiveBeamwidth; + % the rest to code... XXX3 + + end + + end + + + %% EM_SoundSpeedProfile (v2 VERIFIED) + + if isfield(ALLdata,'EM_SoundSpeedProfile') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'SS_1D_Date') + + if update_mode + update_flag = 1; + end + + NumberOfDatagrams = length(ALLdata.EM_SoundSpeedProfile.TypeOfDatagram); + MaxNumberOfEntries = max(ALLdata.EM_SoundSpeedProfile.NumberOfEntries); + + fData.SS_1D_Date = ALLdata.EM_SoundSpeedProfile.Date; + fData.SS_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_SoundSpeedProfile.TimeSinceMidnightInMilliseconds; + fData.SS_1D_ProfileCounter = ALLdata.EM_SoundSpeedProfile.ProfileCounter; + fData.SS_1D_DateWhenProfileWasMade = ALLdata.EM_SoundSpeedProfile.DateWhenProfileWasMade; + fData.SS_1D_TimeSinceMidnightInMillisecondsWhenProfileWasMade = ALLdata.EM_SoundSpeedProfile.TimeSinceMidnightInMillisecondsWhenProfileWasMade; + fData.SS_1D_NumberOfEntries = ALLdata.EM_SoundSpeedProfile.NumberOfEntries; + fData.SS_1D_DepthResolution = ALLdata.EM_SoundSpeedProfile.DepthResolution; + + fData.SS_ED_Depth = nan(MaxNumberOfEntries,NumberOfDatagrams); + fData.SS_ED_SoundSpeed = nan(MaxNumberOfEntries,NumberOfDatagrams); + + for iD = 1:NumberOfDatagrams + + NumberOfEntries = ALLdata.EM_SoundSpeedProfile.NumberOfEntries(iD); + + fData.SS_ED_Depth(1:NumberOfEntries,iD) = cell2mat(ALLdata.EM_SoundSpeedProfile.Depth(iD)); + fData.SS_ED_SoundSpeed(1:NumberOfEntries,iD) = cell2mat(ALLdata.EM_SoundSpeedProfile.SoundSpeed(iD)); + + end + + end + + end + + %% EM_Attitude + + if isfield(ALLdata,'EM_Attitude') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'At_1D_Date') + + if update_mode + update_flag = 1; + end + + NumberOfDatagrams = length(ALLdata.EM_Attitude.TypeOfDatagram); + MaxNumberOfEntries = max(ALLdata.EM_Attitude.NumberOfEntries); + + fData.At_1D_Date = ALLdata.EM_Attitude.Date; + fData.At_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_Attitude.TimeSinceMidnightInMilliseconds; + fData.At_1D_AttitudeCounter = ALLdata.EM_Attitude.AttitudeCounter; + fData.At_1D_NumberOfEntries = ALLdata.EM_Attitude.NumberOfEntries; + + fData.At_ED_TimeInMillisecondsSinceRecordStart = nan(MaxNumberOfEntries, NumberOfDatagrams); + fData.At_ED_SensorStatus = nan(MaxNumberOfEntries, NumberOfDatagrams); + fData.At_ED_Roll = nan(MaxNumberOfEntries, NumberOfDatagrams); + fData.At_ED_Pitch = nan(MaxNumberOfEntries, NumberOfDatagrams); + fData.At_ED_Heave = nan(MaxNumberOfEntries, NumberOfDatagrams); + fData.At_ED_Heading = nan(MaxNumberOfEntries, NumberOfDatagrams); + + for iD = 1:NumberOfDatagrams + + NumberOfEntries = ALLdata.EM_Attitude.NumberOfEntries(iD); + + fData.At_ED_TimeInMillisecondsSinceRecordStart(1:NumberOfEntries, iD) = cell2mat(ALLdata.EM_Attitude.TimeInMillisecondsSinceRecordStart(iD)); + fData.At_ED_SensorStatus(1:NumberOfEntries, iD) = cell2mat(ALLdata.EM_Attitude.SensorStatus(iD)); + fData.At_ED_Roll(1:NumberOfEntries, iD) = cell2mat(ALLdata.EM_Attitude.Roll(iD)); + fData.At_ED_Pitch(1:NumberOfEntries, iD) = cell2mat(ALLdata.EM_Attitude.Pitch(iD)); + fData.At_ED_Heave(1:NumberOfEntries, iD) = cell2mat(ALLdata.EM_Attitude.Heave(iD)); + fData.At_ED_Heading(1:NumberOfEntries, iD) = cell2mat(ALLdata.EM_Attitude.Heading(iD)); + + end + + end + + end + + + %% EM_Height + + if isfield(ALLdata,'EM_Height') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'He_1D_Date') + + if update_mode + update_flag = 1; + end + + % NumberOfDatagrams = length(ALLdata.EM_Height.TypeOfDatagram); + + fData.He_1D_Date = ALLdata.EM_Height.Date; + fData.He_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_Height.TimeSinceMidnightInMilliseconds; + fData.He_1D_HeightCounter = ALLdata.EM_Height.HeightCounter; + fData.He_1D_Height = ALLdata.EM_Height.Height/100; + + end + + end + + %% EM_Position (v2 verified) + + if isfield(ALLdata,'EM_Position') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'Po_1D_Date') + + if update_mode + update_flag = 1; + end + + % NumberOfDatagrams = length(ALLdata.EM_Position.TypeOfDatagram); + + fData.Po_1D_Date = ALLdata.EM_Position.Date; + fData.Po_1D_TimeSinceMidnightInMilliseconds = ALLdata.EM_Position.TimeSinceMidnightInMilliseconds; % in ms + fData.Po_1D_PositionCounter = ALLdata.EM_Position.PositionCounter; + fData.Po_1D_Latitude = ALLdata.EM_Position.Latitude./20000000; % now in decimal degrees + fData.Po_1D_Longitude = ALLdata.EM_Position.Longitude./10000000; % now in decimal degrees + fData.Po_1D_SpeedOfVesselOverGround = ALLdata.EM_Position.SpeedOfVesselOverGround./100; % now in m/s + fData.Po_1D_HeadingOfVessel = ALLdata.EM_Position.HeadingOfVessel./100; % now in degrees relative to north + fData.Po_1D_MeasureOfPositionFixQuality = ALLdata.EM_Position.MeasureOfPositionFixQuality./100; % in meters + fData.Po_1D_PositionSystemDescriptor = ALLdata.EM_Position.PositionSystemDescriptor; % indicator if there are several GPS sources + + end + + end + + + %% EM_Depth + + if isfield(ALLdata,'EM_Depth') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'De_1P_Date') + + if update_mode + update_flag = 1; + end + + NumberOfPings = length(ALLdata.EM_Depth.TypeOfDatagram); % total number of pings in file + MaxNumberOfBeams = max(cellfun(@(x) max(x),ALLdata.EM_Depth.BeamNumber)); % maximum beam number in file + + fData.De_1P_Date = ALLdata.EM_Depth.Date; + fData.De_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_Depth.TimeSinceMidnightInMilliseconds; + fData.De_1P_PingCounter = ALLdata.EM_Depth.PingCounter; + fData.De_1P_HeadingOfVessel = ALLdata.EM_Depth.HeadingOfVessel; + fData.De_1P_SoundSpeedAtTransducer = ALLdata.EM_Depth.SoundSpeedAtTransducer*0.1; + fData.De_1P_TransmitTransducerDepth = ALLdata.EM_Depth.TransmitTransducerDepth + 65536.*ALLdata.EM_Depth.TransducerDepthOffsetMultiplier; + fData.De_1P_MaximumNumberOfBeamsPossible = ALLdata.EM_Depth.MaximumNumberOfBeamsPossible; + fData.De_1P_NumberOfValidBeams = ALLdata.EM_Depth.NumberOfValidBeams; + fData.De_1P_ZResolution = ALLdata.EM_Depth.ZResolution; + fData.De_1P_XAndYResolution = ALLdata.EM_Depth.XAndYResolution; + fData.De_1P_SamplingRate = ALLdata.EM_Depth.SamplingRate; + + % initialize + fData.De_BP_DepthZ = nan(MaxNumberOfBeams,NumberOfPings); + fData.De_BP_AcrosstrackDistanceY = nan(MaxNumberOfBeams,NumberOfPings); + fData.De_BP_AlongtrackDistanceX = nan(MaxNumberOfBeams,NumberOfPings); + fData.De_BP_BeamDepressionAngle = nan(MaxNumberOfBeams,NumberOfPings); + fData.De_BP_BeamAzimuthAngle = nan(MaxNumberOfBeams,NumberOfPings); + fData.De_BP_Range = nan(MaxNumberOfBeams,NumberOfPings); + fData.De_BP_QualityFactor = nan(MaxNumberOfBeams,NumberOfPings); + fData.De_BP_LengthOfDetectionWindow = nan(MaxNumberOfBeams,NumberOfPings); + fData.De_BP_ReflectivityBS = nan(MaxNumberOfBeams,NumberOfPings); + fData.De_B1_BeamNumber = (1:MaxNumberOfBeams)'; + + for iP = 1:NumberOfPings + + BeamNumber = cell2mat(ALLdata.EM_Depth.BeamNumber(iP)); + + fData.De_BP_DepthZ(BeamNumber,iP) = cell2mat(ALLdata.EM_Depth.DepthZ(iP)); + fData.De_BP_AcrosstrackDistanceY(BeamNumber,iP) = cell2mat(ALLdata.EM_Depth.AcrosstrackDistanceY(iP)); + fData.De_BP_AlongtrackDistanceX(BeamNumber,iP) = cell2mat(ALLdata.EM_Depth.AlongtrackDistanceX(iP)); + fData.De_BP_BeamDepressionAngle(BeamNumber,iP) = cell2mat(ALLdata.EM_Depth.BeamDepressionAngle(iP)); + fData.De_BP_BeamAzimuthAngle(BeamNumber,iP) = cell2mat(ALLdata.EM_Depth.BeamAzimuthAngle(iP)); + fData.De_BP_Range(BeamNumber,iP) = cell2mat(ALLdata.EM_Depth.Range(iP)); + fData.De_BP_QualityFactor(BeamNumber,iP) = cell2mat(ALLdata.EM_Depth.QualityFactor(iP)); + fData.De_BP_LengthOfDetectionWindow(BeamNumber,iP) = cell2mat(ALLdata.EM_Depth.LengthOfDetectionWindow(iP)); + fData.De_BP_ReflectivityBS(BeamNumber,iP) = cell2mat(ALLdata.EM_Depth.ReflectivityBS(iP)); + + end + + end + + end + + + %% EM_XYZ88 + + if isfield(ALLdata,'EM_XYZ88') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'X8_1P_Date') + + if update_mode + update_flag = 1; + end + + NumberOfPings = length(ALLdata.EM_XYZ88.TypeOfDatagram); % total number of pings in file + MaxNumberOfBeams = max(ALLdata.EM_XYZ88.NumberOfBeamsInDatagram); % maximum beam number in file + + fData.X8_1P_Date = ALLdata.EM_XYZ88.Date; + fData.X8_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_XYZ88.TimeSinceMidnightInMilliseconds; + fData.X8_1P_PingCounter = ALLdata.EM_XYZ88.PingCounter; + fData.X8_1P_HeadingOfVessel = ALLdata.EM_XYZ88.HeadingOfVessel; + fData.X8_1P_SoundSpeedAtTransducer = ALLdata.EM_XYZ88.SoundSpeedAtTransducer*0.1; + fData.X8_1P_TransmitTransducerDepth = ALLdata.EM_XYZ88.TransmitTransducerDepth; + fData.X8_1P_NumberOfBeamsInDatagram = ALLdata.EM_XYZ88.NumberOfBeamsInDatagram; + fData.X8_1P_NumberOfValidDetections = ALLdata.EM_XYZ88.NumberOfValidDetections; + fData.X8_1P_SamplingFrequencyInHz = ALLdata.EM_XYZ88.SamplingFrequencyInHz; + + % initialize + fData.X8_BP_DepthZ = nan(MaxNumberOfBeams,NumberOfPings); + fData.X8_BP_AcrosstrackDistanceY = nan(MaxNumberOfBeams,NumberOfPings); + fData.X8_BP_AlongtrackDistanceX = nan(MaxNumberOfBeams,NumberOfPings); + fData.X8_BP_DetectionWindowLength = nan(MaxNumberOfBeams,NumberOfPings); + fData.X8_BP_QualityFactor = nan(MaxNumberOfBeams,NumberOfPings); + fData.X8_BP_BeamIncidenceAngleAdjustment = nan(MaxNumberOfBeams,NumberOfPings); + fData.X8_BP_DetectionInformation = nan(MaxNumberOfBeams,NumberOfPings); + fData.X8_BP_RealTimeCleaningInformation = nan(MaxNumberOfBeams,NumberOfPings); + fData.X8_BP_ReflectivityBS = nan(MaxNumberOfBeams,NumberOfPings); + fData.X8_B1_BeamNumber = (1:MaxNumberOfBeams)'; + + for iP = 1:NumberOfPings + N=numel(cell2mat(ALLdata.EM_XYZ88.DepthZ(iP))); + fData.X8_BP_DepthZ(1:N,iP) = cell2mat(ALLdata.EM_XYZ88.DepthZ(iP)); + fData.X8_BP_AcrosstrackDistanceY(1:N,iP) = cell2mat(ALLdata.EM_XYZ88.AcrosstrackDistanceY(iP)); + fData.X8_BP_AlongtrackDistanceX(1:N,iP) = cell2mat(ALLdata.EM_XYZ88.AlongtrackDistanceX(iP)); + fData.X8_BP_DetectionWindowLength(1:N,iP) = cell2mat(ALLdata.EM_XYZ88.DetectionWindowLength(iP)); + fData.X8_BP_QualityFactor(1:N,iP) = cell2mat(ALLdata.EM_XYZ88.QualityFactor(iP)); + fData.X8_BP_BeamIncidenceAngleAdjustment(1:N,iP) = cell2mat(ALLdata.EM_XYZ88.BeamIncidenceAngleAdjustment(iP)); + fData.X8_BP_DetectionInformation(1:N,iP) = cell2mat(ALLdata.EM_XYZ88.DetectionInformation(iP)); + fData.X8_BP_RealTimeCleaningInformation(1:N,iP) = cell2mat(ALLdata.EM_XYZ88.RealTimeCleaningInformation(iP)); + fData.X8_BP_ReflectivityBS(1:N,iP) = cell2mat(ALLdata.EM_XYZ88.ReflectivityBS(iP)); + + end + + end + + end + + + %% EM_SeabedImage + + if isfield(ALLdata,'EM_SeabedImage') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'SI_1P_Date') + + if update_mode + update_flag = 1; + end + + NumberOfPings = length(ALLdata.EM_SeabedImage.TypeOfDatagram); % total number of pings in file + MaxNumberOfBeams = max(cellfun(@(x) max(x),ALLdata.EM_SeabedImage.BeamIndexNumber))+1; % maximum beam number (beam index number +1), in file + MaxNumberOfSamples = max(cellfun(@(x) max(x),ALLdata.EM_SeabedImage.NumberOfSamplesPerBeam)); + + fData.SI_1P_Date = ALLdata.EM_SeabedImage.Date; + fData.SI_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_SeabedImage.TimeSinceMidnightInMilliseconds; + fData.SI_1P_PingCounter = ALLdata.EM_SeabedImage.PingCounter; + fData.SI_1P_MeanAbsorptionCoefficient = ALLdata.EM_SeabedImage.MeanAbsorptionCoefficient; + fData.SI_1P_PulseLength = ALLdata.EM_SeabedImage.PulseLength; + fData.SI_1P_RangeToNormalIncidence = ALLdata.EM_SeabedImage.RangeToNormalIncidence; + fData.SI_1P_StartRangeSampleOfTVGRamp = ALLdata.EM_SeabedImage.StartRangeSampleOfTVGRamp; + fData.SI_1P_StopRangeSampleOfTVGRamp = ALLdata.EM_SeabedImage.StopRangeSampleOfTVGRamp; + fData.SI_1P_NormalIncidenceBS = ALLdata.EM_SeabedImage.NormalIncidenceBS; + fData.SI_1P_ObliqueBS = ALLdata.EM_SeabedImage.ObliqueBS; + fData.SI_1P_TxBeamwidth = ALLdata.EM_SeabedImage.TxBeamwidth; + fData.SI_1P_TVGLawCrossoverAngle = ALLdata.EM_SeabedImage.TVGLawCrossoverAngle; + fData.SI_1P_NumberOfValidBeams = ALLdata.EM_SeabedImage.NumberOfValidBeams; + + % initialize + fData.SI_BP_SortingDirection = nan(MaxNumberOfBeams,NumberOfPings); + fData.SI_BP_NumberOfSamplesPerBeam = nan(MaxNumberOfBeams,NumberOfPings); + fData.SI_BP_CentreSampleNumber = nan(MaxNumberOfBeams,NumberOfPings); + fData.SI_B1_BeamNumber = (1:MaxNumberOfBeams)'; + fData.SI_SBP_SampleAmplitudes = cell(NumberOfPings,1); % saving as sparse + + for iP = 1:NumberOfPings + + % Get data from datagram + BeamNumber = cell2mat(ALLdata.EM_SeabedImage.BeamIndexNumber(iP))+1; + NumberOfSamplesPerBeam = cell2mat(ALLdata.EM_SeabedImage.NumberOfSamplesPerBeam(iP)); + Samples = cell2mat(ALLdata.EM_SeabedImage.SampleAmplitudes(iP).beam(:)); + + % from number of samples per beam, get indices of first and last + % sample for each beam in the Samples data vector + iFirst = [1;cumsum(NumberOfSamplesPerBeam(1:end-1))+1]; + iLast = iFirst+NumberOfSamplesPerBeam-1; + + % store + fData.SI_BP_SortingDirection(BeamNumber,iP) = cell2mat(ALLdata.EM_SeabedImage.SortingDirection(iP)); + fData.SI_BP_NumberOfSamplesPerBeam(BeamNumber,iP) = NumberOfSamplesPerBeam; + fData.SI_BP_CentreSampleNumber(BeamNumber,iP) = cell2mat(ALLdata.EM_SeabedImage.CentreSampleNumber(iP)); + + % initialize the beams/sample array (use zero instead of NaN to + % allow turning it to sparse + temp = zeros(MaxNumberOfSamples,length(BeamNumber)); + + % fill in + for iB = 1:length(BeamNumber) + temp(1:NumberOfSamplesPerBeam(iB),BeamNumber(iB)) = Samples(iFirst(iB):iLast(iB)); + end + + % and save the sparse version + fData.SI_SBP_SampleAmplitudes(iP,1) = {sparse(temp)}; % to use full matrices, fData.SI_SBP_SampleAmplitudes(:,:,iP) = temp; + + end + + end + + end + + + %% EM_SeabedImage89 + + if isfield(ALLdata,'EM_SeabedImage89') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'S8_1P_Date') + + if update_mode + update_flag = 1; + end + + NumberOfPings = length(ALLdata.EM_SeabedImage89.TypeOfDatagram); % total number of pings in file + MaxNumberOfBeams = max(ALLdata.EM_SeabedImage89.NumberOfValidBeams); + MaxNumberOfSamples = max(cellfun(@(x) max(x),ALLdata.EM_SeabedImage89.NumberOfSamplesPerBeam)); + + fData.S8_1P_Date = ALLdata.EM_SeabedImage89.Date; + fData.S8_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_SeabedImage89.TimeSinceMidnightInMilliseconds; + fData.S8_1P_PingCounter = ALLdata.EM_SeabedImage89.PingCounter; + fData.S8_1P_SamplingFrequencyInHz = ALLdata.EM_SeabedImage89.SamplingFrequencyInHz; + fData.S8_1P_RangeToNormalIncidence = ALLdata.EM_SeabedImage89.RangeToNormalIncidence; + fData.S8_1P_NormalIncidenceBS = ALLdata.EM_SeabedImage89.NormalIncidenceBS; + fData.S8_1P_ObliqueBS = ALLdata.EM_SeabedImage89.ObliqueBS; + fData.S8_1P_TxBeamwidthAlong = ALLdata.EM_SeabedImage89.TxBeamwidthAlong; + fData.S8_1P_TVGLawCrossoverAngle = ALLdata.EM_SeabedImage89.TVGLawCrossoverAngle; + fData.S8_1P_NumberOfValidBeams = ALLdata.EM_SeabedImage89.NumberOfValidBeams; + + % initialize + fData.S8_BP_SortingDirection = nan(MaxNumberOfBeams,NumberOfPings); + fData.S8_BP_DetectionInfo = nan(MaxNumberOfBeams,NumberOfPings); + fData.S8_BP_NumberOfSamplesPerBeam = nan(MaxNumberOfBeams,NumberOfPings); + fData.S8_BP_CentreSampleNumber = nan(MaxNumberOfBeams,NumberOfPings); + fData.S8_B1_BeamNumber = (1:MaxNumberOfBeams)'; + fData.S8_SBP_SampleAmplitudes = cell(NumberOfPings,1); + + % in this more recent datagram, all beams are in. No beamnumber anymore + BeamNumber = fData.S8_B1_BeamNumber; + + for iP = 1:NumberOfPings + + % Get data from datagram + NumberOfSamplesPerBeam = cell2mat(ALLdata.EM_SeabedImage89.NumberOfSamplesPerBeam(iP)); + Samples = cell2mat(ALLdata.EM_SeabedImage89.SampleAmplitudes(iP).beam(:)); + + % from number of samples per beam, get indices of first and last + % sample for each beam in the Samples data vector + iFirst = [1;cumsum(NumberOfSamplesPerBeam(1:end-1))+1]; + iLast = iFirst+NumberOfSamplesPerBeam-1; + + % store + fData.S8_BP_SortingDirection(BeamNumber,iP) = cell2mat(ALLdata.EM_SeabedImage89.SortingDirection(iP)); + fData.S8_BP_NumberOfSamplesPerBeam(BeamNumber,iP) = NumberOfSamplesPerBeam; + fData.S8_BP_CentreSampleNumber(BeamNumber,iP) = cell2mat(ALLdata.EM_SeabedImage89.CentreSampleNumber(iP)); + + % initialize the beams/sample array (use zero instead of NaN to + % allow turning it to sparse + temp = zeros(MaxNumberOfSamples,length(BeamNumber)); + + % and fill in + for iB = 1:length(BeamNumber) + temp(1:NumberOfSamplesPerBeam(iB),BeamNumber(iB)) = Samples(iFirst(iB):iLast(iB)); + end + + % and save the sparse version + fData.S8_SBP_SampleAmplitudes(iP,1) = {sparse(temp)}; % to use full matrices, fData.S8_SBP_SampleAmplitudes(:,:,iP) = temp; + + end + + end + + end + + + %% EM_WaterColumn (v2 verified) + + if isfield(ALLdata,'EM_WaterColumn') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'WC_1P_Date') || fData.dr_sub~=dr_sub || fData.db_sub~=db_sub + + if update_mode + update_flag = 1; + end + + % get the number of heads + headNumber = unique(ALLdata.EM_WaterColumn.SystemSerialNumber,'stable'); + + % get the list of pings and the index of first datagram for + % each ping + if length(headNumber) == 1 + % if only one head... + [pingCounters, iFirstDatagram] = unique(ALLdata.EM_WaterColumn.PingCounter,'stable'); + else + % in case there's more than one head, we're going to only + % keep pings for which we have data for all heads + + % pings for first head + pingCounters = unique(ALLdata.EM_WaterColumn.PingCounter(ALLdata.EM_WaterColumn.SystemSerialNumber==headNumber(1)),'stable'); + + % for each other head, get ping numbers and only keep + % intersection + for iH = 2:length(headNumber) + pingCountersOtherHead = unique(ALLdata.EM_WaterColumn.PingCounter(ALLdata.EM_WaterColumn.SystemSerialNumber==headNumber(iH)),'stable'); + pingCounters = intersect(pingCounters, pingCountersOtherHead); + end + + % get the index of first datagram for each ping and each + % head + for iH = 1:length(headNumber) + + iFirstDatagram(:,iH) = arrayfun(@(x) find(ALLdata.EM_WaterColumn.SystemSerialNumber==headNumber(iH) & ALLdata.EM_WaterColumn.PingCounter==x, 1),pingCounters); + + % % originally we would require datagram number 1, but + % % it turns out it doesn't always exist. Keep this + % % code here for now, but the replacement above to + % % just find the first datagram for each ping seems to + % % work fine. + % iFirstDatagram(:,iH) = find( ALLdata.EM_WaterColumn.SystemSerialNumber == headNumber(iH) & ... + % ismember(ALLdata.EM_WaterColumn.PingCounter,pingCounters) & ... + % ALLdata.EM_WaterColumn.DatagramNumbers == 1); + end + end + + % save ping numbers + fData.WC_1P_PingCounter = pingCounters; + + % for the following fields, take value from first datagram in + % first head + fData.WC_1P_Date = ALLdata.EM_WaterColumn.Date(iFirstDatagram(:,1)); + fData.WC_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_WaterColumn.TimeSinceMidnightInMilliseconds(iFirstDatagram(:,1)); + fData.WC_1P_SoundSpeed = ALLdata.EM_WaterColumn.SoundSpeed(iFirstDatagram(:,1))*0.1; + fData.WC_1P_OriginalSamplingFrequencyHz = ALLdata.EM_WaterColumn.SamplingFrequency(iFirstDatagram(:,1)).*0.01; % in Hz + fData.WC_1P_SamplingFrequencyHz = (ALLdata.EM_WaterColumn.SamplingFrequency(iFirstDatagram(:,1)).*0.01)./dr_sub; % in Hz + fData.WC_1P_TXTimeHeave = ALLdata.EM_WaterColumn.TXTimeHeave(iFirstDatagram(:,1)); + fData.WC_1P_TVGFunctionApplied = ALLdata.EM_WaterColumn.TVGFunctionApplied(iFirstDatagram(:,1)); + fData.WC_1P_TVGOffset = ALLdata.EM_WaterColumn.TVGOffset(iFirstDatagram(:,1)); + fData.WC_1P_ScanningInfo = ALLdata.EM_WaterColumn.ScanningInfo(iFirstDatagram(:,1)); + + % test for inconsistencies between heads and raise a warning if + % one is detected + if length(headNumber) > 1 + fields = {'SoundSpeed','SamplingFrequency','TXTimeHeave','TVGFunctionApplied','TVGOffset','ScanningInfo'}; + for iFi = 1:length(fields) + if any(any(ALLdata.EM_WaterColumn.(fields{iFi})(iFirstDatagram(:,1))'.*ones(1,length(headNumber))~=ALLdata.EM_WaterColumn.(fields{iFi})(iFirstDatagram))) + warning('System has more than one head and "%s" data are inconsistent between heads for at least one ping. Using information from first head anyway.',fields{iFi}); + end + end + end + + % for the other fields, sum the numbers from heads + if length(headNumber) > 1 + fData.WC_1P_NumberOfDatagrams = sum(ALLdata.EM_WaterColumn.NumberOfDatagrams(iFirstDatagram),2)'; + fData.WC_1P_NumberOfTransmitSectors = sum(ALLdata.EM_WaterColumn.NumberOfTransmitSectors(iFirstDatagram),2)'; + fData.WC_1P_OriginalTotalNumberOfReceiveBeams = sum(ALLdata.EM_WaterColumn.TotalNumberOfReceiveBeams(iFirstDatagram),2)'; + fData.WC_1P_TotalNumberOfReceiveBeams = sum(ceil(ALLdata.EM_WaterColumn.TotalNumberOfReceiveBeams(iFirstDatagram)/db_sub),2)'; % each head is decimated in beam individually + else + fData.WC_1P_NumberOfDatagrams = ALLdata.EM_WaterColumn.NumberOfDatagrams(iFirstDatagram); + fData.WC_1P_NumberOfTransmitSectors = ALLdata.EM_WaterColumn.NumberOfTransmitSectors(iFirstDatagram); + fData.WC_1P_OriginalTotalNumberOfReceiveBeams = ALLdata.EM_WaterColumn.TotalNumberOfReceiveBeams(iFirstDatagram); + fData.WC_1P_TotalNumberOfReceiveBeams = ceil(ALLdata.EM_WaterColumn.TotalNumberOfReceiveBeams(iFirstDatagram)/db_sub); % each head is decimated in beam individually + end + + % get number of pings, maximum number of transmit sectors, + % maximum number of receive beams and maximum number of samples + % in any given ping to use as the output data dimensions + nPings = length(pingCounters); + maxNTransmitSectors = max(fData.WC_1P_NumberOfTransmitSectors); + maxNBeams = max(fData.WC_1P_OriginalTotalNumberOfReceiveBeams); + maxNBeams_sub = max(fData.WC_1P_TotalNumberOfReceiveBeams); % number of beams to extract (decimated) + maxNSamples = max(cellfun(@(x) max(x), ALLdata.EM_WaterColumn.NumberOfSamples(ismember(ALLdata.EM_WaterColumn.PingCounter,pingCounters)))); + maxNSamples_sub = ceil(maxNSamples/dr_sub); % number of samples to extract (decimated) + + %Samples=cellfun(@plus,ALLdata.EM_WaterColumn.StartRangeSampleNumber,ALLdata.EM_WaterColumn.NumberOfSamples,'un',0); + [maxNSamples_groups,ping_group_start,ping_group_end] = CFF_group_pings(ALLdata.EM_WaterColumn.NumberOfSamples,pingCounters,ALLdata.EM_WaterColumn.PingCounter); + maxNSamples_groups=maxNSamples_groups/dr_sub; + + % path to binary file for WC data + file_binary=cell(1,numel(maxNSamples_groups)); + fileID=-ones(1,numel(maxNSamples_groups)); + + for uig=1:numel(ping_group_start) + file_binary{uig} = fullfile(wc_dir,sprintf('WC_SBP_SampleAmplitudes_%.0f_%.0f.dat',ping_group_start(uig),ping_group_end(uig))); + if ~exist(file_binary{uig},'file') || fData.dr_sub~=dr_sub || fData.db_sub~=db_sub + fileID(uig) = fopen(file_binary{uig},'w+'); + % if we're not here, it means the file already exists and + % already contain the data at the proper sampling. So we + % just need to store the metadata and link to it as + % memmapfile. + end + end + + % initialize data per transmit sector and ping + fData.WC_TP_TiltAngle = nan(maxNTransmitSectors,nPings); + fData.WC_TP_CenterFrequency = nan(maxNTransmitSectors,nPings); + fData.WC_TP_TransmitSectorNumber = nan(maxNTransmitSectors,nPings); + fData.WC_TP_SystemSerialNumber = nan(maxNTransmitSectors,nPings); + + % initialize data per decimated beam and ping + fData.WC_BP_BeamPointingAngle = nan(maxNBeams_sub,nPings)/100; + fData.WC_BP_StartRangeSampleNumber = nan(maxNBeams_sub,nPings); + fData.WC_BP_NumberOfSamples = nan(maxNBeams_sub,nPings); + fData.WC_BP_DetectedRangeInSamples = zeros(maxNBeams_sub,nPings); + fData.WC_BP_TransmitSectorNumber = nan(maxNBeams_sub,nPings); + fData.WC_BP_BeamNumber = nan(maxNBeams_sub,nPings); + fData.WC_BP_SystemSerialNumber = nan(maxNBeams_sub,nPings); + + + ig=1; + % now get data for each ping + % fp=figure(); + % ax=axes(fp); + for iP = 1:nPings + + % ping number (ex: 50455) + pingCounter = fData.WC_1P_PingCounter(1,iP); + if pingCounter-fData.WC_1P_PingCounter(1,1)+1>ping_group_end(ig) + ig=ig+1; + end + % initialize the water column data matrix for that ping. + % Original data are in "int8" format, the NaN equivalent + % will be -128 + if fileID(ig) >= 0 + SB_temp = zeros(maxNSamples_groups(ig),maxNBeams_sub,'int8') - 128; + end + + % intialize number of sectors and beams recorded so far for + % that ping (needed for multiple heads) + nTxSectTot = 0; + nBeamTot = 0; + + for iH = 1:length(headNumber) + + headSSN = headNumber(iH); + + % index of the datagrams making up this ping/head in ALLdata.EM_Watercolumn (ex: 58-59-61-64) + iDatagrams = find( ALLdata.EM_WaterColumn.PingCounter == pingCounter & ... + ALLdata.EM_WaterColumn.SystemSerialNumber == headSSN); + + % actual number of datagrams available (ex: 4) + nDatagrams = length(iDatagrams); + + % some datagrams may be missing. Need to detect and adjust. + % order of the datagrams (ex: 4-3-6-2, the missing one is 1st, 5th and 7th) + datagramOrder = ALLdata.EM_WaterColumn.DatagramNumbers(iDatagrams); + [~,IX] = sort(datagramOrder); + iDatagrams = iDatagrams(IX); % index of the datagrams making up this ping in ALLdata.EM_Watercolumn, but in the right order (ex: 64-59-58-61, missing datagrams are still missing) + nBeamsPerDatagram = ALLdata.EM_WaterColumn.NumberOfBeamsInThisDatagram(iDatagrams); % number of beams in each datagram making up this ping (ex: 56-61-53-28) + + % number of transmit sectors to record + nTxSect = ALLdata.EM_WaterColumn.NumberOfTransmitSectors(iDatagrams(1)); + + % indices of those sectors in output structure + iTxSectDest = nTxSectTot + (1:nTxSect); + + % recording data per transmit sector + fData.WC_TP_TiltAngle(iTxSectDest,iP) = ALLdata.EM_WaterColumn.TiltAngle{iDatagrams(1)}; + fData.WC_TP_CenterFrequency(iTxSectDest,iP) = ALLdata.EM_WaterColumn.CenterFrequency{iDatagrams(1)}; + fData.WC_TP_TransmitSectorNumber(iTxSectDest,iP) = ALLdata.EM_WaterColumn.TransmitSectorNumber{iDatagrams(1)}; + fData.WC_TP_SystemSerialNumber(iTxSectDest,iP) = headSSN; + + % updating total number of sectors recorded so far + nTxSectTot = nTxSectTot + nTxSect; + + % and then read the data in each datagram + for iD = 1:nDatagrams + + % indices of desired beams in this head/datagram + if iD == 1 + % if first datagram, start with first beam + iBeamStart = 1; + else + % if not first datagram, continue the + % decimation where we left it + nBeamsLastDatag = nBeamsPerDatagram(iD-1); + lastRecBeam = iBeamSource(end); + iBeamStart = db_sub - (nBeamsLastDatag-lastRecBeam); + end + iBeamSource = iBeamStart:db_sub:nBeamsPerDatagram(iD); + + % number of beams to record + nBeam = length(iBeamSource); + + % indices of those beams in output structure + iBeamDest = nBeamTot + (1:nBeam); + + fData.WC_BP_BeamPointingAngle(iBeamDest,iP) = ALLdata.EM_WaterColumn.BeamPointingAngle{iDatagrams(iD)}(iBeamSource)/100; + fData.WC_BP_StartRangeSampleNumber(iBeamDest,iP) = round(ALLdata.EM_WaterColumn.StartRangeSampleNumber{iDatagrams(iD)}(iBeamSource)./dr_sub); + fData.WC_BP_NumberOfSamples(iBeamDest,iP) = round(ALLdata.EM_WaterColumn.NumberOfSamples{iDatagrams(iD)}(iBeamSource)./dr_sub); + fData.WC_BP_DetectedRangeInSamples(iBeamDest,iP) = round(ALLdata.EM_WaterColumn.DetectedRangeInSamples{iDatagrams(iD)}(iBeamSource)./dr_sub); + fData.WC_BP_TransmitSectorNumber(iBeamDest,iP) = ALLdata.EM_WaterColumn.TransmitSectorNumber2{iDatagrams(iD)}(iBeamSource); + fData.WC_BP_BeamNumber(iBeamDest,iP) = ALLdata.EM_WaterColumn.BeamNumber{iDatagrams(iD)}(iBeamSource); + fData.WC_BP_SystemSerialNumber(iBeamDest,iP) = headSSN; + + % now getting watercolumn data (beams x samples) + if fileID(ig) >= 0 + + for iB = 1:nBeam + + % actual number of samples in that beam + nSamp = ALLdata.EM_WaterColumn.NumberOfSamples{iDatagrams(iD)}(iBeamSource(iB)); + + % number of samples we're going to record + nSamp_sub = ceil(nSamp/dr_sub); + + % read the data in original file and record + % water column data are recorded in "int8 + % (-128 to 127) with -128 being the NaN + % value, and with a resolution of 0.5dB, + % aka it needs to be multiplied by a factor + % of 1/2 to retrieve the appropriate value, + % aka an int8 record of -41 is actually + % -20.5dB + pos = ALLdata.EM_WaterColumn.SampleAmplitudePosition{iDatagrams(iD)}(iBeamSource(iB)); + fseek(fid_all,pos,'bof'); + SB_temp(1:nSamp_sub,nBeamTot+iB) = fread(fid_all,nSamp_sub,'int8',dr_sub-1); + + end + + end + + % updating total number of beams recorded so far + nBeamTot = nBeamTot + nBeam; + + end + + end + + % store data on binary file + if fileID(ig) >= 0 + fwrite(fileID(ig),SB_temp,'int8'); + % imagesc(ax,SB_temp); + % drawnow; + end + + end + + fData.WC_n_start=ping_group_start; + fData.WC_n_end=ping_group_end; + fData.WC_n_maxNSamples=maxNSamples_groups; + + % close binary data file + fData.WC_SBP_SampleAmplitudes=cell(1,numel(fileID)); + for ig =1:numel(fileID) + if fileID(ig) >= 0 + fclose(fileID(ig)); + end + % and link to it through memmapfile + % remember data is in int8 format + + fData.WC_SBP_SampleAmplitudes{ig} = memmapfile(file_binary{ig},'Format',{'int8' [maxNSamples_groups(ig) maxNBeams_sub fData.WC_n_end(ig)-fData.WC_n_start(ig)+1] 'val'},'repeat',1,'writable',true); + end + + % save info about data format for later access and conversion + % to dB + fData.WC_1_SampleAmplitudes_Class = 'int8'; + fData.WC_1_SampleAmplitudes_Nanval = -128; + fData.WC_1_SampleAmplitudes_Factor = 1/2; + + end + end + + + %% EM_AmpPhase + + if isfield(ALLdata,'EM_AmpPhase') + + % only convert these datagrams if this type doesn't already exist in output + if f_reconvert || ~isfield(fData,'AP_1P_Date') || fData.dr_sub~=dr_sub || fData.db_sub~=db_sub + + if update_mode + update_flag = 1; + end + + % get indices of first datagram for each ping + [pingCounters,iFirstDatagram] = unique(ALLdata.EM_AmpPhase.PingCounter,'stable'); + + % get data dimensions + nPings = length(pingCounters); % total number of pings in file + maxNBeams = max(ALLdata.EM_AmpPhase.TotalNumberOfReceiveBeams); % maximum number of beams for a ping in file + maxNTransmitSectors = max(ALLdata.EM_AmpPhase.NumberOfTransmitSectors); % maximum number of transmit sectors for a ping in file + maxNSamples = max(cellfun(@(x) max(x),ALLdata.EM_AmpPhase.NumberOfSamples)); % max number of samples for a beam in file + + % decimating beams and samples + maxNBeams_sub = ceil(maxNBeams/db_sub); % number of beams to extract + maxNSamples_sub = ceil(maxNSamples/dr_sub); % number of samples to extract + + % read data per ping from first datagram of each ping + fData.AP_1P_Date = ALLdata.EM_AmpPhase.Date(iFirstDatagram); + fData.AP_1P_TimeSinceMidnightInMilliseconds = ALLdata.EM_AmpPhase.TimeSinceMidnightInMilliseconds(iFirstDatagram); + fData.AP_1P_PingCounter = ALLdata.EM_AmpPhase.PingCounter(iFirstDatagram); + fData.AP_1P_NumberOfDatagrams = ALLdata.EM_AmpPhase.NumberOfDatagrams(iFirstDatagram); + fData.AP_1P_NumberOfTransmitSectors = ALLdata.EM_AmpPhase.NumberOfTransmitSectors(iFirstDatagram); + fData.AP_1P_TotalNumberOfReceiveBeams = ALLdata.EM_AmpPhase.TotalNumberOfReceiveBeams(iFirstDatagram); + fData.AP_1P_SoundSpeed = ALLdata.EM_AmpPhase.SoundSpeed(iFirstDatagram)*0.1; + fData.AP_1P_SamplingFrequencyHz = (ALLdata.EM_AmpPhase.SamplingFrequency(iFirstDatagram).*0.01)./dr_sub; % in Hz + fData.AP_1P_TXTimeHeave = ALLdata.EM_AmpPhase.TXTimeHeave(iFirstDatagram); + fData.AP_1P_TVGFunctionApplied = ALLdata.EM_AmpPhase.TVGFunctionApplied(iFirstDatagram); + fData.AP_1P_TVGOffset = ALLdata.EM_AmpPhase.TVGOffset(iFirstDatagram); + fData.AP_1P_ScanningInfo = ALLdata.EM_AmpPhase.ScanningInfo(iFirstDatagram); + + % initialize data per transmit sector and ping + fData.AP_TP_TiltAngle = nan(maxNTransmitSectors,nPings); + fData.AP_TP_CenterFrequency = nan(maxNTransmitSectors,nPings); + fData.AP_TP_TransmitSectorNumber = nan(maxNTransmitSectors,nPings); + + % initialize data per decimated beam and ping + fData.AP_BP_BeamPointingAngle = nan(maxNBeams_sub,nPings); + fData.AP_BP_StartRangeSampleNumber = nan(maxNBeams_sub,nPings); + fData.AP_BP_NumberOfSamples = nan(maxNBeams_sub,nPings); + fData.AP_BP_DetectedRangeInSamples = zeros(maxNBeams_sub,nPings); + fData.AP_BP_TransmitSectorNumber = nan(maxNBeams_sub,nPings); + fData.AP_BP_BeamNumber = nan(maxNBeams_sub,nPings); + + %Samples=cellfun(@plus,ALLdata.EM_AmpPhase.StartRangeSampleNumber,ALLdata.EM_AmpPhase.NumberOfSamples,'un',0); + [maxNSamples_groups,ping_group_start,ping_group_end]=CFF_group_pings(ALLdata.EM_AmpPhase.NumberOfSamples,pingCounters,ALLdata.EM_AmpPhase.PingCounter); + + + % path to binary file for WC data + file_amp_binary=cell(1,numel(maxNSamples_groups)); + file_phase_binary=cell(1,numel(maxNSamples_groups)); + file_amp_id=-ones(1,numel(maxNSamples_groups)); + file_phase_id=-ones(1,numel(maxNSamples_groups)); + + for uig=1:numel(ping_group_start) + file_amp_binary{uig} = fullfile(wc_dir,sprintf('AP_SBP_SampleAmplitudes_%.0f_%.0f.dat',ping_group_start(uig),ping_group_end(uig))); + file_phase_binary{uig} = fullfile(wc_dir,sprintf('AP_SBP_SamplePhase_%.0f_%.0f.dat',ping_group_start(uig),ping_group_end(uig))); + + if exist(file_amp_binary{uig},'file')==0 || fData.dr_sub~=dr_sub || fData.db_sub~=db_sub + file_amp_id(uig) = fopen(file_amp_binary{uig},'w+'); + else + file_amp_id(uig) = -1; + end + + % repeat for phase file + if exist(file_phase_binary{uig},'file')==0 || fData.dr_sub~=dr_sub || fData.db_sub~=db_sub + file_phase_id(uig) = fopen(file_phase_binary{uig},'w+'); + else + file_phase_id(uig) = -1; + end + end + + + ig=1; + disp_wc=0; + if disp_wc + f = figure(); + ax_mag = axes(f,'outerposition',[0 0.5 1 0.5]); + ax_phase = axes(f,'outerposition',[0 0 1 0.5]); + end + % now get data for each ping + for iP = 1:nPings + + if iP>ping_group_end(ig) + ig=ig+1; + end + + % find datagrams composing this ping + pingCounter = fData.AP_1P_PingCounter(1,iP); % ping number (ex: 50455) + % nDatagrams = fData.AP_1P_NumberOfDatagrams(1,iP); % theoretical number of datagrams for this ping (ex: 7) + iDatagrams = find(ALLdata.EM_AmpPhase.PingCounter==pingCounter); % index of the datagrams making up this ping in ALLdata.EM_AmpPhase (ex: 58-59-61-64) + nDatagrams = length(iDatagrams); % actual number of datagrams available (ex: 4) + + % some datagrams may be missing, like in the example. Detect and adjust... + datagramOrder = ALLdata.EM_AmpPhase.DatagramNumbers(iDatagrams); % order of the datagrams (ex: 4-3-6-2, the missing one is 1st, 5th and 7th) + [~,IX] = sort(datagramOrder); + iDatagrams = iDatagrams(IX); % index of the datagrams making up this ping in ALLdata.EM_AmpPhase, but in the right order (ex: 64-59-58-61, missing datagrams are still missing) + nBeamsPerDatagram = ALLdata.EM_AmpPhase.NumberOfBeamsInThisDatagram(iDatagrams); % number of beams in each datagram making up this ping (ex: 56-61-53-28) + + % assuming transmit sectors data are not split between several datagrams, get that data from the first datagram. + nTransmitSectors = fData.AP_1P_NumberOfTransmitSectors(1,iP); % number of transmit sectors in this ping + fData.AP_TP_TiltAngle(1:nTransmitSectors,iP) = ALLdata.EM_AmpPhase.TiltAngle{iDatagrams(1)}; + fData.AP_TP_CenterFrequency(1:nTransmitSectors,iP) = ALLdata.EM_AmpPhase.CenterFrequency{iDatagrams(1)}; + fData.AP_TP_TransmitSectorNumber(1:nTransmitSectors,iP) = ALLdata.EM_AmpPhase.TransmitSectorNumber{iDatagrams(1)}; + + % initialize the water column data matrix for that ping. + + if file_amp_id(ig) >= 0 || file_phase_id(ig) >= 0 + SB2_temp = zeros(maxNSamples_groups(ig),maxNBeams_sub,'int16')-intmin('int16'); + Ph_temp = zeros(maxNSamples_groups(ig),maxNBeams_sub,'int16'); + end + + % and then read the data in each datagram + for iD = 1:nDatagrams + + % index of beams in output structure for this datagram + [iBeams,idx_beams] = unique(ceil((sum(nBeamsPerDatagram(1:iD-1)) + (1:nBeamsPerDatagram(iD)))/db_sub)); + % old approach + % iBeams = sum(nBeamsPerDatagram(1:iD-1)) + (1:nBeamsPerDatagram(iD)); + % idx_beams = (1:numel(iBeams)); + + % ping x beam data + fData.AP_BP_BeamPointingAngle(iBeams,iP) = ALLdata.EM_AmpPhase.BeamPointingAngle{iDatagrams(iD)}(idx_beams)/100; + fData.AP_BP_StartRangeSampleNumber(iBeams,iP) = round(ALLdata.EM_AmpPhase.StartRangeSampleNumber{iDatagrams(iD)}(idx_beams)./dr_sub); + fData.AP_BP_NumberOfSamples(iBeams,iP) = round(ALLdata.EM_AmpPhase.NumberOfSamples{iDatagrams(iD)}(idx_beams)./dr_sub); + fData.AP_BP_DetectedRangeInSamples(iBeams,iP) = round(ALLdata.EM_AmpPhase.DetectedRangeInSamples{iDatagrams(iD)}(idx_beams)./dr_sub); + fData.AP_BP_TransmitSectorNumber(iBeams,iP) = ALLdata.EM_AmpPhase.TransmitSectorNumber2{iDatagrams(iD)}(idx_beams); + fData.AP_BP_BeamNumber(iBeams,iP) = ALLdata.EM_AmpPhase.BeamNumber{iDatagrams(iD)}(idx_beams); + + % now getting watercolumn data (beams x samples) + if file_amp_id(ig) >= 0 || file_phase_id(ig) >= 0 + + for iB = 1:numel(iBeams) + + % actual number of samples in that beam + Ns = ALLdata.EM_AmpPhase.NumberOfSamples{iDatagrams(iD)}(idx_beams(iB)); + + % number of samples we're going to record: + Ns_sub = ceil(Ns/dr_sub); + + % get the data: + if Ns_sub > 0 + + fseek(fid_all,ALLdata.EM_AmpPhase.SamplePhaseAmplitudePosition{iDatagrams(iD)}(idx_beams(iB)),'bof'); + tmp = fread(fid_all,Ns_sub,'uint16',2); + SB2_temp((1:(Ns_sub)),iBeams(iB)) = int16(20*log10(single(tmp)*0.0001)*200); % what is this transformation? XXX2 + + fseek(fid_all,ALLdata.EM_AmpPhase.SamplePhaseAmplitudePosition{iDatagrams(iD)}(idx_beams(iB))+2,'bof'); + tmp = fread(fid_all,Ns_sub,'int16',2); + Ph_temp((1:(Ns_sub)),iBeams(iB)) = int16(-0.0001*single(tmp)*30/pi*180); % what is this transformation? XXX2 + + end + end + end + end + + % store amp data on binary file + if file_amp_id(ig) >= 0 + fwrite(file_amp_id(ig),SB2_temp,'int16'); + end + + % store phase data on binary file + if file_phase_id(ig)>=0 + fwrite(file_phase_id(ig),Ph_temp,'int16'); + end + if disp_wc + imagesc(ax_mag,single(SB2_temp)/200); + hold(ax_mag,'on');plot(ax_mag,fData.AP_BP_DetectedRangeInSamples(:,iP)); + hold(ax_mag,'off'); + caxis(ax_mag,[-100 -20]); + imagesc(ax_phase,single(Ph_temp)/30); + hold(ax_phase,'on');plot(ax_phase,fData.AP_BP_DetectedRangeInSamples(:,iP)); + hold(ax_phase,'off'); + drawnow; + end + + end + + fData.AP_SBP_SampleAmplitudes=cell(1,numel(maxNSamples_groups)); + fData.AP_SBP_SamplePhase=cell(1,numel(maxNSamples_groups)); + + fData.AP_n_start=ping_group_start; + fData.AP_n_end=ping_group_end; + fData.AP_n_maxNSamples=maxNSamples_groups; + + for ig=1:numel(maxNSamples_groups) + % close binary data file + if file_amp_id(ig) >= 0 + fclose(file_amp_id(ig)); + end + + % close binary data file + if file_phase_id(ig) >= 0 + fclose(file_phase_id(ig)); + end + + % and link to them through memmapfile + fData.AP_SBP_SampleAmplitudes{ig} = memmapfile(file_amp_binary{ig},'Format',{'int16' [maxNSamples_groups(ig) maxNBeams fData.AP_n_end(ig)-fData.AP_n_start(ig)+1] 'val'},'repeat',1,'writable',true); + fData.AP_SBP_SamplePhase{ig} = memmapfile(file_phase_binary{ig},'Format',{'int16' [maxNSamples_groups(ig) maxNBeams fData.AP_n_end(ig)-fData.AP_n_start(ig)+1] 'val'},'repeat',1,'writable',true); + end + % save info about data format for later access + fData.AP_1_SampleAmplitudes_Class = 'int16'; + fData.AP_1_SampleAmplitudes_Nanval = intmin('int16'); + fData.AP_1_SampleAmplitudes_Factor = 1/200; + fData.AP_1_SamplePhase_Class = 'int16'; + fData.AP_1_SamplePhase_Nanval = 200; + fData.AP_1_SamplePhase_Factor = 1/30; + + end + + end + + % close the original raw file + fclose(fid_all); + fData.MET_Fmt_version=CFF_get_current_fData_version(); +end diff --git a/read_data_files/Kongsberg/format_all/CFF_decode_RuntimeParameters.m b/read_data_files/Kongsberg/format_all/CFF_decode_RuntimeParameters.m new file mode 100644 index 0000000..6607809 --- /dev/null +++ b/read_data_files/Kongsberg/format_all/CFF_decode_RuntimeParameters.m @@ -0,0 +1,257 @@ +function out = CFF_decode_RuntimeParameters(EM_Runtime) +%CFF_DECODE_RUNTIMEPARAMETERS Read the encoded fields of Runtime Param. +% +% +% Info comes from KONGSBERG. (2022). Kongsberg EM Series Multibeam echo +% sounders - EM datagram formats. Document 850-160692/X (pp. 133). + +% See also CFF_READ_ALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2021-2021; Last revision: 22-11-2021 + + +% everything is dependent on EM number (model) +emNumber = unique(EM_Runtime.EMModelNumber); +if numel(emNumber)>1 + error(['Cannot deal with multiple EM model numbers in a single ' ... + 'EM_Runtime struct. In fact there is no reason why this ' ... + 'should ever happen. Investigate this.']); +end + +% The documentation does not mention the EM 712 but expecting the codes to +% be exactly the same as for EM 710. +if emNumber==712 + emNumber = 710; +end + +% init output +out = struct(); + + +%% Decoding "Operator Station status" +% to do XXX + + +%% Decoding "Processing Unit status (CPU)" +% to do XXX + + +%% Decoding "BSP status" +% to do XXX + + +%% Decoding "Sonar Head or Transceiver status" +% to do XXX + + +%% Decoding "Mode" +data = EM_Runtime.Mode; +sz = size(data); +encodedData = dec2bin(reshape(data,[],1), 8); + +% Ping mode +switch emNumber + + case 3000 + + codeTable = {... + 0, 'Nearfield (4º)';... % xxxx 0000 - Nearfield (4º) + 1, 'Normal (1.5º)';... % xxxx 0001 - Normal (1.5º) + 2, 'Target detect'... % xxxx 0010 - Target detect + }; + codes = bin2dec(encodedData(:,5:8)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.PingMode = reshape(categorical(codeTable(idx,2)),sz); + + case 3002 + + codeTable = {... + 0, 'Wide Tx beamwidth (4°)';... % xxxx 0000 - Wide Tx beamwidth (4°) + 1, 'Normal Tx beamwidth (1.5°)'... % xxxx 0001 - Normal Tx beamwidth (1.5°) + }; + + codes = bin2dec(encodedData(:,5:8)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.PingMode = reshape(categorical(codeTable(idx,2)),sz); + + case {2000, 710, 1002, 300, 302, 120, 122} + + codeTable = {... + 0, 'Very Shallow';... % xxxx 0000 - Very Shallow + 1, 'Shallow';... % xxxx 0001 - Shallow + 2, 'Medium';... % xxxx 0010 - Medium + 3, 'Deep';... % xxxx 0011 - Deep + 4, 'Very deep';... % xxxx 0100 - Very deep + 5, 'Extra deep'... % xxxx 0101 - Extra deep + }; + codes = bin2dec(encodedData(:,5:8)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.PingMode = reshape(categorical(codeTable(idx,2)),sz); + + case 2040 + + codeTable = {... + 0, '200 kHz';... % xxxx 0000 - 200 kHz + 1, '300 kHz';... % xxxx 0001 - 300 kHz + 2, '400 kHz'... % xxxx 0010 - 400 kHz + }; + codes = bin2dec(encodedData(:,5:8)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.PingMode = reshape(categorical(codeTable(idx,2)),sz); + +end + + +% TX pulse form +switch emNumber + + case {2040, 710, 302, 122} + + codeTable = {... + 0, 'CW';... % xx00 xxxx - CW + 1, 'Mixed';... % xx01 xxxx - Mixed + 2, 'FM'... % xx10 xxxx - FM + }; + codes = bin2dec(encodedData(:,3:4)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.TxPulseForm = reshape(categorical(codeTable(idx,2)),sz); + + case 2045 + + codeTable = {... + 0, 'CW';... % xx0x xxxx - CW + 1, 'FM'... % xx1x xxxx - FM + }; + codes = bin2dec(encodedData(:,3)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.TxPulseForm = reshape(categorical(codeTable(idx,2)),sz); + +end + + +% Frequency +switch emNumber + case 2045 + % Frequency (EM2040C) + % Frequency = 180 kHz + 10 kHz * parameter + % Examples: + % xxx0 0000 - 180 kHz + % xxx0 0001 - 190 kHz + % xxx1 0110 - 400 kHz + + parameter = bin2dec(encodedData(:,4:end)); + out.FrequencyKHz = 180 + 10.*parameter; + +end + + +% Dual Swath mode +switch emNumber + case {2040, 710, 302, 122} + + codeTable = {... + 0, 'Dual swath = Off';... % 00xx xxxx - Dual swath = Off + 1, 'Dual swath = Fixed';... % 01xx xxxx - Dual swath = Fixed + 2, 'Dual swath = Dynamic'... % 10xx xxxx - Dual swath = Dynamic + }; + codes = bin2dec(encodedData(:,1:2)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.DualSwathMode = reshape(categorical(codeTable(idx,2)),sz); + +end + + +%% Decoding "Filter Identifier" +% to do XXX + + +%% Decoding "Mode2" +data = EM_Runtime.ReceiverFixedGainSetting; +sz = size(data); +encodedData = dec2bin(reshape(data,[],1), 8); + +% RXarray use (EM2040) +switch emNumber + case 2040 + + codeTable = {... + 0, 'Off (RX inactive)';... % xxxx xx00 - Off (RX inactive) + 1, 'RX 1 (port) active';... % xxxx xx01 - RX 1 (port) active + 2, 'RX 2 (starboard) active';... % xxxx xx10 - RX 2 (starboard) active + 3, 'Both RX units active'... % xxxx xx11 - Both RX units active + }; + codes = bin2dec(encodedData(:,7:8)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.RXarrayUse = reshape(categorical(codeTable(idx,2)),sz); +end + +% Sonar head use (EM2040C) +switch emNumber + case 2045 + + codeTable = {... + 0, 'Off (Both inactive)';... % xxxx xx00 - Off (Both inactive) + 1, 'SH 1 (port) active';... % xxxx xx01 - SH 1 (port) active + 2, 'SH 2 (starboard) active';... % xxxx xx10 - SH 2 (starboard) active + 3, 'Both active'... % xxxx xx11 - Both active + }; + codes = bin2dec(encodedData(:,7:8)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.SonarHeadUse = reshape(categorical(codeTable(idx,2)),sz); +end + +% Pulse length +switch emNumber + + case 2040 + + codeTable = {... + 0, 'Short CW';... % xxxx 00xx - Short CW + 1, 'Medium CW';... % xxxx 01xx - Medium CW + 2, 'Long CW';... % xxxx 10xx - Long CW + 3, 'FM'... % xxxx 11xx - FM + }; + codes = bin2dec(encodedData(:,5:6)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.PulseLength = reshape(categorical(codeTable(idx,2)),sz); + + case 2045 + + codeTable = {... + 0, 'Very Short CW';... % x000 xxxx - Very Short CW + 1, 'Short CW';... % x001 xxxx - Short CW + 2, 'Medium CW';... % x010 xxxx - Medium CW + 3, 'Long CW';... % x011 xxxx - Long CW + 4, 'Very Long CW';... % x100 xxxx - Very Long CW + 5, 'Extra Long CW';... % x101 xxxx - Extra Long CW + 6, 'Short FM';... % x110 xxxx - Short FM + 7, 'Long FM'... % x111 xxxx - Long FM + }; + codes = bin2dec(encodedData(:,2:4)); + [~,idx] = ismember(codes,cell2mat(codeTable(:,1))); + out.PulseLength = reshape(categorical(codeTable(idx,2)),sz); + +end + +% Receiver fixed gain setting in dB +switch emNumber + case {2000, 1002, 3000, 3002, 300, 120} + codes = bin2dec(encodedData(:,1:8)); + out.ReceiverFixedGainSettingDB = codes; +end + + +%% Decoding "sound speed (at the transducer depth)" +% to do XXX + +%% Decoding "beamspacing" +% to do XXX + +%% Decoding "yaw and pitch stabilization" +% to do XXX + +%% Decoding "filter identifier 2" +% to do XXX + diff --git a/read_data_files/Kongsberg/format_all/CFF_decode_X8_DetectionInfo.m b/read_data_files/Kongsberg/format_all/CFF_decode_X8_DetectionInfo.m new file mode 100644 index 0000000..74c48de --- /dev/null +++ b/read_data_files/Kongsberg/format_all/CFF_decode_X8_DetectionInfo.m @@ -0,0 +1,48 @@ +function [S2, cat, ReflectivityCorrection, BScat] = CFF_decode_X8_DetectionInfo(data) + + +dat = reshape(data,[],1); + +% init output +S = nan(size(dat)); + +cat = categorical({... + 'valid, amplitude detection',... % 0 + 'valid, phase detection',... % 1 + 'invalid, normal detection',... % 2 + 'invalid, interpolated or extrapolated from neighbour detections',... % 3 + 'invalid, estimated',... % 4 + 'invalid, rejected candidate',... % 5 + 'invalid, no detection data available'... % 6 + }); + + +dat = dec2bin(dat, 8); + +bit7 = str2num(dat(:,1)); +bits0to3 = bin2dec(dat(:,5:end)); + + +S(~bit7 & bits0to3==0) = 0; +S(~bit7 & bits0to3==1) = 1; +S(bit7 & bits0to3==0) = 2; +S(bit7 & bits0to3==1) = 3; +S(bit7 & bits0to3==2) = 4; +S(bit7 & bits0to3==3) = 5; +S(bit7 & bits0to3==4) = 6; + +S2 = reshape(S,size(data)); + +% additional code for reflectivity: +% "Bit 4 Reflectivity (used in Beam intensity display) correction for +% Lamberts law and for normal incidence: 0= not compensated (xxx0 xxxx) (to +% show beam incidence angle dependency) 1= compensated (xxx1 xxxx) ( uses +% same correction as for seabed image data)" +% This refers to the "detection Infromation" field + +BScat = categorical({... + 'not compensated',... % 0 "(to show beam incidence angle dependency)" + 'compensated'... % 1 "(uses same correction as for seabed image data)" + }); + +ReflectivityCorrection = str2num(dat(:,4)); \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_all/CFF_read_all.m b/read_data_files/Kongsberg/format_all/CFF_read_all.m new file mode 100644 index 0000000..218d2c8 --- /dev/null +++ b/read_data_files/Kongsberg/format_all/CFF_read_all.m @@ -0,0 +1,324 @@ +function [ALLdata,datagrams_parsed_idx] = CFF_read_all(ALLfilename,varargin) +%CFF_READ_ALL Read all file or pair of files +% +% Reads contents of one Kongsberg EM series binary data file in .all +% format (.all or .wcd), or a pair of .all/.wcd files, allowing choice on +% which type of datagrams to parse. +% +% ALLdata = CFF_READ_ALL(ALLfilename) reads all datagrams in a Kongsberg +% file (extension .all or .wcd) ALLfilenamem, and store them in ALLdata. +% +% ALLdata = CFF_READ_ALL(ALLfilename,datagrams) reads only those +% datagrams in ALLfilename that are specified by datagrams, and store +% them in ALLdata. +% +% ALLdata = CFF_READ_ALL(ALLfilename,'datagrams',datagrams) does the +% same. +% +% Considering ALLfilename is the common root filename of a .all/.wcd pair +% (that is, ALLfilename = 'myfile' for a myfile.all and myfile.wcd pair), +% then the above commands will extract the requested datagrams from the +% .wcd file and the remaining in the .all file. +% +% Note this function will extract all datagram types of interest. For +% more control (say you only want the first ten depth datagrams and the +% last position datagram), use CFF_READ_ALL_FROM_FILEINFO. +% +% *INPUT VARIABLES* +% * |ALLfilename|: Required. String filename to parse (extension in .all +% or .wcd and existing file) OR the common root filename of a .all/.wcd +% pair. +% * |datagrams|: Optional. character string, or cell array of character +% string, or numeric values designating the types of datagrams to be +% parsed. If character string or cell array of character string, the +% string must match the datagTypeText of the datagram. If numeric, it +% must match the datagTypeNumber. The possible values are: +% datagTypeNumber = 49. datagTypeText = 'PU STATUS OUTPUT (31H)'; +% datagTypeNumber = 65. datagTypeText = 'ATTITUDE (41H)'; +% datagTypeNumber = 67. datagTypeText = 'CLOCK (43H)'; +% datagTypeNumber = 68. datagTypeText = 'DEPTH DATAGRAM (44H)'; +% datagTypeNumber = 72. datagTypeText = 'HEADING (48H)'; +% datagTypeNumber = 73. datagTypeText = 'INSTALLATION PARAMETERS - START (49H)'; +% datagTypeNumber = 78. datagTypeText = 'RAW RANGE AND ANGLE 78 (4EH)'; +% datagTypeNumber = 79. datagTypeText = 'QUALITY FACTOR DATAGRAM 79 (4FH)'; +% datagTypeNumber = 80. datagTypeText = 'POSITION (50H)'; +% datagTypeNumber = 82. datagTypeText = 'RUNTIME PARAMETERS (52H)'; +% datagTypeNumber = 83. datagTypeText = 'SEABED IMAGE DATAGRAM (53H)'; +% datagTypeNumber = 85. datagTypeText = 'SOUND SPEED PROFILE (55H)'; +% datagTypeNumber = 88. datagTypeText = 'XYZ 88 (58H)'; +% datagTypeNumber = 89. datagTypeText = 'SEABED IMAGE DATA 89 (59H)'; +% datagTypeNumber = 102. datagTypeText = 'RAW RANGE AND BEAM ANGLE (f) (66H)'; +% datagTypeNumber = 104. datagTypeText = 'DEPTH (PRESSURE) OR HEIGHT DATAGRAM (68H)'; +% datagTypeNumber = 105. datagTypeText = 'INSTALLATION PARAMETERS - STOP (69H)'; +% datagTypeNumber = 107. datagTypeText = 'WATER COLUMN DATAGRAM (6BH)'; +% datagTypeNumber = 110. datagTypeText = 'NETWORK ATTITUDE VELOCITY DATAGRAM 110 (6EH)'; +% datagTypeNumber = 114. datagTypeText = 'AMPLITUDE AND PHASE WC DATAGRAM 114 (72H)'; +% +% *OUTPUT VARIABLES* +% * |ALLdata|: structure containing the data. Each field corresponds a +% different type of datagram. The field |ALLfileinfo| is a structure +% containing information about datagrams in ALLfilename, with fields: +% * |ALLfilename|: input file name +% * |filesize|: file size in bytes +% * |datagsizeformat|: endianness of the datagram size field 'b' or 'l' +% * |datagramsformat|: endianness of the datagrams 'b' or 'l' +% * |datagNumberInFile|: number of datagram in file +% * |datagPositionInFile|: position of beginning of datagram in file +% * |datagTypeNumber|: for each datagram, SIMRAD datagram type in +% decimal +% * |datagTypeText|: for each datagram, SIMRAD datagram type +% description +% * |parsed|: 0 for each datagram at this stage. To be later turned to +% 1 for parsing +% * |counter|: the counter of this type of datagram in the file (ie +% first datagram of that type is 1 and last datagram is the total +% number of datagrams of that type) +% * |number|: the number/counter found in the datagram (usually +% different to counter) +% * |size|: for each datagram, datagram size in bytes +% * |syncCounter|: for each datagram, the number of bytes founds +% between this datagram and the previous one (any number different than +% zero indicates a sync error) +% * |emNumber|: EM Model number (eg 2045 for EM2040c) +% * |date|: datagram date in YYYMMDD +% * |timeSinceMidnightInMilliseconds|: time since midnight in msecs +% * |datagrams_parsed_idx|: array of logical values of the same dimension +% as input |datagrams| indicating which of these datagrams have been +% parsed (1) or not (0). If no datagrams were specified in input, this +% output is empty. +% +% *DEVELOPMENT NOTES* +% * Research notes for CFF_ALL_FILE_INFO.m and +% CFF_READ_ALL_FROM_FILEINFO.m apply. +% +% See also CFF_ALL_FILE_INFO, CFF_READ_ALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + + +%% Input arguments management +p = inputParser; + +% name of the .all or .wcd file (or pair) +argName = 'ALLfilename'; +argCheck = @(x) CFF_check_ALLfilename(x); +addRequired(p,argName,argCheck); + +% types of datagram to read +argName = 'datagrams'; +argDefault = []; +argCheck = @(x) isnumeric(x)||iscell(x)||(ischar(x)&&~strcmp(x,'datagrams')); % that last part allows the use of the couple name,param +addOptional(p,argName,argDefault,argCheck); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,ALLfilename,varargin{:}); + +% and get results +ALLfilename = p.Results.ALLfilename; +datagrams_to_parse = p.Results.datagrams; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end +clear p; + +% check input +if ischar(ALLfilename) + % single file .all OR .wcd. Convert filename to cell. + ALLfilename = {ALLfilename}; +else + % matching file pair .all AND .wcd. + % make sure .wcd is listed first because this function only reads in + % the 2nd file what it could not find in the first, and we want to only + % grab from the .all file what is needed and couldn't be found in the + % .wcd file. + if strcmp(CFF_file_extension(ALLfilename{1}),'.all') + ALLfilename = fliplr(ALLfilename); + end +end + + +%% Prep + +% number of files +nFiles = numel(ALLfilename); + +% start message +filename = CFF_file_name(ALLfilename{1},1); +if nFiles == 1 + comms.start(sprintf('Reading data in file %s',filename)); +else + filename_2_ext = CFF_file_extension(ALLfilename{2}); + comms.start(sprintf('Reading data in pair of files %s and %s',filename,filename_2_ext)); +end + +% start progress +comms.progress(0,nFiles); + + +%% FIRST FILE + +% Get info from first (or only) file +if nFiles == 1 + comms.step('Listing datagrams'); +else + comms.step('Listing datagrams in paired file #1/2'); +end +info = CFF_all_file_info(ALLfilename{1}); + +% communicate progress +comms.progress(0.5,nFiles); + +if isempty(datagrams_to_parse) + % parse all datagrams in first file + + info.parsed(:) = 1; + datagrams_parsed_in_first_file = unique(info.datagTypeNumber); + datagrams_parsed_idx = []; + +else + % datagrams to parse are listed in input + + if isnumeric(datagrams_to_parse) + + % datagrams available + datagrams_available = unique(info.datagTypeNumber); + + % find which datagrams can be read here + datagrams_parsable_idx = ismember(datagrams_to_parse,datagrams_available); + + % if any, read those datagrams + if any(datagrams_parsable_idx) + idx = ismember(info.datagTypeNumber,datagrams_to_parse(datagrams_parsable_idx)); + info.parsed(idx) = 1; + end + datagrams_parsed_idx = datagrams_parsable_idx; + + elseif ischar(datagrams_to_parse) || iscell(datagrams_to_parse) + % datagrams is one or several datagTypeText + + if ischar(datagrams_to_parse) + datagrams_to_parse = {datagrams_to_parse}; + end + + % datagrams available + datagrams_available = unique(info.datagTypeText); + + % find which datagrams can be read here + datagrams_parsable_idx = ismember(datagrams_to_parse,datagrams_available); + + % if any, read those datagrams + if any(datagrams_parsable_idx) + idx = ismember(info.datagTypeText,datagrams_to_parse(datagrams_parsable_idx)); + info.parsed(idx) = 1; + end + datagrams_parsed_idx = datagrams_parsable_idx; + + end + +end + +% read data +if nFiles == 1 + comms.step('Reading datagrams'); +else + comms.step('Reading datagrams in paired file #1/2'); +end +ALLdata = CFF_read_all_from_fileinfo(ALLfilename{1}, info); + +% communicate progress +comms.progress(1,nFiles); + + +%% SECOND FILE (if any) +if nFiles>1 + + % parse only if we requested to read all datagrams (in which case, the + % second file might have datagrams not read in the first and we need to + % grab those) OR if we requested a specific set of datagrams and didn't + % get them all from the first file. + if isempty(datagrams_to_parse) || ~all(datagrams_parsed_idx) + + % Get info in second file + comms.step('Listing datagrams in paired file #2/2'); + info = CFF_all_file_info(ALLfilename{2}); + + % communicate progress + comms.progress(1.5,nFiles); + + if isempty(datagrams_to_parse) + % parse all datagrams in second file which we didn't get in the + % first one. + + % datagrams in second file + datagrams_available_in_second_file = unique(info.datagTypeNumber); + + % those in second file that were not in first + datagrams_to_parse_in_second_file = setdiff(datagrams_available_in_second_file,datagrams_parsed_in_first_file); + + % parse those + idx = ismember(info.datagTypeNumber,datagrams_to_parse_in_second_file); + info.parsed(idx) = 1; + + % for output + datagrams_parsed_idx = []; + + else + % datagrams to parse are listed + + if isnumeric(datagrams_to_parse) + + datagrams_available_in_second_file = unique(info.datagTypeNumber); + + % find which remaining datagram types can be read here + datagrams_to_parse_in_second_file_idx = ismember(datagrams_to_parse,datagrams_available_in_second_file) & ~datagrams_parsed_idx; + + % if any, read those datagrams + if any(datagrams_to_parse_in_second_file_idx) + idx = ismember(info.datagTypeNumber,datagrams_to_parse(datagrams_to_parse_in_second_file_idx)); + info.parsed(idx) = 1; + end + datagrams_parsed_idx = datagrams_parsed_idx | datagrams_to_parse_in_second_file_idx; + + elseif ischar(datagrams_to_parse) || iscell(datagrams_to_parse) + % datagrams is one or several datagTypeText + + datagrams_available_in_second_file = unique(info.datagTypeText); + + % find which remaining datagram types can be read here + datagrams_to_parse_in_second_file_idx = ismember(datagrams_to_parse,datagrams_available_in_second_file) & ~datagrams_parsed_idx; + + % if any, read those datagrams + if any(datagrams_to_parse_in_second_file_idx) + idx = ismember(info.datagTypeText,datagrams_to_parse(datagrams_to_parse_in_second_file_idx)); + info.parsed(idx) = 1; + end + datagrams_parsed_idx = datagrams_parsed_idx | datagrams_to_parse_in_second_file_idx; + + end + + end + + % read data in second file + comms.step('Reading datagrams in paired file #2/2'); + ALLdata2 = CFF_read_all_from_fileinfo(ALLfilename{2}, info); + + % combine to data from first file + ALLdata = {ALLdata ALLdata2}; + + end + + % communicate progress + comms.progress(2,nFiles); + +end + + +%% end message +comms.finish('Done'); diff --git a/read_data_files/Kongsberg/format_all/CFF_read_all_from_fileinfo.m b/read_data_files/Kongsberg/format_all/CFF_read_all_from_fileinfo.m new file mode 100644 index 0000000..40e9dbe --- /dev/null +++ b/read_data_files/Kongsberg/format_all/CFF_read_all_from_fileinfo.m @@ -0,0 +1,1421 @@ +function ALLdata = CFF_read_all_from_fileinfo(ALLfilename, ALLfileinfo, varargin) +%CFF_READ_ALL_FROM_FILEINFO Read contents of all file +% +% Reads contents of one Kongsberg EM series binary data file in .all +% format (.all or .wcd), using ALLfileinfo to indicate which datagrams to +% be parsed. +% +% ALLdata = CFF_READ_ALL_FROM_FILEINFO(ALLfilename, ALLfileinfo) reads +% all datagrams in ALLfilename for which ALLfileinfo.parsed equals 1, and +% store them in ALLdata. +% +% *INPUT VARIABLES* +% * |ALLfilename|: Required. String filename to parse (extension in .all +% or .wcd). +% * |ALLfileinfo|: structure containing information about datagrams in +% ALLfilename, with fields: +% * |ALLfilename|: input file name +% * |filesize|: file size in bytes +% * |datagsizeformat|: endianness of the datagram size field 'b' or 'l' +% * |datagramsformat|: endianness of the datagrams 'b' or 'l' +% * |datagNumberInFile|: number of datagram in file +% * |datagPositionInFile|: position of beginning of datagram in file +% * |datagTypeNumber|: for each datagram, SIMRAD datagram type in +% decimal +% * |datagTypeText|: for each datagram, SIMRAD datagram type +% description +% * |parsed|: 0 for each datagram at this stage. To be later turned to +% 1 for parsing +% * |counter|: the counter of this type of datagram in the file (ie +% first datagram of that type is 1 and last datagram is the total +% number of datagrams of that type) +% * |number|: the number/counter found in the datagram (usually +% different to counter) +% * |size|: for each datagram, datagram size in bytes +% * |syncCounter|: for each datagram, the number of bytes founds +% between this datagram and the previous one (any number different than +% zero indicates a sync error) +% * |emNumber|: EM Model number (eg 2045 for EM2040c) +% * |date|: datagram date in YYYMMDD +% * |timeSinceMidnightInMilliseconds|: time since midnight in msecs +% +% *OUTPUT VARIABLES* +% * |ALLdata|: structure containing the data. Each field corresponds a +% different type of datagram. The field |ALLdata.info| contains a copy of +% ALLfileinfo described above. +% +% *DEVELOPMENT NOTES* +% * PU Status output datagram structure seems different to the datagram +% manual description. Find the good description.#edit 21aug2013: updated +% to Rev Q. Need to be checked though. +% * The parsing code for some datagrams still need to be coded. To +% update. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + + +%% Input arguments management +p = inputParser; + +% name of the .all or .wcd file +argName = 'ALLfilename'; +argCheck = @(x) CFF_check_ALLfilename(x); +addRequired(p,argName,argCheck); + +% fileinfo from CFF_ALL_FILE_INFO containing indexes of datagrams to read +argName = 'ALLfileinfo'; +argCheck = @isstruct; +addRequired(p,argName,argCheck); + +% ?? XXX3 +argName = 'OutputFields'; +argCheck = @iscell; +addParameter(p,argName,{},argCheck); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,ALLfilename,ALLfileinfo,varargin{:}); + +% and get results +ALLfilename = p.Results.ALLfilename; +ALLfileinfo = p.Results.ALLfileinfo; +OutputFields = p.Results.OutputFields; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end + + +%% Prep + +% start message +filename = CFF_file_name(ALLfilename,1); +comms.start(sprintf('Reading datagrams in file %s',filename)); + +% get basic info for file opening +filesize = ALLfileinfo.filesize; +datagsizeformat = ALLfileinfo.datagsizeformat; +datagramsformat = ALLfileinfo.datagramsformat; +ALLdata.ALLfilename=ALLfilename; +ALLdata.datagramsformat=datagramsformat; + +% open file +[fid,~] = fopen(ALLfilename, 'r',datagramsformat); + +% parse only datagrams indicated in ALLfileinfo +datagToParse = find(ALLfileinfo.parsed==1); +nDatagsToPars = numel(datagToParse); + +% start progress +comms.progress(0,nDatagsToPars); + + +%% Reading datagrams +for iDatag = datagToParse' + + % current position in file + curpos = ftell(fid); + + % position of datagram in file + datpos = ALLfileinfo.datagPositionInFile(iDatag); + + % go to datagram position + fread(fid, datpos - curpos); + + % start reading + nbDatag = fread(fid,1,'uint32',datagsizeformat); % number of bytes in datagram + stxDatag = fread(fid,1,'uint8'); % STX (always H02) + datagTypeNumber = fread(fid,1,'uint8'); % SIMRAD type of datagram + emNumber = fread(fid,1,'uint16'); % EM Model Number + date = fread(fid,1,'uint32'); % date + timeSinceMidnightInMilliseconds = fread(fid,1,'uint32'); % time since midnight in milliseconds + number = fread(fid,1,'uint16'); % datagram or ping number + systemSerialNumber = fread(fid,1,'uint16'); % EM system serial number + + % reset the parsed switch + parsed = 0; + + switch datagTypeNumber + + case 49 % 'PU STATUS OUTPUT (31H)' + if ~(isempty(OutputFields)||any(strcmp('EM_PUStatus',OutputFields))) + continue; + end + % counter for this type of datagram + try i49=i49+1; catch, i49=1; end + + % parsing + % SOMETHING WRONG WITH THIS DATAGRAM, NEW TEMPLATE? REWRITE + % USING LATEST KONGSBERG DOCUMENTATION XXX1 + % ALLdata.EM_PUStatus.STX(i49) = stxDatag; + % ALLdata.EM_PUStatus.TypeOfDatagram(i49) = datagTypeNumber; + % ALLdata.EM_PUStatus.EMModelNumber(i49) = emNumber; + % ALLdata.EM_PUStatus.Date(i49) = date; + % ALLdata.EM_PUStatus.TimeSinceMidnightInMilliseconds(i49) = timeSinceMidnightInMilliseconds; + % ALLdata.EM_PUStatus.StatusDatagramCounter(i49) = number; + % ALLdata.EM_PUStatus.SystemSerialNumber(i49) = systemSerialNumber; + % + % ALLdata.EM_PUStatus.PingRate(i49) = fread(fid,1,'uint16'); + % ALLdata.EM_PUStatus.PingCounterOfLatestPing(i49) = fread(fid,1,'uint16'); + % ALLdata.EM_PUStatus.DistanceBetweenSwath(i49) = fread(fid,1,'uint8'); + % ALLdata.EM_PUStatus.SensorInputStatusUDPPort2(i49) = fread(fid,1,'uint32'); + % ALLdata.EM_PUStatus.SensorInputStatusSerialPort1(i49) = fread(fid,1,'uint32'); + % ALLdata.EM_PUStatus.SensorInputStatusSerialPort2(i49) = fread(fid,1,'uint32'); + % ALLdata.EM_PUStatus.SensorInputStatusSerialPort3(i49) = fread(fid,1,'uint32'); + % ALLdata.EM_PUStatus.SensorInputStatusSerialPort4(i49) = fread(fid,1,'uint32'); + % ALLdata.EM_PUStatus.PPSStatus(i49) = fread(fid,1,'int8'); + % ALLdata.EM_PUStatus.PositionStatus(i49) = fread(fid,1,'int8'); + % ALLdata.EM_PUStatus.AttitudeStatus(i49) = fread(fid,1,'int8'); + % ALLdata.EM_PUStatus.ClockStatus(i49) = fread(fid,1,'int8'); + % ALLdata.EM_PUStatus.HeadingStatus (i49) = fread(fid,1,'int8'); + % ALLdata.EM_PUStatus.PUStatus(i49) = fread(fid,1,'uint8'); + % ALLdata.EM_PUStatus.LastReceivedHeading(i49) = fread(fid,1,'uint16'); + % ALLdata.EM_PUStatus.LastReceivedRoll(i49) = fread(fid,1,'int16'); + % ALLdata.EM_PUStatus.LastReceivedPitch(i49) = fread(fid,1,'int16'); + % ALLdata.EM_PUStatus.LastReceivedHeave(i49) = fread(fid,1,'int16'); + % ALLdata.EM_PUStatus.SoundSpeedAtTransducer(i49) = fread(fid,1,'uint16'); + % ALLdata.EM_PUStatus.LastReceivedDepth(i49) = fread(fid,1,'uint32'); + % ALLdata.EM_PUStatus.AlongShipVelocity(i49) = fread(fid,1,'int16'); + % ALLdata.EM_PUStatus.AttitudeVelocitySensor(i49) = fread(fid,1,'uint8'); + % ALLdata.EM_PUStatus.MammalProtectionRamp(i49) = fread(fid,1,'uint8'); + % ALLdata.EM_PUStatus.BackscatterAtObliqueAngle(i49) = fread(fid,1,'int8'); + % ALLdata.EM_PUStatus.BackscatterAtNormalIncidence(i49) = fread(fid,1,'int8'); + % ALLdata.EM_PUStatus.FixedGain(i49) = fread(fid,1,'int8'); + % ALLdata.EM_PUStatus.DepthToNormalIncidence(i49) = fread(fid,1,'uint8'); + % ALLdata.EM_PUStatus.RangeToNormalIncidence(i49) = fread(fid,1,'uint16'); + % ALLdata.EM_PUStatus.PortCoverage(i49) = fread(fid,1,'uint8'); + % ALLdata.EM_PUStatus.StarboardCoverage(i49) = fread(fid,1,'uint8'); + % ALLdata.EM_PUStatus.SoundSpeedAtTransducerFoundFromProfile(i49) = fread(fid,1,'uint16'); + % ALLdata.EM_PUStatus.YawStabilization(i49) = fread(fid,1,'int16'); + % ALLdata.EM_PUStatus.PortCoverageOrAcrossShipVelocity(i49) = fread(fid,1,'int16'); + % ALLdata.EM_PUStatus.StarboardCoverageOrDownwardVelocity(i49) = fread(fid,1,'int16'); + % ALLdata.EM_PUStatus.EM2040CPUtemp(i49) = fread(fid,1,'int8'); + % ALLdata.EM_PUStatus.ETX(i49) = fread(fid,1,'uint8'); + % ALLdata.EM_PUStatus.CheckSum(i49) = fread(fid,1,'uint16'); + % + % % ETX check + % if ALLdata.EM_PUStatus.ETX(i49)~=3 + % error('wrong ETX value (ALLdata.EM_PUStatus)'); + % end + % + % % confirm parsing + % parsed = 1; + + case 65 % 'ATTITUDE (41H)' + if ~(isempty(OutputFields)||any(strcmp('EM_Attitude',OutputFields))) + continue; + end + % counter for this type of datagram + try i65=i65+1; catch, i65=1; end + + % parsing + ALLdata.EM_Attitude.NumberOfBytesInDatagram(i65) = nbDatag; + ALLdata.EM_Attitude.STX(i65) = stxDatag; + ALLdata.EM_Attitude.TypeOfDatagram(i65) = datagTypeNumber; + ALLdata.EM_Attitude.EMModelNumber(i65) = emNumber; + ALLdata.EM_Attitude.Date(i65) = date; + ALLdata.EM_Attitude.TimeSinceMidnightInMilliseconds(i65) = timeSinceMidnightInMilliseconds; + ALLdata.EM_Attitude.AttitudeCounter(i65) = number; + ALLdata.EM_Attitude.SystemSerialNumber(i65) = systemSerialNumber; + + ALLdata.EM_Attitude.NumberOfEntries(i65) = fread(fid,1,'uint16'); %N + + % repeat cycle: N entries of 12 bits + temp = ftell(fid); + N = ALLdata.EM_Attitude.NumberOfEntries(i65) ; + ALLdata.EM_Attitude.TimeInMillisecondsSinceRecordStart{i65} = fread(fid,N,'uint16',12-2); + fseek(fid,temp+2,'bof'); % to next data type + ALLdata.EM_Attitude.SensorStatus{i65} = fread(fid,N,'uint16',12-2); + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_Attitude.Roll{i65} = fread(fid,N,'int16',12-2); + fseek(fid,temp+6,'bof'); % to next data type + ALLdata.EM_Attitude.Pitch{i65} = fread(fid,N,'int16',12-2); + fseek(fid,temp+8,'bof'); % to next data type + ALLdata.EM_Attitude.Heave{i65} = fread(fid,N,'int16',12-2); + fseek(fid,temp+10,'bof'); % to next data type + ALLdata.EM_Attitude.Heading{i65} = fread(fid,N,'uint16',12-2); + fseek(fid,2-12,'cof'); % we need to come back after last jump + + ALLdata.EM_Attitude.SensorSystemDescriptor(i65) = fread(fid,1,'uint8'); + ALLdata.EM_Attitude.ETX(i65) = fread(fid,1,'uint8'); + ALLdata.EM_Attitude.CheckSum(i65) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_Attitude.ETX(i65)~=3 + error('wrong ETX value (ALLdata.EM_Attitude)'); + end + + % confirm parsing + parsed = 1; + + case 67 % 'CLOCK (43H)' + if ~(isempty(OutputFields)||any(strcmp('EM_Clock',OutputFields))) + continue; + end + % counter for this type of datagram + try i67=i67+1; catch, i67=1; end + + % parsing + ALLdata.EM_Clock.NumberOfBytesInDatagram(i67) = nbDatag; + ALLdata.EM_Clock.STX(i67) = stxDatag; + ALLdata.EM_Clock.TypeOfDatagram(i67) = datagTypeNumber; + ALLdata.EM_Clock.EMModelNumber(i67) = emNumber; + ALLdata.EM_Clock.Date(i67) = date; + ALLdata.EM_Clock.TimeSinceMidnightInMilliseconds(i67) = timeSinceMidnightInMilliseconds; + ALLdata.EM_Clock.ClockCounter(i67) = number; + ALLdata.EM_Clock.SystemSerialNumber(i67) = systemSerialNumber; + + ALLdata.EM_Clock.DateFromExternalClock(i67) = fread(fid,1,'uint32'); + ALLdata.EM_Clock.TimeSinceMidnightInMillisecondsFromExternalClock(i67) = fread(fid,1,'uint32'); + ALLdata.EM_Clock.OnePPSUse(i67) = fread(fid,1,'uint8'); + ALLdata.EM_Clock.ETX(i67) = fread(fid,1,'uint8'); + ALLdata.EM_Clock.CheckSum(i67) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_Clock.ETX(i67)~=3 + error('wrong ETX value (ALLdata.EM_Clock)'); + end + + % confirm parsing + parsed = 1; + + case 68 % 'DEPTH DATAGRAM (44H)' + if ~(isempty(OutputFields)||any(strcmp('EM_Depth',OutputFields))) + continue; + end + % counter for this type of datagram + try i68=i68+1; catch, i68=1; end + + % parsing + ALLdata.EM_Depth.NumberOfBytesInDatagram(i68) = nbDatag; + ALLdata.EM_Depth.STX(i68) = stxDatag; + ALLdata.EM_Depth.TypeOfDatagram(i68) = datagTypeNumber; + ALLdata.EM_Depth.EMModelNumber(i68) = emNumber; + ALLdata.EM_Depth.Date(i68) = date; + ALLdata.EM_Depth.TimeSinceMidnightInMilliseconds(i68) = timeSinceMidnightInMilliseconds; + ALLdata.EM_Depth.PingCounter(i68) = number; + ALLdata.EM_Depth.SystemSerialNumber(i68) = systemSerialNumber; + + ALLdata.EM_Depth.HeadingOfVessel(i68) = fread(fid,1,'uint16'); + ALLdata.EM_Depth.SoundSpeedAtTransducer(i68) = fread(fid,1,'uint16'); + ALLdata.EM_Depth.TransmitTransducerDepth(i68) = fread(fid,1,'uint16'); + ALLdata.EM_Depth.MaximumNumberOfBeamsPossible(i68) = fread(fid,1,'uint8'); + ALLdata.EM_Depth.NumberOfValidBeams(i68) = fread(fid,1,'uint8'); %N + ALLdata.EM_Depth.ZResolution(i68) = fread(fid,1,'uint8'); + ALLdata.EM_Depth.XAndYResolution(i68) = fread(fid,1,'uint8'); + ALLdata.EM_Depth.SamplingRate(i68) = fread(fid,1,'uint16'); % OR: ALLdata.EM_Depth.DepthDifferenceBetweenSonarHeadsInTheEM3000D(i68) = fread(fid,1,'int16'); + + % repeat cycle: N entries of 16 bits + temp = ftell(fid); + N = ALLdata.EM_Depth.NumberOfValidBeams(i68); + ALLdata.EM_Depth.DepthZ{i68} = fread(fid,N,'int16',16-2); % OR 'uint16' for EM120 and EM300 + fseek(fid,temp+2,'bof'); % to next data type + ALLdata.EM_Depth.AcrosstrackDistanceY{i68} = fread(fid,N,'int16',16-2); + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_Depth.AlongtrackDistanceX{i68} = fread(fid,N,'int16',16-2); + fseek(fid,temp+6,'bof'); % to next data type + ALLdata.EM_Depth.BeamDepressionAngle{i68} = fread(fid,N,'int16',16-2); + fseek(fid,temp+8,'bof'); % to next data type + ALLdata.EM_Depth.BeamAzimuthAngle{i68} = fread(fid,N,'uint16',16-2); + fseek(fid,temp+10,'bof'); % to next data type + ALLdata.EM_Depth.Range{i68} = fread(fid,N,'uint16',16-2); + fseek(fid,temp+12,'bof'); % to next data type + ALLdata.EM_Depth.QualityFactor{i68} = fread(fid,N,'uint8',16-1); + fseek(fid,temp+13,'bof'); % to next data type + ALLdata.EM_Depth.LengthOfDetectionWindow{i68} = fread(fid,N,'uint8',16-1); + fseek(fid,temp+14,'bof'); % to next data type + ALLdata.EM_Depth.ReflectivityBS{i68} = fread(fid,N,'int8',16-1); + fseek(fid,temp+15,'bof'); % to next data type + ALLdata.EM_Depth.BeamNumber{i68} = fread(fid,N,'uint8',16-1); + fseek(fid,1-16,'cof'); % we need to come back after last jump + + ALLdata.EM_Depth.TransducerDepthOffsetMultiplier(i68) = fread(fid,1,'int8'); + ALLdata.EM_Depth.ETX(i68) = fread(fid,1,'uint8'); + ALLdata.EM_Depth.CheckSum(i68) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_Depth.ETX(i68)~=3 + error('wrong ETX value (ALLdata.EM_Depth)'); + end + + % confirm parsing + parsed = 1; + + case 70 % 'RAW RANGE AND BEAM ANGLE (F) (46H)' + if ~(isempty(OutputFields)||any(strcmp('EM_RawRangeBeamAngle',OutputFields))) + continue; + end + % counter for this type of datagram + try i70=i70+1; catch, i70=1; end + + % parsing + % ...to write... + + case 71 % 'SURFACE SOUND SPEED (47H)' + if ~(isempty(OutputFields)||any(strcmp('EM_SurfaceSoundSpeed',OutputFields))) + continue; + end + % counter for this type of datagram + try i71=i71+1; catch, i71=1; end + + % parsing + ALLdata.EM_SurfaceSoundSpeed.NumberOfBytesInDatagram(i71) = nbDatag; + ALLdata.EM_SurfaceSoundSpeed.STX(i71) = stxDatag; + ALLdata.EM_SurfaceSoundSpeed.TypeOfDatagram(i71) = datagTypeNumber; + ALLdata.EM_SurfaceSoundSpeed.EMModelNumber(i71) = emNumber; + ALLdata.EM_SurfaceSoundSpeed.Date(i71) = date; + ALLdata.EM_SurfaceSoundSpeed.TimeSinceMidnightInMilliseconds(i71) = timeSinceMidnightInMilliseconds; + ALLdata.EM_SurfaceSoundSpeed.SoundSpeedCounter(i71) = number; + ALLdata.EM_SurfaceSoundSpeed.SystemSerialNumber(i71) = systemSerialNumber; + + ALLdata.EM_SurfaceSoundSpeed.NumberOfEntries(i71) = fread(fid,1,'uint16'); %N + + % repeat cycle: N entries of 4 bits + temp = ftell(fid); + N = ALLdata.EM_SurfaceSoundSpeed.NumberOfEntries(i71); + ALLdata.EM_SurfaceSoundSpeed.TimeInSecondsSinceRecordStart{i71} = fread(fid,N,'uint16',4-2); + fseek(fid,temp+2,'bof'); % to next data type + ALLdata.EM_SurfaceSoundSpeed.SoundSpeed{i71} = fread(fid,N,'uint16',4-2); + fseek(fid,2-4,'cof'); % we need to come back after last jump + + ALLdata.EM_SurfaceSoundSpeed.Spare(i71) = fread(fid,1,'uint8'); + ALLdata.EM_SurfaceSoundSpeed.ETX(i71) = fread(fid,1,'uint8'); + ALLdata.EM_SurfaceSoundSpeed.CheckSum(i71) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_SurfaceSoundSpeed.ETX(i71)~=3 + error('wrong ETX value (ALLdata.EM_SurfaceSoundSpeed)'); + end + + % confirm parsing + parsed = 1; + + case 72 % 'HEADING (48H)' + if ~(isempty(OutputFields)||any(strcmp('EM_Heading',OutputFields))) + continue; + end + % counter for this type of datagram + try i72=i72+1; catch, i72=1; end + + % parsing + % ...to write... + + case 73 % 'INSTALLATION PARAMETERS - START (49H)' + if ~(isempty(OutputFields)||any(strcmp('EM_InstallationStart',OutputFields))) + continue; + end + % counter for this type of datagram + try i73=i73+1; catch, i73=1; end + + % parsing + ALLdata.EM_InstallationStart.NumberOfBytesInDatagram(i73) = nbDatag; + ALLdata.EM_InstallationStart.STX(i73) = stxDatag; + ALLdata.EM_InstallationStart.TypeOfDatagram(i73) = datagTypeNumber; + ALLdata.EM_InstallationStart.EMModelNumber(i73) = emNumber; + ALLdata.EM_InstallationStart.Date(i73) = date; + ALLdata.EM_InstallationStart.TimeSinceMidnightInMilliseconds(i73) = timeSinceMidnightInMilliseconds; + ALLdata.EM_InstallationStart.SurveyLineNumber(i73) = number; + ALLdata.EM_InstallationStart.SystemSerialNumber(i73) = systemSerialNumber; + + ALLdata.EM_InstallationStart.SerialNumberOfSecondSonarHead(i73) = fread(fid,1,'uint16'); + + % 18 bytes of binary data already recorded and 3 more to come = 21. + % but nbDatag will always be even thanks to SpareByte. so + % nbDatag is 22 if there is no ASCII data and more if there is + % ASCII data. read the rest as ASCII (including SpareByte) with + % 1 byte for 1 character. + ALLdata.EM_InstallationStart.ASCIIData{i73} = fscanf(fid, '%c', nbDatag-21); + + ALLdata.EM_InstallationStart.ETX(i73) = fread(fid,1,'uint8'); + ALLdata.EM_InstallationStart.CheckSum(i73) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_InstallationStart.ETX(i73)~=3 + error('wrong ETX value (ALLdata.EM_InstallationStart)'); + end + + % confirm parsing + parsed = 1; + + case 78 % 'RAW RANGE AND ANGLE 78 (4EH)' + if ~(isempty(OutputFields)||any(strcmp('EM_RawRangeAngle78',OutputFields))) + continue; + end + % counter for this type of datagram + try i78=i78+1; catch, i78=1; end + + % parsing + ALLdata.EM_RawRangeAngle78.NumberOfBytesInDatagram(i78) = nbDatag; + ALLdata.EM_RawRangeAngle78.STX(i78) = stxDatag; + ALLdata.EM_RawRangeAngle78.TypeOfDatagram(i78) = datagTypeNumber; + ALLdata.EM_RawRangeAngle78.EMModelNumber(i78) = emNumber; + ALLdata.EM_RawRangeAngle78.Date(i78) = date; + ALLdata.EM_RawRangeAngle78.TimeSinceMidnightInMilliseconds(i78) = timeSinceMidnightInMilliseconds; + ALLdata.EM_RawRangeAngle78.PingCounter(i78) = number; + ALLdata.EM_RawRangeAngle78.SystemSerialNumber(i78) = systemSerialNumber; + + ALLdata.EM_RawRangeAngle78.SoundSpeedAtTransducer(i78) = fread(fid,1,'uint16'); + ALLdata.EM_RawRangeAngle78.NumberOfTransmitSectors(i78) = fread(fid,1,'uint16'); %Ntx + ALLdata.EM_RawRangeAngle78.NumberOfReceiverBeamsInDatagram(i78) = fread(fid,1,'uint16'); %Nrx + ALLdata.EM_RawRangeAngle78.NumberOfValidDetections(i78) = fread(fid,1,'uint16'); + ALLdata.EM_RawRangeAngle78.SamplingFrequencyInHz(i78) = fread(fid,1,'float32'); + ALLdata.EM_RawRangeAngle78.Dscale(i78) = fread(fid,1,'uint32'); + + % repeat cycle #1: Ntx entries of 24 bits + temp = ftell(fid); + C = 24; + Ntx = ALLdata.EM_RawRangeAngle78.NumberOfTransmitSectors(i78); + ALLdata.EM_RawRangeAngle78.TiltAngle{i78} = fread(fid,Ntx,'int16',C-2); + fseek(fid,temp+2,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.FocusRange{i78} = fread(fid,Ntx,'uint16',C-2); + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.SignalLength{i78} = fread(fid,Ntx,'float32',C-4); + fseek(fid,temp+8,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.SectorTransmitDelay{i78} = fread(fid,Ntx,'float32',C-4); + fseek(fid,temp+12,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.CentreFrequency{i78} = fread(fid,Ntx,'float32',C-4); + fseek(fid,temp+16,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.MeanAbsorptionCoeff{i78} = fread(fid,Ntx,'uint16',C-2); + fseek(fid,temp+18,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.SignalWaveformIdentifier{i78} = fread(fid,Ntx,'uint8',C-1); + fseek(fid,temp+19,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.TransmitSectorNumberTxArrayIndex{i78} = fread(fid,Ntx,'uint8',C-1); + fseek(fid,temp+20,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.SignalBandwidth{i78} = fread(fid,Ntx,'float32',C-4); + fseek(fid,4-C,'cof'); % we need to come back after last jump + + % repeat cycle #2: Nrx entries of 16 bits + temp = ftell(fid); + C = 16; + Nrx = ALLdata.EM_RawRangeAngle78.NumberOfReceiverBeamsInDatagram(i78); + ALLdata.EM_RawRangeAngle78.BeamPointingAngle{i78} = fread(fid,Nrx,'int16',C-2); + fseek(fid,temp+2,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.TransmitSectorNumber{i78} = fread(fid,Nrx,'uint8',C-1); + fseek(fid,temp+3,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.DetectionInfo{i78} = fread(fid,Nrx,'uint8',C-1); + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.DetectionWindowLength{i78} = fread(fid,Nrx,'uint16',C-2); + fseek(fid,temp+6,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.QualityFactor{i78} = fread(fid,Nrx,'uint8',C-1); + fseek(fid,temp+7,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.Dcorr{i78} = fread(fid,Nrx,'int8',C-1); + fseek(fid,temp+8,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.TwoWayTravelTime{i78} = fread(fid,Nrx,'float32',C-4); + fseek(fid,temp+12,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.ReflectivityBS{i78} = fread(fid,Nrx,'int16',C-2); + fseek(fid,temp+14,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.RealTimeCleaningInfo{i78} = fread(fid,Nrx,'int8',C-1); + fseek(fid,temp+15,'bof'); % to next data type + ALLdata.EM_RawRangeAngle78.Spare{i78} = fread(fid,Nrx,'uint8',C-1); + fseek(fid,1-C,'cof'); % we need to come back after last jump + + ALLdata.EM_RawRangeAngle78.Spare2(i78) = fread(fid,1,'uint8'); + ALLdata.EM_RawRangeAngle78.ETX(i78) = fread(fid,1,'uint8'); + ALLdata.EM_RawRangeAngle78.CheckSum(i78) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_RawRangeAngle78.ETX(i78)~=3 + error('wrong ETX value (ALLdata.EM_RawRangeAngle78)'); + end + + % confirm parsing + parsed = 1; + + case 79 % 'QUALITY FACTOR DATAGRAM 79 (4FH)' + if ~(isempty(OutputFields)||any(strcmp('EM_QF',OutputFields))) + continue; + end + % counter for this type of datagram + try i79=i79+1; catch, i79=1; end + + % parsing + ALLdata.EM_QualityFactor79.NumberOfBytesInDatagram(i79) = nbDatag; + ALLdata.EM_QualityFactor79.STX(i79) = stxDatag; + ALLdata.EM_QualityFactor79.TypeOfDatagram(i79) = datagTypeNumber; + ALLdata.EM_QualityFactor79.EMModelNumber(i79) = emNumber; + ALLdata.EM_QualityFactor79.Date(i79) = date; + ALLdata.EM_QualityFactor79.TimeSinceMidnightInMilliseconds(i79) = timeSinceMidnightInMilliseconds; + ALLdata.EM_QualityFactor79.PingCounter(i79) = number; + ALLdata.EM_QualityFactor79.SystemSerialNumber(i79) = systemSerialNumber; + + ALLdata.EM_QualityFactor79.NumberOfReceiveBeams(i79) = fread(fid,1,'uint16'); %Nrx + ALLdata.EM_QualityFactor79.NumberOfParametersPerBeam(i79) = fread(fid,1,'uint8'); %Npar + ALLdata.EM_QualityFactor79.Spare(i79) = fread(fid,1,'uint8'); + + % Repeate cycle. Nrx entries of: 4*Npar bits + % DEV NOTE: This code was written July '21 based on Rev W of + % the .all datagrams documentation, which lists the IFREMER QF + % as the ONLY parameter in this loop, but the format makes + % provision for additional parameters using "Npar". The code + % written below SHOULD work whatever the number of parameters + % but will only read the IMREMER QF. To read additional + % parameters, you'll need to modify the code to make it like + % other loops including fseek between data types and fseek + % after the final jump. + Nrx = ALLdata.EM_QualityFactor79.NumberOfReceiveBeams(i79); + Npar = ALLdata.EM_QualityFactor79.NumberOfParametersPerBeam(i79); + C = 4*Npar; + ALLdata.EM_QualityFactor79.IFREMERqualityFactor{i79} = fread(fid,Nrx,'float32',C-4); + + ALLdata.EM_QualityFactor79.Spare2(i79) = fread(fid,1,'uint8'); + ALLdata.EM_QualityFactor79.ETX(i79) = fread(fid,1,'uint8'); + ALLdata.EM_QualityFactor79.CheckSum(i79) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_QualityFactor79.ETX(i79)~=3 + error('wrong ETX value (ALLdata.EM_QualityFactor79)'); + end + + % confirm parsing + parsed = 1; + + case 80 % 'POSITION (50H)' + if ~(isempty(OutputFields)||any(strcmp('EM_Position',OutputFields))) + continue; + end + % counter for this type of datagram + try i80=i80+1; catch, i80=1; end + + % parsing + ALLdata.EM_Position.NumberOfBytesInDatagram(i80) = nbDatag; + ALLdata.EM_Position.STX(i80) = stxDatag; + ALLdata.EM_Position.TypeOfDatagram(i80) = datagTypeNumber; + ALLdata.EM_Position.EMModelNumber(i80) = emNumber; + ALLdata.EM_Position.Date(i80) = date; + ALLdata.EM_Position.TimeSinceMidnightInMilliseconds(i80) = timeSinceMidnightInMilliseconds; + ALLdata.EM_Position.PositionCounter(i80) = number; + ALLdata.EM_Position.SystemSerialNumber(i80) = systemSerialNumber; + + ALLdata.EM_Position.Latitude(i80) = fread(fid,1,'int32'); + ALLdata.EM_Position.Longitude(i80) = fread(fid,1,'int32'); + ALLdata.EM_Position.MeasureOfPositionFixQuality(i80) = fread(fid,1,'uint16'); + ALLdata.EM_Position.SpeedOfVesselOverGround(i80) = fread(fid,1,'uint16'); + ALLdata.EM_Position.CourseOfVesselOverGround(i80) = fread(fid,1,'uint16'); + ALLdata.EM_Position.HeadingOfVessel(i80) = fread(fid,1,'uint16'); + ALLdata.EM_Position.PositionSystemDescriptor(i80) = fread(fid,1,'uint8'); + ALLdata.EM_Position.NumberOfBytesInInputDatagram(i80) = fread(fid,1,'uint8'); + + % next data size is variable. 34 bits of binary data already + % recorded and 3 more to come = 37. read the rest as ASCII + % (including SpareByte) + ALLdata.EM_Position.PositionInputDatagramAsReceived{i80} = fscanf(fid, '%c', nbDatag-37); + + ALLdata.EM_Position.ETX(i80) = fread(fid,1,'uint8'); + ALLdata.EM_Position.CheckSum(i80) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_Position.ETX(i80)~=3 + error('wrong ETX value (ALLdata.EM_Position)'); + end + + % confirm parsing + parsed = 1; + + case 82 % 'RUNTIME PARAMETERS (52H)' + if ~(isempty(OutputFields)||any(strcmp('EM_Runtime',OutputFields))) + continue; + end + % counter for this type of datagram + try i82=i82+1; catch, i82=1; end + + % parsing + ALLdata.EM_Runtime.NumberOfBytesInDatagram(i82) = nbDatag; + ALLdata.EM_Runtime.STX(i82) = stxDatag; + ALLdata.EM_Runtime.TypeOfDatagram(i82) = datagTypeNumber; + ALLdata.EM_Runtime.EMModelNumber(i82) = emNumber; + ALLdata.EM_Runtime.Date(i82) = date; + ALLdata.EM_Runtime.TimeSinceMidnightInMilliseconds(i82) = timeSinceMidnightInMilliseconds; + ALLdata.EM_Runtime.PingCounter(i82) = number; + ALLdata.EM_Runtime.SystemSerialNumber(i82) = systemSerialNumber; + + ALLdata.EM_Runtime.OperatorStationStatus(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.ProcessingUnitStatus(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.BSPStatus(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.SonarHeadStatus(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.Mode(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.FilterIdentifier(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.MinimumDepth(i82) = fread(fid,1,'uint16'); + ALLdata.EM_Runtime.MaximumDepth(i82) = fread(fid,1,'uint16'); + ALLdata.EM_Runtime.AbsorptionCoefficient(i82) = fread(fid,1,'uint16'); + ALLdata.EM_Runtime.TransmitPulseLength(i82) = fread(fid,1,'uint16'); + ALLdata.EM_Runtime.TransmitBeamwidth(i82) = fread(fid,1,'uint16'); + ALLdata.EM_Runtime.TransmitPowerReMaximum(i82) = fread(fid,1,'int8'); + ALLdata.EM_Runtime.ReceiveBeamwidth(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.ReceiveBandwidth(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.ReceiverFixedGainSetting(i82) = fread(fid,1,'uint8'); % OR mode 2 + ALLdata.EM_Runtime.TVGLawCrossoverAngle(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.SourceOfSoundSpeedAtTransducer(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.MaximumPortSwathWidth(i82) = fread(fid,1,'uint16'); + ALLdata.EM_Runtime.BeamSpacing(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.MaximumPortCoverage(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.YawAndPitchStabilizationMode(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.MaximumStarboardCoverage(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.MaximumStarboardSwathWidth(i82) = fread(fid,1,'uint16'); + ALLdata.EM_Runtime.DurotongSpeed(i82) = fread(fid,1,'uint16'); % OR: ALLdata.EM_Runtime.TransmitAlongTilt(i82) = fread(fid,1,'int16'); + ALLdata.EM_Runtime.HiLoFrequencyAbsorptionCoefficientRatio(i82) = fread(fid,1,'uint8'); % OR filter identifier 2 + ALLdata.EM_Runtime.ETX(i82) = fread(fid,1,'uint8'); + ALLdata.EM_Runtime.CheckSum(i82) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_Runtime.ETX(i82)~=3 + error('wrong ETX value (ALLdata.EM_Runtime)'); + end + + % confirm parsing + parsed = 1; + + case 83 % 'SEABED IMAGE DATAGRAM (53H)' + if ~(isempty(OutputFields)||any(strcmp('EM_SeabedImage',OutputFields))) + continue; + end + % counter for this type of datagram + try i83=i83+1; catch, i83=1; end + + % parsing + ALLdata.EM_SeabedImage.NumberOfBytesInDatagram(i83) = nbDatag; + ALLdata.EM_SeabedImage.STX(i83) = stxDatag; + ALLdata.EM_SeabedImage.TypeOfDatagram(i83) = datagTypeNumber; + ALLdata.EM_SeabedImage.EMModelNumber(i83) = emNumber; + ALLdata.EM_SeabedImage.Date(i83) = date; + ALLdata.EM_SeabedImage.TimeSinceMidnightInMilliseconds(i83) = timeSinceMidnightInMilliseconds; + ALLdata.EM_SeabedImage.PingCounter(i83) = number; + ALLdata.EM_SeabedImage.SystemSerialNumber(i83) = systemSerialNumber; + + ALLdata.EM_SeabedImage.MeanAbsorptionCoefficient(i83) = fread(fid,1,'uint16'); % 'this field had earlier definition' + ALLdata.EM_SeabedImage.PulseLength(i83) = fread(fid,1,'uint16'); % 'this field had earlier definition' + ALLdata.EM_SeabedImage.RangeToNormalIncidence(i83) = fread(fid,1,'uint16'); + ALLdata.EM_SeabedImage.StartRangeSampleOfTVGRamp(i83) = fread(fid,1,'uint16'); + ALLdata.EM_SeabedImage.StopRangeSampleOfTVGRamp(i83) = fread(fid,1,'uint16'); + ALLdata.EM_SeabedImage.NormalIncidenceBS(i83) = fread(fid,1,'int8'); %BSN + ALLdata.EM_SeabedImage.ObliqueBS(i83) = fread(fid,1,'int8'); %BSO + ALLdata.EM_SeabedImage.TxBeamwidth(i83) = fread(fid,1,'uint16'); + ALLdata.EM_SeabedImage.TVGLawCrossoverAngle(i83) = fread(fid,1,'uint8'); + ALLdata.EM_SeabedImage.NumberOfValidBeams(i83) = fread(fid,1,'uint8'); %N + + % repeat cycle: N entries of 6 bits + temp = ftell(fid); + N = ALLdata.EM_SeabedImage.NumberOfValidBeams(i83); + ALLdata.EM_SeabedImage.BeamIndexNumber{i83} = fread(fid,N,'uint8',6-1); + fseek(fid,temp+1,'bof'); % to next data type + ALLdata.EM_SeabedImage.SortingDirection{i83} = fread(fid,N,'int8',6-1); + fseek(fid,temp+2,'bof'); % to next data type + ALLdata.EM_SeabedImage.NumberOfSamplesPerBeam{i83} = fread(fid,N,'uint16',6-2); %Ns + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_SeabedImage.CentreSampleNumber{i83} = fread(fid,N,'uint16',6-2); + fseek(fid,2-6,'cof'); % we need to come back after last jump + + % reading image data + Ns = [ALLdata.EM_SeabedImage.NumberOfSamplesPerBeam{i83}]; + tmp = fread(fid,sum(Ns),'int8'); + ALLdata.EM_SeabedImage.SampleAmplitudes(i83).beam = mat2cell(tmp,Ns); + + % "spare byte if required to get even length (always 0 if used)" + if floor(sum(Ns)/2) == sum(Ns)/2 + % even so far, since ETX is 1 byte, add a spare here + ALLdata.EM_SeabedImage.Data.SpareByte(i83) = fread(fid,1,'uint8'); + else + % odd so far, since ETX is 1 bytes, no spare + ALLdata.EM_SeabedImage.Data.SpareByte(i83) = NaN; + end + ALLdata.EM_SeabedImage.ETX(i83) = fread(fid,1,'uint8'); + ALLdata.EM_SeabedImage.CheckSum(i83) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_SeabedImage.ETX(i83)~=3 + error('wrong ETX value (ALLdata.EM_SeabedImage)'); + end + + % confirm parsing + parsed = 1; + + case 85 % 'SOUND SPEED PROFILE (55H)' + if ~(isempty(OutputFields)||any(strcmp('EM_SoundSpeedProfile',OutputFields))) + continue; + end + % counter for this type of datagram + try i85=i85+1; catch, i85=1; end + + % parsing + ALLdata.EM_SoundSpeedProfile.NumberOfBytesInDatagram(i85) = nbDatag; + ALLdata.EM_SoundSpeedProfile.STX(i85) = stxDatag; + ALLdata.EM_SoundSpeedProfile.TypeOfDatagram(i85) = datagTypeNumber; + ALLdata.EM_SoundSpeedProfile.EMModelNumber(i85) = emNumber; + ALLdata.EM_SoundSpeedProfile.Date(i85) = date; + ALLdata.EM_SoundSpeedProfile.TimeSinceMidnightInMilliseconds(i85) = timeSinceMidnightInMilliseconds; + ALLdata.EM_SoundSpeedProfile.ProfileCounter(i85) = number; + ALLdata.EM_SoundSpeedProfile.SystemSerialNumber(i85) = systemSerialNumber; + + ALLdata.EM_SoundSpeedProfile.DateWhenProfileWasMade(i85) = fread(fid,1,'uint32'); + ALLdata.EM_SoundSpeedProfile.TimeSinceMidnightInMillisecondsWhenProfileWasMade(i85) = fread(fid,1,'uint32'); + ALLdata.EM_SoundSpeedProfile.NumberOfEntries(i85) = fread(fid,1,'uint16'); %N + ALLdata.EM_SoundSpeedProfile.DepthResolution(i85) = fread(fid,1,'uint16'); + + % repeat cycle: N entries of 8 bits + temp = ftell(fid); + N = ALLdata.EM_SoundSpeedProfile.NumberOfEntries(i85); + ALLdata.EM_SoundSpeedProfile.Depth{i85} = fread(fid,N,'uint32',8-4); + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_SoundSpeedProfile.SoundSpeed{i85} = fread(fid,N,'uint32',8-4); + fseek(fid,4-8,'cof'); % we need to come back after last jump + + ALLdata.EM_SoundSpeedProfile.SpareByte(i85) = fread(fid,1,'uint8'); + ALLdata.EM_SoundSpeedProfile.ETX(i85) = fread(fid,1,'uint8'); + ALLdata.EM_SoundSpeedProfile.CheckSum(i85) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_SoundSpeedProfile.ETX(i85)~=3 + error('wrong ETX value (ALLdata.EM_SoundSpeedProfile)'); + end + + % confirm parsing + parsed = 1; + + case 88 % 'XYZ 88 (58H)' + if ~(isempty(OutputFields)||any(strcmp('EM_XYZ88',OutputFields))) + continue; + end + % counter for this type of datagram + try i88=i88+1; catch, i88=1; end + + % parsing + ALLdata.EM_XYZ88.NumberOfBytesInDatagram(i88) = nbDatag; + ALLdata.EM_XYZ88.STX(i88) = stxDatag; + ALLdata.EM_XYZ88.TypeOfDatagram(i88) = datagTypeNumber; + ALLdata.EM_XYZ88.EMModelNumber(i88) = emNumber; + ALLdata.EM_XYZ88.Date(i88) = date; + ALLdata.EM_XYZ88.TimeSinceMidnightInMilliseconds(i88) = timeSinceMidnightInMilliseconds; + ALLdata.EM_XYZ88.PingCounter(i88) = number; + ALLdata.EM_XYZ88.SystemSerialNumber(i88) = systemSerialNumber; + + ALLdata.EM_XYZ88.HeadingOfVessel(i88) = fread(fid,1,'uint16'); + ALLdata.EM_XYZ88.SoundSpeedAtTransducer(i88) = fread(fid,1,'uint16'); + ALLdata.EM_XYZ88.TransmitTransducerDepth(i88) = fread(fid,1,'float32'); + ALLdata.EM_XYZ88.NumberOfBeamsInDatagram(i88) = fread(fid,1,'uint16'); + ALLdata.EM_XYZ88.NumberOfValidDetections(i88) = fread(fid,1,'uint16'); + ALLdata.EM_XYZ88.SamplingFrequencyInHz(i88) = fread(fid,1,'float32'); + ALLdata.EM_XYZ88.ScanningInfo(i88) = fread(fid,1,'uint8'); + ALLdata.EM_XYZ88.Spare1(i88) = fread(fid,1,'uint8'); + ALLdata.EM_XYZ88.Spare2(i88) = fread(fid,1,'uint8'); + ALLdata.EM_XYZ88.Spare3(i88) = fread(fid,1,'uint8'); + + % repeat cycle: N entries of 20 bits + temp = ftell(fid); + C = 20; + N = ALLdata.EM_XYZ88.NumberOfBeamsInDatagram(i88); + ALLdata.EM_XYZ88.DepthZ{i88} = fread(fid,N,'float32',C-4); + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_XYZ88.AcrosstrackDistanceY{i88} = fread(fid,N,'float32',C-4); + fseek(fid,temp+8,'bof'); % to next data type + ALLdata.EM_XYZ88.AlongtrackDistanceX{i88} = fread(fid,N,'float32',C-4); + fseek(fid,temp+12,'bof'); % to next data type + ALLdata.EM_XYZ88.DetectionWindowLength{i88} = fread(fid,N,'uint16',C-2); + fseek(fid,temp+14,'bof'); % to next data type + ALLdata.EM_XYZ88.QualityFactor{i88} = fread(fid,N,'uint8',C-1); + fseek(fid,temp+15,'bof'); % to next data type + ALLdata.EM_XYZ88.BeamIncidenceAngleAdjustment{i88} = fread(fid,N,'int8',C-1); + fseek(fid,temp+16,'bof'); % to next data type + ALLdata.EM_XYZ88.DetectionInformation{i88} = fread(fid,N,'uint8',C-1); + fseek(fid,temp+17,'bof'); % to next data type + ALLdata.EM_XYZ88.RealTimeCleaningInformation{i88} = fread(fid,N,'int8',C-1); + fseek(fid,temp+18,'bof'); % to next data type + ALLdata.EM_XYZ88.ReflectivityBS{i88} = fread(fid,N,'int16',C-2); + fseek(fid,2-C,'cof'); % we need to come back after last jump + + ALLdata.EM_XYZ88.Spare4(i88) = fread(fid,1,'uint8'); + ALLdata.EM_XYZ88.ETX(i88) = fread(fid,1,'uint8'); + ALLdata.EM_XYZ88.CheckSum(i88) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_XYZ88.ETX(i88)~=3 + warning('wrong ETX value (ALLdata.EM_XYZ88)'); + fields_xyz=fieldnames(ALLdata.EM_XYZ88); + for ifi=1:numel(fields_xyz) + if numel(ALLdata.EM_XYZ88.(fields_xyz{ifi}))>=i88 + ALLdata.EM_XYZ88.(fields_xyz{ifi})(i88)=[]; + end + end + i88=i88-1; + parsed=0; + else + + % confirm parsing + parsed = 1; + end + case 89 % 'SEABED IMAGE DATA 89 (59H)' + if ~(isempty(OutputFields)||any(strcmp('EM_SeabedImage89',OutputFields))) + continue; + end + % counter for this type of datagram + try i89=i89+1; catch, i89=1; end + + % parsing + ALLdata.EM_SeabedImage89.NumberOfBytesInDatagram(i89) = nbDatag; + ALLdata.EM_SeabedImage89.STX(i89) = stxDatag; + ALLdata.EM_SeabedImage89.TypeOfDatagram(i89) = datagTypeNumber; + ALLdata.EM_SeabedImage89.EMModelNumber(i89) = emNumber; + ALLdata.EM_SeabedImage89.Date(i89) = date; + ALLdata.EM_SeabedImage89.TimeSinceMidnightInMilliseconds(i89) = timeSinceMidnightInMilliseconds; + ALLdata.EM_SeabedImage89.PingCounter(i89) = number; + ALLdata.EM_SeabedImage89.SystemSerialNumber(i89) = systemSerialNumber; + + ALLdata.EM_SeabedImage89.SamplingFrequencyInHz(i89) = fread(fid,1,'float32'); + ALLdata.EM_SeabedImage89.RangeToNormalIncidence(i89) = fread(fid,1,'uint16'); + ALLdata.EM_SeabedImage89.NormalIncidenceBS(i89) = fread(fid,1,'int16'); %BSN + ALLdata.EM_SeabedImage89.ObliqueBS(i89) = fread(fid,1,'int16'); %BSO + ALLdata.EM_SeabedImage89.TxBeamwidthAlong(i89) = fread(fid,1,'uint16'); + ALLdata.EM_SeabedImage89.TVGLawCrossoverAngle(i89) = fread(fid,1,'uint16'); + ALLdata.EM_SeabedImage89.NumberOfValidBeams(i89) = fread(fid,1,'uint16'); + + % repeat cycle: N entries of 6 bits + temp = ftell(fid); + C = 6; + N = ALLdata.EM_SeabedImage89.NumberOfValidBeams(i89); + ALLdata.EM_SeabedImage89.SortingDirection{i89} = fread(fid,N,'int8',C-1); + fseek(fid,temp+1,'bof'); % to next data type + ALLdata.EM_SeabedImage89.DetectionInfo{i89} = fread(fid,N,'uint8',C-1); + fseek(fid,temp+2,'bof'); % to next data type + ALLdata.EM_SeabedImage89.NumberOfSamplesPerBeam{i89} = fread(fid,N,'uint16',C-2); %Ns + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_SeabedImage89.CentreSampleNumber{i89} = fread(fid,N,'uint16',C-2); + fseek(fid,2-C,'cof'); % we need to come back after last jump + + % reading image data + Ns = [ALLdata.EM_SeabedImage89.NumberOfSamplesPerBeam{i89}]; + tmp = fread(fid,sum(Ns),'int16'); + ALLdata.EM_SeabedImage89.SampleAmplitudes(i89).beam = mat2cell(tmp,Ns); + + ALLdata.EM_SeabedImage89.Spare(i89) = fread(fid,1,'uint8'); + ALLdata.EM_SeabedImage89.ETX(i89) = fread(fid,1,'uint8'); + ALLdata.EM_SeabedImage89.CheckSum(i89) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_SeabedImage89.ETX(i89)~=3 + error('wrong ETX value (ALLdata.EM_SeabedImage89)'); + end + + % confirm parsing + parsed = 1; + + case 102 % 'RAW RANGE AND BEAM ANGLE (f) (66H)' + if ~(isempty(OutputFields)||any(strcmp('EM_RawBeamRangeAngle',OutputFields))) + continue; + end + % counter for this type of datagram + try i102=i102+1; catch, i102=1; end + + % parsing + % ...to write... + + case 104 % 'DEPTH (PRESSURE) OR HEIGHT DATAGRAM (68H)' + if ~(isempty(OutputFields)||any(strcmp('EM_Height',OutputFields))) + continue; + end + % counter for this type of datagram + try i104=i104+1; catch, i104=1; end + + % parsing + ALLdata.EM_Height.NumberOfBytesInDatagram(i104) = nbDatag; + ALLdata.EM_Height.STX(i104) = stxDatag; + ALLdata.EM_Height.TypeOfDatagram(i104) = datagTypeNumber; + ALLdata.EM_Height.EMModelNumber(i104) = emNumber; + ALLdata.EM_Height.Date(i104) = date; + ALLdata.EM_Height.TimeSinceMidnightInMilliseconds(i104) = timeSinceMidnightInMilliseconds; + ALLdata.EM_Height.HeightCounter(i104) = number; + ALLdata.EM_Height.SystemSerialNumber(i104) = systemSerialNumber; + + ALLdata.EM_Height.Height(i104) = fread(fid,1,'int32'); + ALLdata.EM_Height.HeigthType(i104) = fread(fid,1,'uint8'); + ALLdata.EM_Height.ETX(i104) = fread(fid,1,'uint8'); + ALLdata.EM_Height.CheckSum(i104) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_Height.ETX(i104)~=3 + error('wrong ETX value (ALLdata.EM_Height)'); + end + + % confirm parsing + parsed = 1; + + case 105 % 'INSTALLATION PARAMETERS - STOP (69H)' + if ~(isempty(OutputFields)||any(strcmp('EM_InstallationStop',OutputFields))) + continue; + end + % counter for this type of datagram + try i105=i105+1; catch, i105=1; end + + % parsing + ALLdata.EM_InstallationStop.NumberOfBytesInDatagram(i105) = nbDatag; + ALLdata.EM_InstallationStop.STX(i105) = stxDatag; + ALLdata.EM_InstallationStop.TypeOfDatagram(i105) = datagTypeNumber; + ALLdata.EM_InstallationStop.EMModelNumber(i105) = emNumber; + ALLdata.EM_InstallationStop.Date(i105) = date; + ALLdata.EM_InstallationStop.TimeSinceMidnightInMilliseconds(i105) = timeSinceMidnightInMilliseconds; + ALLdata.EM_InstallationStop.SurveyLineNumber(i105) = number; + ALLdata.EM_InstallationStop.SystemSerialNumber(i105) = systemSerialNumber; + + ALLdata.EM_InstallationStop.SerialNumberOfSecondSonarHead(i105) = fread(fid,1,'uint16'); + + % 18 bytes of binary data already recorded and 3 more to come = 21. + % but nbDatag will always be even thanks to SpareByte. so + % nbDatag is 22 if there is no ASCII data and more if there is + % ASCII data. read the rest as ASCII (including SpareByte) with + % 1 byte for 1 character. + ALLdata.EM_InstallationStop.ASCIIData{i105} = fscanf(fid, '%c', nbDatag-21); + + ALLdata.EM_InstallationStop.ETX(i105) = fread(fid,1,'uint8'); + ALLdata.EM_InstallationStop.CheckSum(i105) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_InstallationStop.ETX(i105)~=3 + error('wrong ETX value (ALLdata.EM_InstallationStop)'); + end + + % confirm parsing + parsed = 1; + + case 107 % 'WATER COLUMN DATAGRAM (6BH)' + if ~(isempty(OutputFields)||any(strcmp('EM_WaterColumn',OutputFields))) + continue; + end + % counter for this type of datagram + try i107=i107+1; catch, i107=1; end + + % ----- IMPORTANT NOTE ---------------------------------------- + % This datagram's data is too to be stored in memory. Instead, + % we record the metadata and the position-in-file location of + % the data, which be extracted and stored in binary format at + % the next stage of data conversion. + % ------------------------------------------------------------- + + % parsing + ALLdata.EM_WaterColumn.NumberOfBytesInDatagram(i107) = nbDatag; + + % position at STX identifier + pos_1 = ftell(fid); + + % fields already read + ALLdata.EM_WaterColumn.STX(i107) = stxDatag; + ALLdata.EM_WaterColumn.TypeOfDatagram(i107) = datagTypeNumber; + ALLdata.EM_WaterColumn.EMModelNumber(i107) = emNumber; + ALLdata.EM_WaterColumn.Date(i107) = date; + ALLdata.EM_WaterColumn.TimeSinceMidnightInMilliseconds(i107) = timeSinceMidnightInMilliseconds; + ALLdata.EM_WaterColumn.PingCounter(i107) = number; + ALLdata.EM_WaterColumn.SystemSerialNumber(i107) = systemSerialNumber; + + % remaining fields in start of datagram + ALLdata.EM_WaterColumn.NumberOfDatagrams(i107) = fread(fid,1,'uint16'); + ALLdata.EM_WaterColumn.DatagramNumbers(i107) = fread(fid,1,'uint16'); + ALLdata.EM_WaterColumn.NumberOfTransmitSectors(i107) = fread(fid,1,'uint16'); %Ntx + ALLdata.EM_WaterColumn.TotalNumberOfReceiveBeams(i107) = fread(fid,1,'uint16'); + ALLdata.EM_WaterColumn.NumberOfBeamsInThisDatagram(i107) = fread(fid,1,'uint16'); %Nrx + ALLdata.EM_WaterColumn.SoundSpeed(i107) = fread(fid,1,'uint16'); %SS + ALLdata.EM_WaterColumn.SamplingFrequency(i107) = fread(fid,1,'uint32'); %SF + ALLdata.EM_WaterColumn.TXTimeHeave(i107) = fread(fid,1,'int16'); + ALLdata.EM_WaterColumn.TVGFunctionApplied(i107) = fread(fid,1,'uint8'); %X + ALLdata.EM_WaterColumn.TVGOffset(i107) = fread(fid,1,'int8'); %C + ALLdata.EM_WaterColumn.ScanningInfo(i107) = fread(fid,1,'uint8'); + ALLdata.EM_WaterColumn.Spare1(i107) = fread(fid,1,'uint8'); + ALLdata.EM_WaterColumn.Spare2(i107) = fread(fid,1,'uint8'); + ALLdata.EM_WaterColumn.Spare3(i107) = fread(fid,1,'uint8'); + + % --- repeat cycle #1 ----------------------------------------- + % Ntx entries of 6 bits + temp = ftell(fid); + C = 6; + Ntx = ALLdata.EM_WaterColumn.NumberOfTransmitSectors(i107); + ALLdata.EM_WaterColumn.TiltAngle{i107} = fread(fid,Ntx,'int16',C-2); + fseek(fid,temp+2,'bof'); % to next data type + ALLdata.EM_WaterColumn.CenterFrequency{i107} = fread(fid,Ntx,'uint16',C-2); + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_WaterColumn.TransmitSectorNumber{i107} = fread(fid,Ntx,'uint8',C-1); + fseek(fid,temp+5,'bof'); % to next data type + ALLdata.EM_WaterColumn.Spare{i107} = fread(fid,Ntx,'uint8',C-1); + fseek(fid,1-C,'cof'); % we need to come back after last jump + % --- end of repeat cycle #1 ---------------------------------- + + % --- repeat cycle #2 ----------------------------------------- + % Nrx entries of a possibly variable number of bits. + Nrx = ALLdata.EM_WaterColumn.NumberOfBeamsInThisDatagram(i107); + + % initialize flag of parsing error + wc_parsing_error = 0; + + % initialize outputs + ALLdata.EM_WaterColumn.BeamPointingAngle{i107} = nan(1,Nrx); + ALLdata.EM_WaterColumn.StartRangeSampleNumber{i107} = nan(1,Nrx); + ALLdata.EM_WaterColumn.NumberOfSamples{i107} = nan(1,Nrx); + ALLdata.EM_WaterColumn.DetectedRangeInSamples{i107} = nan(1,Nrx); + ALLdata.EM_WaterColumn.TransmitSectorNumber2{i107} = nan(1,Nrx); + ALLdata.EM_WaterColumn.BeamNumber{i107} = nan(1,Nrx); + ALLdata.EM_WaterColumn.SampleAmplitudePosition{i107} = nan(1,Nrx); + Ns = zeros(1,Nrx); + + % now parse the data + for jj = 1:Nrx + + try + + ALLdata.EM_WaterColumn.BeamPointingAngle{i107}(jj) = fread(fid,1,'int16'); + ALLdata.EM_WaterColumn.StartRangeSampleNumber{i107}(jj) = fread(fid,1,'uint16'); + ALLdata.EM_WaterColumn.NumberOfSamples{i107}(jj) = fread(fid,1,'uint16'); %Ns + ALLdata.EM_WaterColumn.DetectedRangeInSamples{i107}(jj) = fread(fid,1,'uint16'); %DR + ALLdata.EM_WaterColumn.TransmitSectorNumber2{i107}(jj) = fread(fid,1,'uint8'); + ALLdata.EM_WaterColumn.BeamNumber{i107}(jj) = fread(fid,1,'uint8'); + + % record number of samples + Ns(jj) = ALLdata.EM_WaterColumn.NumberOfSamples{i107}(jj); + + % Next are the samples data themselves, which you could + % record with: + % ALLdata.EM_WaterColumn.SampleAmplitude{i107}{jj} = fread(fid,Ns(jj),'int8'); + % But instead, we're only going to record the position + % of the start of data: + ALLdata.EM_WaterColumn.SampleAmplitudePosition{i107}(jj) = ftell(fid); + + % and since we did not read that data, we need to + % move manually onto the next beam: + fseek(fid,Ns(jj),'cof'); + + catch + + % if there's any issue in the recording, flag and exit + % the loop + ALLdata.EM_WaterColumn.NumberOfSamples{i107}(jj) = 0; + Ns(jj) = 0; + wc_parsing_error = 1; + continue; + + end + + end + % --- end of repeat cycle #2 ---------------------------------- + + % read the rest of the datagram. If all went well should be 3 + % or 4 bytes + pos_end = ftell(fid); + tmp_end = fread(fid,nbDatag-(pos_end-pos_1+1)-15,'int8=>int8'); + + % "spare byte if required to get even length (always 0 if used)" + if floor((Nrx*10+sum(Ns))/2) == (Nrx*10+sum(Ns))/2 + % all data parsed so far is an even number. Since ETX is 1 + % byte, add a spare byte here + ALLdata.EM_WaterColumn.Spare4(i107) = double(typecast(tmp_end(1),'uint8')); + + % and remove it from tmp_end + tmp_end(1) = []; + else + % odd. No added spare + ALLdata.EM_WaterColumn.Spare4(i107) = NaN; + end + + % record end of datagram + ALLdata.EM_WaterColumn.ETX(i107) = typecast(tmp_end(1),'uint8'); + ALLdata.EM_WaterColumn.CheckSum(i107) = typecast(tmp_end(2:3),'uint16'); + + % ETX check + if ALLdata.EM_WaterColumn.ETX(i107)~=3 + wc_parsing_error = 1; + end + + if wc_parsing_error == 0 + % HERE if data parsing all went well + + % confirm parsing and exit + parsed = 1; + + else + % HERE if data parsing failed, add a blank datagram in + % output + + % copy field names of previous entries + fields_wc = fieldnames(ALLdata.EM_WaterColumn); + + % add blanks fields for those missing + for ifi = 1:numel(fields_wc) + if numel(ALLdata.EM_WaterColumn.(fields_wc{ifi})) >= i107 + ALLdata.EM_WaterColumn.(fields_wc{ifi})(i107) = []; + end + end + + % failed parsing flag + parsed = 0; + + end + + case 110 % 'NETWORK ATTITUDE VELOCITY DATAGRAM 110 (6EH)' + if ~(isempty(OutputFields)||any(strcmp('EM_NetworkAttitude',OutputFields))) + continue; + end + % counter for this type of datagram + try i110=i110+1; catch, i110=1; end + + % parsing + ALLdata.EM_NetworkAttitude.NumberOfBytesInDatagram(i110) = nbDatag; + ALLdata.EM_NetworkAttitude.STX(i110) = stxDatag; + ALLdata.EM_NetworkAttitude.TypeOfDatagram(i110) = datagTypeNumber; + ALLdata.EM_NetworkAttitude.EMModelNumber(i110) = emNumber; + ALLdata.EM_NetworkAttitude.Date(i110) = date; + ALLdata.EM_NetworkAttitude.TimeSinceMidnightInMilliseconds(i110) = timeSinceMidnightInMilliseconds; + ALLdata.EM_NetworkAttitude.NetworkAttitudeCounter(i110) = number; + ALLdata.EM_NetworkAttitude.SystemSerialNumber(i110) = systemSerialNumber; + + ALLdata.EM_NetworkAttitude.NumberOfEntries(i110) = fread(fid,1,'uint16'); %N + ALLdata.EM_NetworkAttitude.SensorSystemDescriptor(i110) = fread(fid,1,'int8'); + ALLdata.EM_NetworkAttitude.Spare(i110) = fread(fid,1,'uint8'); + + % repeat cycle: N entries of a variable number of bits. Using a for loop + N = ALLdata.EM_NetworkAttitude.NumberOfEntries(i110); + Nx = nan(1,N); + for jj = 1:N + ALLdata.EM_NetworkAttitude.TimeInMillisecondsSinceRecordStart{i110}(jj) = fread(fid,1,'uint16'); + ALLdata.EM_NetworkAttitude.Roll{i110}(jj) = fread(fid,1,'int16'); + ALLdata.EM_NetworkAttitude.Pitch{i110}(jj) = fread(fid,1,'int16'); + ALLdata.EM_NetworkAttitude.Heave{i110}(jj) = fread(fid,1,'int16'); + ALLdata.EM_NetworkAttitude.Heading{i110}(jj) = fread(fid,1,'uint16'); + ALLdata.EM_NetworkAttitude.NumberOfBytesInInputDatagrams{i110}(jj) = fread(fid,1,'uint8'); %Nx + Nx(jj) = ALLdata.EM_NetworkAttitude.NumberOfBytesInInputDatagrams{i110}(jj); + ALLdata.EM_NetworkAttitude.NetworkAttitudeInputDatagramAsReceived{i110}{jj} = fread(fid,Nx(jj),'uint8'); + end + + % "spare byte if required to get even length (always 0 if used)" + if floor((N*11+sum(Nx))/2) == (N*11+sum(Nx))/2 + % even so far, since ETX is 1 byte, add a spare here + ALLdata.EM_NetworkAttitude.Spare2(i110) = fread(fid,1,'uint8'); + else + % odd so far, since ETX is 1 bytes, no spare + ALLdata.EM_NetworkAttitude.Spare2(i110) = NaN; + end + + ALLdata.EM_NetworkAttitude.ETX(i110) = fread(fid,1,'uint8'); + ALLdata.EM_NetworkAttitude.CheckSum(i110) = fread(fid,1,'uint16'); + + % ETX check + if ALLdata.EM_NetworkAttitude.ETX(i110)~=3 + error('wrong ETX value (ALLdata.EM_NetworkAttitude)'); + end + + % confirm parsing + parsed = 1; + + case 114 %'AMPLITUDE AND PHASE WC DATAGRAM 114 (72H)'; + if ~(isempty(OutputFields)||any(strcmp('EM_AmpPhase',OutputFields))) + continue; + end + % counter for this type of datagram + try i114=i114+1; catch, i114=1; end + + % ----- IMPORTANT NOTE ---------------------------------------- + % This datagram's data is too to be stored in memory. Instead, + % we record the metadata and the position-in-file location of + % the data, which be extracted and stored in binary format at + % the next stage of data conversion. + % ------------------------------------------------------------- + + % parsing + ALLdata.EM_AmpPhase.NumberOfBytesInDatagram(i114) = nbDatag; + + % position at STX identifier + pos_1 = ftell(fid); + + % fields already read + ALLdata.EM_AmpPhase.STX(i114) = stxDatag; + ALLdata.EM_AmpPhase.TypeOfDatagram(i114) = datagTypeNumber; + ALLdata.EM_AmpPhase.EMModelNumber(i114) = emNumber; + ALLdata.EM_AmpPhase.Date(i114) = date; + ALLdata.EM_AmpPhase.TimeSinceMidnightInMilliseconds(i114) = timeSinceMidnightInMilliseconds; + ALLdata.EM_AmpPhase.PingCounter(i114) = number; + ALLdata.EM_AmpPhase.SystemSerialNumber(i114) = systemSerialNumber; + + % remaining fields in start of datagram + ALLdata.EM_AmpPhase.NumberOfDatagrams(i114) = fread(fid,1,'uint16'); + ALLdata.EM_AmpPhase.DatagramNumbers(i114) = fread(fid,1,'uint16'); + ALLdata.EM_AmpPhase.NumberOfTransmitSectors(i114) = fread(fid,1,'uint16'); %Ntx + ALLdata.EM_AmpPhase.TotalNumberOfReceiveBeams(i114) = fread(fid,1,'uint16'); + ALLdata.EM_AmpPhase.NumberOfBeamsInThisDatagram(i114) = fread(fid,1,'uint16'); %Nrx + ALLdata.EM_AmpPhase.SoundSpeed(i114) = fread(fid,1,'uint16'); %SS + ALLdata.EM_AmpPhase.SamplingFrequency(i114) = fread(fid,1,'uint32'); %SF + ALLdata.EM_AmpPhase.TXTimeHeave(i114) = fread(fid,1,'int16'); + ALLdata.EM_AmpPhase.TVGFunctionApplied(i114) = fread(fid,1,'uint8'); %X + ALLdata.EM_AmpPhase.TVGOffset(i114) = fread(fid,1,'uint8'); %C + ALLdata.EM_AmpPhase.ScanningInfo(i114) = fread(fid,1,'uint8'); + ALLdata.EM_AmpPhase.Spare1(i114) = fread(fid,1,'uint8'); + ALLdata.EM_AmpPhase.Spare2(i114) = fread(fid,1,'uint8'); + ALLdata.EM_AmpPhase.Spare3(i114) = fread(fid,1,'uint8'); + + % --- repeat cycle #1 ----------------------------------------- + % Ntx entries of 6 bits + temp = ftell(fid); + C = 6; + Ntx = ALLdata.EM_AmpPhase.NumberOfTransmitSectors(i114); + ALLdata.EM_AmpPhase.TiltAngle{i114} = fread(fid,Ntx,'int16',C-2); + fseek(fid,temp+2,'bof'); % to next data type + ALLdata.EM_AmpPhase.CenterFrequency{i114} = fread(fid,Ntx,'uint16',C-2); + fseek(fid,temp+4,'bof'); % to next data type + ALLdata.EM_AmpPhase.TransmitSectorNumber{i114} = fread(fid,Ntx,'uint8',C-1); + fseek(fid,temp+5,'bof'); % to next data type + ALLdata.EM_AmpPhase.Spare{i114} = fread(fid,Ntx,'uint8',C-1); + fseek(fid,1-C,'cof'); % we need to come back after last jump + % --- end of repeat cycle #1 ---------------------------------- + + % --- repeat cycle #2 ----------------------------------------- + % Nrx entries of a possibly variable number of bits. + Nrx = ALLdata.EM_AmpPhase.NumberOfBeamsInThisDatagram(i114); + + pos_2 = ftell(fid); % position at start of data + tmp = fread(fid,nbDatag-(pos_2-pos_1+1)-15,'int8'); % read all that data block + tmp = int8(tmp'); + id = 0; % offset for start of each Nrx block + wc_parsing_error = 0; % initialize flag + + % initialize outputs + ALLdata.EM_AmpPhase.BeamPointingAngle{i114} = nan(1,Nrx); + ALLdata.EM_AmpPhase.StartRangeSampleNumber{i114} = nan(1,Nrx); + ALLdata.EM_AmpPhase.NumberOfSamples{i114} = nan(1,Nrx); + ALLdata.EM_AmpPhase.DetectedRangeInSamples{i114} = nan(1,Nrx); + ALLdata.EM_AmpPhase.TransmitSectorNumber2{i114} = nan(1,Nrx); + ALLdata.EM_AmpPhase.BeamNumber{i114} = nan(1,Nrx); + ALLdata.EM_AmpPhase.SamplePhaseAmplitudePosition{i114} = nan(1,Nrx); + Ns = zeros(1,Nrx); + + % now parse the data + for jj = 1:Nrx + + try + + ALLdata.EM_AmpPhase.BeamPointingAngle{i114}(jj) = typecast(tmp(1+id:2+id),'int16'); + ALLdata.EM_AmpPhase.StartRangeSampleNumber{i114}(jj) = typecast(tmp(3+id:4+id),'uint16'); + ALLdata.EM_AmpPhase.NumberOfSamples{i114}(jj) = typecast(tmp(5+id:6+id),'uint16'); + ALLdata.EM_AmpPhase.DetectedRangeInSamples{i114}(jj) = typecast(tmp(7+id:8+id),'uint16'); + ALLdata.EM_AmpPhase.TransmitSectorNumber2{i114}(jj) = typecast(tmp(9+id),'uint8'); + ALLdata.EM_AmpPhase.BeamNumber{i114}(jj) = typecast(tmp(10+id),'uint8'); + + % recording data position instead of data themselves + ALLdata.EM_AmpPhase.SamplePhaseAmplitudePosition{i114}(jj) = pos_2 + id + 10; + % actual data recording would be: + % ALLdata.EM_AmpPhase.SampleAmplitude{i114}{jj} = tmp((11+id):(11+id+Ns(jj)-1)); + + if ALLdata.EM_AmpPhase.NumberOfSamples{i114}(jj) < 2^16/2 + Ns(jj) = ALLdata.EM_AmpPhase.NumberOfSamples{i114}(jj); + else + % error in number of samples + ALLdata.EM_AmpPhase.NumberOfSamples{i114}(jj) = 0; + Ns(jj) = 0; + end + + % offset to next jj block + id = 10*jj + 4*sum(Ns); + + catch + + % if there's any issue in the recording, flag and exit + % the loop + ALLdata.EM_AmpPhase.NumberOfSamples{i114}(jj) = 0; + Ns(jj) = 0; + wc_parsing_error = 1; + continue; + + end + + end + % --- end of repeat cycle #2 ---------------------------------- + + if wc_parsing_error == 0 + % HERE if data parsing all went well + + % "spare byte if required to get even length (always 0 if used)" + if floor((Nrx*10+4*sum(Ns))/2) == (Nrx*10+4*sum(Ns))/2 + % even so far, since ETX is 1 byte, add a spare here + ALLdata.EM_AmpPhase.Spare4(i114) = double(typecast(tmp(1+id),'uint8')); + id = id+1; + else + % odd so far, since ETX is 1 bytes, no spare + ALLdata.EM_AmpPhase.Spare4(i114) = NaN; + end + + % end of datagram + ALLdata.EM_AmpPhase.ETX(i114) = typecast(tmp(id+1),'uint8'); + ALLdata.EM_AmpPhase.CheckSum(i114) = typecast(tmp(2+id:3+id),'uint16'); + + % ETX check + if ALLdata.EM_AmpPhase.ETX(i114)~=3 + error('wrong ETX value (ALLdata.EM_AmpPhase)'); + end + + % confirm parsing + parsed = 1; + + else + % HERE if data parsing failed, add a blank datagram in + % output + + % copy field names of previous entries + fields_ap = fieldnames(ALLdata.EM_AmpPhase); + + % add blanks fields for those missing + for ifi = 1:numel(fields_ap) + if numel(ALLdata.EM_AmpPhase.(fields_ap{ifi})) >= i114 + ALLdata.EM_AmpPhase.(fields_ap{ifi})(i114) = []; + end + end + + i114 = i114-1; % XXX1 if we do that, then we'll rewrite over the blank record we just entered?? + parsed = 0; + + end + + otherwise + + % datagTypeNumber is not recognized yet + + end + + % modify parsed status in info + ALLfileinfo.parsed(iDatag,1) = parsed; + + % communicate progress + comms.progress(iDatag,nDatagsToPars); + +end + + +%% close fid +fclose(fid); + + +%% modify the ping numbers in case they have gone over intmax('uint16') + +% do with all datagram types that have ping numbers +All_fields = fieldnames(ALLdata); +for ifif = 1:numel(All_fields) + if isstruct(ALLdata.(All_fields{ifif})) && isfield(ALLdata.(All_fields{ifif}),'PingCounter') + + % list of ping numbers + pings = ALLdata.(All_fields{ifif}).PingCounter; + + % search for indices where pings go from 65535 to 0 + idx_rollover = find(diff(pings)==-double(intmax('uint16'))); + + % new ping numbers + idx_high = [idx_rollover+1 numel(pings)]; + for iu = 1:numel(idx_high)-1 + pings(idx_high(iu):idx_high(iu+1)) = pings(idx_high(iu):idx_high(iu+1))+iu*double(intmax('uint16'))+1; + end + + % save back into structure + ALLdata.(All_fields{ifif}).PingCounter = pings; + end +end + + +%% add info to parsed data +ALLdata.info = ALLfileinfo; + + +%% end message +comms.finish('Done'); diff --git a/read_data_files/Kongsberg/format_kmall/CFF_check_KMALLfilename.m b/read_data_files/Kongsberg/format_kmall/CFF_check_KMALLfilename.m new file mode 100644 index 0000000..6c13153 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/CFF_check_KMALLfilename.m @@ -0,0 +1,17 @@ +function out = CFF_check_KMALLfilename(rawfilename) +%CFF_CHECK_KMALLFILENAME Check file exists and has kmall extension +% +% out = CFF_CHECK_KMALLFILENAME(rawFile) checks if single file +% rawFile (char) exists and has '.kmall' or ',kmwcd' extension. +% +% out = CFF_CHECK_KMALLFILENAME(rawFilesPair) checks if pair of files +% rawFilesPair (2x1 cell array of chars) exist, match (same name), and +% have '.kmall' and '.kmwcd' extensions. +% +% See also CFF_CHECK_FILENAME. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +out = CFF_check_filename(rawfilename,{'.kmall','.kmwcd'}); diff --git a/read_data_files/Kongsberg/format_kmall/CFF_convert_KMALLdata_to_fData.m b/read_data_files/Kongsberg/format_kmall/CFF_convert_KMALLdata_to_fData.m new file mode 100644 index 0000000..1b0ce3f --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/CFF_convert_KMALLdata_to_fData.m @@ -0,0 +1,809 @@ +function fData = CFF_convert_KMALLdata_to_fData(KMALLdataGroup,varargin) +%CFF_CONVERT_KMALLDATA_TO_FDATA Convert kmall data to the CoFFee format +% +% Converts Kongsberg EM series data FROM the KMALLdata format (read by +% CFF_READ_KMALL) TO the CoFFee fData format used in processing. +% +% fData = CFF_CONVERT_KMALLDATA_TO_FDATA(KMALLdata) converts the contents +% of one KMALLdata structure to a structure in the fData format. +% +% fData = CFF_CONVERT_KMALLDATA_TO_FDATA(KMALLdataGroup) converts an +% array of two KMALLdata structures into one fData sructure. The pair of +% structure must correspond to an .kmall/.kmwcd pair of files. Do not try +% to use this feature to convert KMALLdata structures from different +% acquisition files. It will not work. Convert each into its own fData +% structure. +% +% Note that the KMALLdata structures are converted to fData in the order +% they are in input, and that the first ones take precedence. Aka in the +% example above, if the second structure contains a type of datagram that +% is already in the first, they will NOT be converted. This is to avoid +% doubling up the data that may exist in duplicate in the pair of raw +% files. You need to order the KMALLdata structures in input in order of +% desired precedence. +% +% fData = CFF_CONVERT_KMALLDATA_TO_FDATA(KMALLdata,dr_sub,db_sub) +% operates the conversion with a sub-sampling of the water-column data +% (either WC or AP datagrams) in range and in beams. For example, to +% sub-sample range by a factor of 10 and beams by a factor of 2, use: +% fData = CFF_CONVERT_KMALLDATA_TO_FDATA(KMALLdata,10,2). + +% DEV NOTE: To date (July 2021), the fData format is based on three +% dimensions ping x beam x sample, but the lowest-level unit in kmall is +% the "swath" to accomodate dual-and multi-swath operating modes. For +% example, in dual-swath mode, a single Tx transducer will transmit 2 +% pulses to create two along-swathes, and in kmall those two swathes are +% recorded with the same ping counter because they were produced at about +% the same time. +% To deal with this, we're going to create new "swath numbers" based on +% the original ping number and swath counter for a ping. +% For example a ping #832 made up of four swathes counted 0-3 will have +% new "swath numbers" of 832.00, 832.01, 832.02, and 832.03. We will +% maintain the current "ping" nomenclature in fData, but using those +% swath numbers. +% Note that if we have single-swath data, then the swath number matches +% the ping number (832). +% Note that this is made more complicated by the fact that an individual +% swathe can have its data on multiple consecutive datagrams, as +% different "Rx fans" (i.e multiple Rx heads) are recorded on separate +% datagrams. +% +% NOTE: a new "inspection" mode with alternating low/high frequency trips +% up the code. To fix eventually when needed XXX2 +% There also seem to be some issues with location of bottom on some +% dual head data. To fix eventually XXX1 +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 21-07-2021 + + +%% Input arguments management +p = inputParser; + +% array of KMALLdata structures +addRequired(p,'KMALLdataGroup',@(x) isstruct(x) || iscell(x)); + +% decimation factor in range and beam +addOptional(p,'dr_sub',1,@(x) isnumeric(x)&&x>0); +addOptional(p,'db_sub',1,@(x) isnumeric(x)&&x>0); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,KMALLdataGroup,varargin{:}) + +% and get results +KMALLdataGroup = p.Results.KMALLdataGroup; +dr_sub = p.Results.dr_sub; +db_sub = p.Results.db_sub; +global comms +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end +clear p; + +if isstruct(KMALLdataGroup) + % just one structure + KMALLdataGroup = {KMALLdataGroup}; +end + +% check input +if numel(KMALLdataGroup)==1 + % single KMALLdata structure + + % check it's from Kongsberg and that source file exist + has_KMALLfilename = isfield(KMALLdataGroup{1}, 'KMALLfilename'); + if ~has_KMALLfilename || ~CFF_check_KMALLfilename(KMALLdataGroup{1}.KMALLfilename) + error('Invalid input'); + end + +elseif numel(KMALLdataGroup)==2 + % pair of KMALLdata structures + + % check it's from a pair of Kongsberg kmall/kmwcd files and that source + % files exist + has_KMALLfilename = cell2mat(cellfun(@(x) isfield(x, 'KMALLfilename'), KMALLdataGroup, 'UniformOutput', false)); + rawfilenames = cellfun(@(x) x.KMALLfilename, KMALLdataGroup, 'UniformOutput', false); + if ~all(has_KMALLfilename) || ~CFF_check_KMALLfilename(rawfilenames) + error('Invalid input'); + end + +else + error('Invalid input'); +end + + +%% Prep + +% start message +comms.start('Converting to fData format'); + +% number of individual KMALLdata structures in input KMALLdataGroup +nStruct = length(KMALLdataGroup); + +% initialize fData, with current version number +fData.MET_Fmt_version = CFF_get_current_fData_version(); + +% initialize source filenames +fData.ALLfilename = cell(1,nStruct); + +% start progress +comms.progress(0,nStruct); + + +%% take one KMALLdata structure at a time and add its contents to fData +for iF = 1:nStruct + + % get current structure + KMALLdata = KMALLdataGroup{iF}; + + % add source filename + KMALLfilename = KMALLdata.KMALLfilename; + fData.ALLfilename{iF} = KMALLfilename; + + % now reading each type of datagram. + % Note we only convert the datagrams if fData does not already contain + % any. + + + %% '#IIP - Installation parameters and sensor setup' + if isfield(KMALLdata,'EMdgmIIP') && ~isfield(fData,'IP_ASCIIparameters') + + comms.step('Converting #IIP'); + + % number of entries + % nD = numel(KMALLdata.EMdgmIIP); + + % get date and time-since-midnight-in-milleseconds from header + % [fData.IP_1D_Date, fData.IP_1D_TimeSinceMidnightInMilliseconds] = CFF_get_date_and_TSMIM_from_kmall_header([KMALLdata.EMdgmIIP.header]); + + % DEV NOTES: Only value needed (to date) is the "sonar + % heading offset". In installation parameters datagrams of .all + % files, we only had one field "S1H" per head. Here we have heading + % values for both the Tx and Rx antennae. So not sure which one we + % should take, or the difference between the two... but for now, + % take the value from Rx. + + % read ASCIIdata + ASCIIdata = KMALLdata.EMdgmIIP(1).install_txt; + + % remove carriage returns, tabs and linefeed + ASCIIdata = regexprep(ASCIIdata,char(9),''); + ASCIIdata = regexprep(ASCIIdata,newline,''); + ASCIIdata = regexprep(ASCIIdata,char(13),''); + + % read some fields and record value in old field for the software + % to pick up + try + IP_ASCIIparameters.TRAI_RX1 = CFF_read_TRAI(ASCIIdata,'TRAI_RX1'); + IP_ASCIIparameters.S1H = IP_ASCIIparameters.TRAI_RX1.H; + catch + % at least in some EM2040C dual head data, I've found this + % field missing and instead having TRAI_HD1 + IP_ASCIIparameters.TRAI_HD1 = CFF_read_TRAI(ASCIIdata,'TRAI_HD1'); + IP_ASCIIparameters.S1H = IP_ASCIIparameters.TRAI_HD1.H; + end + + % finally store in fData + fData.IP_ASCIIparameters = IP_ASCIIparameters; + + end + + + %% '#MRZ - Multibeam (M) raw range (R) and depth(Z) datagram' + if isfield(KMALLdata,'EMdgmMRZ') && ~isfield(fData,'X8_1D_Date') + + comms.step('Converting #MRZ'); + + % DEV NOTE: note we don't decimate beam data here as we do for + % water-column data + + % remove duplicate datagrams + EMdgmMRZ = CFF_remove_duplicate_KMALL_datagrams(KMALLdata.EMdgmMRZ); + + % extract data + header = [EMdgmMRZ.header]; + cmnPart = [EMdgmMRZ.cmnPart]; + pingInfo = [EMdgmMRZ.pingInfo]; + rxInfo = [EMdgmMRZ.rxInfo]; + sounding = [EMdgmMRZ.sounding]; + % CFF_get_kmall_TxRx_info(cmnPart) % evaluate to get info for debugging + + % DEV NOTE: In the .all format, we only record two fields from the + % "Runtime Parameters" datagram: "TransmitPowerReMaximum" for + % radiometric corrections, and "ReceiveBeamwidth" for estimation of + % the bottom echo for its removal. In the .kmall format, these two + % values are found in the MRZ datagrams. + + % Receive beamwidth + Rx_BW = unique([pingInfo.receiveArraySizeUsed_deg]); + if numel(Rx_BW)>1 + comms.error('Found more than one Rx beamwidth records. Keeping only the first value'); + end + fData.Ru_1D_ReceiveBeamwidth = Rx_BW(1); + + % Transmit Power Re Maximum + Tx_pow = unique([pingInfo.transmitPower_dB]); + if numel(Tx_pow)>1 + comms.error('Found more than one Tx Power records. Keeping only the first value'); + end + fData.Ru_1D_TransmitPowerReMaximum = Tx_pow(1); + + % number of datagrams + nDatag = numel(cmnPart); + + % number of pings + dtg_pingCnt = [cmnPart.pingCnt]; % actual ping number for each datagram + dtg_swathAlongPosition = [cmnPart.swathAlongPosition]; % swath number for a ping + dtg_swathCnt = dtg_pingCnt + 0.01.*dtg_swathAlongPosition; % "new ping number" for each datagram + [swath_counter, iFirstDatagram, iC] = unique(dtg_swathCnt,'stable'); % list of swath numbers + nSwaths = numel(swath_counter); % total number of swaths in file + + % number of beams + dtg_nBeams = [rxInfo.numSoundingsMaxMain]; % number of beams per datagram ("main soundings" only. Ignoring "extra detections") + nBeams = arrayfun(@(idx) sum(dtg_nBeams(iC==idx)), 1:nSwaths); % total number of beams per swath + maxnBeams = nanmax(nBeams); % maximum number of "beams per swath" in the file + + % date and time + [dtg_date,dtg_TSMIM] = CFF_get_date_and_TSMIM_from_kmall_header(header); % date and time per datagram + fData.X8_1P_Date = dtg_date(iFirstDatagram); % date per swath + fData.X8_1P_TimeSinceMidnightInMilliseconds = dtg_TSMIM(iFirstDatagram); % time per swath + + % record data per ping + fData.X8_1P_PingCounter = NaN; % unused anyway + fData.X8_1P_HeadingOfVessel = NaN; % unused anyway + fData.X8_1P_SoundSpeedAtTransducer = NaN; % unused anyway + fData.X8_1P_TransmitTransducerDepth = NaN; % unused anyway + fData.X8_1P_NumberOfBeamsInDatagram = NaN; % unused anyway + fData.X8_1P_NumberOfValidDetections = NaN; % unused anyway + fData.X8_1P_SamplingFrequencyInHz = NaN; % unused anyway + + % initialize data per beam and ping + fData.X8_BP_DepthZ = nan(maxnBeams,nSwaths); + fData.X8_BP_AcrosstrackDistanceY = nan(maxnBeams,nSwaths); + fData.X8_BP_AlongtrackDistanceX = NaN; % unused anyway + fData.X8_BP_DetectionWindowLength = NaN; % unused anyway + fData.X8_BP_QualityFactor = NaN; % unused anyway + fData.X8_BP_BeamIncidenceAngleAdjustment = NaN; % unused anyway + fData.X8_BP_DetectionInformation = NaN; % unused anyway + fData.X8_BP_RealTimeCleaningInformation = NaN; % unused anyway + fData.X8_BP_ReflectivityBS = nan(maxnBeams,nSwaths); + fData.X8_B1_BeamNumber = (1:maxnBeams)'; + + % record data per beam and ping + for iS = 1:nSwaths + dtg_iS = find(iC==iS); % indices of datagrams for that swath + nB_tot = 0; % initialize total number of beams recorded so far for that swath + for iD = 1:numel(dtg_iS) + SD = sounding(dtg_iS(iD)); % soundings data for that datagram + nB = dtg_nBeams(dtg_iS(iD)); % number of actual beams in this datagram (ignoring "extra detections") + iB_dst = nB_tot + (1:nB); % indices of beams in output arrays + fData.X8_BP_DepthZ(iB_dst,iS) = SD.z_reRefPoint_m(1:nB); + fData.X8_BP_AcrosstrackDistanceY(iB_dst,iS) = SD.y_reRefPoint_m(1:nB); + fData.X8_BP_ReflectivityBS(iB_dst,iS) = SD.reflectivity1_dB(1:nB); + nB_tot = nB_tot + nB; % update total number of beams recorded so far for this swath + end + end + + % debug graph + debugDisp = 0; + if debugDisp + f = figure(); + ax_z = axes(f,'outerposition',[0 0.66 1 0.3]); + imagesc(ax_z, -fData.X8_BP_DepthZ); + colorbar(ax_z); grid on; title(ax_z, 'bathy'); colormap(ax_z,'jet'); + ax_y = axes(f,'outerposition',[0 0.33 1 0.3]); + imagesc(ax_y, fData.X8_BP_AcrosstrackDistanceY); + colorbar(ax_y); grid on; title(ax_y, 'across-track distance'); + ax_bs = axes(f,'outerposition',[0 0 1 0.3]); + imagesc(ax_bs, fData.X8_BP_ReflectivityBS); + caxis(ax_bs, [prctile(fData.X8_BP_ReflectivityBS(:),5), prctile(fData.X8_BP_ReflectivityBS(:),95)]); + colorbar(ax_bs); grid on; title(ax_bs, 'BS (scaled 5-95th percentile)'); colormap(ax_bs,'gray'); + drawnow; + end + + end + + %% '#MWC - Multibeam (M) water (W) column (C) datagram' + if isfield(KMALLdata,'EMdgmMWC') && ~isfield(fData,'WC_1D_Date') + + comms.step('Converting #MWC'); + + % remove duplicate datagrams + EMdgmMWC = CFF_remove_duplicate_KMALL_datagrams(KMALLdata.EMdgmMWC); + + % extract data + header = [EMdgmMWC.header]; + cmnPart = [EMdgmMWC.cmnPart]; + rxInfo = [EMdgmMWC.rxInfo]; + % CFF_get_kmall_TxRx_info(cmnPart) % evaluate to get info for debugging + + % number of datagrams + nDatag = numel(EMdgmMWC); + + % number of pings + dtg_pingCnt = [cmnPart.pingCnt]; % actual ping number for each datagram + dtg_swathAlongPosition = [cmnPart.swathAlongPosition]; % swath number for a ping + dtg_swathCnt = dtg_pingCnt + 0.01.*dtg_swathAlongPosition; % "new ping number" for each datagram + [swath_counter, iFirstDatagram, iC] = unique(dtg_swathCnt,'stable'); % list of swath numbers + nSwaths = numel(swath_counter); % total number of swaths in file + + % number of beams + dtg_nBeams = [rxInfo.numBeams]; % number of beams per datagram + nBeams = arrayfun(@(idx) sum(dtg_nBeams(iC==idx)), 1:nSwaths); % total number of beams per swath + maxnBeams = nanmax(nBeams); % maximum number of "beams per swath" in the file + maxnBeams_sub = ceil(maxnBeams/db_sub); % maximum number of beams TO READ per swath + + % number of samples + dtg_nSamples = arrayfun(@(idx) [EMdgmMWC(idx).beamData_p(:).numSampleData], 1:nDatag, 'UniformOutput', false); % number of samples per ping per datagram + [maxnSamples_groups, ping_group_start, ping_group_end] = CFF_group_pings(dtg_nSamples, swath_counter, dtg_swathCnt); % making groups of pings to limit size of memmaped files + maxnSamples_groups = ceil(maxnSamples_groups/dr_sub); % maximum number of samples TO READ, per group. + + % add the WCD decimation factors given here in input + fData.dr_sub = dr_sub; + fData.db_sub = db_sub; + + % data per ping + % here taken from first datagram. Ideally, check consistency + % between datagrams for a given ping + [dtg_date,dtg_TSMIM] = CFF_get_date_and_TSMIM_from_kmall_header(header); % date and time per datagram + fData.WC_1P_Date = dtg_date(iFirstDatagram); % date per swath + fData.WC_1P_TimeSinceMidnightInMilliseconds = dtg_TSMIM(iFirstDatagram); % time per swath + fData.WC_1P_PingCounter = swath_counter; + fData.WC_1P_NumberOfDatagrams = NaN; % unused anyway + fData.WC_1P_NumberOfTransmitSectors = NaN; % unused anyway + fData.WC_1P_TotalNumberOfReceiveBeams = NaN; % unused anyway + fData.WC_1P_SoundSpeed = [rxInfo(iFirstDatagram).soundVelocity_mPerSec]; + fData.WC_1P_SamplingFrequencyHz = [rxInfo(iFirstDatagram).sampleFreq_Hz]; + fData.WC_1P_TXTimeHeave = NaN; % unused anyway + fData.WC_1P_TVGFunctionApplied = [rxInfo(iFirstDatagram).TVGfunctionApplied]; + fData.WC_1P_TVGOffset = [rxInfo(iFirstDatagram).TVGoffset_dB]; + fData.WC_1P_ScanningInfo = NaN; % unused anyway + + % data per transmit sector and ping + fData.WC_TP_TiltAngle = NaN; % unused anyway + fData.WC_TP_CenterFrequency = NaN; % unused anyway + fData.WC_TP_TransmitSectorNumber = NaN; % unused anyway + + % initialize data per (decimated) beam and ping + fData.WC_BP_BeamPointingAngle = nan(maxnBeams_sub,nSwaths); + fData.WC_BP_StartRangeSampleNumber = nan(maxnBeams_sub,nSwaths); + fData.WC_BP_NumberOfSamples = nan(maxnBeams_sub,nSwaths); + fData.WC_BP_DetectedRangeInSamples = zeros(maxnBeams_sub,nSwaths); + fData.WC_BP_TransmitSectorNumber = NaN; % unused anyway + fData.WC_BP_BeamNumber = NaN; % unused anyway + fData.WC_BP_SystemSerialNumber = NaN; % unused anyway + + % The actual water-column data will not be saved in fData but in + % binary files. Get the output directory to store those files + wc_dir = CFF_converted_data_folder(KMALLfilename); + + % Clean up that folder first before adding anything to it + CFF_clean_delete_fdata(wc_dir); + + % DEV NOTE: Info format for raw WC data and storage + % In these raw datagrams, you have both amplitude and phase. + % + % Amplitude samples are recorded exactly as in the .all format, + % that is in "int8" (signed integers from -128 to 127) with -128 + % being the NaN value. Raw values needs to be multiplied by a + % factor of 1/2 to retrieve the true value, aka real values go from + % -127/2 = -63.5 dB to 127/2 = 63.5 dB in increments of 0.5 dB + % For storage, we keep the same format in order to save disk space. + % + % Phase might or might not be recorded, and depending on the value + % of the flag may be recorded in 'int8' with a factor of 180./128, + % or in 'int16' with a factor of 0.01. + % + % For storage, we keep the same format in order to save disk space. + + % initialize data-holding binary files for Amplitude + fData = CFF_init_memmapfiles(fData, ... + 'field', 'WC_SBP_SampleAmplitudes', ... + 'wc_dir', wc_dir, ... + 'Class', 'int8', ... + 'Factor', 1./2, ... + 'Nanval', intmin('int8'), ... + 'Offset', 0, ... + 'MaxSamples', maxnSamples_groups, ... + 'MaxBeams', maxnBeams_sub, ... + 'ping_group_start', ping_group_start, ... + 'ping_group_end', ping_group_end); + + % was phase recorded + dtg_phaseFlag = [rxInfo.phaseFlag]; + if all(dtg_phaseFlag==0) + phaseFlag = 0; + elseif all(dtg_phaseFlag==1) + phaseFlag = 1; + elseif all(dtg_phaseFlag==2) + phaseFlag = 2; + else + % hopefully this error should never occur. Otherwise it's + % fixable but have to change the code a bit. + error('phase flag is inconsistent across ping records in this file.') + end + + % record phase data, if available + if phaseFlag + + % two different formats for raw phase, depending on the value + % of the flag. Keep the same for storage + if phaseFlag==1 + phaseFormat = 'int8'; + phaseFactor = 180./128; + else + phaseFormat = 'int16'; + phaseFactor = 0.01; + end + phaseNanValue = intmin(phaseFormat); + + % initialize data-holding binary files for Phase + fData = CFF_init_memmapfiles(fData, ... + 'field', 'WC_SBP_SamplePhase', ... + 'wc_dir', wc_dir, ... + 'Class', phaseFormat, ... + 'Factor', phaseFactor, ... + 'Nanval', phaseNanValue, ... + 'Offset', 0, ... + 'MaxSamples', maxnSamples_groups, ... + 'MaxBeams', maxnBeams_sub, ... + 'ping_group_start', ping_group_start, ... + 'ping_group_end', ping_group_end); + + end + + % Also the samples data were not recorded, only their location in + % the source file, so we need to fopen the source file to grab the + % data. + fid = fopen(KMALLfilename,'r','l'); + + % debug graph + debugDisp = 0; + if debugDisp + f = figure(); + if ~phaseFlag + ax_mag = axes(f,'outerposition',[0 0 1 1]); + title('WCD amplitude'); + else + ax_mag = axes(f,'outerposition',[0 0.5 1 0.5]); + title('WCD amplitude'); + ax_phase = axes(f,'outerposition',[0 0 1 0.5]); + title('WCD phase'); + end + end + + % initialize ping group number + iG = 1; + + % now get data for each swath... + for iS = 1:nSwaths + + % ping group number is the index of the memmaped file in which + % that swath's data will be saved. + if iS > ping_group_end(iG) + iG = iG+1; + end + + % (re-)initialize amplitude and phase arrays for that swath + Mag_tmp = intmin('int8').*ones(maxnSamples_groups(iG),maxnBeams_sub,'int8'); + if phaseFlag + Ph_tmp = phaseNanValue.*ones(maxnSamples_groups(iG),maxnBeams_sub,phaseFormat); + end + + % data for one swath can be spread over several datagrams, + % typically when using dual Rx systems, so we're going to loop + % over all datagrams to grab this swath's entire data + dtg_iS = find(iC==iS); % indices of datagrams for that swath + nB_tot = 0; % initialize total number of beams recorded so far for that swath + iB_src_start = 1; % index of first beam to read in a datagram, start with 1 and to be updated later + + % in each datagram... + for iD = 1:numel(dtg_iS) + + % beamData_p for this datagram + BD = EMdgmMWC(dtg_iS(iD)).beamData_p; + + % important variables for data to grab + nRx = numel(BD.beamPointAngReVertical_deg); % total number of beams in this datagram + iB_src = iB_src_start:db_sub:nRx; % indices of beams to read in this datagram + nB = numel(iB_src); % number of beams to record from this datagram + iB_dst = nB_tot + (1:nB); % indices of those beams in output arrays + + % data per beam + fData.WC_BP_BeamPointingAngle(iB_dst,iS) = BD.beamPointAngReVertical_deg(iB_src); + fData.WC_BP_StartRangeSampleNumber(iB_dst,iS) = BD.startRangeSampleNum(iB_src); + fData.WC_BP_NumberOfSamples(iB_dst,iS) = BD.numSampleData(iB_src); + fData.WC_BP_DetectedRangeInSamples(iB_dst,iS) = BD.detectedRangeInSamplesHighResolution(iB_src); + + % in each beam... + for iB = 1:nB + + % data size + sR = BD.startRangeSampleNum(iB_src(iB)); % start range sample number + nS = BD.numSampleData(iB_src(iB)); % number of samples in this beam + nS_sub = ceil(nS/dr_sub); % number of samples we're going to record + + % get to start of amplitude block + dpif = BD.sampleDataPositionInFile(iB_src(iB)); + fseek(fid,dpif,-1); + + % amplitude block is nS records of 1 byte each. + Mag_tmp(sR+1:sR+nS_sub,iB_dst(iB)) = fread(fid, nS_sub, 'int8=>int8',dr_sub-1); % read with decimation + + if phaseFlag + % go to start of phase block + fseek(fid,dpif+nS,-1); + + if phaseFlag == 1 + % phase block is nS records of 1 byte each. + Ph_tmp(sR+1:sR+nS_sub,iB_dst(iB)) = fread(fid, nS_sub, 'int8=>int8',dr_sub-1); % read with decimation + else + % phase block is nS records of 2 bytes each. + % XXX1 case not tested yet. Find suitable data + % files + Ph_tmp(sR+1:sR+nS_sub,iB_dst(iB)) = fread(fid, nS_sub, 'int16=>int16',2*dr_sub-2); % read with decimation + end + end + end + + % update variables before reading next datagram, if + % necessary + nB_tot = nB_tot + nB; % total number of beams recorded so far for this swath + iB_src_start = iB_src(end) - nRx + db_sub; % index of first beam to read in next datagram + + end + + % debug graph + if debugDisp + % display amplitude + imagesc(ax_mag,double(Mag_tmp)./2); + colorbar(ax_mag) + title(ax_mag, sprintf('Ping %i/%i, WCD amplitude',iS,nSwaths)); + % display phase + if phaseFlag + imagesc(ax_phase,double(Ph_tmp).*phaseFactor); + colorbar(ax_phase) + title(ax_phase, 'WCD phase'); + end + drawnow; + end + + % finished reading this swath's WC data. Store the data in the + % appropriate binary file, at the appropriate ping, through the + % memory mapping + fData.WC_SBP_SampleAmplitudes{iG}.Data.val(:,:,iS-ping_group_start(iG)+1) = Mag_tmp; + if phaseFlag + fData.WC_SBP_SamplePhase{iG}.Data.val(:,:,iS-ping_group_start(iG)+1) = Ph_tmp; + end + + end + + % close the original raw file + fclose(fid); + + end + + %% '#SPO - Sensor (S) data for position (PO)' + if isfield(KMALLdata,'EMdgmSPO') && ~isfield(fData,'Po_1D_Date') + + comms.step('Converting #SPO'); + + % extract data + header = [KMALLdata.EMdgmSPO.header]; + sensorData = [KMALLdata.EMdgmSPO.sensorData]; + + % DEV NOTE: There are many entries here but I found a lot of issues + % in the heading. Digging in the data revealed that many successive + % entries have same values of timeFromSensor_sec, latitude, + % longitude, and other values. Yet speed and heading change with + % every entry... and heading has errors. I suspect those two values + % are (badly) calculated fromt the lat/long. So now we only record + % one entry per unique time stamp. Note the time stamp is in + % seconds so no more than one record per second. The nanosecond + % field is wrong. Alex 12 july 2021 + + % get unique time entries from sensorData + % idx_t = 1:numel(sensorData); % for test to store all + idx_t = [1, find([0, diff([sensorData.timeFromSensor_sec])]~=0)]; + + % number of entries to record + nD = numel(idx_t); + + % get date and time-since-midnight-in-milleseconds from header + [dtg_date,dtg_TSMIM] = CFF_get_date_and_TSMIM_from_kmall_header(header); % date and time per datagram + fData.Po_1D_Date = dtg_date(idx_t); + fData.Po_1D_TimeSinceMidnightInMilliseconds = dtg_TSMIM(idx_t); + + fData.Po_1D_Latitude = [sensorData(idx_t).correctedLat_deg]; % in decimal degrees + fData.Po_1D_Longitude = [sensorData(idx_t).correctedLong_deg]; % in decimal degrees + fData.Po_1D_SpeedOfVesselOverGround = [sensorData(idx_t).speedOverGround_mPerSec]; % in m/s + fData.Po_1D_HeadingOfVessel = [sensorData(idx_t).courseOverGround_deg]; % in degrees relative to north + fData.Po_1D_MeasureOfPositionFixQuality = [sensorData(idx_t).posFixQuality_m]; + fData.Po_1D_PositionSystemDescriptor = zeros(1,nD); % dummy values + + end + + %% communicate progress + comms.progress(iF,nStruct); + +end + + +%% end message +comms.finish('Done'); + + +%% nested functions + + function out_EM_struct = CFF_remove_duplicate_KMALL_datagrams(in_EM_struct) + % DEV NOTE: I have found occurences of duplicate MRZ datagrams in some test + % files. Not sure how common it is, but the conversion code ends up + % duplicating the data too. Instead of modifying the code everywhere to be + % always considering the possibility of duplicates, it's easier to look for + % them at the start and remove them before parsing. + % In the examples I found, it would be sufficient to check for the set + % unicity of the cmnPart fields pingCnt, rxFanIndex, and + % swathAlongPosition. But the range of cases covered by the kmall format + % can be complicated (including systems with dual Rx heads and dual Tx + % heads in multi-swath mode!!! see documentation of EMdgmMbody_def) so to + % be entierely safe, we will instead use ALL the fields of cmnPart in a + % test for set unicity. + + if isfield(in_EM_struct, 'cmnPart') + + cmnPart_table = struct2table([in_EM_struct.cmnPart]); + [~, ia, ~] = unique(cmnPart_table,'rows', 'stable'); + idx_duplicates = ~ismember(1:size(cmnPart_table,1), ia); + + % remove duplicates + out_EM_struct = in_EM_struct(~idx_duplicates); + + % info + if any(idx_duplicates) + infoStr = sprintf('Found and discarded %i duplicate datagrams',sum(idx_duplicates)); + comms.info(infoStr); + end + + else + out_EM_struct = in_EM_struct; + end + + end + +end + + +%% +function out_struct = CFF_read_TRAI(ASCIIdata, TRAI_code) + +out_struct = struct; + +[iS,iE] = regexp(ASCIIdata,[TRAI_code ':.+?,']); + +if isempty(iS) + % no match, exit + return +end + +TRAI_TX1_ASCII = ASCIIdata(iS+9:iE-1); + +yo(:,1) = [1; strfind(TRAI_TX1_ASCII,';')'+1]; % beginning of ASCII field name +yo(:,2) = strfind(TRAI_TX1_ASCII,'=')'-1; % end of ASCII field name +yo(:,3) = strfind(TRAI_TX1_ASCII,'=')'+1; % beginning of ASCII field value +yo(:,4) = [strfind(TRAI_TX1_ASCII,';')'-1;length(TRAI_TX1_ASCII)]; % end of ASCII field value + +for ii = 1:size(yo,1) + + % get field string + field = TRAI_TX1_ASCII(yo(ii,1):yo(ii,2)); + + % try turn value into numeric + value = str2double(TRAI_TX1_ASCII(yo(ii,3):yo(ii,4))); + if length(value)~=1 + % looks like it cant. Keep as string + value = TRAI_TX1_ASCII(yo(ii,3):yo(ii,4)); + end + + % store field/value + out_struct.(field) = value; + +end + +end + + +%% +function [KM_date, TSMIM] = CFF_get_date_and_TSMIM_from_kmall_header(header) + +% get values +time_sec = [header.time_sec]; +time_nanosec = [header.time_nanosec]; + +% convert raw to datetime +dt = datetime(time_sec + time_nanosec.*10^-9,'ConvertFrom','posixtime'); + +% convert datetime to date and TSMIM +KM_date = convertTo(dt, 'yyyymmdd'); +TSMIM = milliseconds(timeofday(dt)); + +end + +%% +function str = CFF_get_kmall_TxRx_info(cmnPart) + +% Single or Dual Tx +iTx = unique([cmnPart.txTransducerInd]); +if numel(iTx)==1 && iTx==0 + str = 'Single Tx, '; +elseif numel(iTx)==2 && all(iTx==[0,1]) + str = 'Dual Tx, '; +else + str = '??? Tx, '; +end + +% Single or Dual Rx +iRx = unique([cmnPart.numRxTransducers]); +iRx2 = unique([cmnPart.rxTransducerInd]); +if numel(iRx)==1 && iRx==1 + % should be single + if numel(iRx2)==1 && iRx2==0 + % confirmed + str = [str, 'Single Rx, ']; + else + % unconfirmed + str = [str, '??? Rx, ']; + end +elseif numel(iRx)==1 && iRx==2 + % should be dual + if numel(iRx2)==2 && all(iRx2==[0,1]) + % confirmed + str = [str, 'Dual Rx, ']; + else + % unconfirmed + str = [str, '??? Rx, ']; + end +else + str = [str, '??? Rx, ']; +end + +% Single or Multi Swath +iSw = unique([cmnPart.swathsPerPing]); +iSw2 = unique([cmnPart.swathAlongPosition]); +if numel(iSw)==1 && iSw==1 + % should be single + if numel(iSw2)==1 && iSw2==0 + % confirmed + str = [str, 'Single Swath.']; + else + % unconfirmed + str = [str, '??? Swath.']; + end +elseif numel(iSw)==1 && iSw==2 + % should be dual + if numel(iSw2)==2 && all(iSw2==[0,1]) + % confirmed + str = [str, 'Dual Swath.']; + else + % unconfirmed + str = [str, '??? Swath.']; + end +elseif numel(iSw)==1 && iSw>2 + % should be multi (more than 2) + if numel(iSw2)==iSw && all(iSw2==[0:iSw-1]) + % confirmed + str = [str, sprintf('Multi Swath (%i).',iSw)]; + else + % unconfirmed + str = [str, '??? Swath.']; + end +else + str = [str, '??? Swath.']; +end + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/CFF_get_kmall_revision.m b/read_data_files/Kongsberg/format_kmall/CFF_get_kmall_revision.m new file mode 100644 index 0000000..8c33d4f --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/CFF_get_kmall_revision.m @@ -0,0 +1,12 @@ +function kmallRev = CFF_get_kmall_revision(EMdgmIIP) +%CFF_GET_KMALL_REVISION Get the KMALL format revision (single letter) +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +idx = strfind(EMdgmIIP.install_txt,'KMALL:Rev ') + 10; +kmallRev = EMdgmIIP.install_txt(idx); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/CFF_is_valid_kmall_format_extension.m b/read_data_files/Kongsberg/format_kmall/CFF_is_valid_kmall_format_extension.m new file mode 100644 index 0000000..4b1115e --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/CFF_is_valid_kmall_format_extension.m @@ -0,0 +1,19 @@ +function bool = CFF_is_valid_kmall_format_extension(file) +%CFF_IS_VALID_KMALL_FORMAT_EXTENSION Check file extension is kmall/wcd +% +% Tests if input file (string, or cell array of strings) has '.kmall', +% '.KMALL', '.kmwcd' or '.KMWCD' extension +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +if ischar(file) + file = {file}; +end + +% function checking if extension is Kongsberg's +isK = @(x) any(strcmp(CFF_file_extension(x),{'.kmall','.KMALL','.kmwcd','.KMWCD'})); + +bool = cellfun(isK,file); diff --git a/read_data_files/Kongsberg/format_kmall/CFF_kmall_file_info.m b/read_data_files/Kongsberg/format_kmall/CFF_kmall_file_info.m new file mode 100644 index 0000000..1c818ac --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/CFF_kmall_file_info.m @@ -0,0 +1,280 @@ +function KMALLfileinfo = CFF_kmall_file_info(KMALLfilename, varargin) +%CFF_KMALL_FILE_INFO Records basic info about contents of .kmall file +% +% Records basic info about the datagrams contained in one Kongsberg EM +% series binary data file in .kmall format (.kmall or .kmwcd). +% +% KMALLfileinfo = CFF_KMALL_FILE_INFO(KMALLfilename) opens file +% KMALLfilename and reads through the start of each datagram to get basic +% information about it, and store it all in KMALLfileinfo. +% +% *INPUT VARIABLES* +% * |KMALLfilename|: Required. String filename to parse (extension in +% .kmall or .kmwcd) +% +% *OUTPUT VARIABLES* +% * |KMALLfileinfo|: structure containing information about datagrams in +% KMALLfilename, with fields: +% * |file_name|: input file name +% * |fileSize|: file size in bytes +% * |dgm_num|: number of datagram in file +% * |dgm_type_code|: datagram type as string, e.g. '#IIP' (Kongsberg +% .kmall format) +% * |dgm_type_text|: datagram type description (Kongsberg .kmall +% format) +% * |dgm_type_version|: version for this type of datagram, as int +% (Kongsberg .kmall format) +% * |dgm_counter|: counter for this type and version of datagram +% in the file. There should not be multiple versions of a same type in +% a same file, but we never know... +% * |dgm_start_pif|: position of beginning of datagram in +% file +% * |dgm_size|: datagram size in bytes +% * |dgm_sys_ID|: System ID. Parameter used for separating datagrams +% from different echosounders. +% * |dgm_EM_ID|: Echo sounder identity, e.g. 124, 304, 712, 2040, +% 2045 (EM 2040C) +% * |sync_counter|: number of bytes found between this datagram and the +% previous one (any number different than zero indicates a sync error) +% * |date_time|: datagram date in datetime format +% * |parsed|: flag for whether the datagram has been parsed. Initiated +% at 0 at this stage. To be later turned to 1 for parsing. +% +% See also CFF_ALL_FILE_INFO, CFF_S7K_FILE_INFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + + +%% Input arguments management +p = inputParser; + +% name of the .kmall or .kmwcd file +argName = 'KMALLfilename'; +argCheck = @(x) CFF_check_KMALLfilename(x); +addRequired(p,argName,argCheck); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,KMALLfilename,varargin{:}); + +% and get results +KMALLfilename = p.Results.KMALLfilename; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end + + +%% Start message +filename = CFF_file_name(KMALLfilename,1); +comms.start(sprintf('Listing datagrams in file %s',filename)); + + +%% Open file and initializing +% kmall files are in little Endian, but just use the default with fopen +[fid,~] = fopen(KMALLfilename, 'r'); + +% go to end of file to get number of bytes in file then rewind +fseek(fid,0,1); +fileSize = ftell(fid); +fseek(fid,0,-1); + +% init output info +KMALLfileinfo.fileName = KMALLfilename; +KMALLfileinfo.fileSize = fileSize; + +% initialize list of datagram types and counter +list_dgmType = {}; +list_dgmType_counter = []; + +% intitializing the counter of datagrams in this file +kk = 0; + +% initializing synchronization counter: the number of bytes that needed to +% be passed before this datagram appeared +sync_counter = 0; + + +%% Start progress +comms.progress(0,fileSize); + + +%% Reading datagrams +next_dgm_start_pif = 0; +while next_dgm_start_pif < fileSize + + %% new datagram begins + dgm_start_pif = ftell(fid); + + % A full kmall datagram is organized as a sequence of: + % * GH - General Header EMdgmHeader (20 bytes, at least for Rev H) + % * DB - Datagram Body (variable size) + % * DS - Datagram size (uint32, aka 4 bytes) + % + % The General Header is read here. It starts with: + % * The datagram size (uint32, aka 4 bytes) + % * The datagram type code (4 char, aka 4 bytes), e.g. '#IIP' + % + % We will test for both datagram completeness and sync by matching the + % two datagram size fields, and checking for the hash symbol at the + % beggining of the datagram type code. + + % parsing general header + header = CFF_read_EMdgmHeader(fid); + + % pif of presumed end of datagram + dgm_end_pif = dgm_start_pif + header.numBytesDgm - 4; + + % get the repeat fileSize at the end of the datagram + if dgm_end_pif < fileSize + fseek(fid, dgm_end_pif, -1); + numBytesDgm_repeat = fread(fid,1,'uint32'); % Datagram length in bytes + next_dgm_start_pif = ftell(fid); + else + % Being here can be due to two things: + % 1) we are in sync but this datagram is incomplete, or + % 2) we are out of sync. + numBytesDgm_repeat = -1; + end + + + %% test for synchronization + % check for matching datagram size, amd the hash symbol of datagram + % type code. + flag_numBytesDgm_match = (header.numBytesDgm == numBytesDgm_repeat); + flag_hash = strcmp(header.dgmType(1), '#'); + if ~flag_numBytesDgm_match || ~flag_hash + % NOT SYNCHRONIZED + % We've either lost sync, or the last datagram is incomplete. + % Go back to new record start, advance one byte, and restart + % reading + fseek(fid, dgm_start_pif+1, -1); + next_dgm_start_pif = -1; + sync_counter = sync_counter+1; % update sync counter + if sync_counter == 1 + % just lost sync, throw a message just now + comms.error('Lost sync while reading datagrams. A datagram may be corrupted. Trying to resync...'); + end + continue + else + % SYNCHRONIZED + if sync_counter + % if we had lost sync, warn here we're back + comms.info(sprintf('Back in sync (%i bytes later). Resume process.',sync_counter)); + % reinitialize sync counter + sync_counter = 0; + end + end + + + %% datagram type counter + + % index of datagram type in the list + idx_dgmType = find(cellfun(@(x) strcmp(header.dgmType,x), list_dgmType)); + + % if type encountered for the first time, add it to the list and + % initialize counter + if isempty(idx_dgmType) + idx_dgmType = numel(list_dgmType) + 1; + list_dgmType{idx_dgmType,1} = header.dgmType; + list_dgmType_counter(idx_dgmType,1) = 0; + end + + % increment counter + list_dgmType_counter(idx_dgmType) = list_dgmType_counter(idx_dgmType) + 1; + + + %% write output KMALLfileinfo + + % Datagram complete + kk = kk + 1; + + % Datagram number in file + KMALLfileinfo.dgm_num(kk,1) = kk; + + % Datagram info + KMALLfileinfo.dgm_type_code{kk,1} = header.dgmType; + KMALLfileinfo.dgm_type_text{kk,1} = get_dgm_type_txt(header.dgmType); + KMALLfileinfo.dgm_type_version(kk,1) = header.dgmVersion; + KMALLfileinfo.dgm_counter(kk,1) = list_dgmType_counter(idx_dgmType); + KMALLfileinfo.dgm_start_pif(kk,1) = dgm_start_pif; + KMALLfileinfo.dgm_size(kk,1) = header.numBytesDgm; + + % System info + KMALLfileinfo.dgm_sys_ID(kk,1) = header.systemID; + KMALLfileinfo.dgm_EM_ID(kk,1) = header.echoSounderID; + + % Time info + KMALLfileinfo.date_time(kk,1) = datetime(header.time_sec + header.time_nanosec.*10^-9,'ConvertFrom','posixtime'); + + % Report any sync issue in reading + KMALLfileinfo.sync_counter(kk,1) = sync_counter; + + + %% prepare for reloop + + % go to end of datagram + fseek(fid, next_dgm_start_pif, -1); + + % communicate progress + comms.progress(next_dgm_start_pif,fileSize); +end + + +%% finalizing + +% adding lists +KMALLfileinfo.list_dgm_type = list_dgmType; +KMALLfileinfo.list_dgm_counter = list_dgmType_counter; + +% initialize parsing field +KMALLfileinfo.parsed = zeros(size(KMALLfileinfo.dgm_num)); + +% closing file +fclose(fid); + +% end message +comms.finish('Done'); + +end + + +%% subfunctions + +%% +function dgm_type_text = get_dgm_type_txt(dgm_type_code) + +list_dgm_type_text = {... + '#IIP - Installation parameters and sensor setup';... + '#IOP - Runtime parameters as chosen by operator';... + '#IBE - Built in test (BIST) error report';... + '#IBR - Built in test (BIST) reply';... + '#IBS - Built in test (BIST) short reply';... + '#MRZ - Multibeam (M) raw range (R) and depth(Z) datagram';... + '#MWC - Multibeam (M) water (W) column (C) datagram';... + '#SPO - Sensor (S) data for position (PO)';... + '#SKM - Sensor (S) KM binary sensor format';... + '#SVP - Sensor (S) data from sound velocity (V) profile (P) or CTD';... + '#SVT - Sensor (S) data for sound velocity (V) at transducer (T)';... + '#SCL - Sensor (S) data from clock (CL)';... + '#SDE - Sensor (S) data from depth (DE) sensor';... + '#SHI - Sensor (S) data for height (HI)';... + '#CPO - Compatibility (C) data for position (PO)';... + '#CHE - Compatibility (C) data for heave (HE)';... + '#FCF - Backscatter calibration (C) file (F) datagram' ... + }; + +idx = find(cellfun(@(x) strcmp(x(1:4),dgm_type_code), list_dgm_type_text)); + +if ~isempty(idx) + dgm_type_text = list_dgm_type_text{idx}; +else + dgm_type_text = sprintf('%i - UNKNOWN DATAGRAM TYPE',dgm_type_code); +end + +end diff --git a/read_data_files/Kongsberg/format_kmall/CFF_read_kmall.m b/read_data_files/Kongsberg/format_kmall/CFF_read_kmall.m new file mode 100644 index 0000000..d80599b --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/CFF_read_kmall.m @@ -0,0 +1,214 @@ +function [KMALLdata,datagrams_parsed_idx] = CFF_read_kmall(KMALLfilename,varargin) +%CFF_READ_KMALL Read kmall file or pair of files +% +% Reads contents of one Kongsberg EM series binary data file in .kmall +% format (.kmall or .kmwcd), or a pair of .kmall/.kmwcd files, allowing +% choice on which type of datagrams to parse. +% +% KMALLdata = CFF_READ_KMALL(KMALLfilename) reads all datagrams in a +% Kongsberg file (extension .kmall or .kmwcd) KMALLfilenamem, and store +% them in KMALLdata. +% +% KMALLdata = CFF_READ_KMALL(KMALLfilename,datagrams) reads only those +% datagrams in KMALLfilename that are specified by datagrams, and store +% them in KMALLdata. +% +% KMALLdata = CFF_READ_KMALL(KMALLfilename,'datagrams',datagrams) does +% the same. +% +% Note this function will extract all datagram types of interest. For +% more control (say you only want the first ten depth datagrams and the +% last position datagram), use CFF_READ_KMALL_FROM_FILEINFO. +% +% See also CFF_KMALL_FILE_INFO, CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + + +%% Input arguments management +p = inputParser; + +% name of the .kmall or .kmwcd file (or pair) +argName = 'KMALLfilename'; +argCheck = @(x) CFF_check_KMALLfilename(x); +addRequired(p,argName,argCheck); + +% types of datagram to read +argName = 'datagrams'; +argDefault = []; +argCheck = @(x) isnumeric(x)||iscell(x)||(ischar(x)&&~strcmp(x,'datagrams')); % that last part allows the use of the couple name,param +addOptional(p,argName,argDefault,argCheck); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,KMALLfilename,varargin{:}); + +% and get results +KMALLfilename = p.Results.KMALLfilename; +datagrams_to_parse = p.Results.datagrams; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end +clear p; + +% check input +if ischar(KMALLfilename) + % single file .kmall OR .kmwcd. Convert filename to cell. + KMALLfilename = {KMALLfilename}; +else + % matching file pair .kmall AND .kmwcd. + % make sure .kmwcd is listed first because this function only reads in + % the 2nd file what it could not find in the first, and we want to only + % grab from the .kmall file what is needed and couldn't be found in the + % .kmwcd file. + if strcmpi(CFF_file_extension(KMALLfilename{1}),'.kmall') + KMALLfilename = fliplr(KMALLfilename); + end +end + + +%% Prep + +% number of files +nFiles = numel(KMALLfilename); + +% start message +filename = CFF_file_name(KMALLfilename{1},1); +if nFiles == 1 + comms.start(sprintf('Reading data in file %s',filename)); +else + filename_2_ext = CFF_file_extension(KMALLfilename{2}); + comms.start(sprintf('Reading data in pair of files %s and %s',filename,filename_2_ext)); +end + +% start progress +comms.progress(0,nFiles); + + +%% FIRST FILE + +% Get info from first (or only) file +if nFiles == 1 + comms.step('Listing datagrams'); +else + comms.step('Listing datagrams in paired file #1/2'); +end +info = CFF_kmall_file_info(KMALLfilename{1}); + +% communicate progress +comms.progress(0.5,nFiles); + +if isempty(datagrams_to_parse) + % parse all datagrams in first file + + info.parsed(:) = 1; + datagrams_parsed_in_first_file = unique(info.dgm_type_code); + datagrams_parsed_idx = []; + +else + % datagrams to parse are listed in input + + if ischar(datagrams_to_parse) + datagrams_to_parse = {datagrams_to_parse}; + end + + % datagrams available + datagrams_available = unique(info.dgm_type_code); + + % find which datagrams can be read here + datagrams_parsable_idx = ismember(datagrams_to_parse,datagrams_available); + + % if any, read those datagrams + if any(datagrams_parsable_idx) + idx = ismember(info.dgm_type_code,datagrams_to_parse(datagrams_parsable_idx)); + info.parsed(idx) = 1; + end + datagrams_parsed_idx = datagrams_parsable_idx; + +end + +% read data +if nFiles == 1 + comms.step('Reading datagrams'); +else + comms.step('Reading datagrams in paired file #1/2'); +end +KMALLdata = CFF_read_kmall_from_fileinfo(KMALLfilename{1}, info); + +% communicate progress +comms.progress(1,nFiles); + + +%% SECOND FILE (if any) +if nFiles>1 + + % parse only if we requested to read all datagrams (in which case, the + % second file might have datagrams not read in the first and we need to + % grab those) OR if we requested a specific set of datagrams and didn't + % get them all from the first file. + if isempty(datagrams_to_parse) || ~all(datagrams_parsed_idx) + + % Get info in second file + comms.step('Listing datagrams in paired file #2/2'); + info = CFF_kmall_file_info(KMALLfilename{2}); + + % communicate progress + comms.progress(1.5,nFiles); + + if isempty(datagrams_to_parse) + % parse all datagrams in second file which we didn't get in the + % first one. + + % datagrams in second file + datagrams_available_in_second_file = unique(info.dgm_type_code); + + % those in second file that were not in first + datagrams_to_parse_in_second_file = setdiff(datagrams_available_in_second_file,datagrams_parsed_in_first_file); + + % parse those + idx = ismember(info.dgm_type_code,datagrams_to_parse_in_second_file); + info.parsed(idx) = 1; + + % for output + datagrams_parsed_idx = []; + + else + % datagrams to parse are listed + + datagrams_available_in_second_file = unique(info.dgm_type_code); + + % find which remaining datagram types can be read here + datagrams_to_parse_in_second_file_idx = ismember(datagrams_to_parse,datagrams_available_in_second_file) & ~datagrams_parsed_idx; + + % if any, read those datagrams + if any(datagrams_to_parse_in_second_file_idx) + idx = ismember(info.dgm_type_code,datagrams_to_parse(datagrams_to_parse_in_second_file_idx)); + info.parsed(idx) = 1; + end + datagrams_parsed_idx = datagrams_parsed_idx | datagrams_to_parse_in_second_file_idx; + + end + + % read data in second file + comms.step('Reading datagrams in paired file #2/2'); + KMALLdata2 = CFF_read_kmall_from_fileinfo(KMALLfilename{2}, info); + + % combine to data from first file + KMALLdata = {KMALLdata KMALLdata2}; + + end + + % communicate progress + comms.progress(2,nFiles); + +end + + +%% end message +comms.finish('Done'); diff --git a/read_data_files/Kongsberg/format_kmall/CFF_read_kmall_from_fileinfo.m b/read_data_files/Kongsberg/format_kmall/CFF_read_kmall_from_fileinfo.m new file mode 100644 index 0000000..015bc23 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/CFF_read_kmall_from_fileinfo.m @@ -0,0 +1,516 @@ +function KMALLdata = CFF_read_kmall_from_fileinfo(KMALLfilename,KMALLfileinfo,varargin) +%CFF_READ_KMALL_FROM_FILEINFO Read contents of kmall file +% +% Reads contents of one Kongsberg EM series binary data file in .kmall +% format (.kmall or .kmwcd), using KMALLfileinfo to indicate which +% datagrams to be parsed. +% +% KMALLdata = CFF_READ_KMALL_FROM_FILEINFO(KMALLfilename, KMALLfileinfo) +% reads all datagrams in KMALLfilename for which KMALLfileinfo.parsed +% equals 1, and store them in KMALLdata. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +global DEBUG; + + +%% HARD-CODED PARAMETERS + +% This code was developped around the following kmall format revisions. To +% update if you verify it works with other revisions. +kmallRevSupported = 'F,H,I'; + + +%% Input arguments management +p = inputParser; + +% name of the .kmall or .kmwcd file +argName = 'KMALLfilename'; +argCheck = @(x) CFF_check_KMALLfilename(x); +addRequired(p,argName,argCheck); + +% fileinfo from CFF_KMALL_FILE_INFO containing indexes of datagrams to read +argName = 'KMALLfileinfo'; +argCheck = @isstruct; +addRequired(p,argName,argCheck); + +% ?? XXX3 +argName = 'OutputFields'; +argCheck = @iscell; +addParameter(p,argName,{},argCheck); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,KMALLfilename,KMALLfileinfo,varargin{:}); + +% and get results +KMALLfilename = p.Results.KMALLfilename; +KMALLfileinfo = p.Results.KMALLfileinfo; +OutputFields = p.Results.OutputFields; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end + + +%% Prep + +% start message +filename = CFF_file_name(KMALLfilename,1); +comms.start(sprintf('Reading datagrams in file %s',filename)); + +% store filename +KMALLdata.KMALLfilename = KMALLfilename; + +% open file +[fid,~] = fopen(KMALLfilename, 'r'); + +% parse only datagrams indicated in KMALLfileinfo +datagToParse = find(KMALLfileinfo.parsed==1); +nDatagsToPars = numel(datagToParse); + +% flag so kmall revision warning only goes off once +kmallRevWarningFlag = 0; + +% start progress +comms.progress(0,nDatagsToPars); + + +%% Reading datagrams +for iDatag = datagToParse' + + % A full kmall datagram is organized as a sequence of: + % * GH - General Header EMdgmHeader (20 bytes, at least for Rev H) + % * DB - Datagram Body (variable size) + % * DS - Datagram size (uint32, aka 4 bytes) + % + % The General Header was read and stored in fileinfo. Here we read the + % datagram body only + % + % Relevant info from the general header + dgm_type_code = KMALLfileinfo.dgm_type_code{iDatag}(2:end); + dgm_start_pif = KMALLfileinfo.dgm_start_pif(iDatag); + + % Go to start of dgm + fseek(fid, dgm_start_pif, -1); + + % set/reset the parsed switch + parsed = 0; + + % set/reset the datagram version warning flag + dtg_warn_flag = 0; + + switch dgm_type_code + + + %% --------- INSTALLATION AND RUNTIME DATAGRAMS (I..) ------------- + + case 'IIP' + % '#IIP - Installation parameters and sensor setup' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iIIP=iIIP+1; catch, iIIP=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmIIP(iIIP) = CFF_read_EMdgmIIP(fid, dtg_warn_flag); + + % extract and check kmall revision + kmallRev = CFF_get_kmall_revision(KMALLdata.EMdgmIIP(iIIP)); + if ~ismember(kmallRev, kmallRevSupported) && ~kmallRevWarningFlag + errStr = sprintf('This file''s kmall format revision (%s) is different to that used to develop the raw data reading code (%s). Data will be read anyway, but there may be issues',kmallRev,kmallRevSupported); + comms.error(errStr); + kmallRevWarningFlag = 1; + end + + parsed = 1; + + case 'IOP' + % '#IOP - Runtime parameters as chosen by operator' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iIOP=iIOP+1; catch, iIOP=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmIOP(iIOP) = CFF_read_EMdgmIOP(fid, dtg_warn_flag); + + parsed = 1; + + case 'IBE' + % '#IBE - Built in test (BIST) error report' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iIBE=iIBE+1; catch, iIBE=1; dtg_warn_flag = 1; end + + % to do maybe one day... XXX3 + % KMALLdata.EMdgmIBE(iIBE) = CFF_read_EMdgmIBE(fid, dtg_warn_flag); + + parsed = 0; + + case 'IBR' + % '#IBR - Built in test (BIST) reply' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iIBR=iIBR+1; catch, iIBR=1; dtg_warn_flag = 1; end + + % to do maybe one day... XXX3 + % KMALLdata.EMdgmIBR(iIBR) = CFF_read_EMdgmIBR(fid, dtg_warn_flag); + + parsed = 0; + + case 'IBS' + % '#IBS - Built in test (BIST) short reply' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iIBS=iIBS+1; catch, iIBS=1; dtg_warn_flag = 1; end + + % to do maybe one day... XXX3 + % KMALLdata.EMdgmIBS(iIBS) = CFF_read_EMdgmIBS(fid, dtg_warn_flag); + + parsed = 0; + + + %% ------------------ MULTIBEAM DATAGRAMS (M..) ------------------- + + case 'MRZ' + % '#MRZ - Multibeam (M) raw range (R) and depth(Z) datagram' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iMRZ=iMRZ+1; catch, iMRZ=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmMRZ(iMRZ) = CFF_read_EMdgmMRZ(fid, dtg_warn_flag); + + parsed = 1; + + if DEBUG + + % create or grab and clear figure + try + figure(f_MRZ); + clf + catch + f_MRZ = figure(); + end + + num_beams = KMALLdata.EMdgmMRZ(iMRZ).rxInfo.numSoundingsMaxMain ... + + KMALLdata.EMdgmMRZ(iMRZ).rxInfo.numExtraDetections; + + % detection info + subplot(221); + plot([KMALLdata.EMdgmMRZ(iMRZ).sounding.qualityFactor]); + hold on + plot([KMALLdata.EMdgmMRZ(iMRZ).sounding.detectionUncertaintyVer_m]); + plot([KMALLdata.EMdgmMRZ(iMRZ).sounding.detectionUncertaintyHor_m]); + xlabel('beam number') + legend('Ifremer quality fact.', 'Vert. uncert. (m)', 'Horz. uncert. (m)'); + title(sprintf('%s\n#MRZ datagram #%i contents\nDetection info', CFF_file_name(KMALLdata.KMALLfilename,1),iMRZ),'Interpreter','none'); + grid on + xlim([1 num_beams]) + + % reflectivity data + subplot(222); + plot([KMALLdata.EMdgmMRZ(iMRZ).sounding.reflectivity1_dB]); + hold on + plot([KMALLdata.EMdgmMRZ(iMRZ).sounding.reflectivity2_dB]); + xlabel('beam number') + legend('Refl. 1 (dB)', 'Refl. 2 (dB)'); + title('Reflectivity data') + grid on + xlim([1 num_beams]) + + % range and angle + subplot(223); + plot([KMALLdata.EMdgmMRZ(iMRZ).sounding.beamAngleReRx_deg], ... + [KMALLdata.EMdgmMRZ(iMRZ).sounding.twoWayTravelTime_sec],'.'); + xlabel('beam angle re. Rx (deg)') + ylabel('two-way travel time (s)') + set(gca, 'YDir','reverse'); + title('Range and angle') + grid on + + % georeferenced depth points + subplot(224); + plot([KMALLdata.EMdgmMRZ(iMRZ).sounding.y_reRefPoint_m], ... + [KMALLdata.EMdgmMRZ(iMRZ).sounding.z_reRefPoint_m],'.'); + xlabel('Horz. dist y (m)') + ylabel('Vert. dist z (m)') + set(gca, 'YDir','reverse') + title('Georeferenced depth points') + axis equal + grid on + + drawnow; + end + + case 'MWC' + % '#MWC - Multibeam (M) water (W) column (C) datagram' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iMWC=iMWC+1; catch, iMWC=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmMWC(iMWC) = CFF_read_EMdgmMWC(fid, dtg_warn_flag); + + parsed = 1; + + if DEBUG + + % save pif + pif_save = ftell(fid); + + % get water-column amplitude for this ping (and phase if it + % exists) + max_samples = max([KMALLdata.EMdgmMWC(iMWC).beamData_p.startRangeSampleNum] ... + + [KMALLdata.EMdgmMWC(iMWC).beamData_p.numSampleData]); + nBeams = KMALLdata.EMdgmMWC(iMWC).rxInfo.numBeams; + Mag_tmp = nan(max_samples, nBeams); + Ph_tmp = nan(max_samples, nBeams); + phaseFlag = KMALLdata.EMdgmMWC(iMWC).rxInfo.phaseFlag; + for iB = 1:nBeams + dpif = KMALLdata.EMdgmMWC(iMWC).beamData_p.sampleDataPositionInFile(iB); + fseek(fid,dpif,-1); + sR = KMALLdata.EMdgmMWC(iMWC).beamData_p.startRangeSampleNum(iB); + nS = KMALLdata.EMdgmMWC(iMWC).beamData_p.numSampleData(iB); + if phaseFlag == 0 + % Only nS records of amplitude of 1 byte + Mag_tmp(sR+1:sR+nS,iB) = fread(fid, nS, 'int8=>int8',0); + elseif phaseFlag == 1 + % nS records of amplitude of 1 byte alternated with nS + % records of phase of 1 byte + Mag_tmp(sR+1:sR+nS,iB) = fread(fid, nS, 'int8=>int8',1); + fseek(fid,dpif+1,-1); % rewind to after the first amplitude record + Ph_tmp(sR+1:sR+nS,iB) = fread(fid, nS, 'int8=>int8',1); + else + % XXX1 this case was not tested yet. Find data for it + % nS records of amplitude of 1 byte alternated with nS + % records of phase of 2 bytes + Mag_tmp(sR+1:sR+nS,iB) = fread(fid, nS, 'int8=>int8',2); + fseek(fid,dpif+1,-1); % rewind to after the first amplitude record + Ph_tmp(sR+1:sR+nS,iB) = fread(fid, nS, 'int16=>int16',1); + end + end + + % reset pif + fseek(fid, pif_save,-1); + + % create or grab and clear figure + try + figure(f_MWC); + clf + catch + f_MWC = figure(); + end + + % plot + if ~phaseFlag + % amplitude only + imagesc(Mag_tmp); + xlabel('beam number'); + ylabel('sample number'); + grid on; colorbar + title(sprintf('%s\n#MWC datagram #%i contents\nAmplitude only', CFF_file_name(KMALLdata.KMALLfilename,1), iMWC),'Interpreter','none'); + else + % amplitude + subplot(121); imagesc(Mag_tmp); + xlabel('beam number'); + ylabel('sample number'); + grid on; colorbar + title(sprintf('%s\n#MWC datagram #%i contents\nAmplitude', CFF_file_name(KMALLdata.KMALLfilename,1), iMWC),'Interpreter','none'); + % phase + subplot(121); imagesc(Ph_tmp); + xlabel('beam number'); + ylabel('sample number'); + grid on; colorbar + title('Phase'); + end + drawnow; + + end + + + %% ------------------- SENSOR DATAGRAMS (S..) --------------------- + + case 'SPO' + % '#SPO - Sensor (S) data for position (PO)' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iSPO=iSPO+1; catch, iSPO=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmSPO(iSPO) = CFF_read_EMdgmSPO(fid, dtg_warn_flag); + + parsed = 1; + + case 'SKM' + % '#SKM - Sensor (S) KM binary sensor format' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iSKM=iSKM+1; catch, iSKM=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmSKM(iSKM) = CFF_read_EMdgmSKM(fid, dtg_warn_flag); + + parsed = 1; + + case 'SVP' + % '#SVP - Sensor (S) data from sound velocity (V) profile (P) or CTD' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iSVP=iSVP+1; catch, iSVP=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmSVP(iSVP) = CFF_read_EMdgmSVP(fid, dtg_warn_flag); + + parsed = 1; + + if DEBUG + depth = [KMALLdata.EMdgmSVP(iSVP).sensorData.depth_m]; + velocity = [KMALLdata.EMdgmSVP(iSVP).sensorData.soundVelocity_mPerSec]; + temp = [KMALLdata.EMdgmSVP(iSVP).sensorData.temp_C]; + salinity = [KMALLdata.EMdgmSVP(iSVP).sensorData.salinity]; + + % create or grab and clear figure + try + figure(f_SVP); + clf + catch + f_SVP = figure(); + end + + % plot + subplot(131); plot(velocity,-depth,'.-'); + ylabel('depth (m)'); xlabel('sound velocity (m/s)'); grid on + subplot(132); plot(temp,-depth,'.-'); + ylabel('depth (m)'); xlabel('temperature (C)'); grid on + title(sprintf('%s\n#SVP datagram #%i contents', CFF_file_name(KMALLdata.KMALLfilename,1), iSVP),'Interpreter','none'); + subplot(133); plot(salinity,-depth,'.-'); + ylabel('depth (m)'); xlabel('salinity'); grid on + drawnow; + end + + case 'SVT' + % '#SVT - Sensor (S) data for sound velocity (V) at transducer (T)' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iSVT=iSVT+1; catch, iSVT=1; dtg_warn_flag = 1; end + + % to do maybe one day... XXX3 + % KMALLdata.EMdgmSVT(iSVT) = CFF_read_EMdgmSVT(fid, dtg_warn_flag); + + parsed = 0; + + case 'SCL' + % '#SCL - Sensor (S) data from clock (CL)' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iSCL=iSCL+1; catch, iSCL=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmSCL(iSCL) = CFF_read_EMdgmSCL(fid, dtg_warn_flag); + + parsed = 1; + + case 'SDE' + % '#SDE - Sensor (S) data from depth (DE) sensor' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iSDE=iSDE+1; catch, iSDE=1; dtg_warn_flag = 1; end + + % to do maybe one day... XXX3 + % KMALLdata.EMdgmSDE(iSDE) = CFF_read_EMdgmSDE(fid, dtg_warn_flag); + + parsed = 0; + + case 'SHI' + % '#SHI - Sensor (S) data for height (HI)' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iSHI=iSHI+1; catch, iSHI=1; dtg_warn_flag = 1; end + + % in progress... + % KMALLdata.EMdgmSHI(iSHI) = CFF_read_EMdgmSHI(fid, dtg_warn_flag); + + parsed = 0; + + + %% --------------- COMPATIBILITY DATAGRAMS (C..) ------------------ + + case 'CPO' + % '#CPO - Compatibility (C) data for position (PO)' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iCPO=iCPO+1; catch, iCPO=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmCPO(iCPO) = CFF_read_EMdgmCPO(fid, dtg_warn_flag); + + parsed = 1; + + case 'CHE' + % '#CHE - Compatibility (C) data for heave (HE)' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iCHE=iCHE+1; catch, iCHE=1; dtg_warn_flag = 1; end + + KMALLdata.EMdgmCHE(iCHE) = CFF_read_EMdgmCHE(fid, dtg_warn_flag); + + parsed = 1; + + + %% --------------------- FILE DATAGRAMS (F..) --------------------- + + case '#FCF - Backscatter calibration (C) file (F) datagram' + % 'YYY' + if ~( isempty(OutputFields) || any(strcmp(dgm_type_code,OutputFields)) ) + continue; + end + try iFCF=iFCF+1; catch, iFCF=1; dtg_warn_flag = 1; end + + % to do maybe one day... XXX3 + % KMALLdata.EMdgmFCF(iFCF) = CFF_read_EMdgmFCF(fid, dtg_warn_flag); + + parsed = 0; + + otherwise + % dgm_type_code not recognized. Skip. + + parsed = 0; + + end + + % modify parsed status in info + KMALLfileinfo.parsed(iDatag,1) = parsed; + + % communicate progress + comms.progress(iDatag,nDatagsToPars); + +end + + +%% finalise + +% close fid +fclose(fid); + +% add info to parsed data +KMALLdata.info = KMALLfileinfo; + +% end message +comms.finish('Done'); + +end + + diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmCHE.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmCHE.m new file mode 100644 index 0000000..83acd8a --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmCHE.m @@ -0,0 +1,43 @@ +function out_struct = CFF_read_EMdgmCHE(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMCHE Read kmall structure #CHE +% +% #CHE - Struct of compatibility heave sensor datagram. +% +% Used for backward compatibility with .all datagram format. Sent before +% #MWC (water column datagram) datagram if compatibility mode is enabled. +% The multibeam datagram body is common with the #MWC datagram. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + + +out_struct.header = CFF_read_EMdgmHeader(fid); + +CHE_VERSION = out_struct.header.dgmVersion; +if CHE_VERSION>0 && dgmVersion_warning_flag + % definitions in this function and subfunctions valid for CHE_VERSION: + % 0 (kmall format revisions F-I, and presumably earlier ones?) + warning('#CHE datagram version (%i) unsupported. Continue reading but there may be issues.',CHE_VERSION); +end + +out_struct.cmnPart = CFF_read_EMdgmMbody(fid); +out_struct.data = CFF_read_EMdgmCHEdata(fid); + +end + + +function out_struct = CFF_read_EMdgmCHEdata(fid) +% #CHE - Heave compatibility data part. Heave reference point is at +% transducer instead of at vessel reference point. +% +% Verified correct for kmall format revisions F-I + +% Heave. Unit meter. Positive downwards. +out_struct.heave_m = fread(fid,1,'float'); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmCPO.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmCPO.m new file mode 100644 index 0000000..4284532 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmCPO.m @@ -0,0 +1,103 @@ +function out_struct = CFF_read_EMdgmCPO(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMCPO Read kmall structure #CPO +% +% #CPO - Struct of compatibility position sensor datagram. +% +% Data from active sensor will be motion corrected if indicated by +% operator. Motion correction is applied to latitude, longitude, speed, +% course and ellipsoidal height. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +CPO_VERSION = out_struct.header.dgmVersion; +if CPO_VERSION>0 && dgmVersion_warning_flag + % definitions in this function and subfunctions valid for CPO_VERSION: + % 0 (kmall format revisions F-I, and presumably earlier ones?) + warning('#CPO datagram version (%i) unsupported. Continue reading but there may be issues.',CPO_VERSION); +end + +out_struct.cmnPart = CFF_read_EMdgmScommon(fid); + +% number of bytes in the actual CPO data is the total datagram size +% (need to remove 4 bytes for the final numBytes field) minus what was +% read in the header (20 bytes, as currently defined), the common part +% (8 bytes, as currently defined), and the data block until the actual +% data (40 bytes, as currently defined) +CPO_data_numBytes = (out_struct.header.numBytesDgm - 4) ... + - 20 ... + - out_struct.cmnPart.numBytesCmnPart ... + - 40; + +out_struct.sensorData = CFF_read_EMdgmCPOdataBlock(fid, CPO_data_numBytes); + +end + + +function out_struct = CFF_read_EMdgmCPOdataBlock(fid, CPO_data_numBytes) +% #CPO - Compatibility sensor position compatibility data block. Data from +% active sensor is referenced to position at antenna footprint at water +% level. Data is corrected for motion ( roll and pitch only) if enabled by +% K-Controller operator. Data given both decoded and corrected (active +% sensors), and raw as received from sensor in text string. +% +% Verified correct for kmall format revisions F-I + +% UTC time from position sensor. Unit seconds. Epoch 1970-01-01. Nanosec +% part to be added for more exact time. +out_struct.timeFromSensor_sec = fread(fid,1,'uint32'); + +% UTC time from position sensor. Unit nano seconds remainder. +out_struct.timeFromSensor_nanosec = fread(fid,1,'uint32'); + +% Only if available as input from sensor. Calculation according to format. +out_struct.posFixQuality_m = fread(fid,1,'float'); + +% Motion corrected (if enabled in K-Controller) data as used in depth +% calculations. Referred to antenna footprint at water level. Unit decimal +% degree. +% For Rev H: Parameter is set to define UNAVAILABLE_LATITUDE if sensor +% inactive. +out_struct.correctedLat_deg = fread(fid,1,'double'); + +% Motion corrected (if enabled in K-Controller) data as used in depth +% calculations. Referred to antenna footprint at water level. Unit decimal +% degree. +% For Rev H: Parameter is set to define UNAVAILABLE_LONGITUDE if sensor +% inactive. +out_struct.correctedLong_deg = fread(fid,1,'double'); + +% Speed over ground. Unit m/s. Motion corrected (if enabled in +% K-Controller) data as used in depth calculations. +% For Rev H: If unavailable or from inactive sensor, value set to define +% UNAVAILABLE_SPEED. +% For Rev I: If unavailable, value set to define UNAVAILABLE_SPEED. +out_struct.speedOverGround_mPerSec = fread(fid,1,'float'); + +% Course over ground. Unit degree. Motion corrected (if enabled in +% K-Controller) data as used in depth calculations. +% For Rev H: If unavailable or from inactive sensor, value set to define +% UNAVAILABLE_COURSE. +% For Rev I: If unavailable, value set to define UNAVAILABLE_COURSE. +out_struct.courseOverGround_deg = fread(fid,1,'float'); + +% Height of antenna footprint above the ellipsoid. Unit meter. Motion +% corrected (if enabled in K-Controller) data as used in depth +% calculations. +% For Rev H: If unavailable or from inactive sensor, value set to define +% UNAVAILABLE_ELLIPSOIDHEIGHT. +% For Rev I: If unavailable, value set to define +% UNAVAILABLE_ELLIPSOIDHEIGHT. +out_struct.ellipsoidHeightReRefPoint_m = fread(fid,1,'float'); + +% Position data as received from sensor, i.e. uncorrected for motion etc. +out_struct.posDataFromSensor = fscanf(fid, '%c', CPO_data_numBytes); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmHeader.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmHeader.m new file mode 100644 index 0000000..331c580 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmHeader.m @@ -0,0 +1,39 @@ +function out_struct = CFF_read_EMdgmHeader(fid) +%CFF_READ_EMDGMHEADER Read header struct of a kmall datagram +% +% Definition of general datagram header. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +% Datagram length in bytes. The length field at the start (4 bytes) and end +% of the datagram (4 bytes) are included in the length count. +out_struct.numBytesDgm = fread(fid,1,'uint32'); + +% Multi beam datagram type definition, e.g. #AAA +out_struct.dgmType = fscanf(fid,'%c',4); + +% Datagram version. +out_struct.dgmVersion = fread(fid,1,'uint8'); + +% System ID. Parameter used for separating datagrams from different +% echosounders if more than one system is connected to SIS/K-Controller. +out_struct.systemID = fread(fid,1,'uint8'); + +% Echo sounder identity, e.g. 124, 304, 712, 2040, 2045 (EM 2040C) +out_struct.echoSounderID = fread(fid,1,'uint16'); + +% UTC time in seconds. Epoch 1970-01-01. time_nanosec part to be added for +% more exact time. +out_struct.time_sec = fread(fid,1,'uint32'); + +% Nano seconds remainder. time_nanosec part to be added to time_sec for +% more exact time. +out_struct.time_nanosec = fread(fid,1,'uint32'); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmIB.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmIB.m new file mode 100644 index 0000000..7abb8c9 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmIB.m @@ -0,0 +1,47 @@ +function out_struct = CFF_read_EMdgmIB(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMIB Read kmall structure #IB +% +% #IB - Results from online built in test (BIST). Definition used for +% three different BIST datagrams, i.e. #IBE (BIST Error report), #IBR +% (BIST reply) or #IBS (BIST short reply). +% +% Verified correct for kmall versions H,I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +if out_struct.header.dgmVersion>0 && dgmVersion_warning_flag + % definition valid for BIST_VERSION 0 (kmall versions H,I) + warning('#IB datagram version (%i) unsupported. Continue reading but there may be issues.',out_struct.header.dgmVersion); +end + +% Size in bytes of body part struct. Used for denoting size of rest of +% the datagram. +out_struct.numBytesCmnPart = fread(fid,1,'uint16'); + +% 0 = last subset of the message +% 1 = more messages to come +out_struct.BISTInfo = fread(fid,1,'uint8'); + +% 0 = plain text +% 1 = use style sheet +out_struct.BISTStyle = fread(fid,1,'uint8'); + +% The BIST number executed. +out_struct.BISTNumber = fread(fid,1,'uint8'); + +% 0 = BIST executed with no errors +% positive number = warning +% negative number = error +out_struct.BISTStatus = fread(fid,1,'int8'); + +% Result of the BIST. Starts with a synopsis of the result, followed by +% detailed descriptions. +out_struct.BISTText = fscanf(fid, '%c',out_struct.numBytesCmnPart-6); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmIIP.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmIIP.m new file mode 100644 index 0000000..1c3c000 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmIIP.m @@ -0,0 +1,41 @@ +function out_struct = CFF_read_EMdgmIIP(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMIIP Read kmall structure #IIP +% +% Definition of #IIP datagram containing installation parameters and +% sensor format settings. +% Details in separate document Installation parameters +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +IIP_VERSION = out_struct.header.dgmVersion; +if IIP_VERSION>0 && dgmVersion_warning_flag + % definitions in this function valid for IIP_VERSION: + % 0 (kmall format revisions F-I, and presumably earlier ones?) + warning('#IIP datagram version (%i) unsupported. Continue reading but there may be issues.',IIP_VERSION); +end + +% Size in bytes of body part struct. Used for denoting size of rest of +% the datagram. +out_struct.numBytesCmnPart = fread(fid,1,'uint16'); + +% Information. For future use. +out_struct.info = fread(fid,1,'uint16'); + +% Status. For future use. +out_struct.status = fread(fid,1,'uint16'); + +% Installation settings as text format. Parameters separated by ; and +% lines separated by , delimiter. +% For detailed description of text strings, see the separate document +% Installation parameters +out_struct.install_txt = fscanf(fid, '%c',out_struct.numBytesCmnPart-6); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmIOP.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmIOP.m new file mode 100644 index 0000000..54d0dc4 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmIOP.m @@ -0,0 +1,43 @@ +function out_struct = CFF_read_EMdgmIOP(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMIOP Read kmall structure #IOP +% +% Definition of #IOP datagram containing runtime parameters, exactly as +% chosen by operator in the K-Controller/SIS menus. +% For detailed description of text strings, see the separate document +% Runtime parameters set by operator. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +IOP_VERSION = out_struct.header.dgmVersion; +if IOP_VERSION>0 && dgmVersion_warning_flag + % definitions in this function valid for IOP_VERSION: + % 0 (kmall format revisions F-I, and presumably earlier ones?) + warning('#IOP datagram version (%i) unsupported. Continue reading but there may be issues.',IOP_VERSION); +end + +% Size in bytes of body part struct. Used for denoting size of rest of +% the datagram. +out_struct.numBytesCmnPart = fread(fid,1,'uint16'); + +% Information. For future use. +out_struct.info = fread(fid,1,'uint16'); + +% Status. For future use. +out_struct.status = fread(fid,1,'uint16'); + +% Runtime paramters as text format. Parameters separated by ; and lines +% separated by , delimiter. Text strings refer to names in menues of +% the K-Controller/SIS. +% For detailed description of text strings, see the separate document +% Runtime parameters set by operator +out_struct.runtime_txt = fscanf(fid, '%c',out_struct.numBytesCmnPart-6); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMRZ.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMRZ.m new file mode 100644 index 0000000..43c7089 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMRZ.m @@ -0,0 +1,741 @@ +function out_struct = CFF_read_EMdgmMRZ(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMMRZ Read kmall structure #MRZ +% +% #MRZ - Multibeam Raw Range and Depth datagram. The datagram also +% contains seabed image data. +% +% Datagram consists of several structs. The MRZ datagram replaces several +% old datagrams: raw range (N 78), depth (XYZ 88), seabed image (Y 89) +% datagram, quality factor (O 79) and runtime (R 52). +% +% Depths points (x,y,z) are calculated in meters, georeferred to the +% position of the vessel reference point at the time of the first +% transmitted pulse of the ping. The depth point coordinates x and y are +% in the surface coordinate system (SCS), and are also given as delta +% latitude and delta longitude, referred to origo of the VCS/SCS, at the +% time of the midpoint of the first transmitted pulse of the ping (equals +% time used in the datagram header timestamp). +% See Coordinate systems for introduction to spatial reference points and +% coordinate systems. Reference points are also described in Reference +% points and offsets. Explanation of the xyz reference points is also +% illustrated in the figure below. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +MRZ_VERSION = out_struct.header.dgmVersion; +if MRZ_VERSION>3 && dgmVersion_warning_flag + % definitions in this function and subfunctions valid for MRZ_VERSION: + % 0 (kmall format revision F, and presumably earlier ones?) + % 1 (kmall format revision G) + % 2 (kmall format revision H) + % 3 (kmall format revision I) + warning('#MRZ datagram version (%i) unsupported. Continue reading but there may be issues.',MRZ_VERSION); +end + +out_struct.partition = CFF_read_EMdgmMpartition(fid); +out_struct.cmnPart = CFF_read_EMdgmMbody(fid); +out_struct.pingInfo = CFF_read_EMdgmMRZ_pingInfo(fid, MRZ_VERSION); + +Ntx = out_struct.pingInfo.numTxSectors; +for iTx = 1:Ntx + out_struct.sectorInfo(iTx) = CFF_read_EMdgmMRZ_txSectorInfo(fid, MRZ_VERSION); +end + +out_struct.rxInfo = CFF_read_EMdgmMRZ_rxInfo(fid); + +Ndc = out_struct.rxInfo.numExtraDetectionClasses; +for iD = 1:Ndc + out_struct.extraDetClassInfo(iD) = CFF_read_EMdgmMRZ_extraDetClassInfo(fid); +end + +Nrx = out_struct.rxInfo.numSoundingsMaxMain; +Nd = out_struct.rxInfo.numExtraDetections; +% DEV NOTE: Normally, if following stricly the "struct" organization of +% kmall data, we would read ONE "sounding" struct per beam, using a for +% loop over all soundings. However, this was taking too much time because +% there are a lot of fields to read in that structure and a lot of beams +% per ping. So, just for this one, we're doing things differently. We're +% recording a SINGLE "sounding" struct, containing arrays for each +% soundings, which can be read efficiently because the total size is the +% same for all beams. +out_struct.sounding = CFF_read_EMdgmMRZ_sounding(fid, Nrx+Nd); + +% Seabed image sample amplitude, in 0.1 dB. Actual number of seabed +% image samples (SIsample_desidB) to be found by summing parameter +% SInumSamples in struct EMdgmMRZ_sounding_def for all beams. Seabed +% image data are raw beam sample data taken from the RX beams. The data +% samples are selected based on the bottom detection ranges. First +% sample for each beam is the one with the lowest range. The centre +% sample from each beam is geo referenced (x, y, z data from the +% detections). The BS corrections applied at the centre sample are the +% same as used for reflectivity2_dB (struct EMdgmMRZ_sounding_def). +Ns = [out_struct.sounding.SInumSamples]; +for iRx = 1:(Nrx+Nd) + out_struct.SIsample_desidB{iRx} = fread(fid,Ns(iRx),'int16'); +end + +end + +function out_struct = CFF_read_EMdgmMRZ_pingInfo(fid, MRZ_VERSION) +% #MRZ - ping info. Information on vessel/system level, i.e. information +% common to all beams in the current ping. +% +% Verified correct for kmall format revisions F-I + +% Number of bytes in current struct. +out_struct.numBytesInfoData = fread(fid,1,'uint16'); + +% Byte alignment. +out_struct.padding0 = fread(fid,1,'uint16'); + +% Ping rate. Filtered/averaged. +out_struct.pingRate_Hz = fread(fid,1,'float'); + +% 0 = Eqidistance +% 1 = Equiangle +% 2 = High density +out_struct.beamSpacing = fread(fid,1,'uint8'); + +% Depth mode. Describes setting of depth in K-Controller. Depth mode +% influences the PUs choice of pulse length and pulse type. If operator has +% manually chosen the depth mode to use, this is flagged by adding 100 to +% the mode index. +% +% Number Auto setting Number Manual setting +% 0 Very shallow 100 Very shallow +% 1 Shallow 101 Shallow +% 2 Medium 102 Medium +% 3 Deep 103 Deep +% 4 Deeper 104 Deeper +% 5 Very deep 105 Very deep +% 6 Extra deep 106 Extra deep +% 7 Extreme deep 107 Extreme deep +out_struct.depthMode = fread(fid,1,'uint8'); + +% For advanced use when depth mode is set manually. 0 = Sub depth mode is +% not used (when depth mode is auto). +out_struct.subDepthMode = fread(fid,1,'uint8'); + +% Achieved distance between swaths, in percent relative to required swath +% distance. +% 0 = function is not used +% 100 = achieved swath distance equals required swath distance. +out_struct.distanceBtwSwath = fread(fid,1,'uint8'); + +% Detection mode. Bottom detection algorithm used. +% 0 = normal +% 1 = waterway +% 2 = tracking +% 3 = minimum depth If system running in simulation mode +% detectionmode + 100 = simulator. +out_struct.detectionMode = fread(fid,1,'uint8'); + +% Pulse forms used for current swath. +% 0 = CW +% 1 = mix +% 2 = FM +out_struct.pulseForm = fread(fid,1,'uint8'); + +% Ping rate. Filtered/averaged. +out_struct.padding1 = fread(fid,1,'uint16'); + +% Ping frequency in hertz. E.g. for EM 2040: 200 000 Hz, 300 000 Hz, 400 +% 000 Hz, 600 000 Hz or 700 000 Hz. If values is less than 100, it refers +% to a code defined in the table below. +% Value Frequency Valid for EM model +% -1 Not used - +% 0 40 - 100 kHz EM 712 +% 1 50 - 100 kHz EM 712 +% 2 70 - 100 kHz EM 712 +% 3 50 kHz EM 712 +% 4 40 kHz EM 712 +% 180 000 - 400 000 180 -400 kHz EM 2040C (10 kHz steps) +% 200 000 200 kHz EM 2040, EM 2040P +% 300 000 300 kHz EM 2040, EM 2040P +% 400 000 400 kHz EM 2040, EM 2040P +% 600 000 600 kHz EM 2040, EM 2040P +% 700 000 700 kHz EM 2040, EM 2040P +out_struct.frequencyMode_Hz = fread(fid,1,'float'); + +% Lowest centre frequency of all sectors in this swath. Unit hertz. E.g. +% for EM 2040: 260 000 Hz. +out_struct.freqRangeLowLim_Hz = fread(fid,1,'float'); + +% Highest centre frequency of all sectors in this swath. Unit hertz. E.g. +% for EM 2040: 320 000 Hz. +out_struct.freqRangeHighLim_Hz = fread(fid,1,'float'); + +% Total signal length of the sector with longest tx pulse. Unit second. +out_struct.maxTotalTxPulseLength_sec = fread(fid,1,'float'); + +% Effective signal length (-3dB envelope) of the sector with longest +% effective tx pulse. Unit second. +out_struct.maxEffTxPulseLength_sec = fread(fid,1,'float'); + +% Effective bandwidth (-3dB envelope) of the sector with highest bandwidth. +out_struct.maxEffTxBandWidth_Hz = fread(fid,1,'float'); + +% Average absorption coefficient, in dB/km, for vertical beam at current +% depth. Not currently in use. +out_struct.absCoeff_dBPerkm = fread(fid,1,'float'); + +% Port sector edge, used by beamformer, Coverage is refered to z of SCS.. +% Unit degree. +out_struct.portSectorEdge_deg = fread(fid,1,'float'); + +% Starboard sector edge, used by beamformer. Coverage is referred to z of +% SCS. Unit degree. +out_struct.starbSectorEdge_deg = fread(fid,1,'float'); + +% Coverage achieved, corrected for raybending. Coverage is referred to z of +% SCS. Unit degree. +out_struct.portMeanCov_deg = fread(fid,1,'float'); + +% Coverage achieved, corrected for raybending. Coverage is referred to z of +% SCS. Unit degree. +out_struct.starbMeanCov_deg = fread(fid,1,'float'); + +% Coverage achieved, corrected for raybending. Coverage is referred to z of +% SCS. Unit meter. +out_struct.portMeanCov_m = fread(fid,1,'int16'); + +% Coverage achieved, corrected for raybending. Unit meter. +out_struct.starbMeanCov_m = fread(fid,1,'int16'); + +% Modes and stabilisation settings as chosen by operator. Each bit refers +% to one setting in K-Controller. Unless otherwise stated, default: 0 = +% off, 1 = on/auto. +% Bit Mode +% 1 Pitch stabilisation +% 2 Yaw stabilisation +% 3 Sonar mode +% 4 Angular coverage mode +% 5 Sector mode +% 6 Swath along position (0 = fixed, 1 = dynamic) +% 7-8 Future use +out_struct.modeAndStabilisation = fread(fid,1,'uint8'); + +% Filter settings as chosen by operator. Refers to settings in runtime +% display of K-Controller. Each bit refers to one filter setting. 0 = off, +% 1 = on/auto. +% Bit Filter choice +% 1 Slope filter +% 2 Aeration filer +% 3 Sector filter +% 4 Interference filter +% 5 Special amplitude detect +% 6-8 Future use +out_struct.runtimeFilter1 = fread(fid,1,'uint8'); + +% Filter settings as chosen by operator. Refers to settings in runtime +% display of K-Controller. 4 bits used per filter. +% Bit Filter choice Setting +% 1-4 Range gate size 0 = small, 1 = normal, 2 = large +% 5-8 Spike filter strength 0 = off, 1= weak, 2 = medium, 3 = strong +% 9-12 Penetration filter 0 = off, 1 = weak, 2 = medium, 3 = strong +% 13-16 Phase ramp 0 = short, 1 = normal, 2 = long +out_struct.runtimeFilter2 = fread(fid,1,'uint16'); + +% Pipe tracking status. Describes how angle and range of top of pipe is +% determined. +% 0 = for future use +% 1 = PU uses guidance from SIS. +out_struct.pipeTrackingStatus = fread(fid,1,'uint32'); + +% Transmit array size used. Direction along ship. Unit degree. +out_struct.transmitArraySizeUsed_deg = fread(fid,1,'float'); + +% Receiver array size used. Direction across ship. Unit degree. +out_struct.receiveArraySizeUsed_deg = fread(fid,1,'float'); + +% Operator selected tx power level re maximum. Unit dB. E.g. 0 dB, -10 dB, +% -20 dB. +out_struct.transmitPower_dB = fread(fid,1,'float'); + +% For marine mammal protection. The parameters describes time remaining +% until max source level (SL) is achieved. Unit %. +out_struct.SLrampUpTimeRemaining = fread(fid,1,'uint16'); + +% Byte alignment. +out_struct.padding2 = fread(fid,1,'uint16'); + +% Yaw correction angle applied. Unit degree. +out_struct.yawAngle_deg = fread(fid,1,'float'); + +% Number of transmit sectors. Also called Ntx in documentation. Denotes how +% many times the struct EMdgmMRZ_txSectorInfo is repeated in the datagram. +out_struct.numTxSectors = fread(fid,1,'uint16'); + +% Number of bytes in the struct EMdgmMRZ_txSectorInfo, containing tx sector +% specific information. The struct is repeated numTxSectors times. +out_struct.numBytesPerTxSector = fread(fid,1,'uint16'); + +% Heading of vessel at time of midpoint of first tx pulse. From active +% heading sensor. +out_struct.headingVessel_deg = fread(fid,1,'float'); + +% At time of midpoint of first tx pulse. Value as used in depth +% calculations. Source of sound speed defined by user in K-Controller. +out_struct.soundSpeedAtTxDepth_mPerSec = fread(fid,1,'float'); + +% Tx transducer depth in meters below waterline, at time of midpoint of +% first tx pulse. For the tx array (head) used by this RX-fan. Use depth of +% TX1 to move depth point (XYZ) from water line to transducer (reference +% point of old datagram format). +out_struct.txTransducerDepth_m = fread(fid,1,'float'); + +% Distance between water line and vessel reference point in meters. At time +% of midpoint of first tx pulse. Measured in the surface coordinate system +% (SCS).See Coordinate systems 'Coordinate systems' for definition. Used +% this to move depth point (XYZ) from vessel reference point to waterline. +out_struct.z_waterLevelReRefPoint_m = fread(fid,1,'float'); + +% Distance between *.all reference point and *.kmall reference point +% (vessel refernece point) in meters, in the surface coordinate system, at +% time of midpoint of first tx pulse. Used this to move depth point (XYZ) +% from vessel reference point to the horisontal location (X,Y) of the +% active position sensor's reference point (old datagram format). +out_struct.x_kmallToall_m = fread(fid,1,'float'); + +% Distance between *.all reference point and *.kmall reference point +% (vessel refernece point) in meters, in the surface coordinate system, at +% time of midpoint of first tx pulse. Used this to move depth point (XYZ) +% from vessel reference point to the horisontal location (X,Y) of the +% active position sensor's reference point (old datagram format). +out_struct.y_kmallToall_m = fread(fid,1,'float'); + +% Method of position determination from position sensor data: +% 0 = last position received +% 1 = interpolated +% 2 = processed +out_struct.latLongInfo = fread(fid,1,'uint8'); + +% Status/quality for data from active position sensor. 0 = valid data, 1 = +% invalid data, 2 = reduced performance +out_struct.posSensorStatus = fread(fid,1,'uint8'); + +% Status/quality for data from active attitude sensor. 0 = valid data, 1 = +% invalid data, 2 = reduced performance +out_struct.attitudeSensorStatus = fread(fid,1,'uint8'); + +% Padding for byte alignment. +out_struct.padding3 = fread(fid,1,'uint8'); + +% Latitude (decimal degrees) of vessel reference point at time of midpoint +% of first tx pulse. Negative on southern hemisphere. Parameter is set to +% define UNAVAILABLE_LATITUDE if not available. +out_struct.latitude_deg = fread(fid,1,'double'); + +% Longitude (decimal degrees) of vessel reference point at time of midpoint +% of first tx pulse. Negative on western hemisphere. Parameter is set to +% define UNAVAILABLE_LONGITUDE if not available. +out_struct.longitude_deg = fread(fid,1,'double'); + +% Height of vessel reference point above the ellipsoid, derived from active +% GGA sensor. ellipsoidHeightReRefPoint_m is GGA height corrected for +% motion and installation offsets of the position sensor. +out_struct.ellipsoidHeightReRefPoint_m = fread(fid,1,'float'); + +if MRZ_VERSION > 0 + + % Backscatter offset set in the installation menu + out_struct.bsCorrectionOffset_dB = fread(fid,1,'float'); + + % Beam intensity data corrected as seabed image data (Lambert and + % normal incidence corrections) + out_struct.lambertsLawApplied = fread(fid,1,'uint8'); + + % Ice window installed + out_struct.iceWindow = fread(fid,1,'uint8'); + + if MRZ_VERSION == 1 + % Padding for byte alignment. + out_struct.padding4 = fread(fid,1,'uint16'); + else + % Sets status for active modes. + % Bit Modes Setting + % 1 EM MultiFrequency Mode 0 = not active, 1 = active + % 2-16 Not in use NA + out_struct.activeModes = fread(fid,1,'uint16'); + end + +end + +end + +function out_struct = CFF_read_EMdgmMRZ_txSectorInfo(fid, MRZ_VERSION) +% #MRZ - sector information. +% +% Information specific to each transmitting sector. sectorInfo is repeated +% numTxSectors (Ntx)- times in datagram. +% +% Verified correct for kmall format revisions F-I + +% TX sector index number, used in the sounding section. Starts at 0. +out_struct.txSectorNumb = fread(fid,1,'uint8'); + +% TX array number. Single TX, txArrNumber = 0. +out_struct.txArrNumber = fread(fid,1,'uint8'); + +% Default = 0. E.g. for EM2040, the transmitted pulse consists of three +% sectors, each transmitted from separate txSubArrays. Orientation and +% numbers are relative the array coordinate system. Sub array installation +% offsets can be found in the installation datagram, #IIP. +% 0 = Port subarray +% 1 = middle subarray +% 2 = starboard subarray +out_struct.txSubArray = fread(fid,1,'uint8'); + +% Byte alignment. +out_struct.padding0 = fread(fid,1,'uint8'); + +% Transmit delay of the current sector/subarray. Delay is the time from the +% midpoint of the current transmission to midpoint of the first transmitted +% pulse of the ping, i.e. relative to the time used in the datagram header. +out_struct.sectorTransmitDelay_sec = fread(fid,1,'float'); + +% Along ship steering angle of the TX beam (main lobe of transmitted +% pulse), angle referred to transducer array coordinate system. Unit +% degree. See Coordinate systems 'Coordinate systems' +out_struct.tiltAngleReTx_deg = fread(fid,1,'float'); + +% Actual SL = txNominalSourceLevel_dB + highVoltageLevel_dB. Unit dB re 1 +% microPascal. +out_struct.txNominalSourceLevel_dB = fread(fid,1,'float'); + +% 0 = no focusing applied. +out_struct.txFocusRange_m = fread(fid,1,'float'); + +% Centre frequency. Unit hertz. +out_struct.centreFreq_Hz = fread(fid,1,'float'); + +% FM mode: effective bandwidth +% CW mode: 1/(effective TX pulse length) +out_struct.signalBandWidth_Hz = fread(fid,1,'float'); + +% Also called pulse length. Unit second. +out_struct.totalSignalLength_sec = fread(fid,1,'float'); + +% Transmit pulse is shaded in time (tapering). Amplitude shading in %. +% cos2- function used for shading the TX pulse in time. +out_struct.pulseShading = fread(fid,1,'uint8'); + +% Transmit signal wave form. +% 0 = CW +% 1 = FM upsweep +% 2 = FM downsweep. +out_struct.signalWaveForm = fread(fid,1,'uint8'); + +% Byte alignment. +out_struct.padding1 = fread(fid,1,'uint16'); + +if MRZ_VERSION > 1 + + % 20 log(Measured high voltage power level at TX pulse / Nominal high + % voltage power level). This parameter will also include the effect of user + % selected transmit power reduction (transmitPower_dB) and mammal + % protection. Actual SL = txNominalSourceLevel_dB + highVoltageLevel_dB. + % Unit dB. + out_struct.highVoltageLevel_dB = fread(fid,1,'float'); + + % Backscatter correction added in sector tracking mode. Unit dB. + out_struct.sectorTrackingCorr_dB = fread(fid,1,'float'); + + % Signal length used for backscatter footprint calculation. This + % compensates for the TX pulse tapering and the RX filter bandwidths. Unit + % second. + out_struct.effectiveSignalLength_sec = fread(fid,1,'float'); + +end + +end + +function out_struct = CFF_read_EMdgmMRZ_rxInfo(fid) +% #MRZ - receiver specific information. +% +% Information specific to the receiver unit used in this swath. +% +% Verified correct for kmall format revisions F-I + +% Bytes in current struct. +out_struct.numBytesRxInfo = fread(fid,1,'uint16'); + +% Maximum number of main soundings (bottom soundings) in this datagram, +% extra detections (soundings in water column) excluded. Also referred to +% as Nrx. Denotes how many bottom points (or loops) given in the struct +% EMdgmMRZ_sounding_def. +out_struct.numSoundingsMaxMain = fread(fid,1,'uint16'); + +% Number of main soundings of valid quality. Extra detections not included. +out_struct.numSoundingsValidMain = fread(fid,1,'uint16'); + +% Bytes per loop of sounding (per depth point), i.e. bytes per loops of the +% struct EMdgmMRZ_sounding_def. +out_struct.numBytesPerSounding = fread(fid,1,'uint16'); + +% Sample frequency divided by water column decimation factor. Unit hertz. +out_struct.WCSampleRate = fread(fid,1,'float'); + +% Sample frequency divided by seabed image decimation factor. Unit hertz. +out_struct.seabedImageSampleRate = fread(fid,1,'float'); + +% Backscatter level, normal incidence. Unit dB +out_struct.BSnormal_dB = fread(fid,1,'float'); + +% Backscatter level, oblique incidence. Unit dB +out_struct.BSoblique_dB = fread(fid,1,'float'); + +% extraDetectionAlarmFlag = sum of alarm flags. Range 0-10. +out_struct.extraDetectionAlarmFlag = fread(fid,1,'uint16'); + +% Sum of extradetection from all classes. Also refered to as Nd. +out_struct.numExtraDetections = fread(fid,1,'uint16'); + +% Range 0-10. +out_struct.numExtraDetectionClasses = fread(fid,1,'uint16'); + +% Number of bytes in the struct EMdgmMRZ_extraDetClassInfo_def. +out_struct.numBytesPerClass = fread(fid,1,'uint16'); + +end + +function out_struct = CFF_read_EMdgmMRZ_extraDetClassInfo(fid) +% #MRZ - Extra detection class information. +% +% To be entered in loop numExtraDetectionClasses - times. +% +% Verified correct for kmall format revisions F-I + +% Number of extra detection in this class. +out_struct.numExtraDetInClass = fread(fid,1,'uint16'); + +% Byte alignment. +out_struct.padding = fread(fid,1,'int8'); + +% 0 = no alarm +% 1 = alarm. +out_struct.alarmFlag = fread(fid,1,'uint8'); + +end + +function out_struct = CFF_read_EMdgmMRZ_sounding(fid, N) +% #MRZ - Data for each sounding, e.g. XYZ, reflectivity, two way travel +% time etc. +% +% Also contains information necessary to read seabed image following this +% datablock (number of samples in SI etc.). To be entered in loop +% (numSoundingsMaxMain + numExtraDetections) times. +% +% Verified correct for kmall format revisions F-I + +structSize = 120; +data = fread(fid,N.*structSize,'uint8=>uint8'); +data = reshape(data, [structSize,N]); + +% Sounding index. Cross reference for seabed image. Valid range: 0 to +% (numSoundingsMaxMain+numExtraDetections)-1, i.e. 0 - (Nrx+Nd)-1. +out_struct.soundingIndex = typecast(reshape(data(1:2,:),1,[]),'uint16'); + +% Transmitting sector number. Valid range: 0-(Ntx-1), where Ntx is +% numTxSectors. +out_struct.txSectorNumb = data(3,:); + + +%% Detection info + +% Bottom detection type. Normal bottom detection, extra detection, or +% rejected. +% 0 = normal detection +% 1 = extra detection +% 2 = rejected detection +% In case 2, the estimated range has been used to fill in amplitude samples +% in the seabed image datagram. +out_struct.detectionType = data(4,:); + +% Method for determining bottom detection, e.g. amplitude or phase. +% 0 = no valid detection +% 1 = amplitude detection +% 2 = phase detection +% 3-15 for future use. +out_struct.detectionMethod = data(5,:); + +% For Kongsberg use. +% out_struct.rejectionInfo1 = data(6,:); + +% For Kongsberg use. +% out_struct.rejectionInfo2 = data(7,:); + +% For Kongsberg use. +% out_struct.postProcessingInfo = data(8,:); + +% Only used by extra detections. Detection class based on detected range. +% Detection class 1 to 7 corresponds to value 0 to 6. If the value is +% between 100 and 106, the class is disabled by the operator. If the value +% is 107, the detections are outside the treshhold limits. +out_struct.detectionClass = data(9,:); + +% Detection confidence level. +out_struct.detectionConfidenceLevel = data(10,:); + +% Byte alignment. +% out_struct.padding = typecast(reshape(data(11:12,:),1,[]),'uint16'); + +% Unit %. rangeFactor = 100 if main detection. +out_struct.rangeFactor = typecast(reshape(data(13:16,:),1,[]),'single'); + +% Estimated standard deviation as % of the detected depth. Quality Factor +% (QF) is calculated from IFREMER Quality Factor (IFQ): +% QF=Est(dz)/z=100*10^-IQF +out_struct.qualityFactor = typecast(reshape(data(17:20,:),1,[]),'single'); + +% Vertical uncertainty, based on quality factor (QF, qualityFactor). +out_struct.detectionUncertaintyVer_m = typecast(reshape(data(21:24,:),1,[]),'single'); + +% Horizontal uncertainty, based on quality factor (QF, qualityFactor). +out_struct.detectionUncertaintyHor_m = typecast(reshape(data(25:28,:),1,[]),'single'); + +% Detection window length. Unit second. Sample data range used in final +% detection. +out_struct.detectionWindowLength_sec = typecast(reshape(data(29:32,:),1,[]),'single'); + +% Measured echo length. Unit second. +out_struct.echoLength_sec = typecast(reshape(data(33:36,:),1,[]),'single'); + + +%% Water column parameters + +% Water column beam number. Info for plotting soundings together with water +% column data. +out_struct.WCBeamNumb = typecast(reshape(data(37:38,:),1,[]),'uint16'); + +% Water column range. Range of bottom detection, in samples. +out_struct.WCrange_samples = typecast(reshape(data(39:40,:),1,[]),'uint16'); + +% Water column nominal beam angle across. Re vertical. +out_struct.WCNomBeamAngleAcross_deg = typecast(reshape(data(41:44,:),1,[]),'single'); + + +%% Reflectivity data (backscatter (BS) data) + +% Mean absorption coefficient, alfa. Used for TVG calculations. Value as +% used. Unit dB/km. +out_struct.meanAbsCoeff_dBPerkm = typecast(reshape(data(45:48,:),1,[]),'single'); + +% Beam intensity, using the traditional KM special TVG. +out_struct.reflectivity1_dB = typecast(reshape(data(49:52,:),1,[]),'single'); + +% Beam intensity (BS), using TVG = X log(R) + 2 alpha R. X (operator +% selected) is common to all beams in datagram. Alpha (variabel +% meanAbsCoeff_dBPerkm) is given for each beam (current struct). +% BS = EL - SL - M + TVG + BScorr, +% where EL= detected echo level (not recorded in datagram), and the rest of +% the parameters are found below. +out_struct.reflectivity2_dB = typecast(reshape(data(53:56,:),1,[]),'single'); + +% Receiver sensitivity (M), in dB, compensated for RX beampattern at actual +% transmit frequency at current vessel attitude. +out_struct.receiverSensitivityApplied_dB = typecast(reshape(data(57:60,:),1,[]),'single'); + +% Source level (SL) applied (dB): +% SL = SLnom + SLcorr +% where SLnom = Nominal maximum SL, recorded per TX sector (variabel +% txNominalSourceLevel_dB in struct EMdgmMRZ_txSectorInfo_def) and SLcorr = +% SL correction relative to nominal TX power based on measured high voltage +% power level and any use of digital power control. SL is corrected for TX +% beampattern along and across at actual transmit frequency at current +% vessel attitude. +out_struct.sourceLevelApplied_dB = typecast(reshape(data(61:64,:),1,[]),'single'); + +% Backscatter (BScorr) calibration offset applied (default = 0 dB). +out_struct.BScalibration_dB = typecast(reshape(data(65:68,:),1,[]),'single'); + +% Time Varying Gain (TVG) used when correcting reflectivity. +out_struct.TVG_dB = typecast(reshape(data(69:72,:),1,[]),'single'); + + +%% Range and angle data + +% Angle relative to the RX transducer array, except for ME70, where the +% angles are relative to the horizontal plane. +out_struct.beamAngleReRx_deg = typecast(reshape(data(73:76,:),1,[]),'single'); + +% Applied beam pointing angle correction. +out_struct.beamAngleCorrection_deg = typecast(reshape(data(77:80,:),1,[]),'single'); + +% Two way travel time (also called range). Unit second. +out_struct.twoWayTravelTime_sec = typecast(reshape(data(81:84,:),1,[]),'single'); + +% Applied two way travel time correction. Unit second. +out_struct.twoWayTravelTimeCorrection_sec = typecast(reshape(data(85:88,:),1,[]),'single'); + + +%% Georeferenced depth points + +% Distance from vessel reference point at time of first tx pulse in ping, +% to depth point. Measured in the surface coordinate system (SCS), see +% Coordinate systems for definition. Unit decimal degrees. +out_struct.deltaLatitude_deg = typecast(reshape(data(89:92,:),1,[]),'single'); + +% Distance from vessel reference point at time of first tx pulse in ping, +% to depth point. Measured in the surface coordinate system (SCS), see +% Coordinate systems for definition. Unit decimal degree. +out_struct.deltaLongitude_deg = typecast(reshape(data(93:96,:),1,[]),'single'); + +% Vertical distance z. Distance from vessel reference point at time of +% first tx pulse in ping, to depth point. Measured in the surface +% coordinate system (SCS), see Coordinate systems for definition. +out_struct.z_reRefPoint_m = typecast(reshape(data(97:100,:),1,[]),'single'); + +% Horizontal distance y. Distance from vessel reference point at time of +% first tx pulse in ping, to depth point. Measured in the surface +% coordinate system (SCS), see Coordinate systems for definition. +out_struct.y_reRefPoint_m = typecast(reshape(data(101:104,:),1,[]),'single'); + +% Horizontal distance x. Distance from vessel reference point at time of +% first tx pulse in ping, to depth point. Measured in the surface +% coordinate system (SCS), see Coordinate systems for definition. +out_struct.x_reRefPoint_m = typecast(reshape(data(105:108,:),1,[]),'single'); + +% Beam incidence angle adjustment (IBA) unit degree. +out_struct.beamIncAngleAdj_deg = typecast(reshape(data(109:112,:),1,[]),'single'); + +% For future use. +out_struct.realTimeCleanInfo = typecast(reshape(data(113:114,:),1,[]),'uint16'); + + +%% Seabed image + +% Seabed image start range, in sample number from transducer. Valid only +% for the current beam. +out_struct.SIstartRange_samples = typecast(reshape(data(115:116,:),1,[]),'uint16'); + +% Seabed image. Number of the centre seabed image sample for the current +% beam. +out_struct.SIcentreSample = typecast(reshape(data(117:118,:),1,[]),'uint16'); + +% Seabed image. Number of range samples from the current beam, used to form +% the seabed image. +out_struct.SInumSamples = typecast(reshape(data(119:120,:),1,[]),'uint16'); + + +%% change out_struct organization +% from one struct with array fields to array of structs with +% single-variable fields, like all other stuctures in this format +% for field = fieldnames(out_struct)' +% for ii = 1:N +% out_struct_2(ii).(field{1}) = out_struct.(field{1})(ii); +% end +% end +% out_struct = out_struct_2; + + + +end diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMWC.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMWC.m new file mode 100644 index 0000000..f36a672 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMWC.m @@ -0,0 +1,229 @@ +function out_struct = CFF_read_EMdgmMWC(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMMWC Read kmall structure #MWC +% +% #MWC - Multibeam Water Column Datagram. Entire datagram containing +% several sub structs. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +MWC_VERSION = out_struct.header.dgmVersion; +if MWC_VERSION>2 && dgmVersion_warning_flag + % definitions in this function and subfunctions valid for MWC_VERSION: + % 0 (kmall format revision F, and presumably earlier ones?) + % 1 (kmall format revision G-H) + % 2 (kmall format revision I) + warning('#MWC datagram version (%i) unsupported. Continue reading but there may be issues.',MWC_VERSION); +end + +out_struct.partition = CFF_read_EMdgmMpartition(fid); +out_struct.cmnPart = CFF_read_EMdgmMbody(fid); +out_struct.txInfo = CFF_read_EMdgmMWCtxInfo(fid); + +Ntx = out_struct.txInfo.numTxSectors; +for iTx = 1:Ntx + out_struct.sectorData(iTx) = CFF_read_EMdgmMWCtxSectorData(fid); +end + +out_struct.rxInfo = CFF_read_EMdgmMWCrxInfo(fid); + +% Pointer to beam related information. Struct defines information about +% data for a beam. Beam information is followed by sample amplitudes in +% 0.5 dB resolution . Amplitude array is followed by phase information +% if phaseFlag >0. These data defined by struct +% EMdgmMWCrxBeamPhase1_def (int8_t) or struct EMdgmMWCrxBeamPhase2_def +% (int16_t) if indicated in the field phaseFlag in struct +% EMdgmMWCrxInfo_def. +% Lenght of data block for each beam depends on the operators choise of +% phase information (see table). +% phaseFlag Beam block size +% 0 numBytesPerBeamEntry + numSampleData* size(sampleAmplitude05dB_p) +% 1 numBytesPerBeamEntry + numSampleData* size(sampleAmplitude05dB_p) + numSampleData* size(EMdgmMWCrxBeamPhase1_def) +% 2 numBytesPerBeamEntry + numSampleData* size(sampleAmplitude05dB_p) + numSampleData* size(EMdgmMWCrxBeamPhase2_def) +phaseFlag = out_struct.rxInfo.phaseFlag; +Nrx = out_struct.rxInfo.numBeams; +out_struct.beamData_p = CFF_read_EMdgmMWCrxBeamData(fid, phaseFlag, Nrx, MWC_VERSION); + +end + + +function out_struct = CFF_read_EMdgmMWCtxInfo(fid) +% #MWC - data block 1: transmit sectors, general info for all sectors +% +% Verified correct for kmall format revisions F-I + +% Number of bytes in current struct. +out_struct.numBytesTxInfo = fread(fid,1,'uint16'); + +% Number of transmitting sectors (Ntx). Denotes the number of times the +% struct EMdgmMWCtxSectorData is repeated in the datagram. +out_struct.numTxSectors = fread(fid,1,'uint16'); + +% Number of bytes in EMdgmMWCtxSectorData. +out_struct.numBytesPerTxSector = fread(fid,1,'uint16'); + +% Byte alignment. +out_struct.padding = fread(fid,1,'int16'); + +% Heave at vessel reference point, at time of ping, i.e. at midpoint of +% first tx pulse in rxfan. +out_struct.heave_m = fread(fid,1,'float'); + +end + + +function out_struct = CFF_read_EMdgmMWCtxSectorData(fid) +% #MWC - data block 1: transmit sector data, loop for all i = numTxSectors. +% +% Verified correct for kmall format revisions F-I + +% Along ship steering angle of the TX beam (main lobe of transmitted +% pulse), angle referred to transducer face. Angle as used by beamformer +% (includes stabilisation). Unit degree. +out_struct.tiltAngleReTx_deg = fread(fid,1,'float'); + +% Centre frequency of current sector. Unit hertz. +out_struct.centreFreq_Hz = fread(fid,1,'float'); + +% Corrected for frequency, sound velocity and tilt angle. Unit degree. +out_struct.txBeamWidthAlong_deg = fread(fid,1,'float'); + +% Transmitting sector number. +out_struct.txSectorNum = fread(fid,1,'uint16'); + +% Byte alignment. +out_struct.padding = fread(fid,1,'int16'); + +end + + +function out_struct = CFF_read_EMdgmMWCrxInfo(fid) +% #MWC - data block 2: receiver, general info +% +% Verified correct for kmall format revisions F-I + +% Number of bytes in current struct. +out_struct.numBytesRxInfo = fread(fid,1,'uint16'); + +% Number of beams in this datagram (Nrx). +out_struct.numBeams = fread(fid,1,'uint16'); + +% Bytes in EMdgmMWCrxBeamData struct, excluding sample amplitudes (which +% have varying lengths) +out_struct.numBytesPerBeamEntry = fread(fid,1,'uint8'); + +% 0 = off +% 1 = low resolution +% 2 = high resolution +out_struct.phaseFlag = fread(fid,1,'uint8'); + +% Time Varying Gain function applied (X). X log R + 2 Alpha R + OFS + C, +% where X and C is documented in #MWC datagram. OFS is gain offset to +% compensate for TX source level, receiver sensitivity etc. +out_struct.TVGfunctionApplied = fread(fid,1,'uint8'); + +% Time Varying Gain offset used (OFS), unit dB. X log R + 2 Alpha R + OFS + +% C, where X and C is documented in #MWC datagram. OFS is gain offset to +% compensate for TX source level, receiver sensitivity etc. +out_struct.TVGoffset_dB = fread(fid,1,'int8'); + +% The sample rate is normally decimated to be approximately the same as the +% bandwidth of the transmitted pulse. Unit hertz. +out_struct.sampleFreq_Hz = fread(fid,1,'float'); + +% Sound speed at transducer, unit m/s. +out_struct.soundVelocity_mPerSec = fread(fid,1,'float'); + +end + + +function out_struct = CFF_read_EMdgmMWCrxBeamData(fid, phaseFlag, Nrx, MWC_VERSION) +% #MWC - data block 2: receiver, specific info for each beam. +% +% Verified correct for kmall format revisions F-I + +for iRx = 1:Nrx + + out_struct.beamPointAngReVertical_deg(iRx) = fread(fid,1,'float'); + + out_struct.startRangeSampleNum(iRx) = fread(fid,1,'uint16'); + + % Two way range in samples. Approximation to calculated distance from + % tx to bottom detection [meters] = soundVelocity_mPerSec * + % detectedRangeInSamples / (sampleFreq_Hz * 2). The detected range is + % set to zero when the beam has no bottom detection. Replaced by + % detectedRangeInSamplesHighResolution for higher precision. + out_struct.detectedRangeInSamples(iRx) = fread(fid,1,'uint16'); + + out_struct.beamTxSectorNum(iRx) = fread(fid,1,'uint16'); + + % Number of sample data for current beam. Also denoted Ns. + out_struct.numSampleData(iRx) = fread(fid,1,'uint16'); + + if MWC_VERSION >= 1 + % The same information as in detectedRangeInSamples with higher + % resolution. Two way range in samples. Approximation to calculated + % distance from tx to bottom detection [meters] = soundVelocity_mPerSec + % * detectedRangeInSamples / (sampleFreq_Hz * 2). The detected range is + % set to zero when the beam has no bottom detection. + out_struct.detectedRangeInSamplesHighResolution(iRx) = fread(fid,1,'float'); + end + + Ns = out_struct.numSampleData(iRx); + + + % ------------------ OPTION 1: ACTUALLY READ DATA --------------------- + % + % % Pointer to start of array with Water Column data. Lenght of array = + % % numSampleData. Sample amplitudes in 0.5 dB resolution. Size of + % % array is numSampleData * int8_t. Amplitude array is followed by + % % phase information if phaseFlag >0. Use (numSampleData * int8_t) to + % % jump to next beam, or to start of phase info for this beam, if + % % phase flag > 0. + % out_struct.sampleAmplitude05dB_p = fread(fid,Ns,'int8'); + % + % switch phaseFlag + % % #MWC - Beam sample phase info, specific for each beam and water + % % column sample. numBeams * numSampleData = (Nrx * Ns) entries. + % case 1 + % % Only added to datagram if phaseFlag = 1. Total size of + % % phase block is numSampleData * int8_t. + % + % % Rx beam phase in 180/128 degree resolution. + % out_struct.rxBeamPhase = fread(fid,Ns,'int8'); + % + % case 2 + % % Only added to datagram if phaseFlag = 2. Total size of + % % phase block is numSampleData * int16_t. + % + % % Rx beam phase in 0.01 degree resolution. + % out_struct.rxBeamPhase = fread(fid,Ns,'int16'); + % + % end + % + % ------------------ END OF OPTION 1 ---------------------------------- + + + % ------------------ OPTION 2: SAVING POSITION IN FILE ---------------- + % instead of reading file as above, we save the position in file for + % later reading. + pif = ftell(fid); + out_struct.sampleDataPositionInFile(iRx) = pif; + + % we still need to fast-forward to the end of the data section so that + % reading can continue from there + dataBlockSizeInBytes = Ns.*(1+phaseFlag); + fseek(fid,dataBlockSizeInBytes,0); + + % ------------------ END OF OPTION 2 ---------------------------------- + +end + +end diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMbody.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMbody.m new file mode 100644 index 0000000..fe81bc6 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMbody.m @@ -0,0 +1,61 @@ +function out_struct = CFF_read_EMdgmMbody(fid) +%CFF_READ_EMDGMMBODY Read Body part of a Multibeam datagram in kmall file +% +% Multibeam (M) datagrams - body part. Start of body of all M datagrams. +% +% Contains information of transmitter and receiver used to find data in +% datagram. The table below illustrates how the indexes will be filled +% out in different system configurations. Each vertical column is data +% from one datagram. See index description table and figure below for +% more information. See the chapter for #MRZ datagram, for more details. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +% Used for denoting size of current struct, EMdgmMbody_def. +out_struct.numBytesCmnPart = fread(fid,1,'uint16'); + +% A ping is made of one or more RX fans and one or more TX pulses +% transmitted at approximately the same time. Ping counter is incremented +% at every set of TX pulses (one or more pulses transmitted at +% approximately the same time). +out_struct.pingCnt = fread(fid,1,'uint16'); + +% Number of rx fans per ping gives information of how many #MRZ datagrams +% are generated per ping. Combined with swathsPerPing, number of datagrams +% to join for a complete swath can be found. +out_struct.rxFansPerPing = fread(fid,1,'uint8'); + +% Index 0 is the aft swath, port side. +out_struct.rxFanIndex = fread(fid,1,'uint8'); + +% Number of swaths per ping. A swath is a complete set of across track +% data. A swath may contain several transmit sectors and RX fans. +out_struct.swathsPerPing = fread(fid,1,'uint8'); + +% Alongship index for the location of the swath in multi swath mode. Index +% 0 is the aftmost swath. +out_struct.swathAlongPosition = fread(fid,1,'uint8'); + +% Transducer used in this rx fan. Index: +% 0 = TRAI_TX1 +% 1 = TRAI_TX2 etc. +out_struct.txTransducerInd = fread(fid,1,'uint8'); + +% Transducer used in this rx fan. Index: +% 0 = TRAI_RX1 +% 1 = TRAI_RX2 etc. +out_struct.rxTransducerInd = fread(fid,1,'uint8'); + +% Total number of receiving units. +out_struct.numRxTransducers = fread(fid,1,'uint8'); + +% For future use. 0 - current algorithm, >0 - future algorithms. +out_struct.algorithmType = fread(fid,1,'uint8'); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMpartition.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMpartition.m new file mode 100644 index 0000000..4ae8300 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmMpartition.m @@ -0,0 +1,32 @@ +function out_struct = CFF_read_EMdgmMpartition(fid) +%CFF_READ_EMDGMMPARTITION Read Partition of M datagram in kmall file +% +% Multibeam (M) datagrams - data partition information. General for all M +% datagrams. +% +% If a multibeam depth datagram (or any other large datagram) exceeds the +% limit of an UDP package (64 kB), the datagram is split into several +% datagrams =< 64 kB before sending from the PU. The parameters in this +% struct will give information of the partitioning of datagrams. +% K-Controller/SIS merges all UDP packets/datagram parts to one datagram, +% and store it as one datagram in the .kmall files. Datagrams stored in +% .kmall files will therefore always have numOfDgm = 1 and dgmNum = 1, +% and may have size > 64 kB. The maximum number of partitions from PU is +% given by MAX_NUM_MWC_DGMS and MAX_NUM_MRZ_DGMS. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +% Number of datagram parts to re-join to get one Multibeam datagram. E.g. +% 3. +out_struct.numOfDgms = fread(fid,1,'uint16'); + +% Datagram part number, e.g. 2 (of 3). +out_struct.dgmNum = fread(fid,1,'uint16'); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSCL.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSCL.m new file mode 100644 index 0000000..4517009 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSCL.m @@ -0,0 +1,56 @@ +function out_struct = CFF_read_EMdgmSCL(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMSCL Read kmall structure #SCL +% +% #SCL - CLock datagram. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +SCL_VERSION = out_struct.header.dgmVersion; +if SCL_VERSION>0 && dgmVersion_warning_flag + % definitions in this function and subfunctions valid for SCL_VERSION: + % 0 (kmall format revisions F-I, and presumably earlier ones?) + warning('#SCL datagram version (%i) unsupported. Continue reading but there may be issues.',SCL_VERSION); +end + +out_struct.cmnPart = CFF_read_EMdgmScommon(fid); + +% number of bytes in the actual SCL data is the total datagram size +% (need to remove 4 bytes for the final numBytes field) minus what was +% read in the header (20 bytes, as currently defined), the common part +% (8 bytes, as currently defined), and the data block until the actual +% data (8 bytes, as currently defined) +SCL_data_numBytes = (out_struct.header.numBytesDgm - 4) ... + - 20 ... + - out_struct.cmnPart.numBytesCmnPart ... + - 8; + +out_struct.sensData = CFF_read_EMdgmSCLdataFromSensor(fid, SCL_data_numBytes); + +end + +function out_struct = CFF_read_EMdgmSCLdataFromSensor(fid, SCL_data_numBytes) +% Part of clock datagram giving offsets and the raw input in text format. +% +% Verified correct for kmall format revisions F-I + +% Offset in seconds from K-Controller operator input. +out_struct.offset_sec = fread(fid,1,'float'); + +% Clock deviation from PU. Difference between time stamp at receive of +% sensor data and time in the clock source. Unit nanoseconds. Difference +% smaller than +/- 1 second if 1PPS is active and sync from ZDA. +out_struct.clockDevPU_nanosec = fread(fid,1,'int32'); + +% Clock data as received from sensor, in text format. Data are uncorrected +% for offsets. +out_struct.dataFromSensor = fscanf(fid, '%c', SCL_data_numBytes); + +end diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSKM.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSKM.m new file mode 100644 index 0000000..593cba9 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSKM.m @@ -0,0 +1,280 @@ +function out_struct = CFF_read_EMdgmSKM(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMSKM Read kmall structure #SKM +% +% #SKM - data from attitude and attitude velocity sensors. +% +% Datagram may contain several sensor measurements. The number of samples +% in datagram is listed in numSamplesArray in the struct +% EMdgmSKMinfo_def. Time given in datagram header, is time of arrival of +% data on serial line or on network. Time inside #KMB sample is time from +% the sensors data. If input is other than KM binary sensor input format, +% the data are converted to the KM binary format by the PU. All +% parameters are uncorrected. For processing of data, installation +% offsets, installation angles and attitude values are needed to correct +% the data for motion. +% +% Verified correct for kmall format revisions F,I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +SKM_VERSION = out_struct.header.dgmVersion; +if SKM_VERSION~=1 && dgmVersion_warning_flag + % definitions in this function and subfunctions valid for SKM_VERSION: + % 1 (kmall format revisions C-I) + warning('#SKM datagram version (%i) unsupported. Continue reading but there may be issues.',SKM_VERSION); +end + +out_struct.infoPart = CFF_read_EMdgmSKMinfo(fid); + +Nsamp = out_struct.infoPart.numSamplesArray; +for iS = 1:Nsamp + out_struct.sample(iS) = CFF_read_EMdgmSKMsample_def(fid); +end + +end + + +function out_struct = CFF_read_EMdgmSKMinfo(fid) +% Sensor (S) output datagram - info of KMB datagrams. +% +% Verified correct for kmall versions F-I + +% Size in bytes of current struct. Used for denoting size of rest of +% datagram in cases where only one datablock is attached. +out_struct.numBytesInfoPart = fread(fid,1,'uint16'); + +% Attitude system number, as numbered in installation parameters. E.g. +% system 0 referes to system ATTI_1 in installation datagram #IIP. +out_struct.sensorSystem = fread(fid,1,'uint8'); + +% Sensor status. Summarise the status fields of all KM binary samples added +% in this datagram (status in struct KMbinary_def). Only available data +% from input sensor format is summarised. Available data found in +% sensorDataContents. +% Bits 0 -7 common to all sensors and #MRZ sensor status: +% Bit number Sensor data +% 0 1 = Sensor is chosen as active +% 1 0 +% 2 0 = Data OK +% 1 = Reduced performance +% 3 0 +% 4 0 = Data OK +% 1 = Invalid data +% 5 0 +% 6 0 = Velocity from sensor +% 1 = Velocity calculated by PU +out_struct.sensorStatus = fread(fid,1,'uint8'); + +% Format of raw data from input sensor, given in numerical code according +% to table below. +% Code Sensor format +% 1 KM binary Sensor Input +% 2 EM 3000 data +% 3 Sagem +% 4 Seapath binary 11 +% 5 Seapath binary 23 +% 6 Seapath binary 26 +% 7 POS M/V GRP 102/103 +out_struct.sensorInputFormat = fread(fid,1,'uint16'); + +% Number of KM binary sensor samples added in this datagram. +out_struct.numSamplesArray = fread(fid,1,'uint16'); + +% Length in bytes of one whole KM binary sensor sample. +out_struct.numBytesPerSample = fread(fid,1,'uint16'); + +% Field to indicate which information is available from the input sensor, +% at the given sensor format. +% 0 = not available +% 1 = data is available +% The bit pattern is used to detemine sensorStatus from status field in +% #KMB samples. Only data available from sensor is check up against +% invalid/reduced performance in status, and summaries in sensorStatus. +% E.g. the binary 23 format does not contain delayed heave. This is +% indicated by setting bit 6 in sensorDataContents to 0. In each sample in +% #KMB output from PU, the status field (struct KMbinary_def) for INVALID +% delayed heave (bit 6) is set to 1. The summaries sensorStatus in struct +% EMdgmSKMinfo_def will then be sets to 0 if all available data is ok. +% Expected data field in sensor input: +% +% Bit number Sensor data +% 0 Horizontal position and velocity +% 1 Roll and pitch +% 2 Heading +% 3 In Rev H: Heave and vertical velocity +% In Rev I: Heave +% 4 Acceleration +% 5 Error fields +% 6 Delayed heave +out_struct.sensorDataContents = fread(fid,1,'uint16'); + +end + +function out_struct = CFF_read_EMdgmSKMsample_def(fid) +% #SKM - all available data. +% +% An implementation of the KM Binary sensor input format. +% +% Verified correct for kmall format revisions F-I + +out_struct.KMdefault = CFF_read_KMbinary(fid); +out_struct.delayedHeave = CFF_read_KMdelayedHeave(fid); + +end + + +function out_struct = CFF_read_KMbinary(fid) +% #SKM - Sensor attitude data block. Data given timestamped, not corrected. +% +% See Coordinate systems for definition of positive angles and axis. +% +% Verified correct for kmall format revisions F-I + +% #KMB +out_struct.dgmType = fscanf(fid,'%c',4); + +% Datagram length in bytes. The length field at the start (4 bytes) and end +% of the datagram (4 bytes) are included in the length count. +out_struct.numBytesDgm = fread(fid,1,'uint16'); + +% Datagram version. +out_struct.dgmVersion = fread(fid,1,'uint16'); + +% UTC time from inside KM sensor data. Unit second. Epoch 1970-01-01 time. +% Nanosec part to be added for more exact time. +% If time is unavailable from attitude sensor input, time of reception on +% serial port is added to this field. +out_struct.time_sec = fread(fid,1,'uint32'); + +% Nano seconds remainder. Nanosec part to be added to time_sec for more +% exact time. +% If time is unavailable from attitude sensor input, time of reception on +% serial port is added to this field. +out_struct.time_nanosec = fread(fid,1,'uint32'); + +% Bit pattern for indicating validity of sensor data, and reduced +% performance. The status word consists of 32 single bit flags numbered +% from 0 to 31, where 0 is the least significant bit. +% Bit number 0-7 indicate if from a sensor data is invalid. 0 = valid data, +% 1 = invalid data. +% Bit number 16-> indicate if data from sensor has reduced performance. 0 = +% valid data, 1 = reduced performance. +% +% Invalid data: +% +% Bit number Sensor data +% 0 Horizontal position and velocity +% 1 Roll and pitch +% 2 Heading +% 3 Heave and vertical velocity +% 4 Acceleration +% 5 For Rev H: Error fields +% For Rev I: Delayed heave +% 6 Delayed heave +% +% Reduced performance: +% +% Bit number Sensor data +% 16 Horizontal position and velocity +% 17 Roll and pitch +% 18 Heading +% 19 Heave and vertical velocity +% 20 Acceleration +% 21 For Rev H: Error fields +% For Rev I: Delayed heave +% 22 Delayed heave +out_struct.status = fread(fid,1,'uint32'); + +% Position in decimal degrees. +out_struct.latitude_deg = fread(fid,1,'double'); + +% Position in decimal degrees. +out_struct.longitude_deg = fread(fid,1,'double'); + +% Height of sensor reference point above the ellipsoid. Positive above +% ellipsoid. ellipsoidHeight_m is not corrected for motion and installation +% offsets of the position sensor. +out_struct.ellipsoidHeight_m = fread(fid,1,'float'); + +% Roll. Unit degree. +out_struct.roll_deg = fread(fid,1,'float'); + +% Pitch. Unit degree +out_struct.pitch_deg = fread(fid,1,'float'); + +% Heading of vessel. Unit degree. Relative to the fixed coordinate system, +% i.e. true north. +out_struct.heading_deg = fread(fid,1,'float'); + +% Heave. Unit meter. Positive downwards. +out_struct.heave_m = fread(fid,1,'float'); + +% Roll rate. Unit degree/s +out_struct.rollRate = fread(fid,1,'float'); + +% Pitch rate. Unit degree/s +out_struct.pitchRate = fread(fid,1,'float'); + +% Yaw (heading) rate. Unit degree/s +out_struct.yawRate = fread(fid,1,'float'); + +% Velocity North (X). Unit m/s +out_struct.velNorth = fread(fid,1,'float'); + +% Velocity East (Y). Unit m/s +out_struct.velEast = fread(fid,1,'float'); + +% Velocity downwards (Z). Unit m/s +out_struct.velDown = fread(fid,1,'float'); + +% Latitude error. Unit meter. +out_struct.latitudeError_m = fread(fid,1,'float'); + +% Longitude error. Unit meter. +out_struct.longitudeError_m = fread(fid,1,'float'); + +% Ellipsoid height error. Unit meter. +out_struct.ellipsoidHeightError_m = fread(fid,1,'float'); + +% Roll error. Unit degree. +out_struct.rollError_deg = fread(fid,1,'float'); + +% Pitch error. Unit degree. +out_struct.pitchError_deg = fread(fid,1,'float'); + +% Heading error. Unit degree. +out_struct.headingError_deg = fread(fid,1,'float'); + +% Heave error. Unit meter. +out_struct.heaveError_m = fread(fid,1,'float'); + +% Unit m/s^2. +out_struct.northAcceleration = fread(fid,1,'float'); + +% Unit m/s^2. +out_struct.eastAcceleration = fread(fid,1,'float'); + +% Unit m/s^2. +out_struct.downAcceleration = fread(fid,1,'float'); + +end + +function out_struct = CFF_read_KMdelayedHeave(fid) +% #SKM - delayed heave. Included if available from sensor. +% +% Verified correct for kmall format revisions F-I + +out_struct.time_sec = fread(fid,1,'uint32'); +out_struct.time_nanosec = fread(fid,1,'uint32'); + +% Delayed heave. Unit meter. +out_struct.delayedHeave_m = fread(fid,1,'float'); + +end + diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSPO.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSPO.m new file mode 100644 index 0000000..9691974 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSPO.m @@ -0,0 +1,101 @@ +function out_struct = CFF_read_EMdgmSPO(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMSPO Read kmall structure #SPO +% +% #SPO - Struct of position sensor datagram. +% +% Data from active sensor will be motion corrected if indicated by +% operator. Motion correction is applied to latitude, longitude, speed, +% course and ellipsoidal height. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +SPO_VERSION = out_struct.header.dgmVersion; +if SPO_VERSION>0 && dgmVersion_warning_flag + % definitions in this function and subfunctions valid for SPO_VERSION: + % 0 (kmall format revisions F-I, and presumably earlier ones?) + warning('#SPO datagram version (%i) unsupported. Continue reading but there may be issues.',SPO_VERSION); +end + +out_struct.cmnPart = CFF_read_EMdgmScommon(fid); + +% number of bytes in the actual SPO data is the total datagram size +% (need to remove 4 bytes for the final numBytes field) minus what was +% read in the header (20 bytes, as currently defined), the common part +% (8 bytes, as currently defined), and the data block until the actual +% data (40 bytes, as currently defined) +SPO_data_numBytes = (out_struct.header.numBytesDgm - 4) ... + - 20 ... + - out_struct.cmnPart.numBytesCmnPart ... + - 40; + +out_struct.sensorData = CFF_read_EMdgmSPOdataBlock(fid, SPO_data_numBytes); + +end + + +function out_struct = CFF_read_EMdgmSPOdataBlock(fid, SPO_data_numBytes) +% #SPO - Sensor position data block. Data from active sensor is corrected +% data for position system installation parameters. Data is also corrected +% for motion ( roll and pitch only) if enabled by K-Controller operator. +% Data given both decoded and corrected (active sensors), and raw as +% received from sensor in text string. +% +% Verified correct for kmall format revisions F-I + +% UTC time from position sensor. Unit seconds. Epoch 1970-01-01. Nanosec +% part to be added for more exact time. +out_struct.timeFromSensor_sec = fread(fid,1,'uint32'); + +% UTC time from position sensor. Unit nano seconds remainder. +out_struct.timeFromSensor_nanosec = fread(fid,1,'uint32'); + +% Only if available as input from sensor. Calculation according to format. +out_struct.posFixQuality_m = fread(fid,1,'float'); + +% Motion corrected (if enabled in K-Controller) data as used in depth +% calculations. Referred to vessel reference point. Unit decimal degree. +% For Rev H: Parameter is set to define UNAVAILABLE_LATITUDE if sensor +% inactive. +out_struct.correctedLat_deg = fread(fid,1,'double'); + +% Motion corrected (if enabled in K-Controller) data as used in depth +% calculations. Referred to vessel reference point. Unit decimal degree. +% For Rev H: Parameter is set to define UNAVAILABLE_LONGITUDE if sensor +% inactive. +out_struct.correctedLong_deg = fread(fid,1,'double'); + +% Speed over ground. Unit m/s. Motion corrected (if enabled in +% K-Controller) data as used in depth calculations. +% For Rev H: If unavailable or from inactive sensor, value set to define +% UNAVAILABLE_SPEED. +% For Rev I: If unavailable, the value set to define UNAVAILABLE_SPEED. +out_struct.speedOverGround_mPerSec = fread(fid,1,'float'); + +% Course over ground. Unit degree. Motion corrected (if enabled in +% K-Controller) data as used in depth calculations. +% For Rev H: If unavailable or from inactive sensor, value set to define +% UNAVAILABLE_COURSE. +% For Rev I: If unavailable, the value set to define UNAVAILABLE_COURSE. +out_struct.courseOverGround_deg = fread(fid,1,'float'); + +% Height of vessel reference point above the ellipsoid. Unit meter. Motion +% corrected (if enabled in K-Controller) data as used in depth +% calculations. +% For Rev H: If unavailable or from inactive sensor, value set to define +% UNAVAILABLE_ELLIPSOIDHEIGHT. +% For Rev I: If unavailable, value set to define +% UNAVAILABLE_ELLIPSOIDHEIGHT. +out_struct.ellipsoidHeightReRefPoint_m = fread(fid,1,'float'); + +% Position data as received from sensor, i.e. uncorrected for motion etc. +out_struct.posDataFromSensor = fscanf(fid, '%c', SPO_data_numBytes); + +end diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSVP.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSVP.m new file mode 100644 index 0000000..15e5885 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSVP.m @@ -0,0 +1,84 @@ +function out_struct = CFF_read_EMdgmSVP(fid, dgmVersion_warning_flag) +%CFF_READ_EMDGMSVP Read kmall structure #SVP +% +% #SVP - Sound Velocity Profile. Data from sound velocity profile or from +% CTD profile. Sound velocity is measured directly or estimated, +% respectively. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +out_struct.header = CFF_read_EMdgmHeader(fid); + +SVP_VERSION = out_struct.header.dgmVersion; +if SVP_VERSION~=1 && dgmVersion_warning_flag + % definitions in this function and subfunctions valid for SVP_VERSION: + % 1 (kmall format revisions F-I) + warning('#SVP datagram version (%i) unsupported. Continue reading but there may be issues.',SVP_VERSION); +end + +% Size in bytes of body part struct. Used for denoting size of rest of +% the datagram. +out_struct.numBytesCmnPart = fread(fid,1,'uint16'); + +% Number of sound velocity samples. +out_struct.numSamples = fread(fid,1,'uint16'); + +% Sound velocity profile format: +% 'S00' = sound velocity profile +% 'S01' = CTD profile +out_struct.sensorFormat = fscanf(fid,'%c',4); + +% Time extracted from the Sound Velocity Profile. Parameter is set to +% zero if not found. +out_struct.time_sec = fread(fid,1,'uint32'); + +% Latitude in degrees. Negative if southern hemisphere. Position +% extracted from the Sound Velocity Profile. Parameter is set to define +% UNAVAILABLE_LATITUDE if not available. +out_struct.latitude_deg = fread(fid,1,'double'); + +% Longitude in degrees. Negative if western hemisphere. Position +% extracted from the Sound Velocity Profile. Parameter is set to define +% UNAVAILABLE_LONGITUDE if not available. +out_struct.longitude_deg = fread(fid,1,'double'); + +% SVP point samples, repeated numSamples times. +for iS = 1:out_struct.numSamples + out_struct.sensorData(iS) = CFF_read_EMdgmSVPpoint(fid); +end + +end + + +function out_struct = CFF_read_EMdgmSVPpoint(fid) +% #SVP - Sound Velocity Profile. Data from one depth point contains +% information specified in this struct. +% +% Verified correct for kmall format revisions F-I + +% Depth at which measurement is taken. Unit m. Valid range from 0.00 m to +% 12000 m. +out_struct.depth_m = fread(fid,1,'float'); + +% Measured sound velocity from profile. Unit m/s. For a CTD profile, this +% will be the calculated sound velocity. +out_struct.soundVelocity_mPerSec = fread(fid,1,'float'); + +% Former absorption coefficient. Voided. +out_struct.padding = fread(fid,1,'uint32'); + +% Water temperature at given depth. Unit Celsius. For a Sound velocity +% profile (S00), this will be set to 0.00. +out_struct.temp_C = fread(fid,1,'float'); + +% Salinity of water at given depth. For a Sound velocity profile (S00), +% this will be set to 0.00. +out_struct.salinity = fread(fid,1,'float'); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmScommon.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmScommon.m new file mode 100644 index 0000000..e6a33f1 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmScommon.m @@ -0,0 +1,68 @@ +function out_struct = CFF_read_EMdgmScommon(fid) +%CFF_READ_EMDGMSCOMMON Read common part of Sensor struct of kmall file +% +% Sensor (S) output datagram - common part for all external sensors. +% +% Verified correct for kmall format revisions F-I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 20-08-2021 + +% Size in bytes of current struct. +out_struct.numBytesCmnPart = fread(fid,1,'uint16'); + +% Sensor system number, as indicated when setting up the system in +% K-Controller installation menu. E.g. position system 0 referes to system +% POSI_1 in installation datagram #IIP. Check if this sensor system is +% active by using #IIP datagram. +% +% #SCL - clock datagram: +% Bit number Sensor system +% 0 Time syncronisation from clock data +% 1 Time syncronisation from active position data +% 2 1 PPS is used +out_struct.sensorSystem = fread(fid,1,'uint16'); + +% Sensor status. To indicate quality of sensor data is valid or invalid. +% Quality may be invalid even if sensor is active and the PU receives data. +% Bit code vary according to type of sensor. +% +% Bits 0 -7 common to all sensors and #MRZ sensor status: +% Bit number Sensor data +% 0 For rev H: 0 = Data OK, 1 = Data OK and sensor is chosen as active +% For rev I: 1 = Sensor is chosen as active +% #SCL only: 1 = Valid data and 1PPS OK +% 1 0 +% 2 0 = Data OK +% 1 = Reduced performance +% #SCL only: 1 = Reduced performance, no time synchronisation of PU +% 3 0 +% 4 0 = Data OK +% 1 = Invalid data +% 5 0 +% 6 0 = Velocity from sensor +% 1 = Velocity calculated by PU +% 7 0 +% +% For #SPO (position) and CPO (position compatibility) datagrams, bit 8-15: +% +% Bit number Sensor data +% 8 0 +% 9 0 = Time from PU used (system) +% 1 = Time from datagram used (e.g. from GGA telegram) +% 10 0 = No motion correction +% 1 = With motion correction +% 11 0 = Normal quality check +% 1 = Operator quality check. Data always valid +% 12 0 +% 13 0 +% 14 0 +% 15 0 +out_struct.sensorStatus = fread(fid,1,'uint16'); + +out_struct.padding = fread(fid,1,'uint16'); + +end \ No newline at end of file diff --git a/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSdataInfo.m b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSdataInfo.m new file mode 100644 index 0000000..1596d04 --- /dev/null +++ b/read_data_files/Kongsberg/format_kmall/kmall_struct/CFF_read_EMdgmSdataInfo.m @@ -0,0 +1,29 @@ +function out_struct = CFF_read_EMdgmSdataInfo(fid) +%CFF_READ_EMDGMSDATAINFO Read Data section of S datagram in kmall file +% +% Information of repeated sensor data in one datagram. +% +% Info about data from sensor. Part included if data from sensor appears +% multiple times in a datagram. +% +% Verified correct for kmall versions H,I +% +% See also CFF_READ_KMALL_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +% Size in bytes of current struct. +out_struct.numBytesInfoPart = fread(fid,1,'uint16'); + +% Number of sensor samples added in datagram. +out_struct.numSamplesArray = fread(fid,1,'uint16'); + +% Length in bytes of one whole sample (decoded and raw data). +out_struct.numBytesPerSample = fread(fid,1,'uint16'); + +% Length in bytes of raw sensor data. +out_struct.numBytesRawSensorData = fread(fid,1,'uint16'); + +end \ No newline at end of file diff --git a/read_data_files/Reson/CFF_check_S7Kfilename.m b/read_data_files/Reson/CFF_check_S7Kfilename.m new file mode 100644 index 0000000..f1356e8 --- /dev/null +++ b/read_data_files/Reson/CFF_check_S7Kfilename.m @@ -0,0 +1,13 @@ +function out = CFF_check_S7Kfilename(rawfilename) +%CFF_CHECK_S7KFILENAME Check file exists and has s7k extension +% +% out = CFF_CHECK_S7KFILENAME(rawFile) checks if single file rawFile +% (char) exists and has '.s7k' extension. +% +% See also CFF_CHECK_FILENAME + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 26-08-2021 + +out = CFF_check_filename(rawfilename,{'.s7k'}); \ No newline at end of file diff --git a/read_data_files/Reson/CFF_convert_S7Kdata_to_fData.m b/read_data_files/Reson/CFF_convert_S7Kdata_to_fData.m new file mode 100644 index 0000000..370389e --- /dev/null +++ b/read_data_files/Reson/CFF_convert_S7Kdata_to_fData.m @@ -0,0 +1,757 @@ +function fData = CFF_convert_S7Kdata_to_fData(S7Kdata,varargin) +%CFF_CONVERT_S7KDATA_TO_FDATA Convert s7k data to the CoFFee format +% +% Converts Teledyne-Reson data FROM the S7Kdata format (read by +% CFF_READ_S7K) TO the CoFFee fData format used in processing. +% IMPORTANT NOTE: THE FDATA FORMAT WAS NOT DESIGNED TO BE A GENERIC +% FORMAT BUT A MATLAB VERSION OF THE KONGSBERG *.ALL FORMAT. AS A RESULT, +% CONVERSION FROM S7K IS NOT OPTIMAL AND REQUIRES SOME TWEAKS. ALSO, WE +% ONLY POPULATE THE FDATA FIELDS THAT ARE ABSOLUTELY NECESSARY FOR +% WATER-COLUMN DISPLAY. YOU MIGHT EXPERIENCE ISSUES TRYING TO DO ANYTHING +% ELSE WITH THAT FDATA. CHECK CFF_CONVERT_ALLDATA_TO_FDATA TO GET AN IDEA +% OF WHAT ALL THOSE FIELDS ACTUALLY ARE. +% +% FDATA = CFF_CONVERT_S7KDATA_TO_FDATA(S7KDATA) converts the contents of +% the S7KDATA structure to a structure in the fData format. +% +% FDATA = CFF_CONVERT_S7KDATA_TO_FDATA(S7KDATA,DR_SUB,DB_SUB) operates +% the conversion with a sub-sampling of the water-column data in range +% and in beams. For example, to sub-sample range by a factor of 10 and +% beams by a factor of 2, use: +% FDATA = CFF_CONVERT_S7KDATA_TO_FDATA(S7KDATA,10,2). +% +% See also CFF_CONVERT_RAW_FILES, CFF_READ_S7K, +% CFF_CONVERT_ALLDATA_TO_FDATA + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 21-07-2022 + + +%% Input arguments management +p = inputParser; + +% array of S7Kdata structures +addRequired(p,'S7Kdata',@(x) isstruct(x) || iscell(x)); + +% decimation factor in range and beam +addOptional(p,'dr_sub',1,@(x) isnumeric(x)&&x>0); +addOptional(p,'db_sub',1,@(x) isnumeric(x)&&x>0); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,S7Kdata,varargin{:}) + +% and get results +S7Kdata = p.Results.S7Kdata; +dr_sub = p.Results.dr_sub; +db_sub = p.Results.db_sub; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end +clear p; + +if iscell(S7Kdata) + S7Kdata = S7Kdata{1}; +end + +% check it's from Teledyne-Reson and that source file exist +has_S7Kfilename = isfield(S7Kdata, 'S7Kfilename'); +if ~has_S7Kfilename || ~CFF_check_S7Kfilename(S7Kdata.S7Kfilename) + error('Invalid input'); +end + + +%% Prep + +% start message +comms.start('Converting to fData format'); + +% initialize fData, with current version number +fData.MET_Fmt_version = CFF_get_current_fData_version(); + +% add source filename +S7Kfilename = S7Kdata.S7Kfilename; +fData.ALLfilename{1} = S7Kfilename; + +% start progress +comms.progress(0,6); + +% now populating fields where we can + + +%% Settings + +comms.step('Converting Settings'); + +% sonar heading offset in degrees +% to check where we can find it... In the meantime, leave it at zero +fData.IP_ASCIIparameters.S1H = 0; + +if isfield(S7Kdata,'R7000_SonarSettings') + fData.Ru_1D_Date = S7Kdata.R7000_SonarSettings.Date; + fData.Ru_1D_TimeSinceMidnightInMilliseconds = S7Kdata.R7000_SonarSettings.TimeSinceMidnightInMilliseconds; + fData.Ru_1D_PingCounter = S7Kdata.R7000_SonarSettings.PingNumber; + fData.Ru_1D_TransmitPowerReMaximum = pow2db(S7Kdata.R7000_SonarSettings.PowerSelection); + fData.Ru_1D_ReceiveBeamwidth = S7Kdata.R7000_SonarSettings.ReceiveBeamWidthRad/pi*180; +end + +comms.progress(1,6); + +%% Navigation + +comms.step('Converting Navigation data'); + +if isfield(S7Kdata,'R1015_Navigation') + fData.Po_1D_Date = S7Kdata.R1015_Navigation.Date; + fData.Po_1D_TimeSinceMidnightInMilliseconds = S7Kdata.R1015_Navigation.TimeSinceMidnightInMilliseconds; + fData.Po_1D_PositionCounter = S7Kdata.R1015_Navigation.PositionCounter; + fData.Po_1D_Latitude = S7Kdata.R1015_Navigation.Latitude/pi*180; % now in deg + fData.Po_1D_Longitude = S7Kdata.R1015_Navigation.Longitude/pi*180; % now in deg + fData.Po_1D_SpeedOfVesselOverGround = S7Kdata.R1015_Navigation.SpeedOfVesselOverGround; + fData.Po_1D_HeadingOfVessel = S7Kdata.R1015_Navigation.Heading/pi*180; % now in deg + fData.Po_1D_MeasureOfPositionFixQuality = S7Kdata.R1015_Navigation.HorizontalPositionAccuracy; + fData.Po_1D_PositionSystemDescriptor = ones(size(S7Kdata.R1015_Navigation.Date)); % dummy values +elseif isfield(S7Kdata,'R1003_Position') + fData.Po_1D_Date = S7Kdata.R1003_Position.Date; + fData.Po_1D_TimeSinceMidnightInMilliseconds = S7Kdata.R1003_Position.TimeSinceMidnightInMilliseconds; + fData.Po_1D_PositionCounter = 1:numel(S7Kdata.R1003_Position.Date); % dummy values + fData.Po_1D_MeasureOfPositionFixQuality = ones(size(S7Kdata.R1003_Position.Date)); % dummy values + fData.Po_1D_PositionSystemDescriptor = S7Kdata.R1003_Position.PositioningMethod; + + if S7Kdata.R1003_Position.PositionTypeFlag(1) == 0 + % geographic coordinates (lat/long) in rad + fData.Po_1D_Latitude = S7Kdata.R1003_Position.LatitudeOrNorthing/pi*180; % now in deg + fData.Po_1D_Longitude = S7Kdata.R1003_Position.LongitudeOrEasting/pi*180; % now in deg + + % calculating speed of vessel and heading based on lat/long + nb_pt = numel(fData.Po_1D_Latitude); + [dist_in_deg,head] = distance([fData.Po_1D_Latitude(1:nb_pt-1)' fData.Po_1D_Longitude(1:nb_pt-1)'],[fData.Po_1D_Latitude(2:nb_pt)' fData.Po_1D_Longitude(2:nb_pt)']); + d_dist = deg2km(dist_in_deg'); + t = datenum(cellfun(@num2str,num2cell(fData.Po_1D_Date),'un',0),'yyyymmdd')'*24*60*60+fData.Po_1D_TimeSinceMidnightInMilliseconds/1e3; + s = d_dist*1000./diff(t); + fData.Po_1D_SpeedOfVesselOverGround = [s(1) s]; + fData.Po_1D_HeadingOfVessel = [head(1) head']; + + else + % grid coordinates (easting/northing) in meters + % Code here conversion back to geographic coordinates if we + % ever come across this case. + end +end + +comms.progress(2,6); + + +%% Height + +comms.step('Converting Height data'); + +if isfield(S7Kdata,'R1015_Navigation') + fData.He_1D_Date = S7Kdata.R1015_Navigation.Date; + fData.He_1D_TimeSinceMidnightInMilliseconds = S7Kdata.R1015_Navigation.TimeSinceMidnightInMilliseconds; + fData.He_1D_HeightCounter = 1:numel(S7Kdata.R1015_Navigation.Date); + fData.He_1D_Height = S7Kdata.R1015_Navigation.VesselHeight; +elseif isfield(S7Kdata,'R1003_Position') + fData.He_1D_Date = S7Kdata.R1003_Position.Date; + fData.He_1D_TimeSinceMidnightInMilliseconds = S7Kdata.R1003_Position.TimeSinceMidnightInMilliseconds; + fData.He_1D_HeightCounter = 1:numel(S7Kdata.R1003_Position.Date); + fData.He_1D_Height = S7Kdata.R1003_Position.Height; +end + +comms.progress(3,6); + + +%% seafloor data (bathy, BS) fields + +comms.step('Converting Seafloor data (bathy, BS)'); + +if isfield(S7Kdata,'R7027_RawDetectionData') + + % number of pings + nPings = numel(S7Kdata.R7027_RawDetectionData.PingNumber); + + % number of beams + % +1 because beam numbers in this field start at 0 + maxnBeams = nanmax(cellfun(@nanmax,S7Kdata.R7027_RawDetectionData.BeamDescriptor)) + 1; + + % date and time + fData.X8_1P_Date = S7Kdata.R7027_RawDetectionData.Date; + fData.X8_1P_TimeSinceMidnightInMilliseconds = S7Kdata.R7027_RawDetectionData.TimeSinceMidnightInMilliseconds; + + % record data per ping + fData.X8_1P_PingCounter = S7Kdata.R7027_RawDetectionData.PingNumber; + fData.X8_1P_HeadingOfVessel = NaN; % unused anyway + fData.X8_1P_SoundSpeedAtTransducer = NaN; % unused anyway + fData.X8_1P_TransmitTransducerDepth = NaN; % unused anyway + fData.X8_1P_NumberOfBeamsInDatagram = NaN; % unused anyway + fData.X8_1P_NumberOfValidDetections = NaN; % unused anyway + fData.X8_1P_SamplingFrequencyInHz = NaN; % unused anyway + + % initialize data per beam and ping + fData.X8_BP_DepthZ = nan(maxnBeams,nPings); + fData.X8_BP_AcrosstrackDistanceY = nan(maxnBeams,nPings); + fData.X8_BP_AlongtrackDistanceX = NaN; % unused anyway + fData.X8_BP_DetectionWindowLength = NaN; % unused anyway + fData.X8_BP_QualityFactor = NaN; % unused anyway + fData.X8_BP_BeamIncidenceAngleAdjustment = NaN; % unused anyway + fData.X8_BP_DetectionInformation = NaN; % unused anyway + fData.X8_BP_RealTimeCleaningInformation = NaN; % unused anyway + fData.X8_BP_ReflectivityBS = nan(maxnBeams,nPings); + fData.X8_B1_BeamNumber = (1:maxnBeams)'; + + % record data per beam and ping + for iP = 1:nPings + iBeam = S7Kdata.R7027_RawDetectionData.BeamDescriptor{iP}+1; % +1 because beam numbers in this field start at 0 + fData.X8_BP_DepthZ(iBeam,iP) = S7Kdata.R7027_RawDetectionData.Depth{iP}; + fData.X8_BP_AcrosstrackDistanceY(iBeam,iP) = S7Kdata.R7027_RawDetectionData.AcrossTrackDistance{iP}; + fData.X8_BP_ReflectivityBS(iBeam,iP) = S7Kdata.R7027_RawDetectionData.Intensity{iP}; + end + + % transform intensity to dB + % DEV NOTE: no idea where Yoann got this formula + fData.X8_BP_ReflectivityBS = 20*log10(fData.X8_BP_ReflectivityBS/65535); + + % debug graph + debugDisp = 0; + if debugDisp + f = figure(); + ax_z = axes(f,'outerposition',[0 0.66 1 0.3]); + imagesc(ax_z, -fData.X8_BP_DepthZ); + colorbar(ax_z); grid on; title(ax_z, 'bathy'); colormap(ax_z,'jet'); + ax_y = axes(f,'outerposition',[0 0.33 1 0.3]); + imagesc(ax_y, fData.X8_BP_AcrosstrackDistanceY); + colorbar(ax_y); grid on; title(ax_y, 'across-track distance'); + ax_bs = axes(f,'outerposition',[0 0 1 0.3]); + imagesc(ax_bs, fData.X8_BP_ReflectivityBS); + caxis(ax_bs, [prctile(fData.X8_BP_ReflectivityBS(:),5), prctile(fData.X8_BP_ReflectivityBS(:),95)]); + colorbar(ax_bs); grid on; title(ax_bs, 'BS (scaled 5-95th percentile)'); colormap(ax_bs,'gray'); + drawnow; + end + +end + +comms.progress(4,6); + + +%% water-column data (amplitude, phase) from 7018 records + +comms.step('Converting Water-column data (amplitude, phase)'); + +if all(isfield(S7Kdata,{'R7018_BeamformedData','R7000_SonarSettings','R7004_BeamGeometry', 'R7027_RawDetectionData'})) + + % I came across one file where not all pings are recorded, also with + % different records having different pings. Since we here combine data + % from different records, we first need to limit the recording to pings + % that are present in all records. To make it more complicated, + % R7004_BeamGeometry does not have a pingNumber field. + R7018pings = S7Kdata.R7018_BeamformedData.PingNumber; + R7000pings = S7Kdata.R7000_SonarSettings.PingNumber; + R7027pings = S7Kdata.R7027_RawDetectionData.PingNumber; + pingNumber = intersect(R7018pings,intersect(R7000pings,R7027pings)); + [~,ipR7018] = ismember(pingNumber,R7018pings); + [~,ipR7000] = ismember(pingNumber,R7000pings); + [~,ipR7027] = ismember(pingNumber,R7027pings); + + % for R7004_BeamGeometry, we will have to assume its contents are the + % same as one of the other record types + if numel(S7Kdata.R7004_BeamGeometry.SonarID) == numel(R7000pings) + ipR7004 = ipR7000; + elseif numel(S7Kdata.R7004_BeamGeometry.SonarID) == numel(R7027pings) + ipR7004 = ipR7027; + elseif numel(S7Kdata.R7004_BeamGeometry.SonarID) == numel(R7018pings) + ipR7004 = ipR7018; + else + error('cannot proceed...') + end + + % number of pings + nPings = numel(pingNumber); + + % number of beams + dtg_nBeams = S7Kdata.R7018_BeamformedData.N(ipR7018); % number of beams per ping + maxnBeams = nanmax(dtg_nBeams); % max number of beams in file + maxnBeams_sub = ceil(maxnBeams/db_sub); % maximum number of beams TO READ per ping + + % number of samples + dtg_nSamples = S7Kdata.R7018_BeamformedData.S(ipR7018); % number of samples per ping + [maxnSamples_groups,ping_group_start,ping_group_end] = CFF_group_pings(dtg_nSamples, pingNumber); % making groups of pings to limit size of memmaped files + maxnSamples_groups = ceil(maxnSamples_groups/dr_sub); % maximum number of samples TO READ, per group. + + % add the WCD decimation factors given here in input + fData.dr_sub = dr_sub; + fData.db_sub = db_sub; + + % data per ping + fData.AP_1P_Date = S7Kdata.R7018_BeamformedData.Date(ipR7018); + fData.AP_1P_TimeSinceMidnightInMilliseconds = S7Kdata.R7018_BeamformedData.TimeSinceMidnightInMilliseconds(ipR7018); + fData.AP_1P_PingCounter = pingNumber; + fData.AP_1P_NumberOfDatagrams = NaN; % unused anyway + fData.AP_1P_NumberOfTransmitSectors = NaN; % unused anyway + fData.AP_1P_TotalNumberOfReceiveBeams = NaN; % unused anyway + fData.AP_1P_SoundSpeed = S7Kdata.R7000_SonarSettings.SoundVelocity(ipR7000); + fData.AP_1P_SamplingFrequencyHz = S7Kdata.R7000_SonarSettings.SampleRate(ipR7000); % in Hz + fData.AP_1P_TXTimeHeave = NaN; % unused anyway + fData.AP_1P_TVGFunctionApplied = nan(size(pingNumber)); % dummy values. to find XXX1 + fData.AP_1P_TVGOffset = zeros(size(pingNumber)); % dummy values. to find XXX1 + fData.AP_1P_ScanningInfo = NaN; % unused anyway + + % initialize data per transmit sector and ping + fData.AP_TP_TiltAngle = NaN; % unused anyway + fData.AP_TP_CenterFrequency = NaN; % unused anyway + fData.AP_TP_TransmitSectorNumber = NaN; % unused anyway + + % initialize data per (decimated) beam and ping + fData.AP_BP_BeamPointingAngle = nan(maxnBeams_sub,nPings); + fData.AP_BP_StartRangeSampleNumber = nan(maxnBeams_sub,nPings); + fData.AP_BP_NumberOfSamples = nan(maxnBeams_sub,nPings); + fData.AP_BP_DetectedRangeInSamples = zeros(maxnBeams_sub,nPings); + fData.AP_BP_TransmitSectorNumber = NaN; % unused anyway + fData.AP_BP_BeamNumber = NaN; % unused anyway + fData.AP_BP_SystemSerialNumber = NaN; % unused anyway + + % The actual water-column data will not be saved in fData but in binary + % files. Get the output directory to store those files + wc_dir = CFF_converted_data_folder(S7Kfilename); + + % Clean up that folder first before adding anything to it + CFF_clean_delete_fdata(wc_dir); + + % DEV NOTE: Info format for raw WC data and storage + % In these raw datagrams, there are both amplitude and phase. + % + % Amplitude samples are in uint16. I don't have doc describing more + % than that but looking at Yoann code, they appear to be in natural + % values, scaled by intmax('uint16'). NaN value is likely + % intmin('uint16'). We need to convert those to dB so we can't reuse + % that format. Here we will store the data as int16 with a factor of + % 1/200. + % + % Phase samples are in radians, format int16, scaled by 10430, in order + % to fit between -pi and pi. We're going to store them as they are, in + % int16, but adjusting the factor so they are in degrees. + + % initialize data-holding binary files + fData = CFF_init_memmapfiles(fData, ... + 'field', 'AP_SBP_SampleAmplitudes', ... + 'wc_dir', wc_dir, ... + 'Class', 'int16', ... + 'Factor', 1/200, ... + 'Nanval', intmin('int16'), ... + 'Offset', 0, ... + 'MaxSamples', maxnSamples_groups, ... + 'MaxBeams', maxnBeams_sub, ... + 'ping_group_start', ping_group_start, ... + 'ping_group_end', ping_group_end); + + fData = CFF_init_memmapfiles(fData, ... + 'field', 'AP_SBP_SamplePhase', ... + 'wc_dir', wc_dir, ... + 'Class', 'int16', ... + 'Factor', (180/pi)/10430, ... + 'Nanval', 200, ... + 'Offset', 0, ... + 'MaxSamples', maxnSamples_groups, ... + 'MaxBeams', maxnBeams_sub, ... + 'ping_group_start', ping_group_start, ... + 'ping_group_end', ping_group_end); + + % Also the samples data were not recorded, only their location in the + % source file, so we need to fopen the source file to grab the data. + fid = fopen(S7Kfilename,'r','l'); + + % debug graph + debugDisp = 0; + if debugDisp + f = figure(); + ax_mag = axes(f,'outerposition',[0 0.5 1 0.5]); + title('WCD amplitude'); + ax_phase = axes(f,'outerposition',[0 0 1 0.5]); + title('WCD phase'); + end + + % initialize ping group number + iG = 1; + + % now get data for each ping + for iP = 1:nPings + + % ping group number is the index of the memmaped file in which that + % swath's data will be saved. + if iP > ping_group_end(iG) + iG = iG+1; + end + + % data per beam + nBeamsInR7004 = S7Kdata.R7004_BeamGeometry.N(ipR7004(iP)); + fData.AP_BP_BeamPointingAngle(1:nBeamsInR7004,iP) = rad2deg(S7Kdata.R7004_BeamGeometry.BeamHorizontalDirectionAngleRad{ipR7004(iP)}); + fData.AP_BP_StartRangeSampleNumber(1:dtg_nBeams(iP),iP) = zeros(dtg_nBeams(iP),1); + fData.AP_BP_NumberOfSamples(1:dtg_nBeams(iP),iP) = dtg_nSamples(iP).*ones(dtg_nBeams(iP),1); + beamsInR7027 = S7Kdata.R7027_RawDetectionData.BeamDescriptor{ipR7027(iP)}+1; + fData.AP_BP_DetectedRangeInSamples(beamsInR7027,iP) = S7Kdata.R7027_RawDetectionData.DetectionPoint{ipR7027(iP)}; + + % initialize the water column data matrix for that ping. + pingMag = intmin('int16').*ones(maxnSamples_groups(iG),maxnBeams_sub,'int16'); + pingPh = 200.*ones(maxnSamples_groups(iG),maxnBeams_sub,'int16'); + + % read amplitude in original format and decode + fseek(fid,S7Kdata.R7018_BeamformedData.BeamformedDataPos(ipR7018(iP)),'bof'); + Mag_tmp = (fread(fid,[dtg_nBeams(iP) dtg_nSamples(iP)],'uint16',2))'; + Mag_tmp(Mag_tmp==double(intmin('uint16'))) = NaN; + Mag_tmp = 20*log10(Mag_tmp/double(intmax('uint16'))); % now in dB + + % re-encode Magnitude for storage + Mag_tmp2 = int16(Mag_tmp*200); + Mag_tmp2(isnan(Mag_tmp)) = intmin('int16'); + + % read phase in original format + fseek(fid,S7Kdata.R7018_BeamformedData.BeamformedDataPos(ipR7018(iP))+2,'bof'); + Ph_tmp = (fread(fid,[dtg_nBeams(iP) dtg_nSamples(iP)],'int16=>int16',2))'; + + % debug graph + if debugDisp + % display amplitude + imagesc(ax_mag,Mag_tmp); + colorbar(ax_mag) + title(ax_mag, sprintf('Ping %i/%i, WCD amplitude',iP,nPings)); + % display phase + imagesc(ax_phase,double(Ph_tmp).*((180/pi)/10430)); + colorbar(ax_phase) + title(ax_phase, 'WCD phase'); + drawnow; + end + + % store Magnitude and Phase + pingMag(1:size(Mag_tmp2,1),:) = Mag_tmp2; + pingPh(1:size(Ph_tmp,1),:) = Ph_tmp; + + % Store the data in the appropriate binary file, at the appropriate + % ping, through the memory mapping + fData.AP_SBP_SampleAmplitudes{iG}.Data.val(:,:,iP-ping_group_start(iG)+1) = pingMag; + fData.AP_SBP_SamplePhase{iG}.Data.val(:,:,iP-ping_group_start(iG)+1) = pingPh; + + end + + % close the original raw file + fclose(fid); + +end + +comms.progress(5,6); + + +%% water-column data (amplitude, phase) from 7042 records + +comms.step('Converting Water-column data (amplitude, phase)'); + +if all(isfield(S7Kdata,{'R7042_CompressedWaterColumnData','R7000_SonarSettings','R7004_BeamGeometry','R7027_RawDetectionData'})) + + % I came across one file where not all pings are recorded, also with + % different records having different pings. Since we here combine data + % from different records, we first need to limit the recording to pings + % that are present in all records. To make it more complicated, + % R7004_BeamGeometry does not have a pingNumber field. + R7042pings = S7Kdata.R7042_CompressedWaterColumnData.PingNumber; + R7000pings = S7Kdata.R7000_SonarSettings.PingNumber; + R7027pings = S7Kdata.R7027_RawDetectionData.PingNumber; + pingNumber = intersect(R7042pings,intersect(R7000pings,R7027pings)); + [~,ipR7042] = ismember(pingNumber,R7042pings); + [~,ipR7000] = ismember(pingNumber,R7000pings); + [~,ipR7027] = ismember(pingNumber,R7027pings); + + % for R7004_BeamGeometry, we will have to assume its contents are the + % same as one of the other record types + if numel(S7Kdata.R7004_BeamGeometry.SonarID) == numel(R7000pings) + ipR7004 = ipR7000; + elseif numel(S7Kdata.R7004_BeamGeometry.SonarID) == numel(R7027pings) + ipR7004 = ipR7027; + elseif numel(S7Kdata.R7004_BeamGeometry.SonarID) == numel(R7042pings) + ipR7004 = ipR7042; + else + error('cannot proceed...') + end + + % number of pings + nPings = numel(pingNumber); + + % number of Tx sectors + maxNTransmitSectors = 1; + + % number of beams + nBeams = cellfun(@numel,S7Kdata.R7042_CompressedWaterColumnData.BeamNumber(ipR7042)); % number of beams per ping + maxnBeams = nanmax(nBeams); % maximum number of beams in file + + % number of samples + % maxNSamples = nanmax(S7Kdata.R7042_CompressedWaterColumnData.FirstSample(ipR7042)+cellfun(@nanmax,S7Kdata.R7042_CompressedWaterColumnData.NumberOfSamples(ipR7042))); + dtg_nSamples = S7Kdata.R7042_CompressedWaterColumnData.NumberOfSamples(ipR7042); % number of samples per datagram and beam + [maxNSamples_groups, ping_group_start, ping_group_end] = CFF_group_pings(dtg_nSamples,pingNumber,pingNumber); % making groups of pings to limit size of memmaped files + + % add the WCD decimation factors given here in input + fData.dr_sub = dr_sub; + fData.db_sub = db_sub; + + % data per ping + fData.AP_1P_Date = S7Kdata.R7042_CompressedWaterColumnData.Date(ipR7042); + fData.AP_1P_TimeSinceMidnightInMilliseconds = S7Kdata.R7042_CompressedWaterColumnData.TimeSinceMidnightInMilliseconds(ipR7042); + fData.AP_1P_PingCounter = pingNumber; + fData.AP_1P_NumberOfDatagrams = ones(size(pingNumber)); + fData.AP_1P_NumberOfTransmitSectors = ones(size(pingNumber)); + fData.AP_1P_TotalNumberOfReceiveBeams = cellfun(@numel,S7Kdata.R7042_CompressedWaterColumnData.BeamNumber(ipR7042)); + fData.AP_1P_SoundSpeed = S7Kdata.R7000_SonarSettings.SoundVelocity(ipR7000); + fData.AP_1P_SamplingFrequencyHz = S7Kdata.R7042_CompressedWaterColumnData.SampleRate(ipR7042); % in Hz + fData.AP_1P_TXTimeHeave = nan(ones(size(pingNumber))); + fData.AP_1P_TVGFunctionApplied = nan(size(pingNumber)); + fData.AP_1P_TVGOffset = zeros(size(pingNumber)); + fData.AP_1P_ScanningInfo = nan(size(pingNumber)); + + % initialize data per transmit sector and ping + fData.AP_TP_TiltAngle = nan(maxNTransmitSectors,nPings); + fData.AP_TP_CenterFrequency = S7Kdata.R7000_SonarSettings.Frequency(ipR7000); + fData.AP_TP_TransmitSectorNumber = nan(maxNTransmitSectors,nPings); + + % initialize data per (decimated) beam and ping + fData.AP_BP_BeamPointingAngle = nan(maxnBeams,nPings); + fData.AP_BP_StartRangeSampleNumber = nan(maxnBeams,nPings); + fData.AP_BP_NumberOfSamples = nan(maxnBeams,nPings); + fData.AP_BP_DetectedRangeInSamples = zeros(maxnBeams,nPings); + fData.AP_BP_TransmitSectorNumber = nan(maxnBeams,nPings); + fData.AP_BP_BeamNumber = nan(maxnBeams,nPings); + + % flags indicating what data are available + [flags,sample_size,mag_fmt,phase_fmt] = CFF_get_R7042_flags(S7Kdata.R7042_CompressedWaterColumnData.Flags(1)); + + % The actual water-column data will not be saved in fData but in binary + % files. Get the output directory to store those files + wc_dir = CFF_converted_data_folder(S7Kfilename); + + % Clean up that folder first before adding anything to it + CFF_clean_delete_fdata(wc_dir); + + % amplitude format + switch mag_fmt + case 'int8' + mag_file_fmt = 'int8'; + mag_fact = 1; + case {'uint16', 'float32'} + mag_file_fmt = 'int16'; + mag_fact = 1/200; + end + + % initialize data-holding binary files for Amplitude + fData = CFF_init_memmapfiles(fData,... + 'field', 'AP_SBP_SampleAmplitudes', ... + 'wc_dir', wc_dir, ... + 'Class', mag_file_fmt, ... + 'Factor', mag_fact, ... + 'Nanval', intmin(mag_file_fmt), ... + 'Offset', 0, ... + 'MaxSamples', maxNSamples_groups, ... + 'MaxBeams', maxnBeams, ... + 'ping_group_start', ping_group_start, ... + 'ping_group_end', ping_group_end); + + % do the same for phase if it's available + if ~flags.magnitudeOnly + + % phase format + switch phase_fmt + case 'int8' + phase_fact = 360/256; + case 'int16' + phase_fact = 180/pi/10430; + end + + % initialize data-holding binary files for Phase + fData = CFF_init_memmapfiles(fData, ... + 'field', 'AP_SBP_SamplePhase', ... + 'wc_dir', wc_dir, ... + 'Class', phase_fmt, ... + 'Factor', phase_fact, ... + 'Nanval', 200, ... + 'Offset', 0, ... + 'MaxSamples', maxNSamples_groups, ... + 'MaxBeams', maxnBeams, ... + 'ping_group_start', ping_group_start, ... + 'ping_group_end', ping_group_end); + + end + + % Also the samples data were not recorded, only their location in the + % source file, so we need to fopen the source file to grab the data. + fid = fopen(S7Kfilename,'r','l'); + + % correct sampling frequency record + if flags.downsamplingType > 0 + fData.AP_1P_SamplingFrequencyHz = fData.AP_1P_SamplingFrequencyHz./flags.downsamplingDivisor; + end + + % initialize ping group counter, to use to specify which memmapfile + % to fill. We start in the first. + iG = 1; + + % debug graph + disp_wc = 0; + if disp_wc + f = figure(); + if flags.magnitudeOnly + ax_mag = axes(f,'outerposition',[0 0 1 1]); + else + ax_mag = axes(f,'outerposition',[0 0.5 1 0.5]); + ax_phase = axes(f,'outerposition',[0 0 1 0.5]); + end + end + + % now get data for each ping + for iP = 1:nPings + + % ping group number is the index of the memmaped file in which that + % swath's data will be saved. + if iP > ping_group_end(iG) + iG = iG+1; + end + + % data per Tx sector + nTransmitSectors = fData.AP_1P_NumberOfTransmitSectors(1,iP); % number of transmit sectors in this ping + fData.AP_TP_TiltAngle(1:nTransmitSectors,iP) = zeros(nTransmitSectors,1); + fData.AP_TP_CenterFrequency(1:nTransmitSectors,iP) = S7Kdata.R7000_SonarSettings.Frequency(ipR7000(iP))*ones(nTransmitSectors,1); + fData.AP_TP_TransmitSectorNumber(1:nTransmitSectors,iP) = 1:nTransmitSectors; + + % data per beam + iBeam = S7Kdata.R7042_CompressedWaterColumnData.BeamNumber{ipR7042(iP)}+1; % beam numbers in this ping + fData.AP_BP_BeamPointingAngle(iBeam,iP) = S7Kdata.R7004_BeamGeometry.BeamHorizontalDirectionAngleRad{ipR7004(iP)}/pi*180; + fData.AP_BP_StartRangeSampleNumber(iBeam,iP) = round(S7Kdata.R7042_CompressedWaterColumnData.FirstSample(ipR7042(iP))); + fData.AP_BP_NumberOfSamples(iBeam,iP) = round(S7Kdata.R7042_CompressedWaterColumnData.NumberOfSamples{ipR7042(iP)}); + fData.AP_BP_DetectedRangeInSamples(S7Kdata.R7027_RawDetectionData.BeamDescriptor{ipR7027(iP)}+1,iP) = round(S7Kdata.R7027_RawDetectionData.DetectionPoint{ipR7027(iP)}/flags.downsamplingDivisor); + fData.AP_BP_TransmitSectorNumber(iBeam,iP) = 1; + fData.AP_BP_BeamNumber(iBeam,iP) = S7Kdata.R7004_BeamGeometry.N(ipR7004(iP)); + + % initialize amplitude and phase matrices + Mag_tmp = ones(maxNSamples_groups(iG),maxnBeams,mag_fmt)*eval([mag_fmt '(-inf)']); + if ~flags.magnitudeOnly + Ph_tmp = zeros(maxNSamples_groups(iG),maxnBeams,phase_fmt); + end + + % number of samples for each beam in this ping + nSamples = S7Kdata.R7042_CompressedWaterColumnData.NumberOfSamples{ipR7042(iP)}; + + % got to start of data in raw file, from here + pos = ftell(fid); + fseek(fid,S7Kdata.R7042_CompressedWaterColumnData.SampleStartPositionInFile{ipR7042(iP)}(1)-pos,'cof'); + + % read the ping's data as int8, between the start position for the + % first beam's data and the end position of the last beam's data + pos_start_ping = S7Kdata.R7042_CompressedWaterColumnData.SampleStartPositionInFile{ipR7042(iP)}(1); + pos_end_ping = S7Kdata.R7042_CompressedWaterColumnData.SampleStartPositionInFile{ipR7042(iP)}(end)+nSamples(end)*sample_size; + DataSamples_tot = fread(fid,pos_end_ping-pos_start_ping+1,'int8=>int8'); + + % index of first sample + start_sample = S7Kdata.R7042_CompressedWaterColumnData.FirstSample(ipR7042(iP))+1; + + % read beam by beam + for jj = 1:S7Kdata.R7004_BeamGeometry.N(ipR7004(iP)) % from R7004??? XXX1 + + % get data for that beam + idx_pp = S7Kdata.R7042_CompressedWaterColumnData.SampleStartPositionInFile{ipR7042(iP)}(jj):(S7Kdata.R7042_CompressedWaterColumnData.SampleStartPositionInFile{ipR7042(iP)}(jj)+nSamples(jj)*sample_size-1); + idx_pp = idx_pp-pos_start_ping+1; + DataSamples_tmp = DataSamples_tot(idx_pp); + + if flags.magnitudeOnly + % data are amplitude only + switch mag_fmt + % read amplitude data + case 'int8' + Mag_tmp((start_sample:start_sample+nSamples(jj)-1),jj) = DataSamples_tmp; + case 'uint16' + Mag_tmp((start_sample:start_sample+nSamples(jj)-1),jj) = typecast(DataSamples_tmp,mag_fmt); + case 'float32' + Mag_tmp((start_sample:start_sample+nSamples(jj)-1),jj) = 10*log10(typecast(DataSamples_tmp,mag_fmt)); + otherwise + warning('WC compression flag issue'); + end + else + % data are amplitude AND phase + switch mag_fmt + % read amplitude data + case 'int8' + Mag_tmp((start_sample:start_sample+nSamples(jj)-1),jj) = DataSamples_tmp(1:2:end,:); + case 'uint16' + idx_tot = rem(1:numel(DataSamples_tmp),4); + idx_mag = idx_tot==1 | idx_tot==2; + Mag_tmp((start_sample:start_sample+nSamples(jj)-1),jj) = typecast(DataSamples_tmp(idx_mag,:),mag_fmt); + otherwise + warning('WC compression flag issue'); + end + switch phase_fmt + % read phase data + case'int8' + Ph_tmp((start_sample:start_sample+nSamples(jj)-1),jj) = DataSamples_tmp(2:2:end,:); + case 'int16' + idx_tot = rem(1:numel(DataSamples_tmp),4); + idx_phase = idx_tot==3 | idx_tot==0; + Ph_tmp((start_sample:start_sample+nSamples(jj)-1),jj) = typecast(DataSamples_tmp(idx_phase,:),phase_fmt); + otherwise + warning('WC compression flag issue'); + end + end + + end + + % debug graph + if disp_wc + % display amplitude + switch mag_fmt + case 'int8' + imagesc(ax_mag,double(Mag_tmp)-128); + case 'uint16' + imagesc(ax_mag,10*log10(double(Mag_tmp)/double(intmax('int16')))); + end + caxis(ax_mag,[-100 -20]); + colorbar + title(sprintf('ping %i/%i',iP,nPings)); + % display phase + if ~flags.magnitudeOnly + imagesc(ax_phase,Ph_tmp*phase_fact); + colorbar + end + drawnow; + end + + % reformat amplitude data for storing + switch mag_fmt + case 'int8' + Mag_tmp = Mag_tmp - int8(128); + case 'uint16' + idx0 = Mag_tmp==0; + Mag_tmp = (10*log10(double(Mag_tmp)/double(intmax('uint16')))/mag_fact); + Mag_tmp(idx0) = -inf; + Mag_tmp = int16(Mag_tmp); + case 'float32' + Mag_tmp = int16(Mag_tmp/mag_fact); + end + + % finished reading this ping's WC data. Store the data in the + % appropriate binary file, at the appropriate ping, through the + % memory mapping + fData.AP_SBP_SampleAmplitudes{iG}.Data.val(:,:,iP-ping_group_start(iG)+1) = Mag_tmp; + if ~flags.magnitudeOnly + fData.AP_SBP_SamplePhase{iG}.Data.val(:,:,iP-ping_group_start(iG)+1) = Ph_tmp; + end + + end + + % close the original raw file + fclose(fid); + +end + +comms.progress(6,6); + + +%% finalise + +% end message +comms.finish('Done'); \ No newline at end of file diff --git a/read_data_files/Reson/CFF_get_R7012_flags.m b/read_data_files/Reson/CFF_get_R7012_flags.m new file mode 100644 index 0000000..ea51979 --- /dev/null +++ b/read_data_files/Reson/CFF_get_R7012_flags.m @@ -0,0 +1,29 @@ +function flags = CFF_get_R7012_flags(flag_dec) +%CFF_GET_R7012_FLAGS One-line description +% +% See also CFF_READ_S7K_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-08-2021 + +if isnumeric(flag_dec) + flag_bin = dec2bin(flag_dec, 16); +else + flag_bin = flag_dec; +end + +flags.pitchStab = 0; +flags.rollStab = 0; +flags.yawStab = 0; +flags.heaveStab = 0; + +flags.pitchStab = bin2dec(flag_bin(16-0)); +flags.rollStab = bin2dec(flag_bin(16-1)); +flags.yawStab = bin2dec(flag_bin(16-2)); +flags.heaveStab = bin2dec(flag_bin(16-3)); + + + + + diff --git a/read_data_files/Reson/CFF_get_R7042_flags.m b/read_data_files/Reson/CFF_get_R7042_flags.m new file mode 100644 index 0000000..5cd4abd --- /dev/null +++ b/read_data_files/Reson/CFF_get_R7042_flags.m @@ -0,0 +1,137 @@ +function [flags,sample_size,mag_fmt,phase_fmt] = CFF_get_R7042_flags(flag_dec) +%CFF_GET_R7042_FLAGS One-line description +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + + +%% init +flags.dataTruncatedBeyondBottom = 0; +flags.magnitudeOnly = 0; +flags.int8BitCompression = 0; +flags.downsamplingDivisor = 0; +flags.downsamplingType = 0; +flags.int32BitsData = 0; +flags.compressionFactorAvailable = 0; +flags.segmentNumbersAvailable = 0; +sample_size = 0; +mag_fmt = ''; +phase_fmt = ''; + + +%% read + +if isnumeric(flag_dec) + flag_bin = dec2bin(flag_dec, 32); +else + flag_bin = flag_dec; +end + +% Bit 0 : Use maximum bottom detection point in each beam to limit data. +% Data is included up to the bottom detection point + 10%. This flag has no +% effect on systems which do not perform bottom detection. +flags.dataTruncatedBeyondBottom = bin2dec(flag_bin(32-0)); + +% Bit 1 : Include intensity data only (strip phase) +flags.magnitudeOnly = bin2dec(flag_bin(32-1)); + +% Bit 2 : Convert mag to dB, then compress from 16 bit to 8 bit +% by truncation of 8 lower bits. Phase compression simply +% truncates lower (least significant) byte of phase data. +flags.int8BitCompression = bin2dec(flag_bin(32-2)); + +% Bit 3 : Reserved. +flags.Reserved = bin2dec(flag_bin(32-3)); + +% Bit 4-7 : Downsampling divisor. Value = (BITS >> 4). Only +% values 2-16 are valid. This field is ignored if downsampling +% is not enabled (type = “none”). +flags.downsamplingDivisor = bin2dec(flag_bin(32-7:32-4)); + +% Bit 8-11 : Downsampling type: +% 0x000 = None +% 0x100 = Middle value +% 0x200 = Peak value +% 0x300 = Average value +flags.downsamplingType = bin2dec(flag_bin(32-11:32-8)); + +% Bit 12: 32 Bits data +flags.int32BitsData = bin2dec(flag_bin(32-12)); + +% Bit 13: Compression factor available +flags.compressionFactorAvailable = bin2dec(flag_bin(32-13)); + +% Bit 14: Segment numbers available +flags.segmentNumbersAvailable = bin2dec(flag_bin(32-14)); + +% Bit 15: First sample contains RxDelay value. +flags.firstSampleContainsRxDelay = bin2dec(flag_bin(32-15)); + +% NOTE +% If downsampling is used (Flags bit 8-11), then the effective Sample Rate +% of the data is changed and is given by the sample rate field. To +% calculate the effective sample rate, the system sample rate (provided in +% the 7000 record) must be divided by the downsampling divisor factor +% specified in bits 4-7. + +% NOTE +% When ‘Bit 2’ is set in the flags of the 7042 record, the record contains +% 8 bit dB values. This should never combined with ‘Bit 12’ indicating that +% intensities are stored as 32 bit values. + + +%% interpret + +% figure the size of a "sample" in bytes based on those flags +if ~flags.int32BitsData + if ~flags.int8BitCompression + if ~flags.magnitudeOnly + % A) 16 bit Mag & 16bit Phase (32 bits total) + sample_size = 4; + mag_fmt = 'uint16'; + phase_fmt = 'int16'; + else + % B) 16 bit Mag (16 bits total, no phase) + sample_size = 2; + mag_fmt = 'uint16'; + phase_fmt = ''; + end + else + if ~flags.magnitudeOnly + % C) 8 bit Mag & 8 bit Phase (16 bits total) + sample_size = 2; + mag_fmt = 'int8'; + phase_fmt = 'int8'; + else + % D) 8 bit Mag (8 bits total, no phase) + sample_size = 1; + mag_fmt = 'int8'; + phase_fmt = ''; + end + end +else + if ~flags.magnitudeOnly + % This case is strange. We have both mag and phase, and bit 12 "32 + % Bits data" is on. One would assume it means magnitude and phase + % are both 32 bits, for a total of 64. OR that magnitude is 32 bits + % and phase is nominal, aka 16 bits, for a total of 48 bits. But + % none of those cases are in the documentation. + % + % What we have in the documentation is case "E) 32 bit Mag & 8 bit + % Phase (40 bits total)", which is strange. Is the phase downgraded + % from 16 to 8 bits to save space? Anyway, we assume that the doc + % is correct and that this case applies here. + + % E) 32 bit Mag & 8 bit Phase (40 bits total) + sample_size = 5; + mag_fmt = 'float32'; + phase_fmt = 'int8'; + else + % F) 32 bit Mag (32 bits total, no phase) + sample_size = 4; + mag_fmt = 'float32'; + phase_fmt = ''; + end +end diff --git a/read_data_files/Reson/CFF_get_Reson_files.m b/read_data_files/Reson/CFF_get_Reson_files.m new file mode 100644 index 0000000..0c0d4fc --- /dev/null +++ b/read_data_files/Reson/CFF_get_Reson_files.m @@ -0,0 +1,21 @@ +function [Reson_files,fileroot_reshaped] = CFF_get_Reson_files(fileroot) +%CFF_GET_RESON_FILES One-line description +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +if ischar(fileroot) + fileroot = {fileroot}; +end + +fileroot_reshaped = reshape(fileroot,[numel(fileroot),1]); + +Reson_files = cell(length(fileroot_reshaped),1); + +for ii = 1:length(fileroot_reshaped) + + Reson_files{ii,1} = strcat(fileroot_reshaped{ii}, '.s7k'); + +end diff --git a/read_data_files/Reson/CFF_is_Reson_file.m b/read_data_files/Reson/CFF_is_Reson_file.m new file mode 100644 index 0000000..3e7c3da --- /dev/null +++ b/read_data_files/Reson/CFF_is_Reson_file.m @@ -0,0 +1,17 @@ +function bool = CFF_is_Reson_file(file) +%CFF_IS_RESON_FILE One-line description +% + + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +if ischar(file) + file = {file}; +end + +% function checking if extension is Reson's +isK = @(x) any(strcmp(CFF_file_extension(x),{'.s7k','.S7K'})); + +bool = cellfun(isK,file); \ No newline at end of file diff --git a/read_data_files/Reson/CFF_read_s7k.m b/read_data_files/Reson/CFF_read_s7k.m new file mode 100644 index 0000000..5d3a830 --- /dev/null +++ b/read_data_files/Reson/CFF_read_s7k.m @@ -0,0 +1,115 @@ +function [S7Kdata,datagrams_parsed_idx] = CFF_read_s7k(S7Kfilename,varargin) +%CFF_READ_S7K Read s7k file +% +% Reads contents of one Teledyne-Reson binary data file in .s7k format, +% allowing choice on which type of datagrams to parse. +% +% S7Kdata = CFF_READ_S7K(S7Kfilename) reads all datagrams in a +% Teledyne-Reson file (extension .s7k) S7Kfilenamem, and store them in +% S7Kdata. +% +% S7Kdata = CFF_READ_S7K(S7Kfilename,datagrams) reads only those +% datagrams in S7Kfilename that are specified by datagrams, and store +% them in S7Kdata. +% +% S7Kdata = CFF_READ_S7K(S7Kfilename,'datagrams',datagrams) does +% the same. +% +% Note this function will extract all datagram types of interest. For +% more control (say you only want the first ten depth datagrams and the +% last position datagram), use CFF_READ_S7K_FROM_FILEINFO. +% +% See also CFF_S7K_FILE_INFO, CFF_READ_S7K_FROM_FILEINFO. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + + +%% Input arguments management +p = inputParser; + +% name of the .s7k file +argName = 'S7Kfilename'; +argCheck = @(x) CFF_check_S7Kfilename(x); +addRequired(p,argName,argCheck); + +% types of datagram to read +argName = 'datagrams'; +argDefault = {}; +argCheck = @(x) isnumeric(x)||isempty(x); +addOptional(p,argName,argDefault,argCheck); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,S7Kfilename,varargin{:}); + +% and get results +S7Kfilename = p.Results.S7Kfilename; +datagrams_to_parse = p.Results.datagrams; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end +clear p; + + +%% Prep + +% start message +filename = CFF_file_name(S7Kfilename,1); +comms.start(sprintf('Reading data in file %s',filename)); + +% start progress +comms.progress(0,1); + + +%% Processing + +% get info from file +comms.step('Listing datagrams'); +info = CFF_s7k_file_info(S7Kfilename); + +% communicate progress +comms.progress(0.5,1); + +if isempty(datagrams_to_parse) + % parse all datagrams in file + idx_to_parse = true(size(info.recordTypeIdentifier)); + datagrams_parsed_idx = []; + +else + % datagrams to parse are listed in input + + % datagrams available + datagrams_available = unique(info.recordTypeIdentifier); + + % find which datagrams can be read here + datagrams_parsable_idx = ismember(datagrams_to_parse,datagrams_available); + + % list datagrams to be parsed + idx_to_parse = ismember(info.recordTypeIdentifier,datagrams_to_parse(datagrams_parsable_idx)); + datagrams_parsed_idx = datagrams_parsable_idx; + +end + +% find and remove possibly corrupted datagrams +idx_corrupted = info.syncCounter~=0; +idx_corrupted = [idx_corrupted(2:end);false]; % the possibly corrupted datagram is the one before the one with syncCounter~=0; +if any(idx_corrupted & idx_to_parse) + comms.info('%i of the %i datagrams to be parsed in this file may be corrupted and will not be parsed.',sum(idx_corrupted & idx_to_parse), sum(idx_to_parse) ); +end + +% parsable datagrams to be parsed +info.parsed(idx_to_parse & ~idx_corrupted) = 1; + +% read data +comms.step('Reading datagrams'); +S7Kdata = CFF_read_s7k_from_fileinfo(S7Kfilename, info); + + +%% end message +comms.finish('Done'); diff --git a/read_data_files/Reson/CFF_read_s7k_from_fileinfo.m b/read_data_files/Reson/CFF_read_s7k_from_fileinfo.m new file mode 100644 index 0000000..e182f23 --- /dev/null +++ b/read_data_files/Reson/CFF_read_s7k_from_fileinfo.m @@ -0,0 +1,1048 @@ +function S7Kdata = CFF_read_s7k_from_fileinfo(S7Kfilename,S7Kfileinfo,varargin) +%CFF_READ_S7K_FROM_FILEINFO Read contents of s7k file +% +% Reads contents of one Kongsberg EM series binary .s7k or .wcd data +% file, using S7Kfileinfo to indicate which datagrams to be parsed. +% +% S7Kdata = CFF_READ_S7K_FROM_FILEINFO(S7Kfilename, S7Kfileinfo) reads +% s7k datagrams in S7Kfilename for which S7Kfileinfo.parsed equals 1, and +% store them in S7Kdata. +% +% *INPUT VARIABLES* +% * |S7Kfilename|: Required. String filename to parse (extension in +% .s7k). +% * |S7Kfileinfo|: structure containing information about datagrams in +% S7Kfilename, with fields: +% * |S7Kfilename|: input file name +% * |filesize|: file size in bytes +% * |datagsizeformat|: endianness of the datagram size field 'b' or 'l' +% * |datagramsformat|: endianness of the datagrams 'b' or 'l' +% * |datagNumberInFile|: number of datagram in file +% * |datagPositionInFile|: position of beginning of datagram in file +% * |datagTypeNumber|: for each datagram, SIMRAD datagram type in +% decimal +% * |datagTypeText|: for each datagram, SIMRAD datagram type +% description +% * |parsed|: 0 for each datagram at this stage. To be later turned to +% 1 for parsing +% * |counter|: the counter of this type of datagram in the file (ie +% first datagram of that type is 1 and last datagram is the total +% number of datagrams of that type) +% * |number|: the number/counter found in the datagram (usually +% different to counter) +% * |size|: for each datagram, datagram size in bytes +% * |syncCounter|: for each datagram, the number of bytes founds +% between this datagram and the previous one (any number different than +% zero indicates a sync error) +% * |emNumber|: EM Model number (eg 2045 for EM2040c) +% * |date|: datagram date in YYYMMDD +% * |timeSinceMidnightInMilliseconds|: time since midnight in msecs +% +% *OUTPUT VARIABLES* +% * |S7Kdata|: structure containing the data. Each field corresponds a +% different type of datagram. The field |S7Kdata.info| contains a copy of +% S7Kfileinfo described above. +% +% *DEVELOPMENT NOTES* +% * PU Status output datagram structure seems different to the datagram +% manual description. Find the good description.#edit 21aug2013: updated +% to Rev Q. Need to be checked though. +% * The parsing code for some datagrams still need to be coded. To +% update. +% +% See also CFF_READ_S7K. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 21-07-2022 + +%% Input arguments management +p = inputParser; + +% name of the .s7k file +argName = 'S7Kfilename'; +argCheck = @(x) CFF_check_S7Kfilename(x); +addRequired(p,argName,argCheck); + +% fileinfo from CFF_S7K_FILE_INFO containing indexes of datagrams to read +argName = 'S7Kfileinfo'; +argCheck = @isstruct; +addRequired(p,argName,argCheck); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,S7Kfilename,S7Kfileinfo,varargin{:}); + +% and get results +S7Kfilename = p.Results.S7Kfilename; +S7Kfileinfo = p.Results.S7Kfileinfo; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end + + +%% Prep + +% start message +filename = CFF_file_name(S7Kfilename,1); +comms.start(sprintf('Reading records in file %s',filename)); + +% store filename +S7Kdata.S7Kfilename = S7Kfilename; + +% open file +[fid,~] = fopen(S7Kfilename, 'r'); + +% parse only datagrams indicated in S7Kfileinfo +datagToParse = find(S7Kfileinfo.parsed==1); +nDatagsToParse = numel(datagToParse); + +% start progress +comms.progress(0,nDatagsToParse); + + +%% Reading datagrams +for iDatag = datagToParse' + + % A full s7k record is organized as a sequence of: + % * DRF - Data Record Frame (64 bytes, at least for protocol version 5) + % * RTH - Record Type Header (variable size) + % * RD - Record Data (optional, variable size) + % * OD - Optional Data (optional, variable size) + % * CS - Checksum (optional, 4 bytes) + + % DRF info was already read so get relevant parameters in fileinfo + pif_recordstart = S7Kfileinfo.recordStartPositionInFile(iDatag); + recordTypeIdentifier = S7Kfileinfo.recordTypeIdentifier(iDatag); + DRF_size = S7Kfileinfo.DRF_size(iDatag); + RTHandRD_size = S7Kfileinfo.RTHandRD_size(iDatag); + OD_size = S7Kfileinfo.OD_size(iDatag); + CS_size = S7Kfileinfo.CS_size(iDatag); + OD_offset = S7Kfileinfo.OD_offset(iDatag); + + % Go to start of dgm + fseek(fid, pif_recordstart+DRF_size, -1); + + % get recordName + recordTypeText = S7Kfileinfo.recordTypeText{iDatag}; + recordName = ['R' regexprep(replace(recordTypeText,' ',''),'\W','_')]; % remove problematic characters + + % get record counter + if isfield(S7Kdata, recordName) + iRec = numel(S7Kdata.(recordName).Date) +1 ; + else + iRec = 1; + end + + % reset the parsed switch + parsed = 0; + + switch recordTypeIdentifier + + case 1003 % Position + % Description: Position Record used in conjunction with Record + % Type 1011. + + % NOTE + % * 1003 record created by the 7k sonar source is using the + % sensor (GPS) data. 1003 created by Teledyne PDS is using + % reference point data. + % * Latency of 1003 created by Teledyne PDS and 7k sonar is + % always 0. + + % start parsing RTH + S7Kdata.(recordName).Datum_id(iRec) = fread(fid,1,'uint32'); % 0 – WGS84; >0 – Reserved + S7Kdata.(recordName).Latency(iRec) = fread(fid,1,'float32'); % In seconds + + % Latitude in radians or northing in meters + S7Kdata.(recordName).LatitudeOrNorthing(iRec) = fread(fid,1,'float64'); + + % Longitude in radians or easting in meters + S7Kdata.(recordName).LongitudeOrEasting(iRec) = fread(fid,1,'float64'); + + S7Kdata.(recordName).Height(iRec) = fread(fid,1,'float64'); % In meters + + % 0 – Geographical coordinates; 1 – Grid coordinates + S7Kdata.(recordName).PositionTypeFlag(iRec) = fread(fid,1,'uint8'); + + S7Kdata.(recordName).UTMZone(iRec) = fread(fid,1,'uint8'); % UTM Zone + + % 0 – Navigation Data; 1 – Dead-Reckoning + S7Kdata.(recordName).QualityFlag(iRec) = fread(fid,1,'uint8'); + + S7Kdata.(recordName).PositioningMethod(iRec) = fread(fid,1,'uint8'); % see doc + S7Kdata.(recordName).NumberOfSatellites(iRec) = fread(fid,1,'uint8'); % Optional + + parsed = 1; + + case 1012 % Roll Pitch Heave + % Description: Motion Data Record. + + % start parsing RTH + S7Kdata.(recordName).Roll(iRec) = fread(fid,1,'float32'); % Vessel roll in radians + S7Kdata.(recordName).Pitch(iRec) = fread(fid,1,'float32'); % Vessel pitch in radians + S7Kdata.(recordName).Heave(iRec) = fread(fid,1,'float32'); % Vessel heave in meters + + parsed = 1; + + case 1013 % Heading + % Description: Vessel Heading Record. + + % start parsing RTH + S7Kdata.(recordName).Heading(iRec) = fread(fid,1,'float32'); % Vessel heading in radians + + parsed = 1; + + case 1015 % Navigation + % Description: This record will be output at the input + % navigation rate. + + % DEV NOTE -- Have not tested this code against actual data... + + % start parsing RTH + + % 1– Ellipsoid; 2 – Geoid; 3 – Chart datum + S7Kdata.(recordName).VerticalReference(iRec) = fread(fid,1,'uint8'); + + % Latitude of vessel reference point in radians -pi/2 to pi/2, -south + S7Kdata.(recordName).Latitude(iRec) = fread(fid,1,'float64'); + + % Longitude of vessel reference point in radians -pi to pi, -west + S7Kdata.(recordName).Longitude(iRec) = fread(fid,1,'float64'); + + % Position accuracy in meters + S7Kdata.(recordName).HorizontalPositionAccuracy(iRec) = fread(fid,1,'float32'); + + % Height of vessel reference point above vertical reference in meters + S7Kdata.(recordName).VesselHeight(iRec) = fread(fid,1,'float32'); + + % In meters + S7Kdata.(recordName).HeightAccuracy(iRec) = fread(fid,1,'float32'); + + % Speed over ground at position time in m/s + S7Kdata.(recordName).SpeedOverGround(iRec) = fread(fid,1,'float32'); + + % Course over ground at position time in radians + S7Kdata.(recordName).CourseOverGround(iRec) = fread(fid,1,'float32'); + + % Heading of vessel at position time in radians + S7Kdata.(recordName).Heading(iRec) = fread(fid,1,'float32'); + + parsed = 1; + + case 7000 % Sonar Settings + % Description: This record is produced by the SeaBat™ 7k sonar + % 7-P processor series. It contains the current sonar settings. + % The 7k sonar source updates this data for each ping. The + % record can be subscribed to from the 7k sonar source. For + % details about requesting and subscribing to records, see + % section 10.62 7500 – Remote Control together with section 11 + % 7k Remote Control Definitions. + + % NOTE + % * Pitch and yaw stabilization are not implemented. + % * When the roll stabilization flag is not zero the beam + % pattern is roll stabilized; beam pattern is relative the + % vertical. + % * Projector beam steering is pitch stabilization. + % * Projector beam steering is not redundant when messages 7004 + % and 7006 are received; this value needs to take into account. + % (Unless the sonar does not have pitch steer capacity.) + + % start parsing RTH + S7Kdata.(recordName).SonarID(iRec) = fread(fid,1,'uint64'); % Sonar serial number + S7Kdata.(recordName).PingNumber(iRec) = fread(fid,1,'uint32'); % Sequential number + + % Flag to indicate multi-ping sequence. Always 0 (zero) if not + % in multi-ping mode; otherwise this is the sequence number of + % the ping in the multi-ping sequence. + S7Kdata.(recordName).MultiPingSequence(iRec) = fread(fid,1,'uint16'); + + S7Kdata.(recordName).Frequency(iRec) = fread(fid,1,'float32'); % Transmit frequency in Hertz + S7Kdata.(recordName).SampleRate(iRec) = fread(fid,1,'float32'); % Sample rate in Hertz + S7Kdata.(recordName).ReceiverBandwidth(iRec) = fread(fid,1,'float32'); % In Hertz + S7Kdata.(recordName).TxPulseWidth(iRec) = fread(fid,1,'float32'); % In seconds + S7Kdata.(recordName).TXPulseIdentifier(iRec) = fread(fid,1,'uint32'); % 0 – CW; 1 – Linear chirp (FM) + S7Kdata.(recordName).TXPulseEnvelopeIdentifier(iRec) = fread(fid,1,'uint32'); % 0 – Tapered rectangular; 1 – Tukey; 2 – Hamming; 3 – Han; 4 – Rectangular + S7Kdata.(recordName).TXPulseEnvelopeParameter(iRec) = fread(fid,1,'float32'); % Some envelopes don’t use this parameter + S7Kdata.(recordName).TXPulseMode(iRec) = fread(fid,1,'uint16'); % 1 – Single ping; 2 – Multi-ping 2; 3 – Multi-ping 3; 4 – Multi-ping 4 + S7Kdata.(recordName).TXPulseReserved(iRec) = fread(fid,1,'uint16'); % Reserved + S7Kdata.(recordName).MaxPingRate(iRec) = fread(fid,1,'float32'); % Maximum ping rate in pings per second + S7Kdata.(recordName).PingPeriod(iRec) = fread(fid,1,'float32'); % Seconds since last ping + S7Kdata.(recordName).RangeSelection(iRec) = fread(fid,1,'float32'); % Range selection in meters + S7Kdata.(recordName).PowerSelection(iRec) = fread(fid,1,'float32'); % Power selection in dB re 1 microPa + S7Kdata.(recordName).GainSelection(iRec) = fread(fid,1,'float32'); % Gain selection in dB + S7Kdata.(recordName).ControlFlags(iRec) = fread(fid,1,'uint32'); % see doc + S7Kdata.(recordName).ProjectIdentifier(iRec) = fread(fid,1,'uint32'); % Projector selection + S7Kdata.(recordName).ProjectorBeamSteeringAngleVerticalRad(iRec) = fread(fid,1,'float32'); % In radians + S7Kdata.(recordName).ProjectorBeamSteeringAngleHorizontalRad(iRec) = fread(fid,1,'float32'); % In radians. Along track beam width + S7Kdata.(recordName).ProjectorBeam3dBWidthVerticalRad(iRec) = fread(fid,1,'float32'); % In radians. Across track beam width + S7Kdata.(recordName).ProjectorBeam3dBWidthHorizontalRad(iRec) = fread(fid,1,'float32'); % In meters + S7Kdata.(recordName).ProjectorBeamFocalPoint(iRec) = fread(fid,1,'float32'); + S7Kdata.(recordName).ProjectorBeamWeightingWindowType(iRec) = fread(fid,1,'uint32'); % 0 – Rectangular; 1 – Chebychev; 2 – Gauss + S7Kdata.(recordName).ProjectorBeamWeightingWindowParameter(iRec) = fread(fid,1,'float32'); % N/A + S7Kdata.(recordName).TransmitFlags(iRec) = fread(fid,1,'uint32'); % see doc + S7Kdata.(recordName).HydrophoneIdentifier(iRec) = fread(fid,1,'uint32'); % Hydrophone selection + S7Kdata.(recordName).ReceiveBeamWeightingWindowType(iRec) = fread(fid,1,'uint32'); % 0 – Chebychev; 1 – Kaiser + S7Kdata.(recordName).ReceiveBeamWeightingWindowParameter(iRec) = fread(fid,1,'float32'); % N/A + S7Kdata.(recordName).ReceiveFlags(iRec) = fread(fid,1,'uint32'); % see doc + S7Kdata.(recordName).ReceiveBeamWidthRad(iRec) = fread(fid,1,'float32'); % Angle in radians + S7Kdata.(recordName).BottomDetectFilter{iRec} = fread(fid,4,'float32'); % [min_range max_range min_depth max_depth] + S7Kdata.(recordName).Absorption(iRec) = fread(fid,1,'float32'); % Absorption in dB/km + S7Kdata.(recordName).SoundVelocity(iRec) = fread(fid,1,'float32'); % Sound velocity in m/s + S7Kdata.(recordName).Spreading(iRec) = fread(fid,1,'float32'); % Spreading loss in dB + S7Kdata.(recordName).Reserved(iRec) = fread(fid,1,'uint16'); % Reserved + + parsed = 1; + + case 7001 % Configuration + % Description: This record is produced by the SeaBat™ 7k sonar + % 7-P processor series. It contains the configuration + % information about the sonar capabilities. Each sonar + % configuration can be found in the record’s module info + % section (see Table 42). The record is created on system + % startup and does not change during operation. The record can + % be manually requested from the 7-P processor. This record is + % not available for subscription. For details about requesting + % and subscribing to records, see section 10.62 7500 – Remote + % Control together with section 11 7k Remote Control + % Definitions. + + % start parsing RTH + S7Kdata.(recordName).SonarId(iRec) = fread(fid,1,'uint64'); % Sonar serial number + + N_info = fread(fid,1,'uint32'); % Number of devices/sonar’s + S7Kdata.(recordName).N(iRec) = N_info; + + % start parsing RD + S7Kdata.(recordName).DeviceID{iRec} = NaN(1,N_info); + S7Kdata.(recordName).DeviceDescription{iRec} = cell(1,N_info); + S7Kdata.(recordName).DeviceAlphaDataCard{iRec} = NaN(1,N_info); + S7Kdata.(recordName).DeviceSerialNumber{iRec} = NaN(1,N_info); + S7Kdata.(recordName).DeviceInfo{iRec} = cell(1,N_info); + + for i_inf = 1:N_info + + S7Kdata.(recordName).DeviceID{iRec}(i_inf) = fread(fid,1,'uint32'); % Unique identifier number + S7Kdata.(recordName).DeviceDescription{iRec}{i_inf} = fread(fid,60,'*char')'; % UTF-8 string + S7Kdata.(recordName).DeviceAlphaDataCard{iRec}(i_inf) = fread(fid,1,'uint32'); % see doc + S7Kdata.(recordName).DeviceSerialNumber{iRec}(i_inf) = fread(fid,1,'uint64'); + + l_tmp = fread(fid,1,'uint32'); % In bytes + S7Kdata.(recordName).DeviceInfo{iRec}{i_inf} = fread(fid,l_tmp,'*char')'; % Varies with device type + + end + + parsed = 1; + + case 7004 % Beam Geometry + % Description: This record is produced by the 7k sonar source. + % It contains the receive beam widths and steering. The 7k + % sonar source updates this data for each ping. The record can + % be manually requested for the last ping or subscribed to from + % the 7k sonar source. For details about requesting and + % subscribing to records, see section 10.62 7500 – Remote + % Control together with section 11 7k Remote Control + % Definitions. + + % NOTE + % * Beam angles are relative to sonar frame when beam + % stabilization is switched off. When enabled it will be + % relative to the vertical. + % * Beam vertical is always zero, angles are relative to sonar + % frame. + + % start parsing RTH + S7Kdata.(recordName).SonarID(iRec) = fread(fid,1,'uint64'); % Sonar serial number + S7Kdata.(recordName).N(iRec) = fread(fid,1,'uint32'); % Number of receiver beams + + N = S7Kdata.(recordName).N(iRec); + + % start parsing RD + + % Angle in radians. The receiver beam steering angle (relative + % to nadir) applied in the alongtrack direction (typically 0). + S7Kdata.(recordName).BeamVerticalDirectionAngleRad{iRec} = fread(fid,N,'float32'); + + % Angle in radians. The receiver beam steering angle (relative + % to nadir) applied in the acrosstrack direction (varies + % according to beam number). Typically -75 to +75 degrees. In + % equidistant mode, this will not change. In equiangular mode, + % steering angles will vary. + S7Kdata.(recordName).BeamHorizontalDirectionAngleRad{iRec} = fread(fid,N,'float32'); + + % Angle in radians. The receiver along-track beam width + % measured at the -3dB points (typically <30deg). + S7Kdata.(recordName).BeamWidth3dBAlongTrackRad{iRec} = fread(fid,N,'float32'); + + % Angle in radians. The receiver across-track beam width + % measured at the -3dB points (typically <5deg). + S7Kdata.(recordName).BeamWidth3dBAcrossTrackRad{iRec} = fread(fid,N,'float32'); + + % Tx Delay for the beam in fractional samples, zero when not + % applicable. + % The Tx Delay is not existing on all sonar models. Up to now + % this is only supported for the HydroSweep sonars (see section + % 9.3). + % When the sonar does not has Tx Delay the item will not be in + % the Record Data, check record length in the Data Record + % Frame. + S7Kdata.(recordName).TxDelay{iRec} = fread(fid,N,'float32'); + + parsed = 1; + + case 7012 % Ping Motion Data + % Description: This record is produced by the 7k sonar source + % series. It contains the description of various parameters + % used in detection computations. The 7k sonar source updates + % this data for each ping. The record can be subscribed to from + % the 7k sonar source. For details about requesting and + % subscribing to records, see section 10.62 7500 – Remote + % Control together with section 11 7k Remote Control + % Definitions. + + % NOTE + % These are not actual steering angles. In order to get actual + % steering angles this data should be used in conjunction with + % base transmit and receive angles from record 7004 – 7k Beam + % Geometry. + + % start parsing RTH + S7Kdata.(recordName).SonarID(iRec) = fread(fid,1,'uint64'); % Sonar serial number + S7Kdata.(recordName).PingNumber(iRec) = fread(fid,1,'uint32'); % Sequential number + + % Flag to indicate multi-ping sequence. + % Always 0 (zero) if not in multi-ping mode; otherwise this + % represents the sequence number of the ping in the multi-ping + % sequence. + S7Kdata.(recordName).MultiPingSequence(iRec) = fread(fid,1,'uint16'); + + S7Kdata.(recordName).NumberOfSamples(iRec) = fread(fid,1,'uint32'); % Number of samples + S7Kdata.(recordName).Flags(iRec) = fread(fid,1,'uint16'); % BIT FIELD. See doc + S7Kdata.(recordName).ErrorFlags(iRec) = fread(fid,1,'uint32'); % BIT FIELD. See doc + S7Kdata.(recordName).SamplingRate(iRec) = fread(fid,1,'float32'); % Sampling frequency in Hz + + % NOTE + % The fields, Pitch, Roll, Heading, and Heave, are present only + % if corresponding flags are set. The new fields may be added + % (refer to the record size in the record header for the total + % size). For sign explanations, see section 2.2 Sign + % Conventions. + + % read and parse flags + flags = CFF_get_R7012_flags(S7Kdata.(recordName).Flags(iRec)); + + % Pitch value at the ping time in radians + if flags.pitchStab > 0 + S7Kdata.(recordName).Pitch(iRec) = fread(fid,1,'float32'); + else + S7Kdata.(recordName).Pitch(iRec) = NaN; + end + + N = S7Kdata.(recordName).NumberOfSamples(iRec); + + % Roll value per sample in radians + if flags.rollStab > 0 + S7Kdata.(recordName).Roll{iRec} = fread(fid,N,'float32'); + else + S7Kdata.(recordName).Roll{iRec} = NaN; + end + + % Heading value per sample in radians + if flags.yawStab > 0 + S7Kdata.(recordName).Heading{iRec} = fread(fid,N,'float32'); + else + S7Kdata.(recordName).Heading{iRec} = NaN; + end + + % Heave value per sample in meters + if flags.heaveStab > 0 + S7Kdata.(recordName).Heave{iRec} = fread(fid,N,'float32'); + else + S7Kdata.(recordName).Heave{iRec} = NaN; + end + + parsed = 1; + + case 7018 % Beamformed Data + % Description: This record is produced by the 7k sonar source + % series. It contains the sonar beam intensity (magnitude) and + % phase data. The 7k sonar source updates this data for each + % ping. The record can be subscribed to from the 7k sonar + % source. For details about requesting and subscribing to + % records, see section 10.62 7500 – Remote Control together + % with section 11 7k Remote Control Definitions. + % This record is available by subscription only. + % Beams and samples are numbered from 0. Data is sample + % followed by beams. + % First sample 0 of all beams then sample 1 of all beams etc. + % The sampling continues until the set range is reached. (Every + % beam will have the same number of samples) + % Data rates: + % Equation for no data reduction, beam limits, and all sonar + % settings: + % beams * data format bits * sample rate * 10% (header overhead) + % Example: + % 128 beams * 32 bits (sonar setting 6) * 34500 samples/s * 1.1 + % = 155.4432 Mbits/s + + % ----- DEV NOTE ---------------------------------------------- + % This datagram's data is too to be stored in memory. Instead, + % we record the metadata and the position-in-file location of + % the data, which be extracted and stored in binary format at + % the next stage of data conversion. + % ------------------------------------------------------------- + + % start parsing RTH + S7Kdata.(recordName).SonarId(iRec) = fread(fid,1,'uint64'); % Sonar serial number + S7Kdata.(recordName).PingNumber(iRec) = fread(fid,1,'uint32'); % Sequential number + + % Flag to indicate multi-ping sequence. + % Always 0 (zero) if not in multi-ping mode; otherwise this + % represents the sequence number of the ping in the multi-ping + % sequence. + S7Kdata.(recordName).MultipingSequence(iRec) = fread(fid,1,'uint16'); + + S7Kdata.(recordName).N(iRec) = fread(fid,1,'uint16'); % Total number of beams in ping record + S7Kdata.(recordName).S(iRec) = fread(fid,1,'uint32'); % Total number of samples per beam in ping record + S7Kdata.(recordName).Reserved{iRec} = fread(fid,8,'uint32'); % Reserved for future use + + % start parsing RD + + % rest of the record are S cycles of N cycles of a amplitude + % (uint16) / phase (int16) pair. Save position. + S7Kdata.(recordName).BeamformedDataPos(iRec) = ftell(fid); + + parsed = 1; + + case 7022 % Sonar Source Version + % Description: This record provides the 7k sonar source version + % as a NULL terminated string. + + % start parsing RTH + + S7Kdata.(recordName).VersionString{iRec} = fread(fid,32,'*char'); % UTF-8 string, max length 31 characters + null + + parsed = 1; + + case 7027 % Raw Detection Data + % Description: This record is produced by the 7k sonar source + % series. It contains noncompensated detection results. The 7k + % sonar source updates this record on every ping. This record + % is available by subscription only. + % Refer to Appendix F on page 246 for a description of handling + % the 7027 record. + + % start parsing RTH + S7Kdata.(recordName).SonarId(iRec) = fread(fid,1,'uint64'); % Sonar serial number + S7Kdata.(recordName).PingNumber(iRec) = fread(fid,1,'uint32'); % Sequential number + + % Flag to indicate multiping sequence. + % Always 0 (zero) if not in multiping mode; otherwise this + % represents the sequence number of the ping in the multiping + % sequence. + S7Kdata.(recordName).MultipingSequence(iRec) = fread(fid,1,'uint16'); + + S7Kdata.(recordName).N(iRec) = fread(fid,1,'uint32'); % Number of detection points + S7Kdata.(recordName).DataFieldSize(iRec) = fread(fid,1,'uint32'); % Size of detection information block in bytes + S7Kdata.(recordName).DetectionAlgorithm(iRec) = fread(fid,1,'uint8'); % see doc + S7Kdata.(recordName).Flags(iRec) = fread(fid,1,'uint32'); % see doc + S7Kdata.(recordName).SamplingRate(iRec) = fread(fid,1,'float32'); % Sonar’s sampling frequency in Hz + + % Applied transmitter steering angle, in radians + % This angle is used for pitch stabilization. It will be zero + % if the system doesn’t have this feature. The value is the + % same as the Projector beam steering angle of the 7000 record. + % They are both filled from the same variable. + S7Kdata.(recordName).TxAngle(iRec) = fread(fid,1,'float32'); + + % Roll value (in radians) applied to gates; zero if roll + % stabilization is ON. + % This value is made available to be able to draw the gating + % lines in the real-time user interface wedge display. + S7Kdata.(recordName).AppliedRoll(iRec) = fread(fid,1,'float32'); + + S7Kdata.(recordName).Reserved{iRec} = fread(fid,15,'uint32'); % Reserved for future use + + % start parsing RD + + % NOTE + % The following data section is repeated for each detection + % point as defined in RTH. The size of each field is always + % defined in RTH. If the size of this definition does not match + % the size specified in the record’s header, the user must + % assume that there is an updated revision of this record and + % that new fields are added at the end. + + % repeat cycle: N entries of S bytes + temp = ftell(fid); + N = S7Kdata.(recordName).N(iRec); + S = S7Kdata.(recordName).DataFieldSize(iRec); + + % Beam number the detection is taken from + S7Kdata.(recordName).BeamDescriptor{iRec} = fread(fid,N,'uint16',S-2); + fseek(fid,temp+2,'bof'); % to next data type + + % Non-corrected fractional sample number with reference to + % receiver’s acoustic center with the zero sample at the + % transmit time + S7Kdata.(recordName).DetectionPoint{iRec} = fread(fid,N,'float32',S-4); + fseek(fid,temp+6,'bof'); % to next data type + + % Beam steering angle with reference to receiver’s acoustic + % center in the sonar reference frame, at the detection point; + % in radians + S7Kdata.(recordName).RxAngle{iRec} = fread(fid,N,'float32',S-4); + fseek(fid,temp+10,'bof'); % to next data type + + % BIT FIELD + % see doc + S7Kdata.(recordName).Flags2{iRec} = fread(fid,N,'uint32',S-4); + fseek(fid,temp+14,'bof'); % to next data type + + % Detection quality + % see doc + S7Kdata.(recordName).Quality{iRec} = fread(fid,N,'uint32',S-4); + fseek(fid,temp+18,'bof'); % to next data type + + % Detection uncertainty represented as an error normalized to + % the detection point + S7Kdata.(recordName).Uncertainty{iRec} = fread(fid,N,'float32',S-4); + fseek(fid,temp+22,'bof'); % to next data type + + % Intensity of detection point + S7Kdata.(recordName).Intensity{iRec} = fread(fid,N,'float32',S-4); + fseek(fid,temp+26,'bof'); % to next data type + + % Minimum sample number of gate limit + S7Kdata.(recordName).MinLimit{iRec} = fread(fid,N,'float32',S-4); + fseek(fid,temp+30,'bof'); % to next data type + + % Maximum sample number of gate limit + S7Kdata.(recordName).MaxLimit{iRec} = fread(fid,N,'float32',S-4); + fseek(fid,4-S,'cof'); % we need to come back after last jump + + % NOTE + % Transmit and receive steering angles provided in this record + % are total steering angles applied. Refer to record 7004 – + % Beam Geometry and/or record 7012 – Ping Motion Data in order + % to isolate steering components. For sign explanations, see + % section 2.2 Sign Conventions. + + if OD_size~=0 + tmp_pos = ftell(fid); + + % start parsing OD + fread(fid,OD_offset-(tmp_pos-pif_recordstart),'uint8'); + + S7Kdata.(recordName).Frequency(iRec) = fread(fid,1,'float32'); % Ping Frequency in Hz + S7Kdata.(recordName).Latitude(iRec) = fread(fid,1,'float64'); % Latitude of vessel reference point in radians -pi/2 to pi/2, south negative + S7Kdata.(recordName).Longitude(iRec) = fread(fid,1,'float64'); % Longitude of vessel reference point in radians -pi to pi, west negative + S7Kdata.(recordName).Heading(iRec) = fread(fid,1,'float32'); % Heading of vessel at transmit time in radians + + % Method used to correct to chart datum. If height source = 1, then Tide = ‘0’. + % 0 – None + % 1 – RTK + % 2 – Tide + S7Kdata.(recordName).HeightSource{iRec} = fread(fid,1,'uint8'); + + S7Kdata.(recordName).Tide(iRec) = fread(fid,1,'float32'); % In meters + S7Kdata.(recordName).Roll(iRec) = fread(fid,1,'float32'); % Roll (in radians) at transmit time + S7Kdata.(recordName).Pitch(iRec) = fread(fid,1,'float32'); % Pitch (in radians) at transmit time + S7Kdata.(recordName).Heave(iRec) = fread(fid,1,'float32'); % Heave (in radians???) at transmit time + S7Kdata.(recordName).VehicleDepth(iRec) = fread(fid,1,'float32'); % Vehicle depth at transmit time in meters + + % The following set of data items are repeated for each + % beam: + tmp_beam_data = fread(fid,[5 N],'float32'); + + % Depth relative chart datum (or relative waterline if + % Height source = 0) (in meters) + S7Kdata.(recordName).Depth{iRec} = tmp_beam_data(1,:); + + % Along track distance in vessel grid (in meters) + S7Kdata.(recordName).AlongTrackDistance{iRec} = tmp_beam_data(2,:); + + % Across track distance in vessel grid (in meters) + S7Kdata.(recordName).AcrossTrackDistance{iRec} = tmp_beam_data(3,:); + + % Beam pointing angle from vertical in radians + S7Kdata.(recordName).PointingAngle{iRec} = tmp_beam_data(4,:); + + % Beam azimuth angle in radians + S7Kdata.(recordName).AzimuthAngle{iRec} = tmp_beam_data(5,:); + + else + + S7Kdata.(recordName).Frequency(iRec) = NaN; + S7Kdata.(recordName).Latitude(iRec) = NaN; + S7Kdata.(recordName).Longitude(iRec) = NaN; + S7Kdata.(recordName).Heading(iRec) = NaN; + S7Kdata.(recordName).HeightSource{iRec} = ''; + S7Kdata.(recordName).Tide(iRec) = NaN; + S7Kdata.(recordName).Roll(iRec) = NaN; + S7Kdata.(recordName).Pitch(iRec) = NaN; + S7Kdata.(recordName).Heave(iRec) = NaN; + S7Kdata.(recordName).VehicleDepth(iRec) = NaN; + S7Kdata.(recordName).Depth{iRec} = NaN(1,N); + S7Kdata.(recordName).AlongTrackDistance{iRec} = NaN(1,N); + S7Kdata.(recordName).AcrossTrackDistance{iRec} = NaN(1,N); + S7Kdata.(recordName).PointingAngle{iRec} = NaN(1,N); + S7Kdata.(recordName).AzimuthAngle{iRec} = NaN(1,N); + + end + + % start parsing CS + if CS_size == 4 + S7Kdata.(recordName).Checksum(iRec) = fread(fid,1,'uint32'); + elseif CS_size == 0 + S7Kdata.(recordName).Checksum(iRec) = NaN; + else + comms.error('%s: unexpected CS size',recordName); + end + % check data integrity with checksum... TO DO XXX2 + + % confirm parsing + parsed = 1; + + case 7042 % Compressed Water Column Data + % Description: This record is produced by the 7k sonar source + % series. It contains compressed water column data. The 7k + % sonar source updates this record on every ping. This record + % is available by subscription only. For details about + % requesting and subscribing to records, see section 10.62 7500 + % – Remote Control together with section 11 7k Remote Control + % Definitions. + + % NOTE + % Remote command 7500 sub7042 is used to configure this record. + + % NOTE + % See Appendix I 7042 Compressed water column data for a + % description of the compression algorithms used to fill the + % 7042 record. + + % The Compressed Water Column record allows for the reduction + % in record data size (vs. the standard 7018 full + % magnitude/phase record) via the several possible options + % listed here. “Downsampling” means that only 1 of N mag/phase + % samples are kept. Where ‘N’ is the “downsampling factor” + % value. The “downsampling type” controls how that 1 value is + % determined. The three choices are: + % 1. Middle of window: + % The 1 sample value kept is the middle one in each “window” + % (e.g. if N = 5, then we keep sample 3, 8, 13, 18, …) + % 2. Peak: + % The 1 sample value kept is the largest of each “window” of N + % samples. + % 3. Average: + % The 1 sample value kept is the average of all N samples in + % each “window”. + + % For example, if the “Remove phase data” option and the + % Downsampling option (with factor 5) are selected, then the + % resulting Compressed Water Column record will be 1/2 * 1/5 = + % 1/10 the size of corresponding 7018 mag+phase Water Column + % Record. + + % ----- DEV NOTE ---------------------------------------------- + % This datagram's data is too to be stored in memory. Instead, + % we record the metadata and the position-in-file location of + % the data, which be extracted and stored in binary format at + % the next stage of data conversion. + % ------------------------------------------------------------- + + % start parsing RTH + S7Kdata.(recordName).SonarId(iRec) = fread(fid,1,'uint64'); % Sonar serial number. + S7Kdata.(recordName).PingNumber(iRec) = fread(fid,1,'uint32'); % Sequential number. + + % Flag to indicate multi-ping sequence. + % Always 0 (zero) if not in Multi-Ping mode; otherwise this + % represents the sequence number of the ping in the multi-ping + % sequence. + S7Kdata.(recordName).MultiPingSequence(iRec) = fread(fid,1,'uint16'); + + S7Kdata.(recordName).Beams(iRec) = fread(fid,1,'uint16'); % Number of beams. + S7Kdata.(recordName).Samples(iRec) = fread(fid,1,'uint32'); % Number of samples (nominal, based on range) + + % Number of samples (maximum over all beams if Flags bit 0 set + % [samples per beam varies]. Otherwise same as Samples(N) ) + % When all beams come with the same number of samples + % 'Compressed Samples' is the same as 'Samples(N)' for each + % beam in the data section of the record. But if bit 0 is set + % in the 'Flags' the beams are individually cut based on bottom + % detection and thus have all different length. 'Compressed + % Samples' then gives you the maximum number of samples of the + % beam with the longest range. Same as the largest value of + % 'Samples(N)' in the data section. + S7Kdata.(recordName).CompressedSamples(iRec) = fread(fid,1,'uint32'); + + % BIT FIELD. See CFF_get_R7042_flags + S7Kdata.(recordName).Flags(iRec) = fread(fid,1,'uint32'); + + % flag processing + [flags,sample_size,~,~] = CFF_get_R7042_flags(S7Kdata.(recordName).Flags(iRec)); + + % First sample included for each beam. Normally zero, unless + % power saving mode “Range Blank” or absolute gate (bit 3) is + % in effect. See RC 1046 for details. Thus, the samples in each + % beam data section will run from F to F+N-1. Construction of a + % correct water column image must take this into account. + S7Kdata.(recordName).FirstSample(iRec) = fread(fid,1,'uint32'); + + S7Kdata.(recordName).SampleRate(iRec) = fread(fid,1,'float32'); % Effective sample rate after downsampling, if specified. + S7Kdata.(recordName).CompressionFactor(iRec) = fread(fid,1,'float32'); % Factor used in intensity (magnitude) compression. + S7Kdata.(recordName).Reserved(iRec) = fread(fid,1,'uint32'); % Zero. Reserved for future use + + % NOTE + % The following data section is repeated for each beam (B) as + % defined in RTH. The size may vary for each beam if bottom + % detection truncation is in effect (Flags bit 0 is set). + % IMPORTANT: This is “reversed” compared to the data ordering + % in the standard 7018 Water Column record! + + % start parsing RD + % repeat cycle: B entries of a possibly variable number of + % bits. Reading everything first and using a for loop to parse + % the data in it + pos_2 = ftell(fid); % position at start of data + RTH_size = 44; + RD_size = RTHandRD_size - RTH_size; + blocktmp = fread(fid,RD_size,'int8=>int8')'; % read all that data block + + wc_parsing_error = 0; % initialize flag + + % initialize outputs + B = S7Kdata.(recordName).Beams(iRec); + S7Kdata.(recordName).BeamNumber{iRec} = NaN(1,B); + S7Kdata.(recordName).SegmentNumber{iRec} = NaN(1,B); + S7Kdata.(recordName).NumberOfSamples{iRec} = NaN(1,B); + S7Kdata.(recordName).SampleStartPositionInFile{iRec} = NaN(1,B); + + Ns = zeros(1,B); % Number of samples in matrix form + id = zeros(1,B+1); % offset for start of each Nrx block + % now parse the data + if flags.segmentNumbersAvailable + for jj = 1:B + try + % Beam Number for this data. + S7Kdata.(recordName).BeamNumber{iRec}(jj) = typecast(blocktmp(1+id(jj):2+id(jj)),'uint16'); + + % Segment number for this beam. Optional field, see ‘Bit 14’ of Flags. + S7Kdata.(recordName).SegmentNumber{iRec}(jj) = typecast(blocktmp(3+id(jj)),'uint8'); + + % Number of samples included for this beam. + S7Kdata.(recordName).NumberOfSamples{iRec}(jj) = typecast(blocktmp(4+id(jj):7+id(jj)),'uint32'); + + % Record position of data + S7Kdata.(recordName).SampleStartPositionInFile{iRec}(jj) = pos_2 + id(jj) + 7; + + Ns(jj) = S7Kdata.(recordName).NumberOfSamples{iRec}(jj); + id(jj) = 7*jj + sum(Ns)*sample_size; + catch + % if any issue in the recording, flag and exit the loop + S7Kdata.(recordName).NumberOfSamples{iRec}(jj) = 0; + Ns(jj) = 0; + wc_parsing_error = 1; + continue; + end + end + else + % same process but without reading segment number + for jj = 1:B + try + % Beam Number for this data. + S7Kdata.(recordName).BeamNumber{iRec}(jj) = typecast(blocktmp(1+id(jj):2+id(jj)),'uint16'); + + % Number of samples included for this beam. + S7Kdata.(recordName).NumberOfSamples{iRec}(jj) = typecast(blocktmp(3+id(jj):6+id(jj)),'uint32'); + + % Record position of data + S7Kdata.(recordName).SampleStartPositionInFile{iRec}(jj) = pos_2 + id(jj) + 6; + + Ns(jj) = S7Kdata.(recordName).NumberOfSamples{iRec}(jj); + id(jj+1) = 6*jj + sum(Ns).*sample_size; + catch + % if any issue in the recording, flag and exit the loop + S7Kdata.(recordName).NumberOfSamples{iRec}(jj) = 0; + Ns(jj) = 0; + wc_parsing_error = 1; + continue; + end + end + end + + if wc_parsing_error == 0 + % HERE if data parsing all went well + + if OD_size~=0 + tmp_pos = ftell(fid); + % start parsing OD + fread(fid,OD_offset-(tmp_pos-pif_recordstart),'uint8'); + tmp_OD = fread(fid,OD_size,'uint8'); + else + tmp_OD = NaN; + end + + % parsing CS + if CS_size == 4 + S7Kdata.(recordName).Checksum(iRec) = fread(fid,1,'uint32'); + elseif CS_size == 0 + S7Kdata.(recordName).Checksum(iRec) = NaN; + else + comms.error('%s: unexpected CS size',recordName); + end + % check data integrity with checksum... TO DO XXX2 + + % confirm parsing + parsed = 1; + + else + % HERE if data parsing failed, add a blank datagram in + % output + comms.error('%s: error while parsing datagram',recordName); + % copy field names of previous entries + fields_wc = recordNames(S7Kdata.(recordName)); + + % add blanks fields for those missing + for ifi = 1:numel(fields_wc) + if numel(S7Kdata.(recordName).(fields_wc{ifi})) >= iRec + S7Kdata.(recordName).(fields_wc{ifi})(iRec) = []; + end + end + + iRec = iRec-1; % XXX1 if we do that, then we'll rewrite over the blank record we just entered?? + parsed = 0; + + end + + case 7200 % File Header + % Description: First record of 7k data file. + + % start parsing RTH + S7Kdata.(recordName).FileIdentifier{iRec} = fread(fid,2,'uint64'); + S7Kdata.(recordName).VersionNumber(iRec) = fread(fid,1,'uint16'); % File format version number + S7Kdata.(recordName).Reserved(iRec) = fread(fid,1,'uint16'); % Reserved + S7Kdata.(recordName).SessionIdentifier{iRec} = fread(fid,2,'uint64'); % User defined session identifier. Used to associate multiple files for a given session. + S7Kdata.(recordName).RecordDataSize(iRec) = fread(fid,1,'uint32'); % Size of record data. 0 – If not present + S7Kdata.(recordName).N(iRec) = fread(fid,1,'uint32'); % Number of devices (N >= 0) + S7Kdata.(recordName).RecordingName{iRec} = fread(fid,64,'*char'); % Null terminated UTF-8 string + S7Kdata.(recordName).RecordingProgramVersionNumber{iRec} = fread(fid,16,'*char'); % Null terminated UTF-8 string + S7Kdata.(recordName).UserDefinedName{iRec} = fread(fid,64,'*char'); % Null terminated UTF-8 string + S7Kdata.(recordName).Notes{iRec} = fread(fid,128,'*char'); % Null terminated UTF-8 string + + % start parsing RD + + % repeat cycle: N entries of 6 bytes + temp = ftell(fid); + N = S7Kdata.(recordName).N(iRec); + S7Kdata.(recordName).DeviceIdentifier{iRec} = fread(fid,N,'uint32',6-4); % Identifier for record type of embedded data + fseek(fid,temp+4,'bof'); % to next data type + S7Kdata.(recordName).SystemEnumerator{iRec} = fread(fid,N,'uint16',6-2); % Identifier for the device enumerator + fseek(fid,2-6,'cof'); % we need to come back after last jump + + % This record may have optional data that contains information + % about the file catalog (7300 record) at the end of the log + % file. The optional data identifier (in the record frame) will + % be 7300. + + if OD_size>= 12 + tmp_pos = ftell(fid); + % start parsing OD + fread(fid,OD_offset-(tmp_pos-pif_recordstart),'uint8'); + S7Kdata.(recordName).Size(iRec) = fread(fid,1,'uint32'); % Size of the file catalog record + S7Kdata.(recordName).Offset(iRec) = fread(fid,1,'uint64'); % File offset of the file catalog record + elseif OD_size == 0 + S7Kdata.(recordName).Size(iRec) = NaN; + S7Kdata.(recordName).Offset(iRec) = NaN; + else + comms.error('%s: unexpected OD size',recordName); + end + + % start parsing CS + if CS_size == 4 + S7Kdata.(recordName).Checksum(iRec) = fread(fid,1,'uint32'); + elseif CS_size == 0 + S7Kdata.(recordName).Checksum(iRec) = NaN; + else + comms.error('%s: unexpected CS size',recordName); + end + % check data integrity with checksum... TO DO XXX2 + + % confirm parsing + parsed = 1; + + case 7610 % Sound Velocity + % Description: This record can be used to set the SeaBat™ 7k + % sonar series systems current sound velocity value. The record + % can be manually requested or subscribed to from the 7k sonar + % source. For details about requesting and subscribing to + % records, see section 10.62 7500 – Remote Control together + % with section 11 7k Remote Control Definitions. + + % start parsing RTH + S7Kdata.(recordName).SoundVelocity(iRec) = fread(fid,1,'float32'); % In meters/second + S7Kdata.(recordName).Temperature(iRec) = fread(fid,1,'float32'); % Kelvin (optional) + S7Kdata.(recordName).Pressure(iRec) = fread(fid,1,'float32'); % Pascal (optional) + + % NOTE + % The Pressure field is for information only. It is not used in + % the 7k sonar source. The 7k sonar source simply passes the + % record along to subscribers and raw data recording with no + % source code changes. + % The field is not present with a version of the IO Module + % older than V4.0.0.8. When the value is zero it is not valid. + + % NOTE + % If filtering is enabled in record 7510 (see section 10.67 + % 7510 – SV Filtering), record 7610 will contain filtered + % values used by 7k sonar source when broadcasted by 7k sonar + % source, except when in manual overwrite mode. + + % NOTE + % Record 7610 is updated for single request, but not broadcast + % when manual value is received via remote control command + % 7610. + % Record 7610 is generated every time surface sound velocity + % value is received via record 7610 even if 7k sonar source is + % in manual overwrite mode and the value was ignored. In this + % case, returned sound velocity value will be unfiltered. + + parsed = 1; + + otherwise + % recordTypeIdentifier is not recognized yet + + end + + % modify parsed status in info + S7Kfileinfo.parsed(iDatag,1) = parsed; + + % and date and time + if parsed == 1 + S7Kdata.(recordName).TimeSinceMidnightInMilliseconds(iRec) = S7Kfileinfo.timeSinceMidnightInMilliseconds(iDatag); + S7Kdata.(recordName).Date(iRec) = str2double(S7Kfileinfo.date{iDatag}); + end + + % communicate progress + comms.progress(iDatag,nDatagsToParse); + +end + + +%% finalise + +% close fid +fclose(fid); + +% add info to parsed data +S7Kdata.info = S7Kfileinfo; + +% end message +comms.finish('Done'); + +end diff --git a/read_data_files/Reson/CFF_s7K_record_types.m b/read_data_files/Reson/CFF_s7K_record_types.m new file mode 100644 index 0000000..66c0d88 --- /dev/null +++ b/read_data_files/Reson/CFF_s7K_record_types.m @@ -0,0 +1,100 @@ +function [listRecordTypeText,s7kFormatVersion] = CFF_s7K_record_types() +%CFF_S7K_RECORD_TYPES List of s7k record types +% +% See also CFF_S7K_FILE_INFO + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2021-2022; Last revision: 21-07-2022 + +% The list of record types below was established from the s7k Data Format +% documentation version 3.12 (May 2020). This may be updated but ensure the +% first five characters make up the record type identifier. +s7kFormatVersion = '3.12'; + +listRecordTypeText = {... + '1000 – Reference Point';... + '1001 – Sensor Offset Position';... + '1002 – Sensor Offset Position Calibrated';... + '1003 – Position';... + '1004 – Custom Attitude Information';... + '1005 – Tide';... + '1006 – Altitude';... + '1007 – Motion Over Ground';... + '1008 – Depth';... + '1009 – Sound Velocity Profile';... + '1010 – CTD';... + '1011 – Geodesy';... + '1012 – Roll Pitch Heave';... + '1013 – Heading';... + '1014 – Survey Line';... + '1015 – Navigation';... + '1016 – Attitude';... + '1017 – Pan Tilt';... + '1020 – Sonar Installation Identifiers';... + '2004 – Sonar Pipe Environment';... + '3001 – Contact Output';... + '7000 – Sonar Settings';... + '7001 – Configuration';... + '7002 – Match Filter';... + '7003 – Firmware and Hardware Configuration';... + '7004 – Beam Geometry';... + '7006 – Bathymetric Data';... % superseded by record 7027 + '7007 – Side Scan Data';... + '7008 – Generic Water Column Data';... % superseded by 7018 and 7028 + '7010 – TVG Values';... + '7011 – Image Data';... + '7012 – Ping Motion Data';... + '7017 – Detection Data Setup';... % deprecated + '7018 – Beamformed Data';... + '7019 – Vernier Processing Data (Raw)';... + '7021 – Built-In Test Environment Data';... + '7022 – Sonar Source Version';... + '7023 – 8k Wet End Version';... + '7027 – Raw Detection Data';... + '7028 – Snippet Data';... + '7029 – Vernier Processing Data (Filtered)';... + '7030 – Sonar Installation Parameters';... + '7031 – Built-In Test Environment Data (Summary)';... + '7041 – Compressed Beamformed Intensity Data';... % deprecated by 7042 + '7042 - Compressed Water Column Data';... + '7047 – Segmented Raw Detection Data';... + '7048 – Calibrated Beam Data';... + '7050 – System Events';... + '7051 – System Event Message';... + '7052 – RDR Recording Status - Detailed';... + '7053 – Subscriptions';... + '7054 – RDR Storage Recording – Short Update';... + '7055 – Calibration Status';... + '7057 – Calibrated Side-Scan Data';... + '7058 – Snippet Scattering Strength';... + '7059 – MB2 specific status';... + '7200 – File Header';... + '7300 – File Catalog Record';... + '7400 – Time Message';... + '7500 – Remote Control';... + '7501 – Remote Control Acknowledge';... + '7502 – Remote Control Not Acknowledge';... + '7503 – Remote Control Sonar Settings';... + '7504 – Common System Settings';... + '7510 – SV Filtering';... + '7511 – System Lock Status';... + '7610 – Sound Velocity';... + '7611 – Absorption Loss';... + '7612 – Spreading Loss'; ... + '7613 – Profile Average Salinity';... + '7614 – Profile Average Temperature';... + '10000 – SBES Channel Settings';... + '10003 - SBES Channel Status';... + '10004 – SBES System Settings';... + '10018 – SBES Echogram Water Column Data';... + '10025 – SBES annotation';... + '10027 – SBES Raw Detection Data';... + '10038 – SBES Raw IQ data';... + '10039 – SBES Clipping data';... + '10049 – SBES Depth Reading';... + '10060 – SBES Port Configuration';... + '10061 – SBES Port Status';... + '10062 – SBES Port QC';... + '10063 – UDP Port Configuration' ... + }; \ No newline at end of file diff --git a/read_data_files/Reson/CFF_s7k_file_info.m b/read_data_files/Reson/CFF_s7k_file_info.m new file mode 100644 index 0000000..b5216c5 --- /dev/null +++ b/read_data_files/Reson/CFF_s7k_file_info.m @@ -0,0 +1,290 @@ +function S7Kfileinfo = CFF_s7k_file_info(S7Kfilename, varargin) +%CFF_S7K_FILE_INFO Records basic info about contents of .s7k file +% +% Records basic info about the datagrams contained in one binary raw data +% file in the Teledyne-Reson format .s7k. +% +% S7Kfileinfo = CFF_S7K_FILE_INFO(S7Kfilename) opens file S7Kfilename and +% reads through the start of each datagram to get basic information about +% it, and store it all in S7Kfileinfo. +% +% *INPUT VARIABLES* +% * |S7Kfilename|: Required. String filename to parse (extension in .s7k) +% +% *OUTPUT VARIABLES* +% * |S7Kfileinfo|: structure containing information about records in +% S7Kfilename, with fields: +% * |S7Kfilename|: input file name +% * |fileSize|: file size in bytes +% * |datagsizeformat|: endianness of the datagram size field. Always +% 'l' for .s7k files +% * |datagramsformat|: endianness of the datagrams 'b' or 'l'. Always +% 'l' for .s7k files +% * |recordNumberInFile|: number of record in file +% * |recordTypeIdentifier|: record type in int (Reson .s7k format) +% * |recordTypeText|: record type description (Reson .s7k format) +% * |recordTypeCounter|: counter of this type of record in the file (ie +% first record of that type is 1 and last record is the total +% number of record of that type) +% * |recordStartPositionInFile|: position of beginning of record in +% file +% * |recordSize|: record size in bytes +% * |DRF_size|: size of the "Data Record Frame" part of the record +% (Reson .s7k format) +% * |RTHandRD_size|: combined size of the "Record Type Header" and +% "Record Data" parts of the record (Reson .s7k format) +% * |OD_offset|: offset of the "Optional Data" part of the record +% (Reson .s7k format) +% * |OD_size|: size of the "Optional Data" part of the record (Reson +% .s7k format) +% * |CS_size|: size of the "Checksum" part of the record (Reson .s7k +% format) +% * |syncCounter|: number of bytes found between this record and the +% previous one (any number different than zero indicates a sync error) +% * |date|: datagram date in YYYMMDD +% * |timeSinceMidnightInMilliseconds|: time since midnight in +% milliseconds +% * |parsed|: flag for whether the record has been parsed. Initiated +% at 0 at this stage. To be later turned to 1 for parsing. +% +% *DEVELOPMENT NOTES* +% * Check regularly with Reson doc to keep updated with new datagrams. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-08-2021 + + +%% Input arguments management +p = inputParser; + +% name of the .s7k file +argName = 'S7Kfilename'; +argCheck = @(x) CFF_check_S7Kfilename(x); +addRequired(p,argName,argCheck); + +% information communication +addParameter(p,'comms',CFF_Comms()); % no communication by default + +% parse inputs +parse(p,S7Kfilename,varargin{:}); + +% and get results +S7Kfilename = p.Results.S7Kfilename; +if ischar(p.Results.comms) + comms = CFF_Comms(p.Results.comms); +else + comms = p.Results.comms; +end + + +%% Start message +filename = CFF_file_name(S7Kfilename,1); +comms.start(sprintf('Listing datagrams in file %s',filename)); + + +%% Open file and initializing +% s7k files are in little Endian, but just use the default with fopen +[fid,~] = fopen(S7Kfilename, 'r'); + +% go to end of file to get number of bytes in file then rewind +fseek(fid,0,1); +fileSize = ftell(fid); +fseek(fid,0,-1); + +% init output info +S7Kfileinfo.fileName = S7Kfilename; +S7Kfileinfo.fileSize = fileSize; + +% initialize the counter of total records in this file, and records of +% given type +kk = 0; +listRecordTypeText = CFF_s7K_record_types(); +listRecordTypeIdentifier = cellfun(@(x) str2double(x(1:5)), listRecordTypeText); +listRecordTypeCounter = zeros(size(listRecordTypeIdentifier)); + +% initializing synchronization counter: the number of bytes that needed to +% be passed before this datagram appeared +syncCounter = 0; + + +%% Start progress +comms.progress(0,fileSize); + + +%% Reading records +pifNextRecordStart = 0; +while pifNextRecordStart < fileSize + + %% new record begins + pifRecordStart = ftell(fid); + + % A full s7k record is organized as a sequence of: + % * DRF - Data Record Frame (64 bytes, at least for protocol version 5) + % * RTH - Record Type Header (variable size) + % * RD - Record Data (optional, variable size) + % * OD - Optional Data (optional, variable size) + % * CS - Checksum (optional, 4 bytes) + + % start parsing DRF + protocolVersion = fread(fid,1,'uint16'); % should be 5 + DRF_offset = fread(fid,1,'uint16'); % should be 60, for version 5 + syncPattern = fread(fid,1,'uint32'); % should be 65535 + + + %% test for synchronization + if protocolVersion~=5 || DRF_offset~=60 || syncPattern~=65535 + % NOT SYNCHRONIZED + % go back to new record start, advance one byte, and restart + % reading + fseek(fid, pifRecordStart+1, -1); + pifNextRecordStart = -1; + syncCounter = syncCounter+1; % update sync counter + if syncCounter == 1 + % just lost sync, throw a message just now + comms.error('Lost sync while reading records. A record may be corrupted. Trying to resync...'); + end + continue; + else + % SYNCHRONIZED + if syncCounter + % if we had lost sync, warn here we're back + comms.info(sprintf('Back in sync (%i bytes later)',syncCounter)); + % reinitialize sync counter + syncCounter = 0; + end + end + + + %% read more information from start of record + + % finish parsing DRF + recordSize = fread(fid,1,'uint32'); + optionalDataOffset = fread(fid,1,'uint32'); + optionalDataIdentifier = fread(fid,1,'uint32'); + sevenKTime_year = fread(fid,1,'uint16'); + sevenKTime_day = fread(fid,1,'uint16'); + sevenKTime_seconds = fread(fid,1,'float32'); + sevenKTime_hours = fread(fid,1,'uint8'); + sevenKTime_minutes = fread(fid,1,'uint8'); + recordVersion = fread(fid,1,'uint16'); + recordTypeIdentifier = fread(fid,1,'uint32'); + deviceIdentifier = fread(fid,1,'uint32'); + reserved1 = fread(fid,1,'uint16'); + systemEnumerator = fread(fid,1,'uint16'); + reserved2 = fread(fid,1,'uint32'); + flags = fread(fid,1,'uint16'); + reserved3 = fread(fid,1,'uint16'); + reserved4 = fread(fid,1,'uint32'); + totalRecordsInFragmentedDataRecordSet = fread(fid,1,'uint32'); + fragmentNumber = fread(fid,1,'uint32'); + + % size of DRF in bytes + DRF_size = DRF_offset + 4; + + % checksum size + if mod(flags,2) + % flag is an odd number, aka the last 4 bytes of the record are the + % checksum + CS_size = 4; + else + % flag is an even number, aka no checksum + CS_size = 0; + end + + % position in file of start of RTH (this is where we should be now) + % pif_RTHstart = pifRecordStart + DRF_size; + + % position in file of next record + pifNextRecordStart = pifRecordStart + recordSize; + + % size of OD and position in file + if optionalDataOffset == 0 + % no OD + OD_size = 0; + % pif_ODstart = NaN; + else + OD_size = recordSize - ( optionalDataOffset + CS_size); + % pif_ODstart = pifRecordStart + optionalDataOffset; + end + + % size of the actual data section (RTH and RD) + RTHandRD_size = recordSize - ( DRF_size + OD_size + CS_size); + + + %% record type counter + + % index of record type in the list + recordType_idx = find(recordTypeIdentifier == listRecordTypeIdentifier); + + if isempty(recordType_idx) + % this record type is not recognized + recordTypeText = sprintf('%i - UNKNOWN RECORD TYPE',recordTypeIdentifier); + recordTypeCounter = NaN; + else + % record type text + recordTypeText = listRecordTypeText{recordType_idx}; + % increment counter for this record type + listRecordTypeCounter(recordType_idx) = listRecordTypeCounter(recordType_idx) + 1; + recordTypeCounter = listRecordTypeCounter(recordType_idx); + end + + + %% write output S7Kfileinfo + + % Record complete + kk = kk + 1; + + % Record number in file + S7Kfileinfo.recordNumberInFile(kk,1) = kk; + + % Type of record info + S7Kfileinfo.recordTypeIdentifier(kk,1) = recordTypeIdentifier; + S7Kfileinfo.recordTypeText{kk,1} = recordTypeText; + S7Kfileinfo.recordTypeCounter(kk,1) = recordTypeCounter; + + % position of start of record in file + S7Kfileinfo.recordStartPositionInFile(kk,1) = pifRecordStart; + + % size of record and its components + S7Kfileinfo.recordSize(kk,1) = recordSize; + S7Kfileinfo.DRF_size(kk,1) = DRF_size; + S7Kfileinfo.RTHandRD_size(kk,1) = RTHandRD_size; + + S7Kfileinfo.OD_offset(kk,1) = optionalDataOffset; + S7Kfileinfo.OD_size(kk,1) = OD_size; + S7Kfileinfo.CS_size(kk,1) = CS_size; + + % report sync issue if any + S7Kfileinfo.syncCounter(kk,1) = syncCounter; + + % record time info + S7Kfileinfo.date{kk,1} = datestr(datenum(sevenKTime_year,0,sevenKTime_day),'yyyymmdd'); + S7Kfileinfo.timeSinceMidnightInMilliseconds(kk,1) = (sevenKTime_hours.*3600 + sevenKTime_minutes.*60 + sevenKTime_seconds).*1000; + + + %% prepare for reloop + + % go to end of record + fseek(fid, pifNextRecordStart, -1); + + % communicate progress + comms.progress(pifNextRecordStart,fileSize); + +end + + +%% finalizing + +% initialize parsing field +S7Kfileinfo.parsed = zeros(size(S7Kfileinfo.recordNumberInFile)); + +% close file +fclose(fid); + +% end message +comms.finish('Done'); + +end + diff --git a/stats/CFF_invpercentile.m b/stats/CFF_invpercentile.m new file mode 100644 index 0000000..19d7a16 --- /dev/null +++ b/stats/CFF_invpercentile.m @@ -0,0 +1,17 @@ +function [V] = CFF_invpercentile(X,P) +%CFF_INVPERCENTILE Calculates inverse percentile +% +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +if all(isnan(X)) + V = NaN; + return +end + +X = X(:); +X = X(~isnan(X)); +X = sort(X); +iP = round(P.*numel(X)./100); +V = X(iP); diff --git a/toolboxes/dispstat/dispstat.m b/toolboxes/dispstat/dispstat.m new file mode 100644 index 0000000..14f938f --- /dev/null +++ b/toolboxes/dispstat/dispstat.m @@ -0,0 +1,82 @@ +function dispstat(TXT,varargin) +% Prints overwritable message to the command line. If you dont want to keep +% this message, call dispstat function with option 'keepthis'. If you want to +% keep the previous message, use option 'keepprev'. First argument must be +% the message. +% IMPORTANT! In the firt call, option 'init' must be used for initialization purposes. +% Options: +% 'init' this must be called in the begining. Otherwise, it can overwrite the previous outputs on the command line. +% 'keepthis' the message will be persistent, wont be overwritable, +% 'keepprev' the previous message wont be overwritten. New message will start from next line, +% 'timestamp' current time hh:mm:ss will be appended to the begining of the message. +% Example: +% clc; +% fprintf('12345677890\n'); +% dispstat('','init') %Initialization. Does not print anything. +% dispstat('Time stamp will be written over this text.'); % First output +% dispstat('is current time.','timestamp','keepthis'); % Overwrites the previous output but this output wont be overwritten. +% dispstat(sprintf('*********\nDeveloped by %s\n*********','Kasim')); % does not overwrites the previous output +% dispstat('','timestamp','keepprev','keepthis'); % does not overwrites the previous output +% dispstat('this wont be overwriten','keepthis'); +% dispstat('dummy dummy dummy'); +% dispstat('final stat'); +% % Output: +% 12345677890 +% 15:15:34 is current time. +% ********* +% Developed by Kasim +% ********* +% 15:15:34 +% this wont be overwriten +% final stat + +% ********** +% **** Options +keepthis = 0; % option for not overwriting +keepprev = 0; +timestamp = 0; % time stamp option +init = 0; % is it initialization step? +if ~isstr(TXT) + return +end +persistent prevCharCnt; +if isempty(prevCharCnt) + prevCharCnt = 0; +end +if nargin == 0 + return +elseif nargin > 1 + for i = 2:nargin + eval([varargin{i-1} '=1;']); + end +end +if init == 1 + prevCharCnt = 0; + return; +end +if isempty(TXT) && timestamp == 0 + return +end +if timestamp == 1 + c = clock; % [year month day hour minute seconds] + txtTimeStamp = sprintf('%02d:%02d:%02d ',c(4),c(5),round(c(6))); +else + txtTimeStamp = ''; +end +if keepprev == 1 + prevCharCnt = 0; +end +% *************** Make safe for fprintf, replace control charachters +TXT = strrep(TXT,'%','%%'); +TXT = strrep(TXT,'\','\\'); +% *************** Print +TXT = [txtTimeStamp TXT '\n']; +fprintf([repmat('\b',1, prevCharCnt) TXT]); +nof_extra = length(strfind(TXT,'%%')); +nof_extra = nof_extra + length(strfind(TXT,'\\')); +nof_extra = nof_extra + length(strfind(TXT,'\n')); +prevCharCnt = length(TXT) - nof_extra; %-1 is for \n +if keepthis == 1 + prevCharCnt = 0; +end +end \ No newline at end of file diff --git a/toolboxes/dispstat/license.txt b/toolboxes/dispstat/license.txt new file mode 100644 index 0000000..af31c69 --- /dev/null +++ b/toolboxes/dispstat/license.txt @@ -0,0 +1,24 @@ +Copyright (c) 2013, kasim tasdemir +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/toolboxes/inpaint_nans/inpaint_nans.m b/toolboxes/inpaint_nans/inpaint_nans.m new file mode 100644 index 0000000..3069951 --- /dev/null +++ b/toolboxes/inpaint_nans/inpaint_nans.m @@ -0,0 +1 @@ +% INPAINT_NANS: in-paints over nans in an array % usage: B=INPAINT_NANS(A) % default method % usage: B=INPAINT_NANS(A,method) % specify method used % % Solves approximation to one of several pdes to % interpolate and extrapolate holes in an array % % arguments (input): % A - nxm array with some NaNs to be filled in % % method - (OPTIONAL) scalar numeric flag - specifies % which approach (or physical metaphor to use % for the interpolation.) All methods are capable % of extrapolation, some are better than others. % There are also speed differences, as well as % accuracy differences for smooth surfaces. % % methods {0,1,2} use a simple plate metaphor. % method 3 uses a better plate equation, % but may be much slower and uses % more memory. % method 4 uses a spring metaphor. % method 5 is an 8 neighbor average, with no % rationale behind it compared to the % other methods. I do not recommend % its use. % % method == 0 --> (DEFAULT) see method 1, but % this method does not build as large of a % linear system in the case of only a few % NaNs in a large array. % Extrapolation behavior is linear. % % method == 1 --> simple approach, applies del^2 % over the entire array, then drops those parts % of the array which do not have any contact with % NaNs. Uses a least squares approach, but it % does not modify known values. % In the case of small arrays, this method is % quite fast as it does very little extra work. % Extrapolation behavior is linear. % % method == 2 --> uses del^2, but solving a direct % linear system of equations for nan elements. % This method will be the fastest possible for % large systems since it uses the sparsest % possible system of equations. Not a least % squares approach, so it may be least robust % to noise on the boundaries of any holes. % This method will also be least able to % interpolate accurately for smooth surfaces. % Extrapolation behavior is linear. % % method == 3 --+ See method 0, but uses del^4 for % the interpolating operator. This may result % in more accurate interpolations, at some cost % in speed. % % method == 4 --+ Uses a spring metaphor. Assumes % springs (with a nominal length of zero) % connect each node with every neighbor % (horizontally, vertically and diagonally) % Since each node tries to be like its neighbors, % extrapolation is as a constant function where % this is consistent with the neighboring nodes. % % method == 5 --+ See method 2, but use an average % of the 8 nearest neighbors to any element. % This method is NOT recommended for use. % % % arguments (output): % B - nxm array with NaNs replaced % % % Example: % [x,y] = meshgrid(0:.01:1); % z0 = exp(x+y); % znan = z0; % znan(20:50,40:70) = NaN; % znan(30:90,5:10) = NaN; % znan(70:75,40:90) = NaN; % % z = inpaint_nans(znan); % % % See also: griddata, interp1 % % Author: John D'Errico % e-mail address: woodchips@rochester.rr.com % Release: 2 % Release date: 4/15/06 function B=inpaint_nans(A,method) % I always need to know which elements are NaN, % and what size the array is for any method [n,m]=size(A); A=A(:); nm=n*m; k=isnan(A(:)); if ~any(k) B=reshape(A,n,m); return; end % list the nodes which are known, and which will % be interpolated nan_list=find(k); known_list=find(~k); % how many nans overall nan_count=length(nan_list); % convert NaN indices to (r,c) form % nan_list==find(k) are the unrolled (linear) indices % (row,column) form [nr,nc]=ind2sub([n,m],nan_list); % both forms of index in one array: % column 1 == unrolled index % column 2 == row index % column 3 == column index nan_list=[nan_list,nr,nc]; % supply default method if (nargin<2) || isempty(method) method = 0; elseif ~ismember(method,0:5) error 'If supplied, method must be one of: {0,1,2,3,4,5}.' end % for different methods switch method case 0 % The same as method == 1, except only work on those % elements which are NaN, or at least touch a NaN. % horizontal and vertical neighbors only talks_to = [-1 0;0 -1;1 0;0 1]; neighbors_list=identify_neighbors(n,m,nan_list,talks_to); % list of all nodes we have identified all_list=[nan_list;neighbors_list]; % generate sparse array with second partials on row % variable for each element in either list, but only % for those nodes which have a row index > 1 or < n L = find((all_list(:,2) > 1) & (all_list(:,2) < n)); nl=length(L); if nl>0 fda=sparse(repmat(all_list(L,1),1,3), ... repmat(all_list(L,1),1,3)+repmat([-1 0 1],nl,1), ... repmat([1 -2 1],nl,1),nm,nm); else fda=spalloc(n*m,n*m,size(all_list,1)*5); end % 2nd partials on column index L = find((all_list(:,3) > 1) & (all_list(:,3) < m)); nl=length(L); if nl>0 fda=fda+sparse(repmat(all_list(L,1),1,3), ... repmat(all_list(L,1),1,3)+repmat([-n 0 n],nl,1), ... repmat([1 -2 1],nl,1),nm,nm); end % eliminate knowns rhs=-fda(:,known_list)*A(known_list); k=find(any(fda(:,nan_list(:,1)),2)); % and solve... B=A; B(nan_list(:,1))=fda(k,nan_list(:,1))\rhs(k); case 1 % least squares approach with del^2. Build system % for every array element as an unknown, and then % eliminate those which are knowns. % Build sparse matrix approximating del^2 for % every element in A. % Compute finite difference for second partials % on row variable first [i,j]=ndgrid(2:(n-1),1:m); ind=i(:)+(j(:)-1)*n; np=(n-2)*m; fda=sparse(repmat(ind,1,3),[ind-1,ind,ind+1], ... repmat([1 -2 1],np,1),n*m,n*m); % now second partials on column variable [i,j]=ndgrid(1:n,2:(m-1)); ind=i(:)+(j(:)-1)*n; np=n*(m-2); fda=fda+sparse(repmat(ind,1,3),[ind-n,ind,ind+n], ... repmat([1 -2 1],np,1),nm,nm); % eliminate knowns rhs=-fda(:,known_list)*A(known_list); k=find(any(fda(:,nan_list),2)); % and solve... B=A; B(nan_list(:,1))=fda(k,nan_list(:,1))\rhs(k); case 2 % Direct solve for del^2 BVP across holes % generate sparse array with second partials on row % variable for each nan element, only for those nodes % which have a row index > 1 or < n L = find((nan_list(:,2) > 1) & (nan_list(:,2) < n)); nl=length(L); if nl>0 fda=sparse(repmat(nan_list(L,1),1,3), ... repmat(nan_list(L,1),1,3)+repmat([-1 0 1],nl,1), ... repmat([1 -2 1],nl,1),n*m,n*m); else fda=spalloc(n*m,n*m,size(nan_list,1)*5); end % 2nd partials on column index L = find((nan_list(:,3) > 1) & (nan_list(:,3) < m)); nl=length(L); if nl>0 fda=fda+sparse(repmat(nan_list(L,1),1,3), ... repmat(nan_list(L,1),1,3)+repmat([-n 0 n],nl,1), ... repmat([1 -2 1],nl,1),n*m,n*m); end % fix boundary conditions at extreme corners % of the array in case there were nans there if ismember(1,nan_list(:,1)) fda(1,[1 2 n+1])=[-2 1 1]; end if ismember(n,nan_list(:,1)) fda(n,[n, n-1,n+n])=[-2 1 1]; end if ismember(nm-n+1,nan_list(:,1)) fda(nm-n+1,[nm-n+1,nm-n+2,nm-n])=[-2 1 1]; end if ismember(nm,nan_list(:,1)) fda(nm,[nm,nm-1,nm-n])=[-2 1 1]; end % eliminate knowns rhs=-fda(:,known_list)*A(known_list); % and solve... B=A; k=nan_list(:,1); B(k)=fda(k,k)\rhs(k); case 3 % The same as method == 0, except uses del^4 as the % interpolating operator. % del^4 template of neighbors talks_to = [-2 0;-1 -1;-1 0;-1 1;0 -2;0 -1; ... 0 1;0 2;1 -1;1 0;1 1;2 0]; neighbors_list=identify_neighbors(n,m,nan_list,talks_to); % list of all nodes we have identified all_list=[nan_list;neighbors_list]; % generate sparse array with del^4, but only % for those nodes which have a row & column index % >= 3 or <= n-2 L = find( (all_list(:,2) >= 3) & ... (all_list(:,2) <= (n-2)) & ... (all_list(:,3) >= 3) & ... (all_list(:,3) <= (m-2))); nl=length(L); if nl>0 % do the entire template at once fda=sparse(repmat(all_list(L,1),1,13), ... repmat(all_list(L,1),1,13) + ... repmat([-2*n,-n-1,-n,-n+1,-2,-1,0,1,2,n-1,n,n+1,2*n],nl,1), ... repmat([1 2 -8 2 1 -8 20 -8 1 2 -8 2 1],nl,1),nm,nm); else fda=spalloc(n*m,n*m,size(all_list,1)*5); end % on the boundaries, reduce the order around the edges L = find((((all_list(:,2) == 2) | ... (all_list(:,2) == (n-1))) & ... (all_list(:,3) >= 2) & ... (all_list(:,3) <= (m-1))) | ... (((all_list(:,3) == 2) | ... (all_list(:,3) == (m-1))) & ... (all_list(:,2) >= 2) & ... (all_list(:,2) <= (n-1)))); nl=length(L); if nl>0 fda=fda+sparse(repmat(all_list(L,1),1,5), ... repmat(all_list(L,1),1,5) + ... repmat([-n,-1,0,+1,n],nl,1), ... repmat([1 1 -4 1 1],nl,1),nm,nm); end L = find( ((all_list(:,2) == 1) | ... (all_list(:,2) == n)) & ... (all_list(:,3) >= 2) & ... (all_list(:,3) <= (m-1))); nl=length(L); if nl>0 fda=fda+sparse(repmat(all_list(L,1),1,3), ... repmat(all_list(L,1),1,3) + ... repmat([-n,0,n],nl,1), ... repmat([1 -2 1],nl,1),nm,nm); end L = find( ((all_list(:,3) == 1) | ... (all_list(:,3) == m)) & ... (all_list(:,2) >= 2) & ... (all_list(:,2) <= (n-1))); nl=length(L); if nl>0 fda=fda+sparse(repmat(all_list(L,1),1,3), ... repmat(all_list(L,1),1,3) + ... repmat([-1,0,1],nl,1), ... repmat([1 -2 1],nl,1),nm,nm); end % eliminate knowns rhs=-fda(:,known_list)*A(known_list); k=find(any(fda(:,nan_list(:,1)),2)); % and solve... B=A; B(nan_list(:,1))=fda(k,nan_list(:,1))\rhs(k); case 4 % Spring analogy % interpolating operator. % list of all springs between a node and a horizontal % or vertical neighbor hv_list=[-1 -1 0;1 1 0;-n 0 -1;n 0 1]; hv_springs=[]; for i=1:4 hvs=nan_list+repmat(hv_list(i,:),nan_count,1); k=(hvs(:,2)>=1) & (hvs(:,2)<=n) & (hvs(:,3)>=1) & (hvs(:,3)<=m); hv_springs=[hv_springs;[nan_list(k,1),hvs(k,1)]]; end % delete replicate springs hv_springs=unique(sort(hv_springs,2),'rows'); % build sparse matrix of connections, springs % connecting diagonal neighbors are weaker than % the horizontal and vertical springs nhv=size(hv_springs,1); springs=sparse(repmat((1:nhv)',1,2),hv_springs, ... repmat([1 -1],nhv,1),nhv,nm); % eliminate knowns rhs=-springs(:,known_list)*A(known_list); % and solve... B=A; B(nan_list(:,1))=springs(:,nan_list(:,1))\rhs; case 5 % Average of 8 nearest neighbors % generate sparse array to average 8 nearest neighbors % for each nan element, be careful around edges fda=spalloc(n*m,n*m,size(nan_list,1)*9); % -1,-1 L = find((nan_list(:,2) > 1) & (nan_list(:,3) > 1)); nl=length(L); if nl>0 fda=fda+sparse(repmat(nan_list(L,1),1,2), ... repmat(nan_list(L,1),1,2)+repmat([-n-1, 0],nl,1), ... repmat([1 -1],nl,1),n*m,n*m); end % 0,-1 L = find(nan_list(:,3) > 1); nl=length(L); if nl>0 fda=fda+sparse(repmat(nan_list(L,1),1,2), ... repmat(nan_list(L,1),1,2)+repmat([-n, 0],nl,1), ... repmat([1 -1],nl,1),n*m,n*m); end % +1,-1 L = find((nan_list(:,2) < n) & (nan_list(:,3) > 1)); nl=length(L); if nl>0 fda=fda+sparse(repmat(nan_list(L,1),1,2), ... repmat(nan_list(L,1),1,2)+repmat([-n+1, 0],nl,1), ... repmat([1 -1],nl,1),n*m,n*m); end % -1,0 L = find(nan_list(:,2) > 1); nl=length(L); if nl>0 fda=fda+sparse(repmat(nan_list(L,1),1,2), ... repmat(nan_list(L,1),1,2)+repmat([-1, 0],nl,1), ... repmat([1 -1],nl,1),n*m,n*m); end % +1,0 L = find(nan_list(:,2) < n); nl=length(L); if nl>0 fda=fda+sparse(repmat(nan_list(L,1),1,2), ... repmat(nan_list(L,1),1,2)+repmat([1, 0],nl,1), ... repmat([1 -1],nl,1),n*m,n*m); end % -1,+1 L = find((nan_list(:,2) > 1) & (nan_list(:,3) < m)); nl=length(L); if nl>0 fda=fda+sparse(repmat(nan_list(L,1),1,2), ... repmat(nan_list(L,1),1,2)+repmat([n-1, 0],nl,1), ... repmat([1 -1],nl,1),n*m,n*m); end % 0,+1 L = find(nan_list(:,3) < m); nl=length(L); if nl>0 fda=fda+sparse(repmat(nan_list(L,1),1,2), ... repmat(nan_list(L,1),1,2)+repmat([n, 0],nl,1), ... repmat([1 -1],nl,1),n*m,n*m); end % +1,+1 L = find((nan_list(:,2) < n) & (nan_list(:,3) < m)); nl=length(L); if nl>0 fda=fda+sparse(repmat(nan_list(L,1),1,2), ... repmat(nan_list(L,1),1,2)+repmat([n+1, 0],nl,1), ... repmat([1 -1],nl,1),n*m,n*m); end % eliminate knowns rhs=-fda(:,known_list)*A(known_list); % and solve... B=A; k=nan_list(:,1); B(k)=fda(k,k)\rhs(k); end % all done, make sure that B is the same shape as % A was when we came in. B=reshape(B,n,m); % ==================================================== % end of main function % ==================================================== % ==================================================== % begin subfunctions % ==================================================== function neighbors_list=identify_neighbors(n,m,nan_list,talks_to) % identify_neighbors: identifies all the neighbors of % those nodes in nan_list, not including the nans % themselves % % arguments (input): % n,m - scalar - [n,m]=size(A), where A is the % array to be interpolated % nan_list - array - list of every nan element in A % nan_list(i,1) == linear index of i'th nan element % nan_list(i,2) == row index of i'th nan element % nan_list(i,3) == column index of i'th nan element % talks_to - px2 array - defines which nodes communicate % with each other, i.e., which nodes are neighbors. % % talks_to(i,1) - defines the offset in the row % dimension of a neighbor % talks_to(i,2) - defines the offset in the column % dimension of a neighbor % % For example, talks_to = [-1 0;0 -1;1 0;0 1] % means that each node talks only to its immediate % neighbors horizontally and vertically. % % arguments(output): % neighbors_list - array - list of all neighbors of % all the nodes in nan_list if ~isempty(nan_list) % use the definition of a neighbor in talks_to nan_count=size(nan_list,1); talk_count=size(talks_to,1); nn=zeros(nan_count*talk_count,2); j=[1,nan_count]; for i=1:talk_count nn(j(1):j(2),:)=nan_list(:,2:3) + ... repmat(talks_to(i,:),nan_count,1); j=j+nan_count; end % drop those nodes which fall outside the bounds of the % original array L = (nn(:,1)<1)|(nn(:,1)>n)|(nn(:,2)<1)|(nn(:,2)>m); nn(L,:)=[]; % form the same format 3 column array as nan_list neighbors_list=[sub2ind([n,m],nn(:,1),nn(:,2)),nn]; % delete replicates in the neighbors list neighbors_list=unique(neighbors_list,'rows'); % and delete those which are also in the list of NaNs. neighbors_list=setdiff(neighbors_list,nan_list,'rows'); else neighbors_list=[]; end \ No newline at end of file diff --git a/toolboxes/textprogressbar/demo_textprogressbar.m b/toolboxes/textprogressbar/demo_textprogressbar.m new file mode 100644 index 0000000..28893c9 --- /dev/null +++ b/toolboxes/textprogressbar/demo_textprogressbar.m @@ -0,0 +1,16 @@ +%demo_textprogressbar +%This a demo for textprogressbar script +textprogressbar('calculating outputs: '); +for i=1:100, + textprogressbar(i); + pause(0.1); +end +textprogressbar('done'); + + +textprogressbar('saving data: '); +for i=1:0.5:80, + textprogressbar(i); + pause(0.05); +end +textprogressbar('terminated'); \ No newline at end of file diff --git a/toolboxes/textprogressbar/license.txt b/toolboxes/textprogressbar/license.txt new file mode 100644 index 0000000..bb993f1 --- /dev/null +++ b/toolboxes/textprogressbar/license.txt @@ -0,0 +1,24 @@ +Copyright (c) 2010, Paul Proteus +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/toolboxes/textprogressbar/textprogressbar.m b/toolboxes/textprogressbar/textprogressbar.m new file mode 100644 index 0000000..9faa77c --- /dev/null +++ b/toolboxes/textprogressbar/textprogressbar.m @@ -0,0 +1,60 @@ +function textprogressbar(c) +% This function creates a text progress bar. It should be called with a +% STRING argument to initialize and terminate. Otherwise the number correspoding +% to progress in % should be supplied. +% INPUTS: C Either: Text string to initialize or terminate +% Percentage number to show progress +% OUTPUTS: N/A +% Example: Please refer to demo_textprogressbar.m + +% Author: Paul Proteus (e-mail: proteus.paul (at) yahoo (dot) com) +% Version: 1.0 +% Changes tracker: 29.06.2010 - First version + +% Inspired by: + +%% Initialization +persistent strCR; % Carriage return pesistent variable + +% Vizualization parameters +strPercentageLength = 10; % Length of percentage string (must be >5) +strDotsMaximum = 10; % The total number of dots in a progress bar + +%% Main + +if isempty(strCR) && ~ischar(c), + % Progress bar must be initialized with a string + error('The text progress must be initialized with a string'); +elseif isempty(strCR) && ischar(c), + % Progress bar - initialization + fprintf('%s',c); + strCR = -1; +elseif ~isempty(strCR) && ischar(c), + % Progress bar - termination + strCR = []; + fprintf([c '\n']); +elseif isnumeric(c) + % Progress bar - normal progress + c = floor(c); + percentageOut = [num2str(c) '%%']; + percentageOut = [percentageOut repmat(' ',1,strPercentageLength-length(percentageOut)-1)]; + nDots = floor(c/100*strDotsMaximum); + dotOut = ['[' repmat('.',1,nDots) repmat(' ',1,strDotsMaximum-nDots) ']']; + strOut = [percentageOut dotOut]; + + % Print it on the screen + if strCR == -1, + % Don't do carriage return during first run + fprintf(strOut); + else + % Do it during all the other runs + fprintf([strCR strOut]); + end + + % Update carriage return + strCR = repmat('\b',1,length(strOut)-1); + +else + % Any other unexpected input + error('Unsupported argument type'); +end diff --git a/watercolumn_general/CFF_get_WC_data.m b/watercolumn_general/CFF_get_WC_data.m new file mode 100644 index 0000000..438548c --- /dev/null +++ b/watercolumn_general/CFF_get_WC_data.m @@ -0,0 +1,181 @@ +function data_tot = CFF_get_WC_data(fData,fieldN,varargin) +%CFF_GET_WC_DATA Grab water column data in a fData structure +% +% Grab water column data in a fData structure, possibly subsampled in +% range or beams, or any pings required, in raw format or true value. +% +% *INPUT VARIABLES* +% * |fData|: Required. Structure for the storage of kongsberg EM series +% multibeam data in a format more convenient for processing. The data is +% recorded as fields coded "a_b_c" where "a" is a code indicating data +% origing, "b" is a code indicating data dimensions, and "c" is the data +% name. See the help of function CFF_convert_ALLdata_to_fData.m for +% description of codes. +% * |fieldN|: Required. Description (Information). +% * |iPing|: Optional. Description (Information). Default []. +% * |dr_sub|: Optional. Description (Information). Default 1. +% * |db_sub|: Optional. Description (Information). Default 1. +% * |output_format|: Optional. Description (Information). 'raw' or 'true' +% (default) + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 11-11-2021 + +% input parser +p = inputParser; +addRequired(p,'fieldN',@ischar); +addOptional(p,'iPing',[],@(x) isnumeric(x) ||isempty(x)); +addOptional(p,'dr_sub',1,@(x) isnumeric(x) && x>0); +addOptional(p,'db_sub',1,@(x) isnumeric(x) && x>0); +addOptional(p,'output_format','true',@(x) ischar(x) && ismember(x,{'raw' 'true'})); +addParameter(p,'iBeam',[],@(x) isnumeric(x) ||isempty(x)); +addParameter(p,'iRange',[],@(x) isnumeric(x) ||isempty(x)); +parse(p,fieldN,varargin{:}) +iPing = round(p.Results.iPing); +iBeam = p.Results.iBeam; +iRange = p.Results.iRange; +dr_sub = p.Results.dr_sub; +db_sub = p.Results.db_sub; +output_format = p.Results.output_format; + +if ~isfield(fData,fieldN) + data_tot = []; + return; +end + +datagramSource = CFF_get_datagramSource(fData); +if ~ismember(datagramSource,{'WC','AP'}) + data_tot = []; + return; +end + +if isempty(iPing) + iPing = 1:cellfun(@(x) nansum(size(x.Data.val,3)),fData.(fieldN)); +end +if isempty(iBeam) + iBeam = 1:cellfun(@(x) nanmax(size(x.Data.val,2)),fData.(fieldN)); +end +if isempty(iRange) + iRange = 1:cellfun(@(x) nanmax(size(x.Data.val,1)),fData.(fieldN)); +end + +% finding relevant groups of pings +p_start = fData.(sprintf('%s_n_start',datagramSource)); +p_end = fData.(sprintf('%s_n_end',datagramSource)); +pingCounter = fData.(sprintf('%s_1P_PingCounter',datagramSource)); +p_end(p_end>numel(pingCounter)) = numel(pingCounter); +p_start(p_start>numel(pingCounter)) = numel(pingCounter); + +% indices of first and last group of pings where requested data is found +istart = find(p_start<=nanmin(iPing),1,'last'); +iend = find(p_end>=nanmax(iPing),1,'first'); + +% init data output +data_tot = nan(ceil(numel(iRange)/dr_sub),ceil(numel(iBeam)/db_sub),numel(iPing),'single'); + +% init ??? +ip = 0; + +% debug graph +debug_disp = 0; +if debug_disp + f = figure(); + ax = axes(f); +end + +% read through each memmapped file +for ig = istart:iend + + % indices of data to grab, with decimation + iRange_src = iRange(1):dr_sub:min([iRange(end) size(fData.(fieldN){ig}.Data.val,1)]); + iBeam_src = iBeam(1):db_sub:min([iBeam(end) size(fData.(fieldN){ig}.Data.val,2)]); + + +% iPing_src = pingCounter(iPing); +% iPing_src_gr = intersect(iPing_src,ping_group_start(ig):ping_group_end(ig)); +% iPing_src = iPing_src_gr-ping_group_start(ig)+1; +% iPing_src(iPing_src>size(fData.(fieldN){ig}.Data.val,3)) = []; + + iPing_src = intersect(iPing,p_start(ig):p_end(ig)); + iPing_src = iPing_src - p_start(ig) + 1; + + + if isempty(iRange_src)||isempty(iBeam_src)||isempty(iPing_src) + data_tot = []; + continue; + end + + % grab data + data = fData.(fieldN){ig}.Data.val(iRange_src,iBeam_src,iPing_src); + + % transform to true values if required + switch output_format + + case 'true' + + % get info about data + idx_undsc = regexp(fieldN,'_'); + datagramSource = fieldN(1:idx_undsc(1)-1); + fieldname = fieldN(idx_undsc(2)+1:end); + + % get NaN value (should be a single value) + Nanval = fData.(sprintf('%s_1_%s_Nanval',datagramSource,fieldname)); + + % get factor (one per memmap file in the new format) + Fact = fData.(sprintf('%s_1_%s_Factor',datagramSource,fieldname)); + if numel(Fact)>1 + Fact = Fact(ig); + end + + % get offset (doesn't exist for older format, one per memmap + % file in the new format) + offset_fieldname = sprintf('%s_1_%s_Offset',datagramSource,fieldname); + if isfield(fData, offset_fieldname) + Offset = fData.(offset_fieldname); + else + Offset = 0; + end + + if numel(Offset)>1 + Offset = Offset(ig); + end + + if ~isa(data,'single') + + % first, convert to single class + data = single(data); + + % reset NaN value + data(data==single(Nanval)) = single(NaN); + + % decode data, minimizing calculation time + if Fact~=1 && Offset~=0 + data = data*Fact+Offset; + elseif Fact~=1 && Offset==0 + data = data*Fact; + elseif Fact==1 && Offset~=0 + data = data+Offset; + end + + end + + end + + if debug_disp + for ii = 1:size(data_tot,3) + imagesc(ax,squeeze(data(:,:,ii))); + drawnow; + end + end + + % add to full array + data_tot(1:size(data,1),1:(size(data,2)),ip+(1:(size(data,3)))) = data; + + % update ? + ip = ip + (size(data,3)); + +end + + + diff --git a/watercolumn_general/CFF_get_WC_size.m b/watercolumn_general/CFF_get_WC_size.m new file mode 100644 index 0000000..2fd1c9d --- /dev/null +++ b/watercolumn_general/CFF_get_WC_size.m @@ -0,0 +1,28 @@ +function [nSamples, nBeams, nPings] = CFF_get_WC_size(fData,varargin) +%CFF_GET_WC_SIZE One-line description +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 11-11-2021 + +% get source datagram +if ~isempty(varargin) && ~isempty(varargin{1}) + datagramSource = varargin{1}; +else + datagramSource = CFF_get_datagramSource(fData); +end + +% get data size +switch datagramSource + case {'WC' 'AP'} + fieldN = sprintf('%s_SBP_SampleAmplitudes',datagramSource); + [nSamples, nBeams, nPings] = cellfun(@(x) size(x.Data.val),fData.(fieldN)); + case 'X8' + nSamples = 1; + [nBeams, nPings] = size(fData.X8_BP_ReflectivityBS); +end + +nSamples = nanmax(nSamples); +nBeams = nanmax(nBeams); +nPings = nansum(nPings); \ No newline at end of file diff --git a/watercolumn_general/CFF_get_fData_wc_grid.m b/watercolumn_general/CFF_get_fData_wc_grid.m new file mode 100644 index 0000000..ea80ae1 --- /dev/null +++ b/watercolumn_general/CFF_get_fData_wc_grid.m @@ -0,0 +1,87 @@ +function data = CFF_get_fData_wc_grid(fData, field, d_lim_sonar_ref, d_lim_bottom_ref) +%CFF_GET_FDATA_WC_GRID Get gridded data from fData. +% +% If 3D grid, option possible to only take data within specified vertical +% bounds. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +if ~iscell(field) + field = {field}; +end + +data = cell(1,numel(field)); +L = cell(1,numel(field)); + +for ui = 1:numel(field) + L{ui} = fData.(['X_NEH_' field{ui}]); + if CFF_is_parallel_computing_available() + if ~isa(L{ui},'gpuArray') + % use GPU processing, turn arrays to gpuArrays + L{ui} = gpuArray(L{ui}); + end + else + if isa(L{ui},'gpuArray') + % not using GPU processing, turn gpuArrays to arrays + L{ui} = gather(L{ui}); + end + end +end + +% in case WC data is in 3D, caculate 2D views depending on display controls +if size(L{1},3)>1 + + switch fData.X_1_gridHeightReference + + case {'depth below sonar' 'Sonar'} + + d_max = 0; + d_min = nanmin(fData.X_BP_bottomHeight(:)); + + d_line_max = nanmin(d_lim_sonar_ref(2),d_max); + d_line_min = nanmax(d_lim_sonar_ref(1),d_min); + + if ~any(~isnan(d_lim_sonar_ref)) + return; + end + + idx_rem = (squeeze(fData.X_11H_gridHeight)+fData.X_1_gridVerticalResolution/2d_line_max); + + case {'height above bottom' 'Bottom'} + + d_max = nanmax(abs(nanmin(fData.X_BP_bottomHeight(:)))); + d_min = 0; + + if ~any(~isnan(d_lim_bottom_ref)) + return; + end + + d_line_max = nanmin(d_lim_bottom_ref(2),d_max); + d_line_min = nanmax(d_lim_bottom_ref(1),d_min); + + idx_rem = (squeeze(fData.X_11H_gridHeight)+fData.X_1_gridVerticalResolution/2d_line_max); + end + + for ui = 1:numel(field) + if ~all(idx_rem) + L{ui}(:,:,idx_rem) = NaN; + switch field{ui} + case 'gridLevel' + data{ui} = 20*log10(nanmean(10.^(L{ui}(:,:,:)/20),3)); + case 'gridDensity' + data{ui} = nansum(L{ui}(:,:,:),3); + case 'gridMaxHorizDist' + data{ui} = nanmax(L{ui}(:,:,:),[],3); + end + else + [~,id_keep] = nanmin(abs(squeeze(fData.X_11H_gridHeight)-d_line_min)); + data{ui} = (L{ui}(:,:,id_keep)); + end + end + +else + data = L; +end \ No newline at end of file diff --git a/watercolumn_general/CFF_init_memmapfiles.m b/watercolumn_general/CFF_init_memmapfiles.m new file mode 100644 index 0000000..ef01db1 --- /dev/null +++ b/watercolumn_general/CFF_init_memmapfiles.m @@ -0,0 +1,150 @@ +function fData = CFF_init_memmapfiles(fData,varargin) +%CFF_INIT_MEMMAPFILES Initializes data-containing memmap files +% +% Create one or several empty binary files of the right size to store ONE +% type of an upcoming large data, link it as a memmap file into a fData +% field, and add info as additional fData fields. +% This function is to be used with the proper varargin parameters +% to initialize the binary files (and link them to fData) with empty +% values, prior to reading the acoustic data and filling the binary +% files. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +%% input parser +p = inputParser; + +addParameter(p,'field','WC_SBP_SampleAmplitudes', @ischar); +addParameter(p,'wc_dir', CFF_converted_data_folder(fData.ALLfilename{1}), @ischar); +addParameter(p,'Class', 'int8', @ischar); +addParameter(p,'Factor', 1/2, @isnumeric); +addParameter(p,'Nanval', intmin('int8'), @isnumeric); +addParameter(p,'Offset', 0, @isnumeric); +addParameter(p,'MaxSamples', 1, @isnumeric); +addParameter(p,'MaxBeams', 1, @isnumeric); +addParameter(p,'ping_group_start', 1, @isnumeric); +addParameter(p,'ping_group_end', 1, @isnumeric); + +parse(p,varargin{:}); + +wc_dir = p.Results.wc_dir; +field = p.Results.field; +Class = p.Results.Class; +Factor = p.Results.Factor; +Nanval = p.Results.Nanval; +Offset = p.Results.Offset; +maxNSamples_groups = p.Results.MaxSamples; +maxNBeams_sub = p.Results.MaxBeams; +ping_group_start = p.Results.ping_group_start; +ping_group_end = p.Results.ping_group_end; + + +%% prep + +% create folder if it does not exist +if ~isfolder(wc_dir) + mkdir(wc_dir); +end + +% number of memmap files requested, one per group of pings +num_files = numel(ping_group_start); + +% if data field already exists, delete it +if isfield(fData,field) + for uig = 1:num_files + fData.(field){uig} = []; + end + fData = rmfield(fData,field) ; +end + +% number of bytes depending on class +switch Class + case {'int8' 'uint8'} + num_bytes = 1; + case {'int16' 'uint16'} + num_bytes = 2; + case {'int32' 'uint32'} + num_bytes = 4; + case {'int64' 'uint64'} + num_bytes = 8; + case {'single'} + num_bytes = 4; + case {'double'} + num_bytes = 8; +end + + +%% create empty binary files +for uig = 1:num_files + + % file name + file_binary = fullfile(wc_dir,sprintf('%s_%.0f_%.0f.dat',field,ping_group_start(uig),ping_group_end(uig))); + + % sizes + nSamples = maxNSamples_groups(uig); + nBeams = maxNBeams_sub; + nPings = (ping_group_end(uig)-ping_group_start(uig)+1); + + % create empty binary file if it does not exist + if ~isfile(file_binary) + + % create and open + fileID = fopen(file_binary,'w+'); + + % initialize file with zeros + total_num_elements = nSamples*nBeams*nPings; + num_skip_bytes = num_bytes*(total_num_elements-1); + fwrite(fileID,0,Class,num_skip_bytes); + + % close file + fclose(fileID); + + else + % ideally, if file already exists, we should test to see if it has + % the right "class [nSamples nBeams nPings]" before + % linking it after. To do this we would need to not have deleted + % the field before. + % + % If it has the right class and size, fill it with NaNs. + % + % If it's not the right class or size, we should delete it and + % recreate a correct one, but it might get tricky as we can't + % delete it if it's memmaped somewhere in the app. + % + % Until we do this, this part here will assume that class and size + % are the same, and that preexisting data won't be an issue. + % + % Anyway, something to think about on the day we encounter an issue + % here... XXX2 + end + + % memory map this binary file as a field in fData + fData.(field){uig} = memmapfile(file_binary,... + 'Format',{Class [nSamples nBeams nPings] 'val'},... + 'repeat',1,... + 'writable',true); + +end + + +%% record in fData the memmapfile info + +% add or overwrite info fields +p_field = strrep(field,'SBP','1'); +fData.(sprintf('%s_Class',p_field)) = Class; +fData.(sprintf('%s_Nanval',p_field)) = Nanval; +fData.(sprintf('%s_Factor',p_field)) = Factor; +fData.(sprintf('%s_Offset',p_field)) = Offset; +switch field(1) + case 'X' + prefix = field(1); + otherwise + prefix = field(1:2); +end +fData.(sprintf('%s_n_start',prefix)) = ping_group_start; +fData.(sprintf('%s_n_end',prefix)) = ping_group_end; +fData.(sprintf('%s_n_maxNSamples',prefix)) = maxNSamples_groups; + diff --git a/watercolumn_processing/CFF_WC_radiometric_corrections.m b/watercolumn_processing/CFF_WC_radiometric_corrections.m new file mode 100644 index 0000000..21ee100 --- /dev/null +++ b/watercolumn_processing/CFF_WC_radiometric_corrections.m @@ -0,0 +1,59 @@ +function [fData] = CFF_WC_radiometric_corrections(fData) +%CFF_WC_RADIOMETRIC_CORRECTIONS One-line description +% +% Apply physical (aka, not aestethic ones) corrections to the dB level in +% water-column data: TVG, dB offset, etc. +% +% *INPUT VARIABLES* +% * |fData|: Required. Structure for the storage of kongsberg EM series +% multibeam data in a format more convenient for processing. The data is +% recorded as fields coded "a_b_c" where "a" is a code indicating data +% origing, "b" is a code indicating data dimensions, and "c" is the data +% name. See the help of function CFF_convert_ALLdata_to_fData.m for +% description of codes. +% +% *OUTPUT VARIABLES* +% * |fData|: fData structure updated with "X_SBP_WaterColumnProcessed" +% now radiometrically corrected +% +% *DEVELOPMENT NOTES* +% Just started this function to integrate the "transmit power re maximum" +% dB offset that is stored in Runtime Parameters (marine mammal +% protection modes I think). But ideally develop this function for future +% compensations of TVG, pulse length, etc. + + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 24-09-2019 + +% extract info about WCD +wcdata_Class = fData.X_1_WaterColumnProcessed_Class; % int8 or int16 +wcdata_Factor = fData.X_1_WaterColumnProcessed_Factor; +wcdata_Nanval = fData.X_1_WaterColumnProcessed_Nanval; + +[nSamples, nBeams, nPings] = CFF_get_WC_size(fData); +% block processing setup +[blocks,info] = CFF_setup_optimized_block_processing(... + nPings,nSamples*nBeams*4,... + 'desiredMaxMemFracToUse',0.1); + +% block processing +for iB = 1:size(blocks,1) + + % list of pings in this block + blockPings = (blocks(iB,1):blocks(iB,2)); + + % grab data in dB + data = CFF_get_WC_data(fData,'X_SBP_WaterColumnProcessed','iPing',blockPings,'output_format','true'); + + % core processing + data = CFF_WC_radiometric_corrections_CORE(data,fData); + + % convert modified data back to raw format and store + data = data./wcdata_Factor; + data(isnan(data)) = wcdata_Nanval; + fData.X_SBP_WaterColumnProcessed.Data.val(:,:,blockPings) = cast(data,wcdata_Class); + +end + diff --git a/watercolumn_processing/CFF_WC_radiometric_corrections_CORE.m b/watercolumn_processing/CFF_WC_radiometric_corrections_CORE.m new file mode 100644 index 0000000..f79ad3f --- /dev/null +++ b/watercolumn_processing/CFF_WC_radiometric_corrections_CORE.m @@ -0,0 +1,143 @@ +function [data,params] = CFF_WC_radiometric_corrections_CORE(data, fData, iPings, varargin) +%CFF_WC_RADIOMETRIC_CORRECTIONS_CORE Apply radiometric correction to WCD +% +% This function radiometrically corrects specific pings of water-column +% data. +% +% DATA = CFF_WC_RADIOMETRIC_CORRECTIONS_CORE(DATA,FDATA,IPINGS) takes +% input DATA (SBP tensor) and radiometrically corrects it to the acoustic +% quantity Sv, using the necessary information in FDATA for the relevant +% ping indices IPINGS. It returns the corrected DATA. +% +% CFF_WC_RADIOMETRIC_CORRECTIONS_CORE(DATA,FDATA,IPINGS,PARAMS) uses +% processing parameters defined as the fields in the PARAMS structure. +% Possible parameters are: +% 'outVal': string for the acoustic quantity desired in output. Possible +% values are 'Sv' (Volume backscattering strength, in dB re 1 m-1, +% default), 'Sa' (Area backscattering strength, in dB re 1(m2 m-2)), and +% 'TS' (Target strength, in dB re 1 m2). For more information see, +% MacLennan et al. (2002) (DOI: 10.1006/jmsc.2001.1158). +% +% CFF_WC_RADIOMETRIC_CORRECTIONS_CORE(...,'comms',COMMS) specifies if and +% how this function communicates on its internal state (progress, info, +% errors). COMMS can be either a CFF_COMMS object, or a text string to +% initiate a new CFF_COMMS object. Options are 'disp', 'textprogressbar', +% 'waitbar', 'oneline', 'multilines'. By default, using an empty +% CFF_COMMS object (i.e. no communication). See CFF_COMMS for more +% information. +% +% [FDATA,PARAMS] = CFF_WC_RADIOMETRIC_CORRECTIONS_CORE(...) also outputs +% the parameters used in processing. +% +% Note: corrections incomplete XXX. +% +% See also CFF_WC_RADIOMETRIC_CORRECTIONS, +% CFF_FILTER_WC_SIDELOBE_ARTIFACT_CORE, CFF_MASK_WC_DATA_CORE. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 27-07-2022 + + +%% Input arguments management +p = inputParser; +addRequired(p,'data',@(x) isnumeric(x)); % data to process +addRequired(p,'fData',@(x) CFF_is_fData_version_current(x)); % source fData +addRequired(p,'iPings',@(x) isnumeric(x)); % indices of pings to process +addOptional(p,'params',struct(),@(x) isstruct(x)); % processing parameters +addParameter(p,'comms',CFF_Comms()); % information communication (none by default) +parse(p,data,fData,iPings,varargin{:}); +params = p.Results.params; +comms = p.Results.comms; +clear p +if ischar(comms) + comms = CFF_Comms(comms); +end + + +%% Prep + +% start message +comms.start('Applying radiometric corrections'); + + +%% Transmit Power level reduction +% This is the "mammal protection" setting, which is recorded in Runtime +% Parameters datagram +TPRM = fData.Ru_1D_TransmitPowerReMaximum; +if numel(unique(TPRM)) == 1 + % This value does not change in the file + TPRM = TPRM(1).*ones(size(data)); +else + % dB offset changed within the file. + % Would need to check when runtime parameters are being issued. Whether + % they are triggered with any change for example. Will likely need to + % extract and compare the time of Ru and WC datagrams to find which db + % offset applies to which pings. + % ... TO DO XXX1 + % for now we will just take the first value and apply to everything + % so that processing can continue... + comms.info('Transmit Power level reduction not constant within the file. Radiometric correction inappropriate'); + TPRM = TPRM(1).*ones(size(data)); +end + + +%% TVG applied in reception +% +% From Kongsberg datagrams manual: +% "The TVG function applied to the data is X logR + 2 Alpha R + OFS + C. +% The parameters X and C is documented in this datagram. OFS is gain offset +% to compensate for TX Source Level, Receiver sensitivity etc." +datagramSource = CFF_get_datagramSource(fData); + +X = fData.(sprintf('%s_1P_TVGFunctionApplied',datagramSource))(iPings); +C = fData.(sprintf('%s_1P_TVGOffset',datagramSource))(iPings); + +% Assuming 30log R if nothing has been specified +X(isnan(X)) = 30; +C(isnan(C)) = 0; + +% X is a parameter in TVG because it defines the output quantity (not +% taking into account constant factors) as follow: +% * For backscatter per unit volume (Sv): 20*log(R) +% * For backscatter per unit surface (Sa/BS): 30*log(R) +% * For target strength (TS): 40*log(R) +% +% Here we allow changing that output quantity by applying + Xcorr*log(R) to +% the data. For example if we want Sv, then the output has to be +% 30*log(R) = X*logR + Xcorr*logR, aka we use Xcorr = 30-X + +% get outVal parameter +if ~isfield(params,'outVal'), params.outVal = 'Sv'; end % default +mustBeMember(params.outVal,{'Sv','Sa','TS'}); % validate +outVal = params.outVal; + +% get Xcorr +switch outVal + case 'Sv' + Xcorr = 20-X; + case 'Sa' + Xcorr = 30-X; + case 'TS' + Xcorr = 40-X; +end +Xcorr = permute(Xcorr,[3,1,2]); + + +%% Full correction +% +% get sample range +nSamples = size(data,1); +interSamplesDistance = CFF_inter_sample_distance(fData,iPings); +datagramSource = fData.MET_datagramSource; +ranges = CFF_get_samples_range( (1:nSamples)', fData.(sprintf('%s_BP_StartRangeSampleNumber',datagramSource))(:,iPings), interSamplesDistance); + +% apply to data +data = data + Xcorr.*log10(ranges) + TPRM; + +% Still need to correct for C, but probably need to do all constant terms +% then. XXX1 + + +%% end message +comms.finish('Done'); diff --git a/watercolumn_processing/CFF_filter_WC_sidelobe_artifact.m b/watercolumn_processing/CFF_filter_WC_sidelobe_artifact.m new file mode 100644 index 0000000..ade9c98 --- /dev/null +++ b/watercolumn_processing/CFF_filter_WC_sidelobe_artifact.m @@ -0,0 +1,86 @@ +function [fData] = CFF_filter_WC_sidelobe_artifact(fData) +%CFF_FILTER_WC_SIDELOBE_ARTIFACT Filter the WC specular/sidelobe artifact +% +% *INPUT VARIABLES* +% * |fData|: Required. Structure for the storage of kongsberg EM series +% multibeam data in a format more convenient for processing. The data is +% recorded as fields coded "a_b_c" where "a" is a code indicating data +% origing, "b" is a code indicating data dimensions, and "c" is the data +% name. See the help of function CFF_convert_ALLdata_to_fData.m for +% description of codes. +% * |method_spec|: Optional/Parameters. Method for removal of specular +% reflection. Default: 2 +% +% *OUTPUT VARIABLES* +% * |fData|: fData structure updated with "X_SBP_WaterColumnProcessed" +% now filtered. +% +% *DEVELOPMENT NOTES* +% * IMPORTANT: only method 2 has been updated. All other methods don't +% work. to update XXX2. +% * dataset have three dimensions: ping #, beam # and sample #. +% Calculating the average backcatter level across samples, would allow +% us to spot the beams that have constantly higher or lower energy in a +% given ping. Doing this only for samples in the watercolumn would allow +% us to normalize the energy in the watercolumn of a ping. Calculating +% the average backcatter across all beams would allow us to spot the +% samples that have constantly higher or lower energy in a given ping. +% * the circular artifact on the bottom is due to specular reflection +% affecting all beams. +% -> remove in each ping by averaging the level at a given range across +% all beams. +% -> working on several pings at a time would work if the responsible +% reflectors are present on successive pings. They also need to stay at +% the same range so that would need some form of heave compensation. For +% heave compensation, maybe use the mean calculated on each ping and line +% up the highest return (specular). +% +% now when the specular artefacts are gone, what of the level being +% uneven across the swath in the water column? A higher level on outer +% beams that seems constant through pings? A higher level on closer +% ranges? +% -> Maybe calculate an average level across all pings for each beam and +% sample? +% -> Maybe such artefact is due to the difference in volume insonified +% that is not properly compensated.... +% -> Since the system is roll-compensated, a given beam correspond to +% different steering angles, hence different beamwidths. +% -> Average not for each beam, but for each steering angle. Sample +% should be fine. + + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +% extract info about WCD +wcdata_Class = fData.X_1_WaterColumnProcessed_Class; % int8 or int16 +wcdata_Factor = fData.X_1_WaterColumnProcessed_Factor; +wcdata_Nanval = fData.X_1_WaterColumnProcessed_Nanval; +[nSamples, nBeams, nPings] = CFF_get_WC_size(fData); + +% block processing setup +[blocks,info] = CFF_setup_optimized_block_processing(... + nPings,nSamples*nBeams*4,... + 'desiredMaxMemFracToUse',0.1); + +% block processing +for iB = 1:size(blocks,1) + + % list of pings in this block + blockPings = (blocks(iB,1):blocks(iB,2)); + + % grab data in dB + data = CFF_get_WC_data(fData,'X_SBP_WaterColumnProcessed','iPing',blockPings,'output_format','true'); + + % core processing + [data, correction] = CFF_filter_WC_sidelobe_artifact_CORE(data, fData, blockPings,[]); + + % convert result back to raw format and store + data = data./wcdata_Factor; + data(isnan(data)) = wcdata_Nanval; + fData.X_SBP_WaterColumnProcessed.Data.val(:,:,blockPings) = cast(data,wcdata_Class); + +end + + diff --git a/watercolumn_processing/CFF_filter_WC_sidelobe_artifact_CORE.m b/watercolumn_processing/CFF_filter_WC_sidelobe_artifact_CORE.m new file mode 100644 index 0000000..7b860e1 --- /dev/null +++ b/watercolumn_processing/CFF_filter_WC_sidelobe_artifact_CORE.m @@ -0,0 +1,260 @@ +function [data, params] = CFF_filter_WC_sidelobe_artifact_CORE(data, fData, iPings, varargin) +%CFF_FILTER_WC_SIDELOBE_ARTIFACT_CORE Filter WCD sidelobe artefact +% +% This function filters the sidelobe artefact in specific pings of +% water-column data. +% +% DATA = CFF_FILTER_WC_SIDELOBE_ARTIFACT_CORE(DATA,FDATA,IPINGS) takes +% input DATA (SBP tensor) and filters the sidelobe artefact in it, using +% the necessary information in FDATA for the relevant ping indices +% IPINGS. It returns the corrected DATA. +% +% CFF_FILTER_WC_SIDELOBE_ARTIFACT_CORE(DATA,FDATA,IPINGS,PARAMS) uses +% processing parameters defined as the fields in the PARAMS structure. +% Possible parameters are: +% 'avgCalc': mode of calculation of the average value across beams. Can +% be 'mean' (default) or 'median'. +% 'refType': type of calculation for reference level: 'constant' +% (constant value) or 'fromPingData' (calculated from the data, default). +% 'refCst': (only used if refType is 'constant') set the constant value +% here (in dB). Default is -70. +% 'refArea': (only used if refType is 'fromPingData') set the reference +% area for the calculation of the reference level here: 'nadirWC' uses +% data from the eleven middle beams before minimum slant range, or +% 'cleanWC' uses all data before minimum slant range (default). +% 'refCalc': (only used if refType is 'fromPingData')) set mode of +% calculation of the reference value from the reference data here. +% Possible values are 'mean', 'median', 'mode', 'perc5', 'perc10', +% 'perc25' (default). +% +% CFF_FILTER_WC_SIDELOBE_ARTIFACT_CORE(...,'comms',COMMS) specifies if +% and how this function communicates on its internal state (progress, +% info, errors). COMMS can be either a CFF_COMMS object, or a text string +% to initiate a new CFF_COMMS object. Options are 'disp', +% 'textprogressbar', 'waitbar', 'oneline', 'multilines'. By default, +% using an empty CFF_COMMS object (i.e. no communication). See CFF_COMMS +% for more information. +% +% [FDATA,PARAMS] = CFF_FILTER_WC_SIDELOBE_ARTIFACT_CORE(...) also outputs +% the parameters used in processing. +% +% Note: development notes at the bottom +% +% See also CFF_FILTER_WC_SIDELOBE_ARTIFACT, +% CFF_WC_RADIOMETRIC_CORRECTIONS_CORE, CFF_MASK_WC_DATA_CORE. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 28-07-2022 + + +global DEBUG; + + +%% Input arguments management +p = inputParser; +addRequired(p,'data',@(x) isnumeric(x)); % data to process +addRequired(p,'fData',@(x) CFF_is_fData_version_current(x)); % source fData +addRequired(p,'iPings',@(x) isnumeric(x)); % indices of pings to process +addOptional(p,'params',struct(),@(x) isstruct(x)); % processing parameters +addParameter(p,'comms',CFF_Comms()); % information communication (none by default) +parse(p,data,fData,iPings,varargin{:}); +params = p.Results.params; +comms = p.Results.comms; +clear p +if ischar(comms) + comms = CFF_Comms(comms); +end + + +%% Prep + +% start message +comms.start('Filtering sidelobe artefact'); + + +%% Calculate average value across beams + +% get avgCalc parameter +if ~isfield(params,'avgCalc'), params.avgCalc = 'mean'; end % default +mustBeMember(params.avgCalc,{'mean','median'}); % validate +avgCalc = params.avgCalc; + +% calculation +switch avgCalc + case 'mean' + avgAcrossBeams = mean(data,2,'omitnan'); + case 'median' + avgAcrossBeams = median(data,2,'omitnan'); +end + + +%% Calculate reference level + +% get refType parameter +if ~isfield(params,'refType'), params.refType = 'fromPingData'; end % default +mustBeMember(params.refType,{'fromPingData','constant'}); % validate +refType = params.refType; + +% calculations +switch refType + + case 'constant' + % constant reference level, as per parameter + + % get refCst parameter + if ~isfield(params,'refCst'), params.refCst = -70; end % default + mustBeNumeric(params.refCst); % validate + refCst = params.refCst; + + % calculate reference level + refLevel = single(refCst.*ones(1,1,numel(iPings))); + + case 'fromPingData' + % reference level calculated from ping data, as per parameters + + % get closest bottom sample (minimum slant range) in each ping + [num_samples, ~, ~] = size(data); + bottomSamples = CFF_get_bottom_sample(fData); + bottomSamples = bottomSamples(:,iPings); + closestBottomSample = nanmin(bottomSamples); + closestBottomSample = nanmin(ceil(closestBottomSample),num_samples); + + % indices for data extraction (getting rid of surface noise) + iSampleStart = ceil(nanmin(closestBottomSample)/10); + iSampleEnd = ceil(nanmax(closestBottomSample)); + + % get refArea parameter + if ~isfield(params,'refArea'), params.refArea = 'cleanWC'; end % default + mustBeMember(params.refArea,{'nadirWC','cleanWC'}); % validate + refArea = params.refArea; + + % defining reference data + switch refArea + + case 'nadirWC' + % using an average noise level from all samples in the + % water column of this ping, above the bottom, within the + % 11 beams closest to nadir. + [~, nBeams, ~] = size(data); + nadirBeams = (floor((nBeams./2)-5):ceil((nBeams./2)+5)); + refData = data(iSampleStart:iSampleEnd,nadirBeams,:); + + case 'cleanWC' + % using an average noise level from all samples in the + % water column of this ping, within minimum slant range, + % aka "clean watercolumn" + refData = data(iSampleStart:iSampleEnd,:,:); + + end + + % nan all samples beyond minimum slant range in the extracted data + iNan = iSampleStart-1+(1:size(refData,1))' >= closestBottomSample; + iNan = permute(iNan,[1 3 2]); + iNan = repmat(iNan,1,size(refData,2),1); + refData(iNan) = NaN; + + % get refCalc parameter + if ~isfield(params,'refCalc'), params.refCalc = 'perc25'; end % default + mustBeMember(params.refCalc,{'mean','median','mode','perc5','perc10','perc25'}); % validate + refCalc = params.refCalc; + + % calculate reference level + switch refCalc + case 'mean' + refLevel = nanmean(refData,[1 2]); + case 'median' + refLevel = nanmedian(refData,[1 2]); + case 'mode' + refLevel = mode(refData,[1 2]); + case 'perc5' + refLevel = prctile(refData,5,[1 2]); + case 'perc10' + refLevel = prctile(refData,10,[1 2]); + case 'perc25' + refLevel = prctile(refData,25,[1 2]); + end + +end + +% display results +if DEBUG + ip = 1; + figure; tiledlayout(1,3); + ax1=nexttile();imagesc(data(:,:,ip)); + grid on;colormap(jet);colorbar;title('raw');c=caxis; + ax2=nexttile();imagesc(data(:,:,ip)-avgAcrossBeams(:,:,ip)); + grid on;colormap(jet);colorbar;title('... -avg'); + ax3=nexttile();imagesc(data(:,:,ip)-avgAcrossBeams(:,:,ip)+refLevel(:,:,ip)); + grid on;colormap(jet);colorbar;title('... +ref');caxis(c); + linkaxes([ax1,ax2,ax3]); +end + +%% Remove average and add reference level +data = data - avgAcrossBeams + refLevel; + + +%% end message +comms.finish('Done'); + + +%% DVPT NOTES +% I originally developed several methods to filter the sidelobe artefact. +% The overall principle is normalization. Just as for seafloor backscatter +% you normalize the level across all angles by removing the average level +% computed across all angles, here with water-column, you normalize the +% level by removing the average level computed across all ranges. +% +% There are several levels of complexity possible. +% +% At the most basic, you really only need to remove the average. The +% resulting data has an average of 0, which is not the normal dB range. +% This is what I did for seafloor backscatter in my first paper. In my +% original code, this was method 1. +% +% So the next level of complexity is to reintroduce a reference level after +% removing the mean. This is the most common procedure, the one retained in +% the code here (formerly known as method 2), and the one termed +% correction "a" in Parnum's thesis. +% +% Now, usually a "normalization" implies also the standard deviation: you +% remove the mean, then divide by the standard deviation. If you want to +% add a new mean (reference level), you do it after those two first steps. +% This is correction "b" in Parnum. Tou'd need to calculate the std as +% stdAcrossBeams = std(data,0,2,'omitnan'); +% and in the final calculation do instead: +% data = (data-avgAcrossBeams)./stdAcrossBeams + refLevel; +% +% Continuing further from Parnum's idea, you could reintroduce a reference +% standard deviation, just as the reference level is actually a reference +% mean. So, in order, you substract the mean, divide by the std, multiply +% by the reference std, and add the reference level. +% The reference std would be calculated in the same loop as that for +% reference level as: +% refStd = nanstd(refData,[1 2]); +% and reintroduced in the final calculation as: +% data = (((data-avgAcrossBeams)./stdAcrossBeams)+refLevel).*refStd; +% +% Another note worth thinking about: Why normalizing only across ranges? +% What about the other dimensions? Normalizing across samples would be +% calculated as: +% avgAcrossSamples = mean(data,1,'omitnan'); +% Across pings as: +% avgAcrossPings = mean(data,3,'omitnan'); +% What about across more than one dimension? Is that possible? +% +% Last note: De Moustier came across a similar solution to filter sidelobe +% artefacts except it consisted in calculating the 75% percentiles across +% all ranges, rather than the mean. Also he did not introduce a reference +% level. It would go as something like this: +% [nSamples, ~, ~] = size(fData.X_SBP_WaterColumnProcessed.Data.val); +% for ip = 1:numel(iPings) +% thisPing = data(:,:,ip); +% sevenfiveperc = nan(nSamples,1); % calc 75th percentile across ranges +% for ismp = 1:nSamples +% X = thisPing(ismp,:,:); +% sevenfiveperc(ismp,1) = CFF_invpercentile(X,75); +% end +% thisPing_corrected = thisPing - sevenfiveperc; +% end + diff --git a/watercolumn_processing/CFF_grid_WC_data.m b/watercolumn_processing/CFF_grid_WC_data.m new file mode 100644 index 0000000..ebce4be --- /dev/null +++ b/watercolumn_processing/CFF_grid_WC_data.m @@ -0,0 +1,579 @@ +function fData = CFF_grid_WC_data(fData,varargin) +%CFF_GRID_WC_DATA One-line description +% +% *INPUT VARIABLES* +% * |fData|: Required. Structure for the storage of kongsberg EM series +% multibeam data in a format more convenient for processing. The data is +% recorded as fields coded "a_b_c" where "a" is a code indicating data +% origing, "b" is a code indicating data dimensions, and "c" is the data +% name. See the help of function CFF_convert_ALLdata_to_fData.m for +% description of codes. +% * |grid_horz_res|: Description (Information). Default: 1 +% * |grid_vert_res|: Description (Information). Default: 1 +% * |grid_type|: Description (Information). '2D' or '3D' (default) +% * |dr_sub|: Description (Information). Default: 4 +% * |db_sub|: Description (Information). Default: 2 +% * |e_lim|: Description (Information). Default: [] +% * |n_lim|: Description (Information). Default: [] +% +% *OUTPUT VARIABLES* +% * |fData|: fData structure updated with fields for gridded data +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +%% input parsing + +% init +p = inputParser; + +% grid mode (2D map or 3D) and resolution (horizontal and vertical) +addParameter(p,'grid_type','3D',@(x) ismember(x,{'2D' '3D'})); +addParameter(p,'grid_horz_res',1,@(x) isnumeric(x)&&x>0); +addParameter(p,'grid_vert_res',1,@(x) isnumeric(x)&&x>0); + +% decimation factors +addParameter(p,'dr_sub',4,@(x) isnumeric(x)&&x>0); +addParameter(p,'db_sub',2,@(x) isnumeric(x)&&x>0); + +% grid limitation parameters +addParameter(p,'grdlim_mode','between',@(x) ismember(x,{'between', 'outside of'})); +addParameter(p,'grdlim_var','Sonar',@(x) ismember(x,{'Sonar', 'Bottom'})); +addParameter(p,'data_type','Processed',@(x) ismember(x,{'Processed', 'Original'})); +addParameter(p,'grdlim_mindist',0,@isnumeric); +addParameter(p,'grdlim_maxdist',inf,@isnumeric); +addParameter(p,'grdlim_east',[],@isnumeric); +addParameter(p,'grdlim_north',[],@isnumeric); + +% parse +parse(p,varargin{:}) + +% get results +grid_type = p.Results.grid_type; +grid_horz_res = p.Results.grid_horz_res; +grid_vert_res = p.Results.grid_vert_res; +dr_sub = p.Results.dr_sub; +db_sub = p.Results.db_sub; +grdlim_mode = p.Results.grdlim_mode; +grdlim_var = p.Results.grdlim_var; +grdlim_mindist = p.Results.grdlim_mindist; +grdlim_maxdist = p.Results.grdlim_maxdist; +grdlim_east = p.Results.grdlim_east; +grdlim_north = p.Results.grdlim_north; +data_type = p.Results.data_type; + + +%% Extract info about WCD +switch data_type + case 'Original' + if isfield(fData,'WC_SBP_SampleAmplitudes') + field_to_grid = 'WC_SBP_SampleAmplitudes'; + else + field_to_grid = 'AP_SBP_SampleAmplitudes'; + end + otherwise + if isfield(fData,'X_SBP_WaterColumnProcessed') + field_to_grid = 'X_SBP_WaterColumnProcessed'; + elseif isfield(fData,'WC_SBP_SampleAmplitudes') + field_to_grid = 'WC_SBP_SampleAmplitudes'; + else + field_to_grid = 'AP_SBP_SampleAmplitudes'; + end +end +% size +[nSamples, nBeams, nPings] = CFF_get_WC_size(fData); + +%% Prepare needed 1xP data for computations + +% Source datagram +datagramSource = CFF_get_datagramSource(fData); + +% inter-sample distance +interSamplesDistance = CFF_inter_sample_distance(fData); % m + +% sonar location +sonarEasting = fData.X_1P_pingE; % m +sonarNorthing = fData.X_1P_pingN; % m +sonarHeight = fData.X_1P_pingH; % m + +% sonar heading +gridConvergence = fData.X_1P_pingGridConv; % deg +vesselHeading = fData.X_1P_pingHeading; % deg +sonarHeadingOffset = fData.IP_ASCIIparameters.S1H; % deg +sonarHeading = deg2rad(-mod(gridConvergence + vesselHeading + sonarHeadingOffset,360)); % rad + +switch grdlim_var + case 'Bottom' + %% Prepare the Height interpolant + % needed to calculate height above seafloor for each sample in case of + % gridding in height above bottom, as well as final gridded bathymetry. + idx_valid_bottom = false(size(fData.X_BP_bottomHeight)); + idx_valid_bottom(1:db_sub:end,:) =true; + idx_valid_bottom = idx_valid_bottom &(~isnan(fData.X_BP_bottomHeight) & ~isinf(fData.X_BP_bottomHeight)); + HeightInterpolant = scatteredInterpolant(fData.X_BP_bottomNorthing(idx_valid_bottom),fData.X_BP_bottomEasting(idx_valid_bottom),fData.X_BP_bottomHeight(idx_valid_bottom),'linear','none'); +end + + +%% GPU setup +if CFF_is_parallel_computing_available() + useGpu = 1; + processingUnit = 'GPU'; +else + useGpu = 0; + processingUnit = 'CPU'; +end + +%% Block processing setup +[blocks,info] = CFF_setup_optimized_block_processing(... + nPings,nSamples*nBeams*4,... + processingUnit,... + 'desiredMaxMemFracToUse',0.2); +nBlocks = size(blocks,1); +%disp(info); + +%% find grid limits + +% initialize vectors +minBlockE = nan(1,nBlocks); +minBlockN = nan(1,nBlocks); +minBlockH = nan(1,nBlocks); +maxBlockE = nan(1,nBlocks); +maxBlockN = nan(1,nBlocks); +maxBlockH = nan(1,nBlocks); + +% find grid limits for each block +for iB = 1:nBlocks + + % list of pings in this block + blockPings = blocks(iB,1):blocks(iB,2); + + % to define the limits of the grid for each block, we'll only consider + % the easting and northing of the first and last sample for the central + % beam and two outer beams, for all pings. + idxSamples = [1 nSamples]'; + startSampleNumber = fData.(sprintf('%s_BP_StartRangeSampleNumber',datagramSource))([1 round(nBeams./2) nBeams],blockPings); + beamPointingAngle = deg2rad(fData.(sprintf('%s_BP_BeamPointingAngle',datagramSource))([1 round(nBeams./2) nBeams],blockPings)); + + % Get easting, northing and height + [blockE, blockN, blockH] = CFF_georeference_sample(idxSamples, startSampleNumber, interSamplesDistance(blockPings), beamPointingAngle, ... + sonarEasting(blockPings), sonarNorthing(blockPings), sonarHeight(blockPings), sonarHeading(blockPings)); + + % these subset of all samples should be enough to find the bounds for + % the entire block + minBlockE(iB) = min(blockE(:)); + maxBlockE(iB) = max(blockE(:)); + minBlockN(iB) = min(blockN(:)); + maxBlockN(iB) = max(blockN(:)); + minBlockH(iB) = min(blockH(:)); + maxBlockH(iB) = max(blockH(:)); + +end + +% Get grid boundaries from the min and max of those blocks +minGridE = floor(min(minBlockE)); +maxGridE = ceil(max(maxBlockE)); +numElemGridE = ceil((maxGridE-minGridE)./grid_horz_res)+1; +minGridN = floor(min(minBlockN)); +maxGridN = ceil(max(maxBlockN)); +numElemGridN = ceil((maxGridN-minGridN)./grid_horz_res)+1; +minGridH = floor(min(minBlockH)); +maxGridH = ceil(max(maxBlockH)); +numElemGridH = ceil((maxGridH-minGridH)./grid_vert_res)+1; + +% the grid defined above in height is referenced to sonar height. It needs +% modified if we request heights reference to seafloor +switch grdlim_var + case 'Bottom' + lowestHeight = nanmin(fData.X_BP_bottomHeight(:)); + % by default, defining grid as extending from 0 (the bottom) to the + % largest height, adding a buffer of a tenth of that largest height + % on each side + minGridH = floor( 0 - abs(lowestHeight./10) ); + maxGridH = ceil( abs(lowestHeight) + abs(lowestHeight./10) ); + numElemGridH = ceil((maxGridH-minGridH)./grid_vert_res)+1; +end + + +%% initalize the grids: +% * weighted sum and total sum of weights. In absence of weights, the total +% sum of weights is simply the count of points, and the weighted sum is +% simply the sum +% * Now also doing a grid containing the maximum horizontal distance to +% nadir, to be used when mosaicking using the "stitching" method. +switch grid_type + case '2D' + gridWeightedSum = zeros(numElemGridN,numElemGridE,'single'); + gridTotalWeight = zeros(numElemGridN,numElemGridE,'single'); + gridMaxHorizDist = nan(numElemGridN,numElemGridE,'single'); + case '3D' + gridWeightedSum = zeros(numElemGridN,numElemGridE,numElemGridH,'single'); + gridTotalWeight = zeros(numElemGridN,numElemGridE,numElemGridH,'single'); + gridMaxHorizDist = nan(numElemGridN,numElemGridE,numElemGridH,'single'); +end + +if useGpu + % turn grids into gpuArrays + gridWeightedSum = gpuArray(gridWeightedSum); + gridTotalWeight = gpuArray(gridTotalWeight); + gridMaxHorizDist = gpuArray(gridMaxHorizDist); +end + + +%% fill the grids with block processing +for iB = 1:nBlocks + + % list of pings in this block + blockPings = blocks(iB,1):blocks(iB,2); + + % get data to grid (possibly decimated in beams and samples + blockL = CFF_get_WC_data(fData,field_to_grid,'iPing',blockPings,'dr_sub',dr_sub,'db_sub',db_sub,'output_format','true'); + + % retrieve the index of samples given the decimation + nSamples_temp = size(blockL,1); + idxSamples = (1:dr_sub:nSamples_temp*dr_sub)'; + + startSampleNumber = fData.(sprintf('%s_BP_StartRangeSampleNumber',datagramSource))(1:db_sub:end,blockPings); + beamPointingAngle = deg2rad(fData.(sprintf('%s_BP_BeamPointingAngle',datagramSource))(1:db_sub:end,blockPings)); + + % get easting, northing, height and across distance from the samples + [blockE, blockN, blockH, blockAccD] = CFF_georeference_sample(idxSamples, startSampleNumber, interSamplesDistance(blockPings), beamPointingAngle, ... + sonarEasting(blockPings), sonarNorthing(blockPings), sonarHeight(blockPings), sonarHeading(blockPings)); + + % blockE and blockN needs to stay double for the interpolant, but + % blockAdd needs to be single for the following calculations + blockAccD = single(blockAccD); + + % if needed, turn height above sonar into height above bottom, which + % depends on height of sonar above bottom. We use the interpolant here, + % which takes a while.... + switch grdlim_var + case 'Bottom' + + %block_bottomHeight = HeightInterpolant(blockN,blockE); + block_bottom_H=permute(fData.X_BP_bottomHeight(1:db_sub:end,blockPings),[3 1 2]); + block_bottomHeight = nan(size(blockH)); + id_interp = (blockH>block_bottom_H);%reducing the amount of data we need to compute heigth above bottom for + + block_bottomHeight(id_interp) = HeightInterpolant(blockN(id_interp),blockE(id_interp)); + + blockH = blockH - block_bottomHeight; + blockH(blockH<0) = NaN; + clear block_bottomHeight + end + + % define weights here + weighting_mode = 'none'; % 'SIAfilter' or 'none' + switch weighting_mode + case 'none' + % all samples have the same weight (1) in the gridding + blockW = ones(size(blockL),class(blockL)); + case 'SIAfilter' + % samples have a weight dependent on how strongly they were + % affected by the SIA filter. The ranges where specular occured + % had a strong mean BS to which we will apply a small weight + % (towards 0), while the ranges that had no speculars had a + % relatively lower mean BS, and those we will apply a larger + % weight (towards 1). + % This correction requires to have saved the sidelobe artifact + % correction at the processing stage. + if isfield(fData, 'X_S1P_sidelobeArtifactCorrection') + + SIAcorrection = fData.X_S1P_sidelobeArtifactCorrection(1:dr_sub:end,1,blockPings); + + % define downramp function, going down linearly from + % (X1,Y1) to (X2,Y2). Equals Y1 for xX2. + downramp_fun = @(x,X1,X2,Y1,Y2) min(max(x.*(Y1-Y2)./(X1-X2)+(Y2.*X1-Y1.*X2)./(X1-X2),Y2),Y1); + + % use inverse percentiles to figure the start and end of + % the ramp. + start_ramp = CFF_invpercentile(SIAcorrection(:),20); + end_ramp = CFF_invpercentile(SIAcorrection(:),80); + + % apply to SIAcorrection to get the weight scores (between + % 0 and 1) + fact = downramp_fun(SIAcorrection,start_ramp,end_ramp,1,0); + blockW = single(repmat(fact,1,size(blockL,2),1)); + else + warning('This weighting mode cannot be used because processed data do not include the needed field. Using no-weighting mode for now.'); + blockW = ones(size(blockL),class(blockL)); + end + end + + % start with removing all data where anything is missing + indNan = isnan(blockL) | isnan(blockW) | isnan(blockE) | isnan(blockN) | isnan(blockH) | isnan(blockAccD); + blockL(indNan) = []; + if isempty(blockL) + continue; + end + blockW(indNan) = []; + blockE(indNan) = []; + blockN(indNan) = []; + blockH(indNan) = []; + blockAccD(indNan) = []; + clear indNan + + % next, in case of 2D gridding, removing samples outside of desired + % layer + switch grid_type + case '2D' + switch grdlim_var + case 'Sonar' + % heights are referenced to sonar and min/max distances + % are in depth (below sonar) + switch grdlim_mode + case 'between' + idx_keep = blockH<=-grdlim_mindist & blockH>=-grdlim_maxdist; + case 'outside of' + idx_keep = blockH>=-grdlim_mindist | blockH<=-grdlim_maxdist; + end + case 'Bottom' + % heights are referenced to bottom and min/max distances + % are in height (above bottom) + switch grdlim_mode + case 'between' + idx_keep = blockH>=grdlim_mindist & blockH<=grdlim_maxdist; + case 'outside of' + idx_keep = blockH<=grdlim_mindist | blockH>=grdlim_maxdist; + end + end + otherwise + idx_keep = true(size(blockH)); + end + + %Only keep samples within the allocated grid (potential issue in bottom reference grids otherwise) + idx_keep = idx_keep &... + (blockE-minGridE)>=0 & (blockE-maxGridE)<=0 & (blockN-minGridN)>=0 & (blockN-maxGridN)<=0 & (blockH-minGridH)>=0 & (blockH-maxGridH)<=0; + + blockL(~idx_keep) = []; + if isempty(blockL) + continue; + end + blockW(~idx_keep) = []; + blockE(~idx_keep) = []; + blockN(~idx_keep) = []; + blockH(~idx_keep) = []; + blockAccD(~idx_keep) = []; + clear idx_keep + + % at this stage, pass blockL and blockW as GPU arrays if using GPUs + if isa(gridWeightedSum,'gpuArray') + blockL = gpuArray(blockL); + blockW = gpuArray(blockW); + blockAccD = gpuArray(blockAccD); + end + + % pass grid level in natural before gridding + blockL = 10.^(blockL./10); + + % also, turn across distance (signed) to horizontal distance from nadir + % (unsigned) + blockD = abs(blockAccD); + clear blockAccD + + % data indices in the full grid + E_idx = round((blockE-minGridE)/grid_horz_res+1); + N_idx = round((blockN-minGridN)/grid_horz_res+1); + clear blockE blockN + + % first index + idx_E_start = min(E_idx); + idx_N_start = min(N_idx); + + % data indices in the small grid built just for this block of pings + E_idx = E_idx - min(E_idx) + 1; + N_idx = N_idx - min(N_idx) + 1; + + % size of this small grid + N_E = max(E_idx); + N_N = max(N_idx); + + % now gridding... + switch grid_type + + case '2D' + + % no need for blockH any more + clear blockH + + % we use the accumarray function to sum all values in both the + % total weight grid, and the weighted sum grid. Prepare the + % common values here + subs = single([N_idx' E_idx']); % indices in the temp grid of each data point + sz = single([N_N N_E]); % size of ouptut + clear N_idx E_idx + + % sum of weights per grid cell, and sum of weighted levels per + % grid cell + gridTotalWeight_forBlock = accumarray(subs,blockW',sz,@sum,single(0)); + gridWeightedSum_forBlock = accumarray(subs,blockW'.*blockL',sz,@sum,single(0)); + + clear blockL blockW + + % maximum horiz distance from nadir per grid cell + gridMaxHorizDist_forBlock = accumarray(subs,blockD',sz,@max,single(NaN)); + clear blockD subs + + % Add the block's small grid of weights sum to the full one, + % and the block's small grid of sum of weighted levels to the + % full one + gridTotalWeight(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1) = ... + gridTotalWeight(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1) + gridTotalWeight_forBlock; + gridWeightedSum(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1) = ... + gridWeightedSum(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1) + gridWeightedSum_forBlock; + clear gridTotalWeight_forBlock gridWeightedSum_forBlock + + % Add the block's small grid of maximum horiz dist to the full + % one: just keep in the grid whatever the maximum is per cell + gridMaxHorizDist(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1) = nanmax( ... + gridMaxHorizDist(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1), gridMaxHorizDist_forBlock); + clear gridMaxHorizDist_forBlock + + case '3D' + + % prepare indices as we did before in E,N, this time in height + H_idx = round((blockH-minGridH)/grid_vert_res+1); + clear blockH + idx_H_start = min(H_idx); + H_idx = H_idx - min(H_idx) + 1; + N_H = max(H_idx); + + % we use the accumarray function to sum all values in both the + % total weight grid, and the weighted sum grid. Prepare the + % common values here + subs = single([N_idx' E_idx' H_idx']); % indices in the temp grid of each data point + sz = single([N_N N_E N_H]); % size of ouptut + clear N_idx E_idx H_idx + + % sum of weights per grid cell, and sum of weighted levels per grid cell + gridTotalWeight_forBlock = accumarray(subs,blockW',sz,@sum,single(0)); + gridWeightedSum_forBlock = accumarray(subs,blockW'.*blockL',sz,@sum,single(0)); + clear blockL blockW + + % maximum horiz distance from nadir per grid cell + gridMaxHorizDist_forBlock = accumarray(subs,blockD',sz,@max,single(NaN)); + clear blockD subs + + % Add the block's small grid of weights sum to the full one, + % and the block's small grid of sum of weighted levels to the + % full one + gridTotalWeight(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1,idx_H_start:idx_H_start+N_H-1) = ... + gridTotalWeight(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1,idx_H_start:idx_H_start+N_H-1) + gridTotalWeight_forBlock; + + gridWeightedSum(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1,idx_H_start:idx_H_start+N_H-1) = ... + gridWeightedSum(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1,idx_H_start:idx_H_start+N_H-1) + gridWeightedSum_forBlock; + clear gridTotalWeight_forBlock gridWeightedSum_forBlock + + % Add the block's small grid of maximum horiz dist to the full + % one: just keep in the grid whatever the maximum is per cell + gridMaxHorizDist(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1,idx_H_start:idx_H_start+N_H-1) = nanmax( ... + gridMaxHorizDist(idx_N_start:idx_N_start+N_N-1,idx_E_start:idx_E_start+N_E-1,idx_H_start:idx_H_start+N_H-1), gridMaxHorizDist_forBlock); + clear gridMaxHorizDist_forBlock + + end + +end + + +%% crop the edges of the grids +% they were built based on the size of the full dataset and so may be much +% bigger than actual data contained +switch grid_type + + case '2D' + + % dimensional sums + sumgridTotalWeight_N = sum(gridTotalWeight,2); + sumgridTotalWeight_E = sum(gridTotalWeight,1); + + % min and max indices for cropping + minNidx = find(sumgridTotalWeight_N,1,'first'); + maxNidx = find(sumgridTotalWeight_N,1,'last'); + minEidx = find(sumgridTotalWeight_E,1,'first'); + maxEidx = find(sumgridTotalWeight_E,1,'last'); + + % crop the grids + gridTotalWeight = gridTotalWeight(minNidx:maxNidx,minEidx:maxEidx); + gridWeightedSum = gridWeightedSum(minNidx:maxNidx,minEidx:maxEidx); + gridMaxHorizDist = gridMaxHorizDist(minNidx:maxNidx,minEidx:maxEidx); + + % define and crop dim vectors + gridNorthing = (0:numElemGridN-1)'.*grid_horz_res + minGridN; + gridEasting = (0:numElemGridE-1) .*grid_horz_res + minGridE; + gridNorthing = gridNorthing(minNidx:maxNidx); + gridEasting = gridEasting(:,minEidx:maxEidx); + + case '3D' + + % dimensional sums + sumgridTotalWeight_1EH = sum(gridTotalWeight,1); + sumgridTotalWeight_N1H = sum(gridTotalWeight,2); + sumgridTotalWeight_N = sum(sumgridTotalWeight_N1H,3); + sumgridTotalWeight_E = sum(sumgridTotalWeight_1EH,3); + sumgridTotalWeight_H = sum(sumgridTotalWeight_1EH,2); + + % min and max indices for cropping + minNidx = find(sumgridTotalWeight_N,1,'first'); + maxNidx = find(sumgridTotalWeight_N,1,'last'); + minEidx = find(sumgridTotalWeight_E,1,'first'); + maxEidx = find(sumgridTotalWeight_E,1,'last'); + minHidx = find(sumgridTotalWeight_H,1,'first'); + maxHidx = find(sumgridTotalWeight_H,1,'last'); + + % crop the grids + gridTotalWeight = gridTotalWeight(minNidx:maxNidx,minEidx:maxEidx,minHidx:maxHidx); + gridWeightedSum = gridWeightedSum(minNidx:maxNidx,minEidx:maxEidx,minHidx:maxHidx); + gridMaxHorizDist = gridMaxHorizDist(minNidx:maxNidx,minEidx:maxEidx,minHidx:maxHidx); + + % define and crop dim vectors + gridNorthing = (0:numElemGridN-1)'.*grid_horz_res + minGridN; + gridEasting = (0:numElemGridE-1) .*grid_horz_res + minGridE; + gridHeight = permute((0:numElemGridH-1).*grid_vert_res + minGridH,[3,1,2]); + gridNorthing = gridNorthing(minNidx:maxNidx); + gridEasting = gridEasting(:,minEidx:maxEidx); + gridHeight = gridHeight(:,:,minHidx:maxHidx); + +end + +%% final calculation: average and back in dB +gridLevel = 10.*log10(gridWeightedSum./gridTotalWeight); +clear gridWeightedSum + + +%% if gridded data were gpuArrays, convert back to regular before storing +% so that data can be used even without the parallel computing toolbox and +% so that loading data don't overload the limited GPU memory. Gather has no +% effect if these were regular arrays +gridLevel = gather(gridLevel); +gridTotalWeight = gather(gridTotalWeight); +gridMaxHorizDist = gather(gridMaxHorizDist); + + +%% saving results: + +% metadata +fData.X_1_gridHeightReference = grdlim_var; +fData.X_1_gridHorizontalResolution = grid_horz_res; + +switch grid_type + case '3D' + fData.X_1_gridVerticalResolution = grid_vert_res; +end + +% grid axes +fData.X_1E_gridEasting = gridEasting; +fData.X_N1_gridNorthing = gridNorthing; + +switch grid_type + case '3D' + fData.X_11H_gridHeight = gridHeight; +end + +% actual grids +fData.X_NEH_gridLevel = gridLevel; +fData.X_NEH_gridDensity = gridTotalWeight; +fData.X_NEH_gridMaxHorizDist = gridMaxHorizDist; + + diff --git a/watercolumn_processing/CFF_group_pings.m b/watercolumn_processing/CFF_group_pings.m new file mode 100644 index 0000000..b12fcad --- /dev/null +++ b/watercolumn_processing/CFF_group_pings.m @@ -0,0 +1,80 @@ +function [maxNSamples_groups,ping_group_start,ping_group_end] = CFF_group_pings(num_samp_per_dtgrm, ping_counter, dtgrm_ping_number) +%CFF_GROUP_PINGS Makes groups of pings based on number of samples +% +% Makes groups of pings based on the max number of samples in each ping. + + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 30-08-2021 + +% get the maximum number of samples for each ping +if iscell(num_samp_per_dtgrm) + % if num_samp_per_dtgrm is a cell array, it holds the number of + % samples per datagram. Group them by ping and find the max + max_num_samp_per_ping = nan(1,numel(ping_counter)); + for ii = 1:numel(ping_counter) + ix = dtgrm_ping_number==ping_counter(ii); + max_num_samp_per_ping(ii) = max(cellfun(@(x) max(x),num_samp_per_dtgrm(ix))); + end +else + % if num_samp_per_dtgrm is normal array, it already is the max number + % of samples per ping + max_num_samp_per_ping = num_samp_per_dtgrm; +end + +% Yoann's mystery algorithm to make groups of sequential pings based on the +% max number of samples they contain +nb_min_s = 50; +nb_min_win = 50; +perc_inc = 10/100; +X_fact = prctile(ceil(max_num_samp_per_ping/nb_min_s)*nb_min_s,90)/prctile(floor(max_num_samp_per_ping/nb_min_s)*nb_min_s,10); +div_factor = (perc_inc/(X_fact-1))*min(max_num_samp_per_ping); +div_factor = ceil(div_factor/nb_min_s)*nb_min_s; +group_by_nb_s = ceil(filter2(ones(1,nb_min_win),ceil(max_num_samp_per_ping/div_factor),'same')./... + filter2(ones(1,nb_min_win),ones(size(ping_counter)),'same')); +idx_change = find(diff(group_by_nb_s)~=0); +idx_change_2 = find(diff(ping_counter)>1)+1; +idx_change = union(idx_change,idx_change_2); + +% % mystery plot for the mystery algorithm +% figure(); +% plot(ping_counter,ceil(max_num_samp_per_ping/div_factor)); +% hold on; +% plot(ping_counter,group_by_nb_s); +% plot(ping_counter,max_num_samp_per_ping/div_factor); +% for uil = 1:numel(idx_change) +% xline(ping_counter(idx_change(uil)),'--k'); +% end + +% and the resulting grouping of pings: +idx_new_group = unique([1 idx_change]); +ping_group_start = ping_counter(idx_new_group); +ping_group_end = ping_counter([idx_new_group(2:end)-1 numel(ping_counter)]); +num_groups = numel(idx_new_group); + +% calculate the max number of samples in a ping, per group +maxNSamples_groups = nan(1,num_groups); +for uig = 1:num_groups + if iscell(num_samp_per_dtgrm) + % indices of datagrams in this group + ix = (dtgrm_ping_number>=ping_group_start(uig))&(dtgrm_ping_number<=ping_group_end(uig)); + maxNSamples_groups(uig) = max(cellfun(@(x) max(x),num_samp_per_dtgrm(ix))); + else + % indices of datagrams in this group + ix = (ping_counter>=ping_group_start(uig))&(ping_counter<=ping_group_end(uig)); + maxNSamples_groups(uig) = max(num_samp_per_dtgrm(ix)); + end +end + +% because ping counters often wrap around (i.e. max ping counter is 65536 +% then it goes back to 1), this can trip up later code, so Yoann here +% changes the ping groups back to 1. +for ui = 1:num_groups + ping_group_start(ui) = find(ping_counter==ping_group_start(ui),1); + ping_group_end(ui) = find(ping_counter==ping_group_end(ui),1); +end + + + + diff --git a/watercolumn_processing/CFF_mask_WC_data.m b/watercolumn_processing/CFF_mask_WC_data.m new file mode 100644 index 0000000..3333f32 --- /dev/null +++ b/watercolumn_processing/CFF_mask_WC_data.m @@ -0,0 +1,72 @@ +function [fData] = CFF_mask_WC_data(fData,varargin) +%CFF_MASK_WC_DATA Mask water-column data to remove unwanted samples +% +% *INPUT VARIABLES* +% * |fData|: Required. Structure for the storage of kongsberg EM series +% multibeam data in a format more convenient for processing. The data is +% recorded as fields coded "a_b_c" where "a" is a code indicating data +% origing, "b" is a code indicating data dimensions, and "c" is the data +% name. See the help of function CFF_convert_ALLdata_to_fData.m for +% description of codes. +% * |remove_angle|: Optional. Steering angle beyond which outer beams are +% removed (in deg ref acoustic axis). Example: 55 -> angles>55 and <-55 +% are removed. Default: inf (all angles are conserved). +% * |remove_closerange|: Optional. Range from sonar (in m) within which +% samples are removed. Example: 4 -> all samples within 4m range from +% sonar are removed. Default: 0 (all samples are conserved). +% * |remove_bottomrange|: Optional. Range from bottom (in m) beyond which +% samples are removed. Range after bottom if positive, before bottom if +% negative. Example: 2 -> all samples 2m AFTER bottom detect and beyond +% are removed. Example: -3 -> all samples 3m BEFORE bottom detect and beyond +% are removed (therefore including bottom detect). Default: inf (all +% samples are conserved). +% * |mypolygon|: Optional. Horizontal polygon (in Easting, Northing +% coordinates) outside of which samples are removed. Defualt: [] (all +% samples are conserved). +% +% *OUTPUT VARIABLES* +% * |fData|: fData structure updated with "X_SBP_WaterColumnProcessed" +% now masked. +% +% *DEVELOPMENT NOTES* +% * check that masking uses filtered bottom if it exists, original bottom +% if not. +% + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +% extract info about WCD +wcdata_Class = fData.X_1_WaterColumnProcessed_Class; % int8 or int16 +wcdata_Factor = fData.X_1_WaterColumnProcessed_Factor; +wcdata_Nanval = fData.X_1_WaterColumnProcessed_Nanval; + +[nSamples, nBeams, nPings] = CFF_get_WC_size(fData); + +% block processing setup +[blocks,info] = CFF_setup_optimized_block_processing(... + nPings,nSamples*nBeams*4,... + 'desiredMaxMemFracToUse',0.1); + +% block processing +for iB = 1:size(blocks,1) + + % list of pings in this block + blockPings = (blocks(iB,1):blocks(iB,2)); + + % grab data in dB + data = CFF_get_WC_data(fData,'X_SBP_WaterColumnProcessed','iPing',blockPings,'output_format','true'); + + % core processing + data = CFF_mask_WC_data_CORE(data, fData, blockPings, varargin{:}); + + % convert modified data back to raw format and store + data = data./wcdata_Factor; + data(isnan(data)) = wcdata_Nanval; + fData.X_SBP_WaterColumnProcessed.Data.val(:,:,blockPings) = cast(data,wcdata_Class); + +end + + + diff --git a/watercolumn_processing/CFF_mask_WC_data_CORE.m b/watercolumn_processing/CFF_mask_WC_data_CORE.m new file mode 100644 index 0000000..b031249 --- /dev/null +++ b/watercolumn_processing/CFF_mask_WC_data_CORE.m @@ -0,0 +1,391 @@ +function [data,params] = CFF_mask_WC_data_CORE(data, fData, iPings, varargin) +%CFF_MASK_WC_DATA_CORE Mask unwanted WCD +% +% This function masks unwanted data in specific pings of water-column +% data. +% +% DATA = CFF_MASK_WC_DATA_CORE(DATA,FDATA,IPINGS) takes input DATA (SBP +% tensor) and masks unwanted data in it, using the necessary information +% in FDATA for the relevant ping indices IPINGS. It returns the corrected +% DATA. +% +% CFF_MASK_WC_DATA_CORE(DATA,FDATA,IPINGS,PARAMS) uses +% processing parameters defined as the fields in the PARAMS structure. +% Possible parameters are: +% 'maxAngle': angle (in degrees) from broadside beyond which data are to +% be discarded. Typically 50 to 60. Default is inf to KEEP all data. +% 'minRange': range (in m) from sonar within which data are to be +% discarded. Typically 1 to 5. Default is 0 to KEEP all data. +% 'maxRangeBelowBottomEcho': range (in m) from the top of the bottom echo +% beyond which data are to be discarded. Typically 0 to remove just the +% echo, or -1 to -10 to be more conservative. Default is inf to KEEP all +% data. +% 'withinPolygon': vertices (in Easting and Northing) of the polygon +% outside of which data are to be discarded. Default is [] to KEEP all +% data. +% 'maxPercentFaultyDetects': proportion (in %) of faulty detects in a +% ping beyond which the entire ping is to be discarded. Typically ~7 to +% remove all but perfect pings, ~ 10 to 20 to allow pings with a few +% faulty detects, or >20 to remove only the most severly affected pings. +% Default is 100 to KEEP all data. +% 'maxRangeBelowMSR': range (in m) from the Minimum Slant Range (MSR) +% beyond which data are to be discarded. Typically 0 to remove all data +% past the MSR, or -1 to -10 to be more conservative. Default is inf to +% KEEP all data. +% +% CFF_MASK_WC_DATA_CORE(...,'comms',COMMS) specifies if +% and how this function communicates on its internal state (progress, +% info, errors). COMMS can be either a CFF_COMMS object, or a text string +% to initiate a new CFF_COMMS object. Options are 'disp', +% 'textprogressbar', 'waitbar', 'oneline', 'multilines'. By default, +% using an empty CFF_COMMS object (i.e. no communication). See CFF_COMMS +% for more information. +% +% [FDATA,PARAMS] = CFF_MASK_WC_DATA_CORE(...) also outputs +% the parameters used in processing. +% +% Note: estimation of bottom echo to be improved +% +% See also CFF_MASK_WC_DATA, CFF_WC_RADIOMETRIC_CORRECTIONS_CORE, +% CFF_FILTER_WC_SIDELOBE_ARTIFACT_CORE. + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann Ladroit +% (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2022; Last revision: 28-07-2022 + + +global DEBUG; + + +%% Input arguments management +p = inputParser; +addRequired(p,'data',@(x) isnumeric(x)); % data to process +addRequired(p,'fData',@(x) CFF_is_fData_version_current(x)); % source fData +addRequired(p,'iPings',@(x) isnumeric(x)); % indices of pings to process +addOptional(p,'params',struct(),@(x) isstruct(x)); % processing parameters +addParameter(p,'comms',CFF_Comms()); % information communication (none by default) +parse(p,data,fData,iPings,varargin{:}); +params = p.Results.params; +comms = p.Results.comms; +clear p +if ischar(comms) + comms = CFF_Comms(comms); +end + + +%% Prep + +% start message +comms.start('Masking unwanted data'); + +% data size +[nSamples,nBeams,nPings] = size(data); + +% source datagram +datagramSource = CFF_get_datagramSource(fData); + +% get some variables needed in several masks +interSamplesDistance = CFF_inter_sample_distance(fData,iPings); +beamPointingAngleDeg = fData.(sprintf('%s_BP_BeamPointingAngle',datagramSource))(:,iPings); +startRangeSampleNumber = fData.(sprintf('%s_BP_StartRangeSampleNumber',datagramSource))(:,iPings); +sampleRange = CFF_get_samples_range((1:nSamples)', startRangeSampleNumber, interSamplesDistance); + +%% Mask 1: Removing beams beyond an angle from broadside +% 'maxAngle': angle (in degrees) from broadside beyond which data are to +% be discarded. Typically 50 to 60. Default is inf to KEEP all data. + +% get maxAngle parameter +if ~isfield(params,'maxAngle'), params.maxAngle = inf; end % default +mustBeNumeric(params.maxAngle); % validate +maxAngle = params.maxAngle; + +if ~isinf(maxAngle) + % build mask: 1: to conserve, 0: to remove + X_BP_OuterBeamsMask = beamPointingAngleDeg>=-abs(maxAngle) & beamPointingAngleDeg<=abs(maxAngle); + X_1BP_OuterBeamsMask = permute(X_BP_OuterBeamsMask ,[3,1,2]); +else + % conserve all data + X_1BP_OuterBeamsMask = true(1,nBeams,nPings); +end + + +%% Mask 2: Removing samples within a range from sonar +% 'minRange': range (in m) from sonar within which data are to be +% discarded. Typically 1 to 5. Default is 0 to KEEP all data. + +% get minRange parameter +if ~isfield(params,'minRange'), params.minRange = 0; end % default +mustBeNumeric(params.minRange); % validate +minRange = params.minRange; + +if minRange>0 + % build mask: 1: to conserve, 0: to remove + X_SBP_CloseRangeMask = sampleRange>=minRange; +else + % conserve all data + X_SBP_CloseRangeMask = true(nSamples,nBeams,nPings); +end + + +%% Mask 3: Removing samples beyond a range below the top of the seabed echo +% 'maxRangeBelowBottomEcho': range (in m) from the top of the bottom echo +% beyond which data are to be discarded. Typically 0 to remove just the +% echo, or -1 to -10 to be more conservative. Default is inf to KEEP all +% data. + +% get maxRangeBelowBottomEcho parameter +if ~isfield(params,'maxRangeBelowBottomEcho'), params.maxRangeBelowBottomEcho = inf; end % default +mustBeNumeric(params.maxRangeBelowBottomEcho); % validate +maxRangeBelowBottomEcho = params.maxRangeBelowBottomEcho; + +if ~isinf(maxRangeBelowBottomEcho) + + % some data needed to find the top of the bottom echo + theta = deg2rad(beamPointingAngleDeg); % beam pointing angle in radians + beamwidth = deg2rad(fData.Ru_1D_ReceiveBeamwidth(1)); % beamwidth + + % the first part is to find the top of the bottom echo, i.e. the range + % in each beam where the echo starts. Some development is still needed + % so for now we're doing a method switch. + % Best method so far is 3. Debug display after the switch. Don't turn + % this into a parameter - the point is to find and retain the best + % method after development, but we keep the old ones just for reference + % for now. + method = 3; + switch method + + case 1 + % first developped version, from Yoann + + % beamwidth including increase with beam steering + psi = beamwidth./cos(abs(theta)).^2/2; + + % transition between normal and grazing incidence + theta_lim = psi/2; + idx_normal = abs(theta) < theta_lim; + idx_grazing = ~idx_normal; + + % prep + R = fData.X_BP_bottomRange(:,iPings); % range of bottom detect + R1 = zeros(size(theta),'single'); % range of echo start + + % compute range for each regime + R1(idx_normal) = ( 1./cos(theta(idx_normal)+psi(idx_normal)/2) - 1./cos(theta(idx_normal)) ) .* R(idx_normal); + R1(idx_grazing) = 2*( sin(theta(idx_grazing)+psi(idx_grazing)/2) - sin(theta(idx_grazing)-psi(idx_grazing)/2) ) .* R(idx_grazing); + + % Alex comments: First, the equation for beamwidth increase + % with beam steering is bizarre. I think it should be + % psi/cos(theta)... Next, I don't get the equation for the + % normal regime, but I can see the equation for the second + % regime is meant to be the horizontal distance of the + % intercept of the beam on a flat seafloor... except I think + % it's missing the abs() function to deal with negative + % steering angles, and it's multiplied by two for some + % reason... + % + % The main issue is: why the horizontal distance? We want the + % RANGE at which the beam FIRST intercepts the seafloor. + % + % So let's not use that one, but keeping it because I don't + % fully understand this and I want to keep it until I'm 100% + % sure it is not correct + + case 2 + % second version, from Alex + + % first, what I think is the actual beamwidth including beam + % steering: + psi = beamwidth./cos(abs(theta)); + + % recalculating the normal/grazing incidence regimes + theta_lim = psi/2; + idx_normal = abs(theta) < theta_lim; + idx_grazing = ~idx_normal; + + % prep + R = fData.X_BP_bottomRange(:,iPings); % range of bottom detect + R1 = zeros(size(theta),'single'); % range of echo start + + % in the grazing regime, assuming a depth D, the range at which + % the echo starts is R1 obtained from: + % cos(theta) = D/R and cos(theta-0.5*psi) = D/R1 + % Aka: R1 = R*(cos(theta)/cos(theta-0.5*psi)) + % Since we here want R-R1, then: + % R1 = R( 1 - (cos(theta)/cos(theta-0.5*psi)) ) + R1(idx_grazing) = R(idx_grazing) .* ( 1 - (cos(abs(theta(idx_grazing)))./cos(abs(theta(idx_grazing))-0.5.*psi(idx_grazing))) ); + + % in the normal regime, we just apply the value at the + % regime-transition aka: R1 = R( 1 - cos(theta) ) + R1(idx_normal) = R(idx_normal) .* ( 1 - cos(abs(theta(idx_normal))) ); + + % Alex comments: it's closer to the bottom echo, but on our + % test data, it looks like the bottom detection is not always + % at the same place in the bottom echo... did we forget some + % angular correction for the placement of the bottom?? + + case 3 + % third version, empirical + + % Since none of the two first versions work too well on our + % test data, we try an empirical approach: We try to + % approximate the range at which the beam footprint starts as + % the minimum range within +-X beams around beam of interest + X = 5; + nbeams = size(theta,1); + R1 = zeros(size(theta),'single'); + for ip = 1:length(iPings) + bottomranges = fData.X_BP_bottomRange(:,iPings(ip)); + minrangefunc = @(ibeam) nanmin(bottomranges(max(1,ibeam-X):min(nbeams,ibeam+X))); + R1(:,ip) = bottomranges - arrayfun(minrangefunc,[1:nbeams]'); + end + + % Alex comment: works better overall, but still not perfect. + + end + + % DEBUG display + if DEBUG + WCD = CFF_get_WC_data(fData,'WC_SBP_SampleAmplitudes',iPings); + WCD_x = 1:size(WCD,2); + WCD_y = interSamplesDistance(1).*[1:size(WCD,1)]; + figure;imagesc(WCD_x,WCD_y,WCD(:,:,1)); colormap('jet'); grid on; hold on + plot(fData.X_BP_bottomRange(:,1),'k.-') + plot(fData.X_BP_bottomRange(:,1) - R1(:,1),'ko-'); + end + + % when we have that range, the rest is easy... + + % calculate max sample beyond which mask is to be applied + X_BP_maxRange = fData.X_BP_bottomRange(:,iPings) - R1 + maxRangeBelowBottomEcho; + X_BP_maxSample = bsxfun(@rdivide,X_BP_maxRange,interSamplesDistance); + X_BP_maxSample = round(X_BP_maxSample); + X_BP_maxSample(X_BP_maxSample>nSamples|isnan(X_BP_maxSample)) = nSamples; + + % build list of indices for each beam & ping + [PP,BB] = meshgrid((1:nPings),(1:nBeams)); + maxSubs = [X_BP_maxSample(:),BB(:),PP(:)]; + + % build mask: 1: to conserve, 0: to remove + X_SBP_BottomRangeMask = false(nSamples,nBeams,nPings); + for ii = 1:size(maxSubs,1) + X_SBP_BottomRangeMask(1:maxSubs(ii,1),maxSubs(ii,2),maxSubs(ii,3)) = true; + end + +else + + % conserve all data + X_SBP_BottomRangeMask = true(nSamples,nBeams,nPings); + +end + + +%% Mask 4: Removing data outside an Easting & Northing polygon +% 'withinPolygon': vertices (in Easting and Northing) of the polygon +% outside of which data are to be discarded. Default is [] to KEEP all +% data. + +% get withinPolygon parameter +if ~isfield(params,'withinPolygon'), params.withinPolygon = []; end % default +mustBeNumeric(params.withinPolygon); % validate (can improve this) +withinPolygon = params.withinPolygon; + +if ~isempty(withinPolygon) + % get easting and northing for all samples + idxSamples = (1:nSamples)'; + sonarEasting = fData.X_1P_pingE(iPings); + sonarNorthing = fData.X_1P_pingN(iPings); + sonarHeight = fData.X_1P_pingH(iPings); + sonarHeading = fData.X_1P_pingHeading(iPings); + [E,N] = CFF_georeference_sample(idxSamples, startRangeSampleNumber, interSamplesDistance, deg2rad(beamPointingAngleDeg), ... + sonarEasting, sonarNorthing, sonarHeight, sonarHeading); + % build mask: 1: to conserve, 0: to remove + X_SBP_PolygonMask = inpolygon(E,N,... + withinPolygon(:,1), ... + withinPolygon(:,2)); +else + % conserve all data + X_SBP_PolygonMask = true(nSamples,nBeams,nPings); +end + + +%% Mask 5: Removing pings that have bad quality +% for now we will use the percentage of faulty bottom detects as a +% threshold to identify bad-quality pings. Aka, if mask_ping=10, then we +% will mask the ping if 10% or more of its bottom detects are faulty. +% Quick data look-up shows that good pings can still have up to 6% faulty +% bottom detects, usually on the outer beams. A ping with some missing +% bottom detects in the data is around 8-15%, so good rule of thumb would be +% to use: +% 'maxPercentFaultyDetects': proportion (in %) of faulty detects in a +% ping beyond which the entire ping is to be discarded. Typically ~7 to +% remove all but perfect pings, ~ 10 to 20 to allow pings with a few +% faulty detects, or >20 to remove only the most severly affected pings. +% Default is 100 to KEEP all data. + +% get maxPercentFaultyDetects parameter +if ~isfield(params,'maxPercentFaultyDetects'), params.maxPercentFaultyDetects = []; end % default +mustBeNumeric(params.maxPercentFaultyDetects); % validate (can improve this) +maxPercentFaultyDetects = params.maxPercentFaultyDetects; + +if maxPercentFaultyDetects<100 + % extract needed data + faultyDetects = fData.(sprintf('%s_BP_DetectedRangeInSamples',datagramSource))(:,iPings)==0; % raw bottom detect + proportionFaultyDetects = 100.*sum(faultyDetects)./nBeams; + % build mask: 1: to conserve, 0: to remove + X_1P_PingMask = proportionFaultyDetects1 + datagramSource = varargin{1}; +else + datagramSource = []; +end +datagramSource = CFF_get_datagramSource(fData,datagramSource); + +% varargin{2}: ellipsoid for CFF_ll2tm conversion +if nargin>2 + ellips = varargin{2}; +else + ellips = 'wgs84'; + %fprintf('ellips not specified. Using ''wgs84''...\n'); +end + +% varargin{3}: TM projection for CFF_ll2tm conversion +if nargin>3 + tmproj = varargin{3}; +else + % to be specified from fir good lat/long value + tmproj = ''; +end + +% varargin{?}: datum conversion? +... + +% varargin{4}: navigation latency +if nargin == 5 + navLat = varargin{4}; +else + navLat = 0; + comms.info('navLat not specified in input. Using 0.'); +end + + +%% EXTRACT PING DATA +% create ping time vectors in serial date number (SDN, Matlab, the whole +% and fractional number of days from January 0, 0000) and Time Since +% Midnight In Milliseconds (TSMIM, Kongsberg). + +pingTSMIM = fData.([datagramSource '_1P_TimeSinceMidnightInMilliseconds']); +pingDate = fData.([datagramSource '_1P_Date']); +pingCounter = fData.([datagramSource '_1P_PingCounter']); +pingDate = datenum(cellfun(@num2str,num2cell(pingDate),'un',0),'yyyymmdd'); +pingSDN = pingDate(:)'+ pingTSMIM/(24*60*60*1000) + navLat./(1000.*60.*60.*24); % apply navigation latency here + + +%% EXTRACT NAVIGATION DATA +% same for navigation. In the future, offer possibility to import +% position/orientation from other files, say SBET + +% test if there are several sources of GPS data +if isfield(fData,'Po_1D_PositionSystemDescriptor') + ID = unique(fData.Po_1D_PositionSystemDescriptor); + if numel(ID) > 1 + % several sources available, we will need to choose one + % start by eliminating those that are obviously bad. + % I have found data where one source had lat/long values that were + % both constant and outside of normal values. You may want to + % devise more tests if you ever come across different examples of + % bad position data + isSingleEntry = arrayfun(@(x) sum(fData.Po_1D_PositionSystemDescriptor==x)==1, ID); % check if single entry (then they will be constant) + isLatAllConst = arrayfun(@(x) all(diff(fData.Po_1D_Latitude(fData.Po_1D_PositionSystemDescriptor==x))==0), ID); % check if all constant values + isLonAllConst = arrayfun(@(x) all(diff(fData.Po_1D_Longitude(fData.Po_1D_PositionSystemDescriptor==x))==0), ID); % check if all constant values + isLatAllBad = arrayfun(@(x) all(abs(fData.Po_1D_Latitude(fData.Po_1D_PositionSystemDescriptor==x))>90), ID); % check if all outside [-90:90] + isLonAllBad = arrayfun(@(x) all(abs(fData.Po_1D_Longitude(fData.Po_1D_PositionSystemDescriptor==x))>180), ID); % check if all outside [-180:180] + idxBadPos = (~isSingleEntry & (isLatAllConst|isLonAllConst)) | isLatAllBad | isLonAllBad; + % removing those bad sources + ID = ID(~idxBadPos); + if numel(ID)==1 + % only one good source left, just use that one + pos_idx = fData.Po_1D_PositionSystemDescriptor==ID; + else + % still several sources available + % find the one with the best fix quality + meanFixQuality = arrayfun(@(x) nanmean(fData.Po_1D_MeasureOfPositionFixQuality(fData.Po_1D_PositionSystemDescriptor==x)), ID); + [~,idx_keep] = min(meanFixQuality); + pos_idx = fData.Po_1D_PositionSystemDescriptor==ID(idx_keep); + comms.info(sprintf('Several sources of GPS data available. Using source with ID: %d',ID(idx_keep))); + end + else + % single source. Use all datagrams. + pos_idx = 1:numel(fData.Po_1D_Latitude); + end +else + % using older version of converted data, throw warning and continue + comms.info('Navigation information in your converted data indicates it is not up to date with this version of CoFFee. Consider reconverting this file, particularly if you see strange patterns in the navigation, or if two GPS sources have been logged in the file.'); + pos_idx = 1:numel(fData.Po_1D_Latitude); +end + +% get data +posLatitude = fData.Po_1D_Latitude(pos_idx); +posLongitude = fData.Po_1D_Longitude(pos_idx); +posHeading = fData.Po_1D_HeadingOfVessel(pos_idx); +posSpeed = fData.Po_1D_SpeedOfVesselOverGround(pos_idx); +posTSMIM = fData.Po_1D_TimeSinceMidnightInMilliseconds(pos_idx); % time since midnight in milliseconds +posDate = datenum(cellfun(@num2str,num2cell(fData.Po_1D_Date(pos_idx)),'un',0),'yyyymmdd'); +posSDN = posDate(:)'+ posTSMIM/(24*60*60*1000); % serial date number + +% define tmproj at this stage, if it was not provided in input +if isempty(tmproj) + [~,~,~,~,tmproj] = CFF_ll2tm(posLongitude(1),posLatitude(1),ellips,'utm'); + tmproj = ['utm' tmproj]; + comms.info(['tmproj not specified in input. Defining it from first position fix: ''' tmproj '''']); +end + + +%% EXTRACT HEIGHT DATA +if isfield(fData,'He_1D_Height') + heiHeight = fData.He_1D_Height; % now m + heiDate = datenum(cellfun(@num2str,num2cell(fData.He_1D_Date),'un',0),'yyyymmdd'); + heiSDN = heiDate(:)' + fData.He_1D_TimeSinceMidnightInMilliseconds/(24*60*60*1000); +else + % no height datagrams, create fake variables + heiHeight = zeros(size(pingTSMIM)); + heiSDN = pingSDN; +end + + +%% PROCESS NAVIGATION AND HEADING +% Get position and heading for each ping. Position and heading were +% recorded at the sensor's time so we need to interpolate them at the same +% time to match ping time. + +% convert posLatitude/posLongitude to easting/northing/grid convergence: +[posE, posN, posGridConv] = CFF_ll2tm(posLongitude, posLatitude, ellips, tmproj); + +% we need at least two position samples to process the navigation. If there +% is only one, make up another one using dead reckoning +if numel(posE)==1 + posE = [posE, posE + posSpeed.*cosd(posHeading)]; + posN = [posN, posN + posSpeed.*sind(posHeading)]; + posGridConv = [posGridConv, posGridConv]; + posHeading = [posHeading, posHeading]; + posSpeed = [posSpeed, posSpeed]; + posTSMIM = [posTSMIM, posTSMIM + 1000]; % + 1 sec + posSDN = [posSDN, posSDN + 1/(24*60*60)]; % + 1 sec +end + + +% convert heading to degrees and allow heading values superior to +% 360 or inferior to 0 (because every time the vessel crossed the NS +% line, the heading jumps from 0 to 360 (or from 360 to 0) and this +% causes a problem for following interpolation): + +posJump = find(diff(posHeading)>300); +negJump = find(diff(posHeading)<-300); +jumps = zeros(1,length(posHeading)); + +if ~isempty(posJump) + for jj = 1:length(posJump) + jumps(posJump(jj)+1:end) = jumps(posJump(jj)+1:end) - 1; + end +end + +if ~isempty(negJump) + for jj = 1:length(negJump) + jumps(negJump(jj)+1:end) = jumps(negJump(jj)+1:end) + 1; + end +end + +posHeading = posHeading + jumps.*360; + +% dirty heading fix if first value is null +if posHeading(1)==0 && abs(posHeading(2))>5 + posLatitude(1) = []; + posLongitude(1) = []; + posHeading(1) = []; + posSpeed(1) = []; + posTSMIM(1) = []; + posDate(1) = []; + posSDN(1) = []; + jumps(1) = []; +end + +% initialize new vectors +pingE = nan(size(pingTSMIM)); +pingN = nan(size(pingTSMIM)); +pingGridConv = nan(size(pingTSMIM)); +pingHeading = nan(size(pingTSMIM)); +pingSpeed = nan(size(pingTSMIM)); + +% interpolate Easting, Northing, Grid Convergence and Heading at ping times +for jj = 1:length(pingTSMIM) + A = posSDN-pingSDN(jj); + iA = find (A == 0); + if A > 0 + % the ping time is older than any navigation time, extrapolate from the first items in navigation array. + pingE(jj) = posE(2) + (posE(2)-posE(1)).*(pingSDN(jj)-posSDN(2))./(posSDN(2)-posSDN(1)); + pingN(jj) = posN(2) + (posN(2)-posN(1)).*(pingSDN(jj)-posSDN(2))./(posSDN(2)-posSDN(1)); + pingGridConv(jj) = posGridConv(2) + (posGridConv(2)-posGridConv(1)).*(pingSDN(jj)-posSDN(2))./(posSDN(2)-posSDN(1)); + pingHeading(jj) = posHeading(2) + (posHeading(2)-posHeading(1)).*(pingSDN(jj)-posSDN(2))./(posSDN(2)-posSDN(1)); + pingSpeed(jj) = posSpeed(2) + (posSpeed(2)-posSpeed(1)).*(pingTSMIM(jj)-posTSMIM(2))./(posTSMIM(2)-posTSMIM(1)); + elseif A < 0 + % the ping time is more recent than any navigation time, extrapolate from the last items in navigation array. + pingE(jj) = posE(end) + (posE(end)-posE(end-1)).*(pingSDN(jj)-posSDN(end))./(posSDN(end)-posSDN(end-1)); + pingN(jj) = posN(end) + (posN(end)-posN(end-1)).*(pingSDN(jj)-posSDN(end))./(posSDN(end)-posSDN(end-1)); + pingGridConv(jj) = posGridConv(end) + (posGridConv(end)-posGridConv(end-1)).*(pingSDN(jj)-posSDN(end))./(posSDN(end)-posSDN(end-1)); + pingHeading(jj) = posHeading(end) + (posHeading(end)-posHeading(end-1)).*(pingSDN(jj)-posSDN(end))./(posSDN(end)-posSDN(end-1)); + pingSpeed(jj) = posSpeed(end) + (posSpeed(end)-posSpeed(end-1)).*(pingTSMIM(jj)-posTSMIM(end))./(posTSMIM(end)-posTSMIM(end-1)); + elseif ~isempty(iA) + % the ping time corresponds to an existing navigation time, get easting and northing from it. + pingE(jj) = posE(iA); + pingN(jj) = posN(iA); + pingGridConv(jj) = posGridConv(iA); + pingHeading(jj) = posHeading(iA); + pingSpeed(jj) = posSpeed(iA); + else + % the ping time is within the limits of the navigation time array but doesn't correspond to any value in it, interpolate from nearest values + iNegA = find(A<0); + [~,iMax] = max(A(iNegA)); + iA(1) = iNegA(iMax); % index of navigation time just older than ping time + iPosA = find(A>0); + [~,iMin] = min(A(iPosA)); + iA(2) = iPosA(iMin); % index of navigation time just more recent ping time + % now extrapolate easting, northing, grid convergence and heading + pingE(jj) = posE(iA(2)) + (posE(iA(2))-posE(iA(1))).*(pingSDN(jj)-posSDN(iA(2)))./(posSDN(iA(2))-posSDN(iA(1))); + pingN(jj) = posN(iA(2)) + (posN(iA(2))-posN(iA(1))).*(pingSDN(jj)-posSDN(iA(2)))./(posSDN(iA(2))-posSDN(iA(1))); + pingGridConv(jj) = posGridConv(iA(2)) + (posGridConv(iA(2))-posGridConv(iA(1))).*(pingSDN(jj)-posSDN(iA(2)))./(posSDN(iA(2))-posSDN(iA(1))); + pingHeading(jj) = posHeading(iA(2)) + (posHeading(iA(2))-posHeading(iA(1))).*(pingSDN(jj)-posSDN(iA(2)))./(posSDN(iA(2))-posSDN(iA(1))); + pingSpeed(jj) = posSpeed(iA(2)) + (posSpeed(iA(2))-posSpeed(iA(1))).*(pingTSMIM(jj)-posTSMIM(iA(2)))./(posTSMIM(iA(2))-posTSMIM(iA(1))); + end +end + +% bring heading back into the interval [0 360] +posHeading = posHeading - jumps.*360; +pingHeading = mod(pingHeading,360); + + +%% PROCESS HEIGHT +% Get height for each ping. Height were recorded at the sensor's time so we +% need to interpolate them at the same time to match ping time. + +% initialize new vectors +pingH = nan(size(pingSDN)); + +% interpolate Height at ping times +for jj = 1:length(pingSDN) + A = heiSDN-pingSDN(jj); + iA = find (A == 0,1); + if A > 0 + % the ping time is older than any height time, extrapolate from the first items in height array. + pingH(jj) = heiHeight(2) + (heiHeight(2)-heiHeight(1)).*(pingSDN(jj)-heiSDN(2))./(heiSDN(2)-heiSDN(1)); + elseif A < 0 + % the ping time is more recent than any height time, extrapolate from the last items in height array. + pingH(jj) = heiHeight(end) + (heiHeight(end)-heiHeight(end-1)).*(pingSDN(jj)-heiSDN(end))./(heiSDN(end)-heiSDN(end-1)); + elseif ~isempty(iA) + % the ping time corresponds to an existing height time, get height + % from it + pingH(jj) = heiHeight(iA); + else + % the ping time is within the limits of the height time array but doesn't correspond to any value in it, interpolate from nearest values + iNegA = find(A<0); + [~,iMax] = max(A(iNegA)); + iA(1) = iNegA(iMax); % index of height time just older than ping time + iPosA = find(A>0); + [~,iMin] = min(A(iPosA)); + iA(2) = iPosA(iMin); % index of height time just more recent ping time + % now extrapolate height + pingH(jj) = heiHeight(iA(2)) + (heiHeight(iA(2))-heiHeight(iA(1))).*(pingSDN(jj)-heiSDN(iA(2)))./(heiSDN(iA(2))-heiSDN(iA(1))); + end +end + + +%% SAVE RESULTS + +% save processed results +fData.X_1P_pingCounter = pingCounter; +fData.X_1P_pingTSMIM = pingTSMIM; +fData.X_1P_pingSDN = pingSDN; +fData.X_1P_pingE = pingE; +fData.X_1P_pingN = pingN; +fData.X_1P_pingH = pingH; +fData.X_1P_pingGridConv = pingGridConv; +fData.X_1P_pingHeading = pingHeading; +fData.X_1P_pingSpeed = pingSpeed; + +% metadata. +% Datagram source is the datagram at the origin of the time vector, to +% which all the "X_1P" fields above correspond. +fData.MET_datagramSource = datagramSource; +fData.MET_navigationLatencyInMilliseconds = navLat; +fData.MET_ellips = ellips; +fData.MET_tmproj = tmproj; + +% sort fields by name +fData = orderfields(fData); diff --git a/zzz_old/obsolete/CFF_filter_WC_bottom_detect.m b/zzz_old/obsolete/CFF_filter_WC_bottom_detect.m new file mode 100644 index 0000000..0c3fac3 --- /dev/null +++ b/zzz_old/obsolete/CFF_filter_WC_bottom_detect.m @@ -0,0 +1,251 @@ +function [fData] = CFF_filter_WC_bottom_detect(fData,varargin) +%CFF_FILTER_WC_BOTTOM_DETECT Filter the bottom detect in watercolumn data +% +% fData = CFF_filter_WC_bottom_detect(fData,varargin) gets the bottom +% sample in fData (fData.X_BP_bottomSample) and filter it according to +% parameters in varargin. The end result is an updated X_BP_bottomSample +% field. +% +% OBSOLETE FUNCTION. USE CFF_FILTER_BOTTOM_DETECT_V2 instead + +% Authors: Alex Schimel (NGU, alexandre.schimel@ngu.no) and Yoann +% Ladroit (NIWA, yoann.ladroit@niwa.co.nz) +% 2017-2021; Last revision: 27-07-2021 + +warning('OBSOLETE FUNCTION. USE CFF_FILTER_BOTTOM_DETECT_V2 instead.'); + +%% INPUT PARSER + +% initialize input parser +p = inputParser; + + +% Required. +validate_fData = @isstruct; +addRequired(p,'fData',validate_fData); + +% 'method': +validate_method = @(x) ismember(x,{'filter','flag'}); +default_method = 'filter'; +addOptional(p,'method',default_method,validate_method); + +% 'pingBeamWindowSize': +validate_pingBeamWindowSize = @(x) validateattributes(x,{'numeric'},{'size',[1,2],'integer','nonnegative'}); +default_pingBeamWindowSize = [5,5]; +addOptional(p,'pingBeamWindowSize',default_pingBeamWindowSize,validate_pingBeamWindowSize); + +% 'maxHorizDist': +validate_maxHorizDist = @(x) validateattributes(x,{'numeric'},{'scalar','positive'}); +default_maxHorizDist = inf; +addOptional(p,'maxHorizDist',default_maxHorizDist,validate_maxHorizDist); + +% 'flagParams': +validate_flagParams = @(x) isstruct(x); +default_flagParams = struct('type','all','variable','vertDist','threshold',1); +addOptional(p,'flagParams',default_flagParams,validate_flagParams); + +% 'interpolate': +validate_interpolate = @(x) ismember(x,{'yes','no'}); +default_interpolate = 'yes'; +addOptional(p,'interpolate',default_interpolate,validate_interpolate); + +% parsing actual inputs +parse(p,fData,varargin{:}); + +% saving results individually +method = p.Results.method; +pingBeamWindowSize = p.Results.pingBeamWindowSize; +maxHorizDist = p.Results.maxHorizDist; +flagParams = p.Results.flagParams; +interpolateFlag = p.Results.interpolate; +clear p + + +%% PRE-PROCESSING + +% extract needed data +b0 = CFF_get_bottom_sample(fData,'which','raw'); % taking raw sample so that we don't refilter something that's been filtered already +%b0 = CFF_get_bottom_sample(fData,'which','processed'); % taking processed sample that is overwritten every time we filter, to allow extra filtering. +bE = fData.X_BP_bottomEasting; +bN = fData.X_BP_bottomNorthing; +bH = fData.X_BP_bottomHeight; + +b0(b0==0) = NaN; % replace no detects by NaNs + +% dimensions +nBeams = size(b0,1); +nPings = size(b0,2); + +% initialize results +b1 = b0; + + +%% PROCESSSING +switch method + + case 'filter' + + if isinf(maxHorizDist) + + % filter method, no limit on horiz distance + for pp = 1:nPings + for bb = 1:nBeams + % find the subset of all bottom detects within set interval in pings and beams + pmin = max(1,pp-pingBeamWindowSize(1)); + pmax = min(nPings,pp+pingBeamWindowSize(1)); + bmin = max(1,bb-pingBeamWindowSize(2)); + bmax = min(nBeams,bb+pingBeamWindowSize(2)); + % get bottom for subset + subBottom = b0(bmin:bmax,pmin:pmax); + % compute median value + b1(bb,pp) = median(subBottom(:),'omitnan'); + end + end + + else + + % filter method, with limit on horiz distance + for pp = 1:nPings + for bb = 1:nBeams + % find the subset of all bottom detects within set interval in pings and beams + pmin = max(1,pp-pingBeamWindowSize(1)); + pmax = min(nPings,pp+pingBeamWindowSize(1)); + bmin = max(1,bb-pingBeamWindowSize(2)); + bmax = min(nBeams,bb+pingBeamWindowSize(2)); + % get bottom for subset + subBottom = b0(bmin:bmax,pmin:pmax); + % get easting and northing + subEasting = bE(bmin:bmax,pmin:pmax); + subNorthing = bN(bmin:bmax,pmin:pmax); + % compute horizontal distance in m + subHzDist = sqrt( (bE(bb,pp)-subEasting).^2 + (bN(bb,pp)-subNorthing).^2 ); + % keep only subset within desired horizontal distance + subBottom(subHzDist>maxHorizDist) = NaN; + % compute median value + b1(bb,pp) = median(subBottom(:),'omitnan'); + end + end + + end + + case 'flag' + + % get flagging type first + switch flagParams.type + case 'all' + f = @(x)all(x); + case 'median' + f = @(x)median(x); + case 'any' + f = @(x)any(x); + otherwise + error('flagParams.type not recognized') + end + + % next, processing for each bottom detect (BT) + for pp = 1:nPings + for bb = 1:nBeams + % first off, flag method is inapplicable if BT doesn't exist. + if isnan(b0(bb,pp)) + % keep b1(bb,pp) as Nan. + continue + end + % find the subset of all bottom detects within set interval in pings and beams + pmin = max(1,pp-pingBeamWindowSize(1)); + pmax = min(nPings,pp+pingBeamWindowSize(1)); + bmin = max(1,bb-pingBeamWindowSize(2)); + bmax = min(nBeams,bb+pingBeamWindowSize(2)); + + subHzDist = sqrt( (bE(bb,pp)-bE(bmin:bmax,pmin:pmax)).^2 + (bN(bb,pp)-bN(bmin:bmax,pmin:pmax)).^2 ); + subHzDist(subHzDist>maxHorizDist) = NaN; + + % if there are no subset left, flag that bottom anyway + if all(isnan(subHzDist(:))) + b1(bb,pp) = NaN; + continue + end + % compute vertical distance in m + subVertDist = bH(bmin:bmax,pmin:pmax)-bH(bb,pp); + % now switch on the flagging variable + switch flagParams.variable + case 'vert' + v = subVertDist(:); + case 'eucl' + % compute euclidian distance in m + subEucliDist = sqrt( subHzDist.^2 + subVertDist.^2); + v = subEucliDist(:); + case 'slope' + % compute slope in degrees + subSlope = atan2(subVertDist,subHzDist) .*pi/180; + v = subSlope(:); + otherwise + error('flagParams.variable not recognized') + end + % finally, apply flagging decision + if f(abs(v) > flagParams.threshold) + b1(bb,pp) = NaN; + end + end + end + + otherwise + + error('method not recognized'); +end + + + +%% INTERPOLATE +switch interpolateFlag + + case 'yes' + + b1 = round(inpaint_nans(b1)); + + % safeguard against inpaint_nans occasionally yielding numbers + % below zeros in areas where there are a lot of nans: + b1(b1<1)=2; + +end + + + +%% TEST DISPLAY +% figure; +% minb = min([b0(:);b1(:)]); maxb= max([b0(:);b1(:)]); +% subplot(221); imagesc(b0); colorbar; Fdata_ID('range of raw bottom'); caxis([minb maxb]) +% subplot(222); imagesc(b1); colorbar; Fdata_ID('range of filtered bottom'); caxis([minb maxb]) +% subplot(223); imagesc(b1-b0); colorbar; Fdata_ID('filtered minus raw') + +%% SAVING RESULTS + +fData = CFF_set_bottom_sample(fData,b1); + +% and parameters +fData.X_1_bottomFilterParameters.method = method; +fData.X_1_bottomFilterParameters.pingBeamWindowSize = pingBeamWindowSize; +fData.X_1_bottomFilterParameters.maxHorizDist = maxHorizDist; +fData.X_1_bottomFilterParameters.flagParams = flagParams; +fData.X_1_bottomFilterParameters.interpolateFlag = interpolateFlag; + +%% RE-PROCESSING BOTTOM FROM RESULTS +fData = CFF_georeference_WC_bottom_detect(fData); + + + +%% obsolete code + +% % OLD method for filtering bottom +% % apply a median filter (medfilt1 should do about the same) +% % fS = ceil((p.Results.beamFilterLength-1)./2); +% fS = pingBeamWindowSize(2); +% for ii = 1+fS:nBeams-fS +% for jj=1:nPings +% tmp = b0(ii,jj-fS:jj+fS); +% tmp = tmp(~isnan(tmp(:))); +% if ~isempty(tmp) +% b2(ii,jj) = median(tmp); +% end +% end +% end + diff --git a/zzz_old/readme.txt b/zzz_old/readme.txt new file mode 100644 index 0000000..1661f74 --- /dev/null +++ b/zzz_old/readme.txt @@ -0,0 +1,6 @@ +'obsolete' is for obsolete functions, which have been replaced by newer ones. Keeping them here for backward compatibility. +For dev: add a note in their docstring (or better a disp) to indicate they are obsolete and indicate the recplacement function. + +'to_sort' are old CoFFee v1 functions, which I still need to go through to see if they are compatible with CoFFee v2. + +Keep the name zzz_old to ensure those functions are found LOWER in the path, compared to newer functions with the same name. \ No newline at end of file diff --git a/functions/DVPT/CFF_all_to_proc.m b/zzz_old/to_sort/DVPT/CFF_all_to_proc.m similarity index 100% rename from functions/DVPT/CFF_all_to_proc.m rename to zzz_old/to_sort/DVPT/CFF_all_to_proc.m diff --git a/functions/DVPT/CFF_convert_mat_to_fpbs.m b/zzz_old/to_sort/DVPT/CFF_convert_mat_to_fpbs.m similarity index 100% rename from functions/DVPT/CFF_convert_mat_to_fpbs.m rename to zzz_old/to_sort/DVPT/CFF_convert_mat_to_fpbs.m diff --git a/functions/DVPT/CFF_get_nav.m b/zzz_old/to_sort/DVPT/CFF_get_nav.m similarity index 100% rename from functions/DVPT/CFF_get_nav.m rename to zzz_old/to_sort/DVPT/CFF_get_nav.m diff --git a/functions/DVPT/CFF_get_soundings.m b/zzz_old/to_sort/DVPT/CFF_get_soundings.m similarity index 100% rename from functions/DVPT/CFF_get_soundings.m rename to zzz_old/to_sort/DVPT/CFF_get_soundings.m diff --git a/functions/DVPT/CFF_pingbeamarray.m b/zzz_old/to_sort/DVPT/CFF_pingbeamarray.m similarity index 100% rename from functions/DVPT/CFF_pingbeamarray.m rename to zzz_old/to_sort/DVPT/CFF_pingbeamarray.m diff --git a/functions/DVPT/CFF_project_nav.m b/zzz_old/to_sort/DVPT/CFF_project_nav.m similarity index 100% rename from functions/DVPT/CFF_project_nav.m rename to zzz_old/to_sort/DVPT/CFF_project_nav.m diff --git a/GUIs/Elevation_Change_Analysis/Elevation_Change_Analysis.fig b/zzz_old/to_sort/GUIs/Elevation_Change_Analysis/Elevation_Change_Analysis.fig similarity index 100% rename from GUIs/Elevation_Change_Analysis/Elevation_Change_Analysis.fig rename to zzz_old/to_sort/GUIs/Elevation_Change_Analysis/Elevation_Change_Analysis.fig diff --git a/GUIs/Elevation_Change_Analysis/Elevation_Change_Analysis.m b/zzz_old/to_sort/GUIs/Elevation_Change_Analysis/Elevation_Change_Analysis.m similarity index 100% rename from GUIs/Elevation_Change_Analysis/Elevation_Change_Analysis.m rename to zzz_old/to_sort/GUIs/Elevation_Change_Analysis/Elevation_Change_Analysis.m diff --git a/GUIs/Elevation_Change_Analysis/example_data/WH1_Z_50cm_UTM54S_LAT_p.asc b/zzz_old/to_sort/GUIs/Elevation_Change_Analysis/example_data/WH1_Z_50cm_UTM54S_LAT_p.asc similarity index 100% rename from GUIs/Elevation_Change_Analysis/example_data/WH1_Z_50cm_UTM54S_LAT_p.asc rename to zzz_old/to_sort/GUIs/Elevation_Change_Analysis/example_data/WH1_Z_50cm_UTM54S_LAT_p.asc diff --git a/GUIs/Elevation_Change_Analysis/example_data/WH1_uncertaintyZ_50cm_UTM54S_p.asc b/zzz_old/to_sort/GUIs/Elevation_Change_Analysis/example_data/WH1_uncertaintyZ_50cm_UTM54S_p.asc similarity index 100% rename from GUIs/Elevation_Change_Analysis/example_data/WH1_uncertaintyZ_50cm_UTM54S_p.asc rename to zzz_old/to_sort/GUIs/Elevation_Change_Analysis/example_data/WH1_uncertaintyZ_50cm_UTM54S_p.asc diff --git a/GUIs/Elevation_Change_Analysis/example_data/WH2_Z_50cm_UTM54S_LAT_p.asc b/zzz_old/to_sort/GUIs/Elevation_Change_Analysis/example_data/WH2_Z_50cm_UTM54S_LAT_p.asc similarity index 100% rename from GUIs/Elevation_Change_Analysis/example_data/WH2_Z_50cm_UTM54S_LAT_p.asc rename to zzz_old/to_sort/GUIs/Elevation_Change_Analysis/example_data/WH2_Z_50cm_UTM54S_LAT_p.asc diff --git a/GUIs/Elevation_Change_Analysis/example_data/WH2_uncertaintyZ_50cm_UTM54S_p.asc b/zzz_old/to_sort/GUIs/Elevation_Change_Analysis/example_data/WH2_uncertaintyZ_50cm_UTM54S_p.asc similarity index 100% rename from GUIs/Elevation_Change_Analysis/example_data/WH2_uncertaintyZ_50cm_UTM54S_p.asc rename to zzz_old/to_sort/GUIs/Elevation_Change_Analysis/example_data/WH2_uncertaintyZ_50cm_UTM54S_p.asc diff --git a/GUIs/Elevation_Change_Analysis/screenshot_1.png b/zzz_old/to_sort/GUIs/Elevation_Change_Analysis/screenshot_1.png similarity index 100% rename from GUIs/Elevation_Change_Analysis/screenshot_1.png rename to zzz_old/to_sort/GUIs/Elevation_Change_Analysis/screenshot_1.png diff --git a/GUIs/Elevation_Change_Analysis/screenshot_2.png b/zzz_old/to_sort/GUIs/Elevation_Change_Analysis/screenshot_2.png similarity index 100% rename from GUIs/Elevation_Change_Analysis/screenshot_2.png rename to zzz_old/to_sort/GUIs/Elevation_Change_Analysis/screenshot_2.png diff --git a/functions/bathymetry/CFF_get_vector_stepsize.m b/zzz_old/to_sort/bathymetry/CFF_get_vector_stepsize.m similarity index 100% rename from functions/bathymetry/CFF_get_vector_stepsize.m rename to zzz_old/to_sort/bathymetry/CFF_get_vector_stepsize.m diff --git a/functions/bathymetry/CFF_slope.m b/zzz_old/to_sort/bathymetry/CFF_slope.m similarity index 100% rename from functions/bathymetry/CFF_slope.m rename to zzz_old/to_sort/bathymetry/CFF_slope.m diff --git a/functions/bathymetry/CFF_weightgrid.m b/zzz_old/to_sort/bathymetry/CFF_weightgrid.m similarity index 100% rename from functions/bathymetry/CFF_weightgrid.m rename to zzz_old/to_sort/bathymetry/CFF_weightgrid.m diff --git a/functions/display/CFF_display_structure.m b/zzz_old/to_sort/display/CFF_display_structure.m similarity index 100% rename from functions/display/CFF_display_structure.m rename to zzz_old/to_sort/display/CFF_display_structure.m diff --git a/functions/display/CFF_nice_easting_northing.m b/zzz_old/to_sort/display/CFF_nice_easting_northing.m similarity index 100% rename from functions/display/CFF_nice_easting_northing.m rename to zzz_old/to_sort/display/CFF_nice_easting_northing.m diff --git a/functions/files_and_folders/CFF_check_file.m b/zzz_old/to_sort/files_and_folders/CFF_check_file.m similarity index 100% rename from functions/files_and_folders/CFF_check_file.m rename to zzz_old/to_sort/files_and_folders/CFF_check_file.m diff --git a/functions/files_and_folders/CFF_check_folder.m b/zzz_old/to_sort/files_and_folders/CFF_check_folder.m similarity index 100% rename from functions/files_and_folders/CFF_check_folder.m rename to zzz_old/to_sort/files_and_folders/CFF_check_folder.m diff --git a/functions/files_and_folders/CFF_closest_existing_folder.m b/zzz_old/to_sort/files_and_folders/CFF_closest_existing_folder.m similarity index 100% rename from functions/files_and_folders/CFF_closest_existing_folder.m rename to zzz_old/to_sort/files_and_folders/CFF_closest_existing_folder.m diff --git a/functions/files_and_folders/CFF_correct_filesep.m b/zzz_old/to_sort/files_and_folders/CFF_correct_filesep.m similarity index 100% rename from functions/files_and_folders/CFF_correct_filesep.m rename to zzz_old/to_sort/files_and_folders/CFF_correct_filesep.m diff --git a/functions/files_and_folders/CFF_default_mat_filename.m b/zzz_old/to_sort/files_and_folders/CFF_default_mat_filename.m similarity index 100% rename from functions/files_and_folders/CFF_default_mat_filename.m rename to zzz_old/to_sort/files_and_folders/CFF_default_mat_filename.m diff --git a/functions/files_and_folders/CFF_file_extension.m b/zzz_old/to_sort/files_and_folders/CFF_file_extension.m similarity index 100% rename from functions/files_and_folders/CFF_file_extension.m rename to zzz_old/to_sort/files_and_folders/CFF_file_extension.m diff --git a/functions/files_and_folders/CFF_files_with_wanted_extension.m b/zzz_old/to_sort/files_and_folders/CFF_files_with_wanted_extension.m similarity index 100% rename from functions/files_and_folders/CFF_files_with_wanted_extension.m rename to zzz_old/to_sort/files_and_folders/CFF_files_with_wanted_extension.m diff --git a/functions/files_and_folders/CFF_full_path.m b/zzz_old/to_sort/files_and_folders/CFF_full_path.m similarity index 100% rename from functions/files_and_folders/CFF_full_path.m rename to zzz_old/to_sort/files_and_folders/CFF_full_path.m diff --git a/functions/gis_DEM_analysis/CFF_DEM_transect.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_DEM_transect.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_DEM_transect.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_DEM_transect.m diff --git a/functions/gis_DEM_analysis/CFF_DOD_analysis.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_DOD_analysis.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_DOD_analysis.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_DOD_analysis.m diff --git a/functions/gis_DEM_analysis/CFF_DOD_spike_filter.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_DOD_spike_filter.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_DOD_spike_filter.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_DOD_spike_filter.m diff --git a/functions/gis_DEM_analysis/CFF_LOD_analysis.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_LOD_analysis.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_LOD_analysis.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_LOD_analysis.m diff --git a/functions/gis_DEM_analysis/CFF_LOD_volumes.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_LOD_volumes.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_LOD_volumes.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_LOD_volumes.m diff --git a/functions/gis_DEM_analysis/CFF_beach_LOD_analysis.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_beach_LOD_analysis.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_beach_LOD_analysis.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_beach_LOD_analysis.m diff --git a/functions/gis_DEM_analysis/CFF_calculate_DOD.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_calculate_DOD.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_calculate_DOD.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_calculate_DOD.m diff --git a/functions/gis_DEM_analysis/CFF_calculate_DPU.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_calculate_DPU.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_calculate_DPU.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_calculate_DPU.m diff --git a/functions/gis_DEM_analysis/CFF_display_LoD_volumes.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_display_LoD_volumes.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_display_LoD_volumes.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_display_LoD_volumes.m diff --git a/functions/gis_DEM_analysis/CFF_volume_interval_analysis.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_volume_interval_analysis.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_volume_interval_analysis.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_volume_interval_analysis.m diff --git a/functions/gis_DEM_analysis/CFF_write_DOD_asc.m b/zzz_old/to_sort/gis_DEM_analysis/CFF_write_DOD_asc.m similarity index 100% rename from functions/gis_DEM_analysis/CFF_write_DOD_asc.m rename to zzz_old/to_sort/gis_DEM_analysis/CFF_write_DOD_asc.m diff --git a/functions/gis_input_output/CFF_load_raster.m b/zzz_old/to_sort/gis_input_output/CFF_load_raster.m similarity index 100% rename from functions/gis_input_output/CFF_load_raster.m rename to zzz_old/to_sort/gis_input_output/CFF_load_raster.m diff --git a/functions/gis_input_output/CFF_num_rasters.m b/zzz_old/to_sort/gis_input_output/CFF_num_rasters.m similarity index 100% rename from functions/gis_input_output/CFF_num_rasters.m rename to zzz_old/to_sort/gis_input_output/CFF_num_rasters.m diff --git a/functions/gis_input_output/CFF_read_arcmap_table_XY.m b/zzz_old/to_sort/gis_input_output/CFF_read_arcmap_table_XY.m similarity index 100% rename from functions/gis_input_output/CFF_read_arcmap_table_XY.m rename to zzz_old/to_sort/gis_input_output/CFF_read_arcmap_table_XY.m diff --git a/functions/gis_input_output/CFF_read_asc.m b/zzz_old/to_sort/gis_input_output/CFF_read_asc.m similarity index 100% rename from functions/gis_input_output/CFF_read_asc.m rename to zzz_old/to_sort/gis_input_output/CFF_read_asc.m diff --git a/functions/gis_input_output/CFF_read_dlm_grid.m b/zzz_old/to_sort/gis_input_output/CFF_read_dlm_grid.m similarity index 100% rename from functions/gis_input_output/CFF_read_dlm_grid.m rename to zzz_old/to_sort/gis_input_output/CFF_read_dlm_grid.m diff --git a/functions/gis_input_output/CFF_read_tfw.m b/zzz_old/to_sort/gis_input_output/CFF_read_tfw.m similarity index 100% rename from functions/gis_input_output/CFF_read_tfw.m rename to zzz_old/to_sort/gis_input_output/CFF_read_tfw.m diff --git a/functions/gis_input_output/CFF_read_tif.m b/zzz_old/to_sort/gis_input_output/CFF_read_tif.m similarity index 100% rename from functions/gis_input_output/CFF_read_tif.m rename to zzz_old/to_sort/gis_input_output/CFF_read_tif.m diff --git a/functions/gis_input_output/CFF_readtable.m b/zzz_old/to_sort/gis_input_output/CFF_readtable.m similarity index 100% rename from functions/gis_input_output/CFF_readtable.m rename to zzz_old/to_sort/gis_input_output/CFF_readtable.m diff --git a/functions/gis_input_output/CFF_write_asc.m b/zzz_old/to_sort/gis_input_output/CFF_write_asc.m similarity index 100% rename from functions/gis_input_output/CFF_write_asc.m rename to zzz_old/to_sort/gis_input_output/CFF_write_asc.m diff --git a/functions/gis_input_output/CFF_write_tif.m b/zzz_old/to_sort/gis_input_output/CFF_write_tif.m similarity index 100% rename from functions/gis_input_output/CFF_write_tif.m rename to zzz_old/to_sort/gis_input_output/CFF_write_tif.m diff --git a/zzz_old/to_sort/gis_raster/CFF_clip_raster.m b/zzz_old/to_sort/gis_raster/CFF_clip_raster.m new file mode 100644 index 0000000..b04f300 --- /dev/null +++ b/zzz_old/to_sort/gis_raster/CFF_clip_raster.m @@ -0,0 +1,67 @@ +function [Z2,X2,Y2] = CFF_clip_raster(Z,X,Y,xv,yv) +% [Z2,X2,Y2] = CFF_clip_raster(Z,X,Y,xv,yv) +% +% DESCRIPTION +% +% clip raster Z with coordinates X,Y to the polygon of vertices xv,yv. All +% grid points outside the polygon are set to NaN and then remove +% unnecessary rows and columns. +% +% USE +% +% ... +% +% PROCESSING SUMMARY +% +% - ... +% - ... +% - ... +% +% INPUT VARIABLES +% +% - varagin +% +% OUTPUT VARIABLES +% +% - NA +% +% RESEARCH NOTES +% +% ... +% +% NEW FEATURES +% +% YYYY-MM-DD: second version. +% YYYY-MM-DD: first version. +% +% EXAMPLE +% +%%% +% Alex Schimel, Deakin University +%%% + +% build mask +mask = nan(size(Z)); +temp = inpolygon(X,Y,xv,yv); +mask(temp) = 1; + +% apply mask +newZ = Z.*mask; + +% find limits of data: +dataZ = ~isnan(newZ); + +rows = double(any(dataZ,2)); +irow_beg = find(rows,1,'first'); +irow_end = find(rows,1,'last'); + +cols = double(any(dataZ,1)); +icol_beg = find(cols,1,'first'); +icol_end = find(cols,1,'last'); + +% output +Z2 = newZ(irow_beg:irow_end,icol_beg:icol_end); +X2 = X(irow_beg:irow_end,icol_beg:icol_end); +Y2 = Y(irow_beg:irow_end,icol_beg:icol_end); + + diff --git a/zzz_old/to_sort/gis_raster/CFF_grid.m b/zzz_old/to_sort/gis_raster/CFF_grid.m new file mode 100644 index 0000000..948d167 --- /dev/null +++ b/zzz_old/to_sort/gis_raster/CFF_grid.m @@ -0,0 +1,86 @@ +function [XX,YY,ZZ] = CFF_grid(x,y,z,res,w) +% [XX,YY,ZZ] = CFF_grid(x,y,z,res,w) +% +% DESCRIPTION +% +% Grid data points (x,y,z) with weights w, at resolution res. +% +% USE +% +% ... +% +% PROCESSING SUMMARY +% +% - ... +% - ... +% - ... +% +% INPUT VARIABLES +% +% - x,y,z: data vectors to be gridded +% - res: grid resolution (in m) +% - w (optional): weight vector for each data point. If not in input, 1 is +% used for each data point (arithmetic mean computation) +% +% OUTPUT VARIABLES +% +% - XX,YY,ZZ: gridded data +% +% RESEARCH NOTES +% +% ... +% +% NEW FEATURES +% +% 2014-10-13: first version. +% +% EXAMPLE +% +% ... +% +%%% +% Alex Schimel, Deakin University +%%% + +if nargin<5 + w = ones(size(x)); +end + +% find grid boundaries +minY = min(y(:)); +maxY = max(y(:)); +minX = min(x(:)); +maxX = max(x(:)); + +% set grid vectors +XX = [floor(minX):res:ceil(maxX)]; +YY = [floor(minY):res:ceil(maxY)]'; + +% initialize the running count, the running weighted sum, and the running +% sum of weights: +% N = zeros(length(YY),length(XX)); +WS = zeros(length(YY),length(XX)); +WW = zeros(length(YY),length(XX)); + +% weight gridding +for ii = 1:length(x(:)) + + if ~isnan(x(ii)) + + % get grid cell index to which this data point will contribute + iR = round(((y(ii)-YY(1))./res)+1); + iC = round(((x(ii)-XX(1))./res)+1); + + % calculate new values with added point + % N(iR,iC) = N(iR,iC) + 1; + WS(iR,iC) = WS(iR,iC) + w(ii).*z(ii); + WW(iR,iC) = WW(iR,iC) + w(ii); + + end +end + +% at the end, divide the weighted sum by the sum of weights to get the +% weighted average values +ZZ = WS./WW; + + diff --git a/zzz_old/to_sort/gis_raster/CFF_interpolate_grid.m b/zzz_old/to_sort/gis_raster/CFF_interpolate_grid.m new file mode 100644 index 0000000..4745572 --- /dev/null +++ b/zzz_old/to_sort/gis_raster/CFF_interpolate_grid.m @@ -0,0 +1,161 @@ +function Zout = CFF_interpolate_grid(X,Y,Z,strel_size) +% Zout = CFF_interpolate_grid(X,Y,Z,strel_size) +% +% DESCRIPTION +% +% Interpolate depth image +% +% USE +% +% ... +% +% PROCESSING SUMMARY +% +% - ... +% - ... +% - ... +% +% INPUT VARIABLES +% +% - varagin +% +% OUTPUT VARIABLES +% +% - NA +% +% RESEARCH NOTES +% +% - Using inpaint_nans (from John d'Erico, found on Mathworks). +% +% - This function separates the full dataset into smaller ones for +% interpolation. This allows not wasting time interpolating vast expenses +% of areas without data and avoid running out on memory (inpaint_nans is a +% glutton). +% +% - For later, build here my second approach idea of filling in cells with +% 8 neighbours, then 7, etc. +% +% NEW FEATURES +% +% 2014-10-13: first version. +% +% EXAMPLE +% +% ... +% +%%% +% Alex Schimel, Deakin University +%%% + +% small grids parameters +nNo = 10; % number of northing (rows) splits +nEa = 10; % number of easting (columns) splits +extra = 10; % number of rows and columns of data we add to the split values + % this allows the extrapolation process to take into account + % extra data on the boundaries although the results past the + % boundaries will be later discarded + +% create the arrays containing the indices of each small grids +% (DTMminE, DTMmaxE, DTMminN, DTMmaxN): +N = ceil(length(Y)./nNo); +E = ceil(length(X)./nEa); + +temp = [1:N:length(Y)]'; +temp = temp*ones(1,nEa); +fine(:,1) = reshape(temp', size(temp,1).*size(temp,2),1); + +temp = [1:N:length(Y)]'+N-1; +temp(end) = length(Y); +temp = temp*ones(1,nEa); +fine(:,2) = reshape(temp', size(temp,1).*size(temp,2),1); + +temp = [1:E:length(X)]'; +temp = temp*ones(1,nNo); +fine(:,3) = reshape(temp, size(temp,1).*size(temp,2),1); + +temp = [1:E:length(X)]'+E-1; +temp(end) = length(X); +temp = temp*ones(1,nNo); +fine(:,4) = reshape(temp, size(temp,1).*size(temp,2),1); + +% create the arrays containing the indices of each small grids + extra for +% the Interpolation +fine2 = fine; +fine2(:,1) = fine(:,1)-extra; +fine2(:,2) = fine(:,2)+extra; +fine2(:,3) = fine(:,3)-extra; +fine2(:,4) = fine(:,4)+extra; + +fine2(fine2<1) = 1; +fine2(fine2(:,1)>length(Y),1) = length(Y); +fine2(fine2(:,2)>length(Y),2) = length(Y); +fine2(fine2(:,3)>length(X),3) = length(X); +fine2(fine2(:,4)>length(X),4) = length(X); + + +% initialize interpolated DTM +Z2 = nan(size(Z)); + +% interpolate small grids separately +for ii = 1:size(fine,1) + + % extract original data (patch+extra) + patch = Z(fine2(ii,1):fine2(ii,2),fine2(ii,3):fine2(ii,4)); + + a = find(sum(~isnan(patch)) ~=0); % list of non-empty columns + b = find(sum(~isnan(patch')) ~=0); % list of non-empty rows + + % interpolate + if ~isempty(a) + % do only if patch is not completely empty + + indexcol = [a(1):a(end)]; + indexrow = [b(1):b(end)]; + + temp = patch(indexrow,indexcol); % remove empty boundaries in patch + InterpPatch = CFF_inpaint_nans(temp,4); % interpolate that, use method specified + temp = -999.*ones(size(patch)); + temp(indexrow,indexcol) = InterpPatch; + InterpPatch = temp; % reintroduce the empty boundaries + + else + % if patch was empty, just fill it with -999. Not NaNs because + % after Interpolation, we have no NaNs left usually. + InterpPatch = -999.*ones(size(patch)); + + end + + % remove extra + InterpPatch(:,end-(fine2(ii,4)-fine(ii,4))+1:end) = []; + InterpPatch(:,1:fine(ii,3)-fine2(ii,3)) = []; + InterpPatch(end-(fine2(ii,2)-fine(ii,2))+1:end,:) = []; + InterpPatch(1:fine(ii,1)-fine2(ii,1),:) = []; + + % replace in new dataset + Z2(fine(ii,1):fine(ii,2),fine(ii,3):fine(ii,4)) = InterpPatch; + +end + +% remove -999 values +Z2(Z2==-999) = NaN; + + +%% MASK EXTRAPOLATION + +% we use the "closing" morphological image processing operation to create a +% mask to remove the extrapolation of the previous processing step. + +% adapt the structuring element to the size of the gaps to close +H = CFF_disk(strel_size); % structuring element + +Mask = Z; +Mask(~isnan(Mask)) = 1; +Mask(isnan(Mask)) = 0; + +% close +NewMask = CFF_imclose(Mask,H); + +NewMask(NewMask==0) = NaN; + +% apply Mask to interpolated DTM: +Zout = Z2.*NewMask; diff --git a/functions/gis_vector/CFF_create_circle_polygon.m b/zzz_old/to_sort/gis_vector/CFF_create_circle_polygon.m similarity index 100% rename from functions/gis_vector/CFF_create_circle_polygon.m rename to zzz_old/to_sort/gis_vector/CFF_create_circle_polygon.m diff --git a/functions/gis_vector/CFF_create_polygons_along_line.m b/zzz_old/to_sort/gis_vector/CFF_create_polygons_along_line.m similarity index 100% rename from functions/gis_vector/CFF_create_polygons_along_line.m rename to zzz_old/to_sort/gis_vector/CFF_create_polygons_along_line.m diff --git a/functions/image_processing/CFF_disk.m b/zzz_old/to_sort/image_processing/CFF_disk.m similarity index 100% rename from functions/image_processing/CFF_disk.m rename to zzz_old/to_sort/image_processing/CFF_disk.m diff --git a/functions/image_processing/CFF_imclose.m b/zzz_old/to_sort/image_processing/CFF_imclose.m similarity index 100% rename from functions/image_processing/CFF_imclose.m rename to zzz_old/to_sort/image_processing/CFF_imclose.m diff --git a/functions/image_processing/CFF_imdilate.m b/zzz_old/to_sort/image_processing/CFF_imdilate.m similarity index 100% rename from functions/image_processing/CFF_imdilate.m rename to zzz_old/to_sort/image_processing/CFF_imdilate.m diff --git a/functions/image_processing/CFF_imerode.m b/zzz_old/to_sort/image_processing/CFF_imerode.m similarity index 100% rename from functions/image_processing/CFF_imerode.m rename to zzz_old/to_sort/image_processing/CFF_imerode.m diff --git a/functions/image_processing/CFF_imfilter.m b/zzz_old/to_sort/image_processing/CFF_imfilter.m similarity index 100% rename from functions/image_processing/CFF_imfilter.m rename to zzz_old/to_sort/image_processing/CFF_imfilter.m diff --git a/functions/image_processing/CFF_imopen.m b/zzz_old/to_sort/image_processing/CFF_imopen.m similarity index 100% rename from functions/image_processing/CFF_imopen.m rename to zzz_old/to_sort/image_processing/CFF_imopen.m diff --git a/functions/image_processing/CFF_inpaint_nans.m b/zzz_old/to_sort/image_processing/CFF_inpaint_nans.m similarity index 100% rename from functions/image_processing/CFF_inpaint_nans.m rename to zzz_old/to_sort/image_processing/CFF_inpaint_nans.m diff --git a/functions/image_processing/CFF_meanfilt2.m b/zzz_old/to_sort/image_processing/CFF_meanfilt2.m similarity index 100% rename from functions/image_processing/CFF_meanfilt2.m rename to zzz_old/to_sort/image_processing/CFF_meanfilt2.m diff --git a/functions/image_processing/CFF_medfilt2.m b/zzz_old/to_sort/image_processing/CFF_medfilt2.m similarity index 100% rename from functions/image_processing/CFF_medfilt2.m rename to zzz_old/to_sort/image_processing/CFF_medfilt2.m diff --git a/functions/image_processing/CFF_spikefilt2.m b/zzz_old/to_sort/image_processing/CFF_spikefilt2.m similarity index 100% rename from functions/image_processing/CFF_spikefilt2.m rename to zzz_old/to_sort/image_processing/CFF_spikefilt2.m diff --git a/functions/image_processing/CFF_stack_offsets.m b/zzz_old/to_sort/image_processing/CFF_stack_offsets.m similarity index 100% rename from functions/image_processing/CFF_stack_offsets.m rename to zzz_old/to_sort/image_processing/CFF_stack_offsets.m diff --git a/functions/mbes_backscatter_area/CFF_annulus_chord_length.m b/zzz_old/to_sort/mbes_backscatter_area/CFF_annulus_chord_length.m similarity index 100% rename from functions/mbes_backscatter_area/CFF_annulus_chord_length.m rename to zzz_old/to_sort/mbes_backscatter_area/CFF_annulus_chord_length.m diff --git a/functions/mbes_backscatter_area/CFF_circle_chord_length.m b/zzz_old/to_sort/mbes_backscatter_area/CFF_circle_chord_length.m similarity index 100% rename from functions/mbes_backscatter_area/CFF_circle_chord_length.m rename to zzz_old/to_sort/mbes_backscatter_area/CFF_circle_chord_length.m diff --git a/functions/mbes_backscatter_area/CFF_common_beam_footprint.m b/zzz_old/to_sort/mbes_backscatter_area/CFF_common_beam_footprint.m similarity index 100% rename from functions/mbes_backscatter_area/CFF_common_beam_footprint.m rename to zzz_old/to_sort/mbes_backscatter_area/CFF_common_beam_footprint.m diff --git a/functions/mbes_backscatter_area/CFF_common_pulse_footprint.m b/zzz_old/to_sort/mbes_backscatter_area/CFF_common_pulse_footprint.m similarity index 100% rename from functions/mbes_backscatter_area/CFF_common_pulse_footprint.m rename to zzz_old/to_sort/mbes_backscatter_area/CFF_common_pulse_footprint.m diff --git a/functions/mbes_backscatter_area/CFF_exact_beam_footprint.m b/zzz_old/to_sort/mbes_backscatter_area/CFF_exact_beam_footprint.m similarity index 100% rename from functions/mbes_backscatter_area/CFF_exact_beam_footprint.m rename to zzz_old/to_sort/mbes_backscatter_area/CFF_exact_beam_footprint.m diff --git a/functions/mbes_backscatter_area/CFF_exact_pulse_footprint.m b/zzz_old/to_sort/mbes_backscatter_area/CFF_exact_pulse_footprint.m similarity index 100% rename from functions/mbes_backscatter_area/CFF_exact_pulse_footprint.m rename to zzz_old/to_sort/mbes_backscatter_area/CFF_exact_pulse_footprint.m diff --git a/functions/mbes_backscatter_area/CFF_footprint_intersection.m b/zzz_old/to_sort/mbes_backscatter_area/CFF_footprint_intersection.m similarity index 100% rename from functions/mbes_backscatter_area/CFF_footprint_intersection.m rename to zzz_old/to_sort/mbes_backscatter_area/CFF_footprint_intersection.m diff --git a/functions/mbes_backscatter_area/script_testing_footprints.m b/zzz_old/to_sort/mbes_backscatter_area/script_testing_footprints.m similarity index 100% rename from functions/mbes_backscatter_area/script_testing_footprints.m rename to zzz_old/to_sort/mbes_backscatter_area/script_testing_footprints.m diff --git a/functions/mbes_processing/CFF_compute_ping_navigation.m b/zzz_old/to_sort/mbes_processing/CFF_compute_ping_navigation.m similarity index 100% rename from functions/mbes_processing/CFF_compute_ping_navigation.m rename to zzz_old/to_sort/mbes_processing/CFF_compute_ping_navigation.m diff --git a/functions/mbes_processing/CFF_georeference_sample.m b/zzz_old/to_sort/mbes_processing/CFF_georeference_sample.m similarity index 100% rename from functions/mbes_processing/CFF_georeference_sample.m rename to zzz_old/to_sort/mbes_processing/CFF_georeference_sample.m diff --git a/functions/mbes_processing/CFF_get_samples_ENH.m b/zzz_old/to_sort/mbes_processing/CFF_get_samples_ENH.m similarity index 100% rename from functions/mbes_processing/CFF_get_samples_ENH.m rename to zzz_old/to_sort/mbes_processing/CFF_get_samples_ENH.m diff --git a/functions/mbes_processing/CFF_get_samples_dist.m b/zzz_old/to_sort/mbes_processing/CFF_get_samples_dist.m similarity index 100% rename from functions/mbes_processing/CFF_get_samples_dist.m rename to zzz_old/to_sort/mbes_processing/CFF_get_samples_dist.m diff --git a/functions/mbes_processing/CFF_get_samples_range.m b/zzz_old/to_sort/mbes_processing/CFF_get_samples_range.m similarity index 100% rename from functions/mbes_processing/CFF_get_samples_range.m rename to zzz_old/to_sort/mbes_processing/CFF_get_samples_range.m diff --git a/functions/mbes_processing/CFF_process_ping.m b/zzz_old/to_sort/mbes_processing/CFF_process_ping.m similarity index 100% rename from functions/mbes_processing/CFF_process_ping.m rename to zzz_old/to_sort/mbes_processing/CFF_process_ping.m diff --git a/functions/mbes_rawfiles/CFF_filelist_for_conversion.m b/zzz_old/to_sort/mbes_rawfiles/CFF_filelist_for_conversion.m similarity index 100% rename from functions/mbes_rawfiles/CFF_filelist_for_conversion.m rename to zzz_old/to_sort/mbes_rawfiles/CFF_filelist_for_conversion.m diff --git a/functions/mbes_rawfiles_kongsberg/CFF_all_file_info.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_all_file_info.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/CFF_all_file_info.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_all_file_info.m diff --git a/functions/mbes_rawfiles_kongsberg/CFF_check_ALLfilename.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_check_ALLfilename.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/CFF_check_ALLfilename.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_check_ALLfilename.m diff --git a/functions/mbes_rawfiles_kongsberg/CFF_convert_ALLdata_to_fData.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_convert_ALLdata_to_fData.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/CFF_convert_ALLdata_to_fData.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_convert_ALLdata_to_fData.m diff --git a/functions/mbes_rawfiles_kongsberg/CFF_get_Kongsberg_files.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_get_Kongsberg_files.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/CFF_get_Kongsberg_files.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_get_Kongsberg_files.m diff --git a/functions/mbes_rawfiles_kongsberg/CFF_is_Kongsberg_file.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_is_Kongsberg_file.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/CFF_is_Kongsberg_file.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_is_Kongsberg_file.m diff --git a/functions/mbes_rawfiles_kongsberg/CFF_list_files_in_dir.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_list_files_in_dir.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/CFF_list_files_in_dir.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_list_files_in_dir.m diff --git a/functions/mbes_rawfiles_kongsberg/CFF_read_all_from_fileinfo.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_read_all_from_fileinfo.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/CFF_read_all_from_fileinfo.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/CFF_read_all_from_fileinfo.m diff --git a/functions/mbes_rawfiles_kongsberg/obs/CFF_all_file_info.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_all_file_info.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/obs/CFF_all_file_info.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_all_file_info.m diff --git a/functions/mbes_rawfiles_kongsberg/obs/CFF_convert_all_to_mat.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_convert_all_to_mat.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/obs/CFF_convert_all_to_mat.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_convert_all_to_mat.m diff --git a/functions/mbes_rawfiles_kongsberg/obs/CFF_convert_all_to_mat_v2.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_convert_all_to_mat_v2.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/obs/CFF_convert_all_to_mat_v2.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_convert_all_to_mat_v2.m diff --git a/functions/mbes_rawfiles_kongsberg/obs/CFF_convert_mat_to_fabc.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_convert_mat_to_fabc.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/obs/CFF_convert_mat_to_fabc.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_convert_mat_to_fabc.m diff --git a/functions/mbes_rawfiles_kongsberg/obs/CFF_read_all.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_read_all.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/obs/CFF_read_all.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_read_all.m diff --git a/functions/mbes_rawfiles_kongsberg/obs/CFF_read_all_from_fileinfo.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_read_all_from_fileinfo.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/obs/CFF_read_all_from_fileinfo.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_read_all_from_fileinfo.m diff --git a/functions/mbes_rawfiles_kongsberg/obs/CFF_save_mat_from_all.m b/zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_save_mat_from_all.m similarity index 100% rename from functions/mbes_rawfiles_kongsberg/obs/CFF_save_mat_from_all.m rename to zzz_old/to_sort/mbes_rawfiles_kongsberg/obs/CFF_save_mat_from_all.m diff --git a/functions/mbes_rawfiles_triton/CFF_convert_xtf_to_all.m b/zzz_old/to_sort/mbes_rawfiles_triton/CFF_convert_xtf_to_all.m similarity index 100% rename from functions/mbes_rawfiles_triton/CFF_convert_xtf_to_all.m rename to zzz_old/to_sort/mbes_rawfiles_triton/CFF_convert_xtf_to_all.m diff --git a/functions/mbes_watercolumn/CFF_filter_WC_bottom_detect.m b/zzz_old/to_sort/mbes_watercolumn/CFF_filter_WC_bottom_detect.m similarity index 100% rename from functions/mbes_watercolumn/CFF_filter_WC_bottom_detect.m rename to zzz_old/to_sort/mbes_watercolumn/CFF_filter_WC_bottom_detect.m diff --git a/functions/mbes_watercolumn/CFF_filter_WC_sidelobe_artifact.m b/zzz_old/to_sort/mbes_watercolumn/CFF_filter_WC_sidelobe_artifact.m similarity index 100% rename from functions/mbes_watercolumn/CFF_filter_WC_sidelobe_artifact.m rename to zzz_old/to_sort/mbes_watercolumn/CFF_filter_WC_sidelobe_artifact.m diff --git a/functions/mbes_watercolumn/CFF_georeference_WC_bottom_detect.m b/zzz_old/to_sort/mbes_watercolumn/CFF_georeference_WC_bottom_detect.m similarity index 100% rename from functions/mbes_watercolumn/CFF_georeference_WC_bottom_detect.m rename to zzz_old/to_sort/mbes_watercolumn/CFF_georeference_WC_bottom_detect.m diff --git a/functions/mbes_watercolumn/CFF_get_WC_data.m b/zzz_old/to_sort/mbes_watercolumn/CFF_get_WC_data.m similarity index 100% rename from functions/mbes_watercolumn/CFF_get_WC_data.m rename to zzz_old/to_sort/mbes_watercolumn/CFF_get_WC_data.m diff --git a/functions/mbes_watercolumn/CFF_grid_WC_data.m b/zzz_old/to_sort/mbes_watercolumn/CFF_grid_WC_data.m similarity index 100% rename from functions/mbes_watercolumn/CFF_grid_WC_data.m rename to zzz_old/to_sort/mbes_watercolumn/CFF_grid_WC_data.m diff --git a/functions/mbes_watercolumn/CFF_initialize_WC_processing.m b/zzz_old/to_sort/mbes_watercolumn/CFF_initialize_WC_processing.m similarity index 100% rename from functions/mbes_watercolumn/CFF_initialize_WC_processing.m rename to zzz_old/to_sort/mbes_watercolumn/CFF_initialize_WC_processing.m diff --git a/functions/mbes_watercolumn/CFF_mask_WC_data.m b/zzz_old/to_sort/mbes_watercolumn/CFF_mask_WC_data.m similarity index 100% rename from functions/mbes_watercolumn/CFF_mask_WC_data.m rename to zzz_old/to_sort/mbes_watercolumn/CFF_mask_WC_data.m diff --git a/functions/mbes_watercolumn/CFF_watercolumn_display.m b/zzz_old/to_sort/mbes_watercolumn/CFF_watercolumn_display.m similarity index 100% rename from functions/mbes_watercolumn/CFF_watercolumn_display.m rename to zzz_old/to_sort/mbes_watercolumn/CFF_watercolumn_display.m diff --git a/functions/mbes_watercolumn/obs/CFF_XYZtoPBS.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_XYZtoPBS.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_XYZtoPBS.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_XYZtoPBS.m diff --git a/functions/mbes_watercolumn/obs/CFF_calculate_ratio_area_insonified.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_calculate_ratio_area_insonified.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_calculate_ratio_area_insonified.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_calculate_ratio_area_insonified.m diff --git a/functions/mbes_watercolumn/obs/CFF_compute_watercolumn_energy.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_compute_watercolumn_energy.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_compute_watercolumn_energy.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_compute_watercolumn_energy.m diff --git a/functions/mbes_watercolumn/obs/CFF_filter_WC_bottom_detect.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_filter_WC_bottom_detect.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_filter_WC_bottom_detect.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_filter_WC_bottom_detect.m diff --git a/functions/mbes_watercolumn/obs/CFF_filter_WC_sidelobe_artifact.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_filter_WC_sidelobe_artifact.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_filter_WC_sidelobe_artifact.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_filter_WC_sidelobe_artifact.m diff --git a/functions/mbes_watercolumn/obs/CFF_filter_watercolumn.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_filter_watercolumn.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_filter_watercolumn.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_filter_watercolumn.m diff --git a/functions/mbes_watercolumn/obs/CFF_find_kelp.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_find_kelp.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_find_kelp.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_find_kelp.m diff --git a/functions/mbes_watercolumn/obs/CFF_grid_WC_bottom_detect.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_grid_WC_bottom_detect.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_grid_WC_bottom_detect.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_grid_WC_bottom_detect.m diff --git a/functions/mbes_watercolumn/obs/CFF_grid_watercolumn.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_grid_watercolumn.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_grid_watercolumn.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_grid_watercolumn.m diff --git a/functions/mbes_watercolumn/obs/CFF_mask_WC_data.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_mask_WC_data.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_mask_WC_data.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_mask_WC_data.m diff --git a/functions/mbes_watercolumn/obs/CFF_process_WC_bottom_detect.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_process_WC_bottom_detect.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_process_WC_bottom_detect.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_process_WC_bottom_detect.m diff --git a/functions/mbes_watercolumn/obs/CFF_process_watercolumn.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_process_watercolumn.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_process_watercolumn.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_process_watercolumn.m diff --git a/functions/mbes_watercolumn/obs/CFF_restrict_gridded_watercolumn.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_restrict_gridded_watercolumn.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_restrict_gridded_watercolumn.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_restrict_gridded_watercolumn.m diff --git a/functions/mbes_watercolumn/obs/CFF_split_WC_by_position_with_locations.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_split_WC_by_position_with_locations.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_split_WC_by_position_with_locations.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_split_WC_by_position_with_locations.m diff --git a/functions/mbes_watercolumn/obs/CFF_watercolumn_sidelobe_filter.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_watercolumn_sidelobe_filter.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_watercolumn_sidelobe_filter.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_watercolumn_sidelobe_filter.m diff --git a/functions/mbes_watercolumn/obs/CFF_watercolumn_stats.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_watercolumn_stats.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_watercolumn_stats.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_watercolumn_stats.m diff --git a/functions/mbes_watercolumn/obs/CFF_weightgrid3.m b/zzz_old/to_sort/mbes_watercolumn/obs/CFF_weightgrid3.m similarity index 100% rename from functions/mbes_watercolumn/obs/CFF_weightgrid3.m rename to zzz_old/to_sort/mbes_watercolumn/obs/CFF_weightgrid3.m diff --git a/functions/misc/CFF_circle.m b/zzz_old/to_sort/misc/CFF_circle.m similarity index 100% rename from functions/misc/CFF_circle.m rename to zzz_old/to_sort/misc/CFF_circle.m diff --git a/functions/misc/CFF_inputparser_test.m b/zzz_old/to_sort/misc/CFF_inputparser_test.m similarity index 100% rename from functions/misc/CFF_inputparser_test.m rename to zzz_old/to_sort/misc/CFF_inputparser_test.m diff --git a/functions/misc/CFF_nanfull.m b/zzz_old/to_sort/misc/CFF_nanfull.m similarity index 100% rename from functions/misc/CFF_nanfull.m rename to zzz_old/to_sort/misc/CFF_nanfull.m diff --git a/functions/misc/CFF_nanfunc3.m b/zzz_old/to_sort/misc/CFF_nanfunc3.m similarity index 100% rename from functions/misc/CFF_nanfunc3.m rename to zzz_old/to_sort/misc/CFF_nanfunc3.m diff --git a/functions/misc/CFF_nanindexable.m b/zzz_old/to_sort/misc/CFF_nanindexable.m similarity index 100% rename from functions/misc/CFF_nanindexable.m rename to zzz_old/to_sort/misc/CFF_nanindexable.m diff --git a/functions/misc/CFF_nanmedian3.m b/zzz_old/to_sort/misc/CFF_nanmedian3.m similarity index 100% rename from functions/misc/CFF_nanmedian3.m rename to zzz_old/to_sort/misc/CFF_nanmedian3.m diff --git a/functions/misc/CFF_nanstat3.m b/zzz_old/to_sort/misc/CFF_nanstat3.m similarity index 100% rename from functions/misc/CFF_nanstat3.m rename to zzz_old/to_sort/misc/CFF_nanstat3.m diff --git a/functions/misc/CFF_nansum3.m b/zzz_old/to_sort/misc/CFF_nansum3.m similarity index 100% rename from functions/misc/CFF_nansum3.m rename to zzz_old/to_sort/misc/CFF_nansum3.m diff --git a/functions/misc/CFF_nice_sprintf.m b/zzz_old/to_sort/misc/CFF_nice_sprintf.m similarity index 100% rename from functions/misc/CFF_nice_sprintf.m rename to zzz_old/to_sort/misc/CFF_nice_sprintf.m diff --git a/functions/navigation/CFF_interpolate_nav.m b/zzz_old/to_sort/navigation/CFF_interpolate_nav.m similarity index 100% rename from functions/navigation/CFF_interpolate_nav.m rename to zzz_old/to_sort/navigation/CFF_interpolate_nav.m diff --git a/functions/navigation/CFF_unwrap_heading.m b/zzz_old/to_sort/navigation/CFF_unwrap_heading.m similarity index 100% rename from functions/navigation/CFF_unwrap_heading.m rename to zzz_old/to_sort/navigation/CFF_unwrap_heading.m diff --git a/functions/statistics/CFF_critical_z_value.m b/zzz_old/to_sort/statistics/CFF_critical_z_value.m similarity index 100% rename from functions/statistics/CFF_critical_z_value.m rename to zzz_old/to_sort/statistics/CFF_critical_z_value.m diff --git a/functions/statistics/CFF_invpercentile.m b/zzz_old/to_sort/statistics/CFF_invpercentile.m similarity index 100% rename from functions/statistics/CFF_invpercentile.m rename to zzz_old/to_sort/statistics/CFF_invpercentile.m diff --git a/functions/statistics/CFF_percentile.m b/zzz_old/to_sort/statistics/CFF_percentile.m similarity index 100% rename from functions/statistics/CFF_percentile.m rename to zzz_old/to_sort/statistics/CFF_percentile.m diff --git a/functions/third_party_code/Che_Hasan_2014/example_Proc_file/Proc_cmst_200511061119.mat b/zzz_old/to_sort/third_party_code/Che_Hasan_2014/example_Proc_file/Proc_cmst_200511061119.mat similarity index 100% rename from functions/third_party_code/Che_Hasan_2014/example_Proc_file/Proc_cmst_200511061119.mat rename to zzz_old/to_sort/third_party_code/Che_Hasan_2014/example_Proc_file/Proc_cmst_200511061119.mat diff --git a/functions/third_party_code/Che_Hasan_2014/inpoly_plosone_v1.m b/zzz_old/to_sort/third_party_code/Che_Hasan_2014/inpoly_plosone_v1.m similarity index 100% rename from functions/third_party_code/Che_Hasan_2014/inpoly_plosone_v1.m rename to zzz_old/to_sort/third_party_code/Che_Hasan_2014/inpoly_plosone_v1.m diff --git a/functions/third_party_code/Che_Hasan_2014/shape_read.m b/zzz_old/to_sort/third_party_code/Che_Hasan_2014/shape_read.m similarity index 100% rename from functions/third_party_code/Che_Hasan_2014/shape_read.m rename to zzz_old/to_sort/third_party_code/Che_Hasan_2014/shape_read.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/MinBound.pdf b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/MinBound.pdf similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/MinBound.pdf rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/MinBound.pdf diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/MinBound.tex b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/MinBound.tex similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/MinBound.tex rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/MinBound.tex diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/convhull.pdf b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/convhull.pdf similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/convhull.pdf rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/convhull.pdf diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/incircle.jpg b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/incircle.jpg similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/incircle.jpg rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/incircle.jpg diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/incircle.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/incircle.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/incircle.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/incircle.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/insphere.jpg b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/insphere.jpg similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/insphere.jpg rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/insphere.jpg diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/insphere.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/insphere.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/insphere.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/insphere.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundcircle.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundcircle.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundcircle.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundcircle.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/minbounddemo.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minbounddemo.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/minbounddemo.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minbounddemo.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundparallelogram.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundparallelogram.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundparallelogram.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundparallelogram.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundquad.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundquad.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundquad.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundquad.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundrect.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundrect.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundrect.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundrect.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundsemicircle.jpg b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundsemicircle.jpg similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundsemicircle.jpg rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundsemicircle.jpg diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundsemicircle.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundsemicircle.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundsemicircle.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundsemicircle.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundsphere.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundsphere.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundsphere.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundsphere.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundtri.m b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundtri.m similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/minboundtri.m rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/minboundtri.m diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/mincircle.jpg b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/mincircle.jpg similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/mincircle.jpg rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/mincircle.jpg diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/rect.jpg b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/rect.jpg similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/rect.jpg rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/rect.jpg diff --git a/functions/third_party_code/MinBoundSuite/MinBoundSuite/tri.jpg b/zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/tri.jpg similarity index 100% rename from functions/third_party_code/MinBoundSuite/MinBoundSuite/tri.jpg rename to zzz_old/to_sort/third_party_code/MinBoundSuite/MinBoundSuite/tri.jpg diff --git a/functions/third_party_code/MinBoundSuite/license.txt b/zzz_old/to_sort/third_party_code/MinBoundSuite/license.txt similarity index 100% rename from functions/third_party_code/MinBoundSuite/license.txt rename to zzz_old/to_sort/third_party_code/MinBoundSuite/license.txt diff --git a/functions/third_party_code/ascgrid/README.md b/zzz_old/to_sort/third_party_code/ascgrid/README.md similarity index 100% rename from functions/third_party_code/ascgrid/README.md rename to zzz_old/to_sort/third_party_code/ascgrid/README.md diff --git a/functions/third_party_code/ascgrid/ascgrid.m b/zzz_old/to_sort/third_party_code/ascgrid/ascgrid.m similarity index 100% rename from functions/third_party_code/ascgrid/ascgrid.m rename to zzz_old/to_sort/third_party_code/ascgrid/ascgrid.m diff --git a/functions/third_party_code/ascgrid/ascread.m b/zzz_old/to_sort/third_party_code/ascgrid/ascread.m similarity index 100% rename from functions/third_party_code/ascgrid/ascread.m rename to zzz_old/to_sort/third_party_code/ascgrid/ascread.m diff --git a/functions/third_party_code/ascgrid/ascwrite.m b/zzz_old/to_sort/third_party_code/ascgrid/ascwrite.m similarity index 100% rename from functions/third_party_code/ascgrid/ascwrite.m rename to zzz_old/to_sort/third_party_code/ascgrid/ascwrite.m diff --git a/functions/third_party_code/ascgrid/pad.m b/zzz_old/to_sort/third_party_code/ascgrid/pad.m similarity index 100% rename from functions/third_party_code/ascgrid/pad.m rename to zzz_old/to_sort/third_party_code/ascgrid/pad.m diff --git a/functions/third_party_code/extrema/extrema.m b/zzz_old/to_sort/third_party_code/extrema/extrema.m similarity index 100% rename from functions/third_party_code/extrema/extrema.m rename to zzz_old/to_sort/third_party_code/extrema/extrema.m diff --git a/functions/third_party_code/extrema/extrema2.m b/zzz_old/to_sort/third_party_code/extrema/extrema2.m similarity index 100% rename from functions/third_party_code/extrema/extrema2.m rename to zzz_old/to_sort/third_party_code/extrema/extrema2.m diff --git a/functions/third_party_code/geom3d/geom3d-demos/demoDrawTubularMesh.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/demoDrawTubularMesh.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/demoDrawTubularMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/demoDrawTubularMesh.m diff --git a/functions/third_party_code/geom3d/geom3d-demos/demoGeom3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/demoGeom3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/demoGeom3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/demoGeom3d.m diff --git a/functions/third_party_code/geom3d/geom3d-demos/demoInertiaEllipsoid.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/demoInertiaEllipsoid.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/demoInertiaEllipsoid.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/demoInertiaEllipsoid.m diff --git a/functions/third_party_code/geom3d/geom3d-demos/demoRevolutionSurface.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/demoRevolutionSurface.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/demoRevolutionSurface.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/demoRevolutionSurface.m diff --git a/functions/third_party_code/geom3d/geom3d-demos/drawSoccerBall.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/drawSoccerBall.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/drawSoccerBall.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/drawSoccerBall.m diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh.html b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh.html similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh.html rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh.html diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_01.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_01.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_01.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_01.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_02.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_02.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_02.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_02.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_03.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_03.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_03.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_03.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_04.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_04.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_04.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoDrawTubularMesh_04.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d.html b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d.html similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d.html rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d.html diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_01.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_01.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_01.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_01.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_02.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_02.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_02.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_02.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_03.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_03.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_03.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoGeom3d_03.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid.html b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid.html similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid.html rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid.html diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid_01.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid_01.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid_01.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid_01.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid_02.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid_02.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid_02.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoInertiaEllipsoid_02.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface.html b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface.html similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface.html rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface.html diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface_01.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface_01.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface_01.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface_01.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface_02.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface_02.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface_02.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/demoRevolutionSurface_02.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall.html b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall.html similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall.html rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall.html diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_01.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_01.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_01.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_01.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_02.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_02.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_02.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_02.png diff --git a/functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_03.png b/zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_03.png similarity index 100% rename from functions/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_03.png rename to zzz_old/to_sort/third_party_code/geom3d/geom3d-demos/html/drawSoccerBall_03.png diff --git a/functions/third_party_code/geom3d/geom3d/Contents.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/Contents.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/Contents.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/Contents.m diff --git a/functions/third_party_code/geom3d/geom3d/anglePoints3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/anglePoints3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/anglePoints3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/anglePoints3d.m diff --git a/functions/third_party_code/geom3d/geom3d/angleSort3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/angleSort3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/angleSort3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/angleSort3d.m diff --git a/functions/third_party_code/geom3d/geom3d/angles3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/angles3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/angles3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/angles3d.m diff --git a/functions/third_party_code/geom3d/geom3d/boundingBox3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/boundingBox3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/boundingBox3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/boundingBox3d.m diff --git a/functions/third_party_code/geom3d/geom3d/box3dVolume.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/box3dVolume.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/box3dVolume.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/box3dVolume.m diff --git a/functions/third_party_code/geom3d/geom3d/boxes3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/boxes3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/boxes3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/boxes3d.m diff --git a/functions/third_party_code/geom3d/geom3d/cart2cyl.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/cart2cyl.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/cart2cyl.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/cart2cyl.m diff --git a/functions/third_party_code/geom3d/geom3d/cart2sph2.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/cart2sph2.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/cart2sph2.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/cart2sph2.m diff --git a/functions/third_party_code/geom3d/geom3d/cart2sph2d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/cart2sph2d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/cart2sph2d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/cart2sph2d.m diff --git a/functions/third_party_code/geom3d/geom3d/changelog.txt b/zzz_old/to_sort/third_party_code/geom3d/geom3d/changelog.txt similarity index 100% rename from functions/third_party_code/geom3d/geom3d/changelog.txt rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/changelog.txt diff --git a/functions/third_party_code/geom3d/geom3d/changes.txt b/zzz_old/to_sort/third_party_code/geom3d/geom3d/changes.txt similarity index 100% rename from functions/third_party_code/geom3d/geom3d/changes.txt rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/changes.txt diff --git a/functions/third_party_code/geom3d/geom3d/circle3dOrigin.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/circle3dOrigin.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/circle3dOrigin.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/circle3dOrigin.m diff --git a/functions/third_party_code/geom3d/geom3d/circle3dPoint.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/circle3dPoint.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/circle3dPoint.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/circle3dPoint.m diff --git a/functions/third_party_code/geom3d/geom3d/circle3dPosition.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/circle3dPosition.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/circle3dPosition.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/circle3dPosition.m diff --git a/functions/third_party_code/geom3d/geom3d/circles3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/circles3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/circles3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/circles3d.m diff --git a/functions/third_party_code/geom3d/geom3d/clipConvexPolygon3dHP.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/clipConvexPolygon3dHP.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/clipConvexPolygon3dHP.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/clipConvexPolygon3dHP.m diff --git a/functions/third_party_code/geom3d/geom3d/clipLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/clipLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/clipLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/clipLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/clipPoints3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/clipPoints3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/clipPoints3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/clipPoints3d.m diff --git a/functions/third_party_code/geom3d/geom3d/clipPolygon3dHP.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/clipPolygon3dHP.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/clipPolygon3dHP.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/clipPolygon3dHP.m diff --git a/functions/third_party_code/geom3d/geom3d/composeTransforms3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/composeTransforms3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/composeTransforms3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/composeTransforms3d.m diff --git a/functions/third_party_code/geom3d/geom3d/createBasisTransform3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createBasisTransform3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createBasisTransform3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createBasisTransform3d.m diff --git a/functions/third_party_code/geom3d/geom3d/createEulerAnglesRotation.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createEulerAnglesRotation.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createEulerAnglesRotation.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createEulerAnglesRotation.m diff --git a/functions/third_party_code/geom3d/geom3d/createLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/createPlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createPlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createPlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createPlane.m diff --git a/functions/third_party_code/geom3d/geom3d/createRotation3dLineAngle.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createRotation3dLineAngle.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createRotation3dLineAngle.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createRotation3dLineAngle.m diff --git a/functions/third_party_code/geom3d/geom3d/createRotationOx.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createRotationOx.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createRotationOx.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createRotationOx.m diff --git a/functions/third_party_code/geom3d/geom3d/createRotationOy.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createRotationOy.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createRotationOy.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createRotationOy.m diff --git a/functions/third_party_code/geom3d/geom3d/createRotationOz.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createRotationOz.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createRotationOz.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createRotationOz.m diff --git a/functions/third_party_code/geom3d/geom3d/createScaling3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createScaling3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createScaling3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createScaling3d.m diff --git a/functions/third_party_code/geom3d/geom3d/createSphere.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createSphere.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createSphere.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createSphere.m diff --git a/functions/third_party_code/geom3d/geom3d/createTranslation3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/createTranslation3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/createTranslation3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/createTranslation3d.m diff --git a/functions/third_party_code/geom3d/geom3d/cyl2cart.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/cyl2cart.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/cyl2cart.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/cyl2cart.m diff --git a/functions/third_party_code/geom3d/geom3d/dihedralAngle.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/dihedralAngle.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/dihedralAngle.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/dihedralAngle.m diff --git a/functions/third_party_code/geom3d/geom3d/distanceLines3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/distanceLines3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/distanceLines3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/distanceLines3d.m diff --git a/functions/third_party_code/geom3d/geom3d/distancePointEdge3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/distancePointEdge3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/distancePointEdge3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/distancePointEdge3d.m diff --git a/functions/third_party_code/geom3d/geom3d/distancePointLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/distancePointLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/distancePointLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/distancePointLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/distancePointPlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/distancePointPlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/distancePointPlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/distancePointPlane.m diff --git a/functions/third_party_code/geom3d/geom3d/distancePoints3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/distancePoints3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/distancePoints3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/distancePoints3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawAxis3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawAxis3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawAxis3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawAxis3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawAxisCube.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawAxisCube.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawAxisCube.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawAxisCube.m diff --git a/functions/third_party_code/geom3d/geom3d/drawBox3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawBox3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawBox3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawBox3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawCircle3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCircle3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawCircle3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCircle3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawCircleArc3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCircleArc3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawCircleArc3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCircleArc3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawCube.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCube.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawCube.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCube.m diff --git a/functions/third_party_code/geom3d/geom3d/drawCuboid.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCuboid.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawCuboid.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCuboid.m diff --git a/functions/third_party_code/geom3d/geom3d/drawCurve3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCurve3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawCurve3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCurve3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawCylinder.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCylinder.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawCylinder.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawCylinder.m diff --git a/functions/third_party_code/geom3d/geom3d/drawEdge3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawEdge3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawEdge3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawEdge3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawEllipse3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawEllipse3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawEllipse3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawEllipse3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawEllipseCylinder.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawEllipseCylinder.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawEllipseCylinder.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawEllipseCylinder.m diff --git a/functions/third_party_code/geom3d/geom3d/drawEllipsoid.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawEllipsoid.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawEllipsoid.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawEllipsoid.m diff --git a/functions/third_party_code/geom3d/geom3d/drawGrid3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawGrid3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawGrid3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawGrid3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawPartialPatch.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPartialPatch.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawPartialPatch.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPartialPatch.m diff --git a/functions/third_party_code/geom3d/geom3d/drawPlane3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPlane3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawPlane3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPlane3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawPoint3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPoint3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawPoint3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPoint3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawPolygon3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPolygon3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawPolygon3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPolygon3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawPolyline3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPolyline3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawPolyline3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawPolyline3d.m diff --git a/functions/third_party_code/geom3d/geom3d/drawSphere.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSphere.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawSphere.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSphere.m diff --git a/functions/third_party_code/geom3d/geom3d/drawSphericalEdge.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSphericalEdge.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawSphericalEdge.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSphericalEdge.m diff --git a/functions/third_party_code/geom3d/geom3d/drawSphericalPolygon.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSphericalPolygon.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawSphericalPolygon.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSphericalPolygon.m diff --git a/functions/third_party_code/geom3d/geom3d/drawSphericalTriangle.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSphericalTriangle.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawSphericalTriangle.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSphericalTriangle.m diff --git a/functions/third_party_code/geom3d/geom3d/drawSurfPatch.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSurfPatch.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawSurfPatch.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawSurfPatch.m diff --git a/functions/third_party_code/geom3d/geom3d/drawTorus.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawTorus.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawTorus.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawTorus.m diff --git a/functions/third_party_code/geom3d/geom3d/drawVector3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/drawVector3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/drawVector3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/drawVector3d.m diff --git a/functions/third_party_code/geom3d/geom3d/ellipsoidSurfaceArea.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/ellipsoidSurfaceArea.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/ellipsoidSurfaceArea.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/ellipsoidSurfaceArea.m diff --git a/functions/third_party_code/geom3d/geom3d/eulerAnglesToRotation3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/eulerAnglesToRotation3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/eulerAnglesToRotation3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/eulerAnglesToRotation3d.m diff --git a/functions/third_party_code/geom3d/geom3d/fillPolygon3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/fillPolygon3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/fillPolygon3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/fillPolygon3d.m diff --git a/functions/third_party_code/geom3d/geom3d/fillSphericalPolygon.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/fillSphericalPolygon.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/fillSphericalPolygon.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/fillSphericalPolygon.m diff --git a/functions/third_party_code/geom3d/geom3d/fillSphericalTriangle.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/fillSphericalTriangle.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/fillSphericalTriangle.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/fillSphericalTriangle.m diff --git a/functions/third_party_code/geom3d/geom3d/fitLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/fitLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/fitLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/fitLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/fitPlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/fitPlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/fitPlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/fitPlane.m diff --git a/functions/third_party_code/geom3d/geom3d/hypot3.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/hypot3.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/hypot3.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/hypot3.m diff --git a/functions/third_party_code/geom3d/geom3d/inertiaEllipsoid.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/inertiaEllipsoid.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/inertiaEllipsoid.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/inertiaEllipsoid.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectBoxes3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectBoxes3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectBoxes3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectBoxes3d.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectEdgePlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectEdgePlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectEdgePlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectEdgePlane.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectLineCylinder.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLineCylinder.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectLineCylinder.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLineCylinder.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectLinePlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLinePlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectLinePlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLinePlane.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectLinePolygon3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLinePolygon3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectLinePolygon3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLinePolygon3d.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectLineSphere.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLineSphere.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectLineSphere.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLineSphere.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectLineTriangle3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLineTriangle3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectLineTriangle3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectLineTriangle3d.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectPlaneLine.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectPlaneLine.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectPlaneLine.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectPlaneLine.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectPlaneSphere.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectPlaneSphere.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectPlaneSphere.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectPlaneSphere.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectPlanes.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectPlanes.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectPlanes.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectPlanes.m diff --git a/functions/third_party_code/geom3d/geom3d/intersectRayPolygon3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectRayPolygon3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/intersectRayPolygon3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/intersectRayPolygon3d.m diff --git a/functions/third_party_code/geom3d/geom3d/isBelowPlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/isBelowPlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/isBelowPlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/isBelowPlane.m diff --git a/functions/third_party_code/geom3d/geom3d/isCoplanar.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/isCoplanar.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/isCoplanar.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/isCoplanar.m diff --git a/functions/third_party_code/geom3d/geom3d/isParallel3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/isParallel3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/isParallel3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/isParallel3d.m diff --git a/functions/third_party_code/geom3d/geom3d/isPerpendicular3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/isPerpendicular3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/isPerpendicular3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/isPerpendicular3d.m diff --git a/functions/third_party_code/geom3d/geom3d/isPointOnLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/isPointOnLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/isPointOnLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/isPointOnLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/linePosition3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/linePosition3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/linePosition3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/linePosition3d.m diff --git a/functions/third_party_code/geom3d/geom3d/lines3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/lines3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/lines3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/lines3d.m diff --git a/functions/third_party_code/geom3d/geom3d/medianPlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/medianPlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/medianPlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/medianPlane.m diff --git a/functions/third_party_code/geom3d/geom3d/mergeBoxes3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/mergeBoxes3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/mergeBoxes3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/mergeBoxes3d.m diff --git a/functions/third_party_code/geom3d/geom3d/midPoint3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/midPoint3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/midPoint3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/midPoint3d.m diff --git a/functions/third_party_code/geom3d/geom3d/normalize3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/normalize3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/normalize3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/normalize3d.m diff --git a/functions/third_party_code/geom3d/geom3d/normalizePlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/normalizePlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/normalizePlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/normalizePlane.m diff --git a/functions/third_party_code/geom3d/geom3d/normalizeVector3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/normalizeVector3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/normalizeVector3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/normalizeVector3d.m diff --git a/functions/third_party_code/geom3d/geom3d/parallelLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/parallelLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/parallelLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/parallelLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/parallelPlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/parallelPlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/parallelPlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/parallelPlane.m diff --git a/functions/third_party_code/geom3d/geom3d/planeNormal.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/planeNormal.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/planeNormal.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/planeNormal.m diff --git a/functions/third_party_code/geom3d/geom3d/planePoint.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/planePoint.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/planePoint.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/planePoint.m diff --git a/functions/third_party_code/geom3d/geom3d/planePosition.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/planePosition.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/planePosition.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/planePosition.m diff --git a/functions/third_party_code/geom3d/geom3d/planes3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/planes3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/planes3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/planes3d.m diff --git a/functions/third_party_code/geom3d/geom3d/planesBisector.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/planesBisector.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/planesBisector.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/planesBisector.m diff --git a/functions/third_party_code/geom3d/geom3d/point3dBounds.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/point3dBounds.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/point3dBounds.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/point3dBounds.m diff --git a/functions/third_party_code/geom3d/geom3d/points3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/points3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/points3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/points3d.m diff --git a/functions/third_party_code/geom3d/geom3d/polygon3dNormalAngle.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/polygon3dNormalAngle.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/polygon3dNormalAngle.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/polygon3dNormalAngle.m diff --git a/functions/third_party_code/geom3d/geom3d/polygonArea3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/polygonArea3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/polygonArea3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/polygonArea3d.m diff --git a/functions/third_party_code/geom3d/geom3d/polygonCentroid3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/polygonCentroid3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/polygonCentroid3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/polygonCentroid3d.m diff --git a/functions/third_party_code/geom3d/geom3d/polygons3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/polygons3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/polygons3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/polygons3d.m diff --git a/functions/third_party_code/geom3d/geom3d/private/localToGlobal3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/private/localToGlobal3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/private/localToGlobal3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/private/localToGlobal3d.m diff --git a/functions/third_party_code/geom3d/geom3d/private/splitPolygons3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/private/splitPolygons3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/private/splitPolygons3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/private/splitPolygons3d.m diff --git a/functions/third_party_code/geom3d/geom3d/projPointOnLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/projPointOnLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/projPointOnLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/projPointOnLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/projPointOnPlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/projPointOnPlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/projPointOnPlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/projPointOnPlane.m diff --git a/functions/third_party_code/geom3d/geom3d/randomAngle3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/randomAngle3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/randomAngle3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/randomAngle3d.m diff --git a/functions/third_party_code/geom3d/geom3d/randomPointInBox3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/randomPointInBox3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/randomPointInBox3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/randomPointInBox3d.m diff --git a/functions/third_party_code/geom3d/geom3d/readme.txt b/zzz_old/to_sort/third_party_code/geom3d/geom3d/readme.txt similarity index 100% rename from functions/third_party_code/geom3d/geom3d/readme.txt rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/readme.txt diff --git a/functions/third_party_code/geom3d/geom3d/recenterTransform3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/recenterTransform3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/recenterTransform3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/recenterTransform3d.m diff --git a/functions/third_party_code/geom3d/geom3d/reverseLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/reverseLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/reverseLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/reverseLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/reversePlane.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/reversePlane.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/reversePlane.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/reversePlane.m diff --git a/functions/third_party_code/geom3d/geom3d/revolutionSurface.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/revolutionSurface.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/revolutionSurface.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/revolutionSurface.m diff --git a/functions/third_party_code/geom3d/geom3d/rotation3dAxisAndAngle.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/rotation3dAxisAndAngle.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/rotation3dAxisAndAngle.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/rotation3dAxisAndAngle.m diff --git a/functions/third_party_code/geom3d/geom3d/rotation3dToEulerAngles.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/rotation3dToEulerAngles.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/rotation3dToEulerAngles.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/rotation3dToEulerAngles.m diff --git a/functions/third_party_code/geom3d/geom3d/rotationOx.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/rotationOx.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/rotationOx.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/rotationOx.m diff --git a/functions/third_party_code/geom3d/geom3d/rotationOy.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/rotationOy.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/rotationOy.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/rotationOy.m diff --git a/functions/third_party_code/geom3d/geom3d/rotationOz.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/rotationOz.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/rotationOz.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/rotationOz.m diff --git a/functions/third_party_code/geom3d/geom3d/scale3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/scale3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/scale3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/scale3d.m diff --git a/functions/third_party_code/geom3d/geom3d/scaling3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/scaling3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/scaling3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/scaling3d.m diff --git a/functions/third_party_code/geom3d/geom3d/sph2cart2.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/sph2cart2.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/sph2cart2.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/sph2cart2.m diff --git a/functions/third_party_code/geom3d/geom3d/sph2cart2d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/sph2cart2d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/sph2cart2d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/sph2cart2d.m diff --git a/functions/third_party_code/geom3d/geom3d/spheres.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/spheres.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/spheres.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/spheres.m diff --git a/functions/third_party_code/geom3d/geom3d/sphericalAngle.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/sphericalAngle.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/sphericalAngle.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/sphericalAngle.m diff --git a/functions/third_party_code/geom3d/geom3d/sphericalVoronoiDomain.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/sphericalVoronoiDomain.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/sphericalVoronoiDomain.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/sphericalVoronoiDomain.m diff --git a/functions/third_party_code/geom3d/geom3d/surfaceCurvature.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/surfaceCurvature.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/surfaceCurvature.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/surfaceCurvature.m diff --git a/functions/third_party_code/geom3d/geom3d/transformLine3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/transformLine3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/transformLine3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/transformLine3d.m diff --git a/functions/third_party_code/geom3d/geom3d/transformPoint3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/transformPoint3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/transformPoint3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/transformPoint3d.m diff --git a/functions/third_party_code/geom3d/geom3d/transformVector3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/transformVector3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/transformVector3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/transformVector3d.m diff --git a/functions/third_party_code/geom3d/geom3d/transforms3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/transforms3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/transforms3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/transforms3d.m diff --git a/functions/third_party_code/geom3d/geom3d/translation3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/translation3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/translation3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/translation3d.m diff --git a/functions/third_party_code/geom3d/geom3d/triangleArea3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/triangleArea3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/triangleArea3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/triangleArea3d.m diff --git a/functions/third_party_code/geom3d/geom3d/vecnorm3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/vecnorm3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/vecnorm3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/vecnorm3d.m diff --git a/functions/third_party_code/geom3d/geom3d/vectorAngle3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/vectorAngle3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/vectorAngle3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/vectorAngle3d.m diff --git a/functions/third_party_code/geom3d/geom3d/vectorCross3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/vectorCross3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/vectorCross3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/vectorCross3d.m diff --git a/functions/third_party_code/geom3d/geom3d/vectorNorm3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/vectorNorm3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/vectorNorm3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/vectorNorm3d.m diff --git a/functions/third_party_code/geom3d/geom3d/vectors3d.m b/zzz_old/to_sort/third_party_code/geom3d/geom3d/vectors3d.m similarity index 100% rename from functions/third_party_code/geom3d/geom3d/vectors3d.m rename to zzz_old/to_sort/third_party_code/geom3d/geom3d/vectors3d.m diff --git a/functions/third_party_code/geom3d/license.txt b/zzz_old/to_sort/third_party_code/geom3d/license.txt similarity index 100% rename from functions/third_party_code/geom3d/license.txt rename to zzz_old/to_sort/third_party_code/geom3d/license.txt diff --git a/functions/third_party_code/geom3d/meshes3d-demos/demoPolyhedra.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/demoPolyhedra.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/demoPolyhedra.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/demoPolyhedra.m diff --git a/functions/third_party_code/geom3d/meshes3d-demos/demoTriangulateFaces.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/demoTriangulateFaces.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/demoTriangulateFaces.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/demoTriangulateFaces.m diff --git a/functions/third_party_code/geom3d/meshes3d-demos/demoVoronoiCell.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/demoVoronoiCell.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/demoVoronoiCell.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/demoVoronoiCell.m diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra.html b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra.html similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra.html rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra.html diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_01.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_01.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_01.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_01.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_02.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_02.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_02.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_02.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_03.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_03.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_03.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_03.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_04.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_04.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_04.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_04.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_05.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_05.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_05.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_05.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_06.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_06.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_06.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_06.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_07.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_07.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_07.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_07.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_08.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_08.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_08.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_08.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_09.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_09.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_09.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoPolyhedra_09.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces.html b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces.html similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces.html rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces.html diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces_01.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces_01.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces_01.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces_01.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces_02.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces_02.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces_02.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoTriangulateFaces_02.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell.html b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell.html similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell.html rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell.html diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_01.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_01.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_01.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_01.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_02.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_02.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_02.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_02.png diff --git a/functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_03.png b/zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_03.png similarity index 100% rename from functions/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_03.png rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d-demos/html/demoVoronoiCell_03.png diff --git a/functions/third_party_code/geom3d/meshes3d/Contents.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/Contents.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/Contents.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/Contents.m diff --git a/functions/third_party_code/geom3d/meshes3d/changelog.txt b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/changelog.txt similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/changelog.txt rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/changelog.txt diff --git a/functions/third_party_code/geom3d/meshes3d/checkMeshAdjacentFaces.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/checkMeshAdjacentFaces.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/checkMeshAdjacentFaces.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/checkMeshAdjacentFaces.m diff --git a/functions/third_party_code/geom3d/meshes3d/clipConvexPolyhedronHP.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/clipConvexPolyhedronHP.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/clipConvexPolyhedronHP.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/clipConvexPolyhedronHP.m diff --git a/functions/third_party_code/geom3d/meshes3d/clipMeshVertices.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/clipMeshVertices.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/clipMeshVertices.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/clipMeshVertices.m diff --git a/functions/third_party_code/geom3d/meshes3d/computeMeshEdges.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/computeMeshEdges.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/computeMeshEdges.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/computeMeshEdges.m diff --git a/functions/third_party_code/geom3d/meshes3d/createCube.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createCube.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createCube.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createCube.m diff --git a/functions/third_party_code/geom3d/meshes3d/createCubeOctahedron.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createCubeOctahedron.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createCubeOctahedron.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createCubeOctahedron.m diff --git a/functions/third_party_code/geom3d/meshes3d/createDodecahedron.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createDodecahedron.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createDodecahedron.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createDodecahedron.m diff --git a/functions/third_party_code/geom3d/meshes3d/createDurerPolyhedron.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createDurerPolyhedron.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createDurerPolyhedron.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createDurerPolyhedron.m diff --git a/functions/third_party_code/geom3d/meshes3d/createIcosahedron.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createIcosahedron.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createIcosahedron.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createIcosahedron.m diff --git a/functions/third_party_code/geom3d/meshes3d/createMengerSponge.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createMengerSponge.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createMengerSponge.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createMengerSponge.m diff --git a/functions/third_party_code/geom3d/meshes3d/createOctahedron.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createOctahedron.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createOctahedron.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createOctahedron.m diff --git a/functions/third_party_code/geom3d/meshes3d/createRhombododecahedron.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createRhombododecahedron.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createRhombododecahedron.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createRhombododecahedron.m diff --git a/functions/third_party_code/geom3d/meshes3d/createSoccerBall.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createSoccerBall.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createSoccerBall.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createSoccerBall.m diff --git a/functions/third_party_code/geom3d/meshes3d/createTetrahedron.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createTetrahedron.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createTetrahedron.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createTetrahedron.m diff --git a/functions/third_party_code/geom3d/meshes3d/createTetrakaidecahedron.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/createTetrakaidecahedron.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/createTetrakaidecahedron.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/createTetrakaidecahedron.m diff --git a/functions/third_party_code/geom3d/meshes3d/cylinderMesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/cylinderMesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/cylinderMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/cylinderMesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/drawFaceNormals.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/drawFaceNormals.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/drawFaceNormals.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/drawFaceNormals.m diff --git a/functions/third_party_code/geom3d/meshes3d/drawMesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/drawMesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/drawMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/drawMesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/drawPolyhedra.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/drawPolyhedra.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/drawPolyhedra.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/drawPolyhedra.m diff --git a/functions/third_party_code/geom3d/meshes3d/drawPolyhedron.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/drawPolyhedron.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/drawPolyhedron.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/drawPolyhedron.m diff --git a/functions/third_party_code/geom3d/meshes3d/faceCentroids.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/faceCentroids.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/faceCentroids.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/faceCentroids.m diff --git a/functions/third_party_code/geom3d/meshes3d/faceNormal.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/faceNormal.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/faceNormal.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/faceNormal.m diff --git a/functions/third_party_code/geom3d/meshes3d/gridmeshToQuadmesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/gridmeshToQuadmesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/gridmeshToQuadmesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/gridmeshToQuadmesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/intersectLineMesh3d.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/intersectLineMesh3d.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/intersectLineMesh3d.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/intersectLineMesh3d.m diff --git a/functions/third_party_code/geom3d/meshes3d/intersectPlaneMesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/intersectPlaneMesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/intersectPlaneMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/intersectPlaneMesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/mergeCoplanarFaces.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/mergeCoplanarFaces.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/mergeCoplanarFaces.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/mergeCoplanarFaces.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshAdjacencyMatrix.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshAdjacencyMatrix.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshAdjacencyMatrix.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshAdjacencyMatrix.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshDihedralAngles.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshDihedralAngles.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshDihedralAngles.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshDihedralAngles.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshEdgeFaces.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshEdgeFaces.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshEdgeFaces.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshEdgeFaces.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshEdgeLength.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshEdgeLength.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshEdgeLength.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshEdgeLength.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshEdges.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshEdges.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshEdges.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshEdges.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshFace.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshFace.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshFace.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshFace.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshFaceEdges.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshFaceEdges.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshFaceEdges.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshFaceEdges.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshFaceNumber.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshFaceNumber.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshFaceNumber.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshFaceNumber.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshFacePolygons.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshFacePolygons.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshFacePolygons.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshFacePolygons.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshReduce.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshReduce.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshReduce.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshReduce.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshSurfaceArea.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshSurfaceArea.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshSurfaceArea.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshSurfaceArea.m diff --git a/functions/third_party_code/geom3d/meshes3d/meshVolume.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshVolume.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/meshVolume.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/meshVolume.m diff --git a/functions/third_party_code/geom3d/meshes3d/minConvexHull.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/minConvexHull.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/minConvexHull.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/minConvexHull.m diff --git a/functions/third_party_code/geom3d/meshes3d/polyhedra.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedra.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/polyhedra.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedra.m diff --git a/functions/third_party_code/geom3d/meshes3d/polyhedronCentroid.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedronCentroid.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/polyhedronCentroid.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedronCentroid.m diff --git a/functions/third_party_code/geom3d/meshes3d/polyhedronMeanBreadth.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedronMeanBreadth.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/polyhedronMeanBreadth.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedronMeanBreadth.m diff --git a/functions/third_party_code/geom3d/meshes3d/polyhedronNormalAngle.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedronNormalAngle.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/polyhedronNormalAngle.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedronNormalAngle.m diff --git a/functions/third_party_code/geom3d/meshes3d/polyhedronSlice.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedronSlice.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/polyhedronSlice.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/polyhedronSlice.m diff --git a/functions/third_party_code/geom3d/meshes3d/private/formatMeshOutput.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/private/formatMeshOutput.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/private/formatMeshOutput.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/private/formatMeshOutput.m diff --git a/functions/third_party_code/geom3d/meshes3d/private/localToGlobal3d.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/private/localToGlobal3d.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/private/localToGlobal3d.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/private/localToGlobal3d.m diff --git a/functions/third_party_code/geom3d/meshes3d/private/parseMeshData.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/private/parseMeshData.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/private/parseMeshData.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/private/parseMeshData.m diff --git a/functions/third_party_code/geom3d/meshes3d/readMesh_off.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/readMesh_off.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/readMesh_off.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/readMesh_off.m diff --git a/functions/third_party_code/geom3d/meshes3d/smoothMesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/smoothMesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/smoothMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/smoothMesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/sphereMesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/sphereMesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/sphereMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/sphereMesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/steinerPolytope.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/steinerPolytope.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/steinerPolytope.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/steinerPolytope.m diff --git a/functions/third_party_code/geom3d/meshes3d/subdivideMesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/subdivideMesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/subdivideMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/subdivideMesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/surfToMesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/surfToMesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/surfToMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/surfToMesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/tetrahedronVolume.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/tetrahedronVolume.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/tetrahedronVolume.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/tetrahedronVolume.m diff --git a/functions/third_party_code/geom3d/meshes3d/torusMesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/torusMesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/torusMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/torusMesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/triangulateFaces.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/triangulateFaces.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/triangulateFaces.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/triangulateFaces.m diff --git a/functions/third_party_code/geom3d/meshes3d/trimMesh.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/trimMesh.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/trimMesh.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/trimMesh.m diff --git a/functions/third_party_code/geom3d/meshes3d/trimeshSurfaceArea.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/trimeshSurfaceArea.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/trimeshSurfaceArea.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/trimeshSurfaceArea.m diff --git a/functions/third_party_code/geom3d/meshes3d/vertexNormal.m b/zzz_old/to_sort/third_party_code/geom3d/meshes3d/vertexNormal.m similarity index 100% rename from functions/third_party_code/geom3d/meshes3d/vertexNormal.m rename to zzz_old/to_sort/third_party_code/geom3d/meshes3d/vertexNormal.m diff --git a/functions/third_party_code/inpaint_nans/inpaint_nans.m b/zzz_old/to_sort/third_party_code/inpaint_nans/inpaint_nans.m similarity index 100% rename from functions/third_party_code/inpaint_nans/inpaint_nans.m rename to zzz_old/to_sort/third_party_code/inpaint_nans/inpaint_nans.m diff --git a/functions/third_party_code/nearestneighbour.m b/zzz_old/to_sort/third_party_code/nearestneighbour.m similarity index 100% rename from functions/third_party_code/nearestneighbour.m rename to zzz_old/to_sort/third_party_code/nearestneighbour.m diff --git a/functions/third_party_code/subdir/subdir.m b/zzz_old/to_sort/third_party_code/subdir/subdir.m similarity index 100% rename from functions/third_party_code/subdir/subdir.m rename to zzz_old/to_sort/third_party_code/subdir/subdir.m