DataController.py

#Import project modules
import Communication
import ZEDController
import GNSSController

#import system modules
from datetime import datetime, timedelta
import os
from threading import Thread

#Import pandas for data formattng/processing
import pandas

#DataController class - Handles moving and storing data to pandas dataframes, along with saving or dumping said data
class DataController(Communication.Observer, Thread):
    #Class constructor
    def __init__(self, product_dir, dur_buffer):
        Thread.__init__(self, name = "thread-datacontroller")

        super().__init__()
        self.dur_buffer = dur_buffer
        self.product_dir = product_dir

        self.bar_list = []
        self.imu_list = []
        self.mag_list = []

        self.img_list = []

        self.us_last_bar = 0
        self.us_last_imu = 0
        self.us_last_mag = 0
        self.us_last_frame = 0

        self.b_sample_frame = False
        self.b_sample_depth = False
        self.b_sample_pc = False
        self.b_sample_sensor = False

        self.last_save = datetime.now()

    #Class destructor
    def __del__(self):
        pass

    def run(self):
        while True:
            self.save()

    #Mutator Methods for variables, which would be protected in c++, returns 0 if toggled successfully
    # mode 
    # 1 -> Toggle Frame
    # 2 -> Toggle Depth
    # 3 -> Toggle PC
    # 4 -> Toggle Sensors
    def setSample(self, mode):
        toggled = False

        if mode & 0b00000001:
            self.b_sample_frame = not self.b_sample_frame
            toggled = True

        if mode & 0b00000010:
            self.b_sample_depth = not self.b_sample_depth
            toggled = True

        if mode & 0b00000100:
            self.b_sample_pc = not self.b_sample_pc
            toggled = True

        if mode & 0b00001000:
            self.b_sample_sensor = not self.b_sample_sensor
            toggled = True

        if not toggled:
            return 1
        
        return 0

    #Controller update function
    def c_update(self, observable, *args, **kwargs):
        print("Starting Updates")
        #If observable calling c_update is ZEDController, process ZED data
        if (isinstance(observable, ZEDController.ZEDController)):

            #This code needs to be fixed, even this causes stutter
            bar_us = observable.timestamp_barometer.get_microseconds()
            new_bar = (self.us_last_bar < bar_us)
            imu_us = observable.timestamp_imu.get_microseconds()
            new_imu = (self.us_last_imu < imu_us)
            mag_us = observable.timestamp_magnetometer.get_microseconds()
            new_mag = (self.us_last_mag < mag_us)
            frame_us = observable.timestamp_frame.get_microseconds()
            new_frame = (self.us_last_frame < frame_us)

            #Ensure barometer data is new, if so, create dict and append to list
            if (new_bar and self.b_sample_sensor):
                print("Processing bar")
                self.us_last_bar = bar_us
                bar_dict = {
                    "timestamp_baro": bar_us,
                    "pressure": observable.pressure,
                    "relative_alt": observable.relative_alt
                }

                self.bar_list.append(bar_dict)

            #Ensure IMU data is new, if so, create dict and append to list
            if (new_imu and self.b_sample_sensor):
                print("Processing IMU")
                self.us_last_imu = imu_us
                imu_dict = {
                    "timestamp_accel": imu_us,
                    "orientation_x": observable.orientation[0],
                    "orientation_y": observable.orientation[1],
                    "orientation_z": observable.orientation[2],
                    "orientation_w": observable.orientation[3],
                    "linear_accel_x": observable.linear_accel[0],
                    "linear_accel_y": observable.linear_accel[1],
                    "linear_accel_z": observable.linear_accel[2],
                    "angular_vel_x": observable.angular_vel[0],
                    "angular_vel_x": observable.angular_vel[1],
                    "angular_vel_x": observable.angular_vel[2]
                }

                self.imu_list.append(imu_dict)

            #Ensure magnometer data is new, if so, create dict and append to list
            if (new_mag and self.b_sample_sensor):
                self.us_last_mag = mag_us
                mag_dict = {
                    "timestamp_mag": mag_us,
                    "magnetic_fld_x": observable.magnetic_fld[0],
                    "magnetic_fld_x": observable.magnetic_fld[1],
                    "magnetic_fld_x": observable.magnetic_fld[2]
                }

                self.mag_list.append(mag_dict)
            
            #TODO: Needs to be improved to fix performance defecit

            #Ensure captured frame is new, if so append to list
            if (new_frame and self.b_sample_frame):
                self.us_last_frame = frame_us
                self.img_list.append(observable.frame)

            if (self.b_sample_depth):
                pass

            if (self.b_sample_pc):
                pass

        #NOt even sure how this runs currently
        if (isinstance(observable, GNSSController.GNSSController)):
            self.gnss_list.append(observable.gnss_dict)

    #Dump - Remove all stored data from the class
    def dump(self):
        self.imu_list = []
        self.mag_list = []
        self.bar_list = []
        self.img_list = []

    #Store - similar to dump except data is stored to disk
    def save(self, fmt = ".csv"):
        if (timedelta(0, self.dur_buffer) < datetime.now() - self.last_save):
            imu_df = pandas.DataFrame(self.imu_list)
            mag_df = pandas.DataFrame(self.mag_list)
            bar_df = pandas.DataFrame(self.bar_list)

            #Reset barometer, IMU, and magnometer data
            self.imu_list = []
            self.mag_list = []
            self.bar_list = []

            #Get current data and time for formatting filename
            date = datetime.now().strftime("%Y-%m-%d")
            time = datetime.now().strftime("%H-%M-%S")
        
            #If a slash was forgootten at the end, append it
            if (self.product_dir[-1] != '/'):
                self.product_dir = self.product_dir + '/'

            #Create directories to prevent errors from occuring, exist_ok = True lets the command know that if teh directories already exist, it is not a problem
            os.makedirs(self.product_dir + date, exist_ok = True)

            #Set default return code to 0 - OK
            ret_code = 0

            #Create filename out of dat time, and product directory
            filename = self.product_dir + date + '/' + time

            #If file type chosen is .csv (Comma Seperated Value format)
            if fmt.lower() == ".csv":
                imu_df.to_csv(filename + "-imu" + fmt.lower())
                mag_df.to_csv(filename + "-mag" + fmt.lower())
                bar_df.to_csv(filename + "-bar" + fmt.lower())

            #If file type chosen is .xlsx (Excel Spreadsheet format)
            elif fmt.lower() == ".xlsx":
                imu_df.to_excel(filename + "-imu" + fmt.lower())
                mag_df.to_excel(filename + "-mag" + fmt.lower())
                bar_df.to_excel(filename + "-bar" + fmt.lower())

            #If file type is chosen is .parq or .parquet (Apache Parquet format)
            elif fmt.lower() in [".parq", ".parquet"]:
                imu_df.to_parquet(filename + "-imu" + fmt.lower())
                mag_df.to_parquet(filename + "-mag" + fmt.lower())
                bar_df.to_parquet(filename + "-bar" + fmt.lower())

            #If filename can't be determined, format to .parq as default
            else:
                print("DataController (NON-FATAL): Couldn't determine format, defaulting to .parq")
                imu_df.to_parquet(filename + "-imu.parq")
                mag_df.to_parquet(filename + "-mag.parq")
                bar_df.to_parquet(filename + "-bar.parq")

                #Boolean AND to the return code to tag 0b10 in, warning INVALID FILE TYPE
                ret_code = ret_code & 2

            #Create string for image directory
            img_dir = self.product_dir + "Images/" + date + '/'

            #Iterate thrugh all images in image_list
            for img in self.img_list:
                #
                try:
                    file = open(img_dir + date + '/' + time + ".raw")

                #
                except:
                    ret_code = ret_code & 4

                #
                else:
                    file.write(img.get_data())

            self.img_list = []

            #Return the state of saving
            return ret_code