# ============================================================================== # # read_table_for_open.py - AcouSed # # Copyright (C) 2024 INRAE # # # # This program is free software: you can redistribute it and/or modify # # it under the terms of the GNU General Public License as published by # # the Free Software Foundation, either version 3 of the License, or # # (at your option) any later version. # # # # This program is distributed in the hope that it will be useful, # # but WITHOUT ANY WARRANTY; without even the implied warranty of # # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # # GNU General Public License for more details. # # # # You should have received a copy of the GNU General Public License # # along with this program. If not, see . # # by Brahim MOUDJED # # ============================================================================== # # -*- coding: utf-8 -*- import sys import numpy as np from PyQt5.QtWidgets import QFileDialog, QApplication, QWidget, QTabWidget import sqlite3 from os import path, chdir import settings as stg from settings import BS_raw_data, acoustic_data from View.acoustic_data_tab import AcousticDataTab class ReadTableForOpen: def __init__(self): pass def open_file_dialog(self): name = QFileDialog.getOpenFileName(caption="Open Acoused file", directory="", filter="Acoused file (*.acd)", options=QFileDialog.DontUseNativeDialog) if name: stg.dirname_open = path.dirname(name[0]) stg.filename_open = path.basename(name[0]) chdir(stg.dirname_open) self.sql_file_to_open = open(stg.filename_open) self.read_table() def read_table(self): stg.read_table_trigger = 1 # connexion to File db cnx = sqlite3.connect(stg.filename_open) # Create database cursor to execute SQL statements and fetch results from SQL queries. cur = cnx.cursor() # -------------------------------------------------------------------------------------------------------------- # +++++++++++++++++++++++++++ # --- Table Acoustic File --- # +++++++++++++++++++++++++++ query0 = f'''SELECT acoustic_data FROM AcousticFile''' data0 = cur.execute(query0).fetchall() print("data0 ", data0) stg.acoustic_data = [x[0] for x in data0] print("stg.acoustic_data ", stg.acoustic_data) for k in range(len(stg.acoustic_data)): print("hello") query = f'''SELECT acoustic_data, acoustic_file, ABS_name, path_BS_noise_data, filename_BS_noise_data, noise_method, noise_value, data_preprocessed FROM AcousticFile WHERE (acoustic_data = {k})''' data = cur.execute(query).fetchall() print("data acoustic file", data) stg.filename_BS_raw_data.append([str(y[1]) + '.aqa' for y in data][0]) stg.ABS_name.append([z[2] for z in data][0]) stg.path_BS_noise_data.append([z[3] for z in data][0]) stg.filename_BS_noise_data.append([z[4] for z in data][0]) stg.noise_method.append([z[5] for z in data][0]) stg.noise_value.append([z[6] for z in data][0]) stg.data_preprocessed.append([z[7] for z in data][0]) print("data acoustic file ", stg.filename_BS_raw_data, stg.ABS_name, stg.path_BS_noise_data, stg.filename_BS_noise_data, stg.noise_method, stg.noise_value, stg.data_preprocessed) # -------------------------------------------------------------------------------------------------------------- # +++++++++++++++++++++ # --- Table Measure --- # +++++++++++++++++++++ stg.date = [0]*len(stg.acoustic_data) stg.hour = [0]*len(stg.acoustic_data) for i in range(len(stg.acoustic_data)): print("i = ", i) query1 = f'''SELECT acoustic_data, Date, Hour, frequency, sound_attenuation, kt_read, kt_corrected, NbProfiles, NbProfilesPerSeconds, NbCells, CellSize, PulseLength, NbPingsPerSeconds, NbPingsAveragedPerProfile, GainRx, GainTx FROM Measure WHERE (acoustic_data = {i})''' data1 = cur.execute(query1).fetchall() print("--------------------------------------") print("data1 ", data1) stg.date[i] = data1[0][1] stg.hour[i] = data1[0][2] stg.freq.append(np.array([x[3] for x in data1])) stg.freq_text.append([str(x[3]*1e-6) + 'MHz' for x in data1]) stg.water_attenuation.append([x[4] for x in data1]) stg.kt_read.append([x[5] for x in data1]) stg.kt_corrected = [x[6] for x in data1] stg.nb_profiles.append([x[7] for x in data1]) stg.nb_profiles_per_sec.append([x[8] for x in data1]) stg.nb_cells.append([x[9] for x in data1]) stg.cell_size.append([x[10] for x in data1]) stg.pulse_length.append([x[11] for x in data1]) stg.nb_pings_per_sec.append([x[12] for x in data1]) stg.nb_pings_averaged_per_profile.append([x[13] for x in data1]) stg.gain_rx.append([x[14] for x in data1]) stg.gain_tx.append([x[15] for x in data1]) print(stg.acoustic_data, stg.freq, stg.water_attenuation, stg.kt_read, stg.kt_corrected, stg.nb_profiles, stg.nb_profiles_per_sec, stg.nb_cells, stg.cell_size, stg.pulse_length, stg.nb_pings_per_sec, stg.nb_pings_averaged_per_profile, stg.gain_rx, stg.gain_tx) print(stg.date) print(stg.hour) # -------------------------------------------------------------------------------------------------------------- # +++++++++++++++++++++++ # --- Table BSRawData --- # ++++++++++++++++++++++ print("len stg.acoustic_data ", len(stg.acoustic_data)) for j in range(len(stg.acoustic_data)): print(f"j = {j}") query2 = f'''SELECT acoustic_data, time, depth, BS_raw_data, time_reshape, depth_reshape, BS_raw_data_reshape, time_cross_section, depth_cross_section, BS_cross_section, BS_stream_bed, depth_bottom, val_bottom, ind_bottom, time_noise, depth_noise, BS_noise_raw_data, SNR_raw_data, SNR_cross_section, SNR_stream_bed, BS_raw_data_pre_process_SNR, BS_raw_data_pre_process_average, BS_cross_section_pre_process_SNR, BS_cross_section_pre_process_average, BS_stream_bed_pre_process_SNR, BS_stream_bed_pre_process_average, BS_mean FROM BSRawData WHERE (acoustic_data = {j})''' data2 = cur.execute(query2).fetchall() print("len data2 ", len(data2)) stg.time.append(np.frombuffer(data2[0][1], dtype=np.float64).reshape((stg.freq[j].shape[0], -1))) print("stg.time[0].shape ", stg.time[j].shape, np.frombuffer(data2[0][1], dtype=np.float64).shape) print(stg.time) stg.depth.append(np.frombuffer(data2[0][2], dtype=np.float64).reshape((stg.freq[j].shape[0], -1))) print("stg.depth[0].shape ", stg.depth[j].shape) print(stg.depth) stg.BS_raw_data.append(np.frombuffer(data2[0][3], dtype=np.float64).reshape((stg.freq[j].shape[0], stg.depth[j].shape[1], stg.time[j].shape[1]))) print("BS_raw_data ", stg.BS_raw_data[j].shape) stg.time_reshape.append(np.frombuffer(data2[0][4], dtype=np.float64).reshape((-1, stg.freq[j].shape[0]))) print("stg.time_reshape[0].shape ", stg.time_reshape[j].shape) stg.depth_reshape.append(np.frombuffer(data2[0][5], dtype=np.float64).reshape((-1, stg.freq[j].shape[0]))) stg.BS_raw_data_reshape.append(np.frombuffer(data2[0][6], dtype=np.float64).reshape((-1, stg.freq[j].shape[0]))) print("time cross section ", stg.time_cross_section, stg.time_cross_section == []) print("np.frombuffer(data2[0][9], dtype=np.float64) ", np.frombuffer(data2[0][9], dtype=np.float64)) if len(np.frombuffer(data2[0][9], dtype=np.float64)) == 0: print("Je suis là") stg.time_cross_section.append(np.array([])) stg.depth_cross_section.append(np.array([])) stg.BS_cross_section.append(np.array([])) else: print("Je suis ici") print(stg.freq[j].shape) print(np.frombuffer(data2[0][7], dtype=np.float64).shape) stg.time_cross_section.append(np.frombuffer(data2[0][7], dtype=np.float64).reshape((stg.freq[j].shape[0], -1))) print("time cross section ", stg.time_cross_section, stg.time_cross_section[j].shape) stg.depth_cross_section.append(np.frombuffer(data2[0][8], dtype=np.float64).reshape((stg.freq[j].shape[0], -1))) print("depth cross section ", stg.depth_cross_section, stg.depth_cross_section[j].shape) stg.BS_cross_section.append(np.frombuffer(data2[0][9], dtype=np.float64).reshape( (stg.freq[j].shape[0], stg.depth_cross_section[j].shape[1], stg.time_cross_section[j].shape[1]))) if len(np.frombuffer(data2[0][10], dtype=np.float64)) == 0: stg.BS_stream_bed.append(np.array([])) else: stg.BS_stream_bed.append(np.frombuffer(data2[0][10], dtype=np.float64).reshape( (stg.freq[j].shape[0], stg.depth_cross_section[j].shape[1], stg.time_cross_section[j].shape[1]))) if len(np.frombuffer(data2[0][11], dtype=np.float64)) == 0: stg.depth_bottom.append(np.array([])) stg.val_bottom.append([]) stg.ind_bottom.append([]) else: stg.depth_bottom.append(np.frombuffer(data2[0][11], dtype=np.float64)) stg.val_bottom.append(np.frombuffer(data2[0][12], dtype=np.float64).tolist()) stg.ind_bottom.append(np.frombuffer(data2[0][13], dtype=np.float64).tolist()) print("stg.depth_bottom ", stg.depth_bottom) print("stg.val_bottom ", stg.val_bottom) print("stg.ind_bottom ", stg.ind_bottom) if len(np.frombuffer(data2[0][14], dtype=np.float64)) == 0: stg.time_noise.append(np.array([])) stg.depth_noise.append(np.array([])) stg.BS_noise_raw_data.append(np.array([])) else: stg.time_noise.append(np.frombuffer(data2[0][14], dtype=np.float64).reshape((stg.freq[j].shape[0], -1))) stg.depth_noise.append(np.frombuffer(data2[0][15], dtype=np.float64).reshape((stg.freq[j].shape[0], -1))) stg.BS_noise_raw_data.append(np.frombuffer(data2[0][16], dtype=np.float64).reshape( (stg.freq[j].shape[0], stg.depth_noise[j].shape[1], stg.time_noise[j].shape[1]))) if len(np.frombuffer(data2[0][17], dtype=np.float64)) == 0: stg.SNR_raw_data.append(np.array([])) else: stg.SNR_raw_data.append(np.frombuffer(data2[0][17], dtype=np.float64).reshape(stg.BS_raw_data[j].shape)) if len(np.frombuffer(data2[0][18], dtype=np.float64)) == 0: stg.SNR_cross_section.append(np.array([])) else: stg.SNR_cross_section.append(np.frombuffer(data2[0][18], dtype=np.float64).reshape(stg.BS_cross_section[j].shape)) if len(np.frombuffer(data2[0][19], dtype=np.float64)) == 0: stg.SNR_stream_bed.append(np.array([])) else: stg.SNR_stream_bed.append(np.frombuffer(data2[0][19], dtype=np.float64).reshape(stg.BS_stream_bed[j].shape)) if len(np.frombuffer(data2[0][20], dtype=np.float64)) == 0: stg.BS_raw_data_pre_process_SNR.append(np.array([])) else: stg.BS_raw_data_pre_process_SNR.append(np.frombuffer(data2[0][20], dtype=np.float64).reshape(stg.BS_raw_data[j].shape)) if len(np.frombuffer(data2[0][21], dtype=np.float64)) == 0: stg.BS_raw_data_pre_process_average.append(np.array([])) else: stg.BS_raw_data_pre_process_average.append(np.frombuffer(data2[0][21], dtype=np.float64).reshape(stg.BS_raw_data[j].shape)) if len(np.frombuffer(data2[0][22], dtype=np.float64)) == 0: stg.BS_cross_section_pre_process_SNR.append(np.array([])) else: stg.BS_cross_section_pre_process_SNR.append(np.frombuffer(data2[0][22], dtype=np.float64).reshape(stg.BS_cross_section[j].shape)) if len(np.frombuffer(data2[0][23], dtype=np.float64)) == 0: stg.BS_cross_section_pre_process_average.append(np.array([])) else: stg.BS_cross_section_pre_process_average.append(np.frombuffer(data2[0][23], dtype=np.float64).reshape(stg.BS_cross_section[j].shape)) if len(np.frombuffer(data2[0][24], dtype=np.float64)) == 0: stg.BS_stream_bed_pre_process_SNR.append(np.array([])) else: stg.BS_stream_bed_pre_process_SNR.append(np.frombuffer(data2[0][24], dtype=np.float64).reshape(stg.BS_stream_bed[j].shape)) if len(np.frombuffer(data2[0][25], dtype=np.float64)) == 0: stg.BS_stream_bed_pre_process_average.append(np.array([])) else: stg.BS_stream_bed_pre_process_average.append(np.frombuffer(data2[0][25], dtype=np.float64).reshape(stg.BS_stream_bed[j].shape)) if len(np.frombuffer(data2[0][26], dtype=np.float64)) == 0: stg.BS_mean.append(np.array([])) else: stg.BS_mean.append(np.frombuffer(data2[0][26], dtype=np.float64).reshape((stg.freq[j].shape[0], -1))) print(stg.BS_mean[j].shape) # -------------------------------------------------------------------------------------------------------------- # +++++++++++++++++++++++ # --- Table Settings --- # +++++++++++++++++++++++ for s in range(len(stg.acoustic_data)): query3 = f'''SELECT acoustic_data, temperature, tmin_index, tmin_value, tmax_index, tmax_value, rmin_index, rmin_value, rmax_index, rmax_value, freq_bottom_detection_index, freq_bottom_detection_value, SNR_filter_value, Nb_cells_to_average_BS_signal FROM Settings WHERE (acoustic_data = {s})''' data3 = cur.execute(query3).fetchall() stg.temperature = [x[1] for x in data3][0] stg.tmin.append([(x[2], x[3]) for x in data3]) stg.tmax.append([(x[4], x[5]) for x in data3]) stg.rmin.append([(x[6], x[7]) for x in data3]) stg.rmax.append([(x[8], x[9]) for x in data3]) stg.freq_bottom_detection.append([(x[10], x[11]) for x in data3]) stg.SNR_filter_value.append([x[12] for x in data3]) stg.Nb_cells_to_average_BS_signal.append([x[13] for x in data3]) # -------------------------------------------------------------------------------------------------------------- # +++++++++++++++++++++++++++ # --- Table Sediment File --- # +++++++++++++++++++++++++++ query4 = f'''SELECT path_fine, filename_fine, radius_grain_fine, path_sand, filename_sand, radius_grain_sand, time_column_label, distance_from_bank_column_label, depth_column_label, Ctot_fine_column_label, D50_fine_column_label, Ctot_sand_column_label, D50_sand_column_label from SedimentsFile''' data4 = cur.execute(query4).fetchall() print("data4 ", data4) stg.path_fine = data4[0][0] stg.filename_fine = data4[0][1] stg.radius_grain_fine = np.array(np.frombuffer(data4[0][2], dtype=np.float64)) stg.path_sand = data4[0][3] stg.filename_sand = data4[0][4] stg.radius_grain_sand = np.array(np.frombuffer(data4[0][5], dtype=np.float64)) stg.columns_fine = [data4[0][6], data4[0][7], data4[0][8], data4[0][9], data4[0][10]] + list(stg.radius_grain_fine) stg.columns_sand = [data4[0][6], data4[0][7], data4[0][8], data4[0][11], data4[0][12]] + list(stg.radius_grain_sand) print("sediment file : ", stg.path_fine, stg.filename_fine, stg.path_sand, stg.filename_sand) print(stg.radius_grain_fine, stg.radius_grain_sand) print('stg.columns_fine ', stg.columns_fine) # -------------------------------------------------------------------------------------------------------------- # +++++++++++++++++++++++++++ # --- Table Sediment Data --- # +++++++++++++++++++++++++++ query5 = f'''SELECT sample_fine_name, sample_fine_index, distance_from_bank_fine, depth_fine, time_fine, Ctot_fine, Ctot_fine_per_cent, D50_fine, frac_vol_fine, frac_vol_fine_cumul, sample_sand_name, sample_sand_index, distance_from_bank_sand, depth_sand, time_sand, Ctot_sand, Ctot_sand_per_cent, D50_sand, frac_vol_sand, frac_vol_sand_cumul from SedimentsData''' data5 = cur.execute(query5).fetchall() stg.frac_vol_fine = [] stg.frac_vol_fine_cumul = [] stg.frac_vol_sand = [] stg.frac_vol_sand_cumul = [] for f in range(len(data5)): stg.sample_fine.append((data5[f][0], data5[f][1])) stg.distance_from_bank_fine.append(data5[f][2]) stg.depth_fine.append(data5[f][3]) stg.time_fine.append(data5[f][4]) stg.Ctot_fine.append(data5[f][5]) stg.Ctot_fine_per_cent.append(data5[f][6]) stg.D50_fine.append(data5[f][7]) print("np.frombuffer(data4[f][8], dtype=np.float64) ", np.frombuffer(data5[f][8], dtype=np.float64)) stg.frac_vol_fine.append(np.frombuffer(data5[f][8], dtype=np.float64)) stg.frac_vol_fine_cumul.append(np.frombuffer(data5[f][9], dtype=np.float64)) stg.sample_sand.append((data5[f][10], data5[f][11])) stg.distance_from_bank_sand.append(data5[f][12]) stg.depth_sand.append(data5[f][13]) stg.time_sand.append(data5[f][14]) stg.Ctot_sand.append(data5[f][15]) stg.Ctot_sand_per_cent.append(data5[f][16]) stg.D50_sand.append(data5[f][17]) stg.frac_vol_sand.append(np.frombuffer(data5[f][18], dtype=np.float64)) stg.frac_vol_sand_cumul.append(np.frombuffer(data5[f][19], dtype=np.float64)) stg.frac_vol_fine = np.array(stg.frac_vol_fine) stg.frac_vol_fine_cumul = np.array(stg.frac_vol_fine_cumul) stg.frac_vol_sand = np.array(stg.frac_vol_sand) stg.frac_vol_sand_cumul = np.array(stg.frac_vol_sand_cumul) # print("data 4 : ", len(data4), data4) print('data 5 :') print(stg.Ctot_fine, stg.sample_sand) print(type(stg.frac_vol_fine_cumul), stg.frac_vol_fine_cumul) # Close database cursor cur.close() # Close database connection cnx.close() print("read table finished") def fill_acoustic_data_tab(self): print("start fill acoustic data tab") # tab_adt = AcousticDataTab(self.master_widget) print("1 AcousticDataTab ", id(AcousticDataTab)) print("tab_adt.combobox_ABS_system_choice ", self.tab.combobox_ABS_system_choice) self.tab.combobox_ABS_system_choice.editTextChanged.connect(self.tab.ABS_system_choice) if stg.ABS_name[0] == "AQUAscat": self.tab.combobox_ABS_system_choice.setCurrentText(stg.ABS_name[0]) print("combobox_ABS_system_choice.setCurrentIndex(1)", self.tab.combobox_ABS_system_choice.itemText(1), self.tab.combobox_ABS_system_choice.itemText(2)) else: self.tab.combobox_ABS_system_choice.setCurrentText(stg.ABS_name[0]) self.tab.plot_backscattered_acoustic_signal_recording() # app = QApplication(sys.argv) # sys.exit(app.exec_()) def reshape_variables(self): for i in stg.acoustic_data: for f, _ in enumerate(stg.freq[i]): if f == 0: depth_temp = np.array([ stg.depth_reshape[i][np.where(stg.depth_reshape[i][:, f] == stg.depth_reshape[i][0, f])[0][0]: np.where(stg.depth_reshape[i][:, f] == stg.depth_reshape[i][0, f])[0][1], f] ]) time_temp = np.array([ stg.time_reshape[i][ np.where(stg.depth_reshape[i][:, f] == stg.depth_reshape[i][0, f])[0], f] ]) else: # print(np.where(stg.depth_reshape[i][:, f] == stg.depth_reshape[i][0, f])) depth_temp = np.insert(depth_temp, depth_temp.shape[0], stg.depth_reshape[i][np.where(stg.depth_reshape[i][:, f] == stg.depth_reshape[i][0, f])[0][0]: np.where(stg.depth_reshape[i][:, f] == stg.depth_reshape[i][0, f])[0][1], f], axis=0) time_temp = np.insert(time_temp, time_temp.shape[0], stg.time_reshape[i][ np.where(stg.depth_reshape[i][:, f] == stg.depth_reshape[i][0, f])[0], f], axis=0) stg.depth.append(depth_temp) stg.time.append(time_temp) stg.BS_raw_data.append(np.reshape(stg.BS_raw_data_reshape[i], (len(stg.freq[i]), stg.depth[i].shape[1], stg.time[i].shape[1])))