parouby
/
acoused
mirror of https://forgemia.inra.fr/theophile.terraz/acoused
1
0
Fork 0
Code Issues Packages Projects Releases Activity
acoused/Model/create_table_for_save_as.py

668 lines
24 KiB
Python
Raw Blame History

# ============================================================================== #
# create_table_for_save_as.py - AcouSed #
# Copyright (C) 2024 INRAE #
# #
# This program is free software: you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation, either version 3 of the License, or #
# (at your option) any later version. #
# #
# This program is distributed in the hope that it will be useful, #
# but WITHOUT ANY WARRANTY; without even the implied warranty of #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
# GNU General Public License for more details. #
# #
# You should have received a copy of the GNU General Public License #
# along with this program. If not, see <https://www.gnu.org/licenses/>. #
# by Brahim MOUDJED #
# ============================================================================== #
# -*- coding: utf-8 -*-
import os
import time
import sqlite3
import logging
import numpy as np
from PyQt5.QtWidgets import QFileDialog, QApplication, QMessageBox
import settings as stg
from settings import ABS_name
logger = logging.getLogger("acoused")
class CreateTableForSaveAs:
def __init__(self):
self.create_AcousticFile = """
CREATE TABLE AcousticFile(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
acoustic_data INTEGER,
acoustic_file STRING,
ABS_name STRING,
path_BS_noise_data STRING,
filename_BS_noise_data STRING,
noise_method FLOAT,
noise_value FLOAT,
data_preprocessed STRING
)
"""
self.create_Measure = """
CREATE TABLE Measure(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
acoustic_data INTEGER,
Date DATE,
Hour TIME,
frequency FLOAT,
sound_attenuation FLOAT,
kt_read FLOAT,
kt_corrected FLOAT,
NbProfiles FLOAT,
NbProfilesPerSeconds FLOAT,
NbCells FLOAT,
CellSize FLOAT,
PulseLength FLOAT,
NbPingsPerSeconds FLOAT,
NbPingsAveragedPerProfile FLOAT,
GainRx FLOAT,
GainTx FLOAT
)
"""
self.create_BSRawData = """
CREATE TABLE BSRawData(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
acoustic_data INTEGER,
time BLOB, depth BLOB, BS_raw_data BLOB,
time_reshape BLOB, depth_reshape BLOB, BS_raw_data_reshape BLOB,
time_cross_section BLOB, depth_cross_section BLOB,
BS_cross_section BLOB, BS_stream_bed BLOB,
depth_bottom BLOB, val_bottom BLOB, ind_bottom BLOB,
time_noise BLOB, depth_noise BLOB, BS_noise_raw_data BLOB,
SNR_raw_data BLOB, SNR_cross_section BLOB, SNR_stream_bed BLOB,
BS_raw_data_pre_process_SNR BLOB,
BS_raw_data_pre_process_average BLOB,
BS_cross_section_pre_process_SNR BLOB,
BS_cross_section_pre_process_average BLOB,
BS_stream_bed_pre_process_SNR BLOB,
BS_stream_bed_pre_process_average BLOB,
BS_mean BLOB
)
"""
self.create_Settings = """
CREATE TABLE Settings(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
acoustic_data INTEGER,
temperature FLOAT,
distance_to_free_surface FLOAT,
tmin_index INTEGER, tmin_value FLOAT,
tmax_index INTEGER, tmax_value FLOAT,
rmin_index INTEGER, rmin_value FLOAT,
rmax_index INTEGER, rmax_value FLOAT,
freq_bottom_detection_index FLOAT,
freq_bottom_detection_value STRING,
depth_bottom_detection_min FLOAT,
depth_bottom_detection_max FLOAT,
depth_bottom_detection_inverval FLOAT,
SNR_filter_value FLOAT,
Nb_cells_to_average_BS_signal FLOAT
)
"""
self.create_SedimentsFile = """
CREATE TABLE SedimentsFile(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
path_fine STRING,
filename_fine STRING,
radius_grain_fine BLOB,
path_sand STRING,
filename_sand STRING,
radius_grain_sand BLOB,
time_column_label STRING,
distance_from_bank_column_label STRING,
depth_column_label STRING,
Ctot_fine_column_label STRING,
D50_fine_column_label STRING,
Ctot_sand_column_label STRING,
D50_sand_column_label STRING
)
"""
self.create_SedimentsData = """
CREATE TABLE SedimentsData(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
sample_fine_name STRING,
sample_fine_index INTEGER,
distance_from_bank_fine FLOAT,
depth_fine FLOAT,
time_fine FLOAT,
Ctot_fine FLOAT,
Ctot_fine_per_cent FLOAT,
D50_fine FLOAT,
frac_vol_fine BLOB,
frac_vol_fine_cumul BLOB,
sample_sand_name STRING,
sample_sand_index INTEGER,
distance_from_bank_sand FLOAT,
depth_sand FLOAT,
time_sand FLOAT,
Ctot_sand FLOAT,
Ctot_sand_per_cent FLOAT,
D50_sand FLOAT,
frac_vol_sand BLOB,
frac_vol_sand_cumul BLOB
)
"""
self.create_Calibration_parameters = """
CREATE TABLE CalibrationParameters(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
acoustic_data INTEGER,
freq_1 INTEGER,
freq_2 INTEGER,
fine_profiles TEXT,
sand_target INTEGER
)
"""
self.create_Calibration = """
CREATE TABLE Calibration(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
path_calibration_file STRING,
filename_calibration_file STRING,
range_lin_interp BLOB,
M_profile_fine BLOB,
ks BLOB,
sv BLOB,
X_exponent BLOB,
alpha_s BLOB,
zeta BLOB,
FCB BLOB,
depth_real BLOB,
lin_reg BLOB
)
"""
self.create_Inversion = """
CREATE TABLE Inversion(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
J_cross_section_freq1 BLOB,
J_cross_section_freq2 BLOB,
VBI_cross_section BLOB,
SSC_fine BLOB,
SSC_sand BLOB
)
"""
self.create_Notes = """
CREATE TABLE Notes(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
notes TEXT
)
"""
def save_as(self):
self.open_file_dialog()
self.save()
def save(self):
start = time.time()
self.create_table()
print(f"end : {time.time() - start} sec")
def open_file_dialog(self):
name, _ = QFileDialog.getSaveFileName(
caption="Save As",
directory="",
filter="AcouSed Files (*.acd)",
options=QFileDialog.DontUseNativeDialog
)
if name != "":
filename = os.path.basename(name)
if os.path.splitext(filename)[1] != ".acd":
filename += ".acd"
logger.debug(f"selected save file: '{filename}'")
stg.dirname_save_as = os.path.dirname(name)
stg.filename_save_as = filename
try:
os.chdir(stg.dirname_save_as)
except OSError as e:
logger.warning(f"chdir: {str(e)}")
else:
msgBox = QMessageBox()
msgBox.setWindowTitle("Save Error")
msgBox.setIcon(QMessageBox.Warning)
msgBox.setText("No file saved")
msgBox.setStandardButtons(QMessageBox.Ok)
msgBox.exec()
def create_table(self):
cnx = sqlite3.connect(stg.filename_save_as)
cur = cnx.cursor()
self.create_table_acoustic_file(cnx, cur)
self.create_table_measure(cnx, cur)
self.create_table_BSRawData(cnx, cur)
self.create_table_settings(cnx, cur)
self.create_table_sediments_file(cnx, cur)
self.create_table_sediments_data(cnx, cur)
self.create_table_calibration_parameters(cnx, cur)
self.create_table_calibration(cnx, cur)
self.create_table_inversion(cnx, cur)
self.create_table_notes(cnx, cur)
cnx.commit()
cur.close()
cnx.close()
def create_table_acoustic_file(self, cnx, cur):
start_table_File = time.time()
cur.execute("DROP TABLE if exists AcousticFile")
cur.execute(self.create_AcousticFile)
for i in stg.acoustic_data:
logger.debug(f"stg.acoustic_data: {stg.acoustic_data[i]}")
logger.debug("stg.filename_BS_raw_data: "
+ f"{stg.filename_BS_raw_data[i]}")
logger.debug(f"stg.ABS_name: {stg.ABS_name}")
logger.debug(f"stg.path_BS_raw_data: {stg.path_BS_raw_data[i]}")
cur.execute(
"""
INSERT into AcousticFile(
acoustic_data,
acoustic_file,
ABS_name,
path_BS_noise_data,
filename_BS_noise_data,
noise_method,
noise_value,
data_preprocessed)
VALUES(?, ?, ?, ?, ?, ?, ?, ?)
""",
(
stg.acoustic_data[i],
stg.filename_BS_raw_data[i].split('.')[0],
stg.ABS_name[i],
stg.path_BS_noise_data[i],
stg.filename_BS_noise_data[i],
stg.noise_method[i],
stg.noise_value[i],
stg.data_preprocessed[i]
)
)
cnx.commit()
logger.info(f"table File : {time.time() - start_table_File} sec")
def create_table_measure(self, cnx, cur):
start_table_Measure = time.time()
# Drop Table if exists
cur.execute("DROP TABLE if exists Measure")
# Execute the CREATE TABLE statement
cur.execute(self.create_Measure)
logger.debug(f"stg.date: {stg.date}, stg.hour: {stg.hour}")
for i in stg.acoustic_data:
for j in range(stg.freq[i].shape[0]):
cur.execute(
"""
INSERT into Measure(
acoustic_data,
Date, Hour,
frequency,
sound_attenuation,
kt_read, kt_corrected,
NbProfiles, NbProfilesPerSeconds,
NbCells, CellSize,
PulseLength,
NbPingsPerSeconds,
NbPingsAveragedPerProfile,
GainRx, GainTx
)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
stg.acoustic_data[i],
stg.date[i].isoformat(),
stg.hour[i].isoformat(),
stg.freq[i][j],
stg.water_attenuation[i][j],
stg.kt_read[j], stg.kt_corrected[j],
stg.nb_profiles[i][j], stg.nb_profiles_per_sec[i][j],
stg.nb_cells[i][j], stg.cell_size[i][j],
stg.pulse_length[i][j],
stg.nb_pings_per_sec[i][j],
stg.nb_pings_averaged_per_profile[i][j],
stg.gain_rx[i][j], stg.gain_tx[i][j]
)
)
# Commit the transaction after executing INSERT.
cnx.commit()
logger.info(f"table Measure : {time.time() - start_table_Measure} sec")
def create_table_BSRawData(self, cnx, cur):
start_table_BSRawData = time.time()
cur.execute('DROP TABLE if exists BSRawData')
# Drop Table BSRawData_i if exists
cur.execute(self.create_BSRawData)
for i in stg.acoustic_data:
cur.execute(
"""
INSERT into BSRawData(
acoustic_data,
time, depth,
BS_raw_data,
time_reshape,
depth_reshape,
BS_raw_data_reshape,
time_cross_section, depth_cross_section,
BS_cross_section, BS_stream_bed,
depth_bottom, val_bottom, ind_bottom,
time_noise, depth_noise, BS_noise_raw_data,
SNR_raw_data, SNR_cross_section, SNR_stream_bed,
BS_raw_data_pre_process_SNR, BS_raw_data_pre_process_average,
BS_cross_section_pre_process_SNR, BS_cross_section_pre_process_average,
BS_stream_bed_pre_process_SNR, BS_stream_bed_pre_process_average,
BS_mean
)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?)
""",
(
stg.acoustic_data[i], stg.time[i].tobytes(),
stg.depth[i].tobytes(), stg.BS_raw_data[i].tobytes(),
stg.time_reshape[i].tobytes(), stg.depth_reshape[i].tobytes(),
stg.BS_raw_data_reshape[i].tobytes(),
stg.time_cross_section[i].tobytes(),
stg.depth_cross_section[i].tobytes(),
stg.BS_cross_section[i].tobytes(), stg.BS_stream_bed[i].tobytes(),
stg.depth_bottom[i].tobytes(), np.array(stg.val_bottom[i]).tobytes(),
np.array(stg.ind_bottom[i]).tobytes(),
stg.time_noise[i].tobytes(), stg.depth_noise[i].tobytes(),
stg.BS_noise_raw_data[i].tobytes(),
stg.SNR_raw_data[i].tobytes(), stg.SNR_cross_section[i].tobytes(),
stg.SNR_stream_bed[i].tobytes(),
stg.BS_raw_data_pre_process_SNR[i].tobytes(),
stg.BS_raw_data_pre_process_average[i].tobytes(),
stg.BS_cross_section_pre_process_SNR[i].tobytes(),
stg.BS_cross_section_pre_process_average[i].tobytes(),
stg.BS_stream_bed_pre_process_SNR[i].tobytes(),
stg.BS_stream_bed_pre_process_average[i].tobytes(),
stg.BS_mean[i].tobytes()
)
)
# Commit the transaction after executing INSERT.
cnx.commit()
logger.info(f"table BSRawData : {time.time() - start_table_BSRawData} sec")
def create_table_settings(self, cnx, cur):
start_table_Settings = time.time()
cur.execute("DROP TABLE if exists Settings")
cur.execute(self.create_Settings)
logger.debug(f"acoustic_data: {stg.acoustic_data}")
logger.debug(f"temperature: {stg.temperature}")
logger.debug(f"rmin: {stg.rmin}, rmax: {stg.rmax}")
logger.debug(f"tmin: {stg.tmin}, tmax: {stg.tmax}")
for i in stg.acoustic_data:
cur.execute(
"""
INSERT into Settings(
acoustic_data, temperature, distance_to_free_surface,
tmin_index, tmin_value, tmax_index, tmax_value,
rmin_index, rmin_value, rmax_index, rmax_value,
freq_bottom_detection_index, freq_bottom_detection_value,
depth_bottom_detection_min,
depth_bottom_detection_max,
depth_bottom_detection_inverval,
SNR_filter_value, Nb_cells_to_average_BS_signal
)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
stg.acoustic_data[i], stg.temperature,
stg.distance_from_ABS_to_free_surface[i],
int(stg.tmin[i][0]), stg.tmin[i][1],
int(stg.tmax[i][0]), stg.tmax[i][1],
int(stg.rmin[i][0]), stg.rmin[i][1],
int(stg.rmax[i][0]), stg.rmax[i][1],
stg.freq_bottom_detection[i][0],
stg.freq_bottom_detection[i][1],
stg.depth_bottom_detection_min,
stg.depth_bottom_detection_max,
stg.depth_bottom_detection_interval,
stg.SNR_filter_value[i],
stg.Nb_cells_to_average_BS_signal[i]
)
)
cnx.commit()
logger.info(f"table Settings : {time.time() - start_table_Settings} sec")
def create_table_sediments_file(self, cnx, cur):
start_table_SedimentsFile = time.time()
cur.execute("DROP TABLE if exists SedimentsFile")
cur.execute(self.create_SedimentsFile)
if stg.path_fine != "" and stg.path_sand != "":
cur.execute(
"""
INSERT into SedimentsFile(
path_fine, filename_fine, radius_grain_fine,
path_sand, filename_sand, radius_grain_sand,
time_column_label, distance_from_bank_column_label,
depth_column_label, Ctot_fine_column_label,
D50_fine_column_label,
Ctot_sand_column_label, D50_sand_column_label
)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
stg.path_fine, stg.filename_fine,
stg.radius_grain_fine.tobytes(),
stg.path_sand, stg.filename_sand,
stg.radius_grain_sand.tobytes(),
stg.columns_fine[0], stg.columns_fine[1],
stg.columns_fine[2], stg.columns_fine[3],
stg.columns_fine[4],
stg.columns_sand[3], stg.columns_sand[4]
)
)
cnx.commit()
logger.info(f"table SedimentsFile : {time.time() - start_table_SedimentsFile} sec")
def create_table_sediments_data(self, cnx, cur):
start_table_SedimentsData = time.time()
cur.execute("DROP TABLE if exists SedimentsData")
cur.execute(self.create_SedimentsData)
for f in range(len(stg.sample_fine)):
cur.execute(
"""
INSERT into SedimentsData(
sample_fine_name, sample_fine_index,
distance_from_bank_fine,
depth_fine, time_fine, Ctot_fine,
Ctot_fine_per_cent, D50_fine,
frac_vol_fine, frac_vol_fine_cumul,
sample_sand_name, sample_sand_index,
distance_from_bank_sand,
depth_sand, time_sand, Ctot_sand,
Ctot_sand_per_cent, D50_sand,
frac_vol_sand, frac_vol_sand_cumul
)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
stg.sample_fine[f][0] , stg.sample_fine[f][1],
stg.distance_from_bank_fine[f], stg.depth_fine[f],
stg.time_fine[f], stg.Ctot_fine[f],
stg.Ctot_fine_per_cent[f], stg.D50_fine[f],
stg.frac_vol_fine[f].tobytes(),
stg.frac_vol_fine_cumul[f].tobytes(),
stg.sample_sand[f][0], stg.sample_sand[f][1],
stg.distance_from_bank_sand[f], stg.depth_sand[f],
stg.time_sand[f], stg.Ctot_sand[f],
stg.Ctot_sand_per_cent[f], stg.D50_sand[f],
stg.frac_vol_sand[f].tobytes(),
stg.frac_vol_sand_cumul[f].tobytes()
)
)
cnx.commit()
logger.info(f"table SedimentsData : {time.time() - start_table_SedimentsData} sec")
def create_table_calibration_parameters(self, cnx, cur):
start_table = time.time()
cur.execute("DROP TABLE if exists CalibrationParameters")
cur.execute(self.create_Calibration_parameters)
if stg.calib_acoustic_data != -1:
cur.execute(
"""
INSERT INTO CalibrationParameters(
acoustic_data,
freq_1, freq_2,
fine_profiles,
sand_target
)
VALUES(?, ?, ?, ?, ?)
""",
(
stg.calib_acoustic_data,
stg.calib_freq_1, stg.calib_freq_2,
",".join(map(str, stg.calib_fine_profiles)),
stg.calib_sand_target,
)
)
cnx.commit()
logger.info(f"table CalibrationParameters : {time.time() - start_table} sec")
def create_table_calibration(self, cnx, cur):
start_table_Calibration = time.time()
cur.execute("DROP TABLE if exists Calibration")
cur.execute(self.create_Calibration)
if len(stg.range_lin_interp) != 0:
cur.execute(
"""
INSERT INTO Calibration(
path_calibration_file, filename_calibration_file,
range_lin_interp, M_profile_fine,
ks, sv, X_exponent, alpha_s, zeta,
FCB, depth_real, lin_reg
)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
stg.path_calibration_file, stg.filename_calibration_file,
stg.range_lin_interp.tobytes(),
stg.M_profile_fine.tobytes(),
np.array(stg.ks).tobytes(), np.array(stg.sv).tobytes(),
np.array(stg.X_exponent).tobytes(),
np.array(stg.alpha_s).tobytes(),
np.array(stg.zeta).tobytes(),
np.array(stg.FCB).tobytes(),
np.array(stg.depth_real).tobytes(),
np.array(stg.lin_reg).tobytes()
)
)
cnx.commit()
logger.info(f"table Calibration : {time.time() - start_table_Calibration} sec")
def create_table_inversion(self, cnx, cur):
start_table_Inversion = time.time()
cur.execute("DROP TABLE if exists Inversion")
cur.execute(self.create_Inversion)
for i in range(len(stg.SSC_fine)):
cur.execute(
"""
INSERT into Inversion(
J_cross_section_freq1, J_cross_section_freq2,
VBI_cross_section, SSC_fine, SSC_sand
)
VALUES(?, ?, ?, ?, ?)
""",
(
stg.J_cross_section[i][0].tobytes(),
stg.J_cross_section[i][1].tobytes(),
stg.VBI_cross_section[i].tobytes(),
stg.SSC_fine[i].tobytes(),
stg.SSC_sand[i].tobytes()
)
)
cnx.commit()
logger.info(f"table Inversion : {time.time() - start_table_Inversion} sec")
def create_table_notes(self, cnx, cur):
start_table = time.time()
cur.execute("DROP TABLE if exists Notes")
cur.execute(self.create_Notes)
for i in range(len(stg.SSC_fine)):
cur.execute(
"""
INSERT into Notes(
notes
)
VALUES(?)
""",
(stg.notes,)
)
cnx.commit()
logger.info(f"table Notes : {time.time() - start_table} sec")
Reference in New Issue View Git Blame Copy Permalink