Names of BS numpy arrays are changed : BS_data and BS_data_section become, respectively, BS_cross_section and BS_stream_bed. In general, name of variables including data under river bottom line is composed with _cross_section. Name of variables with NaN under river bottom line is composed with _stream_bed.

This commit is contained in:
brahim committed 2023-11-13 15:00:28 +01:00
1 parent 1b19bd1175
commit 3dd6a4e310
7 files changed
+827 -429

No files matched your search

+18 -5
View File
@@ -10,7 +10,8 @@ from matplotlib.colors import LogNorm
# path_noise_data = "/home/bmoudjed/Documents/3 SSC acoustic meas project/Graphical interface project/" \
# "Data/AcousticNoise_data/20180107121600.aqa"
class AcousticDataLoader():
class AcousticDataLoader:
def __init__(self, path_BS_raw_data: str):
@@ -48,6 +49,9 @@ class AcousticDataLoader():
self._gain_rx = self._data_BS.RxGain
self._gain_tx = self._data_BS.TxGain
print(self._r[0, :][1] - self._r[1, :][0])
print(self._kt)
# self._snr = np.array([])
# self._snr_reshape = np.array([])
# self._time_snr = np.array([])
@@ -61,11 +65,16 @@ class AcousticDataLoader():
# print(np.where((self._time) == 155)[0][0])
# fig, ax = plt.subplots(nrows=1, ncols=1)
# ax.pcolormesh(self._time, self._r, (self._BS_raw_data[:, 1, :]),
# ax.pcolormesh(self._time[0, :2200], -self._r[0, :], (self._BS_raw_data[0, :, :2200]),
# cmap='viridis',
# norm=LogNorm(vmin=1e-5, vmax=np.max(self._BS_raw_data[0, :, :2200]))) # , shading='gouraud')
# ax.pcolormesh(range(self._BS_raw_data.shape[2]), range(self._BS_raw_data.shape[0]), self._BS_raw_data[:, 1, :], cmap='viridis',
# norm=LogNorm(vmin=1e-5, vmax=np.max(self._BS_raw_data[:, 0, :]))) # , shading='gouraud')
# # ax.pcolormesh(range(self._BS_raw_data.shape[2]), range(self._BS_raw_data.shape[0]), self._BS_raw_data[:, 1, :], cmap='viridis',
# # norm=LogNorm(vmin=1e-5, vmax=np.max(self._BS_raw_data[:, 0, :]))) # , shading='gouraud')
# plt.show()
# --- Plot vertical profile for bottom detection ---
# fig2, ax2 = plt.subplots(nrows=1, ncols=1, layout="constrained")
# ax2.plot(self._BS_raw_data[0, :, 1], -self._r[0], "k.-")
# plt.show()
# fig, ax = plt.subplots(nrows=1, ncols=1)
@@ -77,6 +86,7 @@ class AcousticDataLoader():
# self.reshape_BS_raw_cross_section()
# self.reshape_r()
# self.reshape_t()
# self.compute_r_2D()
def reshape_BS_raw_cross_section(self):
BS_raw_cross_section = np.reshape(self._BS_raw_data,
@@ -96,7 +106,10 @@ class AcousticDataLoader():
return r
def compute_r_2D(self):
r2D = np.repeat(self._r, self._time.size, axis=1)
r2D = np.zeros((self._freq.shape[0], self._r.shape[1], self._time.shape[1]))
for f, _ in enumerate(self._freq):
r2D[f, :, :] = np.repeat(np.transpose(self._r[0, :])[:, np.newaxis], self._time.shape[1], axis=1)
print(r2D.shape)
return r2D
def reshape_t(self):
+135 -58
View File
@@ -1,10 +1,12 @@
from Model.AquascatDataLoader import RawAquascatData
import numpy as np
import pandas as pd
import datetime
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm, BoundaryNorm
from copy import deepcopy
from scipy.signal import savgol_filter
from Model.udt_extract.raw_extract import raw_extract
# raw_20210519_102332.udt raw_20210520_135452.udt raw_20210525_092759.udt raw_20210525_080454.udt
@@ -31,7 +33,7 @@ from Model.udt_extract.raw_extract import raw_extract
# "raw_20210526_153310.udt"
class AcousticDataLoaderUBSediFlow():
class AcousticDataLoaderUBSediFlow:
def __init__(self, path_BS_raw_data: str):
@@ -45,6 +47,10 @@ class AcousticDataLoaderUBSediFlow():
device_name, time_begin, time_end, param_us_dicts, data_us_dicts, data_dicts, settings_dict \
= raw_extract(self.path_BS_raw_data)
print(f"device_name : {device_name}")
print(f"time_begin : {time_begin}")
print(f"settings_dict : {settings_dict}")
# --- Date and Hour of measurements read on udt data file ---
filename = self.path_BS_raw_data[-23:]
date_and_time = datetime.datetime(year=int(filename[4:8]),
@@ -63,28 +69,47 @@ class AcousticDataLoaderUBSediFlow():
self._time = np.array([[]])
self._time_snr = np.array([[]])
self._BS_raw_data = np.array([[[]]])
self._SNR_data = np.array([[[]]])
# self._SNR_data = np.array([[[]]])
time_len = []
time_snr_len = []
for config in param_us_dicts.keys():
print("-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x")
print(f"config : {config} \n")
# print("-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x")
# print(f"config : {config} \n")
for channel in param_us_dicts[config].keys():
print("-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x")
print(f"channel : {channel} \n")
# print(param_us_dicts[1][1])
# print(f"channel : {channel} \n")
print(param_us_dicts[config][channel])
# print(data_us_dicts[config][channel]['echo_avg_profile'])
# --- Frequencies ---
self._freq = np.append(self._freq, param_us_dicts[config][channel]['f0'])
# --- Depth for each frequencies ---
depth = [param_us_dicts[config][channel]['r_cell1'] * i
print("r_dcell : ", param_us_dicts[config][channel]['r_dcell'])
print("n_cell : ", param_us_dicts[config][channel]['n_cell'])
depth = [param_us_dicts[config][channel]['r_dcell'] * i
for i in list(range(param_us_dicts[config][channel]['n_cell']))]
print(f"depth : {depth}")
print(f"lenght of depth : {len(depth)}")
if self._r.shape[1] == 0:
self._r = np.array([depth])
else:
self._r = np.append(self._r, [depth], axis=0)
if len(depth) == self._r.shape[1]:
print("Je suis là")
print(f"depth lenght : {len(depth)}")
print(f"r shape : {self._r.shape}")
self._r = np.append(self._r, np.array([depth]), axis=0)
print("C'est encore moi")
elif len(depth) < self._r.shape[1]:
print(f"depth lenght : {len(depth)}")
self._r = self._r[:, :len(depth)]
self._r = np.append(self._r, np.array([depth]), axis=0)
print(f"r shape : {self._r.shape}")
elif len(depth) > self._r.shape[1]:
print(f"depth lenght : {len(depth)}")
self._r = np.append(self._r, np.array([depth[:self._r.shape[1]]]), axis=0)
print(f"r shape : {self._r.shape}")
print(f"self._r : {self._r.shape}")
# --- BS Time for each frequencies ---
time = [[(t - data_us_dicts[config][channel]['echo_avg_profile']['time'][0]).total_seconds()
@@ -92,72 +117,72 @@ class AcousticDataLoaderUBSediFlow():
time_len = np.append(time_len, len(time[0]))
if len(time_len) == 1:
# print(f"1 time length : {len(time[0])}")
print(f"1 time length : {len(time[0])}")
self._time = np.array(time)
# print(f"self._time.shape {self._time.shape}")
print(f"self._time.shape {self._time.shape}")
elif self._time.shape[1] == len(time[0]):
# print(f"2 time length : {len(time[0])}")
print(f"2 time length : {len(time[0])}")
self._time = np.append(self._time, time, axis=0)
# print(f"self._time.shape {self._time.shape}")
print(f"self._time.shape {self._time.shape}")
elif self._time.shape[1] > len(time[0]):
# print(f"3 time length : {len(time[0])}")
print(f"3 time length : {len(time[0])}")
# print(f"self._time.shape {self._time.shape}")
# print([int(np.min(time_len)) + int(i) - 1 for i in range(1, int(np.max(time_len))-int(np.min(time_len))+1)])
self._time = np.delete(self._time,
[int(np.min(time_len)) + int(i) - 1 for i in range(1, int(np.max(time_len))-int(np.min(time_len))+1)],
axis=1)
self._time = np.append(self._time, time, axis=0)
# print(f"self._time.shape {self._time.shape}")
print(f"self._time.shape {self._time.shape}")
elif self._time.shape[1] < len(time[0]):
# print(f"4 time length : {len(time[0])}")
print(f"4 time length : {len(time[0])}")
time = time[:int(np.max(time_len)) - (int(np.max(time_len)) - int(np.min(time_len)))]
self._time = np.append(self._time, time, axis=0)
# print(f"self._time.shape {self._time.shape}")
print(f"self._time.shape {self._time.shape}")
# --- SNR Time for each frequencies ---
time_snr = [[(t - data_us_dicts[config][channel]['snr_doppler_avg_profile']['time'][0]).total_seconds()
for t in data_us_dicts[config][channel]['snr_doppler_avg_profile']['time']]]
time_snr_len = np.append(time_snr_len, len(time_snr[0]))
if len(time_snr_len) == 1:
# print(f"1 time length : {len(time[0])}")
self._time_snr = np.array(time_snr)
# print(f"self._time.shape {self._time.shape}")
elif self._time_snr.shape[1] == len(time_snr[0]):
# print(f"2 time length : {len(time[0])}")
self._time_snr = np.append(self._time_snr, time_snr, axis=0)
# print(f"self._time.shape {self._time.shape}")
elif self._time_snr.shape[1] > len(time_snr[0]):
# print(f"3 time length : {len(time[0])}")
# print(f"self._time.shape {self._time.shape}")
# print([int(np.min(time_len)) + int(i) - 1 for i in range(1, int(np.max(time_len))-int(np.min(time_len))+1)])
self._time_snr = np.delete(self._time_snr,
[int(np.min(time_snr_len)) + int(i) - 1 for i in
range(1, int(np.max(time_snr_len)) - int(np.min(time_snr_len)) + 1)],
axis=1)
self._time_snr = np.append(self._time_snr, time_snr, axis=0)
# print(f"self._time.shape {self._time.shape}")
elif self._time_snr.shape[1] < len(time_snr[0]):
# print(f"4 time length : {len(time[0])}")
time_snr = time_snr[:int(np.max(time_snr_len)) - (int(np.max(time_snr_len)) - int(np.min(time_snr_len)))]
self._time_snr = np.append(self._time_snr, time_snr, axis=0)
# print(f"self._time.shape {self._time.shape}")
# time_snr = [[(t - data_us_dicts[config][channel]['snr_doppler_avg_profile']['time'][0]).total_seconds()
# for t in data_us_dicts[config][channel]['snr_doppler_avg_profile']['time']]]
# time_snr_len = np.append(time_snr_len, len(time_snr[0]))
#
# if len(time_snr_len) == 1:
# # print(f"1 time length : {len(time[0])}")
# self._time_snr = np.array(time_snr)
# # print(f"self._time.shape {self._time.shape}")
# elif self._time_snr.shape[1] == len(time_snr[0]):
# # print(f"2 time length : {len(time[0])}")
# self._time_snr = np.append(self._time_snr, time_snr, axis=0)
# # print(f"self._time.shape {self._time.shape}")
# elif self._time_snr.shape[1] > len(time_snr[0]):
# # print(f"3 time length : {len(time[0])}")
# # print(f"self._time.shape {self._time.shape}")
# # print([int(np.min(time_len)) + int(i) - 1 for i in range(1, int(np.max(time_len))-int(np.min(time_len))+1)])
# self._time_snr = np.delete(self._time_snr,
# [int(np.min(time_snr_len)) + int(i) - 1 for i in
# range(1, int(np.max(time_snr_len)) - int(np.min(time_snr_len)) + 1)],
# axis=1)
# self._time_snr = np.append(self._time_snr, time_snr, axis=0)
# # print(f"self._time.shape {self._time.shape}")
# elif self._time_snr.shape[1] < len(time_snr[0]):
# # print(f"4 time length : {len(time[0])}")
# time_snr = time_snr[:int(np.max(time_snr_len)) - (int(np.max(time_snr_len)) - int(np.min(time_snr_len)))]
# self._time_snr = np.append(self._time_snr, time_snr, axis=0)
# # print(f"self._time.shape {self._time.shape}")
# --- US Backscatter raw signal + SNR data ---
BS_data = np.array([[]])
if config == 1:
BS_data = np.array([data_us_dicts[config][channel]['echo_avg_profile']['data'][0]])
print("BS_data shape ", BS_data.shape)
# print("BS_data shape ", BS_data.shape)
for i in range(self._time.shape[1]):
BS_data = np.append(BS_data,
np.array([data_us_dicts[config][channel]['echo_avg_profile']['data'][i]]),
axis=0)
print("BS_data shape ", BS_data.shape)
print("0. BS_data shape ", BS_data.shape)
self._BS_raw_data = np.array([BS_data[:self._time.shape[1], :].transpose()])
print("BS_raw_data shape ", self._BS_raw_data.shape)
print("0. BS_raw_data shape ", self._BS_raw_data.shape)
# fig, ax = plt.subplots(nrows=1, ncols=1, layout="constrained")
# pcm = ax.pcolormesh(list(range(self._BS_raw_data.shape[2])), list(range(self._BS_raw_data.shape[1])),
@@ -169,28 +194,40 @@ class AcousticDataLoaderUBSediFlow():
else:
BS_data = np.array([data_us_dicts[config][channel]['echo_avg_profile']['data'][0]])
print("BS_data shape ", BS_data.shape)
# print("BS_data shape ", BS_data.shape)
for i in range(self._time.shape[1]):
BS_data = np.append(BS_data,
np.array(
[data_us_dicts[config][channel]['echo_avg_profile']['data'][i]]),
axis=0)
print("BS_data shape ", BS_data.shape)
print("1. BS_data shape ", BS_data.shape)
if BS_data.shape[0] > self._BS_raw_data.shape[2]:
#-----------------------------------------------------------------------------------------------------------------------
# Ici il faut écrire les conditions sur les tailles selon r et selon time
# donc sur BS_data.shape[0] (time) et BS_data.shape[1] (depth)
#-----------------------------------------------------------------------------------------------------------------------
# 1- time shape > BS data shape
# <=> data recorded with the frequency are longer than data recorded with the other lower frequencies
if (BS_data.shape[0] > self._BS_raw_data.shape[2]):
self._BS_raw_data = np.append(self._BS_raw_data,
np.array([BS_data[:self._BS_raw_data.shape[2], :].transpose()]),
axis=0)
# 2- time shape < BS data shape
# <=> data recorded with the frequency are shorter than data recorded with the other lower frequencies
elif BS_data.shape[0] < self._BS_raw_data.shape[2]:
self._BS_raw_data = np.append(self._BS_raw_data[config-1, :, BS_data.shape[0]],
np.array([BS_data.transpose()]),
axis=0)
# 3- time shape = BS data shape
# <=> data recorded with the frequency have the same duration than data recorded with the other lower frequency
else:
self._BS_raw_data = np.append(self._BS_raw_data, np.array([BS_data.transpose()]), axis=0)
print("BS_raw_data shape ", self._BS_raw_data.shape)
print("1. BS_raw_data shape ", self._BS_raw_data.shape)
# if f == 0:
# print(np.array(data_us_dicts[config][channel]['echo_avg_profile']['data'][0]).shape)
@@ -298,11 +335,11 @@ class AcousticDataLoaderUBSediFlow():
# --- Plot Backscatter US data ---
# fig, ax = plt.subplots(nrows=1, ncols=1, layout="constrained")
# # pcm = ax.pcolormesh(self._time[0, :], self._r[0, :], np.log(self._BS_raw_data[0, :, :]),
# # cmap='Blues')#, shading='gouraud')
# pcm = ax.pcolormesh(list(range(self._BS_raw_data.shape[2])), list(range(self._BS_raw_data.shape[1])),
# np.log(self._BS_raw_data[0, :, :]),
# cmap='Blues') # , shading='gouraud')
# pcm = ax.pcolormesh(self._time[0, :], -self._r[0, :], np.log(self._BS_raw_data[0, :, :]),
# cmap='Blues')#, shading='gouraud')
# # pcm = ax.pcolormesh(list(range(self._BS_raw_data.shape[2])), list(range(self._BS_raw_data.shape[1])),
# # np.log(self._BS_raw_data[0, :, :]),
# # cmap='Blues') # , shading='gouraud')
# # norm=LogNorm(vmin=np.min(self._BS_raw_data[f, :, :]), vmax=np.max(self._BS_raw_data[f, :, :])), shading='gouraud')
# # ax.pcolormesh(range(self._BS_raw_data.shape[2]), range(self._BS_raw_data.shape[0]), self._BS_raw_data[:, 1, :], cmap='viridis',
# # norm=LogNorm(vmin=1e-5, vmax=np.max(self._BS_raw_data[:, 0, :]))) # , shading='gouraud')
@@ -317,12 +354,38 @@ class AcousticDataLoaderUBSediFlow():
# # pcm = ax[f].pcolormesh(list(range(self._BS_raw_data.shape[2])), list(range(self._BS_raw_data.shape[1])),
# # np.log(self._BS_raw_data[f, :, :]),
# # cmap='Blues', shading='gouraud')
# pcm = ax[f].pcolormesh(self._time[f, :], self._r[f, :], np.log(self._BS_raw_data[f, :, :]),
# cmap='viridis', shading='gouraud')
# pcm = ax[f].pcolormesh(self._time[f, 50:247], -self._r[f, :], np.log(self._BS_raw_data[f, :, 50:247]),
# cmap='Blues')#, shading='gouraud')
# # norm=LogNorm(vmin=np.min(self._BS_raw_data[f, :, :]), vmax=np.max(self._BS_raw_data[f, :, :])), shading='gouraud')
# # ax.pcolormesh(range(self._BS_raw_data.shape[2]), range(self._BS_raw_data.shape[0]), self._BS_raw_data[:, 1, :], cmap='viridis',
# # norm=LogNorm(vmin=1e-5, vmax=np.max(self._BS_raw_data[:, 0, :]))) # , shading='gouraud')
# fig.colorbar(pcm, ax=ax[:], shrink=1, location='right')
# # plt.show()
#
# # --- Smooth value with savgol_filter ---
# BS_smooth = deepcopy(self._BS_raw_data[0, :, :])
# for k in range(self._time[0, :].shape[0]):
# BS_smooth[:, k] = savgol_filter(BS_smooth[:, k], 10, 2)
#
# fig1, ax1 = plt.subplots(nrows=1, ncols=1, layout="constrained")
# pcm1 = ax1.pcolormesh(self._time[0, :], -self._r[0, :], np.log(BS_smooth[:, :]), cmap='Blues')
# fig1.colorbar(pcm1, ax=ax1, shrink=1, location='right')
#
# # --- Plot vertical profile for bottom detection ---
# n = 60
# t0 = 200
# t1 = np.where(np.abs(self._time[0, :] - t0) == np.nanmin(np.abs(self._time[0, :] - t0)))[0][0]
# # print(np.abs(self._time[0, :] - 200))
# # print(f"x0 = {x0}")
# r1 = 98
# r2 = 150
# fig2, ax2 = plt.subplots(nrows=1, ncols=n, layout="constrained")
# for i in range(n):
# ax2[i].plot(self._BS_raw_data[0, r1:r2, t1+i], -self._r[0, r1:r2], 'b')
# ax2[i].plot(BS_smooth[r1:r2, t1+i], -self._r[0, r1:r2], 'r')
# ax2[i].set_xticks([])
# if i != 0:
# ax2[i].set_yticks([])
# plt.show()
# --- Plot SNR data ---
@@ -373,6 +436,7 @@ class AcousticDataLoaderUBSediFlow():
# self.reshape_BS_raw_cross_section()
# self.reshape_r()
# self.reshape_t()
# self.compute_r_2D()
# Lecture du fichier excel
@@ -425,6 +489,13 @@ class AcousticDataLoaderUBSediFlow():
# print(r.shape)
return r
def compute_r_2D(self):
r2D = np.zeros((self._freq.shape[0], self._r.shape[1], self._time.shape[1]))
for f, _ in enumerate(self._freq):
r2D[f, :, :] = np.repeat(np.transpose(self._r[0, :])[:, np.newaxis], self._time.shape[1], axis=1)
print("r2D.shape ", r2D.shape)
return r2D
# def compute_r_2D(self):
# r2D = np.repeat(self._r, self._time.size, axis=1)
# return r2D
@@ -443,6 +514,12 @@ class AcousticDataLoaderUBSediFlow():
# print(t.shape)
return t
def detect_bottom(self):
rmin = 2.5
rmax = 3.5
# def concatenate_data(self):
# self.reshape_BS_raw_cross_section()