From 27c08c934bced061870fba9e8c084873bd0dd528 Mon Sep 17 00:00:00 2001 From: Baymax Date: Tue, 23 Jun 2026 04:21:23 +0000 Subject: [PATCH 1/4] changed the naming of the ouputfiles to YYYY_MM_DD format --- ppcpy/io/write2nc.py | 9 ++++++--- ppcpy/visualization/pypolly_readout.py | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ppcpy/io/write2nc.py b/ppcpy/io/write2nc.py index 824d641..7701b73 100644 --- a/ppcpy/io/write2nc.py +++ b/ppcpy/io/write2nc.py @@ -101,7 +101,8 @@ def write_channelwise_2_nc_file(data_cube, root_dir=root_dir, prod_ls=[]): yyyy,mm,dd = date_splitting(data_cube.date) output_path = Path(data_cube.picasso_config_dict["results_folder"],data_cube.device,yyyy,mm,dd) output_path.mkdir(parents=True, exist_ok=True) - output_filename = Path(output_path, f"{data_cube.date}_{data_cube.device}_{prod}.nc") + date_string = datetime.datetime.strptime(data_cube.date, "%Y%m%d").strftime("%Y_%m_%d") + output_filename = Path(output_path, f"{date_string}_{data_cube.device}_{prod}.nc") json2nc_mapping.create_netcdf_from_dict(output_filename, data_cube, json_nc_mapping_dict, compression_level=1, prod=prod) def write2nc_file(data_cube, root_dir=root_dir, prod_ls=[]): @@ -155,7 +156,8 @@ def write2nc_file(data_cube, root_dir=root_dir, prod_ls=[]): yyyy,mm,dd = date_splitting(data_cube.date) output_path = Path(data_cube.picasso_config_dict["results_folder"],data_cube.device,yyyy,mm,dd) output_path.mkdir(parents=True, exist_ok=True) - output_filename = Path(output_path, f"{data_cube.date}_{data_cube.device}_{prod}.nc") + date_string = datetime.datetime.strptime(data_cube.date, "%Y%m%d").strftime("%Y_%m_%d") + output_filename = Path(output_path, f"{date_string}_{data_cube.device}_{prod}.nc") json2nc_mapping.create_netcdf_from_dict(output_filename, data_cube, json_nc_mapping_dict, compression_level=1, prod=prod) @@ -249,7 +251,8 @@ def write_profile2nc_file(data_cube, root_dir:str=root_dir, prod_ls:list=[], col yyyy,mm,dd = date_splitting(data_cube.date) output_path = Path(data_cube.picasso_config_dict["results_folder"],data_cube.device,yyyy,mm,dd) output_path.mkdir(parents=True, exist_ok=True) - output_filename = Path(output_path, f"{data_cube.date}_{data_cube.device}_{start}_{stop}_{prod}.nc") + date_string = datetime.datetime.strptime(data_cube.date, "%Y%m%d").strftime("%Y_%m_%d") + output_filename = Path(output_path, f"{date_string}_{data_cube.device}_{start}_{stop}_{prod}.nc") json2nc_mapping.create_netcdf_from_dict(output_filename, data_cube, json_nc_mapping_dict, compression_level=1, prod=prod, cldFreeIndx=n) diff --git a/ppcpy/visualization/pypolly_readout.py b/ppcpy/visualization/pypolly_readout.py index 7fb86aa..50b6041 100644 --- a/ppcpy/visualization/pypolly_readout.py +++ b/ppcpy/visualization/pypolly_readout.py @@ -61,9 +61,11 @@ def get_nc_filename(date, device, inputfolder, param=""): #file_searchpattern = f"{YYYY}_{MM}_{DD}_*[0-9]_{param}.nc" if not "profile" in param: - file_searchpattern = f"{date}_{device}_{param}.nc" + #file_searchpattern = f"{date}_{device}_{param}.nc" + file_searchpattern = f"{YYYY}_{MM}_{DD}_{device}_{param}.nc" else: - file_searchpattern = f"{date}_{device}_*[0-9][0-9]_{param}.nc" + #file_searchpattern = f"{date}_{device}_*[0-9][0-9]_{param}.nc" + file_searchpattern = f"{YYYY}_{MM}_{DD}_{device}_*[0-9][0-9]_{param}.nc" res_file = Path(r'{}'.format(inputfolder)).glob('{}'.format(file_searchpattern)) ## convert type path to type string From 163a2a73b7d88937e742dcf6af11b748dba430c0 Mon Sep 17 00:00:00 2001 From: Baymax Date: Wed, 1 Jul 2026 11:32:00 +0000 Subject: [PATCH 2/4] corrected SNR,BG,RCS writing to nc file and added RCS,SNR,BG plotting routine --- ppcpy/config/json2nc-mapper_BG.json | 42 +-- ppcpy/config/json2nc-mapper_RCS.json | 42 +-- ppcpy/config/json2nc-mapper_SNR.json | 42 +-- ppcpy/interface/picassopy_operational.py | 8 +- .../visualization/pypolly_display_3d_plots.py | 264 +++++++++++++++--- ppcpy/visualization/pypolly_display_all.py | 50 +++- .../visualization/pypolly_display_profiles.py | 2 +- ppcpy/visualization/pypolly_readout.py | 78 ++++-- 8 files changed, 389 insertions(+), 139 deletions(-) diff --git a/ppcpy/config/json2nc-mapper_BG.json b/ppcpy/config/json2nc-mapper_BG.json index e4068fa..cefa6b0 100644 --- a/ppcpy/config/json2nc-mapper_BG.json +++ b/ppcpy/config/json2nc-mapper_BG.json @@ -75,7 +75,7 @@ "calendar": "julian" } }, - "BG_FR_total_355nm": { + "BG_355_total_FR": { "dtype": "f8", "shape": [ "time" @@ -86,13 +86,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "background FR_total_355nm", - "standard_name": "BG_FR_total_355nm", + "standard_name": "BG_355_total_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "BG_FR_cross_355nm": { + "BG_355_cross_FR": { "dtype": "f8", "shape": [ "time" @@ -103,13 +103,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "background FR_cross_355nm", - "standard_name": "BG_FR_cross_355nm", + "standard_name": "BG_355_cross_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "BG_FR_387nm": { + "BG_387_total_FR": { "dtype": "f8", "shape": [ "time" @@ -126,7 +126,7 @@ "comment": "" } }, - "BG_FR_407nm": { + "BG_407_total_FR": { "dtype": "f8", "shape": [ "time" @@ -143,7 +143,7 @@ "comment": "" } }, - "BG_FR_total_532nm": { + "BG_532_total_FR": { "dtype": "f8", "shape": [ "time" @@ -154,13 +154,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "background FR_total_532nm", - "standard_name": "BG_FR_total_532nm", + "standard_name": "BG_532_total_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "BG_FR_cross_532nm": { + "BG_532_cross_FR": { "dtype": "f8", "shape": [ "time" @@ -171,7 +171,7 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "background FR_cross_532nm", - "standard_name": "BG_FR_cross_532nm", + "standard_name": "BG_532_cross_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" @@ -194,7 +194,7 @@ "comment": "" } }, - "BG_FR_607nm": { + "BG_607_total_FR": { "dtype": "f8", "shape": [ "time" @@ -211,7 +211,7 @@ "comment": "" } }, - "BG_FR_total_1064nm": { + "BG_1064_total_FR": { "dtype": "f8", "shape": [ "time" @@ -222,7 +222,7 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "background FR_total_1064nm", - "standard_name": "BG_FR_total_1064nm", + "standard_name": "BG_1064_total_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" @@ -245,7 +245,7 @@ "comment": "" } }, - "BG_NR_total_355nm": { + "BG_355_total_NR": { "dtype": "f8", "shape": [ "time" @@ -256,13 +256,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "background NR_total_355nm", - "standard_name": "BG_NR_total_355nm", + "standard_name": "BG_355_total_NR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "BG_NR_387nm": { + "BG_387_total_NR": { "dtype": "f8", "shape": [ "time" @@ -279,7 +279,7 @@ "comment": "" } }, - "BG_NR_total_532nm": { + "BG_532_total_NR": { "dtype": "f8", "shape": [ "time" @@ -290,13 +290,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "background NR_total_532nm", - "standard_name": "BG_NR_total_532nm", + "standard_name": "BG_532_total_NR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "BG_NR_cross_532nm": { + "BG_532_cross_NR": { "dtype": "f8", "shape": [ "time" @@ -307,13 +307,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "background NR_cross_532nm", - "standard_name": "BG_NR_cross_532nm", + "standard_name": "BG_532_cross_NR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "BG_NR_607nm": { + "BG_607_total_NR": { "dtype": "f8", "shape": [ "time" diff --git a/ppcpy/config/json2nc-mapper_RCS.json b/ppcpy/config/json2nc-mapper_RCS.json index 103be78..546a0a3 100644 --- a/ppcpy/config/json2nc-mapper_RCS.json +++ b/ppcpy/config/json2nc-mapper_RCS.json @@ -89,7 +89,7 @@ "calendar": "julian" } }, - "RCS_FR_total_355nm": { + "RCS_355_total_FR": { "dtype": "f8", "shape": [ "time", @@ -101,13 +101,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "range corrected signal FR_total_355nm", - "standard_name": "RCS_FR_total_355nm", + "standard_name": "RCS_355_total_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "RCS_FR_cross_355nm": { + "RCS_355_cross_FR": { "dtype": "f8", "shape": [ "time", @@ -119,13 +119,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "range corrected signal FR_cross_355nm", - "standard_name": "RCS_FR_cross_355nm", + "standard_name": "RCS_355_cross_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "RCS_FR_387nm": { + "RCS_387_total_FR": { "dtype": "f8", "shape": [ "time", @@ -143,7 +143,7 @@ "comment": "" } }, - "RCS_FR_407nm": { + "RCS_407_total_FR": { "dtype": "f8", "shape": [ "time", @@ -161,7 +161,7 @@ "comment": "" } }, - "RCS_FR_total_532nm": { + "RCS_532_total_FR": { "dtype": "f8", "shape": [ "time", @@ -173,13 +173,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "range corrected signal FR_total_532nm", - "standard_name": "RCS_FR_total_532nm", + "standard_name": "RCS_532_total_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "RCS_FR_cross_532nm": { + "RCS_532_cross_FR": { "dtype": "f8", "shape": [ "time", @@ -191,7 +191,7 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "range corrected signal FR_cross_532nm", - "standard_name": "RCS_FR_cross_532nm", + "standard_name": "RCS_532_cross_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" @@ -215,7 +215,7 @@ "comment": "" } }, - "RCS_FR_607nm": { + "RCS_607_total_FR": { "dtype": "f8", "shape": [ "time", @@ -233,7 +233,7 @@ "comment": "" } }, - "RCS_FR_total_1064nm": { + "RCS_1064_total_FR": { "dtype": "f8", "shape": [ "time", @@ -245,7 +245,7 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "range corrected signal FR_total_1064nm", - "standard_name": "RCS_FR_total_1064nm", + "standard_name": "RCS_1064_total_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" @@ -269,7 +269,7 @@ "comment": "" } }, - "RCS_NR_total_355nm": { + "RCS_355_total_NR": { "dtype": "f8", "shape": [ "time", @@ -281,13 +281,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "range corrected signal NR_total_355nm", - "standard_name": "RCS_NR_total_355nm", + "standard_name": "RCS_355_total_NR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "RCS_NR_387nm": { + "RCS_387_total_NR": { "dtype": "f8", "shape": [ "time", @@ -305,7 +305,7 @@ "comment": "" } }, - "RCS_NR_total_532nm": { + "RCS_532_total_NR": { "dtype": "f8", "shape": [ "time", @@ -317,13 +317,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "range corrected signal NR_total_532nm", - "standard_name": "RCS_NR_total_532nm", + "standard_name": "RCS_532_total_NR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "RCS_NR_cross_532nm": { + "RCS_532_cross_NR": { "dtype": "f8", "shape": [ "time", @@ -335,13 +335,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "range corrected signal NR_cross_532nm", - "standard_name": "RCS_NR_cross_532nm", + "standard_name": "RCS_532_cross_NR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "RCS_NR_607nm": { + "RCS_607_total_NR": { "dtype": "f8", "shape": [ "time", diff --git a/ppcpy/config/json2nc-mapper_SNR.json b/ppcpy/config/json2nc-mapper_SNR.json index b53f534..2279abd 100644 --- a/ppcpy/config/json2nc-mapper_SNR.json +++ b/ppcpy/config/json2nc-mapper_SNR.json @@ -89,7 +89,7 @@ "calendar": "julian" } }, - "SNR_FR_total_355nm": { + "SNR_355_total_FR": { "dtype": "f8", "shape": [ "time", @@ -101,13 +101,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "signal to noise ratio FR_total_355nm", - "standard_name": "SNR_FR_total_355nm", + "standard_name": "SNR_355_total_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "SNR_FR_cross_355nm": { + "SNR_355_cross_FR": { "dtype": "f8", "shape": [ "time", @@ -119,13 +119,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "signal to noise ratio FR_cross_355nm", - "standard_name": "SNR_FR_cross_355nm", + "standard_name": "SNR_355_cross_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "SNR_FR_387nm": { + "SNR_387_total_FR": { "dtype": "f8", "shape": [ "time", @@ -143,7 +143,7 @@ "comment": "" } }, - "SNR_FR_407nm": { + "SNR_407_total_FR": { "dtype": "f8", "shape": [ "time", @@ -161,7 +161,7 @@ "comment": "" } }, - "SNR_FR_total_532nm": { + "SNR_532_total_FR": { "dtype": "f8", "shape": [ "time", @@ -173,13 +173,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "signal to noise ratio FR_total_532nm", - "standard_name": "SNR_FR_total_532nm", + "standard_name": "SNR_532_total_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "SNR_FR_cross_532nm": { + "SNR_532_cross_FR": { "dtype": "f8", "shape": [ "time", @@ -191,7 +191,7 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "signal to noise ratio FR_cross_532nm", - "standard_name": "SNR_FR_cross_532nm", + "standard_name": "SNR_532_cross_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" @@ -215,7 +215,7 @@ "comment": "" } }, - "SNR_FR_607nm": { + "SNR_607_total_FR": { "dtype": "f8", "shape": [ "time", @@ -233,7 +233,7 @@ "comment": "" } }, - "SNR_FR_total_1064nm": { + "SNR_1064_total_FR": { "dtype": "f8", "shape": [ "time", @@ -245,7 +245,7 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "signal to noise ratio FR_total_1064nm", - "standard_name": "SNR_FR_total_1064nm", + "standard_name": "SNR_1064_total_FR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" @@ -269,7 +269,7 @@ "comment": "" } }, - "SNR_NR_total_355nm": { + "SNR_355_total_NR": { "dtype": "f8", "shape": [ "time", @@ -281,13 +281,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "signal to noise ratio NR_total_355nm", - "standard_name": "SNR_NR_total_355nm", + "standard_name": "SNR_355_total_NR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "SNR_NR_387nm": { + "SNR_387_total_NR": { "dtype": "f8", "shape": [ "time", @@ -305,7 +305,7 @@ "comment": "" } }, - "SNR_NR_total_532nm": { + "SNR_532_total_NR": { "dtype": "f8", "shape": [ "time", @@ -317,13 +317,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "signal to noise ratio NR_total_532nm", - "standard_name": "SNR_NR_total_532nm", + "standard_name": "SNR_532_total_NR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "SNR_NR_cross_532nm": { + "SNR_532_cross_NR": { "dtype": "f8", "shape": [ "time", @@ -335,13 +335,13 @@ "unit": "a.u.", "unit_html": "a.u.", "long_name": "signal to noise ratio NR_cross_532nm", - "standard_name": "SNR_NR_cross_532nm", + "standard_name": "SNR_532_cross_NR", "plot_range": "0., 50.", "plot_scale": "linear", "comment": "" } }, - "SNR_NR_607nm": { + "SNR_607_total_NR": { "dtype": "f8", "shape": [ "time", diff --git a/ppcpy/interface/picassopy_operational.py b/ppcpy/interface/picassopy_operational.py index c898c25..e5cdb87 100644 --- a/ppcpy/interface/picassopy_operational.py +++ b/ppcpy/interface/picassopy_operational.py @@ -149,7 +149,7 @@ ## write channelwise infos to nc-files (e.g.: SNR, Background, RangeCorrectedSignal) write2nc.write_channelwise_2_nc_file(data_cube=data_cube,prod_ls=["SNR","BG","RCS"]) - +exit() ## saturation detection data_cube.SaturationDetect() @@ -230,8 +230,10 @@ ## write depolarization calibration factors and LIDAR calibration constants to sqlite-db base_dir = Path(data_cube.picasso_config_dict['results_folder']) -#db_path = base_dir.joinpath(polly_device,polly_config_dict['calibrationDB']) -db_path = "/data/level1b/polly24h/pollyxt_cpv/pollyxt_cpv_calibration_picassopy.db" +db_path = base_dir.joinpath(polly_device,polly_config_dict['calibrationDB']) +#db_path = "/data/level1b/polly24h/arielle/arielle_calibration_neumeyer.db" +logging.info(f'db_file: {db_path}') +#db_path = "/data/level1b/polly24h/pollyxt_cpv/pollyxt_cpv_calibration_picassopy.db" ## calc. LIDAR calibration constants data_cube.LidarCalibration(db_path=db_path) ## gives also data_cube.pol_cali, data_cube.LCused (e.g.: data_cube.LCused['532_total_FR']) diff --git a/ppcpy/visualization/pypolly_display_3d_plots.py b/ppcpy/visualization/pypolly_display_3d_plots.py index efc15b9..c26669c 100644 --- a/ppcpy/visualization/pypolly_display_3d_plots.py +++ b/ppcpy/visualization/pypolly_display_3d_plots.py @@ -133,7 +133,8 @@ def pollyDisplayAttnBsc(nc_dict, config_dict, polly_conf_dict, saveFolder, wavel ## read from nc file ATT_BETA = nc_dict[f'attenuated_backscatter_{wavelength}nm']['data'] if param == 'FR' or param == 'NR': - SNR = nc_dict[f'SNR_{wavelength}nm']['data'] + if f'SNR_{wavelength}nm' in nc_dict.keys(): + SNR = nc_dict[f'SNR_{wavelength}nm']['data'] quality_mask = nc_dict[f'quality_mask_{wavelength}nm']['data'] elif param == 'OC': quality_mask = np.where(ATT_BETA > 0, 0, 0) @@ -356,6 +357,10 @@ def pollyDisplayAttnBsc(nc_dict, config_dict, polly_conf_dict, saveFolder, wavel ## plotting SNR if param == 'FR' or param == 'NR': + if f'SNR_{wavelength}nm' in nc_dict.keys(): + pass + else: + return None ## fill time gaps in snr matrix SNR, quality_mask_SNR = readout.fill_time_gaps_of_matrix(time, SNR, quality_mask) @@ -2330,7 +2335,7 @@ def pollyDisplay_Overlap(nc_dict,config_dict,polly_conf_dict,outdir,donefilelist ) -def pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelength, param,donefilelist_dict): +def pollyDisplay_preprocessed(nc_dict, config_dict, polly_conf_dict, saveFolder, variable,donefilelist_dict): """ Description ----------- @@ -2338,15 +2343,13 @@ def pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelengt Parameters ---------- + variable: variables for SNR or RCS or BG, e.g. RCS_532_total_NR nc_dict: dict - dict wich stores the RCS data. - wavelength: int - the selected wavelength channel: e.g.: 355/532/1064 nm + dict which stores the preprocessed data (SNR,RCS or BG). Usage ----- - pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelength, param) - param can be one of the following: ['FR','NR'] + pollyDisplay_preprocessed(nc_dict, config_dict, polly_conf_dict, saveFolder, variable,donefilelist_dict) History ------- @@ -2358,21 +2361,23 @@ def pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelengt fontname = config_dict['fontname'] - + var, wavelength, polarization, telescope = readout.gf(variable) + #print(var, wavelength, polarization, telescope) ## read from global config file - if "FR" in param or "RR" in param: - param_l = "FR" - elif "NR" in param: - param_l = "NR" - yLim = polly_conf_dict[f'yLim_{param_l}_RCS'] - zLim = polly_conf_dict[f'zLim_{param_l}_RCS_{wavelength}'] +# yLim = polly_conf_dict[f'yLim_{telescope}_{var}'] + yLim = polly_conf_dict[f'yLim_{telescope}_RCS'] + #zLim = polly_conf_dict[f'zLim_{telescope}_{var}_{wavelength}'] + try: + zLim = polly_conf_dict[f'zLim_{telescope}_{var}_{wavelength}'] + except: + zLim = [None,None] partnerLabel = polly_conf_dict['partnerLabel'] colormap_basic = polly_conf_dict['colormap_basic'] imgFormat = polly_conf_dict['imgFormat'] ## read from nc file - RCS_matrix = nc_dict[f'RCS_{param}_{wavelength}nm'] - quality_mask = np.where(RCS_matrix > 0, 0, -999.0) + data_matrix = nc_dict[variable]['data'] + quality_mask = np.where(data_matrix > 0, 0, -999.0) height = nc_dict['height']['data'] time = nc_dict['time']['data'] @@ -2380,24 +2385,21 @@ def pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelengt pollyVersion = nc_dict['PollyVersion'] location = nc_dict['location'] version = nc_dict['PicassoVersion'] - dataFilename = re.split(r'_RCS',nc_dict['PollyDataFile'])[0] + dataFilename = re.split(f'_{var}',nc_dict['PollyDataFile'])[0] # set the default font matplotlib.rcParams['font.sans-serif'] = fontname matplotlib.rcParams['font.family'] = "sans-serif" - plotfile = f'{dataFilename}_RCS_{param}_{wavelength}.{imgFormat}' - prod_type = f'RCS_{param}_{wavelength}' - + plotfile = f'{dataFilename}_{variable}.{imgFormat}' saveFilename = os.path.join(saveFolder,plotfile) ## fill time gaps in att_bsc matrix -# RCS_matrix, quality_mask_ATT = readout.fill_time_gaps_of_matrix(time, RCS_matrix, quality_mask) - RCS_matrix = readout.fill_time_gaps_of_single_matrix(time, RCS_matrix) + data_matrix = readout.fill_time_gaps_of_single_matrix(time, data_matrix) ## get date and convert to datetime object - date_00 = datetime.strptime(nc_dict['m_date'], '%Y-%m-%d') + date_00 = datetime.strptime(nc_dict['m_date'], '%Y%m%d') date_00 = date_00.timestamp() ## set x-lim to 24h or only to last available timestamp @@ -2414,17 +2416,17 @@ def pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelengt extent = [ x_lims[0], x_lims[-1], max_height[0], max_height[-1] ] # ## mask matrix - RCS_matrix = np.ma.masked_where(RCS_matrix <= 0, RCS_matrix) + data_matrix = np.ma.masked_where(data_matrix <= 0, data_matrix) ## slice matrix to max_height - RCS_matrix = RCS_matrix[:,0:len(max_height)] + data_matrix = data_matrix[:,0:len(max_height)] ## trimm matrix to last available timestamp if neccessary - RCS_matrix = readout.trimm_matrix_to_last_timestamp(flagPlotLastProfilesOnly=config_dict['flagPlotLastProfilesOnly'],matrix=RCS_matrix,mdate=date_00,profile_length=int(np.nanmean(np.diff(time))),last_timestamp=nc_dict['time']['data'][-1]) + data_matrix = readout.trimm_matrix_to_last_timestamp(flagPlotLastProfilesOnly=config_dict['flagPlotLastProfilesOnly'],matrix=data_matrix,mdate=date_00,profile_length=int(np.nanmean(np.diff(time))),last_timestamp=nc_dict['time']['data'][-1]) ## transpose and flip for correct plotting - RCS_matrix= np.ma.transpose(RCS_matrix) ## matrix has to be transposed for usage with pcolormesh! - RCS_matrix= np.flip(RCS_matrix,0) + data_matrix= np.ma.transpose(data_matrix) ## matrix has to be transposed for usage with pcolormesh! + data_matrix= np.flip(data_matrix,0) # define the colormap cmap = load_colormap(name=colormap_basic) @@ -2440,7 +2442,7 @@ def pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelengt fig = plt.figure(figsize=[12, 6]) ax = fig.add_axes([0.11, 0.15, 0.79, 0.75]) pcmesh = ax.imshow( - RCS_matrix / 1e6, + data_matrix / 1e6, cmap=cmap, vmin=zLim[0], vmax=zLim[1], @@ -2470,18 +2472,21 @@ def pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelengt right=True, top=True) ax.set_title( - 'Range Corrected Signal at {wave} nm'.format(wave = wavelength) + + '{var} at {wave} nm'.format(var=var,wave = wavelength) + ' {param} of {instrument} at {location}'.format( - param=param, + param=telescope, instrument=pollyVersion, location=location), fontsize=15) - + if zLim[0] is None or zLim[1] is None: + tick_values = None + else: + tick_values = np.linspace(zLim[0], zLim[1], 5) cb_ax = fig.add_axes([0.92, 0.25, 0.02, 0.55]) cbar = fig.colorbar( pcmesh, cax=cb_ax, - ticks=np.linspace(zLim[0], zLim[1], 5), + ticks=tick_values, orientation='vertical') cbar.ax.tick_params(direction='in', labelsize=15, pad=5) # cbar.ax.set_title(' $Mm^{-1}*sr^{-1}$\n', fontsize=10) @@ -2527,7 +2532,198 @@ def pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelengt remove_widht_height_entry_from_svg_file(saveFilename,imgFormat) + prod_type = variable + ## write2donefilelist + readout.write2donefilelist_dict(donefilelist_dict = donefilelist_dict, + lidar = pollyVersion, + location = nc_dict['location'], + starttime = datetime.utcfromtimestamp(int(nc_dict['time']['data'][0])).strftime('%Y%m%d %H:%M:%S'), + stoptime = datetime.utcfromtimestamp(int(nc_dict['time']['data'][-1])).strftime('%Y%m%d %H:%M:%S'), + last_update = datetime.now(timezone.utc).strftime("%Y%m%d %H:%M:%S"), + wavelength = wavelength, + filename = saveFilename, + level = 0, + info = f"{var} plots for {telescope} at {wavelength}", + nc_zip_file = nc_dict['PollyDataFile'], + nc_zip_file_size = 9000000, + active = 1, + GDAS = 0, + GDAS_timestamp = f"{datetime.utcfromtimestamp(int(nc_dict['time']['data'][0])).strftime('%Y%m%d')} 12:00:00", + lidar_ratio = 50, + software_version = version, + product_type = prod_type, + product_starttime = datetime.utcfromtimestamp(int(nc_dict['time']['data'][0])).strftime('%Y%m%d %H:%M:%S'), + product_stoptime = datetime.utcfromtimestamp(int(nc_dict['time']['data'][-1])).strftime('%Y%m%d %H:%M:%S') + ) + + +def pollyDisplay_preprocessed_BG(nc_dict, config_dict, polly_conf_dict, saveFolder, variable,donefilelist_dict): + """ + Description + ----------- + Display the RangeCorrected (FR/NR) of wavelength [nm] channel from level1 polly nc-file. + + Parameters + ---------- + variable: variables for SNR or RCS or BG, e.g. RCS_532_total_NR + nc_dict: dict + dict which stores the preprocessed data (SNR,RCS or BG). + + Usage + ----- + pollyDisplay_preprocessed(nc_dict, config_dict, polly_conf_dict, saveFolder, variable,donefilelist_dict) + + History + ------- + 2022-09-01. First edition by Andi + """ + ## read from config file + figDPI = config_dict['figDPI'] + flagWatermarkOn = config_dict['flagWatermarkOn'] + fontname = config_dict['fontname'] + + + var, wavelength, polarization, telescope = readout.gf(variable) + ## read from global config file + partnerLabel = polly_conf_dict['partnerLabel'] + colormap_basic = polly_conf_dict['colormap_basic'] + imgFormat = polly_conf_dict['imgFormat'] + + ## read from nc file + data_matrix = nc_dict[variable]['data'] + #quality_mask = np.where(data_matrix > 0, 0, -999.0) + + time = nc_dict['time']['data'] + + timediffs = np.diff(time) + diffs_rounded = np.round(timediffs, decimals=3) + vals, counts = np.unique(diffs_rounded, return_counts=True) + common_delta = vals[np.argmax(counts)] + seconds_in_day = 86400 + + # 00:00:00 UTC dieses Tages + start_of_day = (time[0] // seconds_in_day) * seconds_in_day + + # Bis exakt zum Ende des Tages (00:00:00 UTC des Folgetages) + end_of_day = start_of_day + seconds_in_day + + # --- SCHRITT 3: Den kompletten Tag lückenlos generieren --- + # np.arange stoppt EXKLUSIVE dem Endwert. + # Bei 30s Delta ist der letzte Punkt somit 23:59:30 UTC. + perfect_day_timestamps = np.arange(start_of_day, end_of_day, common_delta) + + + pollyVersion = nc_dict['PollyVersion'] + location = nc_dict['location'] + version = nc_dict['PicassoVersion'] + dataFilename = re.split(f'_{var}',nc_dict['PollyDataFile'])[0] + + # set the default font + matplotlib.rcParams['font.sans-serif'] = fontname + matplotlib.rcParams['font.family'] = "sans-serif" + + plotfile = f'{dataFilename}_{variable}.{imgFormat}' + saveFilename = os.path.join(saveFolder,plotfile) + + ## fill time gaps in att_bsc matrix + data_matrix_h = np.expand_dims(data_matrix, axis=1) ## add dummy-height-dim + data_matrix_h = readout.fill_time_gaps_of_single_matrix(time, data_matrix_h) + + + ## get date and convert to datetime object + date_00 = datetime.strptime(nc_dict['m_date'], '%Y%m%d') + date_00 = date_00.timestamp() + + ## set x-lim to 24h or only to last available timestamp + x_lims = readout.set_x_lims(flagPlotLastProfilesOnly=config_dict['flagPlotLastProfilesOnly'],mdate=date_00,last_timestamp=nc_dict['time']['data'][-1]) + + ## convert these datetime.datetime objects to the correct format for matplotlib to work with. + x_lims = date2num(x_lims) + + ## trimm matrix to last available timestamp if neccessary + data_matrix_h = readout.trimm_matrix_to_last_timestamp(flagPlotLastProfilesOnly=config_dict['flagPlotLastProfilesOnly'],matrix=data_matrix_h,mdate=date_00,profile_length=int(np.nanmean(np.diff(time))),last_timestamp=nc_dict['time']['data'][-1]) + + data_matrix = np.squeeze(data_matrix_h, axis=1)[0:len(perfect_day_timestamps)] + data_matrix = np.where(data_matrix > 0, data_matrix, 0) + + + print(f"plotting {plotfile} ... ") + # display attenuate backscatter + fig = plt.figure(figsize=[12, 6]) + ax = fig.add_axes([0.11, 0.15, 0.79, 0.75]) + ax.plot(perfect_day_timestamps.astype('datetime64[s]'),data_matrix) + # convert the datetime data from a float (which is the output of date2num into a nice datetime string. + ax.xaxis_date() + + ax.set_xlabel('Time [UTC]', fontsize=15) + ax.set_ylabel('Background [a.u.]', fontsize=15) + + ax.xaxis.set_minor_locator(HourLocator(interval=1)) # every hour + if config_dict['flagPlotLastProfilesOnly'] == True: + ax.xaxis.set_major_locator(HourLocator(interval=2)) + else: + ax.xaxis.set_major_locator(HourLocator(byhour = [4,8,12,16,20,24])) + + ax.xaxis.set_major_formatter(DateFormatter('%H:%M')) +# + ax.tick_params( + axis='both', which='major', labelsize=15, right=True, + top=True, width=2, length=5) + ax.tick_params( + axis='both', which='minor', width=1.5, length=3.5, + right=True, top=True) + + ax.set_title( + 'Background Signal at {wave} nm'.format(wave = wavelength) + + ' {param} of {instrument} at {location}'.format( + param=telescope, + instrument=pollyVersion, + location=location), + fontsize=15) + + + # add watermark + if flagWatermarkOn: + #rootDir = os.getcwd() + rootDir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + im_license = matplotlib.image.imread( + os.path.join(rootDir, 'ppcpy', 'img', 'by-sa.png')) + + newax_license = fig.add_axes([0.58, 0.006, 0.14, 0.07], zorder=10) + newax_license.imshow(im_license, alpha=0.8, aspect='equal') + newax_license.axis('off') + + fig.text(0.72, 0.003, 'Preliminary\nResults.', + fontweight='bold', fontsize=12, color='red', + ha='left', va='bottom', alpha=0.8, zorder=10) + + fig.text( + 0.84, 0.003, + u"\u00A9 {1} {0}.\nCC BY SA 4.0 License.".format( + datetime.now().strftime('%Y'), partnerLabel), + fontweight='bold', fontsize=7, color='black', ha='left', + va='bottom', alpha=1, zorder=10) + fig.text( + 0.05, 0.02, + '{0}'.format(nc_dict['m_date']), fontsize=12) + + fig.text( + 0.2, 0.02, + 'Version: {version}'.format( + version=version), + fontsize=12) + + print(saveFilename) + print(figDPI) + fig.savefig(saveFilename,dpi=figDPI) + + plt.close() + remove_widht_height_entry_from_svg_file(saveFilename,imgFormat) + + + prod_type = variable ## write2donefilelist readout.write2donefilelist_dict(donefilelist_dict = donefilelist_dict, lidar = pollyVersion, @@ -2538,7 +2734,7 @@ def pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, saveFolder, wavelengt wavelength = wavelength, filename = saveFilename, level = 0, - info = f"Range corrected Signal plots for {param} at {wavelength}", + info = f"Background Signal plots for {telescope} at {wavelength}", nc_zip_file = nc_dict['PollyDataFile'], nc_zip_file_size = 9000000, active = 1, diff --git a/ppcpy/visualization/pypolly_display_all.py b/ppcpy/visualization/pypolly_display_all.py index 6ec1bd0..55897f9 100644 --- a/ppcpy/visualization/pypolly_display_all.py +++ b/ppcpy/visualization/pypolly_display_all.py @@ -63,7 +63,7 @@ help='the output folder to put the png files to.') my_parser.add_argument('--retrieval', dest='retrieval', metavar='retrieval parameter', default=['all'], - choices=['all','attbsc','voldepol','cloudinfo','target_class','wvmr_rh','quasi_results','profiles','overlap','LC','HKD','longterm_cali','profile_summary','poliphon','RCS'], + choices=['all','attbsc','voldepol','cloudinfo','target_class','wvmr_rh','quasi_results','profiles','overlap','LC','HKD','longterm_cali','profile_summary','poliphon','RCS','SNR','BG'], nargs='+', type=str, help='the retrievals to be plotted; default: "all".') @@ -179,18 +179,52 @@ def main(): nc_files = readout.get_nc_filename(date, device, inputfolder, param='RCS') for data_file in nc_files: nc_dict = readout.read_nc_file(data_file,date,device,location) - param_ls = ['RCS_FR_355nm', 'RCS_FR_cross_355nm', 'RCS_NR_355nm', 'RCS_RR_355nm', 'RCS_FR_387nm', 'RCS_NR_387nm', 'RCS_FR_407nm', 'RCS_NR_407nm', 'RCS_FR_532nm', 'RCS_FR_cross_532nm','RCS_FR_parallel_532nm', 'RCS_NR_532nm', 'RCS_NR_cross_532nm', 'RCS_RR_532nm', 'RCS_FR_607nm', 'RCS_NR_607nm', 'RCS_FR_1064nm', 'RCS_FR_cross_1064nm', 'RCS_RR_1064nm'] + param_ls = ['RCS_355_total_FR', 'RCS_355_cross_FR', 'RCS_387_total_FR', 'RCS_407_total_FR', 'RCS_532_total_FR', 'RCS_532_cross_FR', 'RCS_607_total_FR', 'RCS_1064_total_FR', 'RCS_355_total_NR', 'RCS_387_total_NR', 'RCS_532_total_NR', 'RCS_607_total_NR'] +# param_ls = ['RCS_FR_355nm', 'RCS_FR_cross_355nm', 'RCS_NR_355nm', 'RCS_RR_355nm', 'RCS_FR_387nm', 'RCS_NR_387nm', 'RCS_FR_407nm', 'RCS_NR_407nm', 'RCS_FR_532nm', 'RCS_FR_cross_532nm','RCS_FR_parallel_532nm', 'RCS_NR_532nm', 'RCS_NR_cross_532nm', 'RCS_RR_532nm', 'RCS_FR_607nm', 'RCS_NR_607nm', 'RCS_FR_1064nm', 'RCS_FR_cross_1064nm', 'RCS_RR_1064nm'] for p in param_ls: - p1 = re.split(r'RCS_',p)[1] - param = re.split(r'_[1-9].*nm',p1)[0] - wavelength = re.split(f'{param}_',p1)[-1] - wavelength = re.split(r'nm',wavelength)[0] + print(p) - if np.all(nc_dict[p].mask): ## do not plot empty/non-existing channels + if p not in nc_dict.keys(): ## do not plot empty/non-existing channels continue else: print(f'plotting {p}') - display_3d.pollyDisplayRCS(nc_dict, config_dict, polly_conf_dict, outputfolder, wavelength=wavelength,param=param,donefilelist_dict=donefilelist_dict) + display_3d.pollyDisplay_preprocessed(nc_dict, config_dict, polly_conf_dict, outputfolder, variable=p,donefilelist_dict=donefilelist_dict) + except Exception as e: + logging.exception("An error occurred") + + if ('all' in args.retrieval) or ('SNR' in args.retrieval): + ## plotting SNR plots + try: + nc_files = readout.get_nc_filename(date, device, inputfolder, param='SNR') + for data_file in nc_files: + nc_dict = readout.read_nc_file(data_file,date,device,location) + param_ls = ['SNR_355_total_FR', 'SNR_355_cross_FR', 'SNR_387_total_FR', 'SNR_407_total_FR', 'SNR_532_total_FR', 'SNR_532_cross_FR', 'SNR_607_total_FR', 'SNR_1064_total_FR', 'SNR_355_total_NR', 'SNR_387_total_NR', 'SNR_532_total_NR', 'SNR_607_total_NR'] + for p in param_ls: + print(p) + + if p not in nc_dict.keys(): ## do not plot empty/non-existing channels + continue + else: + print(f'plotting {p}') + display_3d.pollyDisplay_preprocessed(nc_dict, config_dict, polly_conf_dict, outputfolder, variable=p,donefilelist_dict=donefilelist_dict) + except Exception as e: + logging.exception("An error occurred") + + if ('all' in args.retrieval) or ('BG' in args.retrieval): + ## plotting BG plots + try: + nc_files = readout.get_nc_filename(date, device, inputfolder, param='BG') + for data_file in nc_files: + nc_dict = readout.read_nc_file(data_file,date,device,location) + param_ls = ['BG_355_total_FR', 'BG_355_cross_FR', 'BG_387_total_FR', 'BG_407_total_FR', 'BG_532_total_FR', 'BG_532_cross_FR', 'BG_607_total_FR', 'BG_1064_total_FR', 'BG_355_total_NR', 'BG_387_total_NR', 'BG_532_total_NR', 'BG_607_total_NR'] + for p in param_ls: + print(p) + + if p not in nc_dict.keys(): ## do not plot empty/non-existing channels + continue + else: + print(f'plotting {p}') + display_3d.pollyDisplay_preprocessed_BG(nc_dict, config_dict, polly_conf_dict, outputfolder, variable=p,donefilelist_dict=donefilelist_dict) except Exception as e: logging.exception("An error occurred") diff --git a/ppcpy/visualization/pypolly_display_profiles.py b/ppcpy/visualization/pypolly_display_profiles.py index a32812e..30dcec0 100644 --- a/ppcpy/visualization/pypolly_display_profiles.py +++ b/ppcpy/visualization/pypolly_display_profiles.py @@ -1150,7 +1150,7 @@ def plotting_procedure(col,param_dict,parameter,xlabel,xlim=[0,1],ylim=[0,1],sca for n,p in enumerate(param_dict[parameter]["NR"]): - if p == None: + if p == None or p not in nc_dict_profile_NR.keys(): continue color_ls = ['cyan','lime'] diff --git a/ppcpy/visualization/pypolly_readout.py b/ppcpy/visualization/pypolly_readout.py index 50b6041..aa9fb72 100644 --- a/ppcpy/visualization/pypolly_readout.py +++ b/ppcpy/visualization/pypolly_readout.py @@ -420,14 +420,18 @@ def smoothed_data(data,window_size): window_size = 25 - log_klett_355_532 = compute_valid_log(smoothed_data(data=nc_dict['aerBsc_klett_355']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerBsc_klett_532']['data'],window_size=window_size)) - if 'aerBsc_klett_1064' in nc_dict.keys(): + if 'aerBsc_klett_355' in nc_dict.keys() and 'aerBsc_klett_532' in nc_dict.keys(): + log_klett_355_532 = compute_valid_log(smoothed_data(data=nc_dict['aerBsc_klett_355']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerBsc_klett_532']['data'],window_size=window_size)) + if 'aerBsc_klett_532' in nc_dict.keys() and 'aerBsc_klett_1064' in nc_dict.keys(): log_klett_532_1064 = compute_valid_log(smoothed_data(data=nc_dict['aerBsc_klett_532']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerBsc_klett_1064']['data'],window_size=window_size)) - log_raman_355_532 = compute_valid_log(smoothed_data(data=nc_dict['aerBsc_raman_355']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerBsc_raman_532']['data'],window_size=window_size)) - if 'aerBsc_raman_1064' in nc_dict.keys(): + if 'aerBsc_raman_355' in nc_dict.keys() and 'aerBsc_raman_532' in nc_dict.keys(): + log_raman_355_532 = compute_valid_log(smoothed_data(data=nc_dict['aerBsc_raman_355']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerBsc_raman_532']['data'],window_size=window_size)) + if 'aerBsc_raman_532' in nc_dict.keys() and 'aerBsc_raman_1064' in nc_dict.keys(): log_raman_532_1064 = compute_valid_log(smoothed_data(data=nc_dict['aerBsc_raman_532']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerBsc_raman_1064']['data'],window_size=window_size)) - log_LR_355_532 = compute_valid_log(smoothed_data(data=nc_dict['aerLR_raman_355']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerLR_raman_532']['data'],window_size=window_size)) - log_Ext_raman_355_532 = compute_valid_log(smoothed_data(data=nc_dict['aerExt_raman_355']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerExt_raman_532']['data'],window_size=window_size)) + if 'aerLR_raman_355' in nc_dict.keys() and 'aerLR_raman_532' in nc_dict.keys(): + log_LR_355_532 = compute_valid_log(smoothed_data(data=nc_dict['aerLR_raman_355']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerLR_raman_532']['data'],window_size=window_size)) + if 'aerExt_raman_355' in nc_dict.keys() and 'aerExt_raman_532' in nc_dict.keys(): + log_Ext_raman_355_532 = compute_valid_log(smoothed_data(data=nc_dict['aerExt_raman_355']['data'],window_size=window_size),smoothed_data(data=nc_dict['aerExt_raman_532']['data'],window_size=window_size)) # log_klett_355_532 = compute_valid_log(nc_dict['aerBsc_klett_355'],nc_dict['aerBsc_klett_532']) # log_klett_532_1064 = compute_valid_log(nc_dict['aerBsc_klett_532'],nc_dict['aerBsc_klett_1064']) @@ -436,36 +440,40 @@ def smoothed_data(data,window_size): # log_LR_355_532 = compute_valid_log(nc_dict['aerLR_raman_355'],nc_dict['aerLR_raman_532']) # log_Ext_raman_355_532 = compute_valid_log(nc_dict['aerExt_raman_355'],nc_dict['aerExt_raman_532']) - AE_beta_355_532_Klett = log_klett_355_532/np.log(532/355) - if 'aerBsc_klett_1064' in nc_dict.keys(): + if 'log_klett_355_532' in locals(): + AE_beta_355_532_Klett = log_klett_355_532/np.log(532/355) + if 'aerBsc_klett_1064' in nc_dict.keys() and 'log_klett_532_1064' in locals(): AE_beta_532_1064_Klett = log_klett_532_1064/np.log(1064/532) - AE_beta_355_532_Raman = log_raman_355_532/np.log(532/355) - if 'aerBsc_raman_1064' in nc_dict.keys(): + if 'log_raman_355_532' in locals(): + AE_beta_355_532_Raman = log_raman_355_532/np.log(532/355) + if 'aerBsc_raman_1064' in nc_dict.keys() and 'log_raman_532_1064' in locals(): AE_beta_532_1064_Raman = log_raman_532_1064/np.log(1064/532) - AE_LR_355_532_Raman = log_LR_355_532/np.log(532/355) + if 'log_LR_355_532' in locals(): + AE_LR_355_532_Raman = log_LR_355_532/np.log(532/355) #AE_parExt_355_532_Raman = AE_beta_355_532_Raman + AE_LR_355_532_Raman - AE_parExt_355_532_Raman = log_Ext_raman_355_532/np.log(532/355) - - nc_dict['AE_beta_355_532_Klett'] = {} - nc_dict['AE_beta_355_532_Raman'] = {} - nc_dict['AE_beta_532_1064_Klett'] = {} - nc_dict['AE_beta_532_1064_Raman'] = {} - nc_dict['AE_LR_355_532_Raman'] = {} - nc_dict['AE_parExt_355_532_Raman'] = {} - - nc_dict['AE_beta_355_532_Klett']['data'] = AE_beta_355_532_Klett - nc_dict['AE_beta_355_532_Raman']['data'] = AE_beta_355_532_Raman - if 'aerBsc_klett_1064' in nc_dict.keys(): + if 'log_Ext_raman_355_532' in locals(): + AE_parExt_355_532_Raman = log_Ext_raman_355_532/np.log(532/355) + + + if 'AE_beta_355_532_Klett' in locals(): + nc_dict['AE_beta_355_532_Klett'] = {} + nc_dict['AE_beta_355_532_Klett']['data'] = AE_beta_355_532_Klett + if 'AE_beta_355_532_Raman' in locals(): + nc_dict['AE_beta_355_532_Raman'] = {} + nc_dict['AE_beta_355_532_Raman']['data'] = AE_beta_355_532_Raman + if 'AE_beta_532_1064_Klett' in locals(): + nc_dict['AE_beta_532_1064_Klett'] = {} nc_dict['AE_beta_532_1064_Klett']['data'] = AE_beta_532_1064_Klett - else: - del nc_dict['AE_beta_532_1064_Klett'] - if 'aerBsc_raman_1064' in nc_dict.keys(): + if 'AE_beta_532_1064_Raman' in locals(): + nc_dict['AE_beta_532_1064_Raman'] = {} nc_dict['AE_beta_532_1064_Raman']['data'] = AE_beta_532_1064_Raman - else: - del nc_dict['AE_beta_532_1064_Raman'] - nc_dict['AE_LR_355_532_Raman']['data'] = AE_LR_355_532_Raman - nc_dict['AE_parExt_355_532_Raman']['data'] = AE_parExt_355_532_Raman + if 'AE_LR_355_532_Raman' in locals(): + nc_dict['AE_LR_355_532_Raman'] = {} + nc_dict['AE_LR_355_532_Raman']['data'] = AE_LR_355_532_Raman + if 'AE_parExt_355_532_Raman' in locals(): + nc_dict['AE_parExt_355_532_Raman'] = {} + nc_dict['AE_parExt_355_532_Raman']['data'] = AE_parExt_355_532_Raman # nc_dict['AE_beta_355_532_Klett'] = smoothed_data(data=AE_beta_355_532_Klett,window_size=window_size) # nc_dict['AE_beta_355_532_Raman'] = smoothed_data(data=AE_beta_355_532_Raman,window_size=window_size) @@ -731,3 +739,13 @@ def extract_wavelength_from_parameterkey(parameter): else: result = None return result + +def gf(full_var_name): + """ + get parameters var, wavelength, polarization, telescope from full_var_name, e.g RCS_355_total_FR --> returns RCS, 355, total, FR + """ + + splitted = re.split(r'_',full_var_name) + return splitted[0],splitted[1],splitted[2],splitted[3] + + From 4b7efd3ab0e4b6452b55b7ce3db8fd7b7d7e2f24 Mon Sep 17 00:00:00 2001 From: Baymax Date: Mon, 6 Jul 2026 12:27:07 +0000 Subject: [PATCH 3/4] added overlap-write-to-file-procedure; added overlap-plotting-procedure --- .../json2nc-mapper_overlap_profiles.json | 275 ++++++++++++++++++ ppcpy/config/json2nc_translator.json | 60 ++++ ppcpy/interface/picassopy_go.sh | 25 +- ppcpy/interface/picassopy_operational.py | 8 +- ppcpy/interface/picassopy_plotting.sh | 2 +- ppcpy/io/write2nc.py | 35 ++- .../visualization/pypolly_display_3d_plots.py | 7 +- ppcpy/visualization/pypolly_display_all.py | 23 +- .../visualization/pypolly_display_profiles.py | 175 +++++++++++ ppcpy/visualization/pypolly_readout.py | 14 +- 10 files changed, 600 insertions(+), 24 deletions(-) create mode 100644 ppcpy/config/json2nc-mapper_overlap_profiles.json diff --git a/ppcpy/config/json2nc-mapper_overlap_profiles.json b/ppcpy/config/json2nc-mapper_overlap_profiles.json new file mode 100644 index 0000000..617b755 --- /dev/null +++ b/ppcpy/config/json2nc-mapper_overlap_profiles.json @@ -0,0 +1,275 @@ + +{ + "global_attributes": { + "Conventions": "CF-1.0", + "Licence": "Creative Commons Attribution Share Alike 4.0 International (CC BY-SA 4.0)", + "Data_Policy": "Each PollyNET site has Principal Investigator(s) (PI), responsible for deployment, maintenance and data collection. Information on which PI is responsible can be gathered via polly@tropos.de. The PI has priority use of the data collected at the site. The PI is entitled to be informed of any use of that data. Mandatory guidelines for data use and publication: Using PollyNET data or plots (also for presentations/workshops): Please consult with the PI or the PollyNET team (see contact_mail contact) before using data or plots! This will help to avoid misinterpretations of the lidar data and avoid the use of data from periods of malfunction of the instrument. Using PollyNET images/data on external websites: PIs and PollyNET must be asked for agreement and a link directed to polly.tropos.de must be included. Publishing PollyNET data and/or plots data: Offer authorship for the PI(s)! Acknowledge projects which have made the measurements possible according to PI(s) recommendation. PollyNET requests a notification of any published papers or reports or a brief description of other uses (e.g., posters, oral presentations, etc.) of data/plots used from PollyNET. This will help us determine the use of PollyNET data, which is helpful in optimizing product development and acquire new funding for future measurements. It also helps us to keep our product-related references up-to-date.", + "location": "", + "institute": "Ground-based Remote Sensing Group (TROPOS)", + "source": "__device", + "version": "picasso_version", + "reference": "https://polly.tropos.de/", + "contact": "", + "PicassoConfig_Info": "", + "PollyConfig_Info": "", + "CampaignConfig_Info": "", + "PollyData_Info": "", + "history": "" + }, + "dimensions": { + "height": 0, + "reference_height": 2 + }, + "variables": { + "altitude": { + "dtype": "f4", + "shape": [], + "data": null, + "attributes": { + "unit": "m", + "long_name": "Height of lidar above mean sea level", + "standard_name": "altitude" + } + }, + "longitude": { + "dtype": "f4", + "shape": [], + "data": null, + "attributes": { + "unit": "degrees_east", + "long_name": "Longitude of the site", + "standard_name": "longitude", + "axis": "Y" + } + }, + "latitude": { + "dtype": "f4", + "shape": [], + "data": null, + "attributes": { + "unit": "degrees_north", + "long_name": "Latitude of the site", + "standard_name": "latitude", + "axis": "Y" + } + }, + "start_time": { + "dtype": "f4", + "shape": [], + "data": null, + "attributes": { + "unit": "seconds since 1970-01-01 00:00:00 UTC", + "long_name": "Time UTC to finish the current measurement", + "standard_name": "time" + } + }, + "end_time": { + "dtype": "f4", + "shape": [], + "data": null, + "attributes": { + "unit": "seconds since 1970-01-01 00:00:00 UTC", + "long_name": "Time UTC to finish the current measurement", + "standard_name": "time" + } + }, + "height": { + "dtype": "f4", + "shape": ["height"], + "data": null, + "attributes": { + "unit": "m", + "long_name": "Height above the ground", + "standard_name": "height", + "axis": "Z" + } + }, + "tilt_angle": { + "dtype": "f4", + "shape": [], + "data": null, + "attributes": { + "unit": "degrees", + "long_name": "Tilt angle of lidar device", + "standard_name": "tilt_angle" + } + }, + "frnr_overlap_355": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 355nm far-range channel", + "standard_name": "overlap355", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "frnr_overlap_387": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 387nm far-range channel", + "standard_name": "overlap387", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "frnr_overlap_532": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 532nm far-range channel", + "standard_name": "overlap532", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "frnr_overlap_607": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 607nm far-range channel", + "standard_name": "overlap607", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "frnr_overlap_1064": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 1064nm far-range channel", + "standard_name": "overlap1064", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "raman_overlap_355_fr": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 355nm far-range channel (Raman method)", + "standard_name": "overlap355raman_fr", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "raman_overlap_532_fr": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 532nm far-range channel (Raman method)", + "standard_name": "overlap532raman_fr", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "raman_overlap_1064_fr": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 1064nm far-range channel (Raman method)", + "standard_name": "overlap1064raman_fr", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "raman_overlap_355_nr": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 355nm far-range channel (Raman method)", + "standard_name": "overlap355raman_nr", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "raman_overlap_355_nr": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 355nm far-range channel (Raman method)", + "standard_name": "overlap355raman_nr", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + }, + "raman_overlap_355_nr": { + "dtype": "f8", + "shape": ["height"], + "data": null, + "_FillValue": -999, + "attributes": { + "unit": "", + "unit_html": "", + "long_name": "overlap function for 355nm far-range channel (Raman method)", + "standard_name": "overlap355raman_nr", + "plot_range": [0, 1.1], + "plot_scale": "linear", + "source": "__device", + "comment": "This variable is not quality-assured. Only use with instructions from the PollyNET develop team." + } + } + + } +} diff --git a/ppcpy/config/json2nc_translator.json b/ppcpy/config/json2nc_translator.json index 62eb65a..94bedad 100644 --- a/ppcpy/config/json2nc_translator.json +++ b/ppcpy/config/json2nc_translator.json @@ -897,5 +897,65 @@ "parameter":"tcMaskV2" } } + }, + "overlap_profiles": { + "variables": { + "frnr_overlap_355": { + "method":"overlap", + "parameter":"frnr", + "channel":"355_total_FR" + }, + "frnr_overlap_387": { + "method":"overlap", + "parameter":"frnr", + "channel":"387_total_FR" + }, + "frnr_overlap_532": { + "method":"overlap", + "parameter":"frnr", + "channel":"532_total_FR" + }, + "frnr_overlap_607": { + "method":"overlap", + "parameter":"frnr", + "channel":"607_total_FR" + }, + "frnr_overlap_1064": { + "method":"overlap", + "parameter":"frnr", + "channel":"1064_total_FR" + }, + "raman_overlap_355_fr": { + "method":"overlap", + "parameter":"raman", + "channel":"355_total_FR" + }, + "raman_overlap_532_fr": { + "method":"overlap", + "parameter":"raman", + "channel":"532_total_FR" + }, + "raman_overlap_1064_fr": { + "method":"overlap", + "parameter":"raman", + "channel":"1064_total_FR" + }, + "raman_overlap_355_nr": { + "method":"overlap", + "parameter":"raman", + "channel":"355_total_NR" + }, + "raman_overlap_532_nr": { + "method":"overlap", + "parameter":"raman", + "channel":"532_total_NR" + }, + "raman_overlap_1064_nr": { + "method":"overlap", + "parameter":"raman", + "channel":"1064_total_NR" + } + } } + } diff --git a/ppcpy/interface/picassopy_go.sh b/ppcpy/interface/picassopy_go.sh index 4791d37..aef3103 100755 --- a/ppcpy/interface/picassopy_go.sh +++ b/ppcpy/interface/picassopy_go.sh @@ -10,6 +10,7 @@ FLAG_PROCESSING=0 # false TIMESTAMP="" DEVICE="" UNZIPPING="True" +FLAG_PREPROC_ONLY=0 # false PICASSO_CFG="" # ---------- helper functions ---------- @@ -26,6 +27,7 @@ Options: --unzipping Set to True or False, default is True; choose False i.e. for level0b 24h-file --merge_to_single_24h_file Flag to merge all level0 files of the day into one 24h file, default: false --processing Flag if processing or not (e.g. merge-only), default: false + --preproc_only Flag to only run the preprocessing, including the storage of SNR, BG and RCS files, default: false -h, --help Show this help and exit EOF exit 1 @@ -70,6 +72,10 @@ while [[ $# -gt 0 ]]; do FLAG_PROCESSING=1 shift ;; + --preproc_only) + FLAG_PREPROC_ONLY=1 + shift + ;; -h|--help) usage ;; @@ -267,7 +273,24 @@ for TIMESTAMP in ${DATE_LS[@]}; do # Call the actual processing script. # ----------------------------------------------------------------- - if (( FLAG_PROCESSING )); then + if (( FLAG_PREPROC_ONLY )); then + log_info "PreProcessing only" + log_info "Processing file: $rawfile" + PICASSO_OP_SCRIPT="${ROOT_DIR}/ppcpy/interface/picassopy_operational.py" + "$PYTHON_PATH"/python3 "$PICASSO_OP_SCRIPT" \ + --date "$TIMESTAMP" \ + --device "$DEVICE"\ + --base_dir "$BASE_DIR"\ + --picasso_config_file "$PICASSO_CFG" \ + --level0_file_to_process "$rawfile" \ + --preproc_only \ + || { + log_error "Processing of $rawfile failed." + error_list+=("Failed processing: $rawfile") + #exit 1 + continue + } + elif (( FLAG_PROCESSING )); then log_info "Processing file: $rawfile" PICASSO_OP_SCRIPT="${ROOT_DIR}/ppcpy/interface/picassopy_operational.py" "$PYTHON_PATH"/python3 "$PICASSO_OP_SCRIPT" \ diff --git a/ppcpy/interface/picassopy_operational.py b/ppcpy/interface/picassopy_operational.py index e5cdb87..5ddf4c8 100644 --- a/ppcpy/interface/picassopy_operational.py +++ b/ppcpy/interface/picassopy_operational.py @@ -50,6 +50,9 @@ type=str, default=None, help='specify a level0 polly file to be processed') +my_parser.add_argument('--preproc_only', + action='store_true', + help='switch to run only the preprocessing, including the storage of SNR, BG and RCS files') ## init parser args = my_parser.parse_args() @@ -149,7 +152,9 @@ ## write channelwise infos to nc-files (e.g.: SNR, Background, RangeCorrectedSignal) write2nc.write_channelwise_2_nc_file(data_cube=data_cube,prod_ls=["SNR","BG","RCS"]) -exit() +if args.preproc_only: + exit() + ## saturation detection data_cube.SaturationDetect() @@ -244,6 +249,7 @@ ## LC_column_names = ['cali_start_time', 'cali_stop_time', 'liconst', 'uncertainty_liconst', 'wavelength', 'nc_zip_file', 'polly_type', 'cali_method', 'telescope'] ## write profile retrievals to nc files +write2nc.write_profile2nc_file(data_cube=data_cube, prod_ls=["overlap_profiles"]) write2nc.write_profile2nc_file(data_cube=data_cube, prod_ls=["profiles","NR_profiles","OC_profiles"]) ## calc. high resolution retrievals of attenuated backscatter and volume depolarization diff --git a/ppcpy/interface/picassopy_plotting.sh b/ppcpy/interface/picassopy_plotting.sh index 68bfa1b..69788d0 100755 --- a/ppcpy/interface/picassopy_plotting.sh +++ b/ppcpy/interface/picassopy_plotting.sh @@ -138,7 +138,7 @@ main() { ## check number of available level1-files ## 24h-file vs. multiple individual processed files #PATTERN="*[0-9][0-9]_att_bsc*.nc" - PATTERN="*${DEVICE}_att_bsc.nc" + PATTERN="*${DEVICE}_*_RCS.nc" echo $PATTERN YYYY=${DATE:0:4} MM=${DATE:4:2} diff --git a/ppcpy/io/write2nc.py b/ppcpy/io/write2nc.py index 7701b73..d6bbe92 100644 --- a/ppcpy/io/write2nc.py +++ b/ppcpy/io/write2nc.py @@ -99,10 +99,15 @@ def write_channelwise_2_nc_file(data_cube, root_dir=root_dir, prod_ls=[]): """ Create the NetCDF file """ yyyy,mm,dd = date_splitting(data_cube.date) + starttime = data_cube.retrievals_highres['time'][0] + stoptime = data_cube.retrievals_highres['time'][-1] + starttime_utc_str = datetime.datetime.fromtimestamp(float(starttime), tz=datetime.timezone.utc).strftime('%H%M') + stoptime_utc_str = datetime.datetime.fromtimestamp(float(stoptime), tz=datetime.timezone.utc).strftime('%H%M') + utc_str = f'{starttime_utc_str}-{stoptime_utc_str}' output_path = Path(data_cube.picasso_config_dict["results_folder"],data_cube.device,yyyy,mm,dd) output_path.mkdir(parents=True, exist_ok=True) date_string = datetime.datetime.strptime(data_cube.date, "%Y%m%d").strftime("%Y_%m_%d") - output_filename = Path(output_path, f"{date_string}_{data_cube.device}_{prod}.nc") + output_filename = Path(output_path, f"{date_string}_{data_cube.device}_{utc_str}_{prod}.nc") json2nc_mapping.create_netcdf_from_dict(output_filename, data_cube, json_nc_mapping_dict, compression_level=1, prod=prod) def write2nc_file(data_cube, root_dir=root_dir, prod_ls=[]): @@ -154,10 +159,15 @@ def write2nc_file(data_cube, root_dir=root_dir, prod_ls=[]): """ Create the NetCDF file """ yyyy,mm,dd = date_splitting(data_cube.date) + starttime = data_cube.retrievals_highres['time'][0] + stoptime = data_cube.retrievals_highres['time'][-1] + starttime_utc_str = datetime.datetime.fromtimestamp(float(starttime), tz=datetime.timezone.utc).strftime('%H%M') + stoptime_utc_str = datetime.datetime.fromtimestamp(float(stoptime), tz=datetime.timezone.utc).strftime('%H%M') + utc_str = f'{starttime_utc_str}-{stoptime_utc_str}' output_path = Path(data_cube.picasso_config_dict["results_folder"],data_cube.device,yyyy,mm,dd) output_path.mkdir(parents=True, exist_ok=True) date_string = datetime.datetime.strptime(data_cube.date, "%Y%m%d").strftime("%Y_%m_%d") - output_filename = Path(output_path, f"{date_string}_{data_cube.device}_{prod}.nc") + output_filename = Path(output_path, f"{date_string}_{data_cube.device}_{utc_str}_{prod}.nc") json2nc_mapping.create_netcdf_from_dict(output_filename, data_cube, json_nc_mapping_dict, compression_level=1, prod=prod) @@ -228,14 +238,23 @@ def write_profile2nc_file(data_cube, root_dir:str=root_dir, prod_ls:list=[], col method = json_nc_translator[prod]['variables'][var]['method'] ch = json_nc_translator[prod]['variables'][var]['channel'] - - if method in data_cube.retrievals_profile.keys() and ch in data_cube.retrievals_profile[method][n].keys(): - if parameter in data_cube.retrievals_profile[method][n][ch].keys(): - json_nc_mapping_dict['variables'][var]['data'] = data_cube.retrievals_profile[method][n][ch][parameter] + if prod == 'overlap_profiles': + if method in data_cube.retrievals_profile.keys() and parameter in data_cube.retrievals_profile[method].keys() and ch in data_cube.retrievals_profile[method][parameter][n].keys(): + #if parameter in data_cube.retrievals_profile[method][n][ch].keys(): + json_nc_mapping_dict['variables'][var]['data'] = data_cube.retrievals_profile[method][parameter][n][ch]['olFunc'] + #else: + # continue else: continue + else: - continue + if method in data_cube.retrievals_profile.keys() and ch in data_cube.retrievals_profile[method][n].keys(): + if parameter in data_cube.retrievals_profile[method][n][ch].keys(): + json_nc_mapping_dict['variables'][var]['data'] = data_cube.retrievals_profile[method][n][ch][parameter] + else: + continue + else: + continue ### Add molecular profiles and reference hight variable: if collect_debug: @@ -252,7 +271,7 @@ def write_profile2nc_file(data_cube, root_dir:str=root_dir, prod_ls:list=[], col output_path = Path(data_cube.picasso_config_dict["results_folder"],data_cube.device,yyyy,mm,dd) output_path.mkdir(parents=True, exist_ok=True) date_string = datetime.datetime.strptime(data_cube.date, "%Y%m%d").strftime("%Y_%m_%d") - output_filename = Path(output_path, f"{date_string}_{data_cube.device}_{start}_{stop}_{prod}.nc") + output_filename = Path(output_path, f"{date_string}_{data_cube.device}_{start}-{stop}_{prod}.nc") json2nc_mapping.create_netcdf_from_dict(output_filename, data_cube, json_nc_mapping_dict, compression_level=1, prod=prod, cldFreeIndx=n) diff --git a/ppcpy/visualization/pypolly_display_3d_plots.py b/ppcpy/visualization/pypolly_display_3d_plots.py index c26669c..774727a 100644 --- a/ppcpy/visualization/pypolly_display_3d_plots.py +++ b/ppcpy/visualization/pypolly_display_3d_plots.py @@ -2643,7 +2643,12 @@ def pollyDisplay_preprocessed_BG(nc_dict, config_dict, polly_conf_dict, saveFold ## trimm matrix to last available timestamp if neccessary data_matrix_h = readout.trimm_matrix_to_last_timestamp(flagPlotLastProfilesOnly=config_dict['flagPlotLastProfilesOnly'],matrix=data_matrix_h,mdate=date_00,profile_length=int(np.nanmean(np.diff(time))),last_timestamp=nc_dict['time']['data'][-1]) - data_matrix = np.squeeze(data_matrix_h, axis=1)[0:len(perfect_day_timestamps)] + data_matrix = np.squeeze(data_matrix_h, axis=1) + if len(perfect_day_timestamps) < len(data_matrix): + data_matrix = data_matrix[0:len(perfect_day_timestamps)] + elif len(perfect_day_timestamps) > len(data_matrix): + perfect_day_timestamps = perfect_day_timestamps[0:len(data_matrix)] + data_matrix = np.where(data_matrix > 0, data_matrix, 0) diff --git a/ppcpy/visualization/pypolly_display_all.py b/ppcpy/visualization/pypolly_display_all.py index 55897f9..11440a9 100644 --- a/ppcpy/visualization/pypolly_display_all.py +++ b/ppcpy/visualization/pypolly_display_all.py @@ -63,7 +63,7 @@ help='the output folder to put the png files to.') my_parser.add_argument('--retrieval', dest='retrieval', metavar='retrieval parameter', default=['all'], - choices=['all','attbsc','voldepol','cloudinfo','target_class','wvmr_rh','quasi_results','profiles','overlap','LC','HKD','longterm_cali','profile_summary','poliphon','RCS','SNR','BG'], + choices=['all','attbsc','voldepol','cloudinfo','target_class','wvmr_rh','quasi_results','profiles','overlap','LC','HKD','longterm_cali','profile_summary','poliphon','RCS','SNR','BG','overlap_profiles'], nargs='+', type=str, help='the retrievals to be plotted; default: "all".') @@ -446,14 +446,27 @@ def main(): else: pass - if ('all' in args.retrieval) or ('overlap' in args.retrieval): + #if ('all' in args.retrieval) or ('overlap' in args.retrieval): + # ## plotting overlap + # try: + # nc_files = readout.get_nc_filename(date, device, inputfolder, param='overlap') + # for data_file in nc_files: + # nc_dict = readout.read_nc_file(data_file,date,device,location) + # print('plotting overlap:') + # display_3d.pollyDisplay_Overlap(nc_dict, config_dict, polly_conf_dict, outputfolder,donefilelist_dict=donefilelist_dict) + # except Exception as e: + # logging.exception("An error occurred") + + if ('all' in args.retrieval) or ('overlap_profiles' in args.retrieval): ## plotting overlap try: - nc_files = readout.get_nc_filename(date, device, inputfolder, param='overlap') + nc_files = readout.get_nc_filename(date, device, inputfolder, param='overlap_profiles') + print(nc_files) for data_file in nc_files: + print(data_file) nc_dict = readout.read_nc_file(data_file,date,device,location) - print('plotting overlap:') - display_3d.pollyDisplay_Overlap(nc_dict, config_dict, polly_conf_dict, outputfolder,donefilelist_dict=donefilelist_dict) + print('plotting overlap profiles:') + display_profiles.pollyDisplay_Overlap_Profiles(nc_dict, config_dict, polly_conf_dict, outputfolder,donefilelist_dict=donefilelist_dict) except Exception as e: logging.exception("An error occurred") diff --git a/ppcpy/visualization/pypolly_display_profiles.py b/ppcpy/visualization/pypolly_display_profiles.py index 30dcec0..48812d3 100644 --- a/ppcpy/visualization/pypolly_display_profiles.py +++ b/ppcpy/visualization/pypolly_display_profiles.py @@ -1881,4 +1881,179 @@ def plotting_procedure(col,param_dict,parameter,xlabel,xlim=[0,1],ylim=[0,1],sca product_stoptime = datetime.utcfromtimestamp(int(nc_dict_profile['end_time']['data'])).strftime('%Y%m%d %H:%M:%S') ) +def pollyDisplay_Overlap_Profiles(nc_dict,config_dict,polly_conf_dict,outdir,donefilelist_dict): + """ + Description + ----------- + Display the overlap functions from level1 polly nc-file. + + Parameters + ---------- + nc_dict_profile: dict + dict wich stores the overlap data. + + Usage + ----- + pollyDisplay_Overlap(nc_dict,config_dict,polly_conf_dict) + + History + ------- + 2022-09-01. First edition by Andi + """ + + ## read from config file + figDPI = config_dict['figDPI'] + flagWatermarkOn = config_dict['flagWatermarkOn'] + fontname = config_dict['fontname'] + + ## read from global config file + xLim = [-0.1, 1.1] + #yLim = polly_conf_dict['yLim_overlap'] + yLim = [0,2000] + partnerLabel = polly_conf_dict['partnerLabel'] + imgFormat = polly_conf_dict['imgFormat'] + + ## read from nc-file + #overlap355 = nc_dict['OL_355'].reshape(-1) + #overlap355Defaults = nc_dict['OL_355'].reshape(-1) + #overlap532 = nc_dict['OL_532'].reshape(-1) + #overlap532Defaults = nc_dict['OL_532d'].reshape(-1) + overlap355 = nc_dict['frnr_overlap_355']['data'] + overlap532 = nc_dict['frnr_overlap_532']['data'] + if 'raman_overlap_355_fr' in nc_dict.keys(): + overlap355Raman = nc_dict['raman_overlap_355_fr']['data'] + if 'raman_overlap_532_fr' in nc_dict.keys(): + overlap532Raman = nc_dict['raman_overlap_532_fr']['data'] + height = nc_dict['height']['data']/1000 + + pollyVersion = nc_dict['PollyVersion'] + location = nc_dict['location'] + version = nc_dict['PicassoVersion'] + dataFilename = re.split(r'_overlap_profiles',nc_dict['PollyDataFile'])[0] + # set the default font + matplotlib.rcParams['font.family'] = "sans-serif" + + saveFolder = outdir + plotfile = f'{dataFilename}_overlap_profiles.{imgFormat}' + saveFilename = os.path.join(saveFolder,plotfile) + + fig = plt.figure(figsize=[5, 8]) + ax = fig.add_axes([0.21, 0.15, 0.74, 0.75]) + + # display signal + p1, = ax.plot(overlap355, height, color='#1544E9', + linestyle='-', label=r'overlap 355 FR') + p2, = ax.plot(overlap532, height, color='#58B13F', + linestyle='-', label=r'overlap 532 FR') + if 'raman_overlap_355_fr' in nc_dict.keys(): + p5, = ax.plot(overlap355Raman, height, color='#2affff', + linestyle='-', label=r'overlap 355 FR Raman') + if 'raman_overlap_532_fr' in nc_dict.keys(): + p6, = ax.plot(overlap532Raman, height, color='#d4d42a', + linestyle='-', label=r'overlap 532 FR Raman') + + ax.set_xlabel('Overlap', fontsize=15) + ax.set_ylabel('Height (km)', fontsize=15) + +# ax.set_ylim(yLim) + ax.set_ylim(yLim[0]/1000,yLim[1]/1000) + ax.yaxis.set_major_locator(MultipleLocator(0.5)) + ax.yaxis.set_minor_locator(MultipleLocator(0.1)) + ax.set_xlim(xLim) + ax.grid(True) + ax.tick_params(axis='both', which='major', labelsize=15, + right=True, top=True, width=2, length=5) + ax.tick_params(axis='both', which='minor', width=1.5, + length=3.5, right=True, top=True) + +# starttime = time[startInd - 1] +# endtime = time[endInd - 1] + starttime = datetime.utcfromtimestamp(int(nc_dict['start_time']['data'])).strftime('%H:%M') + endtime = datetime.utcfromtimestamp(int(nc_dict['end_time']['data'])).strftime('%H:%M') + print(starttime,endtime) + ax.set_title( + 'Overlap for {instrument} at {location} {date}\n {starttime} - {endtime}'.format( + instrument=pollyVersion, + location=location, + date=nc_dict['m_date'], + starttime=starttime, + endtime=endtime +# starttime=datetime.utcfromtimestamp(int(starttime)).strftime('%Y%m%d %H:%M'), +# endtime=datetime.utcfromtimestamp(int(endtime)).strftime('%H:%M'), +# starttime=starttime.strftime('%Y%m%d %H:%M'), +# endtime=endtime.strftime('%H:%M') + ), + fontsize=14 + ) + + plt.legend(loc='upper right') + + # add watermark + if flagWatermarkOn: + rootDir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# rootDir = os.getcwd() + im_license = matplotlib.image.imread( + os.path.join(rootDir, 'ppcpy', 'img', 'by-sa.png')) + + newax_license = fig.add_axes([0.58, 0.006, 0.14, 0.07], zorder=10) + newax_license.imshow(im_license, alpha=0.8, aspect='equal') + newax_license.axis('off') + + fig.text(0.05, 0.01, 'Preliminary\nResults.', + fontweight='bold', fontsize=12, color='red', + ha='left', va='bottom', alpha=0.8, zorder=10) + + fig.text( + 0.72, 0.003, + u"\u00A9 {1} {0}.\nCC BY SA 4.0 License.".format( + datetime.now().strftime('%Y'), partnerLabel), + fontweight='bold', fontsize=7, color='black', ha='left', + va='bottom', alpha=1, zorder=10) +# fig.text( +# 0.84, 0.003, +# u"\u00A9 {1} {0}.\nCC BY SA 4.0 License.".format( +# datetime.now().strftime('%Y'), partnerLabel), +# fontweight='bold', fontsize=7, color='black', ha='left', +# va='bottom', alpha=1, zorder=10) + +# fig.text( +# 0.05, 0.02, +# '{0}'.format( +## datenum_to_datetime(time[0]).strftime("%Y-%m-%d"), +# args.timestamp), +# fontsize=12) + fig.text( + 0.3, 0.02, + 'Version: {version}'.format( + version=version), + fontsize=12) + print(f"plotting {plotfile} ... ") + fig.savefig(saveFilename,dpi=figDPI) + + plt.close() + remove_widht_height_entry_from_svg_file(saveFilename,imgFormat) + + ## write2donefilelist + readout.write2donefilelist_dict(donefilelist_dict = donefilelist_dict, + lidar = pollyVersion, + location = nc_dict['location'], + starttime = datetime.utcfromtimestamp(int(nc_dict['start_time']['data'])).strftime('%Y%m%d %H:%M:%S'), + stoptime = datetime.utcfromtimestamp(int(nc_dict['end_time']['data'])).strftime('%Y%m%d %H:%M:%S'), + last_update = datetime.now(timezone.utc).strftime("%Y%m%d %H:%M:%S"), + wavelength = 407, + filename = saveFilename, + level = 0, + info = f"overlap function", + nc_zip_file = nc_dict['PollyDataFile'], + nc_zip_file_size = 9000000, + active = 1, + GDAS = 0, + GDAS_timestamp = f"{datetime.utcfromtimestamp(int(nc_dict['start_time']['data'])).strftime('%Y%m%d')} 12:00:00", + lidar_ratio = 50, + software_version = version, + product_type = 'overlap', + product_starttime = datetime.utcfromtimestamp(int(nc_dict['start_time']['data'])).strftime('%Y%m%d %H:%M:%S'), + product_stoptime = datetime.utcfromtimestamp(int(nc_dict['end_time']['data'])).strftime('%Y%m%d %H:%M:%S') + ) diff --git a/ppcpy/visualization/pypolly_readout.py b/ppcpy/visualization/pypolly_readout.py index aa9fb72..629e9fa 100644 --- a/ppcpy/visualization/pypolly_readout.py +++ b/ppcpy/visualization/pypolly_readout.py @@ -59,13 +59,13 @@ def get_nc_filename(date, device, inputfolder, param=""): if path_exist.exists() == True: # print(inputfolder) - #file_searchpattern = f"{YYYY}_{MM}_{DD}_*[0-9]_{param}.nc" - if not "profile" in param: - #file_searchpattern = f"{date}_{device}_{param}.nc" - file_searchpattern = f"{YYYY}_{MM}_{DD}_{device}_{param}.nc" - else: - #file_searchpattern = f"{date}_{device}_*[0-9][0-9]_{param}.nc" - file_searchpattern = f"{YYYY}_{MM}_{DD}_{device}_*[0-9][0-9]_{param}.nc" + file_searchpattern = f"{YYYY}_{MM}_{DD}_{device}_*_{param}.nc" + # if not "profile" in param: + # #file_searchpattern = f"{date}_{device}_{param}.nc" + # file_searchpattern = f"{YYYY}_{MM}_{DD}_{device}_*_{param}.nc" + # else: + # #file_searchpattern = f"{date}_{device}_*[0-9][0-9]_{param}.nc" + # file_searchpattern = f"{YYYY}_{MM}_{DD}_{device}_*[0-9][0-9]_{param}.nc" res_file = Path(r'{}'.format(inputfolder)).glob('{}'.format(file_searchpattern)) ## convert type path to type string From e4739cc1b78fc8875a9d882b0ada440a2faeb811 Mon Sep 17 00:00:00 2001 From: Baymax Date: Thu, 16 Jul 2026 06:45:57 +0000 Subject: [PATCH 4/4] corrected quasi_retrieval plotting procedure --- ppcpy/config/polly_global_config.json | 2 + ppcpy/interface/picassopy_plotting.sh | 52 +++++++------ .../visualization/pypolly_display_3d_plots.py | 75 +++++++++++++++++-- ppcpy/visualization/pypolly_display_all.py | 2 +- 4 files changed, 101 insertions(+), 30 deletions(-) diff --git a/ppcpy/config/polly_global_config.json b/ppcpy/config/polly_global_config.json index 94ad0cc..a4eab0e 100644 --- a/ppcpy/config/polly_global_config.json +++ b/ppcpy/config/polly_global_config.json @@ -294,7 +294,9 @@ "zLim_quasi_beta_355": [0, 10], "zLim_quasi_beta_532": [0, 6], "zLim_quasi_beta_1064": [0, 3], + "zLim_quasi_ext": [-1, 300], "zLim_quasi_Par_DR_532": [0, 0.4], + "zLim_quasi_Vol_DR_532": [0, 0.3], "zLim_FR_RCS_355": [1e-2, 30], "zLim_FR_RCS_387": [1e-2, 30], "zLim_FR_RCS_407": [1e-2, 35], diff --git a/ppcpy/interface/picassopy_plotting.sh b/ppcpy/interface/picassopy_plotting.sh index 69788d0..b98b68a 100755 --- a/ppcpy/interface/picassopy_plotting.sh +++ b/ppcpy/interface/picassopy_plotting.sh @@ -135,30 +135,34 @@ main() { echo $DEVICE for DATE in ${DATE_LS[@]}; do echo $DATE - ## check number of available level1-files - ## 24h-file vs. multiple individual processed files - #PATTERN="*[0-9][0-9]_att_bsc*.nc" - PATTERN="*${DEVICE}_*_RCS.nc" - echo $PATTERN - YYYY=${DATE:0:4} - MM=${DATE:4:2} - DD=${DATE:6:2} - POLLY_LEVEL1_FOLDER="$POLLY_LEVEL1_BASEFOLDER/$DEVICE/$YYYY/$MM/$DD" - echo "$POLLY_LEVEL1_FOLDER" - file_count=$(find "$POLLY_LEVEL1_FOLDER" -type f -name "$PATTERN" | wc -l) - if [ "$file_count" -eq 0 ]; then - echo "No matching file exists in $POLLY_LEVEL1_FOLDER." - echo "Continuing..." - elif [ "$file_count" -eq 1 ]; then - echo "One matching file exists in $POLLY_LEVEL1_FOLDER." - for RETRIEVAL in ${RETRIEVAL_LS[@]}; do - echo $RETRIEVAL - "$PY_FOLDER"python "$PICASSO_DIR"/ppcpy/visualization/pypolly_display_all.py --date $DATE --device $DEVICE --picasso_config $PICASSO_CONFIG_FILE --retrieval $RETRIEVAL --donefilelist $flagDONEFILELIST - done - elif [ "$file_count" -gt 1 ]; then - echo "More than one matching file exists in $POLLY_LEVEL1_FOLDER." - #"$PY_FOLDER"python "$PICASSO_DIR"/ppcpy/visualization/pypolly_display_all.py --date $DATE --device $DEVICE --picasso_config $PICASSO_CONFIG_FILE --retrieval $RETRIEVAL --donefilelist $flagDONEFILELIST - fi + for RETRIEVAL in ${RETRIEVAL_LS[@]}; do + echo $RETRIEVAL + "$PY_FOLDER"python "$PICASSO_DIR"/ppcpy/visualization/pypolly_display_all.py --date $DATE --device $DEVICE --picasso_config $PICASSO_CONFIG_FILE --retrieval $RETRIEVAL --donefilelist $flagDONEFILELIST + done + ### check number of available level1-files + ### 24h-file vs. multiple individual processed files + ##PATTERN="*[0-9][0-9]_att_bsc*.nc" + #PATTERN="*${DEVICE}*_RCS.nc" + #echo $PATTERN + #YYYY=${DATE:0:4} + #MM=${DATE:4:2} + #DD=${DATE:6:2} + #POLLY_LEVEL1_FOLDER="$POLLY_LEVEL1_BASEFOLDER/$DEVICE/$YYYY/$MM/$DD" + #echo "$POLLY_LEVEL1_FOLDER" + #file_count=$(find "$POLLY_LEVEL1_FOLDER" -type f -name "$PATTERN" | wc -l) + #if [ "$file_count" -eq 0 ]; then + # echo "No matching file exists in $POLLY_LEVEL1_FOLDER." + # echo "Continuing..." + #elif [ "$file_count" -eq 1 ]; then + # echo "One matching file exists in $POLLY_LEVEL1_FOLDER." + # for RETRIEVAL in ${RETRIEVAL_LS[@]}; do + # echo $RETRIEVAL + # "$PY_FOLDER"python "$PICASSO_DIR"/ppcpy/visualization/pypolly_display_all.py --date $DATE --device $DEVICE --picasso_config $PICASSO_CONFIG_FILE --retrieval $RETRIEVAL --donefilelist $flagDONEFILELIST + # done + #elif [ "$file_count" -gt 1 ]; then + # echo "More than one matching file exists in $POLLY_LEVEL1_FOLDER." + # #"$PY_FOLDER"python "$PICASSO_DIR"/ppcpy/visualization/pypolly_display_all.py --date $DATE --device $DEVICE --picasso_config $PICASSO_CONFIG_FILE --retrieval $RETRIEVAL --donefilelist $flagDONEFILELIST + #fi # if [[ "$flagWriteIntoTodoList" == "true" ]];then # check_todo_list_consistency diff --git a/ppcpy/visualization/pypolly_display_3d_plots.py b/ppcpy/visualization/pypolly_display_3d_plots.py index 774727a..e9ae93e 100644 --- a/ppcpy/visualization/pypolly_display_3d_plots.py +++ b/ppcpy/visualization/pypolly_display_3d_plots.py @@ -1932,10 +1932,20 @@ def pollyDisplayQR(nc_dict,config_dict, polly_conf_dict, saveFolder, q_param, q_ zLim = polly_conf_dict['zLim_quasi_beta_1064'] elif q_param == "angexp": zLim = polly_conf_dict['zLim_quasi_ANG'] + elif q_param == "bsc_355": + zLim = polly_conf_dict['zLim_quasi_beta_355'] elif q_param == "bsc_532": zLim = polly_conf_dict['zLim_quasi_beta_532'] elif q_param == "par_depol_532": zLim = polly_conf_dict['zLim_quasi_Par_DR_532'] + elif q_param == "vol_depol_532": + zLim = polly_conf_dict['zLim_quasi_Vol_DR_532'] + elif q_param == "ext_355": + zLim = polly_conf_dict['zLim_quasi_ext'] + elif q_param == "ext_532": + zLim = polly_conf_dict['zLim_quasi_ext'] + if q_param == "ext_1064": + zLim = polly_conf_dict['zLim_quasi_ext'] partnerLabel = polly_conf_dict['partnerLabel'] colormap_basic = polly_conf_dict['colormap_basic'] @@ -1945,17 +1955,31 @@ def pollyDisplayQR(nc_dict,config_dict, polly_conf_dict, saveFolder, q_param, q_ if q_param == "angexp": matrix = nc_dict['quasi_ang_532_1064']['data'] - quality_mask = np.where(matrix > 0, 0, 0) + #quality_mask = np.where(matrix > 0, 0, 0) + elif q_param == "bsc_355": + matrix = nc_dict['quasi_bsc_355']['data'] + #quality_mask = nc_dict['quality_mask_355']['data'] elif q_param == "bsc_532": matrix = nc_dict['quasi_bsc_532']['data'] - quality_mask = nc_dict['quality_mask_532']['data'] + #quality_mask = nc_dict['quality_mask_532']['data'] elif q_param == "bsc_1064": matrix = nc_dict['quasi_bsc_1064']['data'] - quality_mask = nc_dict['quality_mask_1064']['data'] + #quality_mask = nc_dict['quality_mask_1064']['data'] + elif q_param == "ext_355": + matrix = nc_dict['quasi_ext_355']['data'] + #quality_mask = nc_dict['quality_mask_355']['data'] + elif q_param == "ext_532": + matrix = nc_dict['quasi_ext_532']['data'] + #quality_mask = nc_dict['quality_mask_532']['data'] + elif q_param == "ext_1064": + matrix = nc_dict['quasi_ext_1064']['data'] + #quality_mask = nc_dict['quality_mask_1064']['data'] + elif q_param == "vol_depol_532": + matrix = nc_dict['quasi_voldepol_532']['data'] elif q_param == "par_depol_532": matrix = nc_dict['quasi_pardepol_532']['data'] #quality_mask = nc_dict['quality_mask_voldepol'] - quality_mask = nc_dict['quality_mask_voldepol_532']['data'] + #quality_mask = nc_dict['quality_mask_voldepol_532']['data'] height = nc_dict['height']['data'] time = nc_dict['time']['data'] @@ -1976,6 +2000,14 @@ def pollyDisplayQR(nc_dict,config_dict, polly_conf_dict, saveFolder, q_param, q_ if q_version == "V2": plotfile = f'{dataFilename}_Quasi_ANGEXP_532_1064_V2.{imgFormat}' quasi_title = f'Quasi BSC Angstroem Exponent 532-1064 ({q_version}) of {pollyVersion} at {location}' + if q_param == "bsc_355": + prodtype = "Quasi_Bsc_355" + wavelength = 355 + if q_version == "V1": + plotfile = f'{dataFilename}_Quasi_Bsc_355.{imgFormat}' + if q_version == "V2": + plotfile = f'{dataFilename}_Quasi_Bsc_355_V2.{imgFormat}' + quasi_title = f'Quasi backscatter coefficient ({q_version}) at 355 nm from {pollyVersion} at {location}' if q_param == "bsc_532": prodtype = "Quasi_Bsc_532" wavelength = 532 @@ -1992,6 +2024,30 @@ def pollyDisplayQR(nc_dict,config_dict, polly_conf_dict, saveFolder, q_param, q_ if q_version == "V2": plotfile = f'{dataFilename}_Quasi_Bsc_1064_V2.{imgFormat}' quasi_title = f'Quasi backscatter coefficient ({q_version}) at 1064 nm from {pollyVersion} at {location}' + if q_param == "ext_355": + prodtype = "Quasi_Ext_355" + wavelength = 355 + if q_version == "V1": + plotfile = f'{dataFilename}_Quasi_Ext_355.{imgFormat}' + if q_version == "V2": + plotfile = f'{dataFilename}_Quasi_Ext_355_V2.{imgFormat}' + quasi_title = f'Quasi extinction ({q_version}) at 355 nm from {pollyVersion} at {location}' + if q_param == "ext_532": + prodtype = "Quasi_Ext_532" + wavelength = 532 + if q_version == "V1": + plotfile = f'{dataFilename}_Quasi_Ext_532.{imgFormat}' + if q_version == "V2": + plotfile = f'{dataFilename}_Quasi_Ext_532_V2.{imgFormat}' + quasi_title = f'Quasi extinction ({q_version}) at 532 nm from {pollyVersion} at {location}' + if q_param == "ext_1064": + prodtype = "Quasi_Ext_1064" + wavelength = 1064 + if q_version == "V1": + plotfile = f'{dataFilename}_Quasi_Ext_1064.{imgFormat}' + if q_version == "V2": + plotfile = f'{dataFilename}_Quasi_Ext_1064_V2.{imgFormat}' + quasi_title = f'Quasi extinction ({q_version}) at 1064 nm from {pollyVersion} at {location}' if q_param == "par_depol_532": prodtype = "Quasi_PDR_532" wavelength = 532 @@ -2000,10 +2056,19 @@ def pollyDisplayQR(nc_dict,config_dict, polly_conf_dict, saveFolder, q_param, q_ if q_version == "V2": plotfile = f'{dataFilename}_Quasi_PDR_532_V2.{imgFormat}' quasi_title = f'Quasi particle depolarization ratio ({q_version}) at 532 nm from {pollyVersion} at {location}' + if q_param == "vol_depol_532": + prodtype = "Quasi_VDR_532" + wavelength = 532 + if q_version == "V1": + plotfile = f'{dataFilename}_Quasi_VDR_532.{imgFormat}' + if q_version == "V2": + plotfile = f'{dataFilename}_Quasi_VDR_532_V2.{imgFormat}' + quasi_title = f'Quasi volume depolarization ratio ({q_version}) at 532 nm from {pollyVersion} at {location}' saveFilename = os.path.join(saveFolder,plotfile) ## fill time gaps in att_bsc matrix + quality_mask = np.full_like(matrix, fill_value=np.nan, dtype=float) ## dummy-quality_mask matrix, quality_mask = readout.fill_time_gaps_of_matrix(time, matrix, quality_mask) ## get date and convert to datetime object @@ -2024,7 +2089,7 @@ def pollyDisplayQR(nc_dict,config_dict, polly_conf_dict, saveFolder, q_param, q_ extent = [ x_lims[0], x_lims[-1], max_height[0], max_height[-1] ] ## mask matrix - matrix = np.ma.masked_where(quality_mask < 0, matrix) + #matrix = np.ma.masked_where(quality_mask < 0, matrix) ## slice matrix to max_height matrix = matrix[:,0:len(max_height)] diff --git a/ppcpy/visualization/pypolly_display_all.py b/ppcpy/visualization/pypolly_display_all.py index 11440a9..808575d 100644 --- a/ppcpy/visualization/pypolly_display_all.py +++ b/ppcpy/visualization/pypolly_display_all.py @@ -333,7 +333,7 @@ def main(): if ('all' in args.retrieval) or ('quasi_results' in args.retrieval): ## plotting Quasi results V1 try: - q_params_ls = ["angexp", "bsc_532", "bsc_1064", "par_depol_532"] + q_params_ls = ["angexp", "bsc_355", "bsc_532", "bsc_1064", "ext_355", "ext_532", "ext_1064", "par_depol_532", "vol_depol_532"] nc_files = readout.get_nc_filename(date, device, inputfolder, param='quasi_results') for data_file in nc_files: nc_dict = readout.read_nc_file(data_file,date,device,location)