From 9a416d0e8be446c64783dde0540806134f854eaa Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Fri, 1 May 2026 14:24:32 -0400 Subject: [PATCH 1/6] fix sum of all pathogen equal to 0 --- SMHviz_plot/utils_data.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SMHviz_plot/utils_data.py b/SMHviz_plot/utils_data.py index 8b4e51e..14d6530 100644 --- a/SMHviz_plot/utils_data.py +++ b/SMHviz_plot/utils_data.py @@ -394,11 +394,14 @@ def prep_multipat_plot_comb(pathogen_information, calc_mean=False): f2.update({"proportion_" + pathogen_name: [med, mean, q1, q2, q3, q4, q5, q6, q7, q8]}) else: f2.update({"proportion_" + pathogen_name: [med, q1, q2, q3, q4, q5, q6, q7, q8]}) + if (all_sample["value"] == 0).any(): + all_sample = all_sample.replace({"value": 0}, 0.00000001) all_sample["proportion_" + pathogen_name] = all_sample["value_" + pathogen_name] / all_sample["value"] + all_sample = all_sample.round({"value": 5}) # Calculate the quantiles for each "value" and "proportion" columns all_quantile = all_sample.groupby(["target_end_date"]).agg(f) all_quantile.columns = all_quantile.columns.get_level_values(0) + "-" + all_quantile.columns.get_level_values(1) detail_quantile = all_sample.groupby(["target_end_date"]).agg(f2) detail_quantile.columns = (detail_quantile.columns.get_level_values(0) + "-" + detail_quantile.columns.get_level_values(1)) - return {"all": all_quantile, "detail": detail_quantile} + return {"all": all_quantile, "detail": detail_quantile} \ No newline at end of file From b381f8c2b239856ee8ca5c01e13f581b65e45d6c Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Tue, 12 May 2026 17:10:08 -0400 Subject: [PATCH 2/6] rm sum of pathogen equal 0 --- SMHviz_plot/utils_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SMHviz_plot/utils_data.py b/SMHviz_plot/utils_data.py index 14d6530..abd7178 100644 --- a/SMHviz_plot/utils_data.py +++ b/SMHviz_plot/utils_data.py @@ -398,9 +398,11 @@ def prep_multipat_plot_comb(pathogen_information, calc_mean=False): all_sample = all_sample.replace({"value": 0}, 0.00000001) all_sample["proportion_" + pathogen_name] = all_sample["value_" + pathogen_name] / all_sample["value"] all_sample = all_sample.round({"value": 5}) + all_sample["tot_proportion"] = all_sample[[col for col in all_sample.columns if col.startswith('proportion_')]].sum(axis=1) # Calculate the quantiles for each "value" and "proportion" columns all_quantile = all_sample.groupby(["target_end_date"]).agg(f) all_quantile.columns = all_quantile.columns.get_level_values(0) + "-" + all_quantile.columns.get_level_values(1) + all_sample = all_sample[all_sample["tot_proportion"] > 0] detail_quantile = all_sample.groupby(["target_end_date"]).agg(f2) detail_quantile.columns = (detail_quantile.columns.get_level_values(0) + "-" + detail_quantile.columns.get_level_values(1)) From 9bc0d222201a2b30a74f4221735331976214a2a5 Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Fri, 15 May 2026 17:46:11 -0400 Subject: [PATCH 3/6] update calculation + rename median --- SMHviz_plot/figures.py | 8 ++-- SMHviz_plot/utils_data.py | 85 ++++++++++++++++++++++++++++++--------- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/SMHviz_plot/figures.py b/SMHviz_plot/figures.py index c3114fc..6b5ea06 100644 --- a/SMHviz_plot/figures.py +++ b/SMHviz_plot/figures.py @@ -1125,7 +1125,7 @@ def make_spaghetti_plot(df, legend_col="model_name", spag_col="type_id", show_le def make_combine_multi_pathogen_plot(list_df, list_pathogen, truth_data=None, opacity=0.2, color=None, palette="turbo", - intervals_dict=None, intervals=None, bar_interval=0.5, bar_calc="med", title=None, + intervals_dict=None, intervals=None, bar_interval=0.5, bar_calc="median", title=None, y_axis_title="", error_bar_pat=None): """Create the Multi-Pathogen Combined plot @@ -1143,7 +1143,7 @@ def make_combine_multi_pathogen_plot(list_df, list_pathogen, truth_data=None, op and 50% quantiles for each "value" and "value_-"columns and (2) "detail": median, 95%, 90%, 80%, and 50% quantiles for each "proportion_-" columns. Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, - 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. + 0.95, 0.975, respectively. The median and mean are noted as "median" and "mean", respectively. :parameter list_df: A dictionary with 2 DataFrame: (1) "all": median, 95%, 90%, 80%, and 50% quantiles for the combined ("value" column) and for each pathogen ("value_" columns) and (2) "detail": median, 95%, 90%, @@ -1176,7 +1176,7 @@ def make_combine_multi_pathogen_plot(list_df, list_pathogen, truth_data=None, op :type intervals: list :parameter bar_interval: Interval to use for the error bar in the second subplot, by default `0.5`. :type bar_interval: float - :parameter bar_calc: Value to use for the bar height, should match columns names. By default, "med" + :parameter bar_calc: Value to use for the bar height, should match columns names. By default, "median" :type bar_calc: str :parameter title: Title of the plot, by default `None` (no title). :type title: str @@ -1326,4 +1326,4 @@ def make_combine_multi_pathogen_plot(list_df, list_pathogen, truth_data=None, op ) if title is not None: fig.update_layout(title=dict(text=title, font=dict(size=18), xanchor="center", xref="paper", x=0.5)) - return fig + return fig \ No newline at end of file diff --git a/SMHviz_plot/utils_data.py b/SMHviz_plot/utils_data.py index abd7178..2dac805 100644 --- a/SMHviz_plot/utils_data.py +++ b/SMHviz_plot/utils_data.py @@ -310,7 +310,7 @@ def q8(x): return x.quantile(0.975) -def med(x): +def median(x): """Calculate the quantile 0.5 (or median) Calculate the quantile 0.5 (or median) on a specific pandas Series @@ -334,16 +334,27 @@ def mean(x): return x.mean() -def prep_multipat_plot_comb(pathogen_information, calc_mean=False): +def prep_multipat_plot_comb(pathogen_information, primary=None, calc_mean=False): """Process Data for Combined Multi-pathogen plot From a dictionary containing each DataFrame associated to a specific pathogen: - `"value"`: Sum of all the "value_" columns ("value_" set to NA for pathogen with empty DataFrame (not selected)) - - Proportion of each pathogen `"proportion_" = "value_" / "value"` - - Calculate the median, 95%, 90%, 80%, and 50% quantiles for each "value" and "proportion" columns (and the mean - if `calc_mean` set to `True`) + - Proportion of each pathogen : + - If two pathogens calculate proportion for primary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - If three pathogens calculate proportion for primary pathogen and secondary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - primary + secondary pathogen: `"proportion___"` = + (`"value_" + "value_"`) / `"value"` + - Calculate the median, and 50% quantiles for each "value" and "proportion" columns (and the mean + if `calc_mean` set to `True`) except: + - For proportion with two pathogens: + - secondary pathogen: `"proportion__median"` = 1 - `"proportion__median" + - For proportion with three pathogens: + - last pathogen: `"proportion__median"` = + `1 - "proportion___median"` Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. @@ -358,6 +369,9 @@ def prep_multipat_plot_comb(pathogen_information, calc_mean=False): :parameter pathogen_information: A dictionary containing multiple dictionary containing a DataFrame (result of sampling process, key: "dataframe") and named with the associated specific pathogen (keys). :type pathogen_information: dict + :parameter primary Name of the pathogen(s) to treat as "primary"/"secondary" for proportion calculation. + If `None` the first pathogen(s) in the `pathogen_information` + :type primary: None | list :parameter calc_mean: Boolean indicating if the mean should be calculated too (in addition to the other quantiles) :type calc_mean: bool :return: A dictionary with 2 objects: (1) "all": median, 95%, 90%, 80%, and 50% quantiles for each "value" and @@ -365,12 +379,12 @@ def prep_multipat_plot_comb(pathogen_information, calc_mean=False): "proportion_-" columns. """ all_sample = pd.DataFrame() - f = {'value': [med, q1, q2, q3, q4, q5, q6, q7, q8]} + f = {'value': [median, q1, q2, q3, q4, q5, q6, q7, q8]} f2 = {} for patho in pathogen_information: # Preparation pathogen_name = patho.lower() - f.update({"value_" + pathogen_name: [med, q1, q2, q3, q4, q5, q6, q7, q8]}) + f.update({"value_" + pathogen_name: [median, q1, q2, q3, q4, q5, q6, q7, q8]}) # Merge all pathogen in one dataframe if len(all_sample) > 0: if len(pathogen_information[patho]["dataframe"]) > 0: @@ -386,24 +400,55 @@ def prep_multipat_plot_comb(pathogen_information, calc_mean=False): all_sample[col] = pd.NA # Calculate sum of all pathogen all_sample["value"] = all_sample[[col for col in all_sample.columns if col.startswith('value_')]].sum(axis=1) + # Calculate the quantiles for each "value" columns + all_quantile = all_sample.groupby(["target_end_date"]).agg(f) + all_quantile.columns = all_quantile.columns.get_level_values(0) + "-" + all_quantile.columns.get_level_values(1) # Calculate proportion of each pathogen + ## Prep column selection + all_sample = all_sample.round({"value": 5}) + list_patho = [] for patho in pathogen_information: - # Preparation - pathogen_name = patho.lower() + if len(pathogen_information[patho]["dataframe"]) > 0: + list_patho.append(patho) + if primary is None: + if len(list_patho) == 2: + primary = [list(pathogen_information.keys())[0]] + if len(list_patho) == 3: + primary = list(pathogen_information.keys())[:2] + else: + if len(list_patho) == 2 and len(primary) != 1: + raise ValueError("Primary should be of length 1") + if len(list_patho) == 2 and len(primary) != 2: + raise ValueError("Primary should be of length 2") + other = [list_patho[-1].lower()] + primary = [x.lower() for x in primary] + ## Calculate proportion + if (all_sample["value"] == 0).any(): + all_sample = all_sample[all_sample["value"] > 0] + all_sample["proportion_" + primary[0]] = all_sample["value_" + primary[0]] / all_sample["value"] + if calc_mean is True: + f2.update({"proportion_" + primary[0]: [median, mean, q4, q5]}) + else: + f2.update({"proportion_" + primary[0]: [median, q4, q5]}) + if len(primary) > 1: + all_sample["proportion_" + primary[0] + "_" + primary[1]] = ( + (all_sample["value_" + primary[0]] + all_sample["value_" + primary[1]]) / + all_sample["value"]) if calc_mean is True: - f2.update({"proportion_" + pathogen_name: [med, mean, q1, q2, q3, q4, q5, q6, q7, q8]}) + f2.update({"proportion_" + primary[0] + "_" + primary[1]: [median, mean]}) else: - f2.update({"proportion_" + pathogen_name: [med, q1, q2, q3, q4, q5, q6, q7, q8]}) - if (all_sample["value"] == 0).any(): - all_sample = all_sample.replace({"value": 0}, 0.00000001) - all_sample["proportion_" + pathogen_name] = all_sample["value_" + pathogen_name] / all_sample["value"] - all_sample = all_sample.round({"value": 5}) - all_sample["tot_proportion"] = all_sample[[col for col in all_sample.columns if col.startswith('proportion_')]].sum(axis=1) - # Calculate the quantiles for each "value" and "proportion" columns - all_quantile = all_sample.groupby(["target_end_date"]).agg(f) - all_quantile.columns = all_quantile.columns.get_level_values(0) + "-" + all_quantile.columns.get_level_values(1) - all_sample = all_sample[all_sample["tot_proportion"] > 0] + f2.update({"proportion_" + primary[0] + "_" + primary[1]: [median]}) + # Calculate the quantiles for each "proportion" columns detail_quantile = all_sample.groupby(["target_end_date"]).agg(f2) detail_quantile.columns = (detail_quantile.columns.get_level_values(0) + "-" + detail_quantile.columns.get_level_values(1)) + if len(primary) > 1: + detail_quantile["proportion_" + other[0] + "-median"] = ( + 1 - detail_quantile["proportion_" + primary[0] + "_" + primary[1] + "-median"]) + detail_quantile["proportion_" + primary[1] + "-median"] = ( + 1 - detail_quantile["proportion_" + other[0] + "-median"] - + detail_quantile["proportion_" + primary[0] + "-median"]) + else: + detail_quantile["proportion_" + other[0] + "-median"] = ( + 1 - detail_quantile["proportion_" + primary[0] + "-median"]) return {"all": all_quantile, "detail": detail_quantile} \ No newline at end of file From d92b6573ae5a9c06393b0f0d59e1ea57bca01958 Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Wed, 20 May 2026 14:50:59 -0400 Subject: [PATCH 4/6] fix primary(ies) information --- SMHviz_plot/utils_data.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/SMHviz_plot/utils_data.py b/SMHviz_plot/utils_data.py index 2dac805..5d84d68 100644 --- a/SMHviz_plot/utils_data.py +++ b/SMHviz_plot/utils_data.py @@ -418,10 +418,13 @@ def prep_multipat_plot_comb(pathogen_information, primary=None, calc_mean=False) else: if len(list_patho) == 2 and len(primary) != 1: raise ValueError("Primary should be of length 1") - if len(list_patho) == 2 and len(primary) != 2: + if len(list_patho) == 3 and len(primary) != 2: raise ValueError("Primary should be of length 2") - other = [list_patho[-1].lower()] primary = [x.lower() for x in primary] + other = [] + for patho in list_patho: + if patho.lower() not in primary: + other.append(patho.lower()) ## Calculate proportion if (all_sample["value"] == 0).any(): all_sample = all_sample[all_sample["value"] > 0] From ab00308172c6bf12b14ec3d30959fa0a4ec1284c Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Fri, 22 May 2026 17:44:01 -0400 Subject: [PATCH 5/6] update calculate to allow storing output and split step --- SMHviz_plot/utils_data.py | 185 ++++++++++++++++++++++++++++---------- 1 file changed, 139 insertions(+), 46 deletions(-) diff --git a/SMHviz_plot/utils_data.py b/SMHviz_plot/utils_data.py index 5d84d68..b7a36da 100644 --- a/SMHviz_plot/utils_data.py +++ b/SMHviz_plot/utils_data.py @@ -334,53 +334,36 @@ def mean(x): return x.mean() -def prep_multipat_plot_comb(pathogen_information, primary=None, calc_mean=False): - """Process Data for Combined Multi-pathogen plot +def calc_value(pathogen_information, write_sample, write_quantiles): + """Process Data for Combined Multi-pathogen plot (internal function_ - From a dictionary containing each DataFrame associated to a specific pathogen: + From a dictionary containing each DataFrame associated to a specific pathogen: - - `"value"`: Sum of all the "value_" columns ("value_" set to NA for pathogen with empty DataFrame - (not selected)) - - Proportion of each pathogen : - - If two pathogens calculate proportion for primary pathogen: - - primary pathogen: `"proportion_" = "value_" / "value"` - - If three pathogens calculate proportion for primary pathogen and secondary pathogen: - - primary pathogen: `"proportion_" = "value_" / "value"` - - primary + secondary pathogen: `"proportion___"` = - (`"value_" + "value_"`) / `"value"` - - Calculate the median, and 50% quantiles for each "value" and "proportion" columns (and the mean - if `calc_mean` set to `True`) except: - - For proportion with two pathogens: - - secondary pathogen: `"proportion__median"` = 1 - `"proportion__median" - - For proportion with three pathogens: - - last pathogen: `"proportion__median"` = - `1 - "proportion___median"` + - `"value"`: Sum of all the "value_" columns ("value_" set to NA for pathogen with empty DataFrame + (not selected)) - Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, - 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. + Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, + 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. - The input `pathogen_information` should be in a specific format: + The input `pathogen_information` should be in a specific format: - - `pathogen_information = {: {"dataframe":}, : {"dataframe":}, etc.}` - - `` is a data frame in the output format of the `sample_df()` function with 3 columns: - "target_end_date", "sample_id_n" and "value_". - - For more information, please consult the `sample_df()` function documentation + - `pathogen_information = {: {"dataframe":}, : {"dataframe":}, etc.}` + - `` is a data frame in the output format of the `sample_df()` function with 3 columns: + "target_end_date", "sample_id_n" and "value_". + - For more information, please consult the `sample_df()` function documentation - :parameter pathogen_information: A dictionary containing multiple dictionary containing a DataFrame (result of - sampling process, key: "dataframe") and named with the associated specific pathogen (keys). - :type pathogen_information: dict - :parameter primary Name of the pathogen(s) to treat as "primary"/"secondary" for proportion calculation. - If `None` the first pathogen(s) in the `pathogen_information` - :type primary: None | list - :parameter calc_mean: Boolean indicating if the mean should be calculated too (in addition to the other quantiles) - :type calc_mean: bool - :return: A dictionary with 2 objects: (1) "all": median, 95%, 90%, 80%, and 50% quantiles for each "value" and - "value_-"columns and (2) "detail": median, 95%, 90%, 80%, and 50% quantiles for each - "proportion_-" columns. + :parameter pathogen_information: A dictionary containing multiple dictionary containing a DataFrame (result of + sampling process, key: "dataframe") and named with the associated specific pathogen (keys). + :type pathogen_information: dict + :parameter write_sample: If not None, write the associated samples in a csv files to the + path and filename inputted + :type write_sample: None | str + :parameter write_quantiles: If not None, write the quantiles associated with the `"value"` + columns in a csv files to the path and filename inputted + :type write_quantiles: None | str """ all_sample = pd.DataFrame() f = {'value': [median, q1, q2, q3, q4, q5, q6, q7, q8]} - f2 = {} for patho in pathogen_information: # Preparation pathogen_name = patho.lower() @@ -403,24 +386,65 @@ def prep_multipat_plot_comb(pathogen_information, primary=None, calc_mean=False) # Calculate the quantiles for each "value" columns all_quantile = all_sample.groupby(["target_end_date"]).agg(f) all_quantile.columns = all_quantile.columns.get_level_values(0) + "-" + all_quantile.columns.get_level_values(1) + if write_sample is not None: + all_sample.to_csv(write_sample) + if write_quantiles is not None: + all_quantile.to_csv(write_quantiles) + return all_quantile, all_sample + + +def calc_proportion(all_sample, primary=None, + calc_mean=False, write_proportion=None): + """Process Data for Combined Multi-pathogen plot (internal function) + + From a DataFrame containing samples for specific pathogens: + + - Proportion of each pathogen : + - If two pathogens calculate proportion for primary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - If three pathogens calculate proportion for primary pathogen and secondary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - primary + secondary pathogen: `"proportion___"` = + (`"value_" + "value_"`) / `"value"` + - Calculate the median, and 50% quantiles for each "value" and "proportion" columns (and the mean + if `calc_mean` set to `True`) except: + - For proportion with two pathogens: + - secondary pathogen: `"proportion__median"` = 1 - `"proportion__median" + - For proportion with three pathogens: + - last pathogen: `"proportion__median"` = + `1 - "proportion___median"` + + Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, + 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. + :parameter all_sample: A DataFrame containing the second result of `calc_value` function ( + sampling process output) + :type all_sample: pd.DataFrame + :parameter primary Name of the pathogen(s) to treat as "primary"/"secondary" for proportion calculation. + If `None` the first pathogen(s) in the `pathogen_information` + :type primary: None | list + :parameter calc_mean: Boolean indicating if the mean should be calculated too (in addition to the other quantiles) + :type calc_mean: bool + :parameter write_proportion: If not None, write the quantiles associated with the `"proportion"` + columns in a csv files to the path and filename inputted + :type write_proportion: None | str + """ # Calculate proportion of each pathogen + f2 = {} ## Prep column selection all_sample = all_sample.round({"value": 5}) - list_patho = [] - for patho in pathogen_information: - if len(pathogen_information[patho]["dataframe"]) > 0: - list_patho.append(patho) + all_sample = all_sample.dropna(axis=1, how='all') + list_patho = list(all_sample.filter(regex='^value_').columns) + list_patho = [s.replace('value_', '') for s in list_patho] if primary is None: if len(list_patho) == 2: - primary = [list(pathogen_information.keys())[0]] + primary = list_patho[0] if len(list_patho) == 3: - primary = list(pathogen_information.keys())[:2] + primary = list_patho[:2] else: if len(list_patho) == 2 and len(primary) != 1: raise ValueError("Primary should be of length 1") if len(list_patho) == 3 and len(primary) != 2: raise ValueError("Primary should be of length 2") - primary = [x.lower() for x in primary] other = [] for patho in list_patho: if patho.lower() not in primary: @@ -437,7 +461,7 @@ def prep_multipat_plot_comb(pathogen_information, primary=None, calc_mean=False) all_sample["proportion_" + primary[0] + "_" + primary[1]] = ( (all_sample["value_" + primary[0]] + all_sample["value_" + primary[1]]) / all_sample["value"]) - if calc_mean is True: + if calc_mean: f2.update({"proportion_" + primary[0] + "_" + primary[1]: [median, mean]}) else: f2.update({"proportion_" + primary[0] + "_" + primary[1]: [median]}) @@ -454,4 +478,73 @@ def prep_multipat_plot_comb(pathogen_information, primary=None, calc_mean=False) else: detail_quantile["proportion_" + other[0] + "-median"] = ( 1 - detail_quantile["proportion_" + primary[0] + "-median"]) + if write_proportion is not None: + detail_quantile.to_csv(write_proportion) + return detail_quantile + + +def prep_multipat_plot_comb(pathogen_information, primary=None, calc_mean=False, + calc_prop=True, write_sample=None, write_quantiles=None, + write_proportion=None): + """Process Data for Combined Multi-pathogen plot + + From a dictionary containing each DataFrame associated to a specific pathogen: + + - `"value"`: Sum of all the "value_" columns ("value_" set to NA for pathogen with empty DataFrame + (not selected)) + - Proportion of each pathogen : + - If two pathogens calculate proportion for primary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - If three pathogens calculate proportion for primary pathogen and secondary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - primary + secondary pathogen: `"proportion___"` = + (`"value_" + "value_"`) / `"value"` + - Calculate the median, and 50% quantiles for each "value" and "proportion" columns (and the mean + if `calc_mean` set to `True`) except: + - For proportion with two pathogens: + - secondary pathogen: `"proportion__median"` = 1 - `"proportion__median" + - For proportion with three pathogens: + - last pathogen: `"proportion__median"` = + `1 - "proportion___median"` + + Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, + 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. + + The input `pathogen_information` should be in a specific format: + + - `pathogen_information = {: {"dataframe":}, : {"dataframe":}, etc.}` + - `` is a data frame in the output format of the `sample_df()` function with 3 columns: + "target_end_date", "sample_id_n" and "value_". + - For more information, please consult the `sample_df()` function documentation + + :parameter pathogen_information: A dictionary containing multiple dictionary containing a DataFrame (result of + sampling process, key: "dataframe") and named with the associated specific pathogen (keys). + :type pathogen_information: dict + :parameter primary Name of the pathogen(s) to treat as "primary"/"secondary" for proportion calculation. + If `None` the first pathogen(s) in the `pathogen_information` + :type primary: None | list + :parameter calc_mean: Boolean indicating if the mean should be calculated too (in addition to the other quantiles) + :type calc_mean: bool + :parameter calc_prop: Boolean indicating if the proportion and associated quantiles are + calculated + :type calc_prop: bool + :parameter write_sample: If not None, write the associated samples in a csv files to the + path and filename inputted + :type write_sample: None | str + :parameter write_quantiles: If not None, write the quantiles associated with the `"value"` + columns in a csv files to the path and filename inputted + :type write_quantiles: None | str + :parameter write_proportion: If not None, write the quantiles associated with the `"proportion"` + columns in a csv files to the path and filename inputted + :type write_proportion: None | str + :return: A dictionary with 2 objects: (1) "all": median, 95%, 90%, 80%, and 50% quantiles for each "value" and + "value_-"columns and (2) "detail": median, 95%, 90%, 80%, and 50% quantiles for each + "proportion_-" columns. + """ + all_quantile, all_sample = calc_value(pathogen_information, write_sample, write_quantiles) + if calc_prop: + detail_quantile = calc_proportion(all_sample, primary=primary, + calc_mean=calc_mean, write_proportion=write_proportion) + else: + detail_quantile = None return {"all": all_quantile, "detail": detail_quantile} \ No newline at end of file From ff42172c68a5b887c8f70fddae464c57256f35ae Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Fri, 29 May 2026 10:52:09 -0400 Subject: [PATCH 6/6] add possibility to write in compress parquet file --- SMHviz_plot/utils_data.py | 20 +++++++++++++++----- pyproject.toml | 3 ++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/SMHviz_plot/utils_data.py b/SMHviz_plot/utils_data.py index b7a36da..d4f60d6 100644 --- a/SMHviz_plot/utils_data.py +++ b/SMHviz_plot/utils_data.py @@ -1,5 +1,7 @@ import numpy as np import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq def calculate_rel_change(row): @@ -359,7 +361,7 @@ def calc_value(pathogen_information, write_sample, write_quantiles): path and filename inputted :type write_sample: None | str :parameter write_quantiles: If not None, write the quantiles associated with the `"value"` - columns in a csv files to the path and filename inputted + columns in a csv or parquet files to the path and filename inputted :type write_quantiles: None | str """ all_sample = pd.DataFrame() @@ -387,9 +389,17 @@ def calc_value(pathogen_information, write_sample, write_quantiles): all_quantile = all_sample.groupby(["target_end_date"]).agg(f) all_quantile.columns = all_quantile.columns.get_level_values(0) + "-" + all_quantile.columns.get_level_values(1) if write_sample is not None: - all_sample.to_csv(write_sample) + if write_sample.endswith(".csv"): + all_sample.to_csv(write_sample) + else: + table = pa.Table.from_pandas(all_sample) + pq.write_table(table, write_sample, compression="GZIP", compression_level=9) if write_quantiles is not None: - all_quantile.to_csv(write_quantiles) + if write_quantiles.endswith(".csv"): + all_quantile.to_csv(write_quantiles) + else: + table = pa.Table.from_pandas(all_quantile) + pq.write_table(table, write_quantiles, compression="GZIP", compression_level=9) return all_quantile, all_sample @@ -528,11 +538,11 @@ def prep_multipat_plot_comb(pathogen_information, primary=None, calc_mean=False, :parameter calc_prop: Boolean indicating if the proportion and associated quantiles are calculated :type calc_prop: bool - :parameter write_sample: If not None, write the associated samples in a csv files to the + :parameter write_sample: If not None, write the associated samples in a csv pr parquet files to the path and filename inputted :type write_sample: None | str :parameter write_quantiles: If not None, write the quantiles associated with the `"value"` - columns in a csv files to the path and filename inputted + columns in a csv or parquet files to the path and filename inputted :type write_quantiles: None | str :parameter write_proportion: If not None, write the quantiles associated with the `"proportion"` columns in a csv files to the path and filename inputted diff --git a/pyproject.toml b/pyproject.toml index cd19a65..6bb4447 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,8 @@ dependencies = [ "datetime", "numpy>=1.23.5", "pandas>=1.5.2", - "plotly>=5.9.0" + "plotly>=5.9.0", + "pyarrow" ] [tool.setuptools.packages.find]