diff --git a/SMHviz_plot/figures.py b/SMHviz_plot/figures.py index c3114fc..6b5ea06 100644 --- a/SMHviz_plot/figures.py +++ b/SMHviz_plot/figures.py @@ -1125,7 +1125,7 @@ def make_spaghetti_plot(df, legend_col="model_name", spag_col="type_id", show_le def make_combine_multi_pathogen_plot(list_df, list_pathogen, truth_data=None, opacity=0.2, color=None, palette="turbo", - intervals_dict=None, intervals=None, bar_interval=0.5, bar_calc="med", title=None, + intervals_dict=None, intervals=None, bar_interval=0.5, bar_calc="median", title=None, y_axis_title="", error_bar_pat=None): """Create the Multi-Pathogen Combined plot @@ -1143,7 +1143,7 @@ def make_combine_multi_pathogen_plot(list_df, list_pathogen, truth_data=None, op and 50% quantiles for each "value" and "value_-"columns and (2) "detail": median, 95%, 90%, 80%, and 50% quantiles for each "proportion_-" columns. Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, - 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. + 0.95, 0.975, respectively. The median and mean are noted as "median" and "mean", respectively. :parameter list_df: A dictionary with 2 DataFrame: (1) "all": median, 95%, 90%, 80%, and 50% quantiles for the combined ("value" column) and for each pathogen ("value_" columns) and (2) "detail": median, 95%, 90%, @@ -1176,7 +1176,7 @@ def make_combine_multi_pathogen_plot(list_df, list_pathogen, truth_data=None, op :type intervals: list :parameter bar_interval: Interval to use for the error bar in the second subplot, by default `0.5`. :type bar_interval: float - :parameter bar_calc: Value to use for the bar height, should match columns names. By default, "med" + :parameter bar_calc: Value to use for the bar height, should match columns names. By default, "median" :type bar_calc: str :parameter title: Title of the plot, by default `None` (no title). :type title: str @@ -1326,4 +1326,4 @@ def make_combine_multi_pathogen_plot(list_df, list_pathogen, truth_data=None, op ) if title is not None: fig.update_layout(title=dict(text=title, font=dict(size=18), xanchor="center", xref="paper", x=0.5)) - return fig + return fig \ No newline at end of file diff --git a/SMHviz_plot/utils_data.py b/SMHviz_plot/utils_data.py index 8b4e51e..d4f60d6 100644 --- a/SMHviz_plot/utils_data.py +++ b/SMHviz_plot/utils_data.py @@ -1,5 +1,7 @@ import numpy as np import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq def calculate_rel_change(row): @@ -310,7 +312,7 @@ def q8(x): return x.quantile(0.975) -def med(x): +def median(x): """Calculate the quantile 0.5 (or median) Calculate the quantile 0.5 (or median) on a specific pandas Series @@ -334,43 +336,40 @@ def mean(x): return x.mean() -def prep_multipat_plot_comb(pathogen_information, calc_mean=False): - """Process Data for Combined Multi-pathogen plot +def calc_value(pathogen_information, write_sample, write_quantiles): + """Process Data for Combined Multi-pathogen plot (internal function_ - From a dictionary containing each DataFrame associated to a specific pathogen: + From a dictionary containing each DataFrame associated to a specific pathogen: - - `"value"`: Sum of all the "value_" columns ("value_" set to NA for pathogen with empty DataFrame - (not selected)) - - Proportion of each pathogen `"proportion_" = "value_" / "value"` - - Calculate the median, 95%, 90%, 80%, and 50% quantiles for each "value" and "proportion" columns (and the mean - if `calc_mean` set to `True`) + - `"value"`: Sum of all the "value_" columns ("value_" set to NA for pathogen with empty DataFrame + (not selected)) - Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, - 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. + Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, + 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. - The input `pathogen_information` should be in a specific format: + The input `pathogen_information` should be in a specific format: - - `pathogen_information = {: {"dataframe":}, : {"dataframe":}, etc.}` - - `` is a data frame in the output format of the `sample_df()` function with 3 columns: - "target_end_date", "sample_id_n" and "value_". - - For more information, please consult the `sample_df()` function documentation + - `pathogen_information = {: {"dataframe":}, : {"dataframe":}, etc.}` + - `` is a data frame in the output format of the `sample_df()` function with 3 columns: + "target_end_date", "sample_id_n" and "value_". + - For more information, please consult the `sample_df()` function documentation - :parameter pathogen_information: A dictionary containing multiple dictionary containing a DataFrame (result of - sampling process, key: "dataframe") and named with the associated specific pathogen (keys). - :type pathogen_information: dict - :parameter calc_mean: Boolean indicating if the mean should be calculated too (in addition to the other quantiles) - :type calc_mean: bool - :return: A dictionary with 2 objects: (1) "all": median, 95%, 90%, 80%, and 50% quantiles for each "value" and - "value_-"columns and (2) "detail": median, 95%, 90%, 80%, and 50% quantiles for each - "proportion_-" columns. + :parameter pathogen_information: A dictionary containing multiple dictionary containing a DataFrame (result of + sampling process, key: "dataframe") and named with the associated specific pathogen (keys). + :type pathogen_information: dict + :parameter write_sample: If not None, write the associated samples in a csv files to the + path and filename inputted + :type write_sample: None | str + :parameter write_quantiles: If not None, write the quantiles associated with the `"value"` + columns in a csv or parquet files to the path and filename inputted + :type write_quantiles: None | str """ all_sample = pd.DataFrame() - f = {'value': [med, q1, q2, q3, q4, q5, q6, q7, q8]} - f2 = {} + f = {'value': [median, q1, q2, q3, q4, q5, q6, q7, q8]} for patho in pathogen_information: # Preparation pathogen_name = patho.lower() - f.update({"value_" + pathogen_name: [med, q1, q2, q3, q4, q5, q6, q7, q8]}) + f.update({"value_" + pathogen_name: [median, q1, q2, q3, q4, q5, q6, q7, q8]}) # Merge all pathogen in one dataframe if len(all_sample) > 0: if len(pathogen_information[patho]["dataframe"]) > 0: @@ -386,19 +385,176 @@ def prep_multipat_plot_comb(pathogen_information, calc_mean=False): all_sample[col] = pd.NA # Calculate sum of all pathogen all_sample["value"] = all_sample[[col for col in all_sample.columns if col.startswith('value_')]].sum(axis=1) - # Calculate proportion of each pathogen - for patho in pathogen_information: - # Preparation - pathogen_name = patho.lower() - if calc_mean is True: - f2.update({"proportion_" + pathogen_name: [med, mean, q1, q2, q3, q4, q5, q6, q7, q8]}) - else: - f2.update({"proportion_" + pathogen_name: [med, q1, q2, q3, q4, q5, q6, q7, q8]}) - all_sample["proportion_" + pathogen_name] = all_sample["value_" + pathogen_name] / all_sample["value"] - # Calculate the quantiles for each "value" and "proportion" columns + # Calculate the quantiles for each "value" columns all_quantile = all_sample.groupby(["target_end_date"]).agg(f) all_quantile.columns = all_quantile.columns.get_level_values(0) + "-" + all_quantile.columns.get_level_values(1) + if write_sample is not None: + if write_sample.endswith(".csv"): + all_sample.to_csv(write_sample) + else: + table = pa.Table.from_pandas(all_sample) + pq.write_table(table, write_sample, compression="GZIP", compression_level=9) + if write_quantiles is not None: + if write_quantiles.endswith(".csv"): + all_quantile.to_csv(write_quantiles) + else: + table = pa.Table.from_pandas(all_quantile) + pq.write_table(table, write_quantiles, compression="GZIP", compression_level=9) + return all_quantile, all_sample + + +def calc_proportion(all_sample, primary=None, + calc_mean=False, write_proportion=None): + """Process Data for Combined Multi-pathogen plot (internal function) + + From a DataFrame containing samples for specific pathogens: + + - Proportion of each pathogen : + - If two pathogens calculate proportion for primary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - If three pathogens calculate proportion for primary pathogen and secondary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - primary + secondary pathogen: `"proportion___"` = + (`"value_" + "value_"`) / `"value"` + - Calculate the median, and 50% quantiles for each "value" and "proportion" columns (and the mean + if `calc_mean` set to `True`) except: + - For proportion with two pathogens: + - secondary pathogen: `"proportion__median"` = 1 - `"proportion__median" + - For proportion with three pathogens: + - last pathogen: `"proportion__median"` = + `1 - "proportion___median"` + + Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, + 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. + :parameter all_sample: A DataFrame containing the second result of `calc_value` function ( + sampling process output) + :type all_sample: pd.DataFrame + :parameter primary Name of the pathogen(s) to treat as "primary"/"secondary" for proportion calculation. + If `None` the first pathogen(s) in the `pathogen_information` + :type primary: None | list + :parameter calc_mean: Boolean indicating if the mean should be calculated too (in addition to the other quantiles) + :type calc_mean: bool + :parameter write_proportion: If not None, write the quantiles associated with the `"proportion"` + columns in a csv files to the path and filename inputted + :type write_proportion: None | str + """ + # Calculate proportion of each pathogen + f2 = {} + ## Prep column selection + all_sample = all_sample.round({"value": 5}) + all_sample = all_sample.dropna(axis=1, how='all') + list_patho = list(all_sample.filter(regex='^value_').columns) + list_patho = [s.replace('value_', '') for s in list_patho] + if primary is None: + if len(list_patho) == 2: + primary = list_patho[0] + if len(list_patho) == 3: + primary = list_patho[:2] + else: + if len(list_patho) == 2 and len(primary) != 1: + raise ValueError("Primary should be of length 1") + if len(list_patho) == 3 and len(primary) != 2: + raise ValueError("Primary should be of length 2") + other = [] + for patho in list_patho: + if patho.lower() not in primary: + other.append(patho.lower()) + ## Calculate proportion + if (all_sample["value"] == 0).any(): + all_sample = all_sample[all_sample["value"] > 0] + all_sample["proportion_" + primary[0]] = all_sample["value_" + primary[0]] / all_sample["value"] + if calc_mean is True: + f2.update({"proportion_" + primary[0]: [median, mean, q4, q5]}) + else: + f2.update({"proportion_" + primary[0]: [median, q4, q5]}) + if len(primary) > 1: + all_sample["proportion_" + primary[0] + "_" + primary[1]] = ( + (all_sample["value_" + primary[0]] + all_sample["value_" + primary[1]]) / + all_sample["value"]) + if calc_mean: + f2.update({"proportion_" + primary[0] + "_" + primary[1]: [median, mean]}) + else: + f2.update({"proportion_" + primary[0] + "_" + primary[1]: [median]}) + # Calculate the quantiles for each "proportion" columns detail_quantile = all_sample.groupby(["target_end_date"]).agg(f2) detail_quantile.columns = (detail_quantile.columns.get_level_values(0) + "-" + detail_quantile.columns.get_level_values(1)) - return {"all": all_quantile, "detail": detail_quantile} + if len(primary) > 1: + detail_quantile["proportion_" + other[0] + "-median"] = ( + 1 - detail_quantile["proportion_" + primary[0] + "_" + primary[1] + "-median"]) + detail_quantile["proportion_" + primary[1] + "-median"] = ( + 1 - detail_quantile["proportion_" + other[0] + "-median"] - + detail_quantile["proportion_" + primary[0] + "-median"]) + else: + detail_quantile["proportion_" + other[0] + "-median"] = ( + 1 - detail_quantile["proportion_" + primary[0] + "-median"]) + if write_proportion is not None: + detail_quantile.to_csv(write_proportion) + return detail_quantile + + +def prep_multipat_plot_comb(pathogen_information, primary=None, calc_mean=False, + calc_prop=True, write_sample=None, write_quantiles=None, + write_proportion=None): + """Process Data for Combined Multi-pathogen plot + + From a dictionary containing each DataFrame associated to a specific pathogen: + + - `"value"`: Sum of all the "value_" columns ("value_" set to NA for pathogen with empty DataFrame + (not selected)) + - Proportion of each pathogen : + - If two pathogens calculate proportion for primary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - If three pathogens calculate proportion for primary pathogen and secondary pathogen: + - primary pathogen: `"proportion_" = "value_" / "value"` + - primary + secondary pathogen: `"proportion___"` = + (`"value_" + "value_"`) / `"value"` + - Calculate the median, and 50% quantiles for each "value" and "proportion" columns (and the mean + if `calc_mean` set to `True`) except: + - For proportion with two pathogens: + - secondary pathogen: `"proportion__median"` = 1 - `"proportion__median" + - For proportion with three pathogens: + - last pathogen: `"proportion__median"` = + `1 - "proportion___median"` + + Each quantile is noted as: q1, q2, q3, q4, q5, q6, q7, q8, corresponding to: 0.025, 0.05, 0.1, 0.25, 0.75, 0.9, + 0.95, 0.975, respectively. The median and mean are noted as "med" and "mean", respectively. + + The input `pathogen_information` should be in a specific format: + + - `pathogen_information = {: {"dataframe":}, : {"dataframe":}, etc.}` + - `` is a data frame in the output format of the `sample_df()` function with 3 columns: + "target_end_date", "sample_id_n" and "value_". + - For more information, please consult the `sample_df()` function documentation + + :parameter pathogen_information: A dictionary containing multiple dictionary containing a DataFrame (result of + sampling process, key: "dataframe") and named with the associated specific pathogen (keys). + :type pathogen_information: dict + :parameter primary Name of the pathogen(s) to treat as "primary"/"secondary" for proportion calculation. + If `None` the first pathogen(s) in the `pathogen_information` + :type primary: None | list + :parameter calc_mean: Boolean indicating if the mean should be calculated too (in addition to the other quantiles) + :type calc_mean: bool + :parameter calc_prop: Boolean indicating if the proportion and associated quantiles are + calculated + :type calc_prop: bool + :parameter write_sample: If not None, write the associated samples in a csv pr parquet files to the + path and filename inputted + :type write_sample: None | str + :parameter write_quantiles: If not None, write the quantiles associated with the `"value"` + columns in a csv or parquet files to the path and filename inputted + :type write_quantiles: None | str + :parameter write_proportion: If not None, write the quantiles associated with the `"proportion"` + columns in a csv files to the path and filename inputted + :type write_proportion: None | str + :return: A dictionary with 2 objects: (1) "all": median, 95%, 90%, 80%, and 50% quantiles for each "value" and + "value_-"columns and (2) "detail": median, 95%, 90%, 80%, and 50% quantiles for each + "proportion_-" columns. + """ + all_quantile, all_sample = calc_value(pathogen_information, write_sample, write_quantiles) + if calc_prop: + detail_quantile = calc_proportion(all_sample, primary=primary, + calc_mean=calc_mean, write_proportion=write_proportion) + else: + detail_quantile = None + return {"all": all_quantile, "detail": detail_quantile} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cd19a65..6bb4447 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,8 @@ dependencies = [ "datetime", "numpy>=1.23.5", "pandas>=1.5.2", - "plotly>=5.9.0" + "plotly>=5.9.0", + "pyarrow" ] [tool.setuptools.packages.find]