feat: add spark support

vorel99 · vorel99 · commit b8069d8b8926 · 2024-09-23T15:37:49.000+02:00
diff --git a/src/ydata_profiling/model/alerts.py b/src/ydata_profiling/model/alerts.py
@@ -170,7 +170,10 @@ def __init__(
     ):
         super().__init__(
             alert_type=AlertType.CONSTANT,
-            values={"n_distinct": values["n_distinct"]},
+            values={
+                "n_distinct": values["n_distinct"],
+                "value_counts_without_nan": values.value_counts_without_nan,
+            },
             column_name=column_name,
             is_empty=is_empty,
         )
diff --git a/src/ydata_profiling/model/pandas/__init__.py b/src/ydata_profiling/model/pandas/__init__.py
@@ -3,10 +3,8 @@
     dataframe_pandas,
     describe_boolean_pandas,
     describe_categorical_pandas,
-    describe_counts_pandas,
     describe_date_pandas,
     describe_file_pandas,
-    describe_generic_pandas,
     describe_image_pandas,
     describe_numeric_pandas,
     describe_path_pandas,
@@ -27,10 +25,8 @@
     "dataframe_pandas",
     "describe_boolean_pandas",
     "describe_categorical_pandas",
-    "describe_counts_pandas",
     "describe_date_pandas",
     "describe_file_pandas",
-    "describe_generic_pandas",
     "describe_image_pandas",
     "describe_numeric_pandas",
     "describe_path_pandas",
diff --git a/src/ydata_profiling/model/pandas/describe_counts_pandas.py b/src/ydata_profiling/model/pandas/describe_counts_pandas.py
diff --git a/src/ydata_profiling/model/pandas/describe_date_pandas.py b/src/ydata_profiling/model/pandas/describe_date_pandas.py
@@ -30,7 +30,7 @@ def pandas_describe_date_1d(
     Returns:
         A dict containing calculated series description values.
     """
-    if summary["value_counts_without_nan"].empty:
+    if summary.value_counts_without_nan.empty:
         values = series.values
         summary.update(
             {
diff --git a/src/ydata_profiling/model/pandas/describe_generic_pandas.py b/src/ydata_profiling/model/pandas/describe_generic_pandas.py
diff --git a/src/ydata_profiling/model/spark/__init__.py b/src/ydata_profiling/model/spark/__init__.py
@@ -3,9 +3,7 @@
     dataframe_spark,
     describe_boolean_spark,
     describe_categorical_spark,
-    describe_counts_spark,
     describe_date_spark,
-    describe_generic_spark,
     describe_numeric_spark,
     describe_supported_spark,
     duplicates_spark,
@@ -21,9 +19,7 @@
     "dataframe_spark",
     "describe_boolean_spark",
     "describe_categorical_spark",
-    "describe_counts_spark",
     "describe_date_spark",
-    "describe_generic_spark",
     "describe_numeric_spark",
     "describe_supported_spark",
     "duplicates_spark",
diff --git a/src/ydata_profiling/model/spark/correlations_spark.py b/src/ydata_profiling/model/spark/correlations_spark.py
@@ -1,4 +1,5 @@
 """Correlations between variables."""
+
 from typing import Optional
 
 import pandas as pd
diff --git a/src/ydata_profiling/model/spark/describe_boolean_spark.py b/src/ydata_profiling/model/spark/describe_boolean_spark.py
@@ -4,12 +4,13 @@
 
 from ydata_profiling.config import Settings
 from ydata_profiling.model.summary_algorithms import describe_boolean_1d
+from ydata_profiling.model.var_description.default import VarDescription
 
 
 @describe_boolean_1d.register
 def describe_boolean_1d_spark(
-    config: Settings, df: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+    config: Settings, df: DataFrame, summary: VarDescription
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a boolean series.
 
     Args:
@@ -20,7 +21,7 @@ def describe_boolean_1d_spark(
         A dict containing calculated series description values.
     """
 
-    value_counts = summary["value_counts"]
+    value_counts = summary.value_counts
 
     # get the most common boolean value and its frequency
     top = value_counts.first()
diff --git a/src/ydata_profiling/model/spark/describe_categorical_spark.py b/src/ydata_profiling/model/spark/describe_categorical_spark.py
@@ -4,12 +4,13 @@
 
 from ydata_profiling.config import Settings
 from ydata_profiling.model.summary_algorithms import describe_categorical_1d
+from ydata_profiling.model.var_description.default import VarDescription
 
 
 @describe_categorical_1d.register
 def describe_categorical_1d_spark(
-    config: Settings, df: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+    config: Settings, df: DataFrame, summary: VarDescription
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a categorical series.
 
     Args:
diff --git a/src/ydata_profiling/model/spark/describe_date_spark.py b/src/ydata_profiling/model/spark/describe_date_spark.py
@@ -6,6 +6,7 @@
 
 from ydata_profiling.config import Settings
 from ydata_profiling.model.summary_algorithms import describe_date_1d
+from ydata_profiling.model.var_description.default import VarDescription
 
 
 def date_stats_spark(df: DataFrame, summary: dict) -> dict:
@@ -21,8 +22,8 @@ def date_stats_spark(df: DataFrame, summary: dict) -> dict:
 
 @describe_date_1d.register
 def describe_date_1d_spark(
-    config: Settings, df: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+    config: Settings, df: DataFrame, summary: VarDescription
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a date series.
 
     Args:
diff --git a/src/ydata_profiling/model/spark/describe_generic_spark.py b/src/ydata_profiling/model/spark/describe_generic_spark.py
diff --git a/src/ydata_profiling/model/spark/describe_numeric_spark.py b/src/ydata_profiling/model/spark/describe_numeric_spark.py
@@ -9,9 +9,10 @@
     describe_numeric_1d,
     histogram_compute,
 )
+from ydata_profiling.model.var_description.default import VarDescription
 
 
-def numeric_stats_spark(df: DataFrame, summary: dict) -> dict:
+def numeric_stats_spark(df: DataFrame, summary: VarDescription) -> dict:
     column = df.columns[0]
 
     expr = [
@@ -29,8 +30,8 @@ def numeric_stats_spark(df: DataFrame, summary: dict) -> dict:
 
 @describe_numeric_1d.register
 def describe_numeric_1d_spark(
-    config: Settings, df: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+    config: Settings, df: DataFrame, summary: VarDescription
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a boolean series.
 
     Args:
@@ -51,7 +52,7 @@ def describe_numeric_1d_spark(
     summary["kurtosis"] = stats["kurtosis"]
     summary["sum"] = stats["sum"]
 
-    value_counts = summary["value_counts"]
+    value_counts = summary.value_counts
 
     n_infinite = (
         value_counts.where(F.col(df.columns[0]).isin([np.inf, -np.inf]))
@@ -106,12 +107,12 @@ def describe_numeric_1d_spark(
     ).stat.approxQuantile("abs_dev", [0.5], quantile_threshold)[0]
 
     # FIXME: move to fmt
-    summary["p_negative"] = summary["n_negative"] / summary["n"]
+    summary["p_negative"] = summary["n_negative"] / summary.n
     summary["range"] = summary["max"] - summary["min"]
     summary["iqr"] = summary["75%"] - summary["25%"]
     summary["cv"] = summary["std"] / summary["mean"] if summary["mean"] else np.NaN
-    summary["p_zeros"] = summary["n_zeros"] / summary["n"]
-    summary["p_infinite"] = summary["n_infinite"] / summary["n"]
+    summary["p_zeros"] = summary["n_zeros"] / summary.n
+    summary["p_infinite"] = summary["n_infinite"] / summary.n
 
     # TODO - enable this feature
     # because spark doesn't have an indexing system, there isn't really the idea of monotonic increase/decrease
@@ -124,14 +125,14 @@ def describe_numeric_1d_spark(
     # display in pandas display
     # the alternative is to do this in spark natively, but it is not trivial
     infinity_values = [np.inf, -np.inf]
-    infinity_index = summary["value_counts_without_nan"].index.isin(infinity_values)
+    infinity_index = summary.value_counts_without_nan.index.isin(infinity_values)
 
     summary.update(
         histogram_compute(
             config,
-            summary["value_counts_without_nan"][~infinity_index].index.values,
+            summary.value_counts_without_nan[~infinity_index].index.values,
             summary["n_distinct"],
-            weights=summary["value_counts_without_nan"][~infinity_index].values,
+            weights=summary.value_counts_without_nan[~infinity_index].values,
         )
     )
 
diff --git a/src/ydata_profiling/model/spark/describe_supported_spark.py b/src/ydata_profiling/model/spark/describe_supported_spark.py
@@ -3,13 +3,17 @@
 from pyspark.sql import DataFrame
 
 from ydata_profiling.config import Settings
+from ydata_profiling.model.spark.var_description.default_spark import (
+    get_default_spark_description,
+)
 from ydata_profiling.model.summary_algorithms import describe_supported
+from ydata_profiling.model.var_description.default import VarDescription
 
 
 @describe_supported.register
 def describe_supported_spark(
     config: Settings, series: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a supported series.
     Args:
         series: The Series to describe.
@@ -18,16 +22,6 @@ def describe_supported_spark(
         A dict containing calculated series description values.
     """
 
-    # number of non-NaN observations in the Series
-    count = summary["count"]
-    n_distinct = summary["value_counts"].count()
+    series_description = get_default_spark_description(config, series, summary)
 
-    summary["n_distinct"] = n_distinct
-    summary["p_distinct"] = n_distinct / count if count > 0 else 0
-
-    n_unique = summary["value_counts"].where("count == 1").count()
-    summary["is_unique"] = n_unique == count
-    summary["n_unique"] = n_unique
-    summary["p_unique"] = n_unique / count if count > 0 else 0
-
-    return config, series, summary
+    return config, series, series_description
diff --git a/src/ydata_profiling/model/spark/describe_text_spark.py b/src/ydata_profiling/model/spark/describe_text_spark.py
@@ -4,12 +4,13 @@
 
 from ydata_profiling.config import Settings
 from ydata_profiling.model.summary_algorithms import describe_text_1d
+from ydata_profiling.model.var_description.default import VarDescription
 
 
 @describe_text_1d.register
 def describe_text_1d_spark(
-    config: Settings, df: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+    config: Settings, df: DataFrame, summary: VarDescription
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a categorical series.
 
     Args:
diff --git a/src/ydata_profiling/model/spark/summary_spark.py b/src/ydata_profiling/model/spark/summary_spark.py
@@ -1,4 +1,5 @@
 """Compute statistical description of datasets."""
+
 import multiprocessing
 from typing import Tuple
 
diff --git a/src/ydata_profiling/model/spark/timeseries_index_spark.py b/src/ydata_profiling/model/spark/timeseries_index_spark.py
@@ -1,4 +1,5 @@
 """Compute statistical description of datasets."""
+
 from pyspark.sql import DataFrame
 
 from ydata_profiling.config import Settings
diff --git a/src/ydata_profiling/model/spark/var_description/counts_spark.py b/src/ydata_profiling/model/spark/var_description/counts_spark.py
diff --git a/src/ydata_profiling/model/spark/var_description/default_spark.py b/src/ydata_profiling/model/spark/var_description/default_spark.py
diff --git a/src/ydata_profiling/model/var_description/default.py b/src/ydata_profiling/model/var_description/default.py

Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,7 @@ def pandas_describe_date_1d(`
`30`	`30`	`Returns:`
`31`	`31`	`A dict containing calculated series description values.`
`32`	`32`	`"""`
`33`		`- if summary["value_counts_without_nan"].empty:`
	`33`	`+ if summary.value_counts_without_nan.empty:`
`34`	`34`	`values = series.values`
`35`	`35`	`summary.update(`
`36`	`36`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`"""Correlations between variables."""`
	`2`	`+`
`2`	`3`	`from typing import Optional`
`3`	`4`
`4`	`5`	`import pandas as pd`