fix: gap identification and improve gap statistics and visualization (#1423)

* fix: wording on time index tab

* fix: improve gap identification

* fix: add minutes and hours as intervals

* fix: add the frequency to the stats tab

* fix: tests

* fix: linter issues

* docs: added gap analysis info to the ts doc

* feat: humanize timespan outputs

* fix: parameter order

* fix: std failing with only one gap

* fix: remove unused import

* fix: adjust the doc msg

Author: alexbarros

docsrc/source/index.rst

@@ -22,6 +22,7 @@
    pages/use_cases/profiling_databases
    pages/use_cases/metadata
    pages/use_cases/custom_report_appearance
+   pages/use_cases/time_series_datasets
 
 .. toctree::
    :maxdepth: 3

docsrc/source/pages/use_cases/time_series_datasets.rst

@@ -2,9 +2,9 @@
 Time-Series data
 ==================
 
-``pandas-profiling`` can be used for a quick Exploratory Data Analysis on time-series data. This is useful for a quick understading on the behaviour of time dependent variables regarding behaviours such as time plots, seasonality, trends and stationarity.
+``ydata-profiling`` can be used for a quick Exploratory Data Analysis on time-series data. This is useful for a quick understading on the behaviour of time dependent variables regarding behaviours such as time plots, seasonality, trends, stationarity and data gaps.
 
-Combined with the profiling reports compare, you're able to compare the evolution and data behaviour through time, in terms of time-series specific statistics such as PACF and ACF plots.
+Combined with the profiling reports compare, you're able to compare the evolution and data behaviour through time, in terms of time-series specific statistics such as PACF and ACF plots. It also provides the identification of gaps in the time series, caused either by missing values or by entries missing in the time index.
 
 The following syntax can be used to generate a profile under the assumption that the dataset includes time dependent features:
 
@@ -60,8 +60,6 @@ In some cases you might be already aware of what variables are expected to be ti
         "NO2 1st Max Value": "timeseries",
         "NO2 1st Max Hour": "timeseries",
         "NO2 AQI": "timeseries",
-        "cos": "numeric",
-        "cat": "numeric",
     }
 
     profile = ProfileReport(

src/ydata_profiling/model/pandas/describe_timeseries_pandas.py

@@ -7,7 +7,6 @@
 from statsmodels.tsa.stattools import adfuller
 
 from ydata_profiling.config import Settings
-from ydata_profiling.model.pandas.utils_pandas import get_period_and_frequency
 from ydata_profiling.model.summary_algorithms import (
     describe_numeric_1d,
     describe_timeseries_1d,
@@ -142,6 +141,25 @@ def get_fft_peaks(
     return threshold, orig_peaks, peaks
 
 
+def identify_gaps(
+    gap: pd.Series, is_datetime: bool, gap_tolerance: int = 2
+) -> Tuple[pd.Series, list]:
+    zero = pd.Timedelta(0) if is_datetime else 0
+    diff = gap.diff()
+
+    non_zero_diff = diff[diff > zero]
+    min_gap_size = gap_tolerance * non_zero_diff.mean()
+
+    gap_stats = non_zero_diff[non_zero_diff > min_gap_size]
+    anchors = gap[diff > min_gap_size].index
+
+    gaps = []
+    for i in anchors:
+        gaps.append(gap.loc[gap.index[[i - 1, i]]].values)
+
+    return gap_stats, gaps
+
+
 def compute_gap_stats(series: pd.Series) -> pd.Series:
     """Computes the intertevals in the series normalized by the period.
 
@@ -157,31 +175,17 @@ def compute_gap_stats(series: pd.Series) -> pd.Series:
     gap = gap.reset_index()[index_name]
     gap.index.name = None
 
-    if isinstance(series.index, pd.DatetimeIndex):
-        period, frequency = get_period_and_frequency(series.index)
-        period = pd.Timedelta(f"{period} {frequency}")
-        base_frequency = pd.Timedelta(f"1 {frequency}")
-    else:
-        period = np.abs(np.diff(series.index)).mean()
-        base_frequency = 1
-
-    diff = gap.diff()
-    anchors = gap[diff > 2 * period].index
-    gaps = []
-    for i in anchors:
-        gaps.append(gap.loc[gap.index[[i - 1, i]]].values)
+    is_datetime = isinstance(series.index, pd.DatetimeIndex)
+    gap_stats, gaps = identify_gaps(gap, is_datetime)
 
     stats = {
-        "period": period / base_frequency,
-        "min": diff.min() / base_frequency,
-        "max": diff.max() / base_frequency,
-        "mean": diff.mean() / base_frequency,
-        "std": diff.std() / base_frequency,
+        "min": gap_stats.min(),
+        "max": gap_stats.max(),
+        "mean": gap_stats.mean(),
+        "std": gap_stats.std() if len(gap_stats) > 1 else 0,
         "series": series,
         "gaps": gaps,
     }
-    if isinstance(series.index, pd.DatetimeIndex):
-        stats["frequency"] = frequency
     return stats
 
 

src/ydata_profiling/model/pandas/timeseries_index_pandas.py

@@ -4,7 +4,6 @@
 from pandas.api.types import is_numeric_dtype
 
 from ydata_profiling.config import Settings
-from ydata_profiling.model.pandas.utils_pandas import get_period_and_frequency
 from ydata_profiling.model.timeseries_index import get_time_index_description
 
 
@@ -21,17 +20,14 @@ def pandas_get_time_index_description(
     length = table_stats["n"]
     start = df.index.min()
     end = df.index.max()
+    period = abs(np.diff(df.index)).mean()
     if isinstance(df.index, pd.DatetimeIndex):
-        period, freq = get_period_and_frequency(df.index)
-    else:
-        freq = None
-        period = abs(np.diff(df.index)).mean()
+        period = pd.Timedelta(period)
 
     return {
         "n_series": n_series,
         "length": length,
         "start": start,
         "end": end,
-        "frequency": freq,
         "period": period,
     }

src/ydata_profiling/model/pandas/utils_pandas.py

@@ -1,7 +1,4 @@
-from typing import Tuple
-
 import numpy as np
-import pandas as pd
 
 
 def weighted_median(data: np.ndarray, weights: np.ndarray) -> int:
@@ -28,21 +25,3 @@ def weighted_median(data: np.ndarray, weights: np.ndarray) -> int:
         else:
             w_median = s_data[idx + 1]
     return w_median
-
-
-def get_period_and_frequency(index: pd.DatetimeIndex) -> Tuple[float, str]:
-    delta = abs(np.diff(index)).mean()
-    delta = pd.Timedelta(delta)
-    if delta.days > 0:
-        frequency = "Days"
-        period = delta / pd.Timedelta(days=1)
-    elif delta.seconds > 0:
-        frequency = "Seconds"
-        period = delta / pd.Timedelta(seconds=1)
-    elif delta.microseconds > 0:
-        frequency = "Microseconds"
-        period = delta / pd.Timedelta(microseconds=1)
-    else:
-        frequency = "Nanoseconds"
-        period = delta.nanoseconds / pd.Timedelta(nanoseconds=1)
-    return period, frequency

src/ydata_profiling/report/formatters.py

@@ -6,6 +6,7 @@
 from typing import Any, Callable, Dict, List, Optional, Union
 
 import numpy as np
+import pandas as pd
 from markupsafe import escape
 
 
@@ -218,6 +219,20 @@ def pluralize(count: Any, singular: str, plural: Optional[str] = None) -> str:
             return concatenate(result)
 
 
+def fmt_timespan_timedelta(
+    delta: Any, detailed: bool = False, max_units: int = 3, precision: int = 10
+) -> str:
+    if isinstance(delta, pd.Timedelta):
+        num_seconds = delta.total_seconds()
+        if delta.microseconds > 0:
+            num_seconds += delta.microseconds * 1e-6
+        if delta.nanoseconds > 0:
+            num_seconds += delta.nanoseconds * 1e-9
+        return fmt_timespan(num_seconds, detailed, max_units)
+    else:
+        return fmt_numeric(delta, precision)
+
+
 @list_args
 def fmt_numeric(value: float, precision: int = 10) -> str:
     """Format any numeric value.

src/ydata_profiling/report/structure/overview.py

@@ -13,6 +13,7 @@
     fmt_numeric,
     fmt_percent,
     fmt_timespan,
+    fmt_timespan_timedelta,
     list_args,
 )
 from ydata_profiling.report.presentation.core import Alerts, Container
@@ -298,18 +299,10 @@ def format_tsindex_limit(limit: Any) -> str:
         },
         {
             "name": "Period",
-            "value": fmt_number(summary.time_index_analysis.period),
+            "value": fmt_timespan_timedelta(summary.time_index_analysis.period),
         },
     ]
 
-    if summary.time_index_analysis.frequency:
-        table_stats.append(
-            {
-                "name": "Frequency",
-                "value": summary.time_index_analysis.frequency,
-            }
-        )
-
     ts_info = Table(table_stats, name="Timeseries statistics", style=config.html.style)
 
     dpi_bak = config.plot.dpi
@@ -318,14 +311,14 @@ def format_tsindex_limit(limit: Any) -> str:
         plot_overview_timeseries(config, summary.variables),
         image_format=config.plot.image_format,
         alt="ts_plot",
-        name="original",
+        name="Original",
         anchor_id="ts_plot_overview",
     )
     timeseries_scaled = ImageWidget(
         plot_overview_timeseries(config, summary.variables, scale=True),
         image_format=config.plot.image_format,
         alt="ts_plot_scaled",
-        name="scaled",
+        name="Scaled",
         anchor_id="ts_plot_scaled_overview",
     )
     config.plot.dpi = dpi_bak

src/ydata_profiling/report/structure/variables/render_timeseries.py

@@ -5,6 +5,7 @@
     fmt_monotonic,
     fmt_numeric,
     fmt_percent,
+    fmt_timespan_timedelta,
 )
 from ydata_profiling.report.presentation.core import (
     Container,
@@ -25,34 +26,40 @@
 def _render_gap_tab(config: Settings, summary: dict) -> Container:
     gap_stats = [
         {
-            "name": "min inverval",
+            "name": "number of gaps",
             "value": fmt_numeric(
+                len(summary["gap_stats"]["gaps"]), precision=config.report.precision
+            ),
+        },
+        {
+            "name": "min",
+            "value": fmt_timespan_timedelta(
                 summary["gap_stats"]["min"], precision=config.report.precision
             ),
         },
         {
-            "name": "max inverval",
-            "value": fmt_numeric(
+            "name": "max",
+            "value": fmt_timespan_timedelta(
                 summary["gap_stats"]["max"], precision=config.report.precision
             ),
         },
         {
-            "name": "mean inverval",
-            "value": fmt_numeric(
+            "name": "mean",
+            "value": fmt_timespan_timedelta(
                 summary["gap_stats"]["mean"], precision=config.report.precision
             ),
         },
         {
-            "name": "interval std",
-            "value": fmt_numeric(
+            "name": "std",
+            "value": fmt_timespan_timedelta(
                 summary["gap_stats"]["std"], precision=config.report.precision
             ),
         },
     ]
 
     gap_table = Table(
         gap_stats,
-        name="Intervals statistics",
+        name="Gap statistics",
         style=config.html.style,
     )