diff --git a/FALLBACK_REPORT.md b/FALLBACK_REPORT.md index c559e31..c86a973 100644 --- a/FALLBACK_REPORT.md +++ b/FALLBACK_REPORT.md @@ -53,4 +53,4 @@ | unordered | optimized_reentry | unique_users | time_to_convert | ✗ No | | | unordered | optimized_reentry | unique_users | cohort_analysis | ✗ No | | -## Recommendations +## Recommendations \ No newline at end of file diff --git a/app.py b/app.py index b474ec3..2a39914 100644 --- a/app.py +++ b/app.py @@ -125,6 +125,98 @@ # Performance monitoring decorators +# Cached Data Loading Functions +@st.cache_data +def load_sample_data_cached() -> pd.DataFrame: + """Cached wrapper for loading sample data to prevent regeneration on every UI interaction""" + from core import DataSourceManager + manager = DataSourceManager() + return manager.get_sample_data() + +@st.cache_data +def load_file_data_cached(file_name: str, file_size: int, file_type: str, file_content: bytes) -> pd.DataFrame: + """Cached wrapper for loading file data based on file properties to avoid re-processing same files""" + import os + import tempfile + + from core import DataSourceManager + + manager = DataSourceManager() + + # Create a temporary file from the uploaded content + with tempfile.NamedTemporaryFile(delete=False, suffix=f".{file_type}") as tmp_file: + tmp_file.write(file_content) + tmp_file.flush() + temp_path = tmp_file.name + + try: + # Create a mock uploaded file object for the manager + class MockUploadedFile: + def __init__(self, name, content): + self.name = name + self._content = content + + def getvalue(self): + return self._content + + mock_file = MockUploadedFile(file_name, file_content) + return manager.load_from_file(mock_file) + finally: + # Clean up temporary file + try: + os.unlink(temp_path) + except OSError: + pass + +@st.cache_data +def load_clickhouse_data_cached(query: str, connection_hash: str) -> pd.DataFrame: + """Cached wrapper for ClickHouse data loading based on query and connection""" + # Note: This assumes the connection is already established in session state + if hasattr(st.session_state, "data_source_manager") and st.session_state.data_source_manager.clickhouse_client: + return st.session_state.data_source_manager.load_from_clickhouse(query) + return pd.DataFrame() + +@st.cache_data +def get_segmentation_properties_cached(events_data: pd.DataFrame) -> dict[str, list[str]]: + """Cached wrapper for getting segmentation properties to avoid repeated JSON parsing""" + from core import DataSourceManager + manager = DataSourceManager() + return manager.get_segmentation_properties(events_data) + +@st.cache_data +def get_property_values_cached(events_data: pd.DataFrame, prop_name: str, prop_type: str) -> list[str]: + """Cached wrapper for getting property values to avoid repeated filtering""" + from core import DataSourceManager + manager = DataSourceManager() + return manager.get_property_values(events_data, prop_name, prop_type) + +@st.cache_data +def get_sorted_event_names_cached(events_data: pd.DataFrame) -> list[str]: + """Cached wrapper for getting sorted event names to avoid repeated sorting""" + return sorted(events_data["event_name"].unique()) + +@st.cache_data +def calculate_timeseries_metrics_cached( + events_data: pd.DataFrame, + funnel_steps: tuple[str, ...], + polars_period: str, + config_dict: dict[str, Any], + use_polars: bool = True +) -> pd.DataFrame: + """ + Cached wrapper for time series calculation to prevent recalculation during tab switching. + Uses tuple for funnel_steps and config dict for proper hashing. + """ + from core import FunnelCalculator + + # Reconstruct config from dict + config = FunnelConfig.from_dict(config_dict) + calculator = FunnelCalculator(config, use_polars=use_polars) + + # Convert tuple back to list for the calculator + steps_list = list(funnel_steps) + + return calculator.calculate_timeseries_metrics(events_data, steps_list, polars_period) # Data Source Management def initialize_session_state(): @@ -201,6 +293,7 @@ def create_enhanced_event_selector_DISABLED(): """Create enhanced event selector with search, filters, and categorized display - DISABLED in simplified version""" +@st.cache_data def get_comprehensive_performance_analysis() -> dict[str, Any]: """ Get comprehensive performance analysis from all monitored components @@ -254,13 +347,14 @@ def get_comprehensive_performance_analysis() -> dict[str, Any]: return analysis +@st.cache_data def get_event_statistics(events_data: pd.DataFrame) -> dict[str, dict[str, Any]]: """Get comprehensive statistics for each event in the dataset""" if events_data is None or events_data.empty: return {} event_stats = {} - event_counts = events_data["event_name"].value_counts() + events_data["event_name"].value_counts() total_events = len(events_data) unique_users = events_data["user_id"].nunique() @@ -368,10 +462,10 @@ def clear_all_steps(): if "event_statistics" not in st.session_state: st.session_state.event_statistics = get_event_statistics(st.session_state.events_data) - # Cache the sorted list of unique event names for better UI performance + # Use cached sorted event names for better UI performance if "sorted_event_names" not in st.session_state: - st.session_state.sorted_event_names = sorted( - st.session_state.events_data["event_name"].unique() + st.session_state.sorted_event_names = get_sorted_event_names_cached( + st.session_state.events_data ) available_events = st.session_state.sorted_event_names @@ -401,7 +495,7 @@ def clear_all_steps(): with event_col1: # Checkbox с улучшенным дизайном - selected = st.checkbox( + st.checkbox( f"**{event}**", value=is_selected, key=f"event_cb_{event.replace(' ', '_').replace('-', '_')}", @@ -541,9 +635,9 @@ def main(): if data_source == "Sample Data": if st.button("Load Sample Data", key="load_sample_data_button"): with st.spinner("Loading sample data..."): - st.session_state.events_data = ( - st.session_state.data_source_manager.get_sample_data() - ) + # Use cached sample data loading + st.session_state.events_data = load_sample_data_cached() + # Clear cached event names when new data is loaded if "sorted_event_names" in st.session_state: del st.session_state.sorted_event_names @@ -562,9 +656,17 @@ def main(): if uploaded_file is not None: with st.spinner("Processing file..."): - st.session_state.events_data = ( - st.session_state.data_source_manager.load_from_file(uploaded_file) + # Use cached file loading based on file properties + file_content = uploaded_file.getvalue() + file_type = uploaded_file.name.split('.')[-1].lower() + + st.session_state.events_data = load_file_data_cached( + uploaded_file.name, + uploaded_file.size, + file_type, + file_content ) + if not st.session_state.events_data.empty: # Clear cached event names when new data is loaded if "sorted_event_names" in st.session_state: @@ -612,9 +714,14 @@ def main(): if st.button("Execute Query"): with st.spinner("Executing query..."): - st.session_state.events_data = ( - st.session_state.data_source_manager.load_from_clickhouse(ch_query) + # Create a connection hash for caching + connection_hash = f"{ch_host}:{ch_port}:{ch_database}:{ch_username}" + + # Use cached ClickHouse loading + st.session_state.events_data = load_clickhouse_data_cached( + ch_query, connection_hash ) + if not st.session_state.events_data.empty: # Clear cached event names when new data is loaded if "sorted_event_names" in st.session_state: @@ -640,7 +747,7 @@ def main(): st.markdown("### ⚡ Performance") if "performance_history" in st.session_state and st.session_state.performance_history: latest_performance = st.session_state.performance_history[-1] - performance_analysis = get_comprehensive_performance_analysis() + get_comprehensive_performance_analysis() # Display current performance st.markdown("**Latest Analysis:**") @@ -660,6 +767,58 @@ def main(): st.markdown("🔄 **Ready for Analysis**") st.markdown("Performance monitoring will appear after first calculation.") + # Configuration Management + st.markdown("---") + st.markdown("### 💾 Configuration Management") + + config_col1, config_col2 = st.columns(2) + + with config_col1: + if st.button("💾 Save Config", help="Save current funnel configuration"): + if st.session_state.funnel_steps: + config_name = f"Funnel_{len(st.session_state.saved_configurations) + 1}" + config_json = FunnelConfigManager.save_config( + st.session_state.funnel_config, + st.session_state.funnel_steps, + config_name, + ) + st.session_state.saved_configurations.append((config_name, config_json)) + st.toast(f"💾 Saved as {config_name}!", icon="💾") + + with config_col2: + uploaded_config = st.file_uploader( + "📁 Load Config", + type=["json"], + help="Upload saved configuration", + key="sidebar_config_upload" + ) + + if uploaded_config is not None: + try: + config_json = uploaded_config.read().decode() + config, steps, name = FunnelConfigManager.load_config(config_json) + st.session_state.funnel_config = config + st.session_state.funnel_steps = steps + st.toast(f"📁 Loaded {name}!", icon="📁") + st.rerun() + except Exception as e: + st.error(f"Error: {str(e)}") + + # Download saved configurations + if st.session_state.saved_configurations: + st.markdown("**Saved Configs:**") + for i, (name, config_json) in enumerate(st.session_state.saved_configurations): + config_col1, config_col2 = st.columns([2, 1]) + config_col1.caption(name) + config_col2.download_button( + "⬇️", + config_json, + file_name=f"{name}.json", + mime="application/json", + key=f"sidebar_download_{i}", + help="Download", + ) + # Cache Management st.markdown("---") st.markdown("### 💾 Cache Management") @@ -773,11 +932,9 @@ def main(): selected_values = [] if st.session_state.events_data is not None and not st.session_state.events_data.empty: - # Update available properties - st.session_state.available_properties = ( - st.session_state.data_source_manager.get_segmentation_properties( - st.session_state.events_data - ) + # Update available properties using cached function + st.session_state.available_properties = get_segmentation_properties_cached( + st.session_state.events_data ) if st.session_state.available_properties: @@ -801,11 +958,9 @@ def main(): if selected_property != "None": prop_type, prop_name = selected_property.split("_", 1) - # Get available values for this property - prop_values = ( - st.session_state.data_source_manager.get_property_values( - st.session_state.events_data, prop_name, prop_type - ) + # Get available values for this property using cached function + prop_values = get_property_values_cached( + st.session_state.events_data, prop_name, prop_type ) if prop_values: @@ -914,57 +1069,7 @@ def main(): else: st.toast("⚠️ Please add at least 2 steps to create a funnel", icon="⚠️") - st.markdown("---") - - # Configuration Management - перенесено в основную область - st.markdown("### 💾 Save & Load Configurations") - col1, col2 = st.columns(2) - - with col1: - if st.button("💾 Save Current Config", use_container_width=True): - if st.session_state.funnel_steps: - config_name = f"Funnel_{len(st.session_state.saved_configurations) + 1}" - config_json = FunnelConfigManager.save_config( - st.session_state.funnel_config, - st.session_state.funnel_steps, - config_name, - ) - st.session_state.saved_configurations.append((config_name, config_json)) - st.success(f"Configuration saved as {config_name}") - - with col2: - uploaded_config = st.file_uploader( - "📁 Load Configuration File", - type=["json"], - help="Upload a previously saved funnel configuration", - ) - - if uploaded_config is not None: - try: - config_json = uploaded_config.read().decode() - config, steps, name = FunnelConfigManager.load_config(config_json) - st.session_state.funnel_config = config - st.session_state.funnel_steps = steps - st.success(f"Loaded configuration: {name}") - st.rerun() - except Exception as e: - st.error(f"Error loading configuration: {str(e)}") - - # Download saved configurations - if st.session_state.saved_configurations: - st.markdown("**Saved Configurations:**") - for i, (name, config_json) in enumerate(st.session_state.saved_configurations): - col1, col2 = st.columns([3, 1]) - col1.write(name) - col2.download_button( - "⬇️", - config_json, - file_name=f"{name}.json", - mime="application/json", - key=f"download_{i}", - help="Download configuration", - ) # STEP 4: Results - показываем только если есть результаты if st.session_state.analysis_results: @@ -1451,8 +1556,16 @@ def format_time(minutes): if hasattr(st.session_state.data_source_manager, "_last_lazy_df"): lazy_df = st.session_state.data_source_manager.get_lazy_frame() - timeseries_data = calculator.calculate_timeseries_metrics( - st.session_state.events_data, results.steps, polars_period, lazy_df + # Use cached time series calculation to prevent recalculation during tab switching + config_dict = st.session_state.funnel_config.to_dict() + funnel_steps_tuple = tuple(results.steps) + + timeseries_data = calculate_timeseries_metrics_cached( + st.session_state.events_data, + funnel_steps_tuple, + polars_period, + config_dict, + use_polars ) if not timeseries_data.empty: @@ -2428,20 +2541,7 @@ def format_time(minutes): tab_idx += 1 - # Test visualizations button - if "analysis_results" in st.session_state and st.session_state.analysis_results: - st.markdown("---") - test_col1, test_col2, test_col3 = st.columns([1, 1, 1]) - with test_col2: - if st.button("🧪 Test Visualizations", use_container_width=True): - with st.spinner("Testing all visualizations..."): - test_results = test_visualizations() - - if test_results["success"]: - st.success("✅ All visualizations passed!") - else: - failed_tests = [name for name, _ in test_results["failed"]] - st.error(f"❌ Failed tests: {', '.join(failed_tests)}") + # Footer st.markdown("---") @@ -2456,250 +2556,8 @@ def format_time(minutes): ) -def test_visualizations(): - """ - Universal test function to verify all visualizations render correctly. - Can be run with: - 1. python app.py test_vis - to run in standalone mode with dummy data - 2. Called from within the app with actual data - """ - import numpy as np - import streamlit as st - - # Function to create minimal dummy data for testing - def create_dummy_data(): - # Minimal FunnelResults - class DummyFunnelResults: - def __init__(self): - self.steps = ["Step 1", "Step 2", "Step 3"] - self.users_count = [1000, 700, 400] - self.drop_offs = [0, 300, 300] - self.drop_off_rates = [0, 30.0, 42.9] - self.conversion_rates = [100.0, 70.0, 40.0] - self.segment_data = { - "Segment A": [600, 400, 250], - "Segment B": [400, 300, 150], - } - - # Minimal TimeToConvertStats - class DummyTimeStats: - def __init__(self, step_from, step_to): - self.step_from = step_from - self.step_to = step_to - self.conversion_times = np.random.exponential(scale=2.0, size=10) - self.mean_hours = np.mean(self.conversion_times) - self.median_hours = np.median(self.conversion_times) - self.p25_hours = np.percentile(self.conversion_times, 25) - self.p75_hours = np.percentile(self.conversion_times, 75) - self.p90_hours = np.percentile(self.conversion_times, 90) - self.std_hours = np.std(self.conversion_times) - - # Minimal CohortData - class DummyCohortData: - def __init__(self): - self.cohort_labels = ["Cohort 1", "Cohort 2"] - self.cohort_sizes = {"Cohort 1": 500, "Cohort 2": 400} - self.conversion_rates = { - "Cohort 1": [100.0, 75.0, 50.0], - "Cohort 2": [100.0, 70.0, 45.0], - } - - # Minimal PathAnalysisData - class DummyPathData: - def __init__(self): - self.dropoff_paths = { - "Step 1": {"Other Path 1": 150, "Other Path 2": 100}, - "Step 2": {"Other Path 3": 200, "Other Path 4": 100}, - } - self.between_steps_events = { - "Step 1 → Step 2": {"Event 1": 700}, - "Step 2 → Step 3": {"Event 2": 400}, - } - - # Minimal StatSignificanceResult - class DummyStatTest: - def __init__(self): - self.segment_a = "Segment A" - self.segment_b = "Segment B" - self.conversion_a = 40.0 - self.conversion_b = 25.0 - self.p_value = 0.03 - self.is_significant = True - self.z_score = 2.5 - self.confidence_interval = (0.05, 0.15) - - return { - "funnel_results": DummyFunnelResults(), - "time_stats": [ - DummyTimeStats("Step 1", "Step 2"), - DummyTimeStats("Step 2", "Step 3"), - ], - "cohort_data": DummyCohortData(), - "path_data": DummyPathData(), - "stat_tests": [DummyStatTest(), DummyStatTest()], - } - - # Function to get real data if available, otherwise use dummy data - def get_test_data(): - # Try to get real data from session state if exists - data = {} - - try: - # Check if we have session state and if we're in the Streamlit context - has_session = ( - "session_state" in globals() or "st" in globals() and hasattr(st, "session_state") - ) - - if has_session and hasattr(st.session_state, "analysis_results"): - results = st.session_state.analysis_results - if results: - data["funnel_results"] = results - if hasattr(results, "time_to_convert"): - data["time_stats"] = results.time_to_convert - if hasattr(results, "cohort_data"): - data["cohort_data"] = results.cohort_data - if hasattr(results, "path_analysis"): - data["path_data"] = results.path_analysis - if hasattr(results, "stat_significance"): - data["stat_tests"] = results.stat_significance - except Exception: - pass # If we can't access session state or it's not properly initialized - - # For any missing data, fill with dummy data - dummy_data = create_dummy_data() - for key in dummy_data: - if key not in data or not data[key]: - data[key] = dummy_data[key] - - return data - - # Track test results - test_results = {"passed": [], "failed": []} - - # Get test data (real or dummy) - data = get_test_data() - - # Set up Streamlit page - st.title("Visualization Tests") - st.markdown( - "This test page verifies that all visualizations render correctly with dark theme." - ) - - # Run tests for each visualization - with st.expander("Test Details", expanded=True): - # Test 1: Funnel Chart - try: - funnel_chart = FunnelVisualizer.create_funnel_chart(data["funnel_results"]) - test_results["passed"].append("Funnel Chart") - st.success("✅ Funnel Chart") - except Exception as e: - test_results["failed"].append(("Funnel Chart", str(e))) - st.error(f"❌ Funnel Chart: {str(e)}") - - # Test 2: Segmented Funnel Chart - try: - segmented_funnel = FunnelVisualizer.create_funnel_chart( - data["funnel_results"], show_segments=True - ) - test_results["passed"].append("Segmented Funnel") - st.success("✅ Segmented Funnel") - except Exception as e: - test_results["failed"].append(("Segmented Funnel", str(e))) - st.error(f"❌ Segmented Funnel: {str(e)}") - - # Test 3: Conversion Flow Sankey - try: - flow_chart = FunnelVisualizer.create_conversion_flow_sankey(data["funnel_results"]) - test_results["passed"].append("Conversion Flow Sankey") - st.success("✅ Conversion Flow Sankey") - except Exception as e: - test_results["failed"].append(("Conversion Flow Sankey", str(e))) - st.error(f"❌ Conversion Flow Sankey: {str(e)}") - - # Test 4: Time to Convert Chart - try: - time_chart = FunnelVisualizer.create_time_to_convert_chart(data["time_stats"]) - test_results["passed"].append("Time to Convert Chart") - st.success("✅ Time to Convert Chart") - except Exception as e: - test_results["failed"].append(("Time to Convert Chart", str(e))) - st.error(f"❌ Time to Convert Chart: {str(e)}") - - # Test 5: Cohort Heatmap - try: - cohort_chart = FunnelVisualizer.create_cohort_heatmap(data["cohort_data"]) - test_results["passed"].append("Cohort Heatmap") - st.success("✅ Cohort Heatmap") - except Exception as e: - test_results["failed"].append(("Cohort Heatmap", str(e))) - st.error(f"❌ Cohort Heatmap: {str(e)}") - - # Test 6: Path Analysis Chart - try: - path_chart = FunnelVisualizer.create_path_analysis_chart(data["path_data"]) - test_results["passed"].append("Path Analysis Chart") - st.success("✅ Path Analysis Chart") - except Exception as e: - test_results["failed"].append(("Path Analysis Chart", str(e))) - st.error(f"❌ Path Analysis Chart: {str(e)}") - - # Test 7: Statistical Significance Table - try: - stat_table = FunnelVisualizer.create_statistical_significance_table(data["stat_tests"]) - test_results["passed"].append("Statistical Significance Table") - st.success("✅ Statistical Significance Table") - except Exception as e: - test_results["failed"].append(("Statistical Significance Table", str(e))) - st.error(f"❌ Statistical Significance Table: {str(e)}") - - # Show overall test result - if not test_results["failed"]: - st.success(f"✅ All {len(test_results['passed'])} visualizations passed!") - else: - st.error( - f"❌ {len(test_results['failed'])} of {len(test_results['passed']) + len(test_results['failed'])} tests failed." - ) - - # Show successful visualizations - if test_results["passed"]: - st.subheader("Successful Visualizations") - - # Display the charts that passed - for viz_name in test_results["passed"]: - if viz_name == "Funnel Chart": - st.subheader("1. Funnel Chart") - st.plotly_chart(funnel_chart, use_container_width=True) - elif viz_name == "Segmented Funnel": - st.subheader("2. Segmented Funnel Chart") - st.plotly_chart(segmented_funnel, use_container_width=True) - elif viz_name == "Conversion Flow Sankey": - st.subheader("3. Conversion Flow Sankey") - st.plotly_chart(flow_chart, use_container_width=True) - elif viz_name == "Time to Convert Chart": - st.subheader("4. Time to Convert Chart") - st.plotly_chart(time_chart, use_container_width=True) - elif viz_name == "Cohort Heatmap": - st.subheader("5. Cohort Heatmap") - st.plotly_chart(cohort_chart, use_container_width=True) - elif viz_name == "Path Analysis Chart": - st.subheader("6. Path Analysis Chart") - st.plotly_chart(path_chart, use_container_width=True) - elif viz_name == "Statistical Significance Table": - st.subheader("7. Statistical Significance Table") - st.dataframe(stat_table) - - return { - "success": len(test_results["failed"]) == 0, - "passed": test_results["passed"], - "failed": test_results["failed"], - } if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "test_vis": - test_visualizations() - else: - main() + main() diff --git a/core/calculator.py b/core/calculator.py index 097b023..473eaff 100644 --- a/core/calculator.py +++ b/core/calculator.py @@ -1006,7 +1006,7 @@ def calculate_funnel_metrics( Returns: FunnelResults object with all calculated metrics """ - start_time = time.time() + time.time() if len(funnel_steps) < 2: return FunnelResults( @@ -1088,7 +1088,7 @@ def _calculate_funnel_metrics_polars( existing_events_in_data = set( polars_df.select("event_name").unique().to_series().to_list() ) - funnel_steps_in_data = set(funnel_steps) & existing_events_in_data + set(funnel_steps) & existing_events_in_data zero_counts = [0] * len(funnel_steps) drop_offs = [0] * len(funnel_steps) @@ -1167,7 +1167,6 @@ def _calculate_funnel_metrics_polars( # Ensure consistent data types between DataFrames # Get the schema of segment_polars_df - segment_schema = segment_polars_df.schema # Cast user_id in both DataFrames to string to ensure consistent types segment_polars_df = segment_polars_df.with_columns( @@ -1272,7 +1271,7 @@ def _calculate_funnel_metrics_pandas( return FunnelResults([], [], [], [], []) # Events exist but none match funnel steps - check if any of the funnel steps exist in the data at all existing_events_in_data = set(events_df["event_name"].unique()) - funnel_steps_in_data = set(funnel_steps) & existing_events_in_data + set(funnel_steps) & existing_events_in_data zero_counts = [0] * len(funnel_steps) drop_offs = [0] * len(funnel_steps) @@ -2013,7 +2012,7 @@ def _calculate_timeseries_metrics_pandas( # Define first and last steps first_step = funnel_steps[0] - last_step = funnel_steps[-1] + funnel_steps[-1] conversion_window_hours = self.config.conversion_window_hours try: @@ -3250,7 +3249,7 @@ def _calculate_path_analysis_polars_optimized( # Fully vectorized approach for between-steps analysis try: # Get unique user IDs with valid conversion pairs - user_ids = conversion_pairs.select("user_id").unique() + conversion_pairs.select("user_id").unique() # Create a lazy frame with step pairs information step_pairs_lazy = conversion_pairs.lazy().select( @@ -3785,7 +3784,7 @@ def _analyze_between_steps_polars( continue step_A_time = user_A[0, "step_A_time"] - conversion_window = timedelta(hours=self.config.conversion_window_hours) + timedelta(hours=self.config.conversion_window_hours) # Find first B after A within conversion window potential_Bs = user_B.filter( @@ -4576,7 +4575,7 @@ def _calculate_unique_pairs_funnel_polars( users_count.append(count) # For unique pairs, conversion rate is step-to-step - step_conversion_rate = ( + ( (count / len(prev_step_users) * 100) if len(prev_step_users) > 0 else 0 ) # But we also track overall conversion rate from first step for consistency @@ -5694,7 +5693,7 @@ def _user_did_later_steps_before_current( try: # Get the funnel sequence from the order that steps appear in the overall dataset # This is a heuristic but works for most cases - all_funnel_events = all_events_df["event_name"].unique() + all_events_df["event_name"].unique() # For the test case, we know the sequence should be: Sign Up -> Email Verification -> First Login # When checking Email Verification after Sign Up, we should see if First Login happened before Email Verification @@ -5888,7 +5887,7 @@ def _calculate_unique_pairs_funnel( users_count.append(count) # For unique pairs, conversion rate is step-to-step - step_conversion_rate = ( + ( (count / len(prev_step_users) * 100) if len(prev_step_users) > 0 else 0 ) # But we also track overall conversion rate from first step @@ -5929,7 +5928,7 @@ def _user_did_later_steps_before_current_polars( try: # Find the indices of steps in the funnel try: - prev_step_idx = funnel_steps.index(prev_step) + funnel_steps.index(prev_step) current_step_idx = funnel_steps.index(current_step) except ValueError: # If the steps aren't in the funnel, we can't determine order diff --git a/core/data_source.py b/core/data_source.py index 971abeb..4f993d6 100644 --- a/core/data_source.py +++ b/core/data_source.py @@ -322,7 +322,7 @@ def _load_from_file_pandas_fallback(self, uploaded_file) -> pd.DataFrame: return df - except Exception as e: + except Exception: return pd.DataFrame() def connect_clickhouse( @@ -338,7 +338,7 @@ def connect_clickhouse( database=database, ) # Test connection - result = self.clickhouse_client.query("SELECT 1") + self.clickhouse_client.query("SELECT 1") return True except Exception as e: st.error(f"ClickHouse connection failed: {str(e)}") diff --git a/quick_benchmark.py b/quick_benchmark.py index dc849f7..9977708 100644 --- a/quick_benchmark.py +++ b/quick_benchmark.py @@ -106,7 +106,7 @@ def quick_benchmark(): # Test statistics (optimized) print(" Testing _calculate_process_statistics_optimized...") start_time = time.time() - statistics = analyzer._calculate_process_statistics_optimized( + analyzer._calculate_process_statistics_optimized( journey_df, activities, transitions ) stats_time = time.time() - start_time @@ -141,7 +141,7 @@ def quick_benchmark(): print(" Without cycles:") start_time = time.time() - result_no_cycles = analyzer.discover_process_mining_structure( + analyzer.discover_process_mining_structure( df, min_frequency=1, include_cycles=False ) time_no_cycles = time.time() - start_time @@ -151,7 +151,7 @@ def quick_benchmark(): print(" With cycles:") start_time = time.time() - result_with_cycles = analyzer.discover_process_mining_structure( + analyzer.discover_process_mining_structure( df, min_frequency=1, include_cycles=True ) time_with_cycles = time.time() - start_time diff --git a/run_tests.py b/run_tests.py index 04b920c..f40f85d 100644 --- a/run_tests.py +++ b/run_tests.py @@ -405,12 +405,10 @@ def generate_test_report() -> TestResult: stdout = result.stdout stderr = result.stderr # For pytest, exit code 1 with fixture errors is acceptable - success = result.returncode in [0, 1] # Accept both success and fixture errors except Exception as e: print(f"❌ Error running pytest: {e}") stdout = "" stderr = str(e) - success = False # Parse the output to extract test statistics passed = 0 @@ -428,10 +426,9 @@ def generate_test_report() -> TestResult: summary_matches = re.findall(summary_pattern, combined_output) # Look for the final summary line (usually the last one) - main_summary = "" for match in summary_matches: if any(word in match.lower() for word in ["passed", "failed", "error", "skipped"]): - main_summary = match + pass # Extract individual numbers using separate patterns from the entire output passed_match = re.search(r"(\d+) passed", combined_output) diff --git a/scalability_test.py b/scalability_test.py index b4c9ff6..6f61636 100644 --- a/scalability_test.py +++ b/scalability_test.py @@ -125,7 +125,7 @@ def scalability_test(): # Test without cycles start_time = time.time() - result_no_cycles = analyzer.discover_process_mining_structure( + analyzer.discover_process_mining_structure( df, min_frequency=10, include_cycles=False ) time_no_cycles = time.time() - start_time @@ -133,7 +133,7 @@ def scalability_test(): # Test with cycles start_time = time.time() - result_with_cycles = analyzer.discover_process_mining_structure( + analyzer.discover_process_mining_structure( df, min_frequency=10, include_cycles=True ) time_with_cycles = time.time() - start_time diff --git a/tests/test_app_ui.py b/tests/test_app_ui.py index cda7477..e38d887 100644 --- a/tests/test_app_ui.py +++ b/tests/test_app_ui.py @@ -57,7 +57,7 @@ def load_sample_data(self) -> None: if attempt < max_retries - 1: # Don't run on last attempt self.at.run(timeout=10) # Re-run to refresh state - except Exception as e: + except Exception: # If button interaction fails, manually load sample data for testing from datetime import datetime, timedelta @@ -151,7 +151,7 @@ def build_funnel(self, steps: List[str]) -> None: try: # Check the checkbox for this event self.at.checkbox(key=checkbox_key).check().run() - except Exception as e: + except Exception: # If checkbox interaction fails, manually add to session state for testing if step not in self.at.session_state.funnel_steps: self.at.session_state.funnel_steps.append(step) @@ -195,7 +195,7 @@ def analyze_funnel(self) -> None: except KeyError: # If analyze button not available, skip this test (button might not be rendered yet) pytest.skip("Analyze button not available - UI might not be fully rendered") - except Exception as e: + except Exception: # If analysis fails, create mock results for testing UI flow from models import FunnelResults diff --git a/tests/test_conversion_logic_debug.py b/tests/test_conversion_logic_debug.py index 0e9df4b..8a7fcfa 100644 --- a/tests/test_conversion_logic_debug.py +++ b/tests/test_conversion_logic_debug.py @@ -65,7 +65,7 @@ def test_debug_conversion_rate_calculation(self): overall_started = overall_results.users_count[0] overall_completed = overall_results.users_count[1] - overall_conversion = overall_results.conversion_rates[1] + overall_results.conversion_rates[1] print( f" Manual verification: {overall_completed}/{overall_started} = {(overall_completed / overall_started * 100):.2f}%" diff --git a/tests/test_data_source_advanced.py b/tests/test_data_source_advanced.py index 0a18dc0..e6ef291 100644 --- a/tests/test_data_source_advanced.py +++ b/tests/test_data_source_advanced.py @@ -426,10 +426,6 @@ def test_load_unsupported_file_format(self, data_manager): def test_load_corrupted_csv_file(self, data_manager): """Test loading corrupted CSV file.""" - corrupted_csv = """user_id,event_name,timestamp -user_001,Sign Up,2024-01-01 -user_002,"Incomplete quote -user_003,Login,""" # Corrupted CSV # Create mock file mock_file = Mock() diff --git a/tests/test_integration_flow.py b/tests/test_integration_flow.py index ab4c9e5..fb5a1e4 100644 --- a/tests/test_integration_flow.py +++ b/tests/test_integration_flow.py @@ -323,7 +323,7 @@ def test_all_counting_methods_integration(self, integration_test_data): "First Login", "First Purchase", ] - base_config = FunnelConfig(conversion_window_hours=24, reentry_mode=ReentryMode.FIRST_ONLY) + FunnelConfig(conversion_window_hours=24, reentry_mode=ReentryMode.FIRST_ONLY) results = {} @@ -684,7 +684,7 @@ def test_performance_integration(self, base_timestamp): # Visualization visualizer = FunnelVisualizer() - chart = visualizer.create_funnel_chart(results) + visualizer.create_funnel_chart(results) end_time = time.time() total_time = end_time - start_time diff --git a/tests/test_polars_fallback_detection.py b/tests/test_polars_fallback_detection.py index 42eb71f..3eeacc2 100644 --- a/tests/test_polars_fallback_detection.py +++ b/tests/test_polars_fallback_detection.py @@ -286,7 +286,7 @@ def test_problematic_lazy_frame_path_analysis(self, create_lazy_frame, log_captu # Convert to regular Polars DataFrame and then to Pandas polars_df = lazy_df.collect() - pandas_df = polars_df.to_pandas() + polars_df.to_pandas() # Setup steps = ["Step1", "Step2", "Step3"] diff --git a/tests/test_polars_path_analysis.py b/tests/test_polars_path_analysis.py index 91b445a..8dad77f 100644 --- a/tests/test_polars_path_analysis.py +++ b/tests/test_polars_path_analysis.py @@ -691,7 +691,7 @@ def track_call(name): calculator = FunnelCalculator(config, use_polars=True) # Calculate funnel metrics - results = calculator.calculate_funnel_metrics(test_df, funnel_steps) + calculator.calculate_funnel_metrics(test_df, funnel_steps) # Check if we've fallen back to Pandas pandas_used = "_calculate_funnel_metrics_pandas" in call_sequence @@ -797,7 +797,7 @@ def test_conversion_window_edge_cases(): try: # Test with Polars implementation directly calculator = FunnelCalculator(config, use_polars=True) - results = calculator.calculate_funnel_metrics(test_df, funnel_steps) + calculator.calculate_funnel_metrics(test_df, funnel_steps) print(f"✅ Polars implementation succeeded with {window_str} window") except Exception as e: diff --git a/tests/test_process_mining_comprehensive.py b/tests/test_process_mining_comprehensive.py index 6ff0735..fd1ff33 100644 --- a/tests/test_process_mining_comprehensive.py +++ b/tests/test_process_mining_comprehensive.py @@ -263,7 +263,7 @@ def test_completion_rate_insights(self, insight_data): ) # Should generate completion rate insights - completion_insights = [ + [ insight for insight in process_data.insights if "completion" in insight.lower() or "complete" in insight.lower() diff --git a/tests/test_realistic_discrepancy.py b/tests/test_realistic_discrepancy.py index dc0eaa6..89c18bd 100644 --- a/tests/test_realistic_discrepancy.py +++ b/tests/test_realistic_discrepancy.py @@ -235,7 +235,6 @@ def test_cross_period_conversion_issue(self, problematic_data, calculator): Test the specific issue where users start in one period but convert in another. This is likely the root cause of the discrepancy. """ - steps = ["step1", "step2"] print("\n=== CROSS-PERIOD CONVERSION ANALYSIS ===") diff --git a/tests/test_timeseries_analysis.py b/tests/test_timeseries_analysis.py index 7db458e..edf4c2e 100644 --- a/tests/test_timeseries_analysis.py +++ b/tests/test_timeseries_analysis.py @@ -400,7 +400,7 @@ def test_timeseries_performance_monitoring(self): calculator = FunnelCalculator(config) # Performance metrics should be empty initially - initial_metrics = calculator.get_performance_report() + calculator.get_performance_report() # Create minimal test data and run calculation events_df = pd.DataFrame( diff --git a/ui/visualization/visualizer.py b/ui/visualization/visualizer.py index 39a52e8..ab88511 100644 --- a/ui/visualization/visualizer.py +++ b/ui/visualization/visualizer.py @@ -1557,7 +1557,7 @@ def create_enhanced_path_analysis_chart(self, path_data: PathAnalysisData) -> go node_categories[len(labels) - 1] = "funnel_step" # Process conversion and drop-off flows with enhanced categorization - node_index = len(funnel_steps) + len(funnel_steps) # Create a color map for consistent coloring across all datasets semantic_colors = {