data_handler.py

import pandas as pd
import numpy as np
import hashlib
from datetime import datetime
from collections import defaultdict
from pathlib import Path


class PowerballDataHandler:
    """Handles data loading, preparation, and feature engineering for Powerball predictions"""
    
    def __init__(self, csv_path):
        self.csv_path = csv_path
        self.data = pd.read_csv(csv_path)
        self.prepare_data()
        
    def prepare_data(self):
        """Prepare and engineer features from raw data"""
        # Now that CSV headers are corrected, we can use the proper column mapping
        corrected_data = self.data.copy()
        
        # Create proper date from the actual Month, Day, Year columns
        corrected_data['Date'] = pd.to_datetime(
            corrected_data[['Year', 'Month', 'Day']],
            errors='coerce'
        )
        
        # Map the lottery numbers correctly with the fixed headers:
        # Num 1-5 are the white balls (1-69 range)
        # Powerball is the red powerball (1-26 range)
        # PowerPlay is the multiplier
        
        corrected_data['White_1'] = corrected_data['Num 1']     # 1st white ball
        corrected_data['White_2'] = corrected_data['Num 2']     # 2nd white ball  
        corrected_data['White_3'] = corrected_data['Num 3']     # 3rd white ball
        corrected_data['White_4'] = corrected_data['Num 4']     # 4th white ball
        corrected_data['White_5'] = corrected_data['Num 5']     # 5th white ball
        corrected_data['Powerball'] = corrected_data['Powerball']   # Powerball (1-26 range)
        
        # Set up for the algorithm
        self.white_cols = ['White_1', 'White_2', 'White_3', 'White_4', 'White_5']
        self.whites = corrected_data[self.white_cols].values
        self.powerballs = corrected_data['Powerball'].values
        
        # Sort by date (most recent first) for time series analysis  
        corrected_data = corrected_data.sort_values('Date', ascending=False).reset_index(drop=True)
        self.data = corrected_data
        
    def calculate_data_hash(self):
        """Calculate hash of data for cache validation"""
        try:
            sorted_data = self.data.sort_values(['Year', 'Month', 'Day']).reset_index(drop=True)
            data_string = ''.join(str(row.values) for _, row in sorted_data.iterrows())
            return hashlib.md5(data_string.encode()).hexdigest()
        except Exception:
            return hashlib.md5(str(datetime.now().timestamp()).encode()).hexdigest()

    def engineer_features(self, lookback_draws=200, adaptive_window=True):
        """Engineer comprehensive features from historical data"""
        if len(self.data) < lookback_draws:
            lookback_draws = len(self.data)
            
        recent_data = self.data.head(lookback_draws).copy()
        features = {}
        
        # 1. Basic frequency analysis for each position
        for i, col in enumerate(self.white_cols):
            freq = recent_data[col].value_counts().to_dict()
            features[f'pos_{i+1}_frequencies'] = freq
            
        # Powerball frequencies
        pb_freq = recent_data['Powerball'].value_counts().to_dict()
        features['powerball_frequencies'] = pb_freq
        
        # 2. Hot/Cold analysis (recent vs historical performance)
        hot_window = min(30, len(recent_data) // 4)
        cold_window = min(100, len(recent_data))
        
        features['hot_numbers'] = {}
        features['cold_numbers'] = {}
        
        for i, col in enumerate(self.white_cols):
            hot_freq = recent_data.head(hot_window)[col].value_counts().to_dict()
            cold_freq = recent_data.tail(cold_window)[col].value_counts().to_dict()
            
            features['hot_numbers'][f'pos_{i+1}'] = hot_freq
            features['cold_numbers'][f'pos_{i+1}'] = cold_freq
            
        # Hot/Cold powerball
        hot_pb_freq = recent_data.head(hot_window)['Powerball'].value_counts().to_dict()
        cold_pb_freq = recent_data.tail(cold_window)['Powerball'].value_counts().to_dict()
        features['hot_numbers']['powerball'] = hot_pb_freq
        features['cold_numbers']['powerball'] = cold_pb_freq
        
        # 3. Gap analysis (draws since last appearance)
        features['gaps'] = {}
        for i, col in enumerate(self.white_cols):
            gaps = {}
            for num in range(1, 70):  # White balls 1-69
                last_seen = None
                for idx, row in recent_data.iterrows():
                    if row[col] == num:
                        last_seen = idx
                        break
                gaps[num] = last_seen if last_seen is not None else len(recent_data)
            features['gaps'][f'pos_{i+1}'] = gaps
            
        # Powerball gaps
        pb_gaps = {}
        for num in range(1, 27):  # Powerball 1-26
            last_seen = None
            for idx, row in recent_data.iterrows():
                if row['Powerball'] == num:
                    last_seen = idx
                    break
            pb_gaps[num] = last_seen if last_seen is not None else len(recent_data)
        features['gaps']['powerball'] = pb_gaps
        
        # 4. Pattern analysis
        features['patterns'] = self._analyze_patterns(recent_data)
        
        # 5. Temporal features
        features['temporal'] = self._analyze_temporal_patterns(recent_data)
        
        # 6. Statistical features
        features['stats'] = self._calculate_statistical_features(recent_data)
        
        return features
    
    def _analyze_patterns(self, data):
        """Analyze number patterns and relationships"""
        patterns = {
            'consecutive_pairs': defaultdict(int),
            'sum_ranges': defaultdict(int),
            'odd_even_patterns': defaultdict(int),
            'high_low_patterns': defaultdict(int)
        }
        
        for _, row in data.iterrows():
            whites = sorted([row[col] for col in self.white_cols])
            
            # Consecutive pairs
            for i in range(len(whites) - 1):
                if whites[i+1] - whites[i] == 1:
                    patterns['consecutive_pairs'][f"{whites[i]}-{whites[i+1]}"] += 1
            
            # Sum ranges
            total_sum = sum(whites)
            if total_sum <= 150:
                patterns['sum_ranges']['low'] += 1
            elif total_sum <= 210:
                patterns['sum_ranges']['medium'] += 1
            else:
                patterns['sum_ranges']['high'] += 1
            
            # Odd/Even patterns
            odd_count = sum(1 for w in whites if w % 2 == 1)
            patterns['odd_even_patterns'][f"{odd_count}_odd"] += 1
            
            # High/Low patterns (>34 is high)
            high_count = sum(1 for w in whites if w > 34)
            patterns['high_low_patterns'][f"{high_count}_high"] += 1
            
        return patterns
    
    def _analyze_temporal_patterns(self, data):
        """Analyze temporal patterns in the data"""
        temporal = {
            'weekday_patterns': defaultdict(lambda: defaultdict(int)),
            'month_patterns': defaultdict(lambda: defaultdict(int)),
            'seasonal_trends': defaultdict(list)
        }
        
        for _, row in data.iterrows():
            date = row['Date']
            if pd.isna(date):
                continue
                
            weekday = date.weekday()
            month = date.month
            
            # Analyze patterns by weekday and month
            for i, col in enumerate(self.white_cols):
                temporal['weekday_patterns'][f'pos_{i+1}'][weekday] += 1
                temporal['month_patterns'][f'pos_{i+1}'][month] += 1
                temporal['seasonal_trends'][f'pos_{i+1}'].append(row[col])
            
            temporal['weekday_patterns']['powerball'][weekday] += 1
            temporal['month_patterns']['powerball'][month] += 1
            temporal['seasonal_trends']['powerball'].append(row['Powerball'])
            
        return temporal
    
    def _calculate_statistical_features(self, data):
        """Calculate various statistical features"""
        stats = {}
        
        # Basic statistics for each position
        for i, col in enumerate(self.white_cols):
            values = data[col].values
            stats[f'pos_{i+1}'] = {
                'mean': np.mean(values),
                'std': np.std(values),
                'median': np.median(values),
                'mode': data[col].mode().iloc[0] if not data[col].mode().empty else 0,
                'skewness': data[col].skew(),
                'kurtosis': data[col].kurtosis()
            }
        
        # Powerball statistics
        pb_values = data['Powerball'].values
        stats['powerball'] = {
            'mean': np.mean(pb_values),
            'std': np.std(pb_values),
            'median': np.median(pb_values),
            'mode': data['Powerball'].mode().iloc[0] if not data['Powerball'].mode().empty else 0,
            'skewness': data['Powerball'].skew(),
            'kurtosis': data['Powerball'].kurtosis()
        }
        
        # Cross-position correlations
        white_data = data[self.white_cols]
        stats['correlations'] = white_data.corr().to_dict()
        
        return stats