models.py

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import TimeSeriesSplit, cross_val_score
from sklearn.linear_model import Ridge, ElasticNet
from sklearn.svm import SVR
import xgboost as xgb
import lightgbm as lgb
from scipy.stats import poisson, gamma
from scipy.special import softmax
import warnings
warnings.filterwarnings('ignore')


class PowerballLSTM(nn.Module):
    """LSTM model for sequence prediction"""
    
    def __init__(self, input_size, hidden_size=128, num_layers=2, output_size=1, dropout=0.2):
        super(PowerballLSTM, self).__init__()
        
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.input_size = input_size
        
        self.lstm = nn.LSTM(
            input_size=input_size,
            hidden_size=hidden_size,
            num_layers=num_layers,
            dropout=dropout if num_layers > 1 else 0,
            batch_first=True
        )
        
        self.dropout = nn.Dropout(dropout)
        self.fc = nn.Linear(hidden_size, output_size)
        
        # Initialize weights
        for name, param in self.lstm.named_parameters():
            if 'bias' in name:
                nn.init.constant_(param, 0.0)
            elif 'weight' in name:
                nn.init.xavier_normal_(param)
    
    def forward(self, x):
        batch_size = x.size(0)
        
        # Initialize hidden state
        h0 = torch.zeros(self.num_layers, batch_size, self.hidden_size)
        c0 = torch.zeros(self.num_layers, batch_size, self.hidden_size)
        
        # Forward propagate LSTM
        lstm_out, _ = self.lstm(x, (h0, c0))
        
        # Take the last output
        lstm_out = lstm_out[:, -1, :]
        
        # Apply dropout
        lstm_out = self.dropout(lstm_out)
        
        # Final layer
        output = self.fc(lstm_out)
        
        return output


class AdvancedPowerballLSTM(nn.Module):
    """Advanced LSTM with attention mechanism for Powerball prediction"""
    
    def __init__(self, sequence_length=20, feature_size=10, max_number=69):
        super(AdvancedPowerballLSTM, self).__init__()
        
        self.sequence_length = sequence_length
        self.feature_size = feature_size
        self.max_number = max_number
        
        # Embedding layer for numbers
        self.embedding = nn.Embedding(max_number + 1, 32)
        
        # LSTM layers
        self.lstm1 = nn.LSTM(32 * 5 + feature_size, 256, batch_first=True, dropout=0.3)
        self.lstm2 = nn.LSTM(256, 128, batch_first=True, dropout=0.3)
        
        # Attention mechanism
        self.attention = nn.MultiheadAttention(128, num_heads=8, batch_first=True)
        
        # Output layers for each position and powerball
        self.white_outputs = nn.ModuleList([
            nn.Sequential(
                nn.Linear(128, 64),
                nn.ReLU(),
                nn.Dropout(0.2),
                nn.Linear(64, max_number)
            ) for _ in range(5)
        ])
        
        self.powerball_output = nn.Sequential(
            nn.Linear(128, 32),
            nn.ReLU(),
            nn.Dropout(0.2),
            nn.Linear(32, 26)
        )
    
    def forward(self, number_sequences, features):
        batch_size = number_sequences.size(0)
        
        # Embed number sequences
        embedded = self.embedding(number_sequences)
        embedded = embedded.view(batch_size, self.sequence_length, -1)
        
        # Concatenate with features
        combined = torch.cat([embedded, features], dim=-1)
        
        # LSTM processing
        lstm_out1, _ = self.lstm1(combined)
        lstm_out2, _ = self.lstm2(lstm_out1)
        
        # Self-attention
        attended, _ = self.attention(lstm_out2, lstm_out2, lstm_out2)
        
        # Take the last time step
        final_hidden = attended[:, -1, :]
        
        # Generate predictions for each position
        white_preds = [output(final_hidden) for output in self.white_outputs]
        powerball_pred = self.powerball_output(final_hidden)
        
        return white_preds, powerball_pred


class EnsembleModelBuilder:
    """Builds and manages ensemble models for prediction"""
    
    def __init__(self, random_state=42):
        self.random_state = random_state
        self.models = {}
        self.scalers = {}
        
    def build_ensemble_models(self, features, target_type='white'):
        """Build ensemble of models for prediction"""
        models = {}
        
        # Prepare feature matrix
        if isinstance(features, dict):
            feature_matrix = self._dict_to_matrix(features, target_type)
        else:
            feature_matrix = features
            
        if len(feature_matrix) == 0:
            print("No features available for model training")
            return models
        
        # Scale features
        scaler = StandardScaler()
        X_scaled = scaler.fit_transform(feature_matrix)
        self.scalers[target_type] = scaler
        
        # Random Forest
        rf_params = {
            'n_estimators': 200,
            'max_depth': 15,
            'min_samples_split': 5,
            'min_samples_leaf': 2,
            'random_state': self.random_state,
            'n_jobs': -1
        }
        models['random_forest'] = RandomForestRegressor(**rf_params)
        
        # XGBoost
        xgb_params = {
            'n_estimators': 150,
            'max_depth': 8,
            'learning_rate': 0.1,
            'subsample': 0.8,
            'colsample_bytree': 0.8,
            'random_state': self.random_state,
            'n_jobs': -1
        }
        models['xgboost'] = xgb.XGBRegressor(**xgb_params)
        
        # LightGBM
        lgb_params = {
            'n_estimators': 150,
            'max_depth': 10,
            'learning_rate': 0.1,
            'subsample': 0.8,
            'colsample_bytree': 0.8,
            'random_state': self.random_state,
            'n_jobs': -1,
            'verbose': -1
        }
        models['lightgbm'] = lgb.LGBMRegressor(**lgb_params)
        
        # Gradient Boosting
        gb_params = {
            'n_estimators': 100,
            'max_depth': 8,
            'learning_rate': 0.1,
            'subsample': 0.8,
            'random_state': self.random_state
        }
        models['gradient_boosting'] = GradientBoostingRegressor(**gb_params)
        
        # Extra Trees
        et_params = {
            'n_estimators': 100,
            'max_depth': 15,
            'min_samples_split': 5,
            'min_samples_leaf': 2,
            'random_state': self.random_state,
            'n_jobs': -1
        }
        models['extra_trees'] = ExtraTreesRegressor(**et_params)
        
        # Neural Network
        mlp_params = {
            'hidden_layer_sizes': (200, 100, 50),
            'activation': 'relu',
            'solver': 'adam',
            'alpha': 0.001,
            'batch_size': 'auto',
            'learning_rate': 'adaptive',
            'max_iter': 500,
            'random_state': self.random_state,
            'early_stopping': True,
            'validation_fraction': 0.1,
            'n_iter_no_change': 20
        }
        models['neural_network'] = MLPRegressor(**mlp_params)
        
        # Support Vector Regression
        svr_params = {
            'kernel': 'rbf',
            'C': 100,
            'gamma': 'scale',
            'epsilon': 0.1
        }
        models['svr'] = SVR(**svr_params)
        
        # Ridge Regression
        ridge_params = {
            'alpha': 1.0,
            'random_state': self.random_state
        }
        models['ridge'] = Ridge(**ridge_params)
        
        self.models[target_type] = models
        return models
    
    def _dict_to_matrix(self, features_dict, target_type):
        """Convert features dictionary to matrix format"""
        feature_matrix = []
        
        # Extract relevant features based on target type
        if target_type == 'white':
            # Use white ball specific features
            for pos in range(1, 6):
                pos_key = f'pos_{pos}'
                if pos_key in features_dict.get('stats', {}):
                    stats = features_dict['stats'][pos_key]
                    feature_row = [
                        stats.get('mean', 0),
                        stats.get('std', 0),
                        stats.get('median', 0),
                        stats.get('skewness', 0),
                        stats.get('kurtosis', 0)
                    ]
                    feature_matrix.append(feature_row)
        
        elif target_type == 'powerball':
            # Use powerball specific features
            if 'powerball' in features_dict.get('stats', {}):
                stats = features_dict['stats']['powerball']
                feature_row = [
                    stats.get('mean', 0),
                    stats.get('std', 0),
                    stats.get('median', 0),
                    stats.get('skewness', 0),
                    stats.get('kurtosis', 0)
                ]
                feature_matrix.append(feature_row)
        
        return np.array(feature_matrix) if feature_matrix else np.array([]).reshape(0, 5)


class LSTMTrainer:
    """Handles LSTM model training and prediction for Powerball numbers"""
    
    def __init__(self, data_handler, random_state=42):
        self.data_handler = data_handler
        self.random_state = random_state
        self.lstm_models = {}
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        
    def prepare_lstm_data(self, sequence_length=20):
        """Prepare data for LSTM training using the FULL dataset"""
        print(f"Preparing LSTM data from {len(self.data_handler.data)} total draws...")
        
        # Use ALL available data, not just 100 draws
        full_data = self.data_handler.data.sort_values('Date').reset_index(drop=True)
        
        sequences_white = []
        sequences_pb = []
        features_list = []
        targets_white = []
        targets_pb = []
        
        # Create sequences from the full dataset
        for i in range(len(full_data) - sequence_length):
            # Get sequence data
            seq_data = full_data.iloc[i:i+sequence_length]
            target_data = full_data.iloc[i+sequence_length]
            
            # Extract white ball sequences
            white_seq = []
            for _, row in seq_data.iterrows():
                white_seq.append([row['White_1'], row['White_2'], row['White_3'], 
                                row['White_4'], row['White_5']])
            
            # Extract powerball sequence
            pb_seq = seq_data['Powerball'].tolist()
            
            # Create features for each time step
            features_seq = []
            for _, row in seq_data.iterrows():
                white_numbers = [row['White_1'], row['White_2'], row['White_3'], 
                               row['White_4'], row['White_5']]
                features = [
                    np.mean(white_numbers),
                    np.std(white_numbers),
                    max(white_numbers),
                    min(white_numbers),
                    row['Powerball'],
                    sum(white_numbers),
                    len(set(white_numbers)),  # Unique count
                    sum(1 for x in white_numbers if x % 2 == 0),  # Even count
                    sum(1 for x in white_numbers if x <= 35),     # Low count
                    i / len(full_data)  # Temporal position
                ]
                features_seq.append(features)
            
            sequences_white.append(white_seq)
            sequences_pb.append(pb_seq)
            features_list.append(features_seq)
            
            # Targets
            targets_white.append([target_data['White_1'], target_data['White_2'], 
                                target_data['White_3'], target_data['White_4'], target_data['White_5']])
            targets_pb.append(target_data['Powerball'])
        
        print(f"Created {len(sequences_white)} training sequences from full dataset")
        
        return {
            'sequences_white': np.array(sequences_white),
            'sequences_pb': np.array(sequences_pb),
            'features': np.array(features_list),
            'targets_white': np.array(targets_white),
            'targets_pb': np.array(targets_pb)
        }
    
    def train_lstm_models(self, sequence_length=20, epochs=100, batch_size=32):
        """Train LSTM models using the full dataset"""
        print("=" * 50)
        print("TRAINING DEEP LEARNING MODELS ON FULL DATASET")
        print("=" * 50)
        
        lstm_data = self.prepare_lstm_data(sequence_length)
        print(f"Training on device: {self.device}")
        print(f"Training data shape: {lstm_data['sequences_white'].shape}")
        print(f"Total training sequences: {len(lstm_data['sequences_white'])}")
        
        self.lstm_models = {}
        
        # Train models for each white ball position
        for pos in range(5):
            print(f"\nTraining LSTM for White Ball Position {pos + 1}...")
            print("-" * 30)
            
            # Prepare position-specific data
            X_sequences = torch.FloatTensor(lstm_data['sequences_white'][:, :, pos]).unsqueeze(-1)
            X_features = torch.FloatTensor(lstm_data['features'])
            y = torch.LongTensor(lstm_data['targets_white'][:, pos] - 1)  # 0-indexed
            
            # Create model - use simpler LSTM for position-specific training
            model = PowerballLSTM(input_size=1, hidden_size=64, num_layers=2, output_size=69)
            model = model.to(self.device)
            
            # Loss and optimizer - optimized for better convergence
            criterion = nn.CrossEntropyLoss()
            optimizer = torch.optim.AdamW(model.parameters(), lr=0.002, weight_decay=0.01)
            # More aggressive learning rate scheduling for better loss improvement
            scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
                optimizer, mode='min', patience=15, factor=0.7, min_lr=1e-6
            )
            
            # Training loop
            model.train()
            best_loss = float('inf')
            patience_counter = 0
            
            print(f"Training for up to {epochs} epochs...")
            for epoch in range(epochs):
                total_loss = 0
                num_batches = 0
                
                # Shuffle data
                indices = torch.randperm(len(X_sequences))
                X_sequences_shuffled = X_sequences[indices]
                X_features_shuffled = X_features[indices]
                y_shuffled = y[indices]
                
                for i in range(0, len(X_sequences), batch_size):
                    batch_seq = X_sequences_shuffled[i:i+batch_size].to(self.device)
                    batch_features = X_features_shuffled[i:i+batch_size].to(self.device)
                    batch_targets = y_shuffled[i:i+batch_size].to(self.device)
                    
                    optimizer.zero_grad()
                    
                    # Forward pass
                    output = model(batch_seq)
                    loss = criterion(output, batch_targets)
                    
                    # Backward pass
                    loss.backward()
                    torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
                    optimizer.step()
                    
                    total_loss += loss.item()
                    num_batches += 1
                
                avg_loss = total_loss / num_batches
                scheduler.step(avg_loss)
                
                if avg_loss < best_loss:
                    best_loss = avg_loss
                    patience_counter = 0
                else:
                    patience_counter += 1
                
                # Enhanced progress reporting for longer training
                if epoch < 20 or epoch % 20 == 0:  # More frequent updates early, then every 20 epochs
                    improvement = f"(↓{best_loss - avg_loss:.4f})" if avg_loss < best_loss else ""
                    print(f"  Epoch {epoch:3d}, Loss: {avg_loss:.4f} {improvement}, LR: {optimizer.param_groups[0]['lr']:.6f}")
                
                # Progress milestones
                if epoch in [25, 50, 75] and epochs >= 100:
                    print(f"  * Training milestone: {epoch}/{epochs} epochs completed, best loss: {best_loss:.4f}")
                
                # Enhanced early stopping with more patience for longer training
                if patience_counter >= 25:  # Increased patience for 100+ epoch training
                    print(f"  Early stopping at epoch {epoch} (no improvement for 25 epochs)")
                    break
                
                # Additional convergence check - stop if loss gets very low
                if avg_loss < 1.0:  # Very good loss for this problem
                    print(f"  Excellent convergence achieved at epoch {epoch} (loss: {avg_loss:.4f})")
                    break
            
            model.eval()
            self.lstm_models[f'white_{pos + 1}'] = model
            print(f"  Final loss for position {pos + 1}: {best_loss:.4f}")
        
        print("\n" + "=" * 50)
        print("DEEP LEARNING MODEL TRAINING COMPLETED!")
        print("=" * 50)
    
    def generate_lstm_predictions(self, num_predictions=10, sequence_length=20, use_intelligent_sequences=True):
        """Generate predictions using trained LSTM models with intelligent sequence usage"""
        if not self.lstm_models:
            print("LSTM models not found. Training new models...")
            self.train_lstm_models(sequence_length=sequence_length)
        
        predictions = []
        
        if use_intelligent_sequences:
            # Use MULTIPLE sequences from different parts of the dataset, not just the most recent
            full_data = self.data_handler.data.sort_values('Date').reset_index(drop=True)
            
            # Define different starting points to sample from various time periods
            sequence_starts = [
                len(full_data) - sequence_length,  # Most recent
                len(full_data) - sequence_length * 2,  # Second most recent
                len(full_data) - sequence_length * 3,  # Third most recent
                len(full_data) // 2,  # Middle of dataset
                len(full_data) // 4,  # Quarter point
                len(full_data) // 3 * 2,  # Two-thirds point
            ]
            
            print(f"Generating {num_predictions} LSTM predictions using MULTIPLE sequences from different time periods...")
            
            predictions_per_sequence = max(1, num_predictions // len(sequence_starts))
            
            for seq_start_idx, seq_start in enumerate(sequence_starts):
                if seq_start < 0 or seq_start + sequence_length >= len(full_data):
                    continue
                    
                # Get data for this sequence
                sequence_data = full_data.iloc[seq_start:seq_start + sequence_length]
                
                # Prepare input sequences and features for this time period
                white_sequences = [[] for _ in range(5)]
                features_seq = []
                
                for _, row in sequence_data.iterrows():
                    white_numbers = [row['White_1'], row['White_2'], row['White_3'], 
                                   row['White_4'], row['White_5']]
                    
                    # Add to position-specific sequences
                    for pos in range(5):
                        white_sequences[pos].append(white_numbers[pos])
                    
                    # Create features
                    features = [
                        np.mean(white_numbers), np.std(white_numbers),
                        max(white_numbers), min(white_numbers), row['Powerball'],
                        sum(white_numbers), len(set(white_numbers)),
                        sum(1 for x in white_numbers if x % 2 == 0),
                        sum(1 for x in white_numbers if x <= 35),
                        len(features_seq) / sequence_length
                    ]
                    features_seq.append(features)
                
                # Generate predictions for this sequence
                for pred_idx in range(predictions_per_sequence):
                    white_prediction = []
                    
                    # Predict each white ball position
                    for pos in range(5):
                        model = self.lstm_models[f'white_{pos + 1}']
                        model = model.to(self.device)
                        model.eval()
                        
                        with torch.no_grad():
                            seq_tensor = torch.FloatTensor([white_sequences[pos]]).unsqueeze(-1).to(self.device)
                            
                            output = model(seq_tensor)
                            probabilities = torch.softmax(output, dim=1)
                            
                            # Use different sampling strategies
                            if pred_idx == 0 and seq_start_idx == 0:
                                # Best prediction from most recent sequence
                                predicted_num = torch.argmax(probabilities, dim=1).item() + 1
                            else:
                                # Sample with increasing temperature for diversity
                                temperature = 1.2 + (pred_idx * 0.2) + (seq_start_idx * 0.1)
                                probabilities = probabilities / temperature
                                probabilities = torch.softmax(probabilities, dim=1)
                                predicted_num = torch.multinomial(probabilities, 1).item() + 1
                            
                            white_prediction.append(predicted_num)
                    
                    # Ensure uniqueness of white balls
                    white_prediction = list(set(white_prediction))
                    while len(white_prediction) < 5:
                        new_num = np.random.randint(1, 70)
                        if new_num not in white_prediction:
                            white_prediction.append(new_num)
                    
                    white_prediction = sorted(white_prediction[:5])
                    
                    # Generate powerball with some intelligence based on sequence
                    recent_pb_values = sequence_data['Powerball'].tolist()
                    if pred_idx == 0:
                        # Use mode or mean for first prediction
                        pb_mean = np.mean(recent_pb_values)
                        powerball_prediction = max(1, min(26, int(np.round(pb_mean))))
                    else:
                        # Use weighted random selection based on recent frequencies
                        pb_counts = {}
                        for pb in recent_pb_values:
                            pb_counts[pb] = pb_counts.get(pb, 0) + 1
                        
                        if pb_counts:
                            # Weight by frequency but add randomness
                            weights = [pb_counts.get(pb, 1) for pb in range(1, 27)]
                            weights = np.array(weights) / sum(weights)
                            powerball_prediction = np.random.choice(range(1, 27), p=weights)
                        else:
                            powerball_prediction = np.random.randint(1, 27)
                    
                    prediction = white_prediction + [powerball_prediction]
                    if prediction not in predictions:  # Avoid duplicates
                        predictions.append(prediction)
                        
                        if len(predictions) >= num_predictions:
                            return predictions[:num_predictions]
            
            # Fill remaining predictions if needed
            while len(predictions) < num_predictions:
                # Use most recent sequence for remaining predictions
                recent_data = full_data.tail(sequence_length)
                
                white_sequences = [[] for _ in range(5)]
                for _, row in recent_data.iterrows():
                    white_numbers = [row['White_1'], row['White_2'], row['White_3'], 
                                   row['White_4'], row['White_5']]
                    for pos in range(5):
                        white_sequences[pos].append(white_numbers[pos])
                
                white_prediction = []
                for pos in range(5):
                    model = self.lstm_models[f'white_{pos + 1}']
                    model = model.to(self.device)
                    model.eval()
                    
                    with torch.no_grad():
                        seq_tensor = torch.FloatTensor([white_sequences[pos]]).unsqueeze(-1).to(self.device)
                        output = model(seq_tensor)
                        probabilities = torch.softmax(output, dim=1)
                        
                        # High temperature for diversity
                        temperature = 2.0
                        probabilities = probabilities / temperature
                        probabilities = torch.softmax(probabilities, dim=1)
                        predicted_num = torch.multinomial(probabilities, 1).item() + 1
                        white_prediction.append(predicted_num)
                
                white_prediction = list(set(white_prediction))
                while len(white_prediction) < 5:
                    new_num = np.random.randint(1, 70)
                    if new_num not in white_prediction:
                        white_prediction.append(new_num)
                
                white_prediction = sorted(white_prediction[:5])
                powerball_prediction = np.random.randint(1, 27)
                
                prediction = white_prediction + [powerball_prediction]
                if prediction not in predictions:
                    predictions.append(prediction)
            
        else:
            # Fallback to original method (single recent sequence)
            recent_data = self.data_handler.data.sort_values('Date').tail(sequence_length)
            
            print(f"Generating {num_predictions} LSTM predictions using {len(recent_data)} recent draws...")
            
            # Prepare input sequences and features
            white_sequences = [[] for _ in range(5)]
            features_seq = []
            
            for _, row in recent_data.iterrows():
                white_numbers = [row['White_1'], row['White_2'], row['White_3'], 
                               row['White_4'], row['White_5']]
                
                # Add to position-specific sequences
                for pos in range(5):
                    white_sequences[pos].append(white_numbers[pos])
                
                # Create features
                features = [
                    np.mean(white_numbers), np.std(white_numbers),
                    max(white_numbers), min(white_numbers), row['Powerball'],
                    sum(white_numbers), len(set(white_numbers)),
                    sum(1 for x in white_numbers if x % 2 == 0),
                    sum(1 for x in white_numbers if x <= 35),
                    len(features_seq) / sequence_length
                ]
                features_seq.append(features)
            
            # Generate multiple predictions
            for pred_idx in range(num_predictions):
                white_prediction = []
                
                # Predict each white ball position
                for pos in range(5):
                    model = self.lstm_models[f'white_{pos + 1}']
                    model = model.to(self.device)
                    model.eval()
                    
                    with torch.no_grad():
                        seq_tensor = torch.FloatTensor([white_sequences[pos]]).unsqueeze(-1).to(self.device)
                        
                        output = model(seq_tensor)
                        probabilities = torch.softmax(output, dim=1)
                        
                        # Add randomization for diversity
                        if pred_idx == 0:
                            # Best prediction for first row
                            predicted_num = torch.argmax(probabilities, dim=1).item() + 1
                        else:
                            # Sample with temperature for diversity
                            temperature = 1.5
                            probabilities = probabilities / temperature
                            probabilities = torch.softmax(probabilities, dim=1)
                            predicted_num = torch.multinomial(probabilities, 1).item() + 1
                        
                        white_prediction.append(predicted_num)
                
                # Ensure uniqueness of white balls
                white_prediction = list(set(white_prediction))
                while len(white_prediction) < 5:
                    new_num = np.random.randint(1, 70)
                    if new_num not in white_prediction:
                        white_prediction.append(new_num)
                
                white_prediction = sorted(white_prediction[:5])
                
                # Generate powerball using classical method (LSTM might overfit on small dataset)
                powerball_prediction = np.random.randint(1, 27)
                
                predictions.append(white_prediction + [powerball_prediction])
        
        return predictions[:num_predictions]