firewall_plot_output.py

import pandas as pd
import matplotlib.pyplot as plt
import os
from datetime import datetime
import numpy as np
import warnings
import matplotlib
matplotlib.use('Agg')  # Use non-interactive backend

# Suppress warnings
warnings.filterwarnings('ignore')

# Complete English environment setup with LARGER BASE FONT SIZE
plt.rcParams['font.family'] = 'DejaVu Sans'
plt.rcParams['axes.unicode_minus'] = False
matplotlib.rcParams['font.sans-serif'] = ['DejaVu Sans', 'Arial', 'Liberation Sans', 'sans-serif']
matplotlib.rcParams['axes.formatter.use_locale'] = False
plt.rcParams['font.size'] = 20  # Increased from 16 to 20 as base font size

def create_output_directories():
    """Create output directories organized by chart type"""
    base_dir = "./Pivot"
    
    directories = {
        'identity_analysis': os.path.join(base_dir, "IdentityAnalysis"),
        'categories_analysis': os.path.join(base_dir, "CategoriesAnalysis"), 
        'destination_ports': os.path.join(base_dir, "DestinationPorts"),
        'source_ips': os.path.join(base_dir, "SourceIPs"),
        'destination_ips': os.path.join(base_dir, "DestinationIPs"),
        'protocols': os.path.join(base_dir, "Protocols"),
        'applications': os.path.join(base_dir, "Applications"),
        'allowed_rules': os.path.join(base_dir, "AllowedRules"),
        'actions': os.path.join(base_dir, "Actions")
    }
    
    for dir_name, dir_path in directories.items():
        os.makedirs(dir_path, exist_ok=True)
        print(f"Created directory: {dir_path}")
    
    return directories

def load_csv_files(input_dir):
    """Load CSV files generated by the CSV generator script"""
    try:
        print("Loading CSV files for plot generation...")
        
        csv_files = {
            'identity_breakdown': 'firewall_identity_breakdown.csv',
            'firewall_categories': 'firewall_categories_breakdown.csv',
            'ad_users_categories': 'ad_users_categories_breakdown.csv',
            'proxy_identity_breakdown': 'proxy_identity_breakdown.csv',
            'proxy_categories': 'proxy_categories_breakdown.csv',
            'proxy_ad_users_categories': 'proxy_ad_users_categories_breakdown.csv',
            'firewall_categories_with_identities': 'firewall_categories_with_identities_breakdown.csv',
            'firewall_top10_summary': 'firewall_top10_categories_summary.csv',
            'firewall_unique_source_ips_by_identity': 'firewall_unique_source_ips_by_identity.csv',
        }
        
        loaded_data = {}
        
        # Load basic files
        for key, filename in csv_files.items():
            filepath = os.path.join(input_dir, filename)
            if os.path.exists(filepath):
                try:
                    df = pd.read_csv(filepath)
                    loaded_data[key] = df
                    print(f"  ✓ Loaded: {filename} ({len(df)} rows)")
                except Exception as e:
                    print(f"  ✗ Error loading {filename}: {e}")
            else:
                print(f"  ✗ File not found: {filename}")
        
        # Load all other CSV files
        all_files = os.listdir(input_dir)
        
        for filename in all_files:
            if filename.endswith('.csv'):
                filepath = os.path.join(input_dir, filename)
                try:
                    df = pd.read_csv(filepath)
                    
                    if filename.startswith('nt_') and 'categories_breakdown' in filename:
                        key = filename.replace('.csv', '')
                        loaded_data[key] = df
                        print(f"  ✓ Loaded: {filename} ({len(df)} rows)")
                    elif filename.startswith('proxy_nt_') and 'categories_breakdown' in filename:
                        key = filename.replace('.csv', '')
                        loaded_data[key] = df
                        print(f"  ✓ Loaded: {filename} ({len(df)} rows)")
                    elif 'protocol_breakdown' in filename:
                        key = filename.replace('.csv', '')
                        loaded_data[key] = df
                        print(f"  ✓ Loaded: {filename} ({len(df)} rows)")
                    elif 'action_breakdown' in filename:
                        key = filename.replace('.csv', '')
                        loaded_data[key] = df
                        print(f"  ✓ Loaded: {filename} ({len(df)} rows)")
                    elif 'allowed_rules_breakdown' in filename:
                        key = filename.replace('.csv', '')
                        loaded_data[key] = df
                        print(f"  ✓ Loaded: {filename} ({len(df)} rows)")
                    elif 'top20' in filename:
                        key = filename.replace('.csv', '')
                        loaded_data[key] = df
                        print(f"  ✓ Loaded: {filename} ({len(df)} rows)")
                    elif 'metadata' in filename:
                        key = filename.replace('.csv', '')
                        loaded_data[key] = df
                        print(f"  ✓ Loaded metadata: {filename}")
                        
                except Exception as e:
                    print(f"  ✗ Error loading {filename}: {e}")
        
        print(f"Successfully loaded {len(loaded_data)} CSV files")
        return loaded_data
        
    except Exception as e:
        print(f"Error loading CSV files: {e}")
        return None

def create_identity_breakdown_bar_chart(data, output_dirs, data_type="firewall"):
    """Create horizontal bar chart showing breakdown of identities"""
    print(f"Creating {data_type} identity breakdown bar chart...")
    
    data_key = f'{data_type}_identity_breakdown' if data_type != 'firewall' else 'identity_breakdown'
    if data_key not in data:
        print(f"{data_type.title()} identity breakdown CSV data not found")
        return
    
    df = data[data_key]
    
    num_categories = len(df)
    max_label_length = max(len(str(cat)) for cat in df['Category'])
    
    fig_height = max(12, num_categories * 1.2)  # Increased
    fig_width = max(20, max_label_length * 0.18)  # Increased
    
    plt.figure(figsize=(fig_width, fig_height))
    y_pos = np.arange(len(df))
    
    colors = []
    for category in df['Category']:
        if 'AD Users' in category:
            colors.append('#ff7f7f')
        elif 'Network Tunnels' in category:
            colors.append('#7fbfff')
        else:
            colors.append('#7fff7f')
    
    bars = plt.barh(y_pos, df['Count'], color=colors, alpha=0.8)
    
    plt.xlabel('Number of Records', fontsize=24)  # Increased
    plt.ylabel('Identity Categories', fontsize=24)  # Increased
    plt.title(f'{data_type.title()} Traffic - Identity Type Breakdown', fontsize=28, pad=35)  # Increased
    
    category_labels = []
    for cat in df['Category']:
        if len(str(cat)) > 35:
            category_labels.append(str(cat)[:32] + "...")
        else:
            category_labels.append(str(cat))
    
    plt.yticks(y_pos, category_labels, fontsize=20)  # Increased
    plt.xticks(fontsize=20)  # Increased
    
    max_count = max(df['Count'])
    for i, (bar, count, percentage) in enumerate(zip(bars, df['Count'], df['Percentage'])):
        width = bar.get_width()
        
        if width < max_count * 0.4:
            text_x = width + max_count * 0.03
            ha = 'left'
            color_text = 'black'
        else:
            text_x = width * 0.92
            ha = 'right'
            color_text = 'white'
        
        label_text = f'{count:,} ({percentage:.1f}%)'
        
        plt.text(text_x, bar.get_y() + bar.get_height()/2, 
                label_text, 
                ha=ha, va='center', fontsize=16, fontweight='bold',  # Increased
                color=color_text, bbox=dict(boxstyle='round,pad=0.3', 
                facecolor='white' if color_text == 'black' else 'black', 
                alpha=0.7, edgecolor='none'))
    
    plt.xlim(0, max_count * 1.35)
    
    total_count = df['Count'].sum()
    plt.figtext(0.5, -0.05, f'Total {data_type} records: {total_count:,}', 
                ha='center', fontsize=18, style='italic', weight='bold')  # Increased
    
    plt.gca().invert_yaxis()
    plt.tight_layout()
    plt.subplots_adjust(left=0.35, bottom=0.18, right=0.95, top=0.88)
    
    output_path = os.path.join(output_dirs['identity_analysis'], f"{data_type}_identity_breakdown.png")
    plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white', pad_inches=0.2)
    plt.close()
    print(f"{data_type.title()} identity breakdown bar chart saved: {output_path}")

def create_categories_breakdown_bar_chart(data, output_dirs, data_type="firewall"):
    """Create horizontal bar chart for categories breakdown"""
    print(f"Creating {data_type} categories breakdown bar chart...")
    
    data_key = f'{data_type}_categories' if data_type != 'firewall' else 'firewall_categories'
    if data_key not in data:
        print(f"{data_type.title()} categories CSV data not found")
        return
    
    df = data[data_key]
    actual_total = df['Count'].sum()
    
    limit = 20
    if len(df) > limit:
        top_categories = df.head(limit)
        others_count = df.iloc[limit:]['Count'].sum()
        others_percentage = df.iloc[limit:]['Percentage'].sum()
        
        if others_count > 0:
            others_row = pd.DataFrame({'Category': ['Others'], 'Count': [others_count], 'Percentage': [others_percentage]})
            plot_data = pd.concat([top_categories, others_row], ignore_index=True)
        else:
            plot_data = top_categories
    else:
        plot_data = df
    
    num_categories = len(plot_data)
    fig_height = max(10, num_categories * 1.0)
    
    plt.figure(figsize=(16, fig_height))
    y_pos = np.arange(len(plot_data))
    
    color = 'lightblue' if data_type == 'firewall' else 'steelblue'
    bars = plt.barh(y_pos, plot_data['Count'], color=color, alpha=0.8)
    
    plt.xlabel('Number of Records', fontsize=20)  # Increased
    plt.ylabel('Categories', fontsize=20)  # Increased
    plt.title(f'{data_type.title()} - Categories Breakdown', fontsize=24, pad=25)  # Increased
    
    category_labels = []
    for cat in plot_data['Category']:
        if len(str(cat)) > 40:
            category_labels.append(str(cat)[:37] + "...")
        else:
            category_labels.append(str(cat))
    
    plt.yticks(y_pos, category_labels, fontsize=18)  # Increased
    
    max_count = max(plot_data['Count'])
    for i, (bar, count, percentage) in enumerate(zip(bars, plot_data['Count'], plot_data['Percentage'])):
        width = bar.get_width()
        
        if data_type == 'proxy':
            if width < max_count * 0.15:
                text_x = width + max_count * 0.02
                ha = 'left'
                text_color = 'black'
            else:
                text_x = width * 0.95
                ha = 'right'
                text_color = 'white'
        else:
            if width < max_count * 0.3:
                text_x = width + max_count * 0.02
                ha = 'left'
                text_color = 'black'
            else:
                text_x = width * 0.95
                ha = 'right'
                text_color = 'white'
            
        plt.text(text_x, bar.get_y() + bar.get_height()/2, 
                f'{count:,} ({percentage:.1f}%)', 
                ha=ha, va='center', fontsize=16, fontweight='bold', color=text_color)  # Increased
    
    plt.gca().invert_yaxis()
    
    num_categories = len(plot_data)
    if num_categories <= 3:
        footer_y = -0.12
        bottom_margin = 0.20
    elif num_categories <= 10:
        footer_y = -0.05
        bottom_margin = 0.15
    else:
        footer_y = 0.02
        bottom_margin = 0.08
    
    plt.figtext(0.5, footer_y, f'Total {data_type} records: {actual_total:,}', 
                ha='center', fontsize=18, style='italic')  # Increased
    
    plt.tight_layout()
    plt.subplots_adjust(left=0.25, bottom=bottom_margin, right=0.95, top=0.92)
    
    output_path = os.path.join(output_dirs['categories_analysis'], f"{data_type}_categories_breakdown.png")
    plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white')
    plt.close()
    print(f"{data_type.title()} categories breakdown bar chart saved: {output_path}")

def create_ad_users_breakdown_bar_chart(data, output_dirs, data_type="firewall"):
    """Create horizontal bar chart for AD Users categories breakdown"""
    print(f"Creating {data_type} AD Users breakdown bar chart...")
    
    data_key = f'{data_type}_ad_users_categories' if data_type != 'firewall' else 'ad_users_categories'
    if data_key not in data:
        print(f"{data_type.title()} AD Users categories CSV data not found")
        return
    
    df = data[data_key]
    
    if df.empty:
        print(f"No {data_type} AD Users data to plot")
        return
    
    if len(df) > 20:
        top_categories = df.head(20)
        others_count = df.iloc[20:]['Count'].sum()
        others_percentage = df.iloc[20:]['Percentage'].sum()
        
        if others_count > 0:
            others_row = pd.DataFrame({'Category': ['Others'], 'Count': [others_count], 'Percentage': [others_percentage]})
            plot_data = pd.concat([top_categories, others_row], ignore_index=True)
        else:
            plot_data = top_categories
    else:
        plot_data = df
    
    num_categories = len(plot_data)
    fig_height = max(12, num_categories * 1.0)
    
    plt.figure(figsize=(18, fig_height))  # Increased
    y_pos = np.arange(len(plot_data))
    
    color = 'lightcoral' if data_type == 'firewall' else 'crimson'
    bars = plt.barh(y_pos, plot_data['Count'], color=color, alpha=0.8)
    
    plt.xlabel('Number of Records', fontsize=20)  # Increased
    plt.ylabel('Categories', fontsize=20)  # Increased
    plt.title(f'{data_type.title()} AD Users - Categories Breakdown', fontsize=22, pad=25)  # Increased
    plt.yticks(y_pos, plot_data['Category'], fontsize=16)  # Increased
    
    max_count = max(plot_data['Count'])
    for i, (bar, count, percentage) in enumerate(zip(bars, plot_data['Count'], plot_data['Percentage'])):
        width = bar.get_width()
        
        if data_type == 'proxy':
            if width < max_count * 0.15:
                text_x = width + max_count * 0.01
                ha = 'left'
                text_color = 'black'
            else:
                text_x = width * 0.95
                ha = 'right'
                text_color = 'white'
        else:
            text_x = width + max_count * 0.01
            ha = 'left'
            text_color = 'black'
            
        plt.text(text_x, bar.get_y() + bar.get_height()/2, 
                f'{count:,} ({percentage:.1f}%)', 
                ha=ha, va='center', fontsize=16, fontweight='bold', color=text_color)  # Increased
    
    plt.gca().invert_yaxis()
    
    total_count = df['Count'].sum()
    num_categories = len(plot_data)
    if num_categories <= 3:
        footer_y = -0.12
    elif num_categories <= 10:
        footer_y = -0.05
    else:
        footer_y = 0.02
    
    plt.figtext(0.5, footer_y, f'Total {data_type} AD Users records: {total_count:,}', 
                ha='center', fontsize=18, style='italic')  # Increased
    
    plt.tight_layout()
    
    output_path = os.path.join(output_dirs['categories_analysis'], f"{data_type}_ad_users_breakdown.png")
    plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white', pad_inches=0.3)
    plt.close()
    print(f"{data_type.title()} AD Users breakdown bar chart saved: {output_path}")

def create_network_tunnels_breakdown_charts(data, output_dirs, data_type="firewall"):
    """Create breakdown bar charts for each Network Tunnels identity"""
    print(f"Creating {data_type} Network Tunnels breakdown bar charts...")
    
    prefix = f"{data_type}_nt_" if data_type != 'firewall' else 'nt_'
    nt_files = [key for key in data.keys() if key.startswith(prefix) and 'categories_breakdown' in key]
    
    if not nt_files:
        print(f"No {data_type} Network Tunnels breakdown data found")
        return
    
    for key in nt_files:
        df = data[key]
        
        if df.empty:
            continue
        
        if len(df) > 20:
            df_limited = df.head(20)
            others_count = df.iloc[20:]['Count'].sum()
            if others_count > 0:
                others_percentage = df.iloc[20:]['Percentage'].sum()
                others_row = pd.DataFrame({'Category': ['Others'], 'Count': [others_count], 'Percentage': [others_percentage]})
                df = pd.concat([df_limited, others_row], ignore_index=True)
            else:
                df = df_limited
        
        if data_type != 'firewall':
            identity_name = key.replace(f'{data_type}_nt_', '').replace('_categories_breakdown', '').replace('_', ' ')
        else:
            identity_name = key.replace('nt_', '').replace('_categories_breakdown', '').replace('_', ' ')
        
        num_categories = len(df)
        fig_height = max(10, num_categories * 1.0)
        
        plt.figure(figsize=(18, fig_height))  # Increased
        y_pos = np.arange(len(df))
        
        color = 'lightgreen' if data_type == 'firewall' else 'forestgreen'
        bars = plt.barh(y_pos, df['Count'], color=color, alpha=0.8)
        
        plt.xlabel('Number of Records', fontsize=20)  # Increased
        plt.ylabel('Categories', fontsize=20)  # Increased
        plt.title(f'{data_type.title()} Network Tunnels ({identity_name}) - Categories Breakdown', fontsize=22, pad=25)  # Increased
        
        category_labels = []
        for cat in df['Category']:
            if len(str(cat)) > 40:
                category_labels.append(str(cat)[:37] + "...")
            else:
                category_labels.append(str(cat))
        
        plt.yticks(y_pos, category_labels, fontsize=16)  # Increased
        
        max_count = max(df['Count'])
        for i, (bar, count, percentage) in enumerate(zip(bars, df['Count'], df['Percentage'])):
            width = bar.get_width()
            
            if data_type == 'proxy':
                if width < max_count * 0.15:
                    text_x = width + max_count * 0.02
                    ha = 'left'
                    text_color = 'black'
                else:
                    text_x = width * 0.95
                    ha = 'right'
                    text_color = 'white'
            else:
                if width < max_count * 0.3:
                    text_x = width + max_count * 0.02
                    ha = 'left'
                    text_color = 'black'
                else:
                    text_x = width * 0.95
                    ha = 'right'
                    text_color = 'white'
                
            plt.text(text_x, bar.get_y() + bar.get_height()/2, 
                    f'{count:,} ({percentage:.1f}%)', 
                    ha=ha, va='center', fontsize=16, fontweight='bold',  # Increased
                    color=text_color)
        
        plt.gca().invert_yaxis()
        
        total_count = df['Count'].sum()
        num_categories = len(df)
        if num_categories <= 3:
            footer_y = -0.12
            bottom_margin = 0.20
        elif num_categories <= 10:
            footer_y = -0.05
            bottom_margin = 0.15
        else:
            footer_y = 0.02
            bottom_margin = 0.08
        
        plt.figtext(0.5, footer_y, f'Total {data_type} Network Tunnels ({identity_name}) records: {total_count:,}', 
                    ha='center', fontsize=18, style='italic')  # Increased
        
        plt.tight_layout()
        plt.subplots_adjust(left=0.25, bottom=bottom_margin, right=0.95, top=0.92)
        
        safe_identity = identity_name.replace('/', '_').replace('\\', '_').replace(':', '_')
        if data_type != 'firewall':
            output_filename = f"{data_type}_nt_{safe_identity}_breakdown.png"
        else:
            output_filename = f"nt_{safe_identity}_breakdown.png"
        output_path = os.path.join(output_dirs['categories_analysis'], output_filename)
        plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white')
        plt.close()
        print(f"{data_type.title()} Network Tunnels ({identity_name}) breakdown bar chart saved: {output_path}")

def create_action_pie_chart(data, filename_key, title, output_dirs):
    """Create pie chart for action breakdown"""
    if filename_key not in data:
        print(f"Data for {filename_key} not found")
        return
    
    df = data[filename_key]
    
    if df.empty:
        print(f"No data to plot for {title}")
        return
    
    plt.figure(figsize=(12, 10))  # Increased
    
    actions = df['Action'].tolist()
    counts = df['Count'].tolist()
    percentages = df['Percentage'].tolist()
    
    colors = []
    for action in actions:
        if 'Allow' in str(action):
            colors.append('#28a745')
        elif 'Block' in str(action) or 'Deny' in str(action):
            colors.append('#dc3545')
        else:
            colors.append('#6c757d')
    
    wedges, texts, autotexts = plt.pie(counts, labels=actions, autopct='%1.1f%%', 
                                       colors=colors, startangle=90, 
                                       textprops={'fontsize': 18})  # Increased
    
    for autotext in autotexts:
        autotext.set_color('white')
        autotext.set_fontweight('bold')
        autotext.set_fontsize(20)  # Increased
    
    for text in texts:
        text.set_fontsize(18)  # Increased
        text.set_fontweight('bold')
    
    plt.title(title, fontsize=22, pad=25, fontweight='bold')  # Increased
    
    total_count = sum(counts)
    plt.figtext(0.5, 0.02, f'Total firewall records: {total_count:,}', 
                ha='center', fontsize=18, style='italic', weight='bold')  # Increased
    
    plt.axis('equal')
    plt.tight_layout()
    
    safe_filename = filename_key.replace('_action_breakdown', '_actions_pie')
    output_path = os.path.join(output_dirs['actions'], f"{safe_filename}.png")
    plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white', pad_inches=0.3)
    plt.close()
    print(f"{title} pie chart saved: {output_path}")

def create_allowed_rules_breakdown_bar_chart(data, filename_key, title, output_dirs):
    """Create horizontal bar chart for allowed rules breakdown"""
    if filename_key not in data:
        print(f"Data for {filename_key} not found")
        return
    
    df = data[filename_key]
    
    if df.empty:
        print(f"No data to plot for {title}")
        return
    
    if len(df) > 20:
        top_rules = df.head(20)
        others_count = df.iloc[20:]['Count'].sum()
        others_percentage = df.iloc[20:]['Percentage'].sum()
        
        if others_count > 0:
            others_row = pd.DataFrame({'Rule_Name': ['Others'], 'Count': [others_count], 'Percentage': [others_percentage]})
            plot_data = pd.concat([top_rules, others_row], ignore_index=True)
        else:
            plot_data = top_rules
    else:
        plot_data = df
    
    num_rules = len(plot_data)
    fig_height = max(12, num_rules * 1.0)  # Increased
    
    plt.figure(figsize=(18, fig_height))  # Increased
    y_pos = np.arange(len(plot_data))
    
    bars = plt.barh(y_pos, plot_data['Count'], color='darkslateblue', alpha=0.8)
    
    plt.xlabel('Number of Allowed Records', fontsize=20)  # Increased
    plt.ylabel('Rule Names', fontsize=20)  # Increased
    plt.title(title, fontsize=22, pad=25)  # Increased
    
    rule_labels = []
    for rule in plot_data['Rule_Name']:
        if len(str(rule)) > 50:
            rule_labels.append(str(rule)[:47] + "...")
        else:
            rule_labels.append(str(rule))
    
    plt.yticks(y_pos, rule_labels, fontsize=16)  # Increased
    
    max_count = max(plot_data['Count'])
    for i, (bar, count, percentage) in enumerate(zip(bars, plot_data['Count'], plot_data['Percentage'])):
        width = bar.get_width()
        
        if width < max_count * 0.25:
            text_x = width + max_count * 0.02
            ha = 'left'
            text_color = 'black'
        else:
            text_x = width * 0.95
            ha = 'right'
            text_color = 'white'
            
        plt.text(text_x, bar.get_y() + bar.get_height()/2, 
                f'{count:,} ({percentage:.1f}%)', 
                ha=ha, va='center', fontsize=16, fontweight='bold', color=text_color)  # Increased
    
    plt.gca().invert_yaxis()
    
    total_count = df['Count'].sum()
    plt.figtext(0.5, 0.02, f'Total allowed records: {total_count:,}', 
                ha='center', fontsize=18, style='italic', weight='bold')  # Increased
    
    plt.tight_layout()
    plt.subplots_adjust(left=0.3, bottom=0.1, right=0.95, top=0.92)
    
    safe_filename = filename_key.replace('_allowed_rules_breakdown', '_allowed_rules')
    output_path = os.path.join(output_dirs['allowed_rules'], f"{safe_filename}.png")
    plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white', pad_inches=0.2)
    plt.close()
    print(f"{title} chart saved: {output_path}")

def create_protocol_pie_chart(data, filename_key, title, output_dirs):
    """Create pie chart for protocol breakdown"""
    if filename_key not in data:
        print(f"Data for {filename_key} not found")
        return
    
    df = data[filename_key]
    
    if df.empty:
        print(f"No data to plot for {title}")
        return
    
    plt.figure(figsize=(14, 12))  # Increased
    
    protocols = df['Protocol'].tolist()
    counts = df['Count'].tolist()
    percentages = df['Percentage'].tolist()
    
    colors = plt.cm.Set3(np.linspace(0, 1, len(protocols)))
    
    wedges, texts, autotexts = plt.pie(counts, labels=protocols, autopct='%1.1f%%', 
                                       colors=colors, startangle=90, 
                                       textprops={'fontsize': 16})  # Increased
    
    for autotext in autotexts:
        autotext.set_color('white')
        autotext.set_fontweight('bold')
        autotext.set_fontsize(15)  # Increased
    
    plt.title(title, fontsize=22, pad=25, fontweight='bold')  # Increased
    
    total_count = sum(counts)
    plt.figtext(0.5, 0.02, f'Total records: {total_count:,}', 
                ha='center', fontsize=18, style='italic', weight='bold')  # Increased
    
    plt.axis('equal')
    plt.tight_layout()
    
    safe_filename = filename_key.replace('_protocol_breakdown', '_protocol_pie')
    output_path = os.path.join(output_dirs['protocols'], f"{safe_filename}.png")
    plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white', pad_inches=0.3)
    plt.close()
    print(f"{title} pie chart saved: {output_path}")

def create_top20_bar_chart(data, filename_key, title, output_dirs, chart_category):
    """Create horizontal bar chart for top 20 data"""
    if filename_key not in data:
        print(f"Data for {filename_key} not found")
        return
    
    df = data[filename_key]
    
    if df.empty:
        print(f"No data to plot for {title}")
        return
    
    metadata_key = filename_key.replace('_top20_', '_metadata_')
    metadata = None
    if metadata_key in data:
        metadata = data[metadata_key].iloc[0].to_dict()
    
    num_items = len(df)
    
    value_col = None
    for col in df.columns:
        if col not in ['Count', 'Percentage']:
            value_col = col
            break
    
    if value_col is None:
        print(f"Could not find value column in {filename_key}")
        return
    
    if 'port' in value_col.lower():
        max_label_length = max(len(f'Port {int(float(port))}') if 'Unknown' not in str(port) else len(str(port)) for port in df[value_col])
    else:
        max_label_length = max(len(str(label)) for label in df[value_col])
    
    fig_height = max(12, num_items * 1.0)  # Increased
    fig_width = max(20, max_label_length * 0.15)  # Increased
    
    plt.figure(figsize=(fig_width, fig_height))
    y_pos = np.arange(len(df))
    
    count_col = 'Count'
    
    if chart_category == 'destination_ports':
        color = 'steelblue'
    elif chart_category == 'source_ips':
        color = 'purple'
    elif chart_category == 'destination_ips':
        color = 'darkred'
    else:
        color = 'gray'
    
    bars = plt.barh(y_pos, df[count_col], color=color, alpha=0.8)
    
    plt.xlabel('Number of Records', fontsize=20)  # Increased
    plt.ylabel(value_col.replace('_', ' ').title(), fontsize=20)  # Increased
    plt.title(title, fontsize=24, pad=30)  # Increased
    
    if 'port' in value_col.lower():
        labels = []
        for port in df[value_col]:
            if 'Unknown' in str(port) or 'Missing' in str(port):
                labels.append(str(port))
            else:
                labels.append(f'Port {int(float(port))}')
    else:
        labels = [str(label) for label in df[value_col]]
    
    formatted_labels = []
    for label in labels:
        if len(str(label)) > 30:
            formatted_labels.append(str(label)[:27] + "...")
        else:
            formatted_labels.append(str(label))
    
    plt.yticks(y_pos, formatted_labels, fontsize=17)  # Increased
    
    max_count = max(df[count_col])
    for i, (bar, count, percentage) in enumerate(zip(bars, df[count_col], df['Percentage'])):
        width = bar.get_width()
        
        if width < max_count * 0.4:
            text_x = width + max_count * 0.03
            ha = 'left'
            color_text = 'black'
            bg_color = 'white'
        else:
            text_x = width * 0.92
            ha = 'right'
            color_text = 'white'
            bg_color = color
            
        label_text = f'{count:,} ({percentage:.1f}%)'
        
        plt.text(text_x, bar.get_y() + bar.get_height()/2, 
                label_text, 
                ha=ha, va='center', fontsize=16, fontweight='bold',  # Increased
                color=color_text, bbox=dict(boxstyle='round,pad=0.3', 
                facecolor=bg_color, alpha=0.7, edgecolor='none'))
    
    plt.gca().invert_yaxis()
    
    if metadata:
        total_records = int(metadata['total_records_all'])
        unique_count = int(metadata['total_unique_values'])
        
        if chart_category in ['source_ips', 'destination_ips']:
            footer_text = f'Total records: {total_records:,} | Unique IP addresses: {unique_count:,}'
        elif chart_category == 'destination_ports':
            footer_text = f'Analysis based on {total_records:,} total records with {unique_count:,} unique destination ports'
        elif chart_category == 'applications':
            footer_text = f'Total records: {total_records:,} | Unique applications: {unique_count:,}'
        else:
            footer_text = f'Total records: {total_records:,} | Unique {value_col.replace("_", " ").lower()}s: {unique_count:,}'
    else:
        displayed_total = df[count_col].sum()
        if chart_category in ['source_ips', 'destination_ips']:
            unique_ips = len(df) - (1 if any('Unknown' in str(ip) or 'Missing' in str(ip) for ip in df[value_col]) else 0)
            footer_text = f'Total records: {displayed_total:,} | Unique IP addresses: {unique_ips:,}'
        else:
            footer_text = f'Total records analyzed: {displayed_total:,}'
    
    num_items = len(df)
    if num_items <= 5:
        footer_y = -0.12
        bottom_margin = 0.25
    elif num_items <= 15:
        footer_y = -0.05
        bottom_margin = 0.20
    else:
        footer_y = 0.02
        bottom_margin = 0.12
    
    plt.figtext(0.5, footer_y, footer_text, 
                ha='center', fontsize=18, style='italic', weight='bold')  # Increased
    
    plt.tight_layout()
    plt.subplots_adjust(left=0.35, bottom=bottom_margin, right=0.95, top=0.88)
    
    output_filename = f"{filename_key}.png"
    output_path = os.path.join(output_dirs[chart_category], output_filename)
    plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white', pad_inches=0.2)
    plt.close()
    print(f"{title} chart saved: {output_path}")

def create_unique_source_ips_table(data, output_dirs):
    """Create table showing unique source IPs count for each identity"""
    print("Creating unique source IPs by identity table...")
    
    if 'firewall_unique_source_ips_by_identity' not in data:
        print("Firewall unique source IPs by identity data not found")
        return
    
    unique_ip_data = data['firewall_unique_source_ips_by_identity']
    
    if unique_ip_data.empty:
        print("No unique source IP data to plot")
        return
    
    fig, ax = plt.subplots(figsize=(16, max(10, len(unique_ip_data) * 1.0)))  # Increased
    ax.axis('tight')
    ax.axis('off')
    
    table_rows = []
    for _, row in unique_ip_data.iterrows():
        identity = row['Identity']
        count = row['Unique_Source_IPs']
        
        if len(identity) > 30:
            display_identity = identity[:27] + "..."
        else:
            display_identity = identity
            
        table_rows.append([display_identity, f"{count:,}"])
    
    table = ax.table(cellText=table_rows,
                    colLabels=['Identity', 'Unique Source IPs'],
                    cellLoc='left',
                    loc='center',
                    colWidths=[0.65, 0.35])
    
    table.auto_set_font_size(False)
    table.set_fontsize(19)  # Increased
    table.scale(1, 2.5)  # Increased
    
    for i in range(2):
        table[(0, i)].set_facecolor('#4CAF50')
        table[(0, i)].set_text_props(weight='bold', color='white')
        table[(0, i)].set_fontsize(22)  # Increased
    
    for i in range(1, len(table_rows) + 1):
        for j in range(2):
            identity_name = table_rows[i-1][0]
            if 'TOTAL' in identity_name:
                table[(i, j)].set_facecolor('#FFC107')
                table[(i, j)].set_text_props(weight='bold')
                table[(i, j)].set_fontsize(20)  # Increased
            elif identity_name == 'AD Users':
                table[(i, j)].set_facecolor('#ffebee')
                table[(i, j)].set_fontsize(19)  # Increased
            elif i % 2 == 0:
                table[(i, j)].set_facecolor('#f8f9fa')
            else:
                table[(i, j)].set_facecolor('white')
    
    plt.title('Firewall - Unique Source IP Addresses by Identity', 
             fontsize=26, fontweight='bold', pad=30)  # Increased
    
    plt.figtext(0.5, 0.02, 
                'Analysis shows the count of unique source IP addresses for each identity type | '
                'Note: "TOTAL (with overlaps)" sums individual counts, "TOTAL (overall unique)" shows actual unique IPs',
                ha='center', fontsize=18, style='italic')  # Increased
    
    output_path = os.path.join(output_dirs['source_ips'], "firewall_unique_source_ips_by_identity_table.png")
    plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white', pad_inches=0.4)
    plt.close()
    print(f"Unique source IPs by identity table saved: {output_path}")

def create_firewall_categories_stacked_bar_chart(data, output_dirs):
    """Create stacked bar chart for top 10 firewall categories with identity breakdown"""
    print("Creating firewall top 10 categories with identity breakdown stacked bar chart...")
    
    if 'firewall_categories_with_identities' not in data:
        print("Firewall categories with identities data not found")
        return
    
    detailed_df = data['firewall_categories_with_identities']
    
    if detailed_df.empty:
        print("No detailed firewall categories data to plot")
        return
    
    if 'firewall_top10_summary' in data:
        summary_df = data['firewall_top10_summary']
        category_order = summary_df['Category'].tolist()
    else:
        category_totals = detailed_df.groupby('Category')['Count'].sum().sort_values(ascending=False)
        category_order = category_totals.index.tolist()
    
    pivot_data = detailed_df.pivot_table(
        index='Category', 
        columns='Identity_Name', 
        values='Count', 
        fill_value=0
    )
    
    pivot_data = pivot_data.reindex(category_order, fill_value=0)
    
    num_categories = len(pivot_data)
    fig_height = max(16, num_categories * 1.6)  # Increased
    fig_width = 26  # Increased
    
    plt.figure(figsize=(fig_width, fig_height))
    
    identity_names = pivot_data.columns
    
    color_map = {}
    predefined_colors = ['#ff7f7f', '#7fbfff', '#90EE90', '#FFB347', '#DDA0DD', 
                        '#F0E68C', '#87CEEB', '#F5DEB3', '#98FB98', '#FFEFD5']
    
    for i, identity in enumerate(identity_names):
        if identity == 'AD Users':
            color_map[identity] = '#ff7f7f'
        elif identity == 'Unknown Network Tunnel':
            color_map[identity] = '#d3d3d3'
        elif identity == 'Mixed':
            color_map[identity] = '#ffd700'
        elif identity == 'Unknown':
            color_map[identity] = '#c0c0c0'
        else:
            color_map[identity] = predefined_colors[i % len(predefined_colors)]
    
    bar_positions = np.arange(len(pivot_data))
    bottom_values = np.zeros(len(pivot_data))
    
    bars_list = []
    for identity in identity_names:
        values = pivot_data[identity].values
        bars = plt.barh(bar_positions, values, left=bottom_values, 
                       label=identity, color=color_map[identity], alpha=0.85)
        bars_list.append(bars)
        bottom_values += values
    
    plt.xlabel('Number of Records', fontsize=22)  # Increased
    plt.ylabel('Categories', fontsize=22)  # Increased
    plt.title('Firewall Top 10 Categories - Identity Breakdown', fontsize=26, pad=35)  # Increased
    
    category_labels = []
    for cat in pivot_data.index:
        if len(str(cat)) > 35:
            category_labels.append(str(cat)[:32] + "...")
        else:
            category_labels.append(str(cat))
    
    plt.yticks(bar_positions, category_labels, fontsize=18)  # Increased
    plt.xticks(fontsize=18)  # Increased
    
    total_values = pivot_data.sum(axis=1).values
    max_total = max(total_values)
    grand_total = sum(total_values)
    
    # Add labels for top 6 categories only
    for i, (category, bar_pos) in enumerate(zip(pivot_data.index, bar_positions)):
        if i >= 6:
            continue
            
        current_left = 0
        category_total = total_values[i]
        
        category_segments = []
        for identity in identity_names:
            segment_value = pivot_data.loc[category, identity]
            if segment_value > 0:
                segment_percentage = (segment_value / category_total) * 100
                category_segments.append((identity, segment_value, segment_percentage))
        
        category_segments.sort(key=lambda x: x[1], reverse=True)
        top_segments = category_segments[:2]
        
        current_left = 0
        
        for identity in identity_names:
            segment_value = pivot_data.loc[category, identity]
            if segment_value > 0:
                segment_width = segment_value
                segment_center = current_left + segment_width / 2
                segment_percentage = (segment_value / category_total) * 100
                
                is_top_segment = any(identity == seg[0] for seg in top_segments)
                if is_top_segment and (segment_percentage > 5 or segment_value > 100):
                    if len(identity) > 15:
                        display_name = identity[:12] + "..."
                    else:
                        display_name = identity
                    
                    text_label = f'{display_name}\n{segment_percentage:.1f}%'
                    text_color = 'white' if segment_percentage > 10 else 'black'
                    
                    plt.text(segment_center, bar_pos, text_label, 
                            ha='center', va='center', fontsize=15, fontweight='bold',  # Increased
                            color=text_color)
                
                current_left += segment_width
    
    # Add total labels
    for i, (total_val, bar_pos) in enumerate(zip(total_values, bar_positions)):
        total_percentage = (total_val / grand_total) * 100
        text_x = total_val + max_total * 0.01
        label_text = f'{total_val:,}\n({total_percentage:.1f}%)'
        
        plt.text(text_x, bar_pos, label_text, 
                ha='left', va='center', fontsize=18, fontweight='bold',  # Increased
                bbox=dict(boxstyle='round,pad=0.3', facecolor='white', alpha=0.9, edgecolor='gray'))
    
    num_identities = len(identity_names)
    if num_identities > 8:
        ncol = 2
    else:
        ncol = 1
    
    legend = plt.legend(bbox_to_anchor=(1.02, 1), loc='upper left', fontsize=17,  # Increased
                       ncol=ncol, frameon=True, fancybox=True, shadow=True)
    legend.get_frame().set_facecolor('white')
    legend.get_frame().set_alpha(0.9)
    
    plt.gca().invert_yaxis()
    plt.xlim(0, max_total * 1.25)
    
    if 'firewall_top10_summary' in data:
        summary_df = data['firewall_top10_summary']
        total_records = summary_df['Count'].sum()
    else:
        total_records = sum(total_values)
    
    plt.figtext(0.5, 0.01, f'Total firewall records represented: {total_records:,} | Chart shows identity breakdown within each category', 
                ha='center', fontsize=18, style='italic', weight='bold')  # Increased
    
    plt.tight_layout()
    plt.subplots_adjust(left=0.20, bottom=0.25, right=0.75, top=0.90)
    
    output_path = os.path.join(output_dirs['categories_analysis'], "firewall_top10_categories_identity_breakdown.png")
    plt.savefig(output_path, dpi=300, bbox_inches='tight', facecolor='white', pad_inches=0.5)
    plt.close()

def main():
    """Main processing function"""
    print("=== Firewall Data Visualization Generator (Large Fonts) ===")
    
    # Set input directory
    input_dir = "./Pivot"
    
    if not os.path.exists(input_dir):
        print(f"Input directory not found: {input_dir}")
        print("Please run the firewall CSV generator script first to generate the required CSV files.")
        return
    
    # Create output directories organized by chart type
    output_dirs = create_output_directories()
    print(f"Output directories created in: {input_dir}")
    
    # Load CSV data
    data = load_csv_files(input_dir)
    if not data:
        print("No CSV data loaded. Please check if CSV files exist.")
        return
    
    # Create visualizations
    print("\n=== Creating Visualizations with Large Fonts ===")
    
    # 1. Identity breakdown bar charts (firewall and proxy)
    create_identity_breakdown_bar_chart(data, output_dirs, "firewall")
    create_identity_breakdown_bar_chart(data, output_dirs, "proxy")
    
    # 2. Categories breakdown bar charts (firewall and proxy)
    create_categories_breakdown_bar_chart(data, output_dirs, "firewall")
    create_categories_breakdown_bar_chart(data, output_dirs, "proxy")
    
    # 2.1. NEW: Create firewall top 10 categories with identity breakdown stacked bar chart
    create_firewall_categories_stacked_bar_chart(data, output_dirs)
    
    # 2.2. NEW: Create unique source IPs by identity table
    create_unique_source_ips_table(data, output_dirs)
    
    # 3. AD Users breakdown bar charts (firewall and proxy)
    create_ad_users_breakdown_bar_chart(data, output_dirs, "firewall")
    create_ad_users_breakdown_bar_chart(data, output_dirs, "proxy")
    
    # 4. Network Tunnels breakdown charts (firewall and proxy)
    create_network_tunnels_breakdown_charts(data, output_dirs, "firewall")
    create_network_tunnels_breakdown_charts(data, output_dirs, "proxy")
    
    # 5. Action pie chart - for firewall overall only
    if 'firewall_action_breakdown' in data:
        create_action_pie_chart(data, 'firewall_action_breakdown', 'Firewall - Actions Distribution', output_dirs)
    
    # 6. Allowed rules breakdown bar charts - automatically find and create charts for all allowed rules files
    allowed_rules_files = [key for key in data.keys() if 'allowed_rules_breakdown' in key]
    
    for key in allowed_rules_files:
        # Extract meaningful names from filename
        if key.startswith('firewall_nt_'):
            # Network Tunnels identity for firewall
            identity_part = key.replace('firewall_nt_', '').replace('_allowed_rules_breakdown', '')
            identity_name = identity_part.replace('_', ' ')
            title = f'Firewall Network Tunnels ({identity_name}) - Allowed Rules Breakdown'
        elif key.startswith('firewall_ad_users'):
            title = 'Firewall AD Users - Allowed Rules Breakdown'
        elif key == 'firewall_allowed_rules_breakdown':
            title = 'Firewall - Allowed Rules Breakdown'
        
        create_allowed_rules_breakdown_bar_chart(data, key, title, output_dirs)
    
    # 7. Protocol pie charts - automatically find and create charts for all protocol files (firewall only)
    protocol_files = [key for key in data.keys() if 'protocol_breakdown' in key and not key.startswith('proxy')]
    
    for key in protocol_files:
        # Extract meaningful names from filename
        if key.startswith('firewall_nt_'):
            # Network Tunnels identity for firewall
            identity_part = key.replace('firewall_nt_', '').replace('_protocol_breakdown', '')
            identity_name = identity_part.replace('_', ' ')
            title = f'Firewall Network Tunnels ({identity_name}) - Protocol Distribution'
        elif key.startswith('firewall_ad_users'):
            title = 'Firewall AD Users - Protocol Distribution'
        elif key == 'firewall_protocol_breakdown':
            title = 'Firewall - Protocol Distribution'
        else:
            title = f'{key.replace("_", " ").title()} - Protocol Distribution'
        
        create_protocol_pie_chart(data, key, title, output_dirs)
    
    # 8. Top 20 charts - automatically find and create charts for all top20 files
    top20_files = [key for key in data.keys() if 'top20' in key]
    
    for key in top20_files:
        # Extract meaningful names from filename
        parts = key.split('_')
        
        if key.startswith('proxy'):
            if 'proxy_nt_' in key:
                # Extract identity name for proxy network tunnels
                identity_part = key.replace('proxy_nt_', '').split('_top20')[0]
                identity_name = identity_part.replace('_', ' ')
                prefix = f'Proxy Network Tunnels ({identity_name})'
            elif 'proxy_ad_users' in key:
                prefix = 'Proxy AD Users'
            else:
                prefix = 'Proxy'
        elif 'firewall' in key:
            prefix = 'Firewall'
        elif 'ad_users' in key:
            prefix = 'AD Users'
        elif key.startswith('nt_'):
            # Extract identity name (preserve original case)
            identity_part = key.replace('nt_', '').split('_top20')[0]
            identity_name = identity_part.replace('_', ' ')
            prefix = f'Network Tunnels ({identity_name})'
        else:
            prefix = 'Unknown'
        
        if 'destination_port' in key:
            if '_tcp_' in key:
                chart_type = 'Top 20 Destination Ports (TCP)'
                chart_category = 'destination_ports'
            elif '_udp_' in key:
                chart_type = 'Top 20 Destination Ports (UDP)'
                chart_category = 'destination_ports'
            else:
                chart_type = 'Top 20 Destination Ports'
                chart_category = 'destination_ports'
        elif 'source_ip' in key:
            chart_type = 'Top 20 Source IPs'
            chart_category = 'source_ips'
        elif 'destination_ip' in key:
            chart_type = 'Top 20 Destination IPs'
            chart_category = 'destination_ips'
        elif 'application' in key:
            chart_type = 'Top 20 Applications'
            chart_category = 'applications'
        else:
            chart_type = 'Top 20'
            chart_category = 'destination_ports'  # default
        
        title = f'{prefix} - {chart_type}'
        
        create_top20_bar_chart(data, key, title, output_dirs, chart_category)
    
    print(f"\nVisualization generation completed! Results saved to:")
    for category, path in output_dirs.items():
        print(f"  {category}: {os.path.abspath(path)}")
    
    # Summary
    print(f"\n=== Summary (Large Fonts Version) ===")
    print(f"CSV files processed: {len(data)}")
    print(f"Chart categories created:")
    print(f"  - Identity Analysis: Identity breakdown charts (LARGE FONTS)")
    print(f"  - Categories Analysis: Categories breakdown charts (Top 20 limit applied, LARGE FONTS)")
    
    nt_breakdown_files = [key for key in data.keys() if 'nt_' in key and 'categories_breakdown' in key]
    if nt_breakdown_files:
        print(f"    • Network Tunnels breakdown ({len(nt_breakdown_files)} identities, Top 20 limit applied, LARGE FONTS)")
    
    # Count charts by category
    dest_port_files = [key for key in data.keys() if 'destination_port' in key]
    source_ip_files = [key for key in data.keys() if 'source_ip' in key]
    dest_ip_files = [key for key in data.keys() if 'destination_ip' in key]
    application_files = [key for key in data.keys() if 'application' in key]
    protocol_files = [key for key in data.keys() if 'protocol_breakdown' in key and not key.startswith('proxy')]
    allowed_rules_files = [key for key in data.keys() if 'allowed_rules_breakdown' in key]
    action_files = [key for key in data.keys() if 'action_breakdown' in key]
    
    if dest_port_files:
        # Count TCP and UDP specific charts
        tcp_charts = len([key for key in dest_port_files if '_tcp_' in key])
        udp_charts = len([key for key in dest_port_files if '_udp_' in key])
        total_charts = len(dest_port_files)
        
        print(f"  - Destination Ports: {total_charts} charts (LARGE FONTS with enhanced readability)")
        if tcp_charts > 0 or udp_charts > 0:
            print(f"    • Overall charts: {total_charts - tcp_charts - udp_charts}")
            print(f"    • TCP-specific charts: {tcp_charts}")
            print(f"    • UDP-specific charts: {udp_charts}")
    if source_ip_files:
        print(f"  - Source IPs: {len(source_ip_files)} charts (LARGE FONTS)")
    if dest_ip_files:
        print(f"  - Destination IPs: {len(dest_ip_files)} charts (LARGE FONTS)")
    if application_files:
        print(f"  - Applications: {len(application_files)} charts (LARGE FONTS)")
    if protocol_files:
        print(f"  - Protocols: {len(protocol_files)} pie charts (firewall only, LARGE FONTS)")
    if allowed_rules_files:
        print(f"  - Allowed Rules: {len(allowed_rules_files)} charts (firewall only, Top 20 limit applied, LARGE FONTS)")
    if action_files:
        print(f"  - Actions: {len(action_files)} pie charts (firewall overall only, LARGE FONTS)")
    
    print("\n=== Font Size Improvements ===")
    print("✓ Base font size increased from 16 to 20")
    print("✓ Title font sizes increased by 20-40%")
    print("✓ Axis label font sizes increased by 25%")
    print("✓ Value label font sizes increased by 30%")
    print("✓ Table font sizes increased significantly")
    print("✓ All charts now have enhanced readability")

if __name__ == "__main__":
    main()