Web_Data_Extractor.txt

/**
 * =============================================================================
 * Modular Web Data Extractor
 * =============================================================================
 * 
 * A flexible, extensible web scraping framework for browser DevTools.
 * This script intelligently detects and extracts tabular and structured data
 * from web pages using a plugin-based architecture where new data sources
 * can be easily added.
 * 
 * ARCHITECTURE OVERVIEW:
 * 
 * 1. Core System
 *    - Registry for detectors and extractors
 *    - Main execution flow
 *    - Configuration management
 * 
 * 2. Detector Modules
 *    - Detect specific data structures in the DOM
 *    - Return candidate elements for extraction
 *    - Include confidence scoring
 * 
 * 3. Extractor Modules
 *    - Extract data from elements found by detectors
 *    - Transform raw DOM data into structured format
 *    - Handle special cases for different data structures
 * 
 * 4. Utility Functions
 *    - DOM helpers
 *    - Text processing
 *    - Structure analysis
 * 
 * =============================================================================
 * HOW TO USE:
 * 
 * 1. Copy this entire script
 * 2. Open browser DevTools on any webpage (F12 or right-click > Inspect)
 * 3. Paste into the Console tab and press Enter
 * 4. The script will automatically detect and extract data
 * 
 * =============================================================================
 * HOW TO EXTEND:
 * 
 * To add a new data source:
 * 1. Add a new detector in the Detectors section
 * 2. Add a corresponding extractor in the Extractors section
 * 3. Register both in the initialization section
 * 
 * See the existing modules for examples.
 * 
 * =============================================================================
 * Author: [Your Name]
 * Version: 1.0.0
 * =============================================================================
 */

// Immediately-invoked function expression (IIFE) to avoid polluting global scope
(function() {
  'use strict';

  /**
   * =============================================================================
   * CORE SYSTEM
   * =============================================================================
   * The core system manages the registry of detectors and extractors,
   * handles execution flow, and provides configuration management.
   */
  
  // Central registry for all detector and extractor modules
  const WebDataExtractor = {
    // Registries
    detectors: [],
    extractors: {},
    
    // Configuration
    config: {
      debug: true, // Set to true for detailed console logs
      maxDetectionAttempts: 5, // Maximum number of detector types to try
      outputFormat: 'html', // 'html', 'json', or 'csv'
      scrollForVirtualized: true, // Whether to scroll to reveal virtualized content
    },
    
    // Register a detector module
    registerDetector: function(detector) {
      if (!detector.name || typeof detector.detect !== 'function' || typeof detector.priority !== 'number') {
        console.error('Invalid detector module:', detector);
        return false;
      }
      
      this.detectors.push(detector);
      // Sort detectors by priority (higher priority will run first)
      this.detectors.sort((a, b) => b.priority - a.priority);
      
      if (this.config.debug) {
        console.log(`Registered detector: ${detector.name} (priority: ${detector.priority})`);
      }
      
      return true;
    },
    
    // Register an extractor module
    registerExtractor: function(extractor) {
      if (!extractor.name || !extractor.forDetector || typeof extractor.extract !== 'function') {
        console.error('Invalid extractor module:', extractor);
        return false;
      }
      
      this.extractors[extractor.forDetector] = extractor;
      
      if (this.config.debug) {
        console.log(`Registered extractor: ${extractor.name} for ${extractor.forDetector}`);
      }
      
      return true;
    },
    
    // Main execution function - made async to properly handle async extractors
    run: async function() {
      console.log('Web Data Extractor running...');
      
      // Data will be collected here
      const extractedData = [];
      
      // Try each detector in order of priority
      for (let i = 0; i < Math.min(this.detectors.length, this.config.maxDetectionAttempts); i++) {
        const detector = this.detectors[i];
        
        if (this.config.debug) {
          console.log(`Trying detector: ${detector.name}`);
        }
        
        // Run the detector
        const detectionResult = detector.detect();
        
        // Skip if nothing detected
        if (!detectionResult.elements || detectionResult.elements.length === 0) {
          if (this.config.debug) {
            console.log(`No elements found by ${detector.name}`);
          }
          continue;
        }
        
        console.log(`${detector.name} found ${detectionResult.elements.length} elements`);
        
        // Get the corresponding extractor
        const extractor = this.extractors[detector.name];
        if (!extractor) {
          console.error(`No extractor found for ${detector.name}`);
          continue;
        }
        
        // Extract data
        try {
          // Key fix: Use await to ensure async extractors complete
          const extractedItems = await Promise.resolve(extractor.extract(detectionResult));
          
          if (extractedItems && extractedItems.length > 0) {
            // Add extraction metadata
            extractedItems.forEach(item => {
              item.source = detector.name;
              extractedData.push(item);
            });
            
            if (this.config.debug) {
              console.log(`Extracted ${extractedItems.length} data items using ${extractor.name}`);
            }
            
            // Stop trying more detectors if we found data and detector says it's reliable
            if (detectionResult.reliable !== false && extractedItems.length > 0) {
              break;
            }
          }
        } catch (error) {
          console.error(`Error in ${extractor.name}:`, error);
        }
      }
      
      // Display the results
      if (extractedData.length > 0) {
        console.log(`Extraction complete. Found ${extractedData.length} data tables.`);
        this.displayResults(extractedData);
        return extractedData;
      } else {
        console.log('No data could be extracted from this page.');
        return [];
      }
    },
    
    // Output the results
    displayResults: function(data) {
      if (this.config.outputFormat === 'json') {
        console.log(JSON.stringify(data, null, 2));
        return;
      }
      
      // Default to HTML display
      const newWindow = window.open('', '_blank');
      if (!newWindow) {
        console.error('Unable to open result window. Check popup blocker settings.');
        return;
      }
      
      const doc = newWindow.document;
      
      // Generate HTML content
      let content = '';
      
      data.forEach((table, index) => {
        content += `
          <div class="table-container">
            <h2>Table ${index + 1}: ${table.id || 'Extracted Data'}</h2>
            <p class="meta">Source: ${table.source}</p>
            <p class="meta">Selector: ${table.selector || 'N/A'}</p>
            
            <table border="1">
              <thead>
                <tr>
                  ${table.headers.map(header => `<th>${header}</th>`).join('')}
                </tr>
              </thead>
              <tbody>
                ${table.rows.map(row => `
                  <tr>
                    ${row.map(cell => `<td>${cell}</td>`).join('')}
                  </tr>
                `).join('')}
              </tbody>
            </table>
          </div>
        `;
      });
      
      // Write the HTML document
      doc.write(`
        <!DOCTYPE html>
        <html>
        <head>
          <title>Extracted Web Data</title>
          <style>
            body { font-family: Arial, sans-serif; margin: 20px; }
            .table-container { margin-bottom: 40px; }
            table { border-collapse: collapse; width: 100%; margin-top: 10px; }
            th { background-color: #f2f2f2; padding: 8px; text-align: left; }
            td { padding: 8px; border: 1px solid #ddd; }
            tr:nth-child(even) { background-color: #f9f9f9; }
            .meta { color: #666; font-size: 14px; margin: 5px 0; }
            h1, h2 { color: #333; }
          </style>
        </head>
        <body>
          <h1>Extracted Web Data</h1>
          ${content}
        </body>
        </html>
      `);
      
      doc.close();
    }
  };

  /**
   * =============================================================================
   * UTILITY FUNCTIONS
   * =============================================================================
   * These utility functions are used by multiple detectors and extractors.
   * They handle common tasks like text processing, DOM analysis, etc.
   */
  
  const Utilities = {
    /**
     * Cleans text by removing extra whitespace and trimming
     * @param {string} text - Text to clean
     * @returns {string} Cleaned text
     */
    cleanText: function(text) {
      if (!text) return '';
      return text.replace(/\s+/g, ' ').trim();
    },
    
    /**
     * Generates a CSS selector for uniquely identifying an element
     * @param {HTMLElement} element - Element to generate selector for
     * @returns {string} CSS selector
     */
    generateSelector: function(element) {
      if (!element || !element.tagName) return 'unknown';
      
      // Try ID first (most specific)
      if (element.id) {
        return `#${element.id}`;
      }
      
      // Try classes
      if (element.classList && element.classList.length > 0) {
        const classes = Array.from(element.classList).join('.');
        const selector = `.${classes}`;
        const matchingElements = document.querySelectorAll(selector);
        if (matchingElements.length === 1) {
          return selector;
        }
      }
      
      // Use tag name with classes
      if (element.classList && element.classList.length > 0) {
        const classes = Array.from(element.classList).join('.');
        return `${element.tagName.toLowerCase()}.${classes}`;
      }
      
      // Use tag name and position
      const siblings = element.parentNode ? element.parentNode.children : [];
      let position = 0;
      
      for (let i = 0; i < siblings.length; i++) {
        if (siblings[i].tagName === element.tagName) {
          position++;
        }
        if (siblings[i] === element) {
          break;
        }
      }
      
      // Return full path
      const parentSelector = (element.parentNode === document) 
        ? '' 
        : this.generateSelector(element.parentNode) + ' > ';
        
      return `${parentSelector}${element.tagName.toLowerCase()}:nth-of-type(${position})`;
    },
    
    /**
     * Gets a signature of an element's structure based on its children
     * @param {HTMLElement} element - Element to analyze
     * @returns {string} Structure signature
     */
    getChildrenStructure: function(element) {
      if (!element || !element.children) return '';
      
      const childTags = Array.from(element.children).map(child => {
        if (!child.classList) return child.tagName;
        return `${child.tagName}${Array.from(child.classList).join('.')}`;
      });
      
      return childTags.join('|');
    },
    
    /**
     * Checks if an element has matching structure to a signature
     * @param {HTMLElement} element - Element to check
     * @param {string} structureSignature - Signature to compare with
     * @returns {boolean} True if structures match
     */
    hasMatchingStructure: function(element, structureSignature) {
      const elementSignature = this.getChildrenStructure(element);
      return elementSignature === structureSignature;
    },
    
    /**
     * Determines if text is likely to be a header based on content heuristics
     * @param {string} text - Text to analyze
     * @returns {boolean} True if likely a header
     */
    isLikelyHeader: function(text) {
      if (!text) return false;
      
      // Headers are typically shorter
      if (text.length > 50) return false;
      
      // Headers often don't contain numbers (except for dates/years)
      const hasNumbers = /\d+/.test(text);
      const isDateOrYear = /^(19|20)\d{2}$/.test(text) || // Year
                         /^\d{1,2}\/\d{1,2}(\/\d{2,4})?$/.test(text); // Date
      
      if (hasNumbers && !isDateOrYear) return false;
      
      // Headers are often title case or uppercase
      const isTitleCase = text !== text.toLowerCase() && 
                        (text === text.charAt(0).toUpperCase() + text.slice(1) ||
                         text.split(' ').every(word => word === word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()));
      
      const isAllCaps = text === text.toUpperCase() && text.length > 1;
      
      // Headers often have certain key terms
      const hasHeaderTerms = /total|sum|average|name|date|time|description|title|category|type|id|number|count|price|cost|amount|quantity|value|rate|percent/i.test(text);
      
      // Calculate a "headerness" score
      let score = 0;
      if (text.length < 20) score += 1;
      if (isTitleCase) score += 1;
      if (isAllCaps) score += 2;
      if (hasHeaderTerms) score += 2;
      if (!hasNumbers || isDateOrYear) score += 1;
      
      return score >= 2;
    },
    
    /**
     * Performs an action with a delay
     * @param {number} ms - Milliseconds to delay
     * @returns {Promise} Promise that resolves after the delay
     */
    delay: function(ms) {
      return new Promise(resolve => setTimeout(resolve, ms));
    }
  };

  /**
   * =============================================================================
   * DETECTORS
   * =============================================================================
   * Detector modules identify specific data structures in the DOM.
   * Each detector returns candidate elements for extraction.
   */
  
  /**
   * HTML Table Detector
   * Finds standard HTML <table> elements
   */
  const HTMLTableDetector = {
    name: 'HTMLTableDetector',
    priority: 100, // Highest priority since HTML tables are most reliable
    
    detect: function() {
      const tables = document.querySelectorAll('table');
      
      return {
        elements: Array.from(tables),
        reliable: tables.length > 0
      };
    }
  };
  
/**
 * AG Grid Detector
 * Finds AG Grid components in the DOM with robust detection strategies
 */
const AGGridDetector = {
  name: 'AGGridDetector',
  priority: 90, // High priority
  
  detect: function() {
    const agGridElements = [];
    
    // Strategy 1: Look for definitive AG Grid containers
    const rootWrappers = document.querySelectorAll('.ag-root-wrapper');
    if (rootWrappers.length > 0) {
      rootWrappers.forEach(wrapper => {
        // Verify it's an AG Grid by checking for child elements
        if (wrapper.querySelector('.ag-root, [role="grid"], .ag-body-viewport, .ag-header')) {
          agGridElements.push(wrapper);
        }
      });
    }
    
    // Strategy 2: Look for AG Grid roots directly
    if (agGridElements.length === 0) {
      const roots = document.querySelectorAll('.ag-root, [role="grid"].ag-root');
      if (roots.length > 0) {
        roots.forEach(root => {
          // Get the wrapper if it exists, otherwise use the root
          const wrapper = root.closest('.ag-root-wrapper') || root;
          if (!agGridElements.includes(wrapper)) {
            agGridElements.push(wrapper);
          }
        });
      }
    }
    
    // Strategy 3: Look for elements with ag-theme classes (common in all implementations)
    if (agGridElements.length === 0) {
      const themedElements = document.querySelectorAll('[class*="ag-theme-"]');
      themedElements.forEach(element => {
        // Verify it's an AG Grid by checking for characteristic children
        if (element.querySelector('.ag-root, .ag-header, .ag-body-viewport, [role="grid"], [role="row"][class*="ag-"]')) {
          if (!agGridElements.includes(element)) {
            agGridElements.push(element);
          }
        }
      });
    }
    
    // Strategy 4: Look for AG Grid body viewport (common in all implementations)
    if (agGridElements.length === 0) {
      const viewports = document.querySelectorAll('.ag-body-viewport');
      viewports.forEach(viewport => {
        // Find the closest grid container
        const container = viewport.closest('.ag-root-wrapper') || 
                         viewport.closest('.ag-root') || 
                         viewport.closest('[role="grid"]') || 
                         viewport.parentElement;
                         
        if (container && !agGridElements.includes(container)) {
          agGridElements.push(container);
        }
      });
    }
    
    // Strategy 5: Look for header containers
    if (agGridElements.length === 0) {
      const headers = document.querySelectorAll('.ag-header');
      headers.forEach(header => {
        // Find the closest grid container
        const container = header.closest('.ag-root-wrapper') || 
                         header.closest('.ag-root') || 
                         header.closest('[role="grid"]') || 
                         header.parentElement;
                         
        if (container && !agGridElements.includes(container)) {
          agGridElements.push(container);
        }
      });
    }
    
    // Strategy 6: Check inside iframes
    try {
      const iframes = document.querySelectorAll('iframe');
      iframes.forEach(iframe => {
        try {
          const iframeDoc = iframe.contentDocument || iframe.contentWindow.document;
          if (iframeDoc) {
            // Look for AG Grid in iframe using the main indicators
            const iframeGrids = iframeDoc.querySelectorAll('.ag-root-wrapper, .ag-root, [role="grid"].ag-root, .ag-body-viewport, .ag-header');
            if (iframeGrids.length > 0) {
              // Found AG Grid in iframe, add the iframe itself
              agGridElements.push(iframe);
            }
          }
        } catch (e) {
          // Silent fail for cross-origin iframes
        }
      });
    } catch (e) {
      console.log('Error checking iframes:', e);
    }
    
    return {
      elements: agGridElements,
      reliable: agGridElements.length > 0
    };
  }
};

  /**
   * Div Table Detector
   * Finds div structures that mimic tables (using CSS Grid or Flexbox)
   */
  const DivTableDetector = {
    name: 'DivTableDetector',
    priority: 80, // Medium-high priority
    
    detect: function() {
      const candidates = [];
      
      // Find potential container elements
      const containers = document.querySelectorAll('div, section, article');
      
      for (const container of containers) {
        // Skip if too small or empty
        if (container.children.length < 2) continue;
        
        // Check for grid/flex layout
        const style = window.getComputedStyle(container);
        const isGrid = style.display === 'grid';
        const isFlex = style.display === 'flex';
        
        if (isGrid || isFlex) {
          // Check for repeated structure in children
          const childStructure = Utilities.getChildrenStructure(container.children[0]);
          let consistentStructure = true;
          
          // Check if at least 3 children have similar structure
          if (container.children.length >= 3) {
            for (let i = 1; i < Math.min(container.children.length, 5); i++) {
              if (!Utilities.hasMatchingStructure(container.children[i], childStructure)) {
                consistentStructure = false;
                break;
              }
            }
            
            if (consistentStructure) {
              candidates.push(container);
            }
          }
        }
      }
      
      return {
        elements: candidates,
        reliable: candidates.length > 0
      };
    }
  };
  
  /**
   * List Detector
   * Finds list structures (ul/ol) that might contain tabular data
   */
  const ListDetector = {
    name: 'ListDetector',
    priority: 70, // Medium priority
    
    detect: function() {
      const lists = Array.from(document.querySelectorAll('ul, ol')).filter(list => {
        const items = list.querySelectorAll('li');
        
        // Need at least 3 items for a meaningful data structure
        if (items.length < 3) return false;
        
        // Check if list items have consistent structure
        const firstItemStructure = Utilities.getChildrenStructure(items[0]);
        
        for (let i = 1; i < Math.min(items.length, 5); i++) {
          if (!Utilities.hasMatchingStructure(items[i], firstItemStructure)) {
            return false;
          }
        }
        
        return true;
      });
      
      return {
        elements: lists,
        reliable: lists.length > 0
      };
    }
  };
  
  /**
   * Link Detector
   * Fallback detector that finds links as a simple data source
   */
  const LinkDetector = {
    name: 'LinkDetector',
    priority: 10, // Lowest priority - fallback option
    
    detect: function() {
      const links = document.querySelectorAll('a');
      
      // Only return links if we have enough to be meaningful
      return {
        elements: links.length >= 5 ? [document.body] : [],
        reliable: false // Links are never considered a reliable data source
      };
    }
  };

  /**
   * =============================================================================
   * EXTRACTORS
   * =============================================================================
   * Extractor modules pull structured data from elements found by detectors.
   * Each extractor corresponds to a specific detector.
   */
  
  /**
   * HTML Table Extractor
   * Extracts data from standard HTML <table> elements
   */
  const HTMLTableExtractor = {
    name: 'HTMLTableExtractor',
    forDetector: 'HTMLTableDetector',
    
    extract: function(detectionResult) {
      const extractedTables = [];
      
      detectionResult.elements.forEach((table, tableIndex) => {
        const tableData = {
          id: `table-${tableIndex}`,
          selector: Utilities.generateSelector(table),
          headers: [],
          rows: []
        };
        
        // Detect headers using comprehensive detection approach
        this.detectTableHeaders(table, tableData);
        
        // Extract rows
        this.extractTableRows(table, tableData);
        
        extractedTables.push(tableData);
      });
      
      return extractedTables;
    },
    
    /**
     * Comprehensive header detection for HTML tables
     * @param {HTMLElement} table - Table element
     * @param {Object} tableData - Table data object to populate
     */
    detectTableHeaders: function(table, tableData) {
      const headerCandidates = [];
      
      // Method 1: Check for semantic thead and th elements (highest confidence)
      const theadElement = table.querySelector('thead');
      if (theadElement) {
        const headerRows = theadElement.querySelectorAll('tr');
        if (headerRows.length > 0) {
          // Use first row in thead
          const headerCells = headerRows[0].querySelectorAll('th, td');
          headerCells.forEach((cell, index) => {
            const text = Utilities.cleanText(cell.textContent);
            headerCandidates[index] = {
              text: text,
              confidence: 10, // High confidence for thead elements
              source: 'thead'
            };
          });
        }
      } else {
        // Method 2: Look for th elements in the first row
        const firstRow = table.querySelector('tr');
        if (firstRow) {
          const thCells = firstRow.querySelectorAll('th');
          if (thCells.length > 0) {
            thCells.forEach((cell, index) => {
              const text = Utilities.cleanText(cell.textContent);
              headerCandidates[index] = {
                text: text,
                confidence: 8, // Good confidence for th elements
                source: 'th'
              };
            });
          } else {
            // Method 3: Use first row cells with styling analysis
            const firstRowCells = firstRow.querySelectorAll('td');
            firstRowCells.forEach((cell, index) => {
              const text = Utilities.cleanText(cell.textContent);
              
              // Calculate confidence based on styling and content
              let confidence = 5; // Base confidence for first row
              
              // Check styling indicators
              const style = window.getComputedStyle(cell);
              if (style.fontWeight === 'bold' || parseInt(style.fontWeight) >= 600) confidence += 1;
              if (style.backgroundColor !== 'rgba(0, 0, 0, 0)' && style.backgroundColor !== 'transparent') confidence += 1;
              if (style.textAlign === 'center') confidence += 0.5;
              
              // Check content indicators
              if (Utilities.isLikelyHeader(text)) confidence += 1.5;
              
              headerCandidates[index] = {
                text: text,
                confidence: confidence,
                source: 'firstRow'
              };
            });
          }
        }
      }
      
      // Method 4: Check for ARIA roles
      const ariaHeaders = table.querySelectorAll('[role="columnheader"], [role="rowheader"]');
      if (ariaHeaders.length > 0) {
        ariaHeaders.forEach(header => {
          // Try to determine the column index
          const headerRow = header.closest('tr');
          if (headerRow) {
            const cells = headerRow.querySelectorAll('td, th');
            for (let i = 0; i < cells.length; i++) {
              if (cells[i] === header) {
                const text = Utilities.cleanText(header.textContent);
                
                // Only override if confidence is higher
                const existingConfidence = headerCandidates[i] ? headerCandidates[i].confidence : 0;
                if (existingConfidence < 9) {
                  headerCandidates[i] = {
                    text: text,
                    confidence: 9, // High confidence for ARIA roles
                    source: 'aria'
                  };
                }
                break;
              }
            }
          }
        });
      }
      
      // Find the maximum number of cells in any row
      const rows = table.querySelectorAll('tr');
      let maxCells = 0;
      
      rows.forEach(row => {
        const cellCount = row.querySelectorAll('td, th').length;
        if (cellCount > maxCells) maxCells = cellCount;
      });
      
      // Create final header array from candidates
      for (let i = 0; i < maxCells; i++) {
        if (headerCandidates[i] && headerCandidates[i].text) {
          tableData.headers.push(headerCandidates[i].text);
        } else {
          // Generate generic header name
          tableData.headers.push(`Column ${i+1}`);
        }
      }
    },
    
    /**
     * Extract rows from HTML table
     * @param {HTMLElement} table - Table element
     * @param {Object} tableData - Table data object to populate
     */
    extractTableRows: function(table, tableData) {
      // Determine where to start extracting rows
      let startIndex = 0;
      
      // Skip header row if it's in the table body (not in thead)
      const hasHeaderRowInBody = !table.querySelector('thead') && 
                                (table.querySelector('tr th') || 
                                table.querySelector('tr [role="columnheader"]'));
      
      if (hasHeaderRowInBody) {
        startIndex = 1;
      }
      
      // Get all rows
      const rows = table.querySelectorAll('tbody tr, tr');
      
      // Extract data from each row
      for (let i = startIndex; i < rows.length; i++) {
        const row = rows[i];
        
        // Skip header rows
        if (row.parentElement && row.parentElement.tagName === 'THEAD') continue;
        
        const rowData = [];
        const cells = row.querySelectorAll('td, th');
        
        cells.forEach(cell => {
          rowData.push(Utilities.cleanText(cell.textContent));
        });
        
        // Only add non-empty rows
        if (rowData.length > 0 && rowData.some(cell => cell)) {
          tableData.rows.push(rowData);
        }
      }
    }
  };
  
/**
 * AG Grid Extractor
 * Extracts data from AG Grid components, handling virtualized content with multi-pass approach
 */
const AGGridExtractor = {
  name: 'AGGridExtractor',
  forDetector: 'AGGridDetector',
  
  extract: async function(detectionResult) {
    const extractedTables = [];
    
    // Process each detected AG Grid
    for (let gridIndex = 0; gridIndex < detectionResult.elements.length; gridIndex++) {
      const gridElement = detectionResult.elements[gridIndex];
      
      const tableData = {
        id: `ag-grid-${gridIndex}`,
        selector: Utilities.generateSelector(gridElement),
        headers: [],
        rows: []
      };
      
      // Step 1: Extract headers
      this.extractHeaders(gridElement, tableData);
      
      // Step 2: Extract visible rows first
      this.extractVisibleRows(gridElement, tableData);
      
      // Step 3: If scrolling is enabled, try to reveal more rows with multi-pass approach
      if (WebDataExtractor.config.scrollForVirtualized) {
        await this.multiPassExtraction(gridElement, tableData);
      }
      
      // Only add grids that have both headers and rows
      if (tableData.headers.length > 0 && tableData.rows.length > 0) {
        extractedTables.push(tableData);
      }
    }
    
    return extractedTables;
  },
  
/**
 * Extract headers from AG Grid
 * @param {HTMLElement} gridElement - AG Grid element
 * @param {Object} tableData - Table data object to populate
 */
extractHeaders: function(gridElement, tableData) {
  // Strategy 1: Look for header cells with role="columnheader"
  const headerCells = gridElement.querySelectorAll('[role="columnheader"], .ag-header-cell');
  
  // First, collect all header information with positions
  const headerInfo = [];
  
  if (headerCells.length > 0) {
    headerCells.forEach((cell, index) => {
      // Try to get the left position to understand visual order
      const style = window.getComputedStyle(cell);
      const leftPos = parseInt(style.left) || parseInt(style.getPropertyValue('left')) || 0;
      
      // Try various ways to get header text
      const headerLabel = cell.querySelector('.ag-header-cell-text, .ag-header-cell-label');
      const cellValue = cell.querySelector('.ag-cell-value');
      
      let headerText = '';
      
      if (headerLabel) {
        headerText = Utilities.cleanText(headerLabel.textContent);
      } else if (cellValue) {
        headerText = Utilities.cleanText(cellValue.textContent);
      } else {
        // Text extraction that filters out menu buttons and icons
        const walker = document.createTreeWalker(
          cell,
          NodeFilter.SHOW_TEXT,
          {
            acceptNode: function(node) {
              if (node.parentElement && 
                 (node.parentElement.classList.contains('ag-header-cell-menu-button') ||
                  node.parentElement.classList.contains('ag-header-icon'))) {
                return NodeFilter.FILTER_REJECT;
              }
              return NodeFilter.FILTER_ACCEPT;
            }
          },
          false
        );
        
        let cellTextContent = '';
        while (walker.nextNode()) {
          cellTextContent += walker.currentNode.textContent;
        }
        
        headerText = Utilities.cleanText(cellTextContent);
      }
      
      if (headerText) {
        // Store visual position information along with header text
        headerInfo.push({
          text: headerText,
          domIndex: index,
          visualPos: leftPos,
          colId: cell.getAttribute('col-id') || `col-${index}`
        });
      }
    });
    
    // Sort headers by visual position (left CSS value) to match what the user sees
    headerInfo.sort((a, b) => a.visualPos - b.visualPos);
    
    // Store column mappings for data extraction
    tableData.columnMappings = headerInfo.map(info => info.colId);
    
    // Set the headers in the tableData
    tableData.headers = headerInfo.map(info => info.text);
    
    // Store visual to DOM index mapping for use in data extraction
    tableData.visualToDomIndex = {};
    headerInfo.forEach((info, visualIndex) => {
      tableData.visualToDomIndex[visualIndex] = info.domIndex;
    });
  }
  
  // Strategy 2: Look for header container if no headers found yet
  if (tableData.headers.length === 0) {
    // Try to find any elements in the header that might contain header text
    const headerContainers = gridElement.querySelectorAll(
      '.ag-header, .ag-header-container, .ag-header-row, .ag-header-row-column'
    );
    
    if (headerContainers.length > 0) {
      // Process each container to find potential header texts
      headerContainers.forEach(container => {
        // Look for specific header text elements first
        const textElements = container.querySelectorAll(
          '.ag-header-cell-text, span:not(.ag-icon), div:not(.ag-header-cell-menu-button)'
        );
        
        if (textElements.length > 0) {
          const texts = [];
          
          textElements.forEach((el, index) => {
            // Try to get visual position
            const style = window.getComputedStyle(el);
            const leftPos = parseInt(style.left) || parseInt(style.getPropertyValue('left')) || index * 100; // Fallback to index-based position
            
            // Only include direct text, not from child elements with their own headers
            if (el.children.length === 0 || !el.querySelector('.ag-header-cell-text')) {
              const text = Utilities.cleanText(el.textContent);
              if (text && text.length < 50) {
                texts.push({
                  text: text,
                  visualPos: leftPos,
                  domIndex: index,
                  colId: `col-${index}`
                });
              }
            }
          });
          
          if (texts.length > 0) {
            // Sort by visual position
            texts.sort((a, b) => a.visualPos - b.visualPos);
            
            // Set headers and mappings
            tableData.headers = texts.map(t => t.text);
            tableData.columnMappings = texts.map(t => t.colId);
            
            // Store visual mapping
            tableData.visualToDomIndex = {};
            texts.forEach((info, visualIndex) => {
              tableData.visualToDomIndex[visualIndex] = info.domIndex;
            });
          }
        }
      });
    }
  }
  
  // Strategy 3: If still no headers, infer from row cell structure
  if (tableData.headers.length === 0) {
    // Check visible cells to determine column count
    const firstRow = gridElement.querySelector('.ag-row');
    if (firstRow) {
      const cells = firstRow.querySelectorAll('.ag-cell');
      if (cells.length > 0) {
        const cellInfo = [];
        
        // Collect position information for cells
        cells.forEach((cell, index) => {
          const style = window.getComputedStyle(cell);
          const leftPos = parseInt(style.left) || parseInt(style.getPropertyValue('left')) || index * 100;
          
          cellInfo.push({
            visualPos: leftPos,
            domIndex: index,
            colId: `col-${index}`
          });
        });
        
        // Sort by visual position
        cellInfo.sort((a, b) => a.visualPos - b.visualPos);
        
        // Generate column mappings
        tableData.columnMappings = cellInfo.map(info => info.colId);
        tableData.visualToDomIndex = {};
        
        cellInfo.forEach((info, visualIndex) => {
          tableData.visualToDomIndex[visualIndex] = info.domIndex;
          
          // Generate generic headers
          if (!tableData.headers[visualIndex]) {
            if (!tableData.headers) tableData.headers = [];
            tableData.headers[visualIndex] = `Column ${visualIndex+1}`;
          }
        });
      }
    }
  }
  
  // Clean up header names if needed (e.g., remove "Name 1" -> "Name")
  if (tableData.headers.length > 0) {
    tableData.headers = tableData.headers.map(header => {
      // Remove numeric suffixes from header names (e.g., "Name 1" -> "Name")
      return header.replace(/\s+\d+$/, '');
    });
  }
},  
  /**
   * Extract visible rows from AG Grid
   * @param {HTMLElement} gridElement - AG Grid element
   * @param {Object} tableData - Table data object to populate
   */
  extractVisibleRows: function(gridElement, tableData) {
    // Use a Map to ensure we don't have duplicate rows
    const uniqueRows = new Map();
    
    // Find all visible rows
    const rows = gridElement.querySelectorAll('.ag-row, [role="row"]');
    
    // Flag to track if we've detected shifted data
    let hasShiftedData = false;
    
    // First, analyze a sample of rows to detect if data appears to be shifted
    const sampleRows = Array.from(rows).slice(0, Math.min(10, rows.length));
    const sampleData = [];
    
    sampleRows.forEach(row => {
      // Skip header rows
      if (row.querySelector('[role="columnheader"]') || 
          row.getAttribute('aria-rowindex') === '1' || 
          row.classList.contains('ag-header-row')) {
        return;
      }
      
      // Create a unique identifier for this row
      const rowId = row.getAttribute('row-id') || 
                   row.getAttribute('row-index') || 
                   row.getAttribute('aria-rowindex') ||
                   `row-${uniqueRows.size}`;
      
      // Skip if we already processed this row
      if (uniqueRows.has(rowId)) return;
      
      // Extract row data for analysis
      const rowData = this.extractRowData(row, tableData);
      if (rowData.some(cell => cell)) {
        sampleData.push(rowData);
      }
    });
    
    // Analyze sample data for consistent patterns of shifted columns
    if (sampleData.length > 0) {
      // Count how many rows have empty columns at the start
      const emptyStartCounts = {};
      
      sampleData.forEach(row => {
        const firstDataIndex = row.findIndex(cell => cell && cell.trim() !== '');
        if (firstDataIndex > 0) {
          emptyStartCounts[firstDataIndex] = (emptyStartCounts[firstDataIndex] || 0) + 1;
        }
      });
      
      // Find the most common pattern of empty start columns
      let mostCommonEmptyStart = 0;
      let maxCount = 0;
      
      Object.entries(emptyStartCounts).forEach(([emptyStart, count]) => {
        if (count > maxCount) {
          maxCount = count;
          mostCommonEmptyStart = parseInt(emptyStart);
        }
      });
      
      // If we have a consistent pattern across multiple rows, apply this correction to all rows
      if (mostCommonEmptyStart > 0 && maxCount >= Math.ceil(sampleData.length * 0.6)) {
        console.log(`Detected shifted data pattern: ${mostCommonEmptyStart} empty columns at start. Will adjust all rows.`);
        hasShiftedData = true;
        tableData.dataShiftCorrection = mostCommonEmptyStart;
      }
    }
    
    // Clear uniqueRows for actual processing
    uniqueRows.clear();
    
    // Now process all rows with our knowledge of data structure
    rows.forEach(row => {
      // Skip header rows
      if (row.querySelector('[role="columnheader"]') || 
          row.getAttribute('aria-rowindex') === '1' || 
          row.classList.contains('ag-header-row')) {
        return;
      }
      
      // Create a unique identifier for this row
      const rowId = row.getAttribute('row-id') || 
                   row.getAttribute('row-index') || 
                   row.getAttribute('aria-rowindex') ||
                   `row-${uniqueRows.size}`;
      
      // Skip if we already processed this row
      if (uniqueRows.has(rowId)) return;
      
      // Extract row data
      let rowData = this.extractRowData(row, tableData);
      
      // Apply consistent shift correction if needed
      if (hasShiftedData && tableData.dataShiftCorrection > 0) {
        rowData = this.shiftRowData(rowData, tableData.dataShiftCorrection);
      }
      
      // Only add non-empty rows
      if (rowData.length > 0 && rowData.some(cell => cell && cell.trim() !== '')) {
        // Generate a unique key based on non-empty content
        const key = rowData.filter(cell => cell && cell.trim() !== '').join('|');
        if (key) {
          uniqueRows.set(key, rowData);
        }
      }
    });
    
    // Convert Map to array and add to tableData
    tableData.rows = [...uniqueRows.values()];
    
    // If we found rows but no headers, generate generic headers
    if (tableData.headers.length === 0 && tableData.rows.length > 0) {
      const columnCount = Math.max(...tableData.rows.map(row => row.length));
      
      for (let i = 0; i < columnCount; i++) {
        tableData.headers.push(`Column ${i+1}`);
      }
    }
    
    // Normalize row lengths to match header count
    this.normalizeRowLengths(tableData);
  },
  
  /**
   * Shift row data to correct alignment
   * @param {Array} rowData - Array of cell values
   * @param {number} shiftAmount - Number of positions to shift left
   * @returns {Array} Corrected array of cell values
   */
  shiftRowData: function(rowData, shiftAmount) {
    const result = Array(rowData.length).fill('');
    
    // Shift data leftward by shiftAmount positions
    for (let i = shiftAmount; i < rowData.length; i++) {
      const targetIndex = i - shiftAmount;
      if (targetIndex >= 0 && targetIndex < result.length) {
        result[targetIndex] = rowData[i];
      }
    }
    
    return result;
  },
  
/**
 * Extract data from a single AG Grid row
 * @param {HTMLElement} row - Row element
 * @param {Object} tableData - Table data object (for column mappings)
 * @returns {Array} Array of cell values
 */
extractRowData: function(row, tableData) {
  const headerCount = tableData.headers.length;
  const rowData = Array(headerCount).fill('');
  
  // Get all cells in the row as an array for easier manipulation
  const cells = Array.from(row.querySelectorAll('.ag-cell, [role="gridcell"], [col-id]'));
  
  // Collect cell information with positions
  const cellInfo = [];
  
  cells.forEach((cell, domIndex) => {
    // Get visual position from style
    const style = window.getComputedStyle(cell);
    const leftPos = parseInt(style.left) || parseInt(style.getPropertyValue('left')) || 0;
    
    // Extract text content from the cell
    let cellText = '';
    
    // Try to get from value elements first
    const valueElement = cell.querySelector('.ag-cell-value, .text-overflow');
    if (valueElement) {
      cellText = Utilities.cleanText(valueElement.textContent);
    } else {
      // Get all text nodes directly
      const walker = document.createTreeWalker(
        cell, 
        NodeFilter.SHOW_TEXT,
        {
          acceptNode: (node) => {
            const parent = node.parentElement;
            if (parent && (
                parent.classList.contains('ag-hidden') || 
                parent.style.display === 'none')) {
              return NodeFilter.FILTER_REJECT;
            }
            return NodeFilter.FILTER_ACCEPT;
          }
        }
      );
      
      // Collect text from all text nodes
      while (walker.nextNode()) {
        cellText += walker.currentNode.textContent;
      }
      
      cellText = Utilities.cleanText(cellText);
    }
    
    // Store cell information with position
    cellInfo.push({
      text: cellText,
      domIndex: domIndex,
      visualPos: leftPos,
      colId: cell.getAttribute('col-id')
    });
  });
  
  // Sort cells by visual position to match what the user sees
  cellInfo.sort((a, b) => a.visualPos - b.visualPos);
  
  // Map cells to columns based on visual position
  cellInfo.forEach((info, visualIndex) => {
    if (visualIndex < headerCount) {
      rowData[visualIndex] = info.text;
    }
  });
  
  // Special handling for Name column, if we detect it's in the wrong place
  const nameIndex = tableData.headers.findIndex(h => h.toLowerCase().includes('name'));
  if (nameIndex > 0 && rowData[0] === '' && rowData[nameIndex] !== '') {
    // If we have a name in a different column but first column is empty, shift it to participant
    if (tableData.headers[0].toLowerCase().includes('participant')) {
      rowData[0] = rowData[nameIndex];
      rowData[nameIndex] = '';
    }
  }
  
  return rowData;
},

  
  /**
   * Multi-pass extraction with thorough scrolling to reveal virtualized content
   * @param {HTMLElement} gridElement - AG Grid element
   * @param {Object} tableData - Table data object to populate
   */
  multiPassExtraction: async function(gridElement, tableData) {
    // Find the scrollable viewport
    const viewport = gridElement.querySelector('.ag-body-viewport');
    
    if (!viewport) {
      console.log('No scrollable viewport found for AG Grid');
      return;
    }
    
    // Store original scroll position to restore later
    const originalScrollTop = viewport.scrollTop;
    const scrollHeight = viewport.scrollHeight;
    
    // Use a Map to track unique rows
    // IMPORTANT: We use a custom row key function that ignores empty cells
    // This helps prevent duplicate detection issues caused by partial data loading
    const uniqueRowsMap = new Map();
    
    // Function to generate a more robust key for a row
    const generateRowKey = (rowData) => {
      // Filter out empty cells and join with a separator unlikely to appear in the data
      return rowData.filter(cell => cell && cell.trim() !== '').join('||❖||');
    };
    
    // Initialize with existing rows
    tableData.rows.forEach(row => {
      const key = generateRowKey(row);
      if (key) { // Only add rows with actual content
        uniqueRowsMap.set(key, row);
      }
    });
    
    try {
      console.log(`Starting multi-pass extraction for AG Grid (height: ${scrollHeight}px)`);
      
      // Try to identify any sorting arrows and sort the grid if possible
      try {
        const sortableHeaders = gridElement.querySelectorAll('.ag-header-cell[aria-sort="none"], .ag-header-cell:not([aria-sort])');
        if (sortableHeaders.length > 0) {
          // Try to click a column that might have reliable sorting (like ID, Name, etc.)
          const preferredHeaders = Array.from(sortableHeaders).filter(header => {
            const text = header.textContent.toLowerCase();
            return text.includes('id') || text.includes('name') || text.includes('date');
          });
          
          const headerToClick = preferredHeaders.length > 0 ? preferredHeaders[0] : sortableHeaders[0];
          console.log('Sorting grid by column:', headerToClick.textContent.trim());
          
          // Click to sort
          headerToClick.click();
          await Utilities.delay(800); // Wait for sort to complete
        }
      } catch (e) {
        console.log('Error attempting to sort grid:', e);
      }
      
      // THOROUGH MULTI-PASS EXTRACTION
      // Calculate scroll steps based on grid height
      const scrollSteps = Math.max(10, Math.min(30, Math.floor(scrollHeight / 400)));
      const scrollIncrement = Math.floor(scrollHeight / scrollSteps);
      
      console.log(`Using ${scrollSteps} scroll positions with increment of ${scrollIncrement}px`);
      
      // Perform multiple passes with different scroll patterns - limit to 2 passes
      for (let pass = 1; pass <= 2; pass++) {
        console.log(`Starting pass ${pass} of grid extraction`);
        
        // Each pass uses different scroll steps to try to reveal different content
        const increment = pass === 1 ? scrollIncrement : Math.floor(scrollIncrement / 2);
        const startOffset = pass === 1 ? 0 : Math.floor(increment / 2);
        
        // Additional delay for later passes to allow grid to settle
        const delayTime = pass === 1 ? 300 : 400;
        
        for (let position = startOffset; position <= scrollHeight; position += increment) {
          // Set scroll position
          viewport.scrollTop = position;
          
          // Wait for rendering
          await Utilities.delay(delayTime);
          
          // Extract newly visible rows
          const rows = gridElement.querySelectorAll('.ag-row, [role="row"]');
          let newRowsInStep = 0;
          
          // Process rows
          for (const row of rows) {
            // Skip header rows
            if (row.querySelector('[role="columnheader"]') || 
                row.getAttribute('aria-rowindex') === '1' || 
                row.classList.contains('ag-header-row')) {
              continue;
            }
            
            // Extract row data
            const rowData = this.extractRowData(row, tableData);
            
            // Only add non-empty rows
            if (rowData.length > 0 && rowData.some(cell => cell && cell.trim() !== '')) {
              const key = generateRowKey(rowData);
              if (key && !uniqueRowsMap.has(key)) {
                uniqueRowsMap.set(key, rowData);
                newRowsInStep++;
              }
            }
          }
          
          console.log(`Pass ${pass}: Position ${position}/${scrollHeight}, found ${newRowsInStep} new rows, total: ${uniqueRowsMap.size}`);
        }
        
        // Extra check at the very bottom
        viewport.scrollTop = scrollHeight;
        await Utilities.delay(delayTime + 200);
        
        // Check rows at bottom
        const bottomRows = gridElement.querySelectorAll('.ag-row, [role="row"]');
        bottomRows.forEach(row => {
          if (!row.querySelector('[role="columnheader"]') && 
              row.getAttribute('aria-rowindex') !== '1' && 
              !row.classList.contains('ag-header-row')) {
            
            const rowData = this.extractRowData(row, tableData);
            if (rowData.some(cell => cell && cell.trim() !== '')) {
              const key = generateRowKey(rowData);
              if (key && !uniqueRowsMap.has(key)) {
                uniqueRowsMap.set(key, rowData);
              }
            }
          }
        });
      }
      
      // Final check at the top to ensure we get the first rows
      viewport.scrollTop = 0;
      await Utilities.delay(400);
      
      // Check rows at top once more
      const topRows = gridElement.querySelectorAll('.ag-row, [role="row"]');
      topRows.forEach(row => {
        if (!row.querySelector('[role="columnheader"]') && 
            row.getAttribute('aria-rowindex') !== '1' && 
            !row.classList.contains('ag-header-row')) {
          
          const rowData = this.extractRowData(row, tableData);
          if (rowData.some(cell => cell && cell.trim() !== '')) {
            const key = generateRowKey(rowData);
            if (key && !uniqueRowsMap.has(key)) {
              uniqueRowsMap.set(key, rowData);
            }
          }
        }
      });
      
      // Update tableData with all unique rows
      tableData.rows = [...uniqueRowsMap.values()];
      
      // Normalize row lengths again with any new data
      this.normalizeRowLengths(tableData);
      
      // Post-extraction cleanup: remove rows that are entirely empty or have too many empty cells
      const headerCount = tableData.headers.length;
      tableData.rows = tableData.rows.filter(row => {
        const nonEmptyCells = row.filter(cell => cell && cell.trim() !== '').length;
        return nonEmptyCells >= Math.max(2, Math.floor(headerCount * 0.2)); // At least 2 cells or 20% of headers
      });
      
      console.log(`Multi-pass extraction complete. Total rows after cleanup: ${tableData.rows.length}`);
      
    } catch (e) {
      console.error('Error during multi-pass extraction:', e);
    } finally {
      // Restore original scroll position
      viewport.scrollTop = originalScrollTop;
    }
  },
  
  /**
   * Ensure all rows have the same length as the headers
   * @param {Object} tableData - Table data object to normalize
   */
  normalizeRowLengths: function(tableData) {
    if (!tableData.headers || tableData.headers.length === 0) return;
    
    const headerCount = tableData.headers.length;
    
    tableData.rows = tableData.rows.map(row => {
      // If row is too short, pad with empty strings
      if (row.length < headerCount) {
        return [...row, ...Array(headerCount - row.length).fill('')];
      }
      // If row is too long, trim
      else if (row.length > headerCount) {
        return row.slice(0, headerCount);
      }
      return row;
    });
  }
};




  /**
   * Div Table Extractor
   * Extracts data from div structures that mimic tables
   */
  const DivTableExtractor = {
    name: 'DivTableExtractor',
    forDetector: 'DivTableDetector',
    
    extract: function(detectionResult) {
      const extractedTables = [];
      
      detectionResult.elements.forEach((container, tableIndex) => {
        const tableData = {
          id: `div-table-${tableIndex}`,
          selector: Utilities.generateSelector(container),
          headers: [],
          rows: []
        };
        
        // Step 1: Extract headers
        this.detectHeaders(container, tableData);
        
        // Step 2: Extract rows
        this.extractRows(container, tableData);
        
        // Only add tables with data
        if (tableData.headers.length > 0 && tableData.rows.length > 0) {
          extractedTables.push(tableData);
        }
      });
      
      return extractedTables;
    },
    
    /**
     * Detect headers in a div-based table
     * @param {HTMLElement} container - Container element
     * @param {Object} tableData - Table data object to populate
     */
    detectHeaders: function(container, tableData) {
      const headerCandidates = [];
      let maxColumnCount = 0;
      
      // Method 1: Look for semantic header elements
      const semanticHeaders = container.querySelectorAll('[role="columnheader"], [role="rowheader"], [class*="header"], [class*="heading"], [class*="title"]');
      
      if (semanticHeaders.length > 0) {
        // Check if these are in the first row
        const firstItem = container.children[0];
        const headersInFirstRow = Array.from(semanticHeaders).filter(header => firstItem.contains(header));
        
        if (headersInFirstRow.length > 0) {
          headersInFirstRow.forEach((header, index) => {
            headerCandidates[index] = Utilities.cleanText(header.textContent);
          });
          
          maxColumnCount = Math.max(maxColumnCount, headersInFirstRow.length);
        } else {
          // Headers might be in their own containers
          semanticHeaders.forEach((header, index) => {
            headerCandidates[index] = Utilities.cleanText(header.textContent);
          });
          
          maxColumnCount = Math.max(maxColumnCount, semanticHeaders.length);
        }
      }
      
      // Method 2: Analyze styling of first row elements
      if (container.children.length > 0) {
        const firstItem = container.children[0];
        const potentialHeaderElements = firstItem.querySelectorAll('div, span, p, h1, h2, h3, h4, h5, h6');
        
        if (potentialHeaderElements.length > 0) {
          potentialHeaderElements.forEach((element, index) => {
            // Skip if we already have a header for this position
            if (headerCandidates[index]) return;
            
            const text = Utilities.cleanText(element.textContent);
            
            // Check styling
            const style = window.getComputedStyle(element);
            const isBold = style.fontWeight === 'bold' || parseInt(style.fontWeight) >= 600;
            const hasBackground = style.backgroundColor !== 'rgba(0, 0, 0, 0)' && style.backgroundColor !== 'transparent';
            
            // If it looks like a header by styling or content
            if (isBold || hasBackground || Utilities.isLikelyHeader(text)) {
              headerCandidates[index] = text;
            }
          });
          
          maxColumnCount = Math.max(maxColumnCount, potentialHeaderElements.length);
        }
      }
      
      // Create final header array, filling in any gaps
      for (let i = 0; i < maxColumnCount; i++) {
        if (headerCandidates[i]) {
          tableData.headers.push(headerCandidates[i]);
        } else {
          tableData.headers.push(`Column ${i+1}`);
        }
      }
    },
    
    /**
     * Extract rows from a div-based table
     * @param {HTMLElement} container - Container element
     * @param {Object} tableData - Table data object to populate
     */
    extractRows: function(container, tableData) {
      // Start from the second child, assuming first is headers
      const startIndex = 1;
      
      for (let i = startIndex; i < container.children.length; i++) {
        const row = container.children[i];
        const rowData = [];
        
        // If it's a simple structure, extract text directly
        if (row.children.length === 0) {
          rowData.push(Utilities.cleanText(row.textContent));
        } else {
          // Extract from child elements
          const cells = row.querySelectorAll('div, span, p');
          cells.forEach(cell => {
            rowData.push(Utilities.cleanText(cell.textContent));
          });
        }
        
        // Only add non-empty rows
        if (rowData.length > 0 && rowData.some(cell => cell)) {
          tableData.rows.push(rowData);
        }
      }
      
      // Normalize row lengths to match header count
      if (tableData.headers.length > 0) {
        const headerCount = tableData.headers.length;
        
        tableData.rows = tableData.rows.map(row => {
          // If row is too short, pad with empty strings
          if (row.length < headerCount) {
            return [...row, ...Array(headerCount - row.length).fill('')];
          }
          // If row is too long, trim
          else if (row.length > headerCount) {
            return row.slice(0, headerCount);
          }
          return row;
        });
      }
    }
  };
  
  /**
   * List Extractor
   * Extracts data from list structures (ul/ol)
   */
  const ListExtractor = {
    name: 'ListExtractor',
    forDetector: 'ListDetector',
    
    extract: function(detectionResult) {
      const extractedTables = [];
      
      detectionResult.elements.forEach((list, listIndex) => {
        const tableData = {
          id: `list-table-${listIndex}`,
          selector: Utilities.generateSelector(list),
          headers: [],
          rows: []
        };
        
        const items = list.querySelectorAll('li');
        
        // Determine headers from first item
        if (items.length > 0) {
          const firstItem = items[0];
          
          // Check if the item has labeled parts (strong, b, em, etc.)
          const labels = firstItem.querySelectorAll('strong, b, em, i, [class*="label"], [class*="title"]');
          
          if (labels.length > 0) {
            labels.forEach(label => {
              tableData.headers.push(Utilities.cleanText(label.textContent));
            });
          } else {
            // If no labels, try to infer structure from content
            tableData.headers = ['Item'];
            if (firstItem.querySelectorAll('a').length > 0) {
              tableData.headers.push('Link');
            }
          }
        }
        
        // Extract rows from list items
        items.forEach(item => {
          if (tableData.headers.length === 1) {
            // Simple list
            tableData.rows.push([Utilities.cleanText(item.textContent)]);
          } else if (tableData.headers.includes('Link')) {
            // List with links
            const link = item.querySelector('a');
            if (link) {
              tableData.rows.push([
                Utilities.cleanText(link.textContent),
                link.href
              ]);
            } else {
              tableData.rows.push([Utilities.cleanText(item.textContent), '']);
            }
          } else {
            // Complex list items with multiple parts
            const rowData = [];
            
            // Try to extract data based on labels or structure
            const parts = item.querySelectorAll('div, span, p, a');
            parts.forEach(part => {
              rowData.push(Utilities.cleanText(part.textContent));
            });
            
            if (rowData.length === 0) {
              rowData.push(Utilities.cleanText(item.textContent));
            }
            
            tableData.rows.push(rowData);
          }
        });
        
        extractedTables.push(tableData);
      });
      
      return extractedTables;
    }
  };
  
  /**
   * Link Extractor
   * Extracts links as a fallback data source
   */
  const LinkExtractor = {
    name: 'LinkExtractor',
    forDetector: 'LinkDetector',
    
    extract: function(detectionResult) {
      const links = document.querySelectorAll('a');
      
      const tableData = {
        id: 'all-links',
        selector: 'a',
        headers: ['Text', 'URL'],
        rows: []
      };
      
      links.forEach(link => {
        const text = Utilities.cleanText(link.textContent);
        if (text) {
          tableData.rows.push([text, link.href]);
        }
      });
      
      return [tableData];
    }
  };

  /**
   * =============================================================================
   * INITIALIZATION
   * =============================================================================
   * Register all modules and launch the extraction process
   */
  
  // Register all detectors
  WebDataExtractor.registerDetector(HTMLTableDetector);
  WebDataExtractor.registerDetector(AGGridDetector);
  WebDataExtractor.registerDetector(DivTableDetector);
  WebDataExtractor.registerDetector(ListDetector);
  WebDataExtractor.registerDetector(LinkDetector);
  
  // Register all extractors
  WebDataExtractor.registerExtractor(HTMLTableExtractor);
  WebDataExtractor.registerExtractor(AGGridExtractor);
  WebDataExtractor.registerExtractor(DivTableExtractor);
  WebDataExtractor.registerExtractor(ListExtractor);
  WebDataExtractor.registerExtractor(LinkExtractor);
  
  // Run the extraction process - we'll use an async wrapper to handle the Promise
  (async function() {
    try {
      const data = await WebDataExtractor.run();
      console.log('Web Data Extractor completed successfully');
    } catch (error) {
      console.error('Error running Web Data Extractor:', error);
    }
  })();
  
  // Export the extractor to global scope for external access
  window.WebDataExtractor = WebDataExtractor;
  
  console.log('Web Data Extractor initialized and running...');
})();