Get_Well_API_Data.py

# -*- coding: utf-8 -*-
"""
Created on Mon Aug 17 18:25:11 2026

@author: epotts

Code to extract large amounts of Oil and Gas Well Data from WellFinder
"""

import requests
from bs4 import BeautifulSoup
import re
import time
import os
from datetime import datetime


# ============================================================
# SETTINGS
# ============================================================

# Update API Well List
APIS = [
    "403726384",
    "403705121",
    "403726339",
    "407100115",
    "407100153",
]

# Update for your email!
EMAIL = "epotts@toddgroundwater.com"

BASE_URL = "https://filerequest.conservation.ca.gov"


# ============================================================
# LOG FILE
# ============================================================

timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

LOG_FILE = os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    f"Well_File_Request_Log_{timestamp}.txt"
)


def log(message=""):
    """
    Write detailed information to the TXT log file.
    """

    with open(
        LOG_FILE,
        "a",
        encoding="utf-8"
    ) as f:

        f.write(str(message) + "\n")


# ============================================================
# SESSION
# ============================================================

session = requests.Session()

session.headers.update({
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) "
        "Gecko/20100101 Firefox/153.0"
    ),
    "Accept-Language": "en-US,en;q=0.9",
})


# ============================================================
# GET ALL FILES FOR A WELL
# ============================================================

def get_files_for_well(csv_api):

    site_api = csv_api[1:]

    url = f"{BASE_URL}/WellRecord?api={site_api}"

    log()
    log("=" * 80)
    log("STEP 1 — GET WELL RECORD")
    log("=" * 80)
    log()
    log(f"CSV API:   {csv_api}")
    log(f"SITE API:  {site_api}")
    log(f"URL:       {url}")
    log()

    response = session.get(
        url,
        timeout=30
    )

    log(
        f"Status: {response.status_code}"
    )

    if response.status_code != 200:

        log(
            "ERROR: Could not load WellRecord."
        )

        return []

    soup = BeautifulSoup(
        response.text,
        "html.parser"
    )

    files = []

    # --------------------------------------------------------
    # Find all checkboxes whose onchange contains:
    #
    # ToggleItemSelected(1234567)
    # --------------------------------------------------------

    checkboxes = soup.find_all(
        "input",
        {
            "type": "checkbox"
        }
    )

    for checkbox in checkboxes:

        onchange = checkbox.get(
            "onchange",
            ""
        )

        match = re.search(
            r"ToggleItemSelected\((\d+)\)",
            onchange
        )

        if not match:
            continue

        file_id = match.group(1)

        # ----------------------------------------------------
        # Find the containing table row
        # ----------------------------------------------------

        row = checkbox.find_parent("tr")

        filename = ""

        if row:

            cells = row.find_all("td")

            if cells:

                filename = cells[0].get_text(
                    " ",
                    strip=True
                )

        files.append({
            "id": file_id,
            "filename": filename
        })

    # --------------------------------------------------------
    # Log results
    # --------------------------------------------------------

    log()
    log(
        f"FILES FOUND: {len(files)}"
    )
    log()

    for i, file in enumerate(files, 1):

        log(
            f"{i:3d}. "
            f"ID={file['id']}   "
            f"{file['filename']}"
        )

    return files


# ============================================================
# SELECT A FILE
# ============================================================

def select_file(
    file_id,
    well_api
):

    url = f"{BASE_URL}/IsFileSelectedToggle"

    headers = {
        "X-Requested-With": "XMLHttpRequest",

        "Referer": (
            f"{BASE_URL}/WellRecord?api={well_api}"
        ),

        "Origin": BASE_URL,
    }

    response = session.post(
        url,
        data={
            "Id": file_id
        },
        headers=headers,
        timeout=30
    )

    return response


# ============================================================
# GET REQUEST FILES PAGE
# ============================================================

def get_request_page(file_ids):

    ids = ",".join(file_ids)

    url = f"{BASE_URL}/RequestFiles?files={ids}"

    log()
    log("=" * 80)
    log("STEP 3 — GET REQUEST FILES PAGE")
    log("=" * 80)
    log()
    log(url)

    response = session.get(
        url,
        timeout=30
    )

    log()
    log(
        f"Status: {response.status_code}"
    )

    log(
        f"Final URL: {response.url}"
    )

    if response.status_code != 200:

        log(
            "ERROR loading RequestFiles page."
        )

        return None, None

    soup = BeautifulSoup(
        response.text,
        "html.parser"
    )

    # --------------------------------------------------------
    # Find CSRF verification token
    # --------------------------------------------------------

    token_input = soup.find(
        "input",
        {
            "name": "__RequestVerificationToken"
        }
    )

    if not token_input:

        log()
        log(
            "ERROR: Could not find verification token."
        )

        return response, None

    token = token_input.get(
        "value"
    )

    log()
    log(
        "Verification token found."
    )

    log(
        f"Token length: {len(token)}"
    )

    log(
        f"Token starts: {token[:20]}..."
    )

    # --------------------------------------------------------
    # Get filename from hidden FileName field
    # --------------------------------------------------------

    filename_input = soup.find(
        "input",
        {
            "name": "FileName"
        }
    )

    filename = ""

    if filename_input:

        filename = filename_input.get(
            "value",
            ""
        )

    log()
    log(
        f"Filename: {filename}"
    )

    return response, {
        "token": token,
        "filename": filename,
        "file_ids": ids
    }


# ============================================================
# SUBMIT REQUEST
# ============================================================

def submit_request(
    file_ids,
    token,
    filename,
    well_api
):

    url = f"{BASE_URL}/RequestFile"

    data = {
        "FileId": ",".join(file_ids),
        "FileName": filename,
        "Name": "",
        "Email": EMAIL,
        "__RequestVerificationToken": token
    }

    headers = {
        "Referer": (
            f"{BASE_URL}/RequestFiles?files="
            + ",".join(file_ids)
        ),

        "Origin": BASE_URL,
    }

    log()
    log("=" * 80)
    log("STEP 5 — SUBMIT FILE REQUEST")
    log("=" * 80)
    log()

    log(
        "POST:"
    )

    log(
        url
    )

    log()
    log(
        "DATA:"
    )

    log(
        f"FileId:   {data['FileId']}"
    )

    log(
        f"FileName: {data['FileName']}"
    )

    log(
        f"Name:     {data['Name']}"
    )

    log(
        f"Email:    {data['Email']}"
    )

    log(
        f"Token:    {token[:20]}..."
    )

    response = session.post(
        url,
        data=data,
        headers=headers,
        allow_redirects=False,
        timeout=30
    )

    log()
    log("=" * 80)
    log("RESULT")
    log("=" * 80)
    log()

    log(
        f"Status: {response.status_code}"
    )

    location = response.headers.get(
        "Location"
    )

    if location:

        log(
            f"Location: {location}"
        )

    if response.status_code == 302:

        if location == "/Success":

            log()
            log(
                "SUCCESS — REQUEST SUBMITTED"
            )

            return True

        else:

            log()
            log(
                "302 redirect received, "
                "but destination was unexpected."
            )

    else:

        log()
        log(
            "WARNING: Request did not return "
            "the expected HTTP 302."
        )

    return False


# ============================================================
# PROCESS ONE WELL
# ============================================================

def process_well(csv_api):

    site_api = csv_api[1:]

    # --------------------------------------------------------
    # STEP 1
    # --------------------------------------------------------

    files = get_files_for_well(
        csv_api
    )

    if not files:

        log()
        log(
            "NO FILES FOUND — SKIPPING WELL."
        )

        return False

    file_ids = [
        file["id"]
        for file in files
    ]

    # --------------------------------------------------------
    # STEP 2 — SELECT ALL FILES
    # --------------------------------------------------------

    log()
    log("=" * 80)
    log("STEP 2 — SELECT ALL FILES")
    log("=" * 80)
    log()

    for file in files:

        file_id = file["id"]

        response = select_file(
            file_id,
            site_api
        )

        log(
            f"File {file_id}: "
            f"HTTP {response.status_code}"
        )

        if response.status_code != 200:

            log(
                "WARNING: File selection "
                "may have failed."
            )

        time.sleep(0.25)

    # --------------------------------------------------------
    # STEP 3
    # --------------------------------------------------------

    response, request_info = get_request_page(
        file_ids
    )

    if request_info is None:

        log()
        log(
            "ERROR: Could not get request page."
        )

        return False

    # --------------------------------------------------------
    # STEP 4 / 5
    # --------------------------------------------------------

    success = submit_request(
        file_ids=file_ids,

        token=request_info[
            "token"
        ],

        filename=request_info[
            "filename"
        ],

        well_api=site_api
    )

    return success


# ============================================================
# MAIN
# ============================================================

# Create the log header

log()
log("=" * 80)
log(
    "CALIFORNIA WELL RECORD FILE REQUEST AUTOMATION"
)
log("=" * 80)
log()
log(
    f"Started: "
    f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
)
log()
log(
    f"Wells to process: {len(APIS)}"
)
log(
    f"Email: {EMAIL}"
)
log()


# ============================================================
# PROCESS WELLS
# ============================================================

results = []

successful = 0
failed = 0


for number, csv_api in enumerate(
    APIS,
    1
):

    try:

        success = process_well(
            csv_api
        )

        results.append({
            "api": csv_api,
            "site_api": csv_api[1:],
            "success": success
        })

        # ----------------------------------------------------
        # ONLY OUTPUT TO CONSOLE
        # ----------------------------------------------------

        if success:

            print(
                f"{csv_api} = SUCCESS"
            )

            successful += 1

        else:

            print(
                f"{csv_api} = FAILED"
            )

            failed += 1

    except Exception as e:

        # ----------------------------------------------------
        # Detailed error goes into log
        # ----------------------------------------------------

        log()
        log("=" * 80)
        log("ERROR PROCESSING WELL")
        log("=" * 80)
        log()
        log(
            f"API: {csv_api}"
        )
        log()
        log(
            f"Error: {str(e)}"
        )

        results.append({
            "api": csv_api,
            "site_api": csv_api[1:],
            "success": False
        })

        print(
            f"{csv_api} = FAILED"
        )

        failed += 1

    # --------------------------------------------------------
    # Pause between wells
    # --------------------------------------------------------

    if number < len(APIS):

        time.sleep(2)


# ============================================================
# FINAL SUMMARY — LOG FILE ONLY
# ============================================================

log()
log()
log("=" * 80)
log("FINAL SUMMARY")
log("=" * 80)
log()

for result in results:

    status = (
        "SUCCESS"
        if result["success"]
        else "FAILED"
    )

    log(
        f"{result['api']}  "
        f"({result['site_api']})  "
        f"{status}"
    )

log()
log(
    f"Total wells: {len(results)}"
)

log(
    f"Successful: {successful}"
)

log(
    f"Failed: {failed}"
)

log()
log(
    f"Finished: "
    f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
)

log()
log("=" * 80)
log("DONE")
log("=" * 80)


# ============================================================
# CONSOLE FINAL MESSAGE
# ============================================================

print()
print(
    f"Complete: {successful} successful, "
    f"{failed} failed."
)

print(
    f"Detailed log: {LOG_FILE}"
)