From 47c1f92b6049a25b6b68c589562f49e69fbd60a3 Mon Sep 17 00:00:00 2001 From: laurenzzumschlinge Date: Thu, 9 Jul 2026 11:10:08 +0200 Subject: [PATCH 1/4] feat: Add Charity Commission API scraper and unit tests --- .../raw/register_of_charities_results.json | 527 ++++++++++++++++++ src/scrapers/register_of_charities.py | 366 ++++++++++++ src/tests/test_register_of_charities.py | 153 +++++ 3 files changed, 1046 insertions(+) create mode 100644 src/data/raw/register_of_charities_results.json create mode 100644 src/scrapers/register_of_charities.py create mode 100644 src/tests/test_register_of_charities.py diff --git a/src/data/raw/register_of_charities_results.json b/src/data/raw/register_of_charities_results.json new file mode 100644 index 0000000..2219471 --- /dev/null +++ b/src/data/raw/register_of_charities_results.json @@ -0,0 +1,527 @@ +[ + { + "registered_charity_number": 249658, + "suffix": 0, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=249658&subid=0", + "all_details": { + "organisation_number": 249658, + "reg_charity_number": 249658, + "group_subsid_suffix": 0, + "charity_name": "(WEST LANCASHIRE BRANCH) OF THE BRITISH RED CROSS SOCIETY", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1966-10-13T00:00:00", + "date_of_removal": "1993-07-02T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": "Removed", + "removal_reason": "CEASED TO EXIST", + "cio_ind": false, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + }, + { + "registered_charity_number": 254059, + "suffix": 0, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=254059&subid=0", + "all_details": { + "organisation_number": 254059, + "reg_charity_number": 254059, + "group_subsid_suffix": 0, + "charity_name": "ABERGAVENNY BRITISH RED CROSS SOCIETY AND ST JOHN AMBULANCE HALL", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1967-11-02T00:00:00", + "date_of_removal": "2003-07-22T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": "Removed", + "removal_reason": "CEASED TO EXIST", + "cio_ind": false, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + }, + { + "registered_charity_number": 266624, + "suffix": 0, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=266624&subid=0", + "all_details": { + "organisation_number": 266624, + "reg_charity_number": 266624, + "group_subsid_suffix": 0, + "charity_name": "AVON BRANCH OF THE BRITISH RED CROSS SOCIETY", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1973-12-14T00:00:00", + "date_of_removal": "1998-10-23T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": "Removed", + "removal_reason": "CEASED TO EXIST", + "cio_ind": false, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + }, + { + "registered_charity_number": 249170, + "suffix": 0, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=249170&subid=0", + "all_details": { + "organisation_number": 249170, + "reg_charity_number": 249170, + "group_subsid_suffix": 0, + "charity_name": "BRECONSHIRE BRANCH OF THE BRITISH RED CROSS SOCIETY", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1966-09-05T00:00:00", + "date_of_removal": "1966-09-05T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": "Removed", + "removal_reason": "CEASED TO EXIST", + "cio_ind": false, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + }, + { + "registered_charity_number": 242323, + "suffix": 0, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=242323&subid=0", + "all_details": { + "organisation_number": 242323, + "reg_charity_number": 242323, + "group_subsid_suffix": 0, + "charity_name": "BRITISH RED CROSS - BEDFORDSHIRE BRANCH", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1965-06-16T00:00:00", + "date_of_removal": "1998-10-23T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": "Removed", + "removal_reason": "CEASED TO EXIST", + "cio_ind": false, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [ + { + "other_name": "THE BEDFORDSHIRE BRANCH OF THE BRITISH RED CROSS SOCIETY", + "name_type": "O" + } + ], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + }, + { + "registered_charity_number": 247125, + "suffix": 0, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=247125&subid=0", + "all_details": { + "organisation_number": 247125, + "reg_charity_number": 247125, + "group_subsid_suffix": 0, + "charity_name": "BRITISH RED CROSS - NORTHAMPTONSHIRE BRANCH", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1966-03-07T00:00:00", + "date_of_removal": "1999-02-26T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": "Removed", + "removal_reason": "CEASED TO EXIST", + "cio_ind": false, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [ + { + "other_name": "THE NORTHAMPTONSHIRE BRANCH OF THE BRITISH RED CROSS SOCIETY", + "name_type": "O" + } + ], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + }, + { + "registered_charity_number": 502853, + "suffix": 0, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=502853&subid=0", + "all_details": { + "organisation_number": 502853, + "reg_charity_number": 502853, + "group_subsid_suffix": 0, + "charity_name": "BRITISH RED CROSS - SOUTH WEST WALES BRANCH", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1973-12-07T00:00:00", + "date_of_removal": "1999-08-18T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": "Removed", + "removal_reason": "CEASED TO EXIST", + "cio_ind": false, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [ + { + "other_name": "DYFED COUNTY BRANCH OF THE BRITISH RED CROSS SOCIETY", + "name_type": "O" + } + ], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + }, + { + "registered_charity_number": 210119, + "suffix": 1, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=210119&subid=1", + "all_details": { + "organisation_number": 5823, + "reg_charity_number": 210119, + "group_subsid_suffix": 1, + "charity_name": "BRITISH RED CROSS (WHITAKER ENDOWMENT)", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": null, + "cif_cdf_ind": null, + "cio_dissolution_ind": null, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1962-09-22T00:00:00", + "date_of_removal": "1996-10-08T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": null, + "removal_reason": null, + "cio_ind": null, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + }, + { + "registered_charity_number": 1033268, + "suffix": 0, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=1033268&subid=0", + "all_details": { + "organisation_number": 1033268, + "reg_charity_number": 1033268, + "group_subsid_suffix": 0, + "charity_name": "BRITISH RED CROSS SOCIETY COMMON DEPOSIT FUND", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1995-04-28T00:00:00", + "date_of_removal": "1999-10-20T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": "Removed", + "removal_reason": "CEASED TO EXIST", + "cio_ind": false, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + }, + { + "registered_charity_number": 506481, + "suffix": 0, + "link": "https://register-of-charities.charitycommission.gov.uk/charity-details/?regid=506481&subid=0", + "all_details": { + "organisation_number": 506481, + "reg_charity_number": 506481, + "group_subsid_suffix": 0, + "charity_name": "BRITISH RED CROSS SOCIETY LINCOLNSHIRE BRANCH", + "charity_type": null, + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": false, + "date_of_interim_manager_appt": null, + "reg_status": "RM", + "date_of_registration": "1977-07-07T00:00:00", + "date_of_removal": "1998-10-23T00:00:00", + "latest_acc_fin_year_start_date": null, + "latest_acc_fin_year_end_date": null, + "latest_income": null, + "latest_expenditure": null, + "address_line_one": null, + "address_line_two": null, + "address_line_three": null, + "address_line_four": null, + "address_line_five": null, + "address_post_code": null, + "phone": null, + "email": null, + "web": null, + "charity_co_reg_number": null, + "reporting_status": "Removed", + "removal_reason": "CEASED TO EXIST", + "cio_ind": false, + "last_modified_time": "2011-03-12T13:05:56.5", + "trustee_names": [], + "who_what_where": [], + "CharityAoOCountryContinent": [], + "CharityAoOLocalAuthority": [], + "CharityAoORegion": [], + "other_names": [], + "constituency_name": [] + }, + "assets_liabilities": [], + "primary_grants": null, + "who_what_how": [], + "financial_history": [] + } +] \ No newline at end of file diff --git a/src/scrapers/register_of_charities.py b/src/scrapers/register_of_charities.py new file mode 100644 index 0000000..5b33f95 --- /dev/null +++ b/src/scrapers/register_of_charities.py @@ -0,0 +1,366 @@ +import os +import time +import json +import logging +import argparse +import requests + +# Load environment variables from .env file if it exists +def load_env(): + scrapers_dir = os.path.dirname(os.path.abspath(__file__)) + src_dir = os.path.dirname(scrapers_dir) + workspace_root = os.path.dirname(src_dir) + env_path = os.path.join(workspace_root, ".env") + if os.path.exists(env_path): + with open(env_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" in line: + key, val = line.split("=", 1) + key = key.strip() + val = val.strip().strip("'\"") + if key: + os.environ[key] = val + +load_env() + +# Configure logger +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.StreamHandler() + ] +) + +class CharityCommissionAPI: + """ + Client for interacting with the official Charity Commission for England and Wales API. + """ + BASE_URL = "https://api.charitycommission.gov.uk/register/api" + + def __init__(self, api_key=None, timeout=10.0, max_retries=3, backoff_factor=2): + # Read API key from parameter or environment variables + self.api_key = api_key if api_key is not None else os.environ.get("CHARITY_COMMISSION_API_KEY") + self.timeout = timeout + self.max_retries = max_retries + self.backoff_factor = backoff_factor + + self.session = requests.Session() + if self.api_key: + self.session.headers.update({ + "Ocp-Apim-Subscription-Key": self.api_key, + "Cache-Control": "no-cache" + }) + else: + logging.warning("No API key provided. Requests to the Charity Commission API may fail with HTTP 401 Unauthorized.") + + def make_request(self, method, endpoint, **kwargs): + """ + Make an HTTP request with automatic retries, exponential backoff, and timeout. + """ + url = f"{self.BASE_URL}/{endpoint.lstrip('/')}" + + # Ensure timeout is set + if "timeout" not in kwargs: + kwargs["timeout"] = self.timeout + + for attempt in range(self.max_retries): + try: + response = self.session.request(method, url, **kwargs) + # Retry on transient server errors or rate limits (e.g. 429, 5xx) + if response.status_code in [429, 500, 502, 503, 504]: + logging.warning( + f"Transient HTTP status {response.status_code} for URL: {url}. " + f"Attempt {attempt + 1}/{self.max_retries}" + ) + if attempt < self.max_retries - 1: + time.sleep(self.backoff_factor ** attempt) + continue + return response + except requests.exceptions.RequestException as e: + logging.warning( + f"Request failed for URL: {url} (Error: {e}). " + f"Attempt {attempt + 1}/{self.max_retries}" + ) + if attempt < self.max_retries - 1: + time.sleep(self.backoff_factor ** attempt) + continue + raise e + raise requests.exceptions.RequestException( + f"Failed to fetch {url} after {self.max_retries} attempts." + ) + + def all_charity_details(self, registered_number, suffix=0): + """ + Retrieves comprehensive details about a charity (V2 endpoint). + URL: allcharitydetailsV2/{RegisteredNumber}/{suffix} + """ + endpoint = f"allcharitydetailsV2/{registered_number}/{suffix}" + response = self.make_request("GET", endpoint) + if response.status_code == 200: + return response.json() + elif response.status_code == 404: + logging.warning(f"Charity {registered_number} (suffix {suffix}) not found.") + return None + else: + response.raise_for_status() + + def charity_assets_liabilities(self, registered_number, suffix=0): + """ + Retrieves asset and liability information for a charity. + URL: charityassetsliabilities/{RegisteredNumber}/{suffix} + """ + endpoint = f"charityassetsliabilities/{registered_number}/{suffix}" + response = self.make_request("GET", endpoint) + if response.status_code == 200: + return response.json() + elif response.status_code == 404: + logging.warning(f"Charity assets/liabilities not found for {registered_number} (suffix {suffix}).") + return None + else: + response.raise_for_status() + + def check_primary_grants(self, registered_number, suffix=0): + """ + Checks primary grants information for a charity. + URL: checkprimarygrants/{RegisteredNumber}/{suffix} + """ + endpoint = f"checkprimarygrants/{registered_number}/{suffix}" + response = self.make_request("GET", endpoint) + if response.status_code == 200: + return response.json() + elif response.status_code == 404: + logging.warning(f"Primary grants not found for {registered_number} (suffix {suffix}).") + return None + else: + response.raise_for_status() + + def charity_who_what_how(self, registered_number, suffix=0): + """ + Retrieves who, what, and how information for a charity. + URL: charitywhowhathow/{RegisteredNumber}/{suffix} + """ + endpoint = f"charitywhowhathow/{registered_number}/{suffix}" + response = self.make_request("GET", endpoint) + if response.status_code == 200: + return response.json() + elif response.status_code == 404: + logging.warning(f"Who/What/How info not found for {registered_number} (suffix {suffix}).") + return None + else: + response.raise_for_status() + + def charity_financial_history(self, registered_number, suffix=0): + """ + Retrieves financial history for a charity. + URL: charityfinancialhistory/{RegisteredNumber}/{suffix} + """ + endpoint = f"charityfinancialhistory/{registered_number}/{suffix}" + response = self.make_request("GET", endpoint) + if response.status_code == 200: + return response.json() + elif response.status_code == 404: + logging.warning(f"Financial history not found for {registered_number} (suffix {suffix}).") + return None + else: + response.raise_for_status() + + def search_charity_name(self, charity_name): + """ + Searches for a charity by name. + URL: searchCharityName/{charityname} + """ + endpoint = f"searchCharityName/{charity_name}" + response = self.make_request("GET", endpoint) + if response.status_code == 200: + return response.json() + else: + response.raise_for_status() + + +# Module-level convenience functions +def get_all_charity_details(registered_number, suffix=0, api_key=None): + client = CharityCommissionAPI(api_key=api_key) + return client.all_charity_details(registered_number, suffix) + +def get_charity_assets_liabilities(registered_number, suffix=0, api_key=None): + client = CharityCommissionAPI(api_key=api_key) + return client.charity_assets_liabilities(registered_number, suffix) + +def get_check_primary_grants(registered_number, suffix=0, api_key=None): + client = CharityCommissionAPI(api_key=api_key) + return client.check_primary_grants(registered_number, suffix) + +def get_charity_who_what_how(registered_number, suffix=0, api_key=None): + client = CharityCommissionAPI(api_key=api_key) + return client.charity_who_what_how(registered_number, suffix) + +def get_charity_financial_history(registered_number, suffix=0, api_key=None): + client = CharityCommissionAPI(api_key=api_key) + return client.charity_financial_history(registered_number, suffix) + +def search_charity_name(charity_name, api_key=None): + client = CharityCommissionAPI(api_key=api_key) + return client.search_charity_name(charity_name) + + +def scrape(registered_numbers=None, search_name=None, limit=None, sleep_time=1.0, timeout=10.0, api_key=None): + """ + Scrapes data for one or more charities. + Can either search for a charity name or fetch details for specific registered numbers. + """ + client = CharityCommissionAPI(api_key=api_key, timeout=timeout) + results = [] + + if search_name: + logging.info(f"Searching for charities with name matching: '{search_name}'") + try: + search_results = client.search_charity_name(search_name) + if not search_results: + logging.info("No matching charities found.") + return [] + + reg_numbers = [] + for item in search_results: + reg_no = item.get("reg_charity_number") or item.get("registeredCharityNumber") or item.get("charityNumber") or item.get("regno") + suffix = item.get("group_subsid_suffix") if item.get("group_subsid_suffix") is not None else (item.get("suffix") or 0) + if reg_no: + reg_numbers.append((int(reg_no), int(suffix))) + except Exception as e: + logging.error(f"Failed to search for charity name '{search_name}': {e}") + return [] + elif registered_numbers: + reg_numbers = [] + for item in registered_numbers: + if isinstance(item, tuple): + reg_numbers.append((int(item[0]), int(item[1]))) + elif isinstance(item, (int, str)): + reg_numbers.append((int(item), 0)) + else: + logging.error("Either registered_numbers or search_name must be provided to scrape.") + return [] + + if limit: + reg_numbers = reg_numbers[:limit] + + total = len(reg_numbers) + logging.info(f"Starting scraping details for {total} charity records...") + + for i, (reg_no, suffix) in enumerate(reg_numbers): + logging.info(f"[{i+1}/{total}] Fetching details for Charity Registration: {reg_no}, Suffix: {suffix}") + charity_data = { + "registered_charity_number": reg_no, + "suffix": suffix, + "link": f"https://register-of-charities.charitycommission.gov.uk/charity-details/?regid={reg_no}&subid={suffix}" + } + + try: + charity_data["all_details"] = client.all_charity_details(reg_no, suffix) + time.sleep(sleep_time) + + charity_data["assets_liabilities"] = client.charity_assets_liabilities(reg_no, suffix) + time.sleep(sleep_time) + + charity_data["primary_grants"] = client.check_primary_grants(reg_no, suffix) + time.sleep(sleep_time) + + charity_data["who_what_how"] = client.charity_who_what_how(reg_no, suffix) + time.sleep(sleep_time) + + charity_data["financial_history"] = client.charity_financial_history(reg_no, suffix) + time.sleep(sleep_time) + + except Exception as e: + logging.error(f"Error scraping details for charity {reg_no} (suffix {suffix}): {e}") + charity_data["error"] = str(e) + + results.append(charity_data) + + return results + + +def save_data(data, path): + logging.info(f"Saving scraped data to {path}...") + try: + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=4) + logging.info("Data saved successfully.") + except Exception as e: + logging.error(f"Failed to save data to {path}: {e}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Scrape Charity Commission API.") + parser.add_argument( + "--reg-numbers", + type=str, + nargs="+", + help="List of registered charity numbers to scrape (e.g. 219907 283322)" + ) + parser.add_argument( + "--search", + type=str, + help="Name search term to find and scrape charities" + ) + parser.add_argument( + "--suffix", + type=int, + default=0, + help="Suffix for the registered numbers (defaults to 0)" + ) + parser.add_argument( + "--api-key", + type=str, + help="Charity Commission API subscription key (defaults to environment variable)" + ) + parser.add_argument( + "--output", + type=str, + default=os.path.join(os.path.dirname(__file__), "../data/raw/register_of_charities_results.json"), + help="Path where the output JSON file should be saved." + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Limit the number of charities to scrape." + ) + parser.add_argument( + "--sleep", + type=float, + default=1.0, + help="Sleep time in seconds between API requests." + ) + parser.add_argument( + "--timeout", + type=float, + default=10.0, + help="HTTP request timeout in seconds." + ) + args = parser.parse_args() + + # Determine registered numbers format + registered_numbers = None + if args.reg_numbers: + registered_numbers = [(int(num), args.suffix) for num in args.reg_numbers] + + if not registered_numbers and not args.search: + parser.error("At least one of --reg-numbers or --search must be specified.") + + results = scrape( + registered_numbers=registered_numbers, + search_name=args.search, + limit=args.limit, + sleep_time=args.sleep, + timeout=args.timeout, + api_key=args.api_key + ) + + if results: + save_data(results, args.output) + else: + logging.error("No data retrieved. File was not saved.") \ No newline at end of file diff --git a/src/tests/test_register_of_charities.py b/src/tests/test_register_of_charities.py new file mode 100644 index 0000000..37fe123 --- /dev/null +++ b/src/tests/test_register_of_charities.py @@ -0,0 +1,153 @@ +import unittest +from unittest.mock import patch, MagicMock +import os +import sys +import requests + +# Add project root to sys.path so we can import scrapers +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) + +from scrapers.register_of_charities import ( + CharityCommissionAPI, + get_all_charity_details, + get_charity_assets_liabilities, + get_check_primary_grants, + get_charity_who_what_how, + get_charity_financial_history, + search_charity_name, + scrape +) + +class TestCharityCommissionAPI(unittest.TestCase): + + @patch("scrapers.register_of_charities.requests.Session.request") + def test_api_key_header(self, mock_request): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_request.return_value = mock_resp + + # Instantiate with api_key + api = CharityCommissionAPI(api_key="test-secret-key") + api.all_charity_details(12345, 0) + + # Check that request was called with key in session headers + self.assertEqual(api.session.headers.get("Ocp-Apim-Subscription-Key"), "test-secret-key") + + @patch("scrapers.register_of_charities.requests.Session.request") + def test_all_charity_details_success(self, mock_request): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"charity_name": "Test Charity"} + mock_request.return_value = mock_resp + + details = get_all_charity_details(12345, 0, api_key="dummy") + self.assertEqual(details, {"charity_name": "Test Charity"}) + + @patch("scrapers.register_of_charities.requests.Session.request") + def test_charity_assets_liabilities_success(self, mock_request): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"assets": 1000} + mock_request.return_value = mock_resp + + res = get_charity_assets_liabilities(12345, 0, api_key="dummy") + self.assertEqual(res, {"assets": 1000}) + + @patch("scrapers.register_of_charities.requests.Session.request") + def test_check_primary_grants_success(self, mock_request): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"grants": []} + mock_request.return_value = mock_resp + + res = get_check_primary_grants(12345, 0, api_key="dummy") + self.assertEqual(res, {"grants": []}) + + @patch("scrapers.register_of_charities.requests.Session.request") + def test_charity_who_what_how_success(self, mock_request): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"activities": "helping people"} + mock_request.return_value = mock_resp + + res = get_charity_who_what_how(12345, 0, api_key="dummy") + self.assertEqual(res, {"activities": "helping people"}) + + @patch("scrapers.register_of_charities.requests.Session.request") + def test_charity_financial_history_success(self, mock_request): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = [{"year": 2023}] + mock_request.return_value = mock_resp + + res = get_charity_financial_history(12345, 0, api_key="dummy") + self.assertEqual(res, [{"year": 2023}]) + + @patch("scrapers.register_of_charities.requests.Session.request") + def test_search_charity_name_success(self, mock_request): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = [{"reg_charity_number": 12345, "group_subsid_suffix": 0}] + mock_request.return_value = mock_resp + + res = search_charity_name("Test", api_key="dummy") + self.assertEqual(res, [{"reg_charity_number": 12345, "group_subsid_suffix": 0}]) + + @patch("scrapers.register_of_charities.requests.Session.request") + def test_all_charity_details_not_found(self, mock_request): + mock_resp = MagicMock() + mock_resp.status_code = 404 + mock_request.return_value = mock_resp + + details = get_all_charity_details(12345, 0, api_key="dummy") + self.assertIsNone(details) + + @patch("scrapers.register_of_charities.requests.Session.request") + @patch("scrapers.register_of_charities.time.sleep") + def test_scrape_by_registered_numbers(self, mock_sleep, mock_request): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.side_effect = [ + {"name": "Details"}, # all_charity_details + {"assets": 100}, # assets_liabilities + {"grants": ["A"]}, # check_primary_grants + {"who": "what"}, # charity_who_what_how + {"history": []} # charity_financial_history + ] + mock_request.return_value = mock_resp + + results = scrape(registered_numbers=[12345], sleep_time=0.01, api_key="dummy") + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["registered_charity_number"], 12345) + self.assertEqual(results[0]["all_details"], {"name": "Details"}) + self.assertEqual(results[0]["assets_liabilities"], {"assets": 100}) + self.assertEqual(results[0]["primary_grants"], {"grants": ["A"]}) + self.assertEqual(results[0]["who_what_how"], {"who": "what"}) + self.assertEqual(results[0]["financial_history"], {"history": []}) + + @patch("scrapers.register_of_charities.requests.Session.request") + @patch("scrapers.register_of_charities.time.sleep") + def test_scrape_by_search_name(self, mock_sleep, mock_request): + mock_resp_search = MagicMock() + mock_resp_search.status_code = 200 + mock_resp_search.json.return_value = [{"reg_charity_number": 12345, "group_subsid_suffix": 0}] + + mock_resp_details = MagicMock() + mock_resp_details.status_code = 200 + mock_resp_details.json.side_effect = [ + {"name": "Details"}, # all_charity_details + {"assets": 100}, # assets_liabilities + {"grants": []}, # check_primary_grants + {"who": "what"}, # charity_who_what_how + {"history": []} # charity_financial_history + ] + + mock_request.side_effect = [mock_resp_search, mock_resp_details, mock_resp_details, mock_resp_details, mock_resp_details, mock_resp_details] + + results = scrape(search_name="Test", sleep_time=0.01, api_key="dummy") + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["registered_charity_number"], 12345) + self.assertEqual(results[0]["all_details"], {"name": "Details"}) + +if __name__ == "__main__": + unittest.main() From cc301a403c17ee9dff37a3bdadf22d427a4cd0ac Mon Sep 17 00:00:00 2001 From: lauxenz <47084606+lauxenz@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:45:25 +0200 Subject: [PATCH 2/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/scrapers/register_of_charities.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scrapers/register_of_charities.py b/src/scrapers/register_of_charities.py index 5b33f95..517747a 100644 --- a/src/scrapers/register_of_charities.py +++ b/src/scrapers/register_of_charities.py @@ -173,8 +173,9 @@ def search_charity_name(self, charity_name): Searches for a charity by name. URL: searchCharityName/{charityname} """ - endpoint = f"searchCharityName/{charity_name}" - response = self.make_request("GET", endpoint) +from urllib.parse import quote +endpoint = f"searchCharityName/{quote(str(charity_name), safe='')}" +response = self.make_request("GET", endpoint) if response.status_code == 200: return response.json() else: From e4b9a834fde4139b9b55b23d7fec4a6b130d183f Mon Sep 17 00:00:00 2001 From: lauxenz <47084606+lauxenz@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:45:57 +0200 Subject: [PATCH 3/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/scrapers/register_of_charities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scrapers/register_of_charities.py b/src/scrapers/register_of_charities.py index 517747a..050db51 100644 --- a/src/scrapers/register_of_charities.py +++ b/src/scrapers/register_of_charities.py @@ -22,7 +22,7 @@ def load_env(): key = key.strip() val = val.strip().strip("'\"") if key: - os.environ[key] = val + os.environ.setdefault(key, val) load_env() From 79bba79a50d7b74c31ca63f0fbcacc004fe9f779 Mon Sep 17 00:00:00 2001 From: lauxenz <47084606+lauxenz@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:46:10 +0200 Subject: [PATCH 4/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/tests/test_register_of_charities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_register_of_charities.py b/src/tests/test_register_of_charities.py index 37fe123..98e5130 100644 --- a/src/tests/test_register_of_charities.py +++ b/src/tests/test_register_of_charities.py @@ -2,7 +2,7 @@ from unittest.mock import patch, MagicMock import os import sys -import requests + # Add project root to sys.path so we can import scrapers sys.path.append(os.path.join(os.path.dirname(__file__), ".."))