diff --git a/src/preprocessing/consolidate.py b/src/preprocessing/consolidate.py index e7e49e1..21daaa0 100644 --- a/src/preprocessing/consolidate.py +++ b/src/preprocessing/consolidate.py @@ -4,6 +4,11 @@ import logging from urllib.parse import urlparse +try: + from preprocessing.quality import is_informative_value, is_placeholder_value +except ModuleNotFoundError: + from quality import is_informative_value, is_placeholder_value + # Configure logger logger = logging.getLogger("consolidate") if not logger.handlers: @@ -20,7 +25,7 @@ def _extract_number(val_str): Extracts a numeric float value from a string, ignoring currency symbols and years. Handles standard number formats (thousands separators, decimals). """ - if not val_str or "not publicly available" in val_str.lower() or "not disclosed" in val_str.lower(): + if not val_str or is_placeholder_value(val_str): return None # Remove year patterns like (2024) or [2022] or (2023-24) to avoid matching the year as the number @@ -257,18 +262,24 @@ def normalize_to_clean_schema(member, source_name): # Standalone Hinchilla financial conversion (or general Hinchilla standardization) annual_giving = p_info.get("annual_giving", "") + annual_income = p_info.get("annual_income", "") + annual_expenditure = p_info.get("annual_expenditure", "") average_grant = p_info.get("average_grant", "") grant_range = p_info.get("grant_range", "") expenditure = p_info.get("expenditure", "") if source_name == "Hinchilla": annual_giving = convert_gbp_to_eur(annual_giving) + annual_income = convert_gbp_to_eur(annual_income) + annual_expenditure = convert_gbp_to_eur(annual_expenditure) average_grant = convert_gbp_to_eur(average_grant) grant_range = convert_gbp_to_eur(grant_range) expenditure = convert_gbp_to_eur(expenditure) funding_info = { "annual_giving": annual_giving, + "annual_income": annual_income, + "annual_expenditure": annual_expenditure, "average_grant": average_grant, "grant_range": grant_range, "funding_model": p_info.get("funding_model", ""), @@ -276,6 +287,8 @@ def normalize_to_clean_schema(member, source_name): "success_rate": p_info.get("success_rate", ""), "decision_time": p_info.get("decision_time", ""), "expenditure": expenditure, + "number_of_grants": p_info.get("number_of_grants", ""), + "quick_stats": p_info.get("quick_stats", {}), "charity_number": p_info.get("charityNumber", "") or p_info.get("charity_number", ""), "application_portal": p_info.get("applicationPortal", "") or p_info.get("application_portal", ""), "sources": p_info.get("sources", []) @@ -317,9 +330,9 @@ def merge_members(p_member, h_member): # Helper to resolve field value and check discrepancies def resolve_field(field_name, p_val, h_val, is_financial=False): - if not p_val or p_val.lower() in ["not publicly available", "not disclosed", ""]: - return h_val if h_val else p_val - if not h_val or h_val.lower() in ["not publicly available", "not disclosed", ""]: + if not is_informative_value(p_val): + return h_val if is_informative_value(h_val) else p_val + if not is_informative_value(h_val): return p_val # Both are non-empty. Compare. @@ -389,7 +402,11 @@ def resolve_field(field_name, p_val, h_val, is_financial=False): # Union metadata for key in ["success_rate", "decision_time", "charity_number", "application_portal"]: - if h_fund.get(key): + if is_informative_value(h_fund.get(key)): + p_fund[key] = h_fund[key] + + for key in ["annual_income", "annual_expenditure", "number_of_grants", "quick_stats"]: + if is_informative_value(h_fund.get(key)) and not is_informative_value(p_fund.get(key)): p_fund[key] = h_fund[key] # Text merge (prefer longer) diff --git a/src/preprocessing/extract_geo_topic.py b/src/preprocessing/extract_geo_topic.py index 39344a4..0662b76 100644 --- a/src/preprocessing/extract_geo_topic.py +++ b/src/preprocessing/extract_geo_topic.py @@ -509,6 +509,182 @@ def extract_geos_final(raw_text): found[macro].add(country_name) return {k: sorted(list(v)) for k, v in found.items()} + + def normalize_location_text(raw_text): + return re.sub(r"[^a-z0-9]+", " ", raw_text.lower()).strip() + + def contains_location_phrase(text_clean, phrase): + phrase_clean = normalize_location_text(phrase) + if not phrase_clean: + return False + pattern = r"(?,]+") + +def extract_email_from_text(text): + if not text: + return "" + for match in EMAIL_RE.finditer(text): + email = match.group(0).strip().rstrip(".,;:)]}") + local, _, domain = email.partition("@") + if local and "." in domain and not domain.startswith(".") and not domain.endswith("."): + return email + return "" + +def extract_urls_from_text(text): + if not text: + return [] + return [match.group(0).rstrip(".,;:)]}") for match in URL_RE.finditer(text)] + +def is_application_url(url): + if not url or not isinstance(url, str): + return False + url = url.strip() + if not url.startswith(("http://", "https://")): + return False + url_lower = url.lower() + return any(indicator in url_lower for indicator in APPLICATION_URL_INDICATORS) + +def infer_application_portal(urls): + for url in urls: + if is_application_url(url): + return url.strip() return "" def scrape(limit=None, sleep_time=1.0, timeout=10.0, completed_slugs=None): @@ -293,6 +350,12 @@ def scrape(limit=None, sleep_time=1.0, timeout=10.0, completed_slugs=None): # Geographic Focus includes areaOfOperation, Funding Priorities, and Quick Stats info geo_focus_combined = f"Area of Operation: {area_of_operation}\n\n{funding_priorities}\n\n{quick_stats_text}" + text_for_contact = "\n".join([overview, funding_priorities, quick_stats_text]) + website = meta_data.get("website", "") + email = meta_data.get("email", "") or extract_email_from_text(text_for_contact) + application_portal = meta_data.get("applicationPortal", "") or infer_application_portal( + [website] + extract_urls_from_text(text_for_contact) + ) member["philea_info"] = { # Normal fields expected by extract_geo_topic.py @@ -304,18 +367,55 @@ def scrape(limit=None, sleep_time=1.0, timeout=10.0, completed_slugs=None): "charityNumber": meta_data.get("charityNumber", ""), "areaOfOperation": area_of_operation, "expenditure": meta_data.get("expenditure", ""), - "website": meta_data.get("website", ""), + "website": website, "phone": meta_data.get("phone", ""), - "email": meta_data.get("email", ""), + "email": email, "address": meta_data.get("address", ""), - "applicationPortal": meta_data.get("applicationPortal", ""), + "applicationPortal": application_portal, # Financial stats from Quick Stats - "annual_giving": get_case_insensitive(quick_stats, ["Annual Giving", "Annual giving"]), - "success_rate": get_case_insensitive(quick_stats, ["Success Rate", "Success rate"]), - "decision_time": get_case_insensitive(quick_stats, ["Decision Time", "Decision time"]), - "grant_range": get_case_insensitive(quick_stats, ["Grant Range", "Grant range", "Average Grant", "Average grant"]), - "funding_model": get_case_insensitive(quick_stats, ["Funding Model", "Funding model", "Application Method", "Application method"]), + "quick_stats": quick_stats, + "annual_giving": pick_quick_stat(quick_stats, [ + "Annual Giving", + "Annual Grant Distribution", + "Annual Grants", + "AAC's Own Annual Grants", + ]), + "annual_income": pick_quick_stat(quick_stats, ["Annual Income", "Total Income"]), + "annual_expenditure": pick_quick_stat(quick_stats, ["Annual Expenditure", "Charitable Expenditure"]), + "success_rate": pick_quick_stat(quick_stats, [ + "Success Rate", + "Award Rate", + "Acceptance Rate", + "Funding Success Rate", + ]), + "decision_time": pick_quick_stat(quick_stats, [ + "Decision Time", + "Decision Timeline", + "Response Time", + "Review Time", + "Turnaround Time", + ]), + "grant_range": pick_quick_stat(quick_stats, [ + "Grant Range", + "Average Grant", + "Grant Amount", + "Grant Size", + "Award Range", + "Typical Grant", + "Amount", + ]), + "average_grant": pick_quick_stat(quick_stats, ["Average Grant", "Typical Grant"]), + "funding_model": pick_quick_stat(quick_stats, [ + "Funding Model", + "Application Method", + "Application Process", + "Application", + "Application Schedule", + "Grant Distribution", + "Distribution Method", + ]), + "number_of_grants": pick_quick_stat(quick_stats, ["Number of Grants", "Grants Awarded", "Projects Funded Globally"]), } scraped_successfully += 1 diff --git a/src/tests/test_hinchilla.py b/src/tests/test_hinchilla.py index dca6c56..ea6a11d 100644 --- a/src/tests/test_hinchilla.py +++ b/src/tests/test_hinchilla.py @@ -38,6 +38,20 @@ def test_parse_rsc_payload_newline_in_text(self): self.assertEqual(blocks["e"]["type"], "text") self.assertEqual(blocks["e"]["content"], "test") + def test_parse_rsc_payload_uses_byte_lengths_for_text(self): + # RSC T lengths are byte lengths. Non-ASCII text must not move the parser + # into the middle of a text segment and create fake blocks like "period:". + text = "Grant range: £250 - £1,000\nCooling off period: 3 months" + content = f"d:T{len(text.encode('utf-8')):x},{text}e:T4,test" + blocks = parse_rsc_payload(content) + + self.assertEqual(len(blocks), 2) + self.assertEqual(blocks["d"]["type"], "text") + self.assertEqual(blocks["d"]["content"], text) + self.assertEqual(blocks["e"]["type"], "text") + self.assertEqual(blocks["e"]["content"], "test") + self.assertNotIn("period", blocks) + def test_resolve_rsc_references(self): blocks = { "d": {"type": "text", "content": "Funding Details Text"}, @@ -121,7 +135,7 @@ def test_scrape_limited(self, mock_make_request): mock_detail_resp.status_code = 200 mock_detail_resp.text = """ d:T17,Funding Priorities Text - 9:[["$", "$Lc", null, {"data": {"name": "Three Peas", "areaOfOperation": "Greece, Czechia", "expenditure": "90000", "website": "https://threepeas.org"}, "sections": [{"title": "Overview", "content": "About Three Peas"}, {"title": "Funding Priorities", "content": "$d"}, {"title": "Quick Stats", "content": "- **Annual Giving**: 90k"}]}]] + 9:[["$", "$Lc", null, {"data": {"name": "Three Peas", "areaOfOperation": "Greece, Czechia", "expenditure": "90000", "website": "https://threepeas.org"}, "sections": [{"title": "Overview", "content": "About Three Peas"}, {"title": "Funding Priorities", "content": "$d"}, {"title": "Quick Stats", "content": "- **Annual Grant Distribution**: £3 million+\\n- **Annual Income**: £4.66 million\\n- **Annual Expenditure**: £1.2 million\\n- **Application Process**: No public application process\\n- **Number of Grants**: 68 grants awarded\\n- **Average Grant**: £6,000"}]}]] """ mock_make_request.side_effect = [mock_dir_resp, mock_detail_resp, mock_detail_resp] @@ -136,7 +150,14 @@ def test_scrape_limited(self, mock_make_request): self.assertEqual(philea_info["Programme Areas"], "Funding Priorities Text") self.assertIn("Greece, Czechia", philea_info["Geographic Focus"]) self.assertEqual(philea_info["website"], "https://threepeas.org") - self.assertEqual(philea_info["annual_giving"], "90k") + self.assertEqual(philea_info["annual_giving"], "£3 million+") + self.assertEqual(philea_info["annual_income"], "£4.66 million") + self.assertEqual(philea_info["annual_expenditure"], "£1.2 million") + self.assertEqual(philea_info["grant_range"], "£6,000") + self.assertEqual(philea_info["average_grant"], "£6,000") + self.assertEqual(philea_info["funding_model"], "No public application process") + self.assertEqual(philea_info["number_of_grants"], "68 grants awarded") + self.assertEqual(philea_info["quick_stats"]["Application Process"], "No public application process") if __name__ == "__main__": unittest.main() diff --git a/src/tests/test_pipeline.py b/src/tests/test_pipeline.py index a43ae49..54d63a8 100644 --- a/src/tests/test_pipeline.py +++ b/src/tests/test_pipeline.py @@ -8,7 +8,10 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from scrapers.philea import make_request, scrape +from scrapers.hinchilla import extract_email_from_text, infer_application_portal +from preprocessing.consolidate import normalize_to_clean_schema from preprocessing.extract_geo_topic import extract_tags, extract_geo +from preprocessing.quality import has_technical_value, is_informative_value, is_placeholder_value class TestPhileaScraper(unittest.TestCase): @@ -157,6 +160,62 @@ def test_extract_geo_abbreviation_and_alternation(self): self.assertIn("Europe (Western / General)", members[0]["geo_locations"]) self.assertIn("United Kingdom", members[0]["geo_locations"]["Europe (Western / General)"]) + def test_extract_geo_uk_local_counties(self): + members = [ + { + "name": "3R Foundation", + "philea_info": { + "Geographic Focus": "Local community funding.", + "areaOfOperation": "Cumbria, Lancashire" + } + } + ] + extract_geo(members) + self.assertIn("Europe (Western / General)", members[0]["geo_locations"]) + self.assertIn("United Kingdom", members[0]["geo_locations"]["Europe (Western / General)"]) + + def test_extract_geo_fallback_leaves_ambiguous_places_without_context_unresolved(self): + members = [ + { + "name": "Lancaster Fund", + "philea_info": { + "Geographic Focus": "Local community funding.", + "areaOfOperation": "Lancaster" + } + }, + { + "name": "Reading Fund", + "philea_info": { + "Geographic Focus": "Local community funding.", + "areaOfOperation": "Reading" + } + }, + { + "name": "Georgia Fund", + "philea_info": { + "Geographic Focus": "Local community funding.", + "areaOfOperation": "Georgia" + } + }, + { + "name": "Jordan Fund", + "philea_info": { + "Geographic Focus": "Local community funding.", + "areaOfOperation": "Jordan" + } + }, + { + "name": "Victoria Fund", + "philea_info": { + "Geographic Focus": "Local community funding.", + "areaOfOperation": "Victoria" + } + } + ] + extract_geo(members) + for member in members: + self.assertEqual(member["geo_locations"], {}) + def test_extract_geo_substring_safety(self): members = [ { @@ -193,5 +252,60 @@ def test_extract_geo_fallback(self): self.assertIn("Europe (Western / General)", members[0]["geo_locations"]) self.assertIn("United Kingdom", members[0]["geo_locations"]["Europe (Western / General)"]) +class TestHinchillaQualityPreprocessing(unittest.TestCase): + + def test_placeholder_values_are_not_informative(self): + placeholders = [ + "Not available", + "Not publicly available", + "Not published", + "Not specified", + "Not disclosed", + "Not publicly disclosed", + "Data not available", + "Data not publicly available", + "N/A", + "Unknown", + "Not publicly disclosed (operates through partnerships)", + ] + + for value in placeholders: + self.assertTrue(has_technical_value(value)) + self.assertTrue(is_placeholder_value(value)) + self.assertFalse(is_informative_value(value)) + + self.assertTrue(is_informative_value("£250 - £1,000")) + self.assertFalse(is_placeholder_value("No public application process")) + + def test_infer_application_portal_from_existing_url(self): + url = "https://www.3rc.org.uk/grant-application" + + self.assertEqual(infer_application_portal([url]), url) + self.assertEqual(infer_application_portal(["Invitation only", "Rolling basis via online portal"]), "") + + def test_extract_email_from_existing_hinchilla_text(self): + text = "Online application form submission via email to grants@rgs.org." + + self.assertEqual(extract_email_from_text(text), "grants@rgs.org") + + def test_annual_income_is_preserved_but_not_used_as_annual_giving(self): + raw_hinchilla = { + "name": "Income Only Trust", + "philea_info": { + "annual_income": "£100", + "number_of_grants": "68 grants awarded", + "quick_stats": { + "Annual Income": "£100", + "Number of Grants": "68 grants awarded", + }, + }, + } + + clean = normalize_to_clean_schema(raw_hinchilla, "Hinchilla") + + self.assertEqual(clean["funding_info"]["annual_giving"], "") + self.assertEqual(clean["funding_info"]["annual_income"], "€120 (converted from GBP)") + self.assertEqual(clean["funding_info"]["number_of_grants"], "68 grants awarded") + if __name__ == "__main__": unittest.main()