Improve Hinchilla scraping and preprocessing quality

src/preprocessing/consolidate.py

@@ -4,6 +4,11 @@
 import logging
 from urllib.parse import urlparse
 
+try:
+    from preprocessing.quality import is_informative_value, is_placeholder_value
+except ModuleNotFoundError:
+    from quality import is_informative_value, is_placeholder_value
+
 # Configure logger
 logger = logging.getLogger("consolidate")
 if not logger.handlers:
@@ -20,7 +25,7 @@ def _extract_number(val_str):
     Extracts a numeric float value from a string, ignoring currency symbols and years.
     Handles standard number formats (thousands separators, decimals).
     """
-    if not val_str or "not publicly available" in val_str.lower() or "not disclosed" in val_str.lower():
+    if not val_str or is_placeholder_value(val_str):
         return None
 
     # Remove year patterns like (2024) or [2022] or (2023-24) to avoid matching the year as the number
@@ -257,25 +262,33 @@ def normalize_to_clean_schema(member, source_name):
 
     # Standalone Hinchilla financial conversion (or general Hinchilla standardization)
     annual_giving = p_info.get("annual_giving", "")
+    annual_income = p_info.get("annual_income", "")
+    annual_expenditure = p_info.get("annual_expenditure", "")
     average_grant = p_info.get("average_grant", "")
     grant_range = p_info.get("grant_range", "")
     expenditure = p_info.get("expenditure", "")
 
     if source_name == "Hinchilla":
         annual_giving = convert_gbp_to_eur(annual_giving)
+        annual_income = convert_gbp_to_eur(annual_income)
+        annual_expenditure = convert_gbp_to_eur(annual_expenditure)
         average_grant = convert_gbp_to_eur(average_grant)
         grant_range = convert_gbp_to_eur(grant_range)
         expenditure = convert_gbp_to_eur(expenditure)
 
     funding_info = {
         "annual_giving": annual_giving,
+        "annual_income": annual_income,
+        "annual_expenditure": annual_expenditure,
         "average_grant": average_grant,
         "grant_range": grant_range,
         "funding_model": p_info.get("funding_model", ""),
         "application_details": p_info.get("application_details", ""),
         "success_rate": p_info.get("success_rate", ""),
         "decision_time": p_info.get("decision_time", ""),
         "expenditure": expenditure,
+        "number_of_grants": p_info.get("number_of_grants", ""),
+        "quick_stats": p_info.get("quick_stats", {}),
         "charity_number": p_info.get("charityNumber", "") or p_info.get("charity_number", ""),
         "application_portal": p_info.get("applicationPortal", "") or p_info.get("application_portal", ""),
         "sources": p_info.get("sources", [])
@@ -317,9 +330,9 @@ def merge_members(p_member, h_member):
 
     # Helper to resolve field value and check discrepancies
     def resolve_field(field_name, p_val, h_val, is_financial=False):
-        if not p_val or p_val.lower() in ["not publicly available", "not disclosed", ""]:
-            return h_val if h_val else p_val
-        if not h_val or h_val.lower() in ["not publicly available", "not disclosed", ""]:
+        if not is_informative_value(p_val):
+            return h_val if is_informative_value(h_val) else p_val
+        if not is_informative_value(h_val):
             return p_val
 
         # Both are non-empty. Compare.
@@ -389,7 +402,11 @@ def resolve_field(field_name, p_val, h_val, is_financial=False):
 
     # Union metadata
     for key in ["success_rate", "decision_time", "charity_number", "application_portal"]:
-        if h_fund.get(key):
+        if is_informative_value(h_fund.get(key)):
+            p_fund[key] = h_fund[key]
+
+    for key in ["annual_income", "annual_expenditure", "number_of_grants", "quick_stats"]:
+        if is_informative_value(h_fund.get(key)) and not is_informative_value(p_fund.get(key)):
             p_fund[key] = h_fund[key]
 
     # Text merge (prefer longer)

src/preprocessing/extract_geo_topic.py

@@ -509,6 +509,182 @@ def extract_geos_final(raw_text):
                     found[macro].add(country_name)
 
         return {k: sorted(list(v)) for k, v in found.items()}
+
+    def normalize_location_text(raw_text):
+        return re.sub(r"[^a-z0-9]+", " ", raw_text.lower()).strip()
+
+    def contains_location_phrase(text_clean, phrase):
+        phrase_clean = normalize_location_text(phrase)
+        if not phrase_clean:
+            return False
+        pattern = r"(?<![a-z0-9])" + re.escape(phrase_clean) + r"(?![a-z0-9])"
+        return re.search(pattern, text_clean) is not None
+
+    def merge_country_result(found, country_name):
+        macro = COUNTRY_TO_MACRO.get(country_name)
+        if not macro:
+            return
+        if macro not in found:
+            found[macro] = set()
+        elif not isinstance(found[macro], set):
+            found[macro] = set(found[macro])
+        found[macro].add(country_name)
+
+    def normalize_geo_result(found):
+        return {k: sorted(list(v)) for k, v in found.items() if v}
+
+    def stringify_geo_value(value):
+        if isinstance(value, str):
+            return value
+        if isinstance(value, list):
+            return " ".join(stringify_geo_value(item) for item in value)
+        if isinstance(value, dict):
+            return " ".join(stringify_geo_value(item) for item in value.values())
+        return ""
+
+    def collect_fallback_text(member, info):
+        field_markers = (
+            "address",
+            "area",
+            "country",
+            "description",
+            "eligib",
+            "geo",
+            "location",
+            "operation",
+            "program",
+            "programme",
+            "region",
+        )
+        fallback_parts = [
+            member.get("address", ""),
+            member.get("country", ""),
+            member.get("position", {}).get("country", "") if isinstance(member.get("position"), dict) else "",
+            member.get("name", ""),
+        ]
+        for source in (member, info):
+            if not isinstance(source, dict):
+                continue
+            for key, value in source.items():
+                key_lower = str(key).lower()
+                if any(marker in key_lower for marker in field_markers):
+                    fallback_parts.append(stringify_geo_value(value))
+        return " ".join(part for part in fallback_parts if part).strip()
+
+    LOCAL_FALLBACK_GAZETTEER = {
+        "United Kingdom": {
+            "terms": [
+                "cumbria", "lancashire", "north lancashire", "south cumbria",
+                "morecambe", "kendal", "carnforth", "greater manchester",
+                "manchester", "liverpool", "leeds", "bristol", "birmingham",
+                "newcastle upon tyne", "glasgow", "edinburgh", "cardiff",
+                "belfast", "yorkshire", "derbyshire", "nottinghamshire",
+                "surrey", "kent", "sussex", "berkshire"
+            ],
+            "contextual_terms": {
+                "lancaster": ["cumbria", "lancashire", "morecambe", "kendal", "carnforth", "north west england", "uk", "england"],
+                "reading": ["berkshire", "thames valley", "uk", "england", "united kingdom"],
+            },
+        },
+        "Ireland": {"terms": ["dublin", "cork", "galway", "limerick"]},
+        "France": {"terms": ["paris", "lyon", "marseille", "bordeaux", "normandy", "brittany"]},
+        "Germany": {"terms": ["berlin", "munich", "hamburg", "frankfurt", "bavaria", "north rhine westphalia"]},
+        "Switzerland": {"terms": ["zurich", "geneva", "basel", "bern"]},
+        "Austria": {"terms": ["vienna", "salzburg", "tyrol"]},
+        "Netherlands": {"terms": ["amsterdam", "rotterdam", "utrecht", "eindhoven"]},
+        "Belgium": {"terms": ["antwerp", "flanders", "wallonia"]},
+        "Spain": {"terms": ["madrid", "barcelona", "catalonia", "andalusia", "valencia"]},
+        "Italy": {"terms": ["rome", "milan", "naples", "venice", "sicily", "sardinia", "tuscany"]},
+        "Portugal": {"terms": ["lisbon", "porto", "algarve"]},
+        "Greece": {"terms": ["athens", "crete", "thessaloniki"]},
+        "Denmark": {"terms": ["copenhagen", "jutland"]},
+        "Sweden": {"terms": ["stockholm", "gothenburg", "malmo"]},
+        "Norway": {"terms": ["oslo", "bergen", "trondheim"]},
+        "Finland": {"terms": ["helsinki", "espoo", "tampere"]},
+        "Poland": {"terms": ["warsaw", "krakow", "gdansk"]},
+        "Ukraine": {"terms": ["kyiv", "kiev", "lviv", "odesa", "odessa"]},
+        "Czech Republic": {"terms": ["prague", "brno"]},
+        "Romania": {"terms": ["bucharest", "cluj", "transylvania"]},
+        "United States": {
+            "terms": [
+                "california", "new york", "new york city", "texas", "florida",
+                "illinois", "chicago", "massachusetts", "boston", "colorado",
+                "denver", "michigan", "detroit", "pennsylvania", "philadelphia",
+                "ohio", "oregon", "portland", "seattle", "los angeles",
+                "san francisco", "washington dc", "district of columbia"
+            ],
+            "contextual_terms": {
+                "washington": ["seattle", "puget sound", "usa", "united states", "pacific northwest"],
+            },
+        },
+        "Canada": {"terms": ["ontario", "quebec", "toronto", "montreal", "vancouver", "british columbia", "alberta"]},
+        "Mexico": {"terms": ["mexico city", "yucatan", "oaxaca"]},
+        "Brazil": {"terms": ["sao paulo", "rio de janeiro", "bahia", "amazonas"]},
+        "Argentina": {"terms": ["buenos aires", "patagonia"]},
+        "Colombia": {"terms": ["bogota", "medellin"]},
+        "Chile": {"terms": ["santiago", "valparaiso"]},
+        "South Africa": {"terms": ["cape town", "johannesburg", "gauteng", "kwazulu natal"]},
+        "Kenya": {"terms": ["nairobi", "mombasa"]},
+        "Nigeria": {"terms": ["lagos", "abuja"]},
+        "Ghana": {"terms": ["accra", "kumasi"]},
+        "Tanzania": {"terms": ["dar es salaam", "zanzibar"]},
+        "India": {"terms": ["delhi", "new delhi", "mumbai", "bangalore", "bengaluru", "karnataka", "maharashtra"]},
+        "China": {"terms": ["beijing", "shanghai", "guangdong", "shenzhen"]},
+        "Japan": {"terms": ["tokyo", "osaka", "kyoto"]},
+        "South Korea": {"terms": ["seoul", "busan"]},
+        "Indonesia": {"terms": ["jakarta", "bali", "java"]},
+        "Thailand": {"terms": ["bangkok", "chiang mai"]},
+        "Vietnam": {"terms": ["hanoi", "ho chi minh city"]},
+        "Australia": {
+            "terms": ["sydney", "melbourne", "brisbane", "perth", "adelaide", "new south wales", "queensland", "western australia", "tasmania"],
+            "contextual_terms": {
+                "victoria": ["australia", "melbourne", "new south wales", "queensland", "tasmania"],
+            },
+        },
+        "New Zealand": {"terms": ["auckland", "wellington", "christchurch"]},
+        "Israel": {"terms": ["tel aviv", "jerusalem", "haifa"]},
+        "Palestine": {"terms": ["west bank", "gaza"]},
+        "Egypt": {"terms": ["cairo", "alexandria"]},
+        "Lebanon": {"terms": ["beirut"]},
+        "Morocco": {"terms": ["casablanca", "marrakesh", "rabat"]},
+        "United Arab Emirates": {"terms": ["dubai", "abu dhabi"]},
+    }
+
+    AMBIGUOUS_FALLBACK_CONTEXT = {
+        "Georgia": ["tbilisi", "caucasus", "black sea", "armenia", "azerbaijan", "eastern europe"],
+        "Jordan": ["amman", "middle east", "mena", "levant", "palestine", "syria", "iraq", "israel"],
+    }
+
+    def filter_ambiguous_fallback_geos(geos, raw_text):
+        text_clean = normalize_location_text(raw_text)
+        filtered = {macro: set(countries) for macro, countries in geos.items()}
+        for country_name, context_terms in AMBIGUOUS_FALLBACK_CONTEXT.items():
+            macro = COUNTRY_TO_MACRO.get(country_name)
+            if not macro or country_name not in filtered.get(macro, set()):
+                continue
+            if not any(contains_location_phrase(text_clean, term) for term in context_terms):
+                filtered[macro].discard(country_name)
+        return normalize_geo_result(filtered)
+
+    def extract_geos_from_fallback_gazetteer(raw_text):
+        if not raw_text:
+            return {}
+        text_clean = normalize_location_text(raw_text)
+        found = {}
+        for country_name, config in LOCAL_FALLBACK_GAZETTEER.items():
+            for term in config.get("terms", []):
+                if contains_location_phrase(text_clean, term):
+                    merge_country_result(found, country_name)
+                    break
+            if country_name in [country for countries in found.values() for country in countries]:
+                continue
+            for term, context_terms in config.get("contextual_terms", {}).items():
+                if contains_location_phrase(text_clean, term) and any(
+                    contains_location_phrase(text_clean, context) for context in context_terms
+                ):
+                    merge_country_result(found, country_name)
+                    break
+        return normalize_geo_result(found)
 
     parsed_geo_results = {}
     for member in members:
@@ -530,12 +706,7 @@ def extract_geos_final(raw_text):
 
         # If no geolocations could be extracted from Geographic Focus, fall back to other fields
         if not geos:
-            addr = member.get("address", "") or info.get("address", "") or ""
-            country = member.get("country", "") or member.get("position", {}).get("country", "") or ""
-            aop = info.get("areaOfOperation", "") or ""
-            about = info.get("About", "") or ""
-            name = member.get("name", "")
-            fallback_text = f"{addr} {country} {aop} {about} {name}".strip()
+            fallback_text = collect_fallback_text(member, info)
             if fallback_text:
                 robust = extract_geos_robust(fallback_text)
                 final_geos = extract_geos_final(fallback_text)
@@ -544,6 +715,12 @@ def extract_geos_final(raw_text):
                     if macro not in geos:
                         geos[macro] = []
                     geos[macro] = sorted(list(set(geos[macro]) | set(countries)))
+                geos = filter_ambiguous_fallback_geos(geos, fallback_text)
+                fallback_geos = extract_geos_from_fallback_gazetteer(fallback_text)
+                for macro, countries in fallback_geos.items():
+                    if macro not in geos:
+                        geos[macro] = []
+                    geos[macro] = sorted(list(set(geos[macro]) | set(countries)))
 
         parsed_geo_results[member_name] = geos
 
@@ -593,4 +770,4 @@ def load_data(path):
         extract_geo(members)
         save_data(members, args.output)
     except Exception as e:
-        logging.error(f"Preprocessing pipeline failed: {e}")
+        logging.error(f"Preprocessing pipeline failed: {e}")

src/preprocessing/quality.py

@@ -0,0 +1,42 @@
+import re
+
+
+_PLACEHOLDER_RE = re.compile(
+    r"^\s*(?:"
+    r"n/?a|"
+    r"unknown|"
+    r"not\s+available|"
+    r"not\s+publicly\s+available|"
+    r"not\s+published|"
+    r"not\s+specified|"
+    r"not\s+publicly\s+specified|"
+    r"not\s+disclosed|"
+    r"not\s+publicly\s+disclosed|"
+    r"not\s+applicable|"
+    r"data\s+not\s+available|"
+    r"data\s+not\s+publicly\s+available|"
+    r"information\s+not\s+available|"
+    r"information\s+not\s+publicly\s+available"
+    r")(?:\s|$|[().,:;-])",
+    re.IGNORECASE,
+)
+
+
+def has_technical_value(value):
+    if value is None:
+        return False
+    if isinstance(value, str):
+        return bool(value.strip())
+    if isinstance(value, (list, tuple, set, dict)):
+        return bool(value)
+    return True
+
+
+def is_placeholder_value(value):
+    if not isinstance(value, str):
+        return False
+    return bool(_PLACEHOLDER_RE.match(value.strip()))
+
+
+def is_informative_value(value):
+    return has_technical_value(value) and not is_placeholder_value(value)