From c7839fbf1bcd237f6872a3ad405f33da36831275 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 15:05:42 +0000 Subject: [PATCH 1/3] fix: prevent fuzzy merge from collapsing distinct conferences A canonical-csv run (d76e737) renamed PyCon South Africa to PyCon Africa and deleted the real PyCon Africa entry. Root cause: conference_scorer takes the max of several fuzzy strategies, and token_set_ratio returns 100 whenever one name's tokens are a subset of the other's - so 'PyCon South Africa' vs 'PyCon Africa' scored 100 and was auto-merged as an 'exact' match with no confirmation. Both YAML rows then mapped to the same remote row and were collapsed into one by deduplication. Fixes: - fuzzy_match only auto-merges when names are genuinely identical; score-100-but-different pairs now go through the confirmation path, which defaults to 'no' in CI - remote rows already claimed by an identically-named YAML entry can never be fuzzy-merged by another conference - add PyCon Africa <-> PyCon South Africa to rejections.yml - add utils/check_data_loss.py and a workflow step that fails the canonical-source-update run before committing if any (conference, year) entry disappears from the data files - regression tests reproducing the incident Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019bLVuyt4qdsvSpucA2qvgH --- .github/workflows/canonical-source-update.yml | 10 ++ tests/test_fuzzy_match.py | 109 ++++++++++++++++++ utils/check_data_loss.py | 90 +++++++++++++++ utils/tidy_conf/data/rejections.yml | 9 +- utils/tidy_conf/interactive_merge.py | 102 +++++++++++++--- 5 files changed, 302 insertions(+), 18 deletions(-) create mode 100644 utils/check_data_loss.py diff --git a/.github/workflows/canonical-source-update.yml b/.github/workflows/canonical-source-update.yml index 6ae3d71a803..1dfe86c35de 100644 --- a/.github/workflows/canonical-source-update.yml +++ b/.github/workflows/canonical-source-update.yml @@ -117,6 +117,7 @@ jobs: run: | cp _data/conferences.yml /tmp/conferences_before.yml cp _data/archive.yml /tmp/archive_before.yml 2>/dev/null || true + cp _data/legacy.yml /tmp/legacy_before.yml 2>/dev/null || true - name: Setup Pixi uses: prefix-dev/setup-pixi@v0.10.0 @@ -150,6 +151,15 @@ jobs: # Capture which source was processed for commit message echo "source_label=$SOURCE" >> $GITHUB_OUTPUT + - name: Verify no conference data loss + run: | + # An automated merge must never delete or rename existing conferences. + # This blocks e.g. two distinct conferences being fuzzy-matched into one + # (PyCon Africa vs PyCon South Africa) before anything is committed. + pixi run python ./utils/check_data_loss.py \ + --before /tmp/conferences_before.yml /tmp/archive_before.yml /tmp/legacy_before.yml \ + --after _data/conferences.yml _data/archive.yml _data/legacy.yml + - name: Check for changes id: check_changes run: | diff --git a/tests/test_fuzzy_match.py b/tests/test_fuzzy_match.py index f7b11ea2fd2..4216a4512c3 100644 --- a/tests/test_fuzzy_match.py +++ b/tests/test_fuzzy_match.py @@ -495,6 +495,115 @@ def test_below_90_percent_no_prompt(self, mock_title_mappings): assert len(remote) >= 1 +class TestSubsetNameCollision: + """Regression tests for distinct conferences whose names score 100. + + token_set_ratio returns 100 when one name's tokens are a subset of the + other's (e.g. "PyCon Africa" vs "PyCon South Africa"). Such pairs must + never be auto-merged as "exact" matches - that previously renamed + PyCon South Africa to PyCon Africa and deleted the real PyCon Africa + entry (commit d76e737). + """ + + @staticmethod + def _africa_yaml(include_africa: bool) -> pd.DataFrame: + rows = { + "conference": ["PyCon South Africa"], + "year": [2026], + "cfp": ["2026-06-01 23:59:00"], + "link": ["https://za.pycon.org/"], + "place": ["Rondebosch, South Africa"], + "start": ["2026-10-14"], + "end": ["2026-10-18"], + } + df = pd.DataFrame(rows) + if include_africa: + africa = pd.DataFrame( + { + "conference": ["PyCon Africa"], + "year": [2026], + "cfp": ["2026-04-16 23:59:00"], + "link": ["https://africa.pycon.org/"], + "place": ["Kampala, Uganda"], + "start": ["2026-10-07"], + "end": ["2026-10-11"], + }, + ) + df = pd.concat([df, africa], ignore_index=True) + return df + + @staticmethod + def _africa_remote() -> pd.DataFrame: + return pd.DataFrame( + { + "conference": ["PyCon Africa"], + "year": [2026], + "cfp": [""], + "link": ["https://africa.pycon.org/"], + "place": ["Kampala, Uganda"], + "start": ["2026-10-07"], + "end": ["2026-10-11"], + "sponsor": ["https://africa.pycon.org/2026/sponsor-us/"], + }, + ) + + def test_subset_name_is_not_an_exact_match(self, mock_title_mappings): + """A token-subset name pair must not auto-merge without confirmation. + + Contract: "PyCon South Africa" vs "PyCon Africa" scores 100 via + token_set_ratio, but the names are not identical, so it must go + through the fuzzy confirmation path (which defaults to "no" in CI). + """ + df_yml = self._africa_yaml(include_africa=False) + df_remote = self._africa_remote() + + # Non-interactive / user rejects: conferences stay separate + with patch("builtins.input", return_value="n"): + result, _remote, _report = fuzzy_match(df_yml, df_remote) + + conf_list = result["conference"].tolist() + assert "PyCon South Africa" in conf_list, f"PyCon South Africa was renamed/merged away: {conf_list}" + + za_row = result[result["conference"] == "PyCon South Africa"].iloc[0] + assert za_row["link"] == "https://za.pycon.org/" + assert za_row["start"] == "2026-10-14" + + def test_both_africa_conferences_survive_merge(self, mock_title_mappings): + """Reproduce the d76e737 incident: both conferences must survive. + + YAML has PyCon Africa (Kampala) and PyCon South Africa (Rondebosch); + the remote CSV has only PyCon Africa. The remote row belongs to its + identically-named YAML entry - PyCon South Africa must be left + completely alone, without prompting. + """ + df_yml = self._africa_yaml(include_africa=True) + df_remote = self._africa_remote() + + with patch( + "builtins.input", + side_effect=AssertionError("Should not prompt: remote row belongs to identical YAML entry"), + ): + result, _remote, _report = fuzzy_match(df_yml, df_remote) + + conf_list = result["conference"].tolist() + assert "PyCon Africa" in conf_list, f"PyCon Africa was lost: {conf_list}" + assert "PyCon South Africa" in conf_list, f"PyCon South Africa was lost: {conf_list}" + + za_row = result[result["conference"] == "PyCon South Africa"].iloc[0] + assert za_row["link"] == "https://za.pycon.org/" + assert za_row["place"] == "Rondebosch, South Africa" + assert za_row["start"] == "2026-10-14" + assert pd.isna(za_row.get("sponsor")) or za_row.get("sponsor") in ( + "", + None, + ), "PyCon South Africa must not inherit PyCon Africa's sponsor link" + + africa_row = result[result["conference"] == "PyCon Africa"].iloc[0] + assert africa_row["place"] == "Kampala, Uganda" + assert africa_row["start"] == "2026-10-07" + assert africa_row["sponsor"] == "https://africa.pycon.org/2026/sponsor-us/" + + class TestDataPreservation: """Test that original data is preserved through fuzzy matching.""" diff --git a/utils/check_data_loss.py b/utils/check_data_loss.py new file mode 100644 index 00000000000..4f316088e0e --- /dev/null +++ b/utils/check_data_loss.py @@ -0,0 +1,90 @@ +"""Guard against silent conference loss during automated data merges. + +Compares snapshots of the conference data files taken before a merge with the +files after the merge. Every (conference, year) pair that existed before must +still exist afterwards - in any of the given files, since conferences may +legitimately move between conferences.yml, archive.yml and legacy.yml. + +Exits non-zero if any conference disappeared, so CI can block the commit. +A legitimate rename (via titles.yml mappings) will also trip this check; +that is intentional - renames of existing entries should be reviewed by a +human, not auto-committed. +""" + +import argparse +import sys +from pathlib import Path + +import yaml + + +def load_conference_keys(paths: list[str]) -> set[tuple[str, int | str]]: + """Collect (conference, year) pairs from a list of YAML data files. + + Missing files are skipped, so the same invocation works whether or not + optional files like legacy.yml exist. + """ + keys = set() + for path in paths: + file = Path(path) + if not file.exists(): + continue + with file.open(encoding="utf-8") as f: + data = yaml.safe_load(f) + if not data: + continue + for entry in data: + if not isinstance(entry, dict): + continue + conference = entry.get("conference") + year = entry.get("year") + if conference: + keys.add((str(conference).strip(), year)) + return keys + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Fail if conferences disappeared from the data files.", + ) + parser.add_argument( + "--before", + nargs="+", + required=True, + help="Data files snapshotted before the merge", + ) + parser.add_argument( + "--after", + nargs="+", + required=True, + help="Data files after the merge", + ) + args = parser.parse_args() + + before = load_conference_keys(args.before) + after = load_conference_keys(args.after) + missing = before - after + + if missing: + print( + f"ERROR: {len(missing)} conference(s) disappeared during the merge:", + file=sys.stderr, + ) + for conference, year in sorted(missing, key=str): + print(f" - {conference} ({year})", file=sys.stderr) + print( + "\nAn automated merge must never delete or rename existing conferences.\n" + "This usually means two distinct conferences were fuzzy-matched into one\n" + "(e.g. 'PyCon Africa' vs 'PyCon South Africa'). Add the pair to\n" + "utils/tidy_conf/data/rejections.yml, or if the rename is intentional,\n" + "apply it manually.", + file=sys.stderr, + ) + return 1 + + print(f"OK: all {len(before)} conference entries survived the merge.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/tidy_conf/data/rejections.yml b/utils/tidy_conf/data/rejections.yml index d9c6e345a38..4fa46225e02 100644 --- a/utils/tidy_conf/data/rejections.yml +++ b/utils/tidy_conf/data/rejections.yml @@ -1,4 +1,3 @@ ---- alt_name: AfroPython Conference: variations: @@ -14,6 +13,11 @@ alt_name: variations: - PyCon Australia - PyCon AU + PyCon Africa: + variations: + - PyCon South Africa + - PyCon ZA + - PyConZA PyCon Austria: variations: - PyCon Australia @@ -30,6 +34,9 @@ alt_name: PyCon Latin America: variations: - PyCon Latam + PyCon South Africa: + variations: + - PyCon Africa Python Austria: variations: - PyCon Australia diff --git a/utils/tidy_conf/interactive_merge.py b/utils/tidy_conf/interactive_merge.py index d511b802d6c..c26225249a8 100644 --- a/utils/tidy_conf/interactive_merge.py +++ b/utils/tidy_conf/interactive_merge.py @@ -55,6 +55,29 @@ } +def is_identical_name(s1: str, s2: str) -> bool: + """Check whether two conference names are genuinely identical. + + A fuzzy score of 100 does NOT imply identity: token_set_ratio returns 100 + whenever one name's tokens are a subset of the other's (e.g. "PyCon Africa" + vs "PyCon South Africa"). Only names that are equal after case and + whitespace normalization may be auto-merged without confirmation. + + Parameters + ---------- + s1 : str + First conference name to compare + s2 : str + Second conference name to compare + + Returns + ------- + bool + True if the names are identical after normalization + """ + return " ".join(s1.lower().split()) == " ".join(s2.lower().split()) + + def is_placeholder_value(value) -> bool: """Check if a value is a placeholder (TBA, TBD, None, empty). @@ -276,6 +299,31 @@ def is_excluded(name1, name2): """Check if two conference names are in the combined exclusion list.""" return frozenset([name1, name2]) in all_exclusions + def best_match(title_match): + """Extract (title, score) from a process.extract result (2- or 3-tuple).""" + match_result = title_match[0] + if len(match_result) == 3: + title, prob, _ = match_result + else: + title, prob = match_result + return title, prob + + # Remote titles already spoken for by a YAML row with the genuinely + # identical name. No other YAML row may merge into these - otherwise two + # distinct conferences collapse into one row and one of them is lost + # (e.g. "PyCon South Africa" must not merge into remote "PyCon Africa" + # when a YAML "PyCon Africa" also exists). + identical_titles = set() + for _, row in df.iterrows(): + if isinstance(row["title_match"], str) or not row["title_match"]: + continue + title, _prob = best_match(row["title_match"]) + if is_identical_name(row["conference"], title): + identical_titles.add(title) + + # Remote titles claimed by a YAML row during this run (title -> yaml name) + claimed_titles = {} + # Process matches and track in report for i, row in df.iterrows(): if isinstance(row["title_match"], str): @@ -283,12 +331,7 @@ def is_excluded(name1, name2): if not row["title_match"]: continue - # Handle both 2-tuple and 3-tuple results from process.extract - match_result = row["title_match"][0] - if len(match_result) == 3: - title, prob, _ = match_result - else: - title, prob = match_result + title, prob = best_match(row["title_match"]) conference_name = row["conference"] year = row.get("year", 0) @@ -311,21 +354,45 @@ def is_excluded(name1, name2): df.at[i, "title_match"] = conference_name # Use original name, not index record.match_type = "excluded" record.action = "kept_yaml" - elif prob >= EXACT_MATCH_THRESHOLD: - logger.debug( - f"Exact match: '{conference_name}' -> '{title}' (score: {prob})", - ) - df.at[i, "title_match"] = title - record.match_type = "exact" - record.action = "merged" + elif prob >= EXACT_MATCH_THRESHOLD and is_identical_name(conference_name, title): + # Only genuinely identical names merge automatically. A score of + # 100 alone is NOT sufficient: token_set_ratio scores 100 whenever + # one name's tokens are a subset of the other's. + if title in claimed_titles and claimed_titles[title] != conference_name: + logger.warning( + f"Remote '{title}' already claimed by '{claimed_titles[title]}', " + f"keeping '{conference_name}' separate", + ) + df.at[i, "title_match"] = conference_name + record.match_type = "exact" + record.action = "kept_yaml" + else: + logger.debug( + f"Exact match: '{conference_name}' -> '{title}' (score: {prob})", + ) + df.at[i, "title_match"] = title + claimed_titles[title] = conference_name + record.match_type = "exact" + record.action = "merged" elif prob >= FUZZY_MATCH_THRESHOLD: + if title in identical_titles or title in claimed_titles: + # Remote row already belongs to its identically-named YAML + # entry (or was claimed earlier) - never merge another + # conference into it. + logger.info( + f"Skipping fuzzy match: remote '{title}' already belongs to an " + f"identically-named entry, keeping '{conference_name}' separate", + ) + df.at[i, "title_match"] = conference_name + record.match_type = "fuzzy" + record.action = "kept_yaml" # Prompt user for fuzzy matches that aren't excluded - logger.info( - f"Fuzzy match candidate: '{conference_name}' -> '{title}' (score: {prob})", - ) - if not query_yes_no( + elif not query_yes_no( f"Do '{row['conference']}' and '{title}' match? (y/n): ", ): + logger.info( + f"Fuzzy match rejected: '{conference_name}' vs '{title}' (score: {prob})", + ) new_rejections[title].append(conference_name) new_rejections[conference_name].append(title) df.at[i, "title_match"] = conference_name # Use original name, not index @@ -334,6 +401,7 @@ def is_excluded(name1, name2): else: new_mappings[conference_name].append(title) df.at[i, "title_match"] = title + claimed_titles[title] = conference_name record.match_type = "fuzzy" record.action = "merged" else: From d550afdf47df6d6755bb822d3d043a7eb810da86 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 15:37:17 +0000 Subject: [PATCH 2/3] fix: don't treat Google Maps viewport URLs as Mastodon profiles TBA enrichment's generic Mastodon fallback accepted any URL with '/@' in the path, so a venue map link like google.com/maps/place/Venue/@52.386807,4.8698442,17z was written into the mastodon field (29e9231, FastAPI Conference). The generic fallback now requires the path to be exactly /@handle (or /@user@instance). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019bLVuyt4qdsvSpucA2qvgH --- tests/test_youtube_extraction.py | 23 +++++++++++++++++++++++ utils/enrich_tba.py | 9 +++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/test_youtube_extraction.py b/tests/test_youtube_extraction.py index 87da528d1fa..e4eee280bd6 100644 --- a/tests/test_youtube_extraction.py +++ b/tests/test_youtube_extraction.py @@ -75,6 +75,29 @@ def test_generic_mastodon_still_works(self, mock_links): assert "mastodon" in result assert "youtube" not in result + @patch("enrich_tba.get_all_links") + def test_google_maps_viewport_not_mastodon(self, mock_links): + """Google Maps /@lat,lng,zoom viewport URLs must not be detected as mastodon. + + Regression test: a venue map link like + https://www.google.com/maps/place/Transformatorhuis/@52.386807,4.8698442,17z + contains "/@" but is not a Mastodon profile. + """ + mock_links.return_value = [ + "https://www.google.com/maps/place/Transformatorhuis/@52.386807,4.8698442,17z", + ] + result = extract_links_from_url("https://fastapiconf.com") + assert "mastodon" not in result + + @patch("enrich_tba.get_all_links") + def test_user_at_instance_profile_still_works(self, mock_links): + """Full /@user@instance profile paths are still detected as mastodon.""" + mock_links.return_value = [ + "https://social.example.org/@pyconf@mastodon.social", + ] + result = extract_links_from_url("https://pyconf.org") + assert result.get("mastodon") == "https://social.example.org/@pyconf@mastodon.social" + @patch("enrich_tba.get_all_links") def test_youtube_first_seen_wins(self, mock_links): """Only the first YouTube link is kept.""" diff --git a/utils/enrich_tba.py b/utils/enrich_tba.py index bed4ec4bcfe..93a24a6e10e 100644 --- a/utils/enrich_tba.py +++ b/utils/enrich_tba.py @@ -272,6 +272,11 @@ def _domain_matches(domain: str, hosts: tuple[str, ...]) -> bool: return any(domain == h or domain.endswith(f".{h}") for h in hosts) +# A Mastodon profile URL's path is exactly "/@handle" (or "/@user@instance"). +# Requiring this shape prevents false positives from other "/@" URLs, e.g. +# Google Maps viewports like "/maps/place/Venue/@52.386807,4.8698442,17z". +MASTODON_PROFILE_RE = re.compile(r"^/@[A-Za-z0-9_][\w.-]*(?:@[\w.-]+)?/?$") + # Known Mastodon instances (common ones in tech/Python community) MASTODON_INSTANCES = { "mastodon.social", @@ -366,8 +371,8 @@ def extract_links_from_url(url: str) -> dict[str, str]: found["mastodon"] = link seen_types.add("mastodon") logger.debug(f" Found mastodon: {link}") - elif "/@" in parsed_link.path: - # Generic /@username pattern - likely Mastodon-compatible + elif MASTODON_PROFILE_RE.match(parsed_link.path): + # Path is exactly /@username - likely Mastodon-compatible found["mastodon"] = link seen_types.add("mastodon") logger.debug(f" Found mastodon (generic): {link}") From 93f3df1165cecdbef3b89d71ab2b8f287c3f2417 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 15:43:02 +0000 Subject: [PATCH 3/3] feat: drop sub-page link fields that just repeat the homepage Upstream sources sometimes fill every URL column with the conference homepage (e.g. python-organizers' Proposal URL gave PyCon DE 2027 'cfp_link: https://2027.pycon.de/'). A cfp_link/sponsor/finaid identical to the main link carries no information, so the sort pipeline now drops them. URLs only count as identical when they differ at most by scheme, 'www.' prefix, or trailing slash - a different path, subdomain, query, or #anchor is a different pointer and survives. Existing offenders on main (PyCon Hong Kong cfp_link, PyCon Greece finaid, PyCon Italy finaid, PyCon Germany finaid) will be cleaned by the next recurring sort run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019bLVuyt4qdsvSpucA2qvgH --- tests/test_redundant_links.py | 122 ++++++++++++++++++++++++++++++++++ utils/sort_yaml.py | 6 ++ utils/tidy_conf/links.py | 65 ++++++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 tests/test_redundant_links.py diff --git a/tests/test_redundant_links.py b/tests/test_redundant_links.py new file mode 100644 index 00000000000..d47d7af1b8e --- /dev/null +++ b/tests/test_redundant_links.py @@ -0,0 +1,122 @@ +"""Tests for dropping sub-page link fields that just repeat the homepage. + +Upstream sources sometimes fill every URL column with the conference +homepage (e.g. python-organizers' Proposal URL). A cfp_link/sponsor/finaid +identical to the main link carries no information and must be dropped +during sanitation - but a different path, subdomain, query, or #anchor is a +different pointer and must survive. +""" + +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from tidy_conf.links import drop_redundant_link_fields +from tidy_conf.links import normalize_url_pointer + + +class TestNormalizeUrlPointer: + """Test URL normalization for same-pointer comparison.""" + + def test_scheme_ignored(self): + assert normalize_url_pointer("http://pycon.de/") == normalize_url_pointer("https://pycon.de/") + + def test_www_prefix_ignored(self): + assert normalize_url_pointer("https://www.pycon.de/") == normalize_url_pointer("https://pycon.de/") + + def test_trailing_slash_ignored(self): + assert normalize_url_pointer("https://2027.pycon.de/") == normalize_url_pointer("https://2027.pycon.de") + + def test_fragment_is_different_pointer(self): + assert normalize_url_pointer("http://pycon.sg/#sponsors") != normalize_url_pointer("http://pycon.sg/") + + def test_path_is_different_pointer(self): + assert normalize_url_pointer("https://pycon.de/sponsoring/") != normalize_url_pointer("https://pycon.de/") + + def test_subdomain_is_different_pointer(self): + assert normalize_url_pointer("https://cfp.pycon.de/") != normalize_url_pointer("https://pycon.de/") + + def test_query_is_different_pointer(self): + assert normalize_url_pointer("https://pycon.de/?page=cfp") != normalize_url_pointer("https://pycon.de/") + + +class TestDropRedundantLinkFields: + """Test removal of sub-page links identical to the main link.""" + + def test_cfp_link_same_as_link_dropped(self): + """Reproduces the PyCon DE 2027 case: cfp_link is just the homepage.""" + data = [ + { + "conference": "PyCon DE & PyData", + "year": 2027, + "link": "https://2027.pycon.de/", + "cfp_link": "https://2027.pycon.de/", + "cfp": "TBA", + }, + ] + result = drop_redundant_link_fields(data) + assert "cfp_link" not in result[0] + assert result[0]["link"] == "https://2027.pycon.de/" + + def test_all_redundant_sub_fields_dropped(self): + data = [ + { + "conference": "Cheeky Conf", + "year": 2026, + "link": "https://cheeky.conf/", + "cfp_link": "https://cheeky.conf", + "sponsor": "http://www.cheeky.conf/", + "finaid": "https://cheeky.conf/", + }, + ] + result = drop_redundant_link_fields(data) + assert "cfp_link" not in result[0] + assert "sponsor" not in result[0] + assert "finaid" not in result[0] + + def test_anchor_on_homepage_kept(self): + """A #anchor is a different pointer (e.g. PyCon SG's sponsor link).""" + data = [ + { + "conference": "PyCon Singapore", + "year": 2026, + "link": "http://pycon.sg/", + "sponsor": "http://pycon.sg/index.html#sponsors", + }, + ] + result = drop_redundant_link_fields(data) + assert result[0]["sponsor"] == "http://pycon.sg/index.html#sponsors" + + def test_sub_page_and_subdomain_kept(self): + data = [ + { + "conference": "PyCon Africa", + "year": 2026, + "link": "https://africa.pycon.org/", + "cfp_link": "https://africa.pycon.org/2026/talks/proposals/", + "sponsor": "https://africa.pycon.org/2026/sponsor-us/", + "finaid": "https://africa.pycon.org/2026/opportunity-grants/", + }, + { + "conference": "Sub Conf", + "year": 2026, + "link": "https://sub.conf/", + "cfp_link": "https://cfp.sub.conf/", + }, + ] + result = drop_redundant_link_fields(data) + assert result[0]["cfp_link"] == "https://africa.pycon.org/2026/talks/proposals/" + assert result[0]["sponsor"] == "https://africa.pycon.org/2026/sponsor-us/" + assert result[0]["finaid"] == "https://africa.pycon.org/2026/opportunity-grants/" + assert result[1]["cfp_link"] == "https://cfp.sub.conf/" + + def test_entry_without_link_untouched(self): + data = [{"conference": "No Link Conf", "year": 2026, "cfp_link": "https://example.com/"}] + result = drop_redundant_link_fields(data) + assert result[0]["cfp_link"] == "https://example.com/" + + def test_entry_without_sub_fields_untouched(self): + data = [{"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"}] + result = drop_redundant_link_fields(data) + assert result[0] == {"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"} diff --git a/utils/sort_yaml.py b/utils/sort_yaml.py index 2eba1ae2ed3..c627ee95136 100644 --- a/utils/sort_yaml.py +++ b/utils/sort_yaml.py @@ -22,6 +22,7 @@ from tidy_conf.latlon import add_latlon from tidy_conf.links import check_link_availability from tidy_conf.links import check_mastodon_migration + from tidy_conf.links import drop_redundant_link_fields from tidy_conf.links import get_cache from tidy_conf.schema import Conference from tidy_conf.schema import get_schema @@ -35,6 +36,7 @@ from .tidy_conf.latlon import add_latlon from .tidy_conf.links import check_link_availability from .tidy_conf.links import check_mastodon_migration + from .tidy_conf.links import drop_redundant_link_fields from .tidy_conf.links import get_cache from .tidy_conf.schema import Conference from .tidy_conf.schema import get_schema @@ -289,6 +291,10 @@ def sort_data(base="", prefix="", skip_links=False): logger.info("🏷️ Cleaning titles") data = tidy_titles(data) + # Drop sub-page links that just repeat the homepage + logger.info("🔗 Dropping redundant link fields") + data = drop_redundant_link_fields(data) + # Add Sub logger.info("🏢 Adding submission types") data = auto_add_sub(data) diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index c630d352a23..a3adec72417 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -9,6 +9,71 @@ import requests from tqdm import tqdm +# Sub-page link fields that must point somewhere other than the main link. +# A cfp_link/sponsor/finaid that is just the homepage again carries no +# information (e.g. "cfp_link: https://2027.pycon.de/" on the PyCon DE entry +# whose link is https://2027.pycon.de/) - it should be a different page, a +# subdomain, or at least a #anchor on the homepage. +REDUNDANT_LINK_FIELDS = ("cfp_link", "sponsor", "finaid") + + +def normalize_url_pointer(url: str) -> tuple: + """Normalize a URL for same-pointer comparison. + + Two URLs are the same pointer when they only differ by scheme + (http/https), a "www." prefix, host case, or a trailing slash. + Anything else - path, subdomain, query, or #fragment - makes them + different pointers. + + Parameters + ---------- + url : str + URL to normalize + + Returns + ------- + tuple + Comparable (host, path, params, query, fragment) tuple + """ + parsed = urlparse(str(url).strip()) + netloc = parsed.netloc.lower().removeprefix("www.") + return (netloc, parsed.path.rstrip("/"), parsed.params, parsed.query, parsed.fragment) + + +def drop_redundant_link_fields(data: list) -> list: + """Drop sub-page link fields that are the same pointer as the main link. + + Goes through cfp_link/sponsor/finaid on each conference and + removes any that just repeat the conference's main link. This catches + upstream sources that fill every URL column with the homepage. + + Parameters + ---------- + data : list + List of conference dictionaries + + Returns + ------- + list + The same list with redundant link fields removed + """ + for q in data: + if not isinstance(q, dict): + continue + link = q.get("link") + if not link: + continue + base = normalize_url_pointer(link) + for field in REDUNDANT_LINK_FIELDS: + value = q.get(field) + if value and normalize_url_pointer(value) == base: + tqdm.write( + f"Dropping redundant '{field}' from {q.get('conference')} {q.get('year')}: " + f"'{value}' is the same pointer as the main link", + ) + del q[field] + return data + def get_cache_location(): # Check if the URL is cached