diff --git a/.github/workflows/canonical-source-update.yml b/.github/workflows/canonical-source-update.yml index 6ae3d71a80..1dfe86c35d 100644 --- a/.github/workflows/canonical-source-update.yml +++ b/.github/workflows/canonical-source-update.yml @@ -117,6 +117,7 @@ jobs: run: | cp _data/conferences.yml /tmp/conferences_before.yml cp _data/archive.yml /tmp/archive_before.yml 2>/dev/null || true + cp _data/legacy.yml /tmp/legacy_before.yml 2>/dev/null || true - name: Setup Pixi uses: prefix-dev/setup-pixi@v0.10.0 @@ -150,6 +151,15 @@ jobs: # Capture which source was processed for commit message echo "source_label=$SOURCE" >> $GITHUB_OUTPUT + - name: Verify no conference data loss + run: | + # An automated merge must never delete or rename existing conferences. + # This blocks e.g. two distinct conferences being fuzzy-matched into one + # (PyCon Africa vs PyCon South Africa) before anything is committed. + pixi run python ./utils/check_data_loss.py \ + --before /tmp/conferences_before.yml /tmp/archive_before.yml /tmp/legacy_before.yml \ + --after _data/conferences.yml _data/archive.yml _data/legacy.yml + - name: Check for changes id: check_changes run: | diff --git a/tests/test_fuzzy_match.py b/tests/test_fuzzy_match.py index f7b11ea2fd..4216a4512c 100644 --- a/tests/test_fuzzy_match.py +++ b/tests/test_fuzzy_match.py @@ -495,6 +495,115 @@ def test_below_90_percent_no_prompt(self, mock_title_mappings): assert len(remote) >= 1 +class TestSubsetNameCollision: + """Regression tests for distinct conferences whose names score 100. + + token_set_ratio returns 100 when one name's tokens are a subset of the + other's (e.g. "PyCon Africa" vs "PyCon South Africa"). Such pairs must + never be auto-merged as "exact" matches - that previously renamed + PyCon South Africa to PyCon Africa and deleted the real PyCon Africa + entry (commit d76e737). + """ + + @staticmethod + def _africa_yaml(include_africa: bool) -> pd.DataFrame: + rows = { + "conference": ["PyCon South Africa"], + "year": [2026], + "cfp": ["2026-06-01 23:59:00"], + "link": ["https://za.pycon.org/"], + "place": ["Rondebosch, South Africa"], + "start": ["2026-10-14"], + "end": ["2026-10-18"], + } + df = pd.DataFrame(rows) + if include_africa: + africa = pd.DataFrame( + { + "conference": ["PyCon Africa"], + "year": [2026], + "cfp": ["2026-04-16 23:59:00"], + "link": ["https://africa.pycon.org/"], + "place": ["Kampala, Uganda"], + "start": ["2026-10-07"], + "end": ["2026-10-11"], + }, + ) + df = pd.concat([df, africa], ignore_index=True) + return df + + @staticmethod + def _africa_remote() -> pd.DataFrame: + return pd.DataFrame( + { + "conference": ["PyCon Africa"], + "year": [2026], + "cfp": [""], + "link": ["https://africa.pycon.org/"], + "place": ["Kampala, Uganda"], + "start": ["2026-10-07"], + "end": ["2026-10-11"], + "sponsor": ["https://africa.pycon.org/2026/sponsor-us/"], + }, + ) + + def test_subset_name_is_not_an_exact_match(self, mock_title_mappings): + """A token-subset name pair must not auto-merge without confirmation. + + Contract: "PyCon South Africa" vs "PyCon Africa" scores 100 via + token_set_ratio, but the names are not identical, so it must go + through the fuzzy confirmation path (which defaults to "no" in CI). + """ + df_yml = self._africa_yaml(include_africa=False) + df_remote = self._africa_remote() + + # Non-interactive / user rejects: conferences stay separate + with patch("builtins.input", return_value="n"): + result, _remote, _report = fuzzy_match(df_yml, df_remote) + + conf_list = result["conference"].tolist() + assert "PyCon South Africa" in conf_list, f"PyCon South Africa was renamed/merged away: {conf_list}" + + za_row = result[result["conference"] == "PyCon South Africa"].iloc[0] + assert za_row["link"] == "https://za.pycon.org/" + assert za_row["start"] == "2026-10-14" + + def test_both_africa_conferences_survive_merge(self, mock_title_mappings): + """Reproduce the d76e737 incident: both conferences must survive. + + YAML has PyCon Africa (Kampala) and PyCon South Africa (Rondebosch); + the remote CSV has only PyCon Africa. The remote row belongs to its + identically-named YAML entry - PyCon South Africa must be left + completely alone, without prompting. + """ + df_yml = self._africa_yaml(include_africa=True) + df_remote = self._africa_remote() + + with patch( + "builtins.input", + side_effect=AssertionError("Should not prompt: remote row belongs to identical YAML entry"), + ): + result, _remote, _report = fuzzy_match(df_yml, df_remote) + + conf_list = result["conference"].tolist() + assert "PyCon Africa" in conf_list, f"PyCon Africa was lost: {conf_list}" + assert "PyCon South Africa" in conf_list, f"PyCon South Africa was lost: {conf_list}" + + za_row = result[result["conference"] == "PyCon South Africa"].iloc[0] + assert za_row["link"] == "https://za.pycon.org/" + assert za_row["place"] == "Rondebosch, South Africa" + assert za_row["start"] == "2026-10-14" + assert pd.isna(za_row.get("sponsor")) or za_row.get("sponsor") in ( + "", + None, + ), "PyCon South Africa must not inherit PyCon Africa's sponsor link" + + africa_row = result[result["conference"] == "PyCon Africa"].iloc[0] + assert africa_row["place"] == "Kampala, Uganda" + assert africa_row["start"] == "2026-10-07" + assert africa_row["sponsor"] == "https://africa.pycon.org/2026/sponsor-us/" + + class TestDataPreservation: """Test that original data is preserved through fuzzy matching.""" diff --git a/tests/test_redundant_links.py b/tests/test_redundant_links.py new file mode 100644 index 0000000000..d47d7af1b8 --- /dev/null +++ b/tests/test_redundant_links.py @@ -0,0 +1,122 @@ +"""Tests for dropping sub-page link fields that just repeat the homepage. + +Upstream sources sometimes fill every URL column with the conference +homepage (e.g. python-organizers' Proposal URL). A cfp_link/sponsor/finaid +identical to the main link carries no information and must be dropped +during sanitation - but a different path, subdomain, query, or #anchor is a +different pointer and must survive. +""" + +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from tidy_conf.links import drop_redundant_link_fields +from tidy_conf.links import normalize_url_pointer + + +class TestNormalizeUrlPointer: + """Test URL normalization for same-pointer comparison.""" + + def test_scheme_ignored(self): + assert normalize_url_pointer("http://pycon.de/") == normalize_url_pointer("https://pycon.de/") + + def test_www_prefix_ignored(self): + assert normalize_url_pointer("https://www.pycon.de/") == normalize_url_pointer("https://pycon.de/") + + def test_trailing_slash_ignored(self): + assert normalize_url_pointer("https://2027.pycon.de/") == normalize_url_pointer("https://2027.pycon.de") + + def test_fragment_is_different_pointer(self): + assert normalize_url_pointer("http://pycon.sg/#sponsors") != normalize_url_pointer("http://pycon.sg/") + + def test_path_is_different_pointer(self): + assert normalize_url_pointer("https://pycon.de/sponsoring/") != normalize_url_pointer("https://pycon.de/") + + def test_subdomain_is_different_pointer(self): + assert normalize_url_pointer("https://cfp.pycon.de/") != normalize_url_pointer("https://pycon.de/") + + def test_query_is_different_pointer(self): + assert normalize_url_pointer("https://pycon.de/?page=cfp") != normalize_url_pointer("https://pycon.de/") + + +class TestDropRedundantLinkFields: + """Test removal of sub-page links identical to the main link.""" + + def test_cfp_link_same_as_link_dropped(self): + """Reproduces the PyCon DE 2027 case: cfp_link is just the homepage.""" + data = [ + { + "conference": "PyCon DE & PyData", + "year": 2027, + "link": "https://2027.pycon.de/", + "cfp_link": "https://2027.pycon.de/", + "cfp": "TBA", + }, + ] + result = drop_redundant_link_fields(data) + assert "cfp_link" not in result[0] + assert result[0]["link"] == "https://2027.pycon.de/" + + def test_all_redundant_sub_fields_dropped(self): + data = [ + { + "conference": "Cheeky Conf", + "year": 2026, + "link": "https://cheeky.conf/", + "cfp_link": "https://cheeky.conf", + "sponsor": "http://www.cheeky.conf/", + "finaid": "https://cheeky.conf/", + }, + ] + result = drop_redundant_link_fields(data) + assert "cfp_link" not in result[0] + assert "sponsor" not in result[0] + assert "finaid" not in result[0] + + def test_anchor_on_homepage_kept(self): + """A #anchor is a different pointer (e.g. PyCon SG's sponsor link).""" + data = [ + { + "conference": "PyCon Singapore", + "year": 2026, + "link": "http://pycon.sg/", + "sponsor": "http://pycon.sg/index.html#sponsors", + }, + ] + result = drop_redundant_link_fields(data) + assert result[0]["sponsor"] == "http://pycon.sg/index.html#sponsors" + + def test_sub_page_and_subdomain_kept(self): + data = [ + { + "conference": "PyCon Africa", + "year": 2026, + "link": "https://africa.pycon.org/", + "cfp_link": "https://africa.pycon.org/2026/talks/proposals/", + "sponsor": "https://africa.pycon.org/2026/sponsor-us/", + "finaid": "https://africa.pycon.org/2026/opportunity-grants/", + }, + { + "conference": "Sub Conf", + "year": 2026, + "link": "https://sub.conf/", + "cfp_link": "https://cfp.sub.conf/", + }, + ] + result = drop_redundant_link_fields(data) + assert result[0]["cfp_link"] == "https://africa.pycon.org/2026/talks/proposals/" + assert result[0]["sponsor"] == "https://africa.pycon.org/2026/sponsor-us/" + assert result[0]["finaid"] == "https://africa.pycon.org/2026/opportunity-grants/" + assert result[1]["cfp_link"] == "https://cfp.sub.conf/" + + def test_entry_without_link_untouched(self): + data = [{"conference": "No Link Conf", "year": 2026, "cfp_link": "https://example.com/"}] + result = drop_redundant_link_fields(data) + assert result[0]["cfp_link"] == "https://example.com/" + + def test_entry_without_sub_fields_untouched(self): + data = [{"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"}] + result = drop_redundant_link_fields(data) + assert result[0] == {"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"} diff --git a/tests/test_youtube_extraction.py b/tests/test_youtube_extraction.py index 87da528d1f..e4eee280bd 100644 --- a/tests/test_youtube_extraction.py +++ b/tests/test_youtube_extraction.py @@ -75,6 +75,29 @@ def test_generic_mastodon_still_works(self, mock_links): assert "mastodon" in result assert "youtube" not in result + @patch("enrich_tba.get_all_links") + def test_google_maps_viewport_not_mastodon(self, mock_links): + """Google Maps /@lat,lng,zoom viewport URLs must not be detected as mastodon. + + Regression test: a venue map link like + https://www.google.com/maps/place/Transformatorhuis/@52.386807,4.8698442,17z + contains "/@" but is not a Mastodon profile. + """ + mock_links.return_value = [ + "https://www.google.com/maps/place/Transformatorhuis/@52.386807,4.8698442,17z", + ] + result = extract_links_from_url("https://fastapiconf.com") + assert "mastodon" not in result + + @patch("enrich_tba.get_all_links") + def test_user_at_instance_profile_still_works(self, mock_links): + """Full /@user@instance profile paths are still detected as mastodon.""" + mock_links.return_value = [ + "https://social.example.org/@pyconf@mastodon.social", + ] + result = extract_links_from_url("https://pyconf.org") + assert result.get("mastodon") == "https://social.example.org/@pyconf@mastodon.social" + @patch("enrich_tba.get_all_links") def test_youtube_first_seen_wins(self, mock_links): """Only the first YouTube link is kept.""" diff --git a/utils/check_data_loss.py b/utils/check_data_loss.py new file mode 100644 index 0000000000..4f316088e0 --- /dev/null +++ b/utils/check_data_loss.py @@ -0,0 +1,90 @@ +"""Guard against silent conference loss during automated data merges. + +Compares snapshots of the conference data files taken before a merge with the +files after the merge. Every (conference, year) pair that existed before must +still exist afterwards - in any of the given files, since conferences may +legitimately move between conferences.yml, archive.yml and legacy.yml. + +Exits non-zero if any conference disappeared, so CI can block the commit. +A legitimate rename (via titles.yml mappings) will also trip this check; +that is intentional - renames of existing entries should be reviewed by a +human, not auto-committed. +""" + +import argparse +import sys +from pathlib import Path + +import yaml + + +def load_conference_keys(paths: list[str]) -> set[tuple[str, int | str]]: + """Collect (conference, year) pairs from a list of YAML data files. + + Missing files are skipped, so the same invocation works whether or not + optional files like legacy.yml exist. + """ + keys = set() + for path in paths: + file = Path(path) + if not file.exists(): + continue + with file.open(encoding="utf-8") as f: + data = yaml.safe_load(f) + if not data: + continue + for entry in data: + if not isinstance(entry, dict): + continue + conference = entry.get("conference") + year = entry.get("year") + if conference: + keys.add((str(conference).strip(), year)) + return keys + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Fail if conferences disappeared from the data files.", + ) + parser.add_argument( + "--before", + nargs="+", + required=True, + help="Data files snapshotted before the merge", + ) + parser.add_argument( + "--after", + nargs="+", + required=True, + help="Data files after the merge", + ) + args = parser.parse_args() + + before = load_conference_keys(args.before) + after = load_conference_keys(args.after) + missing = before - after + + if missing: + print( + f"ERROR: {len(missing)} conference(s) disappeared during the merge:", + file=sys.stderr, + ) + for conference, year in sorted(missing, key=str): + print(f" - {conference} ({year})", file=sys.stderr) + print( + "\nAn automated merge must never delete or rename existing conferences.\n" + "This usually means two distinct conferences were fuzzy-matched into one\n" + "(e.g. 'PyCon Africa' vs 'PyCon South Africa'). Add the pair to\n" + "utils/tidy_conf/data/rejections.yml, or if the rename is intentional,\n" + "apply it manually.", + file=sys.stderr, + ) + return 1 + + print(f"OK: all {len(before)} conference entries survived the merge.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/enrich_tba.py b/utils/enrich_tba.py index bed4ec4bcf..93a24a6e10 100644 --- a/utils/enrich_tba.py +++ b/utils/enrich_tba.py @@ -272,6 +272,11 @@ def _domain_matches(domain: str, hosts: tuple[str, ...]) -> bool: return any(domain == h or domain.endswith(f".{h}") for h in hosts) +# A Mastodon profile URL's path is exactly "/@handle" (or "/@user@instance"). +# Requiring this shape prevents false positives from other "/@" URLs, e.g. +# Google Maps viewports like "/maps/place/Venue/@52.386807,4.8698442,17z". +MASTODON_PROFILE_RE = re.compile(r"^/@[A-Za-z0-9_][\w.-]*(?:@[\w.-]+)?/?$") + # Known Mastodon instances (common ones in tech/Python community) MASTODON_INSTANCES = { "mastodon.social", @@ -366,8 +371,8 @@ def extract_links_from_url(url: str) -> dict[str, str]: found["mastodon"] = link seen_types.add("mastodon") logger.debug(f" Found mastodon: {link}") - elif "/@" in parsed_link.path: - # Generic /@username pattern - likely Mastodon-compatible + elif MASTODON_PROFILE_RE.match(parsed_link.path): + # Path is exactly /@username - likely Mastodon-compatible found["mastodon"] = link seen_types.add("mastodon") logger.debug(f" Found mastodon (generic): {link}") diff --git a/utils/sort_yaml.py b/utils/sort_yaml.py index 2eba1ae2ed..c627ee9513 100644 --- a/utils/sort_yaml.py +++ b/utils/sort_yaml.py @@ -22,6 +22,7 @@ from tidy_conf.latlon import add_latlon from tidy_conf.links import check_link_availability from tidy_conf.links import check_mastodon_migration + from tidy_conf.links import drop_redundant_link_fields from tidy_conf.links import get_cache from tidy_conf.schema import Conference from tidy_conf.schema import get_schema @@ -35,6 +36,7 @@ from .tidy_conf.latlon import add_latlon from .tidy_conf.links import check_link_availability from .tidy_conf.links import check_mastodon_migration + from .tidy_conf.links import drop_redundant_link_fields from .tidy_conf.links import get_cache from .tidy_conf.schema import Conference from .tidy_conf.schema import get_schema @@ -289,6 +291,10 @@ def sort_data(base="", prefix="", skip_links=False): logger.info("🏷️ Cleaning titles") data = tidy_titles(data) + # Drop sub-page links that just repeat the homepage + logger.info("🔗 Dropping redundant link fields") + data = drop_redundant_link_fields(data) + # Add Sub logger.info("🏢 Adding submission types") data = auto_add_sub(data) diff --git a/utils/tidy_conf/data/rejections.yml b/utils/tidy_conf/data/rejections.yml index d9c6e345a3..4fa46225e0 100644 --- a/utils/tidy_conf/data/rejections.yml +++ b/utils/tidy_conf/data/rejections.yml @@ -1,4 +1,3 @@ ---- alt_name: AfroPython Conference: variations: @@ -14,6 +13,11 @@ alt_name: variations: - PyCon Australia - PyCon AU + PyCon Africa: + variations: + - PyCon South Africa + - PyCon ZA + - PyConZA PyCon Austria: variations: - PyCon Australia @@ -30,6 +34,9 @@ alt_name: PyCon Latin America: variations: - PyCon Latam + PyCon South Africa: + variations: + - PyCon Africa Python Austria: variations: - PyCon Australia diff --git a/utils/tidy_conf/interactive_merge.py b/utils/tidy_conf/interactive_merge.py index d511b802d6..c26225249a 100644 --- a/utils/tidy_conf/interactive_merge.py +++ b/utils/tidy_conf/interactive_merge.py @@ -55,6 +55,29 @@ } +def is_identical_name(s1: str, s2: str) -> bool: + """Check whether two conference names are genuinely identical. + + A fuzzy score of 100 does NOT imply identity: token_set_ratio returns 100 + whenever one name's tokens are a subset of the other's (e.g. "PyCon Africa" + vs "PyCon South Africa"). Only names that are equal after case and + whitespace normalization may be auto-merged without confirmation. + + Parameters + ---------- + s1 : str + First conference name to compare + s2 : str + Second conference name to compare + + Returns + ------- + bool + True if the names are identical after normalization + """ + return " ".join(s1.lower().split()) == " ".join(s2.lower().split()) + + def is_placeholder_value(value) -> bool: """Check if a value is a placeholder (TBA, TBD, None, empty). @@ -276,6 +299,31 @@ def is_excluded(name1, name2): """Check if two conference names are in the combined exclusion list.""" return frozenset([name1, name2]) in all_exclusions + def best_match(title_match): + """Extract (title, score) from a process.extract result (2- or 3-tuple).""" + match_result = title_match[0] + if len(match_result) == 3: + title, prob, _ = match_result + else: + title, prob = match_result + return title, prob + + # Remote titles already spoken for by a YAML row with the genuinely + # identical name. No other YAML row may merge into these - otherwise two + # distinct conferences collapse into one row and one of them is lost + # (e.g. "PyCon South Africa" must not merge into remote "PyCon Africa" + # when a YAML "PyCon Africa" also exists). + identical_titles = set() + for _, row in df.iterrows(): + if isinstance(row["title_match"], str) or not row["title_match"]: + continue + title, _prob = best_match(row["title_match"]) + if is_identical_name(row["conference"], title): + identical_titles.add(title) + + # Remote titles claimed by a YAML row during this run (title -> yaml name) + claimed_titles = {} + # Process matches and track in report for i, row in df.iterrows(): if isinstance(row["title_match"], str): @@ -283,12 +331,7 @@ def is_excluded(name1, name2): if not row["title_match"]: continue - # Handle both 2-tuple and 3-tuple results from process.extract - match_result = row["title_match"][0] - if len(match_result) == 3: - title, prob, _ = match_result - else: - title, prob = match_result + title, prob = best_match(row["title_match"]) conference_name = row["conference"] year = row.get("year", 0) @@ -311,21 +354,45 @@ def is_excluded(name1, name2): df.at[i, "title_match"] = conference_name # Use original name, not index record.match_type = "excluded" record.action = "kept_yaml" - elif prob >= EXACT_MATCH_THRESHOLD: - logger.debug( - f"Exact match: '{conference_name}' -> '{title}' (score: {prob})", - ) - df.at[i, "title_match"] = title - record.match_type = "exact" - record.action = "merged" + elif prob >= EXACT_MATCH_THRESHOLD and is_identical_name(conference_name, title): + # Only genuinely identical names merge automatically. A score of + # 100 alone is NOT sufficient: token_set_ratio scores 100 whenever + # one name's tokens are a subset of the other's. + if title in claimed_titles and claimed_titles[title] != conference_name: + logger.warning( + f"Remote '{title}' already claimed by '{claimed_titles[title]}', " + f"keeping '{conference_name}' separate", + ) + df.at[i, "title_match"] = conference_name + record.match_type = "exact" + record.action = "kept_yaml" + else: + logger.debug( + f"Exact match: '{conference_name}' -> '{title}' (score: {prob})", + ) + df.at[i, "title_match"] = title + claimed_titles[title] = conference_name + record.match_type = "exact" + record.action = "merged" elif prob >= FUZZY_MATCH_THRESHOLD: + if title in identical_titles or title in claimed_titles: + # Remote row already belongs to its identically-named YAML + # entry (or was claimed earlier) - never merge another + # conference into it. + logger.info( + f"Skipping fuzzy match: remote '{title}' already belongs to an " + f"identically-named entry, keeping '{conference_name}' separate", + ) + df.at[i, "title_match"] = conference_name + record.match_type = "fuzzy" + record.action = "kept_yaml" # Prompt user for fuzzy matches that aren't excluded - logger.info( - f"Fuzzy match candidate: '{conference_name}' -> '{title}' (score: {prob})", - ) - if not query_yes_no( + elif not query_yes_no( f"Do '{row['conference']}' and '{title}' match? (y/n): ", ): + logger.info( + f"Fuzzy match rejected: '{conference_name}' vs '{title}' (score: {prob})", + ) new_rejections[title].append(conference_name) new_rejections[conference_name].append(title) df.at[i, "title_match"] = conference_name # Use original name, not index @@ -334,6 +401,7 @@ def is_excluded(name1, name2): else: new_mappings[conference_name].append(title) df.at[i, "title_match"] = title + claimed_titles[title] = conference_name record.match_type = "fuzzy" record.action = "merged" else: diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index c630d352a2..a3adec7241 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -9,6 +9,71 @@ import requests from tqdm import tqdm +# Sub-page link fields that must point somewhere other than the main link. +# A cfp_link/sponsor/finaid that is just the homepage again carries no +# information (e.g. "cfp_link: https://2027.pycon.de/" on the PyCon DE entry +# whose link is https://2027.pycon.de/) - it should be a different page, a +# subdomain, or at least a #anchor on the homepage. +REDUNDANT_LINK_FIELDS = ("cfp_link", "sponsor", "finaid") + + +def normalize_url_pointer(url: str) -> tuple: + """Normalize a URL for same-pointer comparison. + + Two URLs are the same pointer when they only differ by scheme + (http/https), a "www." prefix, host case, or a trailing slash. + Anything else - path, subdomain, query, or #fragment - makes them + different pointers. + + Parameters + ---------- + url : str + URL to normalize + + Returns + ------- + tuple + Comparable (host, path, params, query, fragment) tuple + """ + parsed = urlparse(str(url).strip()) + netloc = parsed.netloc.lower().removeprefix("www.") + return (netloc, parsed.path.rstrip("/"), parsed.params, parsed.query, parsed.fragment) + + +def drop_redundant_link_fields(data: list) -> list: + """Drop sub-page link fields that are the same pointer as the main link. + + Goes through cfp_link/sponsor/finaid on each conference and + removes any that just repeat the conference's main link. This catches + upstream sources that fill every URL column with the homepage. + + Parameters + ---------- + data : list + List of conference dictionaries + + Returns + ------- + list + The same list with redundant link fields removed + """ + for q in data: + if not isinstance(q, dict): + continue + link = q.get("link") + if not link: + continue + base = normalize_url_pointer(link) + for field in REDUNDANT_LINK_FIELDS: + value = q.get(field) + if value and normalize_url_pointer(value) == base: + tqdm.write( + f"Dropping redundant '{field}' from {q.get('conference')} {q.get('year')}: " + f"'{value}' is the same pointer as the main link", + ) + del q[field] + return data + def get_cache_location(): # Check if the URL is cached