From 17deeb9832d8982cd454c3850a083132744b9cd1 Mon Sep 17 00:00:00 2001 From: Brian Macy Date: Mon, 5 Oct 2026 21:21:14 +0000 Subject: [PATCH] Carry secondary practice locations on the provider Each secondary practice location (pl_pfile) used to become a nameless NPI-LOCATIONS record. No name key, name+place key or category key can be built for a nameless record, so providers were invisible at those 1.2M addresses (810,348 NPIs). Now map_npi adds each location's address to the provider as an extra ADDR_TYPE SECONDARY address (deduplicated against the provider's own addresses, after them in FEATURES) and its telephone and fax numbers as PHONE features (add_phone drops a repeat; a fax keeps PHONE_TYPE FAX). The NPI-LOCATIONS records are written only with --locationRecords. Checked against the previous output (9,443,429 providers): 735,071 changed, every one by additions only (1,111,747 secondary addresses, 852,163 phones, 308,029 faxes); officials and affiliations identical; the 4,594 deactivated NPIs that had locations carry them on their NPI_DEACTIVE record. Tests: tests/test_secondary_locations.py (address and phone carried, deduped, location records only with the flag). --- CHANGELOG.md | 5 + README.md | 7 ++ src/npi_mapper.py | 170 ++++++++++++++++++-------- tests/fixtures/npidata_1063947125.csv | 2 + tests/test_secondary_locations.py | 137 +++++++++++++++++++++ 5 files changed, 270 insertions(+), 51 deletions(-) create mode 100644 tests/fixtures/npidata_1063947125.csv create mode 100644 tests/test_secondary_locations.py diff --git a/CHANGELOG.md b/CHANGELOG.md index eb7bc38..0685169 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Secondary practice locations (`pl_pfile`) are carried on the provider: each address as an extra `ADDR_TYPE: SECONDARY` address + (deduplicated against the provider's own) and each telephone and fax number as `PHONE_NUMBER` features. Tests in + `tests/test_secondary_locations.py`. - `ShuffleWriter`, a bounded-memory reservoir that shuffles records as they are written, plus `-S/--shuffleBuffer` to size it (default 250000 records per output file; 0 writes straight through and disables shuffling). NPPES ships sorted by NPI, so the records belonging to one @@ -23,6 +26,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- The nameless `NPI-LOCATIONS` records are no longer written by default (their addresses and numbers are on the provider); + `--locationRecords` writes them as before. - `NPI-AFFILIATIONS` records are now clustered by ER content. `er_record_id()` derives the `RECORD_ID` as a sha1 over the record's normalized (strip, collapse whitespace, upper), sorted features, EXCLUDING the `REL_*` disclosure features, so two rows that describe the diff --git a/README.md b/README.md index 4dd83c3..35df464 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,13 @@ Place the following files on a directory of your choice ... - [npi_mapper.py] - [npi_config_updates.g2c] +### Secondary practice locations + +The secondary practice locations (`pl_pfile`) are carried on the provider itself: each location's address as an extra +`ADDR_TYPE: SECONDARY` address (once, and not when it repeats an address the provider already has) and its telephone and fax +numbers as `PHONE_NUMBER` features (a fax keeps `PHONE_TYPE: FAX`). The nameless `NPI-LOCATIONS` records are no longer written; +`--locationRecords` writes them as well. + ### Configuring Senzing _Note:_ This only needs to be performed one time! In fact you may want to add these configuration updates to a master configuration file for all your data sources. diff --git a/src/npi_mapper.py b/src/npi_mapper.py index 45665b0..9bc815c 100644 --- a/src/npi_mapper.py +++ b/src/npi_mapper.py @@ -59,6 +59,7 @@ # - removed all name defaulting on locations, turns ou its not reliable at all # # ---------------------------------------------------------------------------------------------------- +import re import csv import hashlib import json @@ -75,6 +76,7 @@ import signal import random + # NPPES placeholder values that appear in name fields and must never be emitted as a name. # "" arrives with the undocumented "Provider Other Organization Name Type Code" 6. NAME_PLACEHOLDERS = {"", "NONE", ""} @@ -349,10 +351,100 @@ def flush_affiliate_records(out_file): # ------------------------------------------------------------- # Map Provider Locations Reference file for this NPI # ------------------------------------------------------------- -def map_locations(inNPI, inName, inType): +def address_identity(feature): + """Street + ZIP5 of an address feature, upper-cased and stripped to letters and digits; None without a street line. + + Two addresses with the same identity are the same place for the purpose of not repeating an address on a provider. + """ + line1 = re.sub(r"[^0-9A-Z]", "", str(feature.get("ADDR_LINE1", "")).upper()) + if not line1: + return None + zip5 = re.sub(r"[^0-9]", "", str(feature.get("ADDR_POSTAL_CODE", "")))[:5] + return line1 + "|" + zip5 + + +def secondary_addresses(locations, provider_features): + """The secondary practice-location addresses to add to the provider: each once, none already on the provider. + + `locations` are the ADDR_TYPE SECONDARY address features map_locations() produced for this NPI, in its deterministic order. + """ + seen = {address_identity(f) for f in provider_features if "ADDR_LINE1" in f} + out = [] + for feature in locations: + ident = address_identity(feature) + if ident is None or ident in seen: + continue + seen.add(ident) + out.append(feature) + return out + + +# Whether to also write the nameless NPI-LOCATIONS records. Off by default: each secondary location's address and phone numbers +# are carried on the provider itself (map_locations / map_npi), where name+place and category search can see them, and the +# separate records only added 1.2M nameless entities that no name key can find. +EMIT_LOCATION_RECORDS = False + + +def write_location_record(inNPI, rsltRecord, hdr1): + """Write one nameless NPI-LOCATIONS record (only with --locationRecords).""" global NPILocations_row_count global JSON_row_count + loc_data = {} + loc_features = [] + loc_data["DATA_SOURCE"] = "NPI-LOCATIONS" + loc_data["RECORD_ID"] = derived_record_id(inNPI, *[rsltRecord[col] for col in hdr1]) + loc_data["FEATURES"] = loc_features + add_feature(loc_features, {"RECORD_TYPE": "ORGANIZATION"}) + updateStat("DATA_SOURCES", loc_data["DATA_SOURCE"]) + updateStat(loc_data["DATA_SOURCE"], "ORGANIZATION") + + # No name on purpose: the name of the organization at a secondary location cannot be stated reliably. + + if rsltRecord["ADDR1"]: + # ADDR_TYPE BUSINESS is load-bearing here, not decoration: these records carry no name + # at all, so the address IS their identity, and BUSINESS is what marks it as a distinct + # physical location so two practice locations do not collapse into one entity. + updateStat(loc_data["DATA_SOURCE"], "ADDR_LINE1", rsltRecord["ADDR1"]) + address = {"ADDR_TYPE": "BUSINESS", "ADDR_LINE1": rsltRecord["ADDR1"]} + if rsltRecord["ADDR2"] and rsltRecord["ADDR2"] != "NONE": + updateStat(loc_data["DATA_SOURCE"], "ADDR_LINE2", rsltRecord["ADDR2"]) + address["ADDR_LINE2"] = rsltRecord["ADDR2"] + address["ADDR_CITY"] = rsltRecord["CITY"] + address["ADDR_STATE"] = rsltRecord["STATE"] + address["ADDR_POSTAL_CODE"] = rsltRecord["POSTAL_CODE"] + address["ADDR_COUNTRY"] = rsltRecord["COUNTRY"] + add_feature(loc_features, address) + + if rsltRecord["PH1"]: + updateStat(loc_data["DATA_SOURCE"], "PHONE", rsltRecord["PH1"]) + add_phone(loc_features, rsltRecord["PH1"]) + if rsltRecord["PH2"]: + updateStat(loc_data["DATA_SOURCE"], "FAX", rsltRecord["PH2"]) + add_phone(loc_features, rsltRecord["PH2"], "FAX") + + # Disclose rel to NPI + add_feature( + loc_features, + { + "REL_POINTER_DOMAIN": "NPI", + "REL_POINTER_KEY": inNPI, + "REL_POINTER_ROLE": "Secondary Location", + }, + ) + + Locations_outFile.write(json.dumps(drop_empty_attributes(loc_data)) + "\n") + JSON_row_count += 1 + NPILocations_row_count += 1 + + +def map_locations(inNPI, inName, inType): + """The secondary practice locations of one NPI, from the pl_pfile, in a deterministic order. + + Returns a list of dicts: "address" (an ADDR_TYPE SECONDARY address feature, or None without a street line) and "phone" / + "fax" (the location's numbers, or None). map_npi puts them on the provider; the nameless NPI-LOCATIONS record is written + only with --locationRecords. + """ sql = "select distinct " sql += ' "Provider Secondary Practice Location Address- Address Line 1" as ADDR1,' sql += ' "Provider Secondary Practice Location Address- Address Line 2" as ADDR2,' @@ -369,65 +461,23 @@ def map_locations(inNPI, inName, inType): cursor1 = plObj.execute(sql, (str(inNPI),)) hdr1 = [col[0] for col in plObj.description] resultRow = cursor1.fetchone() + locations = [] while resultRow: rsltRecord = dict(zip(hdr1, resultRow)) - - loc_data = {} - loc_features = [] - loc_data["DATA_SOURCE"] = "NPI-LOCATIONS" - loc_data["RECORD_ID"] = derived_record_id(inNPI, *[rsltRecord[col] for col in hdr1]) - loc_data["FEATURES"] = loc_features - add_feature(loc_features, {"RECORD_TYPE": "ORGANIZATION"}) - updateStat("DATA_SOURCES", loc_data["DATA_SOURCE"]) - updateStat(loc_data["DATA_SOURCE"], "ORGANIZATION") - - if ( - False - ): # --cannot reliably say this is the name of the organization at that location - if inType == "1": - add_feature(loc_features, {"NAME_TYPE": "PRIMARY", "NAME_FULL": inName}) - updateStat(loc_data["DATA_SOURCE"], "NAME_FULL(PERSON)", inName) - else: - add_feature(loc_features, {"NAME_TYPE": "PRIMARY", "NAME_ORG": inName}) - updateStat(loc_data["DATA_SOURCE"], "NAME_ORG(ORGANIZATION)", inName) - + address = None if rsltRecord["ADDR1"]: - # ADDR_TYPE BUSINESS is load-bearing here, not decoration: these records carry no name - # at all, so the address IS their identity, and BUSINESS is what marks it as a distinct - # physical location so two practice locations do not collapse into one entity. - updateStat(loc_data["DATA_SOURCE"], "ADDR_LINE1", rsltRecord["ADDR1"]) - address = {"ADDR_TYPE": "BUSINESS", "ADDR_LINE1": rsltRecord["ADDR1"]} + address = {"ADDR_TYPE": "SECONDARY", "ADDR_LINE1": rsltRecord["ADDR1"]} if rsltRecord["ADDR2"] and rsltRecord["ADDR2"] != "NONE": - updateStat(loc_data["DATA_SOURCE"], "ADDR_LINE2", rsltRecord["ADDR2"]) address["ADDR_LINE2"] = rsltRecord["ADDR2"] address["ADDR_CITY"] = rsltRecord["CITY"] address["ADDR_STATE"] = rsltRecord["STATE"] address["ADDR_POSTAL_CODE"] = rsltRecord["POSTAL_CODE"] address["ADDR_COUNTRY"] = rsltRecord["COUNTRY"] - add_feature(loc_features, address) - - if rsltRecord["PH1"]: - updateStat(loc_data["DATA_SOURCE"], "PHONE", rsltRecord["PH1"]) - add_phone(loc_features, rsltRecord["PH1"]) - if rsltRecord["PH2"]: - updateStat(loc_data["DATA_SOURCE"], "FAX", rsltRecord["PH2"]) - add_phone(loc_features, rsltRecord["PH2"], "FAX") - - # Disclose rel to NPI - add_feature( - loc_features, - { - "REL_POINTER_DOMAIN": "NPI", - "REL_POINTER_KEY": inNPI, - "REL_POINTER_ROLE": "Secondary Location", - }, - ) - - Locations_outFile.write(json.dumps(drop_empty_attributes(loc_data)) + "\n") - JSON_row_count += 1 - NPILocations_row_count += 1 - + locations.append({"address": address, "phone": rsltRecord["PH1"] or None, "fax": rsltRecord["PH2"] or None}) + if EMIT_LOCATION_RECORDS: + write_location_record(inNPI, rsltRecord, hdr1) resultRow = cursor1.fetchone() + return locations # @@ -1274,7 +1324,17 @@ def map_npi(input_row): NPIOfficials_row_count += 1 # Map the Provider Locations reference data if there are any for this NPI - map_locations(input_row["NPI"], npi_name, input_row["Entity Type Code"]) + # and carry each secondary location's address on the provider too (ADDR_TYPE SECONDARY, after its own addresses) + locations = map_locations(input_row["NPI"], npi_name, input_row["Entity Type Code"]) + for address in secondary_addresses([loc["address"] for loc in locations if loc["address"]], features): + add_feature(features, address) + updateStat(json_data["DATA_SOURCE"], "ADDRESS(SECONDARY)", address.get("ADDR_LINE1")) + # ...and its telephone and fax numbers (add_phone drops a number the provider already has) + for loc in locations: + if loc["phone"] and add_phone(features, loc["phone"]): + updateStat(json_data["DATA_SOURCE"], "PHONE(SECONDARY)", loc["phone"]) + if loc["fax"] and add_phone(features, loc["fax"], "FAX"): + updateStat(json_data["DATA_SOURCE"], "FAX(SECONDARY)", loc["fax"]) # Map the Endpoint reference data if there are any for this NPI # --jb: some endpoints like email and website belong to the npi, others are affiliates @@ -1465,7 +1525,15 @@ def signal_handler(signal, frame): "loading them in file order serialises the consumer fleet on one lock -- see ShuffleWriter. " "Cost is roughly this many records resident per output file.", ) + argParser.add_argument( + "--locationRecords", + dest="locationRecords", + action="store_true", + default=False, + help="also write the nameless NPI-LOCATIONS records (default off: secondary locations are carried on the provider)", + ) parms = argParser.parse_args() + EMIT_LOCATION_RECORDS = parms.locationRecords if (parms.filePeriod and len(parms.filePeriod) > 0) and ( parms.sourceDir and len(parms.sourceDir) > 0 diff --git a/tests/fixtures/npidata_1063947125.csv b/tests/fixtures/npidata_1063947125.csv new file mode 100644 index 0000000..2845d0c --- /dev/null +++ b/tests/fixtures/npidata_1063947125.csv @@ -0,0 +1,2 @@ +"NPI","Entity Type Code","Replacement NPI","Employer Identification Number (EIN)","Provider Organization Name (Legal Business Name)","Provider Last Name (Legal Name)","Provider First Name","Provider Middle Name","Provider Name Prefix Text","Provider Name Suffix Text","Provider Credential Text","Provider Other Organization Name","Provider Other Organization Name Type Code","Provider Other Last Name","Provider Other First Name","Provider Other Middle Name","Provider Other Name Prefix Text","Provider Other Name Suffix Text","Provider Other Credential Text","Provider Other Last Name Type Code","Provider First Line Business Mailing Address","Provider Second Line Business Mailing Address","Provider Business Mailing Address City Name","Provider Business Mailing Address State Name","Provider Business Mailing Address Postal Code","Provider Business Mailing Address Country Code (If outside U.S.)","Provider Business Mailing Address Telephone Number","Provider Business Mailing Address Fax Number","Provider First Line Business Practice Location Address","Provider Second Line Business Practice Location Address","Provider Business Practice Location Address City Name","Provider Business Practice Location Address State Name","Provider Business Practice Location Address Postal Code","Provider Business Practice Location Address Country Code (If outside U.S.)","Provider Business Practice Location Address Telephone Number","Provider Business Practice Location Address Fax Number","Provider Enumeration Date","Last Update Date","NPI Deactivation Reason Code","NPI Deactivation Date","NPI Reactivation Date","Provider Sex Code","Authorized Official Last Name","Authorized Official First Name","Authorized Official Middle Name","Authorized Official Title or Position","Authorized Official Telephone Number","Healthcare Provider Taxonomy Code_1","Provider License Number_1","Provider License Number State Code_1","Healthcare Provider Primary Taxonomy Switch_1","Healthcare Provider Taxonomy Code_2","Provider License Number_2","Provider License Number State Code_2","Healthcare Provider Primary Taxonomy Switch_2","Healthcare Provider Taxonomy Code_3","Provider License Number_3","Provider License Number State Code_3","Healthcare Provider Primary Taxonomy Switch_3","Healthcare Provider Taxonomy Code_4","Provider License Number_4","Provider License Number State Code_4","Healthcare Provider Primary Taxonomy Switch_4","Healthcare Provider Taxonomy Code_5","Provider License Number_5","Provider License Number State Code_5","Healthcare Provider Primary Taxonomy Switch_5","Healthcare Provider Taxonomy Code_6","Provider License Number_6","Provider License Number State Code_6","Healthcare Provider Primary Taxonomy Switch_6","Healthcare Provider Taxonomy Code_7","Provider License Number_7","Provider License Number State Code_7","Healthcare Provider Primary Taxonomy Switch_7","Healthcare Provider Taxonomy Code_8","Provider License Number_8","Provider License Number State Code_8","Healthcare Provider Primary Taxonomy Switch_8","Healthcare Provider Taxonomy Code_9","Provider License Number_9","Provider License Number State Code_9","Healthcare Provider Primary Taxonomy Switch_9","Healthcare Provider Taxonomy Code_10","Provider License Number_10","Provider License Number State Code_10","Healthcare Provider Primary Taxonomy Switch_10","Healthcare Provider Taxonomy Code_11","Provider License Number_11","Provider License Number State Code_11","Healthcare Provider Primary Taxonomy Switch_11","Healthcare Provider Taxonomy Code_12","Provider License Number_12","Provider License Number State Code_12","Healthcare Provider Primary Taxonomy Switch_12","Healthcare Provider Taxonomy Code_13","Provider License Number_13","Provider License Number State Code_13","Healthcare Provider Primary Taxonomy Switch_13","Healthcare Provider Taxonomy Code_14","Provider License Number_14","Provider License Number State Code_14","Healthcare Provider Primary Taxonomy Switch_14","Healthcare Provider Taxonomy Code_15","Provider License Number_15","Provider License Number State Code_15","Healthcare Provider Primary Taxonomy Switch_15","Other Provider Identifier_1","Other Provider Identifier Type Code_1","Other Provider Identifier State_1","Other Provider Identifier Issuer_1","Other Provider Identifier_2","Other Provider Identifier Type Code_2","Other Provider Identifier State_2","Other Provider Identifier Issuer_2","Other Provider Identifier_3","Other Provider Identifier Type Code_3","Other Provider Identifier State_3","Other Provider Identifier Issuer_3","Other Provider Identifier_4","Other Provider Identifier Type Code_4","Other Provider Identifier State_4","Other Provider Identifier Issuer_4","Other Provider Identifier_5","Other Provider Identifier Type Code_5","Other Provider Identifier State_5","Other Provider Identifier Issuer_5","Other Provider Identifier_6","Other Provider Identifier Type Code_6","Other Provider Identifier State_6","Other Provider Identifier Issuer_6","Other Provider Identifier_7","Other Provider Identifier Type Code_7","Other Provider Identifier State_7","Other Provider Identifier Issuer_7","Other Provider Identifier_8","Other Provider Identifier Type Code_8","Other Provider Identifier State_8","Other Provider Identifier Issuer_8","Other Provider Identifier_9","Other Provider Identifier Type Code_9","Other Provider Identifier State_9","Other Provider Identifier Issuer_9","Other Provider Identifier_10","Other Provider Identifier Type Code_10","Other Provider Identifier State_10","Other Provider Identifier Issuer_10","Other Provider Identifier_11","Other Provider Identifier Type Code_11","Other Provider Identifier State_11","Other Provider Identifier Issuer_11","Other Provider Identifier_12","Other Provider Identifier Type Code_12","Other Provider Identifier State_12","Other Provider Identifier Issuer_12","Other Provider Identifier_13","Other Provider Identifier Type Code_13","Other Provider Identifier State_13","Other Provider Identifier Issuer_13","Other Provider Identifier_14","Other Provider Identifier Type Code_14","Other Provider Identifier State_14","Other Provider Identifier Issuer_14","Other Provider Identifier_15","Other Provider Identifier Type Code_15","Other Provider Identifier State_15","Other Provider Identifier Issuer_15","Other Provider Identifier_16","Other Provider Identifier Type Code_16","Other Provider Identifier State_16","Other Provider Identifier Issuer_16","Other Provider Identifier_17","Other Provider Identifier Type Code_17","Other Provider Identifier State_17","Other Provider Identifier Issuer_17","Other Provider Identifier_18","Other Provider Identifier Type Code_18","Other Provider Identifier State_18","Other Provider Identifier Issuer_18","Other Provider Identifier_19","Other Provider Identifier Type Code_19","Other Provider Identifier State_19","Other Provider Identifier Issuer_19","Other Provider Identifier_20","Other Provider Identifier Type Code_20","Other Provider Identifier State_20","Other Provider Identifier Issuer_20","Other Provider Identifier_21","Other Provider Identifier Type Code_21","Other Provider Identifier State_21","Other Provider Identifier Issuer_21","Other Provider Identifier_22","Other Provider Identifier Type Code_22","Other Provider Identifier State_22","Other Provider Identifier Issuer_22","Other Provider Identifier_23","Other Provider Identifier Type Code_23","Other Provider Identifier State_23","Other Provider Identifier Issuer_23","Other Provider Identifier_24","Other Provider Identifier Type Code_24","Other Provider Identifier State_24","Other Provider Identifier Issuer_24","Other Provider Identifier_25","Other Provider Identifier Type Code_25","Other Provider Identifier State_25","Other Provider Identifier Issuer_25","Other Provider Identifier_26","Other Provider Identifier Type Code_26","Other Provider Identifier State_26","Other Provider Identifier Issuer_26","Other Provider Identifier_27","Other Provider Identifier Type Code_27","Other Provider Identifier State_27","Other Provider Identifier Issuer_27","Other Provider Identifier_28","Other Provider Identifier Type Code_28","Other Provider Identifier State_28","Other Provider Identifier Issuer_28","Other Provider Identifier_29","Other Provider Identifier Type Code_29","Other Provider Identifier State_29","Other Provider Identifier Issuer_29","Other Provider Identifier_30","Other Provider Identifier Type Code_30","Other Provider Identifier State_30","Other Provider Identifier Issuer_30","Other Provider Identifier_31","Other Provider Identifier Type Code_31","Other Provider Identifier State_31","Other Provider Identifier Issuer_31","Other Provider Identifier_32","Other Provider Identifier Type Code_32","Other Provider Identifier State_32","Other Provider Identifier Issuer_32","Other Provider Identifier_33","Other Provider Identifier Type Code_33","Other Provider Identifier State_33","Other Provider Identifier Issuer_33","Other Provider Identifier_34","Other Provider Identifier Type Code_34","Other Provider Identifier State_34","Other Provider Identifier Issuer_34","Other Provider Identifier_35","Other Provider Identifier Type Code_35","Other Provider Identifier State_35","Other Provider Identifier Issuer_35","Other Provider Identifier_36","Other Provider Identifier Type Code_36","Other Provider Identifier State_36","Other Provider Identifier Issuer_36","Other Provider Identifier_37","Other Provider Identifier Type Code_37","Other Provider Identifier State_37","Other Provider Identifier Issuer_37","Other Provider Identifier_38","Other Provider Identifier Type Code_38","Other Provider Identifier State_38","Other Provider Identifier Issuer_38","Other Provider Identifier_39","Other Provider Identifier Type Code_39","Other Provider Identifier State_39","Other Provider Identifier Issuer_39","Other Provider Identifier_40","Other Provider Identifier Type Code_40","Other Provider Identifier State_40","Other Provider Identifier Issuer_40","Other Provider Identifier_41","Other Provider Identifier Type Code_41","Other Provider Identifier State_41","Other Provider Identifier Issuer_41","Other Provider Identifier_42","Other Provider Identifier Type Code_42","Other Provider Identifier State_42","Other Provider Identifier Issuer_42","Other Provider Identifier_43","Other Provider Identifier Type Code_43","Other Provider Identifier State_43","Other Provider Identifier Issuer_43","Other Provider Identifier_44","Other Provider Identifier Type Code_44","Other Provider Identifier State_44","Other Provider Identifier Issuer_44","Other Provider Identifier_45","Other Provider Identifier Type Code_45","Other Provider Identifier State_45","Other Provider Identifier Issuer_45","Other Provider Identifier_46","Other Provider Identifier Type Code_46","Other Provider Identifier State_46","Other Provider Identifier Issuer_46","Other Provider Identifier_47","Other Provider Identifier Type Code_47","Other Provider Identifier State_47","Other Provider Identifier Issuer_47","Other Provider Identifier_48","Other Provider Identifier Type Code_48","Other Provider Identifier State_48","Other Provider Identifier Issuer_48","Other Provider Identifier_49","Other Provider Identifier Type Code_49","Other Provider Identifier State_49","Other Provider Identifier Issuer_49","Other Provider Identifier_50","Other Provider Identifier Type Code_50","Other Provider Identifier State_50","Other Provider Identifier Issuer_50","Is Sole Proprietor","Is Organization Subpart","Parent Organization LBN","Parent Organization TIN","Authorized Official Name Prefix Text","Authorized Official Name Suffix Text","Authorized Official Credential Text","Healthcare Provider Taxonomy Group_1","Healthcare Provider Taxonomy Group_2","Healthcare Provider Taxonomy Group_3","Healthcare Provider Taxonomy Group_4","Healthcare Provider Taxonomy Group_5","Healthcare Provider Taxonomy Group_6","Healthcare Provider Taxonomy Group_7","Healthcare Provider Taxonomy Group_8","Healthcare Provider Taxonomy Group_9","Healthcare Provider Taxonomy Group_10","Healthcare Provider Taxonomy Group_11","Healthcare Provider Taxonomy Group_12","Healthcare Provider Taxonomy Group_13","Healthcare Provider Taxonomy Group_14","Healthcare Provider Taxonomy Group_15","Certification Date" +"1063947125","2","","","LOVETTSVILLE DENTAL ARTS","","","","","","","","","","","","","","","","2 N LIGHT ST","","LOVETTSVILLE","VA","201808613","US","","","2 N LIGHT ST","","LOVETTSVILLE","VA","201808613","US","5408224224","","04/26/2017","05/09/2017","","","","","MEYER","BRANDON","","OWNER","5408224224","261QD0000X","","","Y","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","N","","","","","","","","","","","","","","","","","","","","","" diff --git a/tests/test_secondary_locations.py b/tests/test_secondary_locations.py new file mode 100644 index 0000000..8dc82d4 --- /dev/null +++ b/tests/test_secondary_locations.py @@ -0,0 +1,137 @@ +"""Tests for the secondary practice locations carried on the provider (npi_mapper.map_locations / map_npi). + +The provider below is a verbatim NPPES row (tests/fixtures/npidata_1063947125.csv); the pl_pfile rows are inserted into the +in-memory PL table the mapper reads. +""" + +import csv +import io +import json +import os +import sqlite3 +import sys + +import pytest + +SRC_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "src") +FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures") +sys.path.insert(0, SRC_DIR) + + +@pytest.fixture(name="mapper") +def fixture_mapper(): + """npi_mapper with the globals its __main__ block would set, backed by EMPTY reference tables.""" + # npi_mapper imports pandas at module level; only this fixture needs it, so import it here. + # pylint: disable-next=import-outside-toplevel + import npi_mapper + + conn = sqlite3.connect(":memory:") + conn.execute( + 'create table OTHERNAME (NPI, "Provider Other Organization Name", "Provider Other Organization Name Type Code")' + ) + pl_cols = [ + "Provider Secondary Practice Location Address- Address Line 1", + "Provider Secondary Practice Location Address- Address Line 2", + "Provider Secondary Practice Location Address - City Name", + "Provider Secondary Practice Location Address - State Name", + "Provider Secondary Practice Location Address - Postal Code", + "Provider Secondary Practice Location Address - Country Code (If outside U.S.)", + "Provider Secondary Practice Location Address - Telephone Number", + "Provider Practice Location Address - Fax Number", + ] + conn.execute("create table PL (NPI, %s)" % ", ".join('"%s"' % c for c in pl_cols)) + ep_cols = [ + "Affiliation", + "Endpoint", + "Affiliation Legal Business Name", + "Affiliation Address Line One", + "Affiliation Address Line Two", + "Affiliation Address City", + "Affiliation Address State", + "Affiliation Address Country", + "Affiliation Address Postal Code", + ] + conn.execute("create table ENDPOINT (NPI, %s)" % ", ".join('"%s"' % c for c in ep_cols)) + officials = io.StringIO() + locations = io.StringIO() + settings = { + "conn": conn, + "statPack": {}, + "Officials_outFile": officials, + "Locations_outFile": locations, + "JSON_row_count": 0, + "NPIOfficials_row_count": 0, + "NPILocations_row_count": 0, + "idValuesToIgnore": {"=========": True, "NONE": True}, + } + for name, value in settings.items(): + setattr(npi_mapper, name, value) + yield npi_mapper + conn.close() + + +PL_ROWS = [ + # a new place for the provider + ("1063947125", "6855 Wilson Blvd", "Ste 2", "Jacksonville", "FL", "322103600", "", "5615095009", ""), + # the provider's own practice address (must not be repeated), same street/ZIP5 spelled differently + ("1063947125", "2 n. light st", "", "Lovettsville", "VA", "20180", "", "5551230000", ""), + # the same new place again with another phone: one address on the provider, two location records + ("1063947125", "6855 Wilson Blvd", "Ste 2", "Jacksonville", "FL", "322103600", "", "9045550000", "9045551111"), +] + + +def _provider_row(): + with open(os.path.join(FIXTURES, "npidata_1063947125.csv"), encoding="utf-8") as handle: + return next(csv.DictReader(handle)) + + +def test_secondary_locations_become_provider_addresses_and_phones(mapper): + """Each secondary location's address and phone numbers are on the provider (ADDR_TYPE SECONDARY), once, not repeated.""" + mapper.conn.executemany("insert into PL values (?,?,?,?,?,?,?,?,?)", PL_ROWS) + produced = json.loads(mapper.map_npi(_provider_row())) + secondary = [f for f in produced["FEATURES"] if f.get("ADDR_TYPE") == "SECONDARY"] + assert secondary == [ + { + "ADDR_TYPE": "SECONDARY", + "ADDR_LINE1": "6855 Wilson Blvd", + "ADDR_LINE2": "Ste 2", + "ADDR_CITY": "Jacksonville", + "ADDR_STATE": "FL", + "ADDR_POSTAL_CODE": "322103600", + } + ] + # the provider's own addresses come first, so a consumer reading "the first BUSINESS address" is unchanged + types = [f["ADDR_TYPE"] for f in produced["FEATURES"] if "ADDR_TYPE" in f] + assert types.index("SECONDARY") > max(i for i, t in enumerate(types) if t != "SECONDARY") + # every location's telephone is on the provider, each number once (also for the location at the provider's own address) + phones = [f["PHONE_NUMBER"] for f in produced["FEATURES"] if "PHONE_NUMBER" in f and "PHONE_TYPE" not in f] + for number in ("5615095009", "5551230000", "9045550000"): + assert phones.count(number) == 1 + assert [f for f in produced["FEATURES"] if f.get("PHONE_TYPE") == "FAX" and f["PHONE_NUMBER"] == "9045551111"] + # no nameless location records by default + assert mapper.Locations_outFile.getvalue() == "" + + +def test_location_records_only_with_the_flag(mapper): + mapper.conn.executemany("insert into PL values (?,?,?,?,?,?,?,?,?)", PL_ROWS) + mapper.EMIT_LOCATION_RECORDS = True + try: + mapper.map_npi(_provider_row()) + finally: + mapper.EMIT_LOCATION_RECORDS = False + lines = [json.loads(x) for x in mapper.Locations_outFile.getvalue().splitlines()] + assert len(lines) == len(PL_ROWS) + assert all(r["DATA_SOURCE"] == "NPI-LOCATIONS" for r in lines) + + +def test_address_identity_and_dedupe(mapper): + ident = mapper.address_identity + assert ident({"ADDR_LINE1": "44 S. Kidwell Ave", "ADDR_POSTAL_CODE": "20180-1234"}) == "44SKIDWELLAVE|20180" + assert ident({"ADDR_CITY": "Nowhere"}) is None + provider = [{"ADDR_TYPE": "BUSINESS", "ADDR_LINE1": "44 S KIDWELL AVE", "ADDR_POSTAL_CODE": "201801234"}] + locations = [ + {"ADDR_TYPE": "SECONDARY", "ADDR_LINE1": "44 S Kidwell Ave.", "ADDR_POSTAL_CODE": "20180"}, + {"ADDR_TYPE": "SECONDARY", "ADDR_LINE1": "1 Main St", "ADDR_POSTAL_CODE": "20176"}, + {"ADDR_TYPE": "SECONDARY", "ADDR_LINE1": "1 MAIN ST", "ADDR_POSTAL_CODE": "20176-0001"}, + ] + assert mapper.secondary_addresses(locations, provider) == [locations[1]]