From f047e8d82d55c36468675bd17e7977b3da03f451 Mon Sep 17 00:00:00 2001 From: Akeem King <13573860+KingAkeem@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:16:44 -0400 Subject: [PATCH] Integrate Social Mapping across the platform --- .dockerignore | 7 + .env.example | 7 + .github/workflows/integration.yml | 50 +++++ .gitignore | 1 + .gitmodules | 11 +- CHANGELOG.md | 6 + Dockerfile | 9 +- README.md | 93 ++++----- dataset_service/README.md | 45 ++--- dataset_service/models.py | 2 +- dataset_service/repository.py | 234 +++++++++++++++++++++- dataset_service/server.py | 41 ++++ docker-compose.yml | 35 ++++ examples/datasets/geospatial_entities.csv | 4 + nginx.conf | 17 ++ scripts/import_geospatial_demo.py | 42 ++++ scripts/test_social_mapping_e2e.py | 29 +++ social_mapping | 1 + tests/test_dataset_service.py | 82 ++++++++ tests/test_nginx_routes.py | 3 + tests/test_social_mapping_e2e.py | 70 +++++++ whoisit | 2 +- 22 files changed, 711 insertions(+), 80 deletions(-) create mode 100644 .github/workflows/integration.yml create mode 100644 examples/datasets/geospatial_entities.csv create mode 100755 scripts/import_geospatial_demo.py create mode 100755 scripts/test_social_mapping_e2e.py create mode 160000 social_mapping create mode 100644 tests/test_social_mapping_e2e.py diff --git a/.dockerignore b/.dockerignore index 8b70911..5a2b863 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,11 @@ .git +.venv **/.git +**/.venv +**/node_modules +**/dist +**/build +**/__pycache__ __pycache__ *.pyc *.pyo @@ -8,3 +14,4 @@ __pycache__ .env .env.* .DS_Store +*.log diff --git a/.env.example b/.env.example index a1fa4f5..2176b54 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,13 @@ # Create it in an X Developer Portal project/app. Keep it secret. TWEEPY_BEARER_TOKEN=your_tweepy_bearer_token +# social_mapping: Nominatim resolves only investigator-entered search origins. +# Keep the identifying User-Agent and a contact/project URL when using the public service. +NOMINATIM_BASE_URL=https://nominatim.openstreetmap.org +NOMINATIM_USER_AGENT=social-mapping/1.0 (+https://github.com/osint-services/social_mapping) +GEOCODER_CACHE_DAYS=30 +GEOCODER_MIN_INTERVAL_SECONDS=1 + # phone_search: Twilio API credentials from the Twilio Console. # The Account SID normally starts with AC. Keep the auth token secret. TWILIO_ACCOUNT_SID=your_twilio_account_sid diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..5e62c8d --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,50 @@ +name: Platform integration + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - run: >- + pip install pytest pytest-asyncio + -r dataset_service/requirements.txt + -r profile_checker/requirements.txt + -r profile_search/requirements.txt + -r phone_search/requirements.txt + - run: pytest -q tests + - run: docker compose config --quiet + + compose-e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - run: docker compose up --build -d dataset_service social_mapping proxy + - name: Wait for map readiness + run: | + for attempt in {1..30}; do + if curl --fail --silent http://localhost/map/readyz >/dev/null; then + exit 0 + fi + sleep 2 + done + docker compose logs social_mapping dataset_service proxy + exit 1 + - run: python3 scripts/test_social_mapping_e2e.py + - if: always() + run: docker compose down --volumes diff --git a/.gitignore b/.gitignore index f02ff5d..0d706a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +.venv/ __pycache__/ *.py[cod] finder.log diff --git a/.gitmodules b/.gitmodules index faca96f..3d32e7c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,15 @@ [submodule "profile_checker"] path = profile_checker - url = git@github.com:osint-services/profile_checker + url = https://github.com/osint-services/profile_checker.git [submodule "profile_search"] path = profile_search - url = git@github.com:osint-services/profile_search + url = https://github.com/osint-services/profile_search.git [submodule "phone_search"] path = phone_search - url = git@github.com:osint-services/phone_search + url = https://github.com/osint-services/phone_search.git [submodule "whoisit"] path = whoisit - url = git@github.com:osint-services/whoisit.git + url = https://github.com/osint-services/whoisit.git +[submodule "social_mapping"] + path = social_mapping + url = https://github.com/osint-services/social_mapping.git diff --git a/CHANGELOG.md b/CHANGELOG.md index c35ee3f..587ef55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. ## [Unreleased] ### Added +- Integrated `social_mapping` as a pinned submodule and Compose service with `/map/*` nginx routing, readiness checks, and persistent geocoder cache storage. +- Added explicit geospatial dataset fields, additive migration, radius search, GeoJSON provenance, and associated phone metadata. +- Added synthetic London, Arlington, and Portland entities plus credential-free import and end-to-end helper scripts. +- Added a platform CI path that validates tests, Compose configuration, and the complete synthetic London map flow. - Added a sparse entity model that associates optional profile and phone identifiers while preserving their type-specific metadata. - Added a persistent SQLite dataset service for mapped profile and phone imports. - Added CSV, JSON, JSONL, and NDJSON parsing, previews, automatic field suggestions, per-row validation, provenance, and raw-record retention. @@ -15,6 +19,8 @@ All notable changes to this project will be documented in this file. - Expanded regression coverage for dataset import, fuzzy profile search, normalized phone search, deletion, health endpoints, and client-side parsers. ### Changed +- Routed the native whoisit Map workspace and standalone Social Mapping client through the same canonical `/map/search` API. +- Switched public submodule URLs to HTTPS so recursive clones do not require organization SSH keys. - Replaced separate profile and phone dataset ingestion with one entity mapping pipeline; legacy API imports remain compatible and existing SQLite databases migrate in place. - Unified username and phone lookup into one Search workspace with conservative auto-detection, explicit Profile and Phone modes, and source filters that control which APIs and datasets are queried. - Reworked the Electron interface into cohesive Search, Datasets, Integrations, and History workspaces with type-specific master/detail results. diff --git a/Dockerfile b/Dockerfile index 897b2cc..5462b21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,13 +9,18 @@ COPY ./profile_checker /app/profile_checker COPY ./profile_search /app/profile_search COPY ./phone_search /app/phone_search COPY ./dataset_service /app/dataset_service +COPY ./social_mapping /app/social_mapping_repo # Install the selected service dependencies ARG APP_DIR=profile_checker -RUN pip install --no-cache-dir -r /app/${APP_DIR}/requirements.txt +RUN if [ "$APP_DIR" = "social_mapping" ]; then \ + pip install --no-cache-dir -r /app/social_mapping_repo/requirements.txt; \ + else \ + pip install --no-cache-dir -r /app/${APP_DIR}/requirements.txt; \ + fi ENV PYTHONUNBUFFERED=1 -ENV PYTHONPATH=/app +ENV PYTHONPATH=/app:/app/social_mapping_repo EXPOSE 8000 diff --git a/README.md b/README.md index 037715e..546d68b 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,37 @@ # OSINT Services Platform -The platform combines live public-data APIs with user-supplied datasets behind one nginx endpoint and one Electron investigation workspace. +The platform combines public-data services, user-supplied datasets, and evidence-oriented mapping behind one nginx endpoint and the [`whoisit`](https://github.com/osint-services/whoisit) desktop workspace. ## Architecture ```text -whoisit desktop app - | - v -nginx :80 +whoisit desktop app and standalone Social Mapping client + | + v + nginx :80 |-- /scan/* -> profile_checker (public username availability) - |-- /focus* -> profile_search (X profile metadata via Tweepy) - |-- /phone_search* -> phone_search (caller-name data via Twilio) - `-- /datasets* -> dataset_service (SQLite entity imports and local search) + |-- /focus* -> profile_search (X profile metadata) + |-- /phone_search* -> phone_search (caller-name data) + |-- /datasets* -> dataset_service (SQLite imports and geo search) + `-- /map/* -> social_mapping (GeoJSON aggregation, X posts, origin geocoding) ``` -Docker Compose runs the four FastAPI services, nginx, and a persistent `dataset_data` volume. The Electron UI provides one search workspace with Auto, Profile, and Phone modes plus All, Live APIs, and Imported datasets source filters. It queries enabled sources independently and combines successful results, so one unavailable provider does not prevent other sources from being searched. +Docker Compose runs the five FastAPI services, nginx, a persistent `dataset_data` volume, and a separate durable `social_mapping_cache` volume. The map service queries datasets internally and can use the same X bearer-token configuration as profile search. `whoisit` calls `/map/search` natively; it does not embed the standalone client. -## Requirements +## Requirements and setup -- Docker with `docker compose` -- Node.js 18 or newer to build or run the Electron UI -- An X API bearer token for profile inspection -- Twilio Account SID and Auth Token for live phone lookup - -## Setup +- Docker with Compose v2 +- Node.js 22 or newer to build the current desktop UI +- Optional X bearer token for profile inspection and X map posts +- Optional Twilio credentials for live phone lookup ```bash git submodule update --init --recursive cp .env.example .env -``` - -Add your provider credentials to `.env`, then start the API stack: - -```bash ./scripts/start.sh ``` -Run the desktop UI from a second terminal: +Run the desktop UI from another terminal: ```bash cd whoisit @@ -45,18 +39,27 @@ npm install npm start ``` -The UI checks service readiness on launch and can attempt to start the Compose stack when Docker is available. Its Integrations workspace shows the configured and reachable state for X/Tweepy, Twilio Lookup, and local datasets. Users can replace provider credentials there without exposing existing values to the renderer. Packaged builds are created with `npm run package`. +The imported-dataset map works without live credentials. Load the synthetic fixture and verify London end to end: + +```bash +python3 scripts/import_geospatial_demo.py +python3 scripts/test_social_mapping_e2e.py +``` ## Environment values | Variable | Required | Used by | Meaning | | --- | --- | --- | --- | -| `TWEEPY_BEARER_TOKEN` | For live profile inspection | `profile_search` | X API v2 application bearer token. Despite the variable name, the credential is issued by X; Tweepy is the Python client. | -| `TWILIO_ACCOUNT_SID` | For live phone lookup | `phone_search` | Twilio account identifier, normally beginning with `AC`. | -| `TWILIO_AUTH_TOKEN` | For live phone lookup | `phone_search` | Secret used with the Account SID to authenticate Twilio API calls. | -| `DATASET_DB_PATH` | No | `dataset_service` | SQLite file location. Compose sets this to `/data/datasets.db` inside its persistent volume. | +| `TWEEPY_BEARER_TOKEN` | No | `profile_search`, `social_mapping` | X API v2 bearer token; map searches degrade to datasets when absent | +| `TWILIO_ACCOUNT_SID` | No | `phone_search` | Twilio account identifier | +| `TWILIO_AUTH_TOKEN` | No | `phone_search` | Twilio API secret | +| `DATASET_DB_PATH` | No | `dataset_service` | Compose uses `/data/datasets.db` in persistent storage | +| `NOMINATIM_BASE_URL` | No | `social_mapping` | Server-side search-origin geocoder | +| `NOMINATIM_USER_AGENT` | Recommended | `social_mapping` | Identifying User-Agent and contact/project URL | +| `GEOCODER_CACHE_DAYS` | No | `social_mapping` | Durable normalized place-cache refresh interval; default 30 | +| `GEOCODER_MIN_INTERVAL_SECONDS` | No | `social_mapping` | Process-wide Nominatim request spacing; default 1 | -Do not commit `.env`. Imported datasets may contain sensitive or licensed information; only import data you are authorized to retain and use. +Do not commit `.env`. Imported datasets may contain sensitive or licensed information; import only data you are authorized to retain and use. ## API routes @@ -64,34 +67,34 @@ Do not commit `.env`. Imported datasets may contain sensitive or licensed inform | --- | --- | | `GET /scan/{username}` | Check supported sites for a public username | | `GET /focus?url=https://x.com/{username}` | Retrieve expanded X profile metadata | -| `GET /phone_search?phone_number=+18135551212` | Retrieve live caller-name metadata | +| `GET /phone_search?phone_number=...` | Retrieve live caller-name metadata | | `GET /datasets` | List imported datasets | -| `GET /datasets/schema/entity` | Get the unified sparse entity schema and sample | -| `POST /datasets/import` | Import mapped entities containing profile identifiers, phone identifiers, or both | -| `GET /datasets/search/profiles?query=...&fuzzy=true` | Search imported profiles | -| `GET /datasets/search/phones?phone_number=...` | Search imported phone records | -| `DELETE /datasets/{dataset_id}` | Delete a dataset and its records | +| `POST /datasets/import` | Import sparse entities, profiles, or phone records | +| `GET /datasets/search/geo` | Radius-search only records with explicit coordinate pairs | +| `GET /map/search` | Return combined GeoJSON for a place or coordinate origin | +| `GET /map/healthz` | Map process liveness | +| `GET /map/readyz` | Map cache and configuration readiness | + +Dataset imports accept nullable `latitude`, `longitude`, `location_accuracy`, and `location_source` fields. Latitude and longitude must be supplied as a valid pair. The migration is additive for existing SQLite databases. Free-text record locations are not geocoded into evidence coordinates. Associated phone records can appear in marker details but never produce or imply phone/device locations. -The import API accepts sparse entity rows after the UI parses CSV, JSON, JSONL, or NDJSON. Each row needs at least one username, profile URL, or E.164 phone number; profile and phone fields can coexist and remain associated in search results. A single import is limited to 10,000 records and the UI limits files to 10 MB. Canonical records preserve their source row, dataset provenance, observation time, and optional confidence score. Legacy typed imports remain API-compatible and existing databases migrate in place. +`/search/tweets?city=&distance=` remains a deprecated compatibility alias until the next major Social Mapping release. ## Operations and tests ```bash -./scripts/build.sh # rebuild images -./scripts/logs.sh # stream Compose logs -./scripts/stop.sh # stop the stack -.venv/bin/pytest -q # backend tests -cd whoisit && npm test # importer parser tests +./scripts/build.sh +./scripts/logs.sh +./scripts/stop.sh +.venv/bin/pytest -q tests +docker compose config --quiet ``` -Liveness endpoints are available at `/scan/healthz`, `/phone_search/healthz`, and `/datasets/healthz`. Profile inspection also exposes `/focus/readyz`, which verifies that its X credential is configured. +CI mocks external X and Nominatim behavior, runs platform tests, validates Compose, and starts the dataset/map/proxy path for a credential-free synthetic London test. Live X testing is optional and deliberately separate so routine CI never spends X credits. ## Documentation +- [Social Mapping](social_mapping/README.md) - [Desktop UI guide](whoisit/README.md) - [Dataset service guide](dataset_service/README.md) -- [Sample combined entity dataset](examples/datasets/sample_entities.csv) -- [Sample profile dataset](examples/datasets/sample_profiles.csv) -- [Sample phone dataset](examples/datasets/sample_phone_records.jsonl) +- [Synthetic geospatial fixture](examples/datasets/geospatial_entities.csv) - [Changelog](CHANGELOG.md) -- [2026-07-17 release notes](CHANGELOG-2026-07-17.md) diff --git a/dataset_service/README.md b/dataset_service/README.md index 5104616..8070d9e 100644 --- a/dataset_service/README.md +++ b/dataset_service/README.md @@ -1,42 +1,41 @@ # Dataset service -The dataset service provides normalized, persistent identity sources for the platform's profile and phone searches. It uses SQLite and has no external provider dependency. +The dataset service stores normalized identity evidence for profile, phone, and geospatial searches. It uses SQLite and has no external provider dependency. -## Supported input +## Imports and coordinates -The Electron UI parses `.csv`, `.json`, `.jsonl`, and `.ndjson` files, then maps every import through the entity schema returned by: +The desktop UI parses CSV, JSON, JSONL, and NDJSON and maps rows through `GET /datasets/schema/entity`. Each sparse entity needs a username, profile URL, or E.164 phone number. Optional explicit geospatial fields are: -```http -GET /datasets/schema/entity -``` +- `latitude` and `longitude`, supplied together and within valid ranges; +- `location_accuracy`, such as `exact`, `place`, or `approximate`; and +- `location_source`, describing where the coordinate evidence came from. -Each sparse entity row requires at least one searchable identifier: a username, profile URL, or E.164 phone number such as `+12025550101`. All other fields are optional. When both profile and phone fields exist, the service stores typed identifiers under one entity so a match for either identifier exposes the association. Invalid booleans, negative metrics, confidence outside `0..1`, and malformed identifiers are rejected per row without discarding valid rows. +Free-text `location` remains descriptive text. The service never geocodes it. Existing databases receive these nullable columns through an additive startup migration. -The legacy `profile` and `phone` schemas and import values remain accepted for API compatibility, but the desktop application uses only the entity pipeline. Existing SQLite databases are migrated in place when the service starts. +Associated profile and phone identifiers remain attached to the entity. Phone information may be returned as marker metadata, but a phone row never creates or implies a device location. -Each accepted entity includes: +## Geospatial search -- its dataset and source; -- an observation timestamp; -- optional confidence; -- normalized searchable identifiers; and -- the complete original source row for audit. +```http +GET /datasets/search/geo?latitude=51.5074&longitude=-0.1278&radius_miles=5&query=demo&limit=100 +``` -## Storage and limits +The route returns a GeoJSON `FeatureCollection` containing radius-filtered entity and standalone profile records with source, accuracy, confidence, timestamp, distance, provenance, and associated identifiers. -Compose stores `/data/datasets.db` in the named `dataset_data` volume. Deleting a dataset cascades to its entities and typed identifiers. Each import accepts 1–10,000 JSON object rows; the desktop UI additionally enforces a 10 MB file limit. +## Storage and limits -The API is intentionally local and currently has no authentication layer. Do not expose the nginx proxy to an untrusted network without adding access control. +Compose stores `/data/datasets.db` in `dataset_data`. Deleting a dataset cascades to its entities and typed identifiers. Imports accept 1 to 10,000 JSON object rows; the desktop UI also limits files to 10 MB. The API is local and has no authentication layer, so do not expose nginx to an untrusted network without access control. ## Sample data -The platform repository includes a combined fictional entity dataset plus separate fixtures for exercising compatibility and source-specific fields: - -- [`examples/datasets/sample_entities.csv`](../examples/datasets/sample_entities.csv) -- [`examples/datasets/sample_profiles.csv`](../examples/datasets/sample_profiles.csv) -- [`examples/datasets/sample_phone_records.jsonl`](../examples/datasets/sample_phone_records.jsonl) +- [`geospatial_entities.csv`](../examples/datasets/geospatial_entities.csv): synthetic London, Arlington, and Portland entities with explicit coordinates and reserved fictional phone numbers. +- [`sample_entities.csv`](../examples/datasets/sample_entities.csv): combined identities. +- [`sample_profiles.csv`](../examples/datasets/sample_profiles.csv): profile compatibility import. +- [`sample_phone_records.jsonl`](../examples/datasets/sample_phone_records.jsonl): phone compatibility import. -Import any of them from the same desktop Datasets workflow; canonical headers are auto-mapped. The combined file demonstrates a username and phone on one entity, a profile-only entity, and a phone-only entity. Search for `demo_ada_1843` or `+12025550101` to see the association in both directions. The phone values use the reserved North American `555-01xx` fictional range and do not identify real subscribers. +```bash +python3 scripts/import_geospatial_demo.py +``` ## Development diff --git a/dataset_service/models.py b/dataset_service/models.py index ee385c5..4b17143 100644 --- a/dataset_service/models.py +++ b/dataset_service/models.py @@ -15,7 +15,7 @@ class DatasetImportRequest(BaseModel): record_type: RecordType = RecordType.entity filename: str | None = Field(default=None, max_length=255) mapping: dict[str, str] = Field(default_factory=dict) - rows: list[dict[str, Any]] = Field(min_items=1, max_items=10_000) + rows: list[dict[str, Any]] = Field(min_length=1, max_length=10_000) class RejectedRow(BaseModel): diff --git a/dataset_service/repository.py b/dataset_service/repository.py index a3f0b33..8bf09d0 100644 --- a/dataset_service/repository.py +++ b/dataset_service/repository.py @@ -1,4 +1,5 @@ import json +from math import asin, cos, radians, sin, sqrt import re import sqlite3 from datetime import datetime, timezone @@ -32,6 +33,10 @@ "observed_at", "confidence", "source", + "latitude", + "longitude", + "location_accuracy", + "location_source", ) PHONE_FIELDS = ( @@ -72,6 +77,10 @@ observed_at TEXT, confidence REAL, source TEXT, + latitude REAL, + longitude REAL, + location_accuracy TEXT, + location_source TEXT, raw_json TEXT NOT NULL ); @@ -101,6 +110,10 @@ observed_at TEXT, confidence REAL, source TEXT, + latitude REAL, + longitude REAL, + location_accuracy TEXT, + location_source TEXT, raw_json TEXT NOT NULL ); @@ -174,6 +187,31 @@ def parse_confidence(value: Any) -> float | None: return confidence +def parse_coordinate_pair(latitude_value: Any, longitude_value: Any) -> tuple[float | None, float | None]: + has_latitude = latitude_value is not None and str(latitude_value).strip() != "" + has_longitude = longitude_value is not None and str(longitude_value).strip() != "" + if has_latitude != has_longitude: + raise ValueError("latitude and longitude must be provided together") + if not has_latitude: + return None, None + latitude = float(latitude_value) + longitude = float(longitude_value) + if not -90 <= latitude <= 90: + raise ValueError("latitude must be between -90 and 90") + if not -180 <= longitude <= 180: + raise ValueError("longitude must be between -180 and 180") + return latitude, longitude + + +def haversine_miles(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + earth_radius_miles = 3958.7613 + lat1_r, lon1_r, lat2_r, lon2_r = map(radians, (lat1, lon1, lat2, lon2)) + dlat = lat2_r - lat1_r + dlon = lon2_r - lon1_r + value = sin(dlat / 2) ** 2 + cos(lat1_r) * cos(lat2_r) * sin(dlon / 2) ** 2 + return 2 * earth_radius_miles * asin(sqrt(value)) + + def normalize_username(value: Any) -> str | None: username = clean_text(value) return username.lstrip("@").lower() if username else None @@ -225,6 +263,22 @@ def initialize(self) -> None: self.migrate_legacy_schema(connection) else: connection.executescript(SCHEMA_SQL) + self.ensure_geospatial_columns(connection) + + @staticmethod + def ensure_geospatial_columns(connection: sqlite3.Connection) -> None: + for table in ("entities", "profile_records"): + columns = {row["name"] for row in connection.execute(f"PRAGMA table_info({table})")} + for name, column_type in ( + ("latitude", "REAL"), + ("longitude", "REAL"), + ("location_accuracy", "TEXT"), + ("location_source", "TEXT"), + ): + if name not in columns: + connection.execute(f"ALTER TABLE {table} ADD COLUMN {name} {column_type}") + connection.execute("CREATE INDEX IF NOT EXISTS idx_entity_coordinates ON entities(latitude, longitude)") + connection.execute("CREATE INDEX IF NOT EXISTS idx_profile_coordinates ON profile_records(latitude, longitude)") @staticmethod def migrate_legacy_schema(connection: sqlite3.Connection) -> None: @@ -341,10 +395,12 @@ def import_dataset(self, request: DatasetImportRequest) -> DatasetImportResult: """ INSERT INTO entities ( id, dataset_id, display_name, bio, location, website, - observed_at, confidence, source, raw_json + observed_at, confidence, source, latitude, longitude, + location_accuracy, location_source, raw_json ) VALUES ( :id, :dataset_id, :display_name, :bio, :location, :website, - :observed_at, :confidence, :source, :raw_json + :observed_at, :confidence, :source, :latitude, :longitude, + :location_accuracy, :location_source, :raw_json ) """, [dict(record, dataset_id=dataset_id) for record in entities], @@ -357,13 +413,15 @@ def import_dataset(self, request: DatasetImportRequest) -> DatasetImportResult: profile_url, display_name, bio, location, website, verified, protected, created_at, profile_image_url, profile_banner_url, followers_count, following_count, post_count, listed_count, - like_count, media_count, observed_at, confidence, source, raw_json + like_count, media_count, observed_at, confidence, source, + latitude, longitude, location_accuracy, location_source, raw_json ) VALUES ( :id, :dataset_id, :entity_id, :platform, :username, :username_normalized, :profile_url, :display_name, :bio, :location, :website, :verified, :protected, :created_at, :profile_image_url, :profile_banner_url, :followers_count, :following_count, :post_count, :listed_count, - :like_count, :media_count, :observed_at, :confidence, :source, :raw_json + :like_count, :media_count, :observed_at, :confidence, :source, + :latitude, :longitude, :location_accuracy, :location_source, :raw_json ) """, [ @@ -411,6 +469,7 @@ def prepare_entity( raise ValueError("username, profile_url, or phone_number is required") entity_id = str(uuid4()) + latitude, longitude = parse_coordinate_pair(values["latitude"], values["longitude"]) observed_at = clean_text(values["observed_at"]) or utc_now() confidence = parse_confidence(values["confidence"]) source = clean_text(values["source"]) or dataset_name @@ -426,6 +485,10 @@ def prepare_entity( "observed_at": observed_at, "confidence": confidence, "source": source, + "latitude": latitude, + "longitude": longitude, + "location_accuracy": clean_text(values["location_accuracy"]), + "location_source": clean_text(values["location_source"]), "raw_json": json.dumps(raw_row, default=str), } @@ -452,6 +515,7 @@ def prepare_profile( if not username_normalized: raise ValueError("username or profile_url is required") username = username or username_normalized + latitude, longitude = parse_coordinate_pair(values["latitude"], values["longitude"]) return { "id": str(uuid4()), @@ -477,6 +541,10 @@ def prepare_profile( "observed_at": clean_text(values["observed_at"]) or utc_now(), "confidence": parse_confidence(values["confidence"]), "source": clean_text(values["source"]) or dataset_name, + "latitude": latitude, + "longitude": longitude, + "location_accuracy": clean_text(values["location_accuracy"]), + "location_source": clean_text(values["location_source"]), "raw_json": json.dumps(raw_row, default=str), } @@ -577,6 +645,156 @@ def search_phones(self, phone_number: str) -> list[dict[str, Any]]: records.append(record) return records + def search_geo( + self, + *, + latitude: float, + longitude: float, + radius_miles: float, + query: str | None, + limit: int, + ) -> list[dict[str, Any]]: + parse_coordinate_pair(latitude, longitude) + if radius_miles <= 0 or radius_miles > 1000: + raise ValueError("radius_miles must be greater than 0 and no more than 1000") + latitude_delta = radius_miles / 69.0 + longitude_scale = max(cos(radians(latitude)), 0.01) + longitude_delta = radius_miles / (69.172 * longitude_scale) + normalized_query = query.strip().casefold() if query and query.strip() else None + + with self.connect() as connection: + entity_rows = connection.execute( + """ + SELECT entities.*, datasets.name AS dataset_name + FROM entities + JOIN datasets ON datasets.id = entities.dataset_id + WHERE latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ? + """, + ( + latitude - latitude_delta, + latitude + latitude_delta, + longitude - longitude_delta, + longitude + longitude_delta, + ), + ).fetchall() + profile_rows = connection.execute( + """ + SELECT profile_records.*, datasets.name AS dataset_name + FROM profile_records + JOIN datasets ON datasets.id = profile_records.dataset_id + WHERE entity_id IS NULL + AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ? + """, + ( + latitude - latitude_delta, + latitude + latitude_delta, + longitude - longitude_delta, + longitude + longitude_delta, + ), + ).fetchall() + + candidates: list[dict[str, Any]] = [] + for row in entity_rows: + summary = self.entity_summary(connection, row["id"]) or {} + searchable = " ".join( + str(value or "") + for value in (row["display_name"], row["bio"], row["location"], row["source"]) + ).casefold() + if normalized_query and normalized_query not in searchable: + continue + distance = haversine_miles(latitude, longitude, row["latitude"], row["longitude"]) + if distance > radius_miles: + continue + candidates.append( + self.geo_feature( + feature_id=f"dataset:entity:{row['id']}", + latitude=row["latitude"], + longitude=row["longitude"], + title=row["display_name"] or "Imported entity", + text=row["bio"], + record_type="entity", + row=row, + distance=distance, + associated_profiles=summary.get("profiles", []), + associated_phones=summary.get("phone_numbers", []), + ) + ) + + for row in profile_rows: + searchable = " ".join( + str(value or "") + for value in (row["username"], row["display_name"], row["bio"], row["location"], row["source"]) + ).casefold() + if normalized_query and normalized_query not in searchable: + continue + distance = haversine_miles(latitude, longitude, row["latitude"], row["longitude"]) + if distance > radius_miles: + continue + candidates.append( + self.geo_feature( + feature_id=f"dataset:profile:{row['id']}", + latitude=row["latitude"], + longitude=row["longitude"], + title=row["display_name"] or f"@{row['username']}", + text=row["bio"], + record_type="profile", + row=row, + distance=distance, + associated_profiles=[ + { + "id": row["id"], + "platform": row["platform"], + "username": row["username"], + "profile_uri": row["profile_url"], + } + ], + associated_phones=[], + ) + ) + + candidates.sort(key=lambda item: item["properties"]["distance_miles"]) + return candidates[:limit] + + @staticmethod + def geo_feature( + *, + feature_id: str, + latitude: float, + longitude: float, + title: str, + text: str | None, + record_type: str, + row: sqlite3.Row, + distance: float, + associated_profiles: list[dict[str, Any]], + associated_phones: list[dict[str, Any]], + ) -> dict[str, Any]: + return { + "type": "Feature", + "id": feature_id, + "geometry": {"type": "Point", "coordinates": [longitude, latitude]}, + "properties": { + "source": "datasets", + "record_type": record_type, + "title": title, + "text": text, + "location_accuracy": row["location_accuracy"] or "unknown", + "location_source": row["location_source"] or "dataset_import", + "confidence": row["confidence"], + "timestamp": row["observed_at"], + "distance_miles": round(distance, 3), + "associated_profiles": associated_profiles, + "associated_phones": associated_phones, + "provenance": { + "provider": "platform_dataset", + "dataset_id": row["dataset_id"], + "dataset_name": row["dataset_name"], + "record_id": row["id"], + "record_source": row["source"], + }, + }, + } + def dataset_records( self, dataset_id: str, limit: int, offset: int ) -> tuple[str, list[dict[str, Any]]] | None: @@ -643,6 +861,10 @@ def entity_summary( "bio": entity["bio"], "location": entity["location"], "website": entity["website"], + "latitude": entity["latitude"], + "longitude": entity["longitude"], + "location_accuracy": entity["location_accuracy"], + "location_source": entity["location_source"], "profiles": [ { "id": row["id"], @@ -708,6 +930,10 @@ def profile_response(row: sqlite3.Row) -> dict[str, Any]: "name": row["display_name"], "bio": row["bio"], "location": row["location"], + "latitude": row["latitude"], + "longitude": row["longitude"], + "location_accuracy": row["location_accuracy"], + "location_source": row["location_source"], "website": row["website"], "verified": ( bool(row["verified"]) if row["verified"] is not None else None diff --git a/dataset_service/server.py b/dataset_service/server.py index 2307f1e..a8123c0 100644 --- a/dataset_service/server.py +++ b/dataset_service/server.py @@ -25,6 +25,10 @@ FieldDefinition(key="display_name", label="Display name", description="Human-readable account name."), FieldDefinition(key="bio", label="Bio", description="Public profile description."), FieldDefinition(key="location", label="Location", description="Public or source-provided location."), + FieldDefinition(key="latitude", label="Latitude", description="Explicit evidence latitude; must be paired with longitude."), + FieldDefinition(key="longitude", label="Longitude", description="Explicit evidence longitude; must be paired with latitude."), + FieldDefinition(key="location_accuracy", label="Location accuracy", description="Accuracy label such as exact, place, or approximate."), + FieldDefinition(key="location_source", label="Location source", description="Source or method that supplied the coordinate pair."), FieldDefinition(key="website", label="Website", description="Website associated with the account."), FieldDefinition(key="verified", label="Verified", description="Boolean verification status."), FieldDefinition(key="protected", label="Protected", description="Boolean protected/private status."), @@ -94,6 +98,10 @@ FieldDefinition(key="display_name", label="Display name", description="Person, organization, or account name."), FieldDefinition(key="bio", label="Bio", description="Source-provided identity or profile description."), FieldDefinition(key="location", label="Location", description="Public or source-provided location."), + FieldDefinition(key="latitude", label="Latitude", description="Explicit evidence latitude; must be paired with longitude."), + FieldDefinition(key="longitude", label="Longitude", description="Explicit evidence longitude; must be paired with latitude."), + FieldDefinition(key="location_accuracy", label="Location accuracy", description="Accuracy label such as exact, place, or approximate."), + FieldDefinition(key="location_source", label="Location source", description="Source or method that supplied the coordinate pair."), FieldDefinition(key="website", label="Website", description="Website associated with the entity or profile."), FieldDefinition(key="verified", label="Verified", description="Boolean profile verification status."), FieldDefinition(key="protected", label="Protected", description="Boolean protected/private profile status."), @@ -123,6 +131,10 @@ "profile_url": "https://profiles.example/demo_ada_1843", "phone_number": "+12025550101", "display_name": "Ada Example", + "latitude": 51.5074, + "longitude": -0.1278, + "location_accuracy": "exact", + "location_source": "source_export", "carrier_name": "Example Wireless", "line_type": "mobile", "observed_at": "2026-07-17T12:00:00Z", @@ -211,6 +223,35 @@ def search_phones(phone_number: str = Query(min_length=1)) -> dict: raise HTTPException(status_code=422, detail=str(exc)) from exc return {"query": phone_number, "records": records} + @app.get("/datasets/search/geo") + def search_geo( + latitude: float = Query(ge=-90, le=90), + longitude: float = Query(ge=-180, le=180), + radius_miles: float = Query(default=25, gt=0, le=1000), + query: str | None = Query(default=None, max_length=512), + limit: int = Query(default=100, ge=1, le=500), + ) -> dict: + try: + features = repository.search_geo( + latitude=latitude, + longitude=longitude, + radius_miles=radius_miles, + query=query, + limit=limit, + ) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + return { + "type": "FeatureCollection", + "features": features, + "metadata": { + "origin": {"latitude": latitude, "longitude": longitude}, + "radius_miles": radius_miles, + "query": query, + "limit": limit, + }, + } + return app diff --git a/docker-compose.yml b/docker-compose.yml index e9cae25..de63150 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,39 @@ services: command: uvicorn dataset_service.server:app --host 0.0.0.0 --port 8000 volumes: - dataset_data:/data + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz')"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - platform_net + + social_mapping: + build: + context: . + dockerfile: Dockerfile + args: + APP_DIR: social_mapping + environment: + - DATASET_SERVICE_URL=http://dataset_service:8000 + - TWEEPY_BEARER_TOKEN=${TWEEPY_BEARER_TOKEN:-} + - NOMINATIM_BASE_URL=${NOMINATIM_BASE_URL:-https://nominatim.openstreetmap.org} + - NOMINATIM_USER_AGENT=${NOMINATIM_USER_AGENT:-social-mapping/1.0 (+https://github.com/osint-services/social_mapping)} + - GEOCODER_CACHE_PATH=/data/geocoder-cache.db + - GEOCODER_CACHE_DAYS=${GEOCODER_CACHE_DAYS:-30} + - GEOCODER_MIN_INTERVAL_SECONDS=${GEOCODER_MIN_INTERVAL_SECONDS:-1} + command: uvicorn social_mapping.server:app --host 0.0.0.0 --port 8000 + depends_on: + dataset_service: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/map/readyz')"] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - social_mapping_cache:/data networks: - platform_net @@ -57,6 +90,7 @@ services: - profile_search - phone_search - dataset_service + - social_mapping ports: - "80:80" volumes: @@ -70,3 +104,4 @@ networks: volumes: dataset_data: + social_mapping_cache: diff --git a/examples/datasets/geospatial_entities.csv b/examples/datasets/geospatial_entities.csv new file mode 100644 index 0000000..92ef975 --- /dev/null +++ b/examples/datasets/geospatial_entities.csv @@ -0,0 +1,4 @@ +username,platform,profile_url,phone_number,display_name,bio,location,latitude,longitude,location_accuracy,location_source,observed_at,confidence,source +london_map_demo,DemoNet,https://profiles.example/london_map_demo,+12025550111,London Map Demo,A synthetic London record for geospatial stack testing.,London,51.5074,-0.1278,exact,synthetic_fixture,2026-08-21T12:00:00Z,1.0,OSINT Services synthetic geospatial fixture +arlington_map_demo,CodeCommons,https://profiles.example/arlington_map_demo,+12025550112,Arlington Map Demo,A synthetic Arlington Virginia record for geospatial stack testing.,Arlington VA,38.8816,-77.0910,exact,synthetic_fixture,2026-08-21T12:00:00Z,1.0,OSINT Services synthetic geospatial fixture +portland_map_demo,PhotoSquare,https://profiles.example/portland_map_demo,+12025550113,Portland Map Demo,A synthetic Portland Oregon record for geospatial stack testing.,"Portland, OR",45.5152,-122.6784,exact,synthetic_fixture,2026-08-21T12:00:00Z,1.0,OSINT Services synthetic geospatial fixture diff --git a/nginx.conf b/nginx.conf index 5d4028b..966df02 100644 --- a/nginx.conf +++ b/nginx.conf @@ -11,6 +11,7 @@ http { set $profile_search_upstream profile_search:8000; set $phone_search_upstream phone_search:8000; set $dataset_service_upstream dataset_service:8000; + set $social_mapping_upstream social_mapping:8000; location = /scan/healthz { proxy_pass http://$profile_checker_upstream/healthz; @@ -76,5 +77,21 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } + location /map/ { + proxy_pass http://$social_mapping_upstream; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /search/tweets { + proxy_pass http://$social_mapping_upstream/search/tweets; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } } diff --git a/scripts/import_geospatial_demo.py b/scripts/import_geospatial_demo.py new file mode 100755 index 0000000..c69ec23 --- /dev/null +++ b/scripts/import_geospatial_demo.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +import csv +import json +from pathlib import Path +import sys +from urllib import error, request + + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = ROOT / "examples" / "datasets" / "geospatial_entities.csv" + + +def main() -> int: + base_url = (sys.argv[1] if len(sys.argv) > 1 else "http://localhost").rstrip("/") + with SAMPLE.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + payload = json.dumps( + { + "name": "Synthetic geospatial entities", + "filename": SAMPLE.name, + "record_type": "entity", + "rows": rows, + } + ).encode("utf-8") + import_request = request.Request( + f"{base_url}/datasets/import", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with request.urlopen(import_request, timeout=20) as response: + result = json.load(response) + except error.HTTPError as exc: + print(exc.read().decode("utf-8"), file=sys.stderr) + return 1 + print(f"Imported {result['imported']} synthetic geospatial entities into dataset {result['dataset_id']}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_social_mapping_e2e.py b/scripts/test_social_mapping_e2e.py new file mode 100755 index 0000000..7f9f287 --- /dev/null +++ b/scripts/test_social_mapping_e2e.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from urllib import parse, request + + +def main() -> int: + base_url = (sys.argv[1] if len(sys.argv) > 1 else "http://localhost").rstrip("/") + subprocess.run([sys.executable, "scripts/import_geospatial_demo.py", base_url], check=True) + query = parse.urlencode( + {"latitude": 51.5074, "longitude": -0.1278, "radius_miles": 5, "sources": "datasets"} + ) + with request.urlopen(f"{base_url}/map/search?{query}", timeout=20) as response: + payload = json.load(response) + matches = [feature for feature in payload["features"] if feature["properties"]["title"] == "London Map Demo"] + if len(matches) != 1: + raise RuntimeError("London synthetic marker was not returned") + properties = matches[0]["properties"] + if properties["source"] != "datasets" or properties["location_accuracy"] != "exact": + raise RuntimeError("London marker is missing source or accuracy metadata") + if not properties["associated_phones"]: + raise RuntimeError("Associated phone metadata was not returned") + print("Social Mapping end-to-end check passed with the synthetic London marker.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/social_mapping b/social_mapping new file mode 160000 index 0000000..4fd2fed --- /dev/null +++ b/social_mapping @@ -0,0 +1 @@ +Subproject commit 4fd2fedd183df9607862ead17a6d6e9055086a12 diff --git a/tests/test_dataset_service.py b/tests/test_dataset_service.py index 5f086ef..22561e0 100644 --- a/tests/test_dataset_service.py +++ b/tests/test_dataset_service.py @@ -1,5 +1,6 @@ import csv import json +import sqlite3 import sys from pathlib import Path @@ -9,6 +10,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from dataset_service.repository import SCHEMA_SQL from dataset_service.server import create_app @@ -29,6 +31,7 @@ def test_health_and_schema_are_available(tmp_path): "profile_url", "phone_number", ] + assert any(field["key"] == "latitude" for field in schema.json()["fields"]) def test_entity_import_accepts_sparse_and_combined_identifiers(tmp_path): @@ -290,3 +293,82 @@ def test_documented_sample_files_import_and_search(tmp_path): assert phones[0]["raw"]["case_note"].endswith( "used to exercise false and null values." ) + + +def test_geospatial_import_radius_query_and_associated_phone_metadata(tmp_path): + client = make_client(tmp_path) + with (ROOT / "examples" / "datasets" / "geospatial_entities.csv").open(newline="", encoding="utf-8") as file: + rows = list(csv.DictReader(file)) + imported = client.post( + "/datasets/import", + json={"name": "Synthetic geospatial entities", "record_type": "entity", "rows": rows}, + ) + assert imported.status_code == 201 + assert imported.json()["imported"] == 3 + + response = client.get( + "/datasets/search/geo", + params={"latitude": 51.5074, "longitude": -0.1278, "radius_miles": 5}, + ) + assert response.status_code == 200 + payload = response.json() + assert payload["type"] == "FeatureCollection" + assert len(payload["features"]) == 1 + marker = payload["features"][0] + assert marker["geometry"]["coordinates"] == [-0.1278, 51.5074] + assert marker["properties"]["title"] == "London Map Demo" + assert marker["properties"]["location_accuracy"] == "exact" + assert marker["properties"]["provenance"]["dataset_name"] == "Synthetic geospatial entities" + assert marker["properties"]["associated_phones"][0]["phone_number"] == "+12025550111" + + filtered = client.get( + "/datasets/search/geo", + params={"latitude": 51.5074, "longitude": -0.1278, "radius_miles": 5, "query": "missing"}, + ) + assert filtered.json()["features"] == [] + + +def test_invalid_coordinate_pairs_are_rejected(tmp_path): + client = make_client(tmp_path) + response = client.post( + "/datasets/import", + json={ + "name": "Invalid coordinates", + "record_type": "entity", + "rows": [ + {"username": "missing_longitude", "latitude": 20}, + {"username": "bad_latitude", "latitude": 95, "longitude": 0}, + ], + }, + ) + assert response.status_code == 201 + assert response.json()["imported"] == 0 + assert response.json()["rejected"] == 2 + assert "provided together" in response.json()["rejected_rows"][0]["reason"] + assert "between -90 and 90" in response.json()["rejected_rows"][1]["reason"] + + +def test_existing_database_receives_additive_geospatial_migration(tmp_path): + database = tmp_path / "existing.db" + legacy_schema = SCHEMA_SQL + for column in ( + " latitude REAL,\n", + " longitude REAL,\n", + " location_accuracy TEXT,\n", + " location_source TEXT,\n", + ): + legacy_schema = legacy_schema.replace(column, "") + with sqlite3.connect(database) as connection: + connection.executescript(legacy_schema) + connection.execute( + "INSERT INTO datasets VALUES (?, ?, ?, ?, ?, ?, ?)", + ("existing", "Existing dataset", "profile", None, "2026-01-01T00:00:00Z", 0, 0), + ) + + client = TestClient(create_app(database)) + assert client.get("/datasets").json()[0]["name"] == "Existing dataset" + with sqlite3.connect(database) as connection: + entity_columns = {row[1] for row in connection.execute("PRAGMA table_info(entities)")} + profile_columns = {row[1] for row in connection.execute("PRAGMA table_info(profile_records)")} + for columns in (entity_columns, profile_columns): + assert {"latitude", "longitude", "location_accuracy", "location_source"} <= columns diff --git a/tests/test_nginx_routes.py b/tests/test_nginx_routes.py index 24a318c..b433186 100644 --- a/tests/test_nginx_routes.py +++ b/tests/test_nginx_routes.py @@ -13,6 +13,7 @@ def test_nginx_uses_docker_dns_for_runtime_service_resolution(): "profile_search", "phone_search", "dataset_service", + "social_mapping", ): assert f"set ${service}_upstream {service}:8000;" in config assert f"proxy_pass http://${service}_upstream" in config @@ -29,3 +30,5 @@ def test_nginx_keeps_health_and_application_routes_separate(): assert "location /phone_search" in config assert "location = /datasets/healthz" in config assert "location /datasets" in config + assert "location /map/" in config + assert "location = /search/tweets" in config diff --git a/tests/test_social_mapping_e2e.py b/tests/test_social_mapping_e2e.py new file mode 100644 index 0000000..9c8a127 --- /dev/null +++ b/tests/test_social_mapping_e2e.py @@ -0,0 +1,70 @@ +import csv +import sys +from pathlib import Path + +from fastapi.testclient import TestClient + + +ROOT = Path(__file__).resolve().parents[1] +SOCIAL_MAPPING_ROOT = ROOT / "social_mapping" +for path in (ROOT, SOCIAL_MAPPING_ROOT): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) + +from dataset_service.server import create_app as create_dataset_app +from social_mapping import server as map_server + + +class FixedGeocoder: + async def resolve(self, query): + assert query == "London" + return { + "latitude": 51.5074, + "longitude": -0.1278, + "label": "London, United Kingdom", + "provider": "nominatim", + "cached": True, + } + + +class LocalDatasetProvider: + def __init__(self, repository): + self.repository = repository + + async def search(self, **parameters): + return self.repository.search_geo(**parameters) + + +def test_import_and_london_map_search_without_live_credentials(tmp_path, monkeypatch): + dataset_app = create_dataset_app(tmp_path / "e2e.db") + dataset_client = TestClient(dataset_app) + with (ROOT / "examples" / "datasets" / "geospatial_entities.csv").open(newline="", encoding="utf-8") as file: + rows = list(csv.DictReader(file)) + imported = dataset_client.post( + "/datasets/import", + json={"name": "Synthetic geospatial entities", "record_type": "entity", "rows": rows}, + ) + assert imported.status_code == 201 + + monkeypatch.setattr(map_server, "get_geocoder", lambda: FixedGeocoder()) + monkeypatch.setattr( + map_server, + "get_providers", + lambda: { + "datasets": LocalDatasetProvider(dataset_app.state.repository), + "x": LocalDatasetProvider(dataset_app.state.repository), + }, + ) + map_client = TestClient(map_server.app) + response = map_client.get( + "/map/search", + params={"place": "London", "radius_miles": 5, "sources": "datasets"}, + ) + assert response.status_code == 200 + payload = response.json() + assert payload["providers"]["datasets"]["status"] == "ok" + marker = payload["features"][0] + assert marker["properties"]["title"] == "London Map Demo" + assert marker["properties"]["source"] == "datasets" + assert marker["properties"]["location_accuracy"] == "exact" + assert marker["properties"]["associated_phones"][0]["phone_number"] == "+12025550111" diff --git a/whoisit b/whoisit index 7e4f8ae..56bb939 160000 --- a/whoisit +++ b/whoisit @@ -1 +1 @@ -Subproject commit 7e4f8ae4cf8209d3b6c14e5d1cb8abf6465acee1 +Subproject commit 56bb939871051aad4549f4bea5cb5fb6594df714