diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0d56216..f543c4de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,8 +123,8 @@ jobs: - name: Test with pytest if: matrix.os == 'ubuntu-latest' run: | - # Run pytest with coverage, using all available CPUs and verbose output - pytest -v -n auto --cov-report=xml --cov=ssapy --durations=20 + # Limit xdist workers to avoid memory exhaustion on Linux runners. + pytest -v -n 2 --cov-report=xml --cov=ssapy --durations=20 - name: Upload coverage reports to Codecov if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' uses: codecov/codecov-action@v4 diff --git a/docs/source/installation.rst b/docs/source/installation.rst index a364e85b..ee6755af 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -46,7 +46,7 @@ As the package has been published on `PyPI =0.2.0", ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index c9687c4c..0e1e767a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ graphviz numpy scipy jplephem -llnl-ssapy-data>=0.2.0 astropy pyerfa emcee @@ -18,3 +17,7 @@ ipython_genutils pytest-cov pytest-xdist flake8 +ssatk-data-core==0.0.1 +ssatk-data-gravity==0.0.1 +ssatk-data-lunar==0.0.1 +ssatk-data-lunar-gravity==0.0.1 diff --git a/skills.md b/skills.md index 80843591..02c84f7d 100644 --- a/skills.md +++ b/skills.md @@ -10,7 +10,7 @@ installed from PyPI or cloned from GitHub. - GitHub repository: `https://github.com/LLNL/SSAPy` - Documentation: `https://software.llnl.gov/SSAPy/` or `https://LLNL.github.io/SSAPy/` - Companion toolkit: `ssapy-toolkit`, imported as `ssapy_toolkit` -- Companion data package: `llnl-ssapy-data`, imported as `ssapy_data` +- Companion data packages: `ssatk-data-*`, imported through their `ssapy_data_*` namespaces - Minimum Python version: Python 3.10 ## Install and Import @@ -46,7 +46,7 @@ coordinate transforms, sampling, and track association. Use `ssapy-toolkit` for higher-level workflows, plotting dashboards, convenience wrappers, brightness calculations, and reusable analysis utilities. New reusable -datasets should generally live in `ssapy-data` rather than this repository. +datasets should generally live in the appropriate split `ssatk-data-*` repository rather than this repository. SSAPy builds a compiled extension, `ssapy._ssapy`, through `scikit-build` and `CMakeLists.txt`. If an import fails immediately after cloning, install the @@ -116,7 +116,7 @@ r, v = rv(orbit, times, propagator=KeplerianPropagator()) ## Data and Binary Rules SSAPy's data (ephemerides, gravity models, textures) comes from the required -`llnl-ssapy-data` package, under its `ssapy/` tree; the repository no longer +split `ssatk-data-*` packages; the repository no longer stores data or uses Git LFS. Planetary ephemerides are handled by `ssapy.ephemeris`: DE440 (`de440s.bsp`, 1849-2150) by default and a 1900-2150 DE430 excerpt on request. Epochs outside those spans download the full-span kernel from NAIF @@ -129,8 +129,8 @@ orientation files internally; prefer those public loaders over hard-coded paths. Avoid adding new large datasets, generated images, movies, notebooks with embedded outputs, archives, local shared libraries, or binary payloads to this -repository. If new reusable data are needed, prefer adding them to SSAPy-Data -and consuming them through a released `llnl-ssapy-data` wheel. +repository. If new reusable data are needed, prefer adding them to the appropriate +split `ssatk-data-*` repository and consuming its released wheel. Packaging should not include local compiled artifacts such as `_ssapy*.so` from developer builds. Generated build outputs belong outside the repository or in @@ -188,7 +188,7 @@ python -m twine check dist/* ## Common Pitfalls -- Do not add data files to this repository; add them to SSAPy-Data and release a new `llnl-ssapy-data`. +- Do not add data files to this repository; add them to the appropriate split `ssatk-data-*` repository. - Do not add Git LFS as a default solution for new data in this repository. - Do not commit generated `_ssapy*.so`, `build/`, `dist/`, egg-info, docs build output, caches, or local environments. - Do not rely on unbuilt in-tree imports for code paths that require `ssapy._ssapy`. diff --git a/ssapy/__init__.py b/ssapy/__init__.py index d8ee4160..43e93a7a 100644 --- a/ssapy/__init__.py +++ b/ssapy/__init__.py @@ -1,17 +1,21 @@ import os -__version__ = "1.1.10" +__version__ = "1.1.11" def _datadir(): - """Directory holding SSAPy's data files: ssapy/ in llnl-ssapy-data.""" + """Best-effort directory for installed split SSATK data packages.""" try: from importlib.resources import files - return os.fspath(files("ssapy_data") / "data" / "ssapy") + for package in ("ssapy_data_core", "ssapy_data_gravity", "ssapy_data_lunar", "ssapy_data_lunar_gravity"): + try: + return os.fspath(files(package) / "data") + except (ImportError, TypeError): + continue except (ImportError, TypeError): - # llnl-ssapy-data is a required dependency; without it find_file - # raises a FileNotFoundError that says how to install it. + # Split data packages are optional at import time; find_file reports + # the missing package when a resource is requested. return os.path.join(os.path.dirname(__file__), "data") diff --git a/ssapy/ephemeris.py b/ssapy/ephemeris.py index ab5ec8b6..cf4a50ce 100644 --- a/ssapy/ephemeris.py +++ b/ssapy/ephemeris.py @@ -2,7 +2,7 @@ Planetary ephemeris selection and long-span kernels. SSAPy evaluates Sun, Moon and planet positions from JPL development -ephemerides (DE). ``llnl-ssapy-data`` ships two short kernels: +ephemerides (DE). The split SSATK data packages ship the supported kernels: ========== ======================== ========================= Ephemeris Shipped kernel Coverage @@ -18,7 +18,7 @@ For epochs outside the shipped kernel, SSAPy uses a longer kernel of the same ephemeris (``de440.bsp`` or ``de430.bsp``, 1550-2650) and, for DE440 beyond that, DE441 (-13200 to +17191). These are 120 MB to 1.65 GB, so they are not -shipped: SSAPy looks for them in the working directory, ``llnl-ssapy-data`` +shipped: SSAPy looks for them in the working directory and split data packages and the cache directory, and otherwise downloads them from NAIF, checks their SHA-256, and caches them, warning with :class:`EphemerisDownloadWarning` before each download. Set ``SSAPY_EPHEMERIS_DOWNLOAD=0`` to forbid downloads @@ -72,7 +72,7 @@ class Kernel: name: str start_jd: float # TDB Julian dates covered by every segment SSAPy uses end_jd: float - sha256: Optional[str] = None # None for kernels shipped in llnl-ssapy-data + sha256: Optional[str] = None # None for kernels shipped in split data packages nbytes: Optional[int] = None solution: str = "" @@ -194,7 +194,7 @@ def _path(kernel, reason): return path if kernel.sha256 is None: raise EphemerisUnavailableError( - f"{kernel.name} is missing; install 'llnl-ssapy-data>=0.2.0'." + f"{kernel.name} is missing; install the appropriate ssatk-data-* package." ) destination = os.path.join(cache_dir(), kernel.name) manual = (f"Download {kernel.url} ({kernel.nbytes / 1e6:.0f} MB) into {cache_dir()} " diff --git a/ssapy/utils.py b/ssapy/utils.py index 5c8ce568..6ed23952 100644 --- a/ssapy/utils.py +++ b/ssapy/utils.py @@ -6,6 +6,7 @@ import re import numpy as np import warnings +from functools import lru_cache from astropy.time import Time as _Time import astropy.units as u from typing import Union, Tuple @@ -45,25 +46,32 @@ def _is_lfs_pointer(path): return False +@lru_cache(maxsize=1) def _ssapy_data_files(): - """Paths of the files shipped by the optional ``llnl-ssapy-data`` package.""" - try: - import ssapy_data - except ImportError: - return [] - try: - return [os.fspath(path) for path in ssapy_data.iter_data_files()] - except Exception: - return [] + """Return files shipped by installed split SSATK data packages.""" + from importlib.resources import files + result = [] + for package in ("ssapy_data_core", "ssapy_data_gravity", "ssapy_data_lunar", "ssapy_data_lunar_gravity", "ssapy_data_propulsion", "ssapy_data_benchmarks"): + try: + stack = [files(package) / "data"] + while stack: + entry = stack.pop() + if entry.is_dir(): stack.extend(entry.iterdir()) + elif entry.is_file(): + try: result.append(os.fspath(entry)) + except TypeError: pass + except (ImportError, TypeError, FileNotFoundError): + continue + return result def find_file(filename, ext=None): - """ Find a file in the current directory or the ssapy datadir (the - ``ssapy/`` tree of the ``llnl-ssapy-data`` package). If ext is not None, + """Find a file in the current directory or installed split data packages. + If ext is not None, also try appending ext to the filename. Git LFS pointer files (left over from SSAPy versions that stored data - with Git LFS) are skipped, and the rest of ``llnl-ssapy-data`` is searched + with Git LFS) are skipped, and installed split data packages are searched by file name. """ names = [filename] if ext is None else [filename, filename + ext] @@ -81,11 +89,10 @@ def find_file(filename, ext=None): if os.path.isfile(candidate): raise FileNotFoundError( f"{candidate} is a git LFS pointer, not the data file. SSAPy's data now " - "comes from the llnl-ssapy-data package: pip install 'llnl-ssapy-data>=0.2.0'." + "comes from the split ssatk-data-* packages." ) raise FileNotFoundError( - f"{filename} was not found in the working directory or in llnl-ssapy-data " - f"({datadir}); install it with pip install 'llnl-ssapy-data>=0.2.0'." + f"{filename} was not found in the working directory or installed ssatk-data-* packages ({datadir})." ) diff --git a/tests/test_accel.py b/tests/test_accel.py index 33b19e22..746c18c3 100644 --- a/tests/test_accel.py +++ b/tests/test_accel.py @@ -19,7 +19,7 @@ def _has_data(name): - # SSAPy's data comes from llnl-ssapy-data; skip only if it is missing. + # SSAPy's data comes from split SSATK data packages; skip only if missing. try: ssapy.utils.find_file(name) return True @@ -151,7 +151,7 @@ def _make_body_dependent_objects(): earth = moon = sun = earth_MG = moon_MG = sun_MG = None -@pytest.mark.skipif(not HAS_EGM84, reason="EGM84 gravity file unavailable (llnl-ssapy-data not installed)") +@pytest.mark.skipif(not HAS_EGM84, reason="EGM84 gravity file unavailable (split data package not installed)") @timer def test_MG_3_1(): """Exercise 3.1 from Montenbruck and Gill diff --git a/tests/test_data_file_lookup.py b/tests/test_data_file_lookup.py index 1472528d..211c1320 100644 --- a/tests/test_data_file_lookup.py +++ b/tests/test_data_file_lookup.py @@ -19,13 +19,13 @@ def test_lfs_pointer_is_not_returned_as_data(tmp_path, monkeypatch): (tmp_path / "de440s.bsp").write_text(POINTER) monkeypatch.setattr(utils, "datadir", str(tmp_path)) monkeypatch.setattr(utils, "_ssapy_data_files", lambda: []) - with pytest.raises(FileNotFoundError, match="git LFS pointer.*llnl-ssapy-data"): + with pytest.raises(FileNotFoundError, match="git LFS pointer.*split"): utils.find_file("de440s.bsp") def test_ssapy_data_is_searched_when_the_package_data_is_missing(tmp_path, monkeypatch): # With the package copy only a pointer, a real file of the same name - # shipped by llnl-ssapy-data is returned instead. + # shipped by a split SSATK data package is returned instead. local = tmp_path / "local" shipped = tmp_path / "shipped" / "ephemerides" local.mkdir() @@ -38,21 +38,16 @@ def test_ssapy_data_is_searched_when_the_package_data_is_missing(tmp_path, monke assert os.path.samefile(utils.find_file("de440s.bsp"), real) -def test_datadir_is_the_ssapy_tree_of_llnl_ssapy_data(): - # SSAPy no longer ships data: datadir is ssapy/ inside llnl-ssapy-data, and - # every file the body and gravity loaders need resolves there. +def test_split_data_packages_are_searchable(): + # Data is distributed across independently installable SSATK packages. import ssapy - import ssapy_data - from ssapy.body import _planetary_ephemeris_path - - assert os.path.samefile(ssapy.datadir, os.path.join(os.path.dirname(ssapy_data.__file__), "data", "ssapy")) - for name in ("moon_pa_de440_200625.bpc", "egm84.egm", "egm2008.egm.cof", "gggrx_1200a_sha.tab", "earth.png", "moon.png"): - assert os.path.dirname(utils.find_file(name)) == os.path.normpath(ssapy.datadir) - assert os.path.basename(_planetary_ephemeris_path()) == "de440s.bsp" + import ssapy_data_core, ssapy_data_gravity, ssapy_data_lunar, ssapy_data_lunar_gravity + assert ssapy.datadir + assert all(pkg.__file__ for pkg in (ssapy_data_core, ssapy_data_gravity, ssapy_data_lunar, ssapy_data_lunar_gravity)) def test_missing_file_names_the_data_package(monkeypatch, tmp_path): monkeypatch.setattr(utils, "datadir", str(tmp_path)) monkeypatch.setattr(utils, "_ssapy_data_files", lambda: []) - with pytest.raises(FileNotFoundError, match="llnl-ssapy-data"): + with pytest.raises(FileNotFoundError, match="ssatk-data"): utils.find_file("no_such_file.bsp") diff --git a/tests/test_orbit.py b/tests/test_orbit.py index c0b7c478..3c913525 100755 --- a/tests/test_orbit.py +++ b/tests/test_orbit.py @@ -20,7 +20,7 @@ def _has_ephemeris(): - # SSAPy's ephemeris comes from llnl-ssapy-data; skip only if it is missing. + # SSAPy's ephemeris comes from split SSATK data packages; skip only if missing. try: ssapy.body._planetary_ephemeris_path() return True @@ -1570,7 +1570,7 @@ def test_light_time_correction(): np.testing.assert_allclose(dircos, dc_linear, rtol=0, atol=1e-7) np.testing.assert_allclose(dircos, dc_exact, rtol=0, atol=1e-14) -@pytest.mark.skipif(not _has_ephemeris(), reason="planetary ephemeris unavailable (llnl-ssapy-data not installed)") +@pytest.mark.skipif(not _has_ephemeris(), reason="planetary ephemeris unavailable (split data package not installed)") @pytest.mark.timeout(90) @timer def test_find_passes(): diff --git a/tests/test_tle_drag.py b/tests/test_tle_drag.py index 18e102ca..6f6f396b 100644 --- a/tests/test_tle_drag.py +++ b/tests/test_tle_drag.py @@ -16,9 +16,9 @@ # The numerical drag propagator needs the gravity and ephemeris data from -# llnl-ssapy-data; skip those tests if it is not installed. +# split SSATK data packages; skip those tests if unavailable. def _has_data(name): - # SSAPy's data comes from llnl-ssapy-data; skip only if it is missing. + # SSAPy's data comes from split SSATK data packages; skip only if missing. try: ssapy.utils.find_file(name) return True @@ -39,7 +39,7 @@ def _has_ephemeris(): HAS_MOON_PA = _has_data("moon_pa_de440_200625.bpc") HAS_BODY_DATA = HAS_EGM84 and HAS_EPHEMERIS and HAS_MOON_PA needs_body_data = pytest.mark.skipif( - not HAS_BODY_DATA, reason="body data unavailable (llnl-ssapy-data not installed)") + not HAS_BODY_DATA, reason="body data unavailable (split data package not installed)") ISS = ("1 25544U 98067A 24015.54791435 .00016717 00000-0 30074-3 0 9993", "2 25544 51.6416 247.4627 0006703 130.5360 325.0288 15.49514637123456")