From 05122a24c63252a8efa556fa9b7709ccfecf28af Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Fri, 4 Sep 2026 13:32:55 -0400 Subject: [PATCH] feat: let a service opt out of the HTTPS proxy Some services cannot work behind the per-job subdomain at all. One whose OAuth callback is registered against a fixed host and port is the concrete case: republished at job-. it fails the callback no matter what the proxy does. There was no way to say so, so enabling the proxy broke such an app with no recourse short of unregistering it. An entry point can now set `service_proxy: false`. The flag is snapshotted onto the job row at submit time so editing the manifest cannot change the decision under a running job, and it is enforced in both places rather than only at publish time: the job detail endpoint publishes the direct URL, and the resolve endpoint refuses the hostname. The label is unguessable but derivable by anyone holding the signing key, so leaving resolution open would make the opt-out advisory rather than real. Refusals count as `refused_proxy_disabled` beside the existing reasons, so an operator can see an app opting out rather than inferring it from traffic that never arrives. Validated as service-only, matching auto_url. The check keys on the value being false rather than on the field being set, because model_dump writes the True default onto every entry point and a round-tripped job manifest would otherwise fail to revalidate. Co-Authored-By: Claude Fable 5.1 --- docs/ServiceProxy.md | 6 ++++ .../a9c3e05f1b47_add_job_service_proxy.py | 29 +++++++++++++++++ fileglancer/apps/jobs.py | 1 + fileglancer/database.py | 9 ++++++ fileglancer/model.py | 18 +++++++++++ fileglancer/server.py | 20 +++++++++--- tests/test_apps.py | 26 ++++++++++++++++ tests/test_service_proxy.py | 31 +++++++++++++++++-- 8 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 fileglancer/alembic/versions/a9c3e05f1b47_add_job_service_proxy.py diff --git a/docs/ServiceProxy.md b/docs/ServiceProxy.md index 941dded8..da2e1112 100644 --- a/docs/ServiceProxy.md +++ b/docs/ServiceProxy.md @@ -36,6 +36,12 @@ Leave `service_proxy_domain` empty to disable; the direct `http://:` `session_secret_key` is required when the proxy domain is set, and the server refuses to start without it. An unset key is generated at random per process, so under `uvicorn --workers N` each worker would sign hostnames with a different key and most proxied requests would be refused. Rotating it invalidates live service URLs, on top of the session revocation rotation already causes. +## Apps that opt out + +An individual service can decline to be republished by setting `service_proxy: false` on its entry point. Its direct `http://:` URL is published instead, and the resolve endpoint refuses its hostname, so it cannot be reached through the proxy even by someone who can derive the signed label. Refusals are counted as `refused_proxy_disabled` in the aggregate resolve log line. + +The flag is snapshotted onto the job row at submit time, so editing the manifest does not change the decision for a job that is already running. It exists for services that cannot work behind the proxy at all — one whose OAuth callback is registered against a fixed host and port, say — and not as a performance or preference switch. An app that opts out gives up transport encryption on every hop, including the one from the user's browser, so it should not be sent credentials worth protecting. + ## Reverse proxy configuration Fileglancer does not proxy the traffic itself. It exposes `GET /api/apps/resolve`, which reads the `Host` header and answers `204` with `X-Fg-Upstream: :`, or `403`. The reverse proxy resolves each request through it and connects to the upstream directly, so no proxied bytes pass through the application server. diff --git a/fileglancer/alembic/versions/a9c3e05f1b47_add_job_service_proxy.py b/fileglancer/alembic/versions/a9c3e05f1b47_add_job_service_proxy.py new file mode 100644 index 00000000..d3995057 --- /dev/null +++ b/fileglancer/alembic/versions/a9c3e05f1b47_add_job_service_proxy.py @@ -0,0 +1,29 @@ +"""add service_proxy opt-out to jobs + +Revision ID: a9c3e05f1b47 +Revises: c3e9b7f41a28 +Create Date: 2026-09-04 00:00:00.000000 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'a9c3e05f1b47' +down_revision = 'c3e9b7f41a28' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Every existing job was eligible for the proxy, so a server default + # backfills them rather than leaving a nullable column whose NULL would + # have to mean "true" at every read site. + op.add_column('jobs', sa.Column('service_proxy', sa.Boolean(), + nullable=False, + server_default=sa.true())) + + +def downgrade() -> None: + op.drop_column('jobs', 'service_proxy') diff --git a/fileglancer/apps/jobs.py b/fileglancer/apps/jobs.py index c5603438..b7f385f5 100644 --- a/fileglancer/apps/jobs.py +++ b/fileglancer/apps/jobs.py @@ -859,6 +859,7 @@ async def submit_job( requirements=effective_requirements, commit_sha=executed_sha, code_repo_url=executed_repo_url, + service_proxy=entry_point.service_proxy, ) job_id = db_job.id diff --git a/fileglancer/database.py b/fileglancer/database.py index 9e4f56ff..0496e094 100644 --- a/fileglancer/database.py +++ b/fileglancer/database.py @@ -5,6 +5,7 @@ from functools import lru_cache from sqlalchemy import create_engine, Boolean, Column, String, Integer, DateTime, JSON, UniqueConstraint, func +from sqlalchemy import true as sa_true from sqlalchemy.orm import sessionmaker, declarative_base, Session from sqlalchemy.engine.url import make_url from sqlalchemy.pool import StaticPool @@ -201,6 +202,12 @@ class JobDB(Base): # indexed read, instead of a per-user worker RPC and an NFS stat on every # proxied request. NULL until the job publishes a URL and it is first read. service_url = Column(String, nullable=True) + # Whether this service may be republished at the HTTPS proxy URL. Taken + # from the entry point's service_proxy flag and snapshotted at submit time, + # so editing the manifest cannot change the decision under a job that is + # already running. False publishes the direct http://host:port URL instead + # and makes the resolve endpoint refuse the job's proxy hostname. + service_proxy = Column(Boolean, nullable=False, server_default=sa_true()) # Commit whose code this job executed (the code repo's SHA when the # manifest declares a separate repo_url, else the app repo's SHA). NULL for # jobs submitted before commit pinning existed. @@ -1146,6 +1153,7 @@ def create_job(session: Session, username: str, app_url: str, app_name: str, requirements: Optional[List[str]] = None, commit_sha: Optional[str] = None, code_repo_url: Optional[str] = None, + service_proxy: bool = True, clean_env: bool = False) -> JobDB: """Create a new job record""" now = datetime.now(UTC) @@ -1174,6 +1182,7 @@ def create_job(session: Session, username: str, app_url: str, app_name: str, requirements=requirements, commit_sha=commit_sha, code_repo_url=code_repo_url, + service_proxy=service_proxy, status="PENDING", created_at=now, status_updated_at=now, diff --git a/fileglancer/model.py b/fileglancer/model.py index 414ed656..4e8fc562 100644 --- a/fileglancer/model.py +++ b/fileglancer/model.py @@ -585,6 +585,18 @@ class AppEntryPoint(BaseModel): ), default=None, ) + service_proxy: bool = Field( + description=( + "For service entry points only: whether Fileglancer may republish " + "this service at an HTTPS proxy URL when the server has a service " + "proxy configured. Set it to false only when the service cannot " + "work behind the proxy at all, e.g. it pins an OAuth callback to a " + "fixed host and port. Users then reach it at the direct " + "http://host:port URL, with no transport encryption on any hop, so " + "nothing sensitive should be sent to a service that opts out." + ), + default=True, + ) requirements: List[str] = Field( description="Required tools for this entry point, e.g. ['apptainer']. Merged with manifest-level requirements.", default=[], @@ -720,6 +732,12 @@ def check_conda_container_exclusive(self): raise ValueError("auto_url is only valid for service entry points (type: service)") if self.service_url_suffix is not None and not self.auto_url: raise ValueError("service_url_suffix requires auto_url to be set") + # Checked by value rather than by model_fields_set, like the parameter + # validators above: manifests round-trip through model_dump, which + # writes the True default onto every entry point, so only an explicit + # false is distinguishable — and only false means anything here. + if not self.service_proxy and self.type != "service": + raise ValueError("service_proxy is only valid for service entry points (type: service)") return self diff --git a/fileglancer/server.py b/fileglancer/server.py index 46e5fac9..6d35546e 100644 --- a/fileglancer/server.py +++ b/fileglancer/server.py @@ -2757,10 +2757,13 @@ async def get_job(job_id: int, f"Could not resolve or cache the service URL for job {job_id}", exc_info=True) # The cached value stays raw — it is the proxy's upstream. Only what - # goes back to the browser is rewritten. - proxied = apps_module.build_proxied_service_url( - service_url, job_id, settings.apps.service_proxy_domain, - settings.session_secret_key) + # goes back to the browser is rewritten, and only for an app that + # has not opted out of being republished. + proxied = None + if db_job.service_proxy: + proxied = apps_module.build_proxied_service_url( + service_url, job_id, settings.apps.service_proxy_domain, + settings.session_secret_key) return _convert_job(db_job, service_url=proxied or service_url, files=files, phase=phase) @@ -2776,7 +2779,8 @@ async def resolve_service_upstream(request: Request): location `internal` so it is not reachable from outside. Returns 204 with X-Fg-Upstream on success and 403 for everything else, so - auth_request denies the request. + auth_request denies the request. An app whose manifest sets + service_proxy: false is refused here too, not merely left unpublished. Successful resolutions are cached for a few seconds, which is also the window in which a job that has just stopped can still be proxied. See @@ -2808,6 +2812,12 @@ async def resolve_service_upstream(request: Request): or db_job.status != 'RUNNING'): apps_module.record_resolve("refused_not_running") raise HTTPException(status_code=403, detail="No running service for this host") + # Refused here as well as suppressed at publish time: the label is + # unguessable, but an app that opted out should not be reachable + # through the proxy even by someone who has the hostname. + if not db_job.service_proxy: + apps_module.record_resolve("refused_proxy_disabled") + raise HTTPException(status_code=403, detail="This service is not published through the proxy") upstream = apps_module.upstream_from_service_url( db_job.service_url, allowed_zone=settings.apps.service_proxy_upstream_zone, diff --git a/tests/test_apps.py b/tests/test_apps.py index cbd4e2e2..21fb2116 100644 --- a/tests/test_apps.py +++ b/tests/test_apps.py @@ -953,6 +953,32 @@ def test_auto_url_rejected_on_job(self): type="job", auto_url=True) +class TestServiceProxyOptOut: + """service_proxy lets a service that cannot work behind the HTTPS proxy keep + its direct URL. It defaults on, so opting out is always explicit.""" + + def test_defaults_true(self): + ep = AppEntryPoint(id="t", name="T", command="echo", type="service") + assert ep.service_proxy is True + + def test_can_be_disabled_on_a_service(self): + ep = AppEntryPoint(id="t", name="T", command="echo", + type="service", service_proxy=False) + assert ep.service_proxy is False + + def test_rejected_on_a_job(self): + with pytest.raises(ValidationError, match="service_proxy is only valid for service"): + AppEntryPoint(id="t", name="T", command="echo", + type="job", service_proxy=False) + + def test_default_survives_a_job_entry_point(self): + """The True default is written onto every entry point by model_dump, so + the validator must key on the value being false, not on the field being + set — otherwise a round-tripped job manifest fails to revalidate.""" + ep = AppEntryPoint(id="t", name="T", command="echo", type="job") + assert AppEntryPoint(**ep.model_dump()).service_proxy is True + + class TestServiceUrlSuffix: """service_url_suffix is a restricted template validated for shell-safety.""" diff --git a/tests/test_service_proxy.py b/tests/test_service_proxy.py index b6c270ca..2eb44948 100644 --- a/tests/test_service_proxy.py +++ b/tests/test_service_proxy.py @@ -100,13 +100,15 @@ def _build(proxy_domain="", upstream_zone=""): fileglancer.database._migrations_run = False -def _seed_service_job(db_url, status="RUNNING", entry_point_type="service"): +def _seed_service_job(db_url, status="RUNNING", entry_point_type="service", + service_proxy=True): session = get_db_session(db_url) try: job = create_job( session, OWNER, "https://github.com/owner/repo", "My App", "serve", "Server", {}, entry_point_type=entry_point_type, + service_proxy=service_proxy, ) job.status = status session.commit() @@ -190,8 +192,9 @@ def _resolve(app, host): return TestClient(app).get("/api/apps/resolve", headers={"Host": host}) -def _seed_running_service_with_url(db_url, url="http://node01:41235/lab?token=abc"): - job_id = _seed_service_job(db_url) +def _seed_running_service_with_url(db_url, url="http://node01:41235/lab?token=abc", + service_proxy=True): + job_id = _seed_service_job(db_url, service_proxy=service_proxy) session = get_db_session(db_url) try: set_job_service_url(session, job_id, url) @@ -316,6 +319,28 @@ def test_job_detail_publishes_raw_url_when_proxy_disabled(app_factory, monkeypat assert resp.json()["service_url"] == "http://node01:41235/lab?token=abc" +def test_job_detail_publishes_raw_url_when_the_app_opts_out(app_factory, monkeypatch): + """An app that sets service_proxy: false keeps its direct URL even on a + server where the proxy is configured and every other app is republished.""" + app, db_url = app_factory(PROXY_DOMAIN) + job_id = _seed_service_job(db_url, service_proxy=False) + resp = _get_job_with_worker_url( + app, job_id, "http://node01:41235/lab?token=abc", monkeypatch) + assert resp.status_code == 200 + assert resp.json()["service_url"] == "http://node01:41235/lab?token=abc" + + +def test_resolve_refuses_an_app_that_opted_out(app_factory): + """Suppressing the published URL is not enough on its own: the hostname is + derivable by anyone holding the signing key, and an opted-out app must not + be reachable through the proxy even by someone who has it.""" + app, db_url = app_factory(PROXY_DOMAIN) + job_id = _seed_running_service_with_url(db_url, service_proxy=False) + resp = _resolve(app, _host(job_id)) + assert resp.status_code == 403 + assert apps.resolve_counts() == {"refused_proxy_disabled": 1} + + def test_job_detail_caches_the_raw_url_not_the_proxied_one(app_factory, monkeypatch): """The cached value is the upstream, so it must stay in its raw form.""" app, db_url = app_factory(PROXY_DOMAIN)