diff --git a/.github/workflows/Docs.yml b/.github/workflows/Docs.yml index 5366075..c775a67 100644 --- a/.github/workflows/Docs.yml +++ b/.github/workflows/Docs.yml @@ -4,6 +4,8 @@ on: push: branches: - main + tags: + - 'v*' release: types: [published] workflow_dispatch: @@ -21,12 +23,19 @@ jobs: steps: - uses: actions/checkout@v7 - # Determine deploy target from ref + # Determine deploy target from ref. + # A tag push is the reliable release signal: TagBot pushes tags over an SSH deploy + # key (which does trigger workflows), but creates the GitHub Release with + # GITHUB_TOKEN — and token-authored events never start a workflow run. - name: Set deploy version id: version run: | if [ "${{ github.event_name }}" = "release" ]; then VERSION="${{ github.event.release.tag_name }}" + elif [ "${{ github.ref_type }}" = "tag" ]; then + VERSION="${{ github.ref_name }}" + fi + if [ -n "$VERSION" ]; then echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "is_release=true" >> "$GITHUB_OUTPUT" else @@ -46,6 +55,8 @@ jobs: - uses: quarto-dev/quarto-actions/setup@v2 - name: Set build date run: sed -i "s/__BUILD_DATE__/$(date -u +'%Y-%m-%d')/" docs/_quarto.yml + # Retry: QuartoNotebookRunner precompiles outside the cached depot on first + # run, which can exceed the render timeout; the second attempt succeeds. - name: Render docs run: quarto render docs || quarto render docs @@ -75,6 +86,13 @@ jobs: rm -rf "gh-pages-deploy/${VERSION}" cp -r docs/_site "gh-pages-deploy/${VERSION}" + # Inject version + base path into every rendered page so the version + # selector doesn't have to guess from the URL. + find "gh-pages-deploy/${VERSION}" -name '*.html' -exec sed -i \ + -e "s|__CURRENT_VERSION__|${VERSION}|g" \ + -e "s|__BASE_PATH__|/${REPO_NAME}/|g" \ + {} + + cd gh-pages-deploy # If release tag: update versions.json and stable redirect @@ -111,8 +129,9 @@ jobs: STABLE_EOF fi - # Create stable placeholder if no releases yet - if [ ! -d stable ]; then + # (Re)write stable placeholder while no releases exist. Once versions.json + # is non-empty, the release branch above owns stable/ so we leave it alone. + if [ ! -f versions.json ] || [ "$(tr -d ' \n' < versions.json)" = "[]" ]; then mkdir -p stable cat > stable/index.html << PLACEHOLDER_EOF @@ -120,13 +139,14 @@ jobs: - No Stable Release Yet + ${REPO_NAME} — No Stable Release Yet +
-

No Stable Release Yet

-

This package has not published a release.
Check out the development docs instead.

+

${REPO_NAME}

+

No stable version has been released yet.
Redirecting to the development docs in 5 seconds...

diff --git a/.github/workflows/DocsBackfill.yml b/.github/workflows/DocsBackfill.yml index 919299c..88e4100 100644 --- a/.github/workflows/DocsBackfill.yml +++ b/.github/workflows/DocsBackfill.yml @@ -19,23 +19,19 @@ jobs: with: fetch-depth: 0 - # Save docs infrastructure from main (without coverage — too slow for backfill) + # Save docs infrastructure from main. Regenerating coverage for every tag is + # too slow, so the coverage page is swapped for a placeholder — that keeps the + # sidebar structure intact and lets LocalCoverage drop out of the environment. - name: Save docs infrastructure run: | cp -r docs /tmp/docs-infrastructure - rm -f /tmp/docs-infrastructure/coverage.qmd + cat > /tmp/docs-infrastructure/pages/coverage.qmd << 'COVERAGE_EOF' + --- + title: "Coverage" + --- - # Remove coverage chapter from _quarto.yml - python3 -c " - import re - with open('/tmp/docs-infrastructure/_quarto.yml') as f: - content = f.read() - content = re.sub(r'\n\s*- coverage\.qmd', '', content) - with open('/tmp/docs-infrastructure/_quarto.yml', 'w') as f: - f.write(content) - " - - # Remove LocalCoverage from docs/Project.toml + Coverage is reported for the current release only. + COVERAGE_EOF sed -i '/LocalCoverage/d' /tmp/docs-infrastructure/Project.toml - uses: julia-actions/setup-julia@v3 @@ -112,6 +108,14 @@ jobs: fi cp -r docs/_site "gh-pages-deploy/$TAG" + + # Inject version + base path into every rendered page so the version + # selector doesn't render raw placeholders. + find "gh-pages-deploy/$TAG" -name '*.html' -exec sed -i \ + -e "s|__CURRENT_VERSION__|$TAG|g" \ + -e "s|__BASE_PATH__|/${REPO_NAME}/|g" \ + {} + + BUILT+=("$TAG") echo "Success: $TAG" done diff --git a/.github/workflows/LiveTests.yml b/.github/workflows/LiveTests.yml new file mode 100644 index 0000000..5ab127b --- /dev/null +++ b/.github/workflows/LiveTests.yml @@ -0,0 +1,33 @@ +name: LiveTests +on: + push: + branches: + - main + - master + pull_request: + workflow_dispatch: + +# The public Overpass API asks users not to run multiple scripts in parallel, +# so the live tests run on a single job and never concurrently for the same ref. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + live: + name: Live Overpass API + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + actions: write + contents: read + env: + OVERPASS_LIVE_TESTS: "true" + steps: + - uses: actions/checkout@v7 + - uses: julia-actions/setup-julia@v3 + with: + version: '1' + - uses: julia-actions/cache@v3 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index 7dc0810..dab83d3 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -8,11 +8,6 @@ on: lookback: description: "[DEPRECATED] No longer has any effect" default: "3" -permissions: - actions: read - contents: write - issues: read - pull-requests: read jobs: TagBot: if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' diff --git a/CLAUDE.md b/CLAUDE.md index 8cd05eb..7bab23f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,7 @@ # Development - Run tests: `julia --project -e 'using Pkg; Pkg.test()'` +- Live tests against the public Overpass API (`test/test_service.jl`) are opt-in: `OVERPASS_LIVE_TESTS=true`. They run in CI only in the LiveTests workflow, on a single job. - Build docs: `quarto render docs` - `docs/` has its own Project.toml for doc-specific dependencies. - Each .qmd file in the docs should have `engine: julia` in the YAML frontmatter @@ -11,7 +12,6 @@ - `api.qmd` must always be the last item before the "Reference" section in `_quarto.yml` - `api.qmd` lives in its own `part: "API"` to visually separate it from other doc pages -- The API page uses open cards (not collapsible blocks) to display exports # Style diff --git a/docs/.gitignore b/docs/.gitignore index ad29309..83e8ce3 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,2 +1,5 @@ /.quarto/ +/_site/ **/*.quarto_ipynb +/pages/coverage/ +/pages/lcov.info diff --git a/docs/_quarto.yml b/docs/_quarto.yml index 9d11a48..f332cc7 100644 --- a/docs/_quarto.yml +++ b/docs/_quarto.yml @@ -2,7 +2,7 @@ project: type: book output-dir: _site -engine: julia +engines: ['julia'] julia: project: "." @@ -13,6 +13,8 @@ book: keyboard-shortcut: - "/" navbar: + logo: assets/logo.svg + logo-href: https://rallypoint1.com right: - icon: star href: https://github.com/RallypointOne/OverpassAPI.jl/stargazers @@ -27,11 +29,11 @@ book: - overpass-ql.qmd - part: "API" chapters: - - api.qmd + - pages/api.qmd - part: "Reference" chapters: - changelog.qmd - - coverage.qmd + - pages/coverage.qmd page-footer: center: "Built on __BUILD_DATE__" @@ -41,11 +43,10 @@ language: format: html: - css: styles.css + css: assets/styles.css number-sections: false theme: - light: flatly - dark: darkly - respect-user-color-scheme: true + - flatly + - assets/theme.scss include-after-body: - - file: _version-selector.html + - file: assets/_version-selector.html diff --git a/docs/api.qmd b/docs/api.qmd deleted file mode 100644 index c09b31c..0000000 --- a/docs/api.qmd +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: "API" -engine: julia -toc: true -toc-depth: 4 ---- - -```{julia} -#| echo: false -#| output: false -using OverpassAPI -``` - -```{julia} -#| echo: false -#| output: asis -export_names = filter(!=(:OverpassAPI), names(OverpassAPI)) - -classify(b) = b isa Type ? :type : b isa Function ? :function : :constant - -for (kind, label) in [(:type, "Types"), (:function, "Functions"), (:constant, "Constants")] - items = filter(n -> classify(getfield(OverpassAPI, n)) == kind, export_names) - isempty(items) && continue - println("## $label\n") - for name in items - binding = getfield(OverpassAPI, name) - doc = string(Docs.doc(binding)) - println(""" - ::: {.card .mb-3} - ::: {.card-header .api-item-header} - #### `$name` {#$(lowercase(string(name)))} - ::: - ::: {.card-body} - $doc - ::: - ::: - """) - end -end -``` diff --git a/docs/_version-selector.html b/docs/assets/_version-selector.html similarity index 50% rename from docs/_version-selector.html rename to docs/assets/_version-selector.html index 2d3a91e..aa926ef 100644 --- a/docs/_version-selector.html +++ b/docs/assets/_version-selector.html @@ -1,21 +1,14 @@ diff --git a/docs/pages/api.qmd b/docs/pages/api.qmd new file mode 100644 index 0000000..7f3f7ba --- /dev/null +++ b/docs/pages/api.qmd @@ -0,0 +1,78 @@ +--- +title: "API" +toc: true +toc-depth: 4 +--- + +```{julia} +#| echo: false +#| output: false +using OverpassAPI +``` + +```{julia} +#| echo: false +#| output: asis +export_names = filter(!=(:OverpassAPI), names(OverpassAPI)) + +classify(name, b) = startswith(string(name), "@") ? :macro : + b isa Type ? :type : b isa Function ? :function : :constant + +# `@` is a legal HTML id character but breaks the querySelector calls Quarto's nav makes. +anchor(name) = lowercase(replace(string(name), "@" => "macro-")) + +if isempty(export_names) + println("This package does not export any names yet.\n") +else + println(""" +
+ + +
+ """) +end + +for (kind, label) in [(:type, "Types"), (:function, "Functions"), (:macro, "Macros"), (:constant, "Constants")] + items = filter(n -> classify(n, getfield(OverpassAPI, n)) == kind, export_names) + isempty(items) && continue + println("## $label\n") + for name in items + binding = getfield(OverpassAPI, name) + doc = string(Docs.doc(binding)) + println(""" +
+

`$name`

 + + ::: {.card-body} + $doc + ::: + +
+ """) + end +end +``` + + diff --git a/docs/pages/coverage.qmd b/docs/pages/coverage.qmd new file mode 100644 index 0000000..dd1937b --- /dev/null +++ b/docs/pages/coverage.qmd @@ -0,0 +1,42 @@ +--- +title: "Coverage" +resources: + - coverage/ +--- + +```{julia} +#| echo: false +#| output: false +using LocalCoverage +coverage = generate_coverage("OverpassAPI"; run_test=true) +covdir = joinpath(@__DIR__, "coverage") +if coverage.lines_tracked == 0 + # No coverable lines yet (e.g. a freshly generated package). `genhtml` (lcov 2.x) + # treats an empty tracefile as a fatal error, so emit a placeholder instead. + mkpath(covdir) + write(joinpath(covdir, "index.html"), + "" * + "

No coverage data yet — add code and tests to generate a report.

" * + "") +else + html_coverage(coverage; dir=covdir, open=false) +end +cov_dir = joinpath(dirname(dirname(@__DIR__)), "coverage") +mv(joinpath(cov_dir, "lcov.info"), joinpath(@__DIR__, "lcov.info"); force=true) +rm(cov_dir; recursive=true, force=true) +``` + + + + diff --git a/docs/styles.css b/docs/styles.css deleted file mode 100644 index 15cf6b9..0000000 --- a/docs/styles.css +++ /dev/null @@ -1,79 +0,0 @@ -/* Sidebar section spacing */ -.sidebar-item-section { - margin-top: 2rem !important; -} - -.sidebar-item-section > .sidebar-item-container > .sidebar-item-text { - font-weight: bold; -} - -/* Fix heading inline code in dark mode */ -h1 code, h2 code, h3 code, h4 code { - background: none; - color: inherit; -} - -/* Larger search results panel */ -.aa-Panel, -#quarto-search-results .aa-Panel { - max-width: 600px; - font-size: 1rem; -} - -/* Fix search bar in dark mode */ -.aa-Autocomplete .aa-Form, -.aa-DetachedFormContainer .aa-Form { - background-color: var(--bs-body-bg); - color: var(--bs-body-color); - border-color: var(--bs-border-color); -} - -.aa-Input { - color: var(--bs-body-color) !important; -} - -.aa-Input::placeholder { - color: var(--bs-secondary-color) !important; -} - -.aa-SubmitButton svg, -.aa-Form .aa-Label svg, -.aa-Form .aa-LoadingIndicator svg { - color: var(--bs-body-color) !important; -} - -/* Sun/moon icons for light/dark toggle */ -.quarto-color-scheme-toggle .bi::before { - background-color: var(--bs-body-color) !important; - background-image: none !important; - -webkit-mask-size: 1rem 1rem; - mask-size: 1rem 1rem; - -webkit-mask-repeat: no-repeat; - mask-repeat: no-repeat; -} - -.quarto-color-scheme-toggle:not(.alternate) .bi::before { - -webkit-mask-image: url('data:image/svg+xml,'); - mask-image: url('data:image/svg+xml,'); -} - -/* API item card headers */ -.api-item-header { - background-color: var(--bs-primary) !important; -} - -.api-item-header h4 { - margin: 0 !important; - color: white !important; - font-size: 1rem; -} - -.api-item-header h4 code { - color: white !important; - background: none !important; -} - -.quarto-color-scheme-toggle.alternate .bi::before { - -webkit-mask-image: url('data:image/svg+xml,'); - mask-image: url('data:image/svg+xml,'); -} diff --git a/src/OverpassAPI.jl b/src/OverpassAPI.jl index 5dcbdc5..628dce2 100644 --- a/src/OverpassAPI.jl +++ b/src/OverpassAPI.jl @@ -20,6 +20,8 @@ The default Overpass API endpoint: `"https://overpass-api.de/api/interpreter"`. """ const DEFAULT_ENDPOINT = "https://overpass-api.de/api/interpreter" +const USER_AGENT = "OverpassAPI.jl (+https://github.com/RallypointOne/OverpassAPI.jl)" + #--------------------------------------------------------------------------------# Types """ @@ -409,13 +411,24 @@ julia> r = query("node[amenity=cafe](35.9,-79.1,36.1,-78.8); out geom;") ``` """ function query(ql::String; bbox::Union{Extent, Nothing}=nothing, endpoint::String=DEFAULT_ENDPOINT) - q = contains(ql, "[out:json]") ? ql : "[out:json];" * ql - if !isnothing(bbox) - x = bbox.X - y = bbox.Y - q = "[bbox:$(y[1]),$(x[1]),$(y[2]),$(x[2])];" * q + # Overpass allows one settings statement, so `[out:json]` and the bbox are + # merged into the query's own leading `[...]` block when it has one. + bbox_setting = isnothing(bbox) ? "" : + "[bbox:$(bbox.Y[1]),$(bbox.X[1]),$(bbox.Y[2]),$(bbox.X[2])]" + body = lstrip(ql) + if startswith(body, "[") + k = findfirst(';', body) + isnothing(k) && error("settings statement without a terminating ';' in query: $ql") + head, rest = body[1:prevind(body, k)], body[k:end] + contains(head, "[out:") || (head = "[out:json]" * head) + q = head * bbox_setting * rest + else + q = "[out:json]" * bbox_setting * ";" * body end - resp = HTTP.post(endpoint, [], HTTP.Form(Dict("data" => q))) + # Overpass expects a URL-encoded form body; it rejects multipart/form-data with HTTP 400. + # The public instances ask clients to identify themselves via User-Agent. + headers = ["Content-Type" => "application/x-www-form-urlencoded", "User-Agent" => USER_AGENT] + resp = HTTP.post(endpoint, headers, "data=" * HTTP.escapeuri(q); status_exception = false) if resp.status != 200 error("Overpass API error (HTTP $(resp.status)): $(String(resp.body))") end @@ -429,7 +442,7 @@ end Execute a [`QLStatement`](@ref) built with [`OQL`](@ref). The `out` keyword controls the output verbosity/geometry (default `:geom`). -Accepts a `Symbol` (e.g. `:geom`, `:body`, `:center`, `:count`) or a `String` +Accepts a `Symbol` (e.g. `:geom`, `:body`, `:center`, `:skel`) or a `String` for full control (e.g. `"body qt 100"`). ### Examples diff --git a/test/runtests.jl b/test/runtests.jl index 83a6b5e..6a60c86 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -352,4 +352,6 @@ const MIXED_JSON = JSON3.read(""" @test contains(repr(Relation(id=1)), "Relation(1") @test contains(repr(parse_response(MIXED_JSON)), "OverpassResponse") end + + get(ENV, "OVERPASS_LIVE_TESTS", "false") == "true" && include("test_service.jl") end diff --git a/test/test_service.jl b/test/test_service.jl new file mode 100644 index 0000000..61a7eb7 --- /dev/null +++ b/test/test_service.jl @@ -0,0 +1,85 @@ +# Live queries against the public Overpass API. Opt in with OVERPASS_LIVE_TESTS=true. +# OVERPASS_ENDPOINT selects a mirror, e.g. https://overpass.kumi.systems/api/interpreter. +using OverpassAPI +using Extents: Extents, Extent +using GeoInterface +using Test + +const LIVE_ENDPOINT = get(ENV, "OVERPASS_ENDPOINT", DEFAULT_ENDPOINT) + +# Downtown Durham, NC. Small enough that every query below returns in a few seconds. +const LIVE_EXT = Extent(X=(-78.905, -78.895), Y=(35.993, 36.001)) + +# The public endpoint rate-limits (HTTP 429) and sheds load (HTTP 504). Its usage policy asks +# for a 30 s pause before retrying; any other error propagates unchanged. +function live_query(args...; kw...) + for delay in (30, 60) + try + return query(args...; kw..., endpoint=LIVE_ENDPOINT) + catch e + e isa ErrorException && occursin(r"HTTP (429|504)", e.msg) || rethrow() + @warn "Overpass API busy, retrying in $(delay)s" + sleep(delay) + end + end + query(args...; kw..., endpoint=LIVE_ENDPOINT) +end + +@testset "Live Overpass service" begin + cafes = live_query("node[amenity=cafe]$(bbox_string(LIVE_EXT)); out;") + cafe_ids = Set(n.id for n in nodes(cafes)) + + @testset "raw QL with inline bbox" begin + @test cafes.version == 0.6 + @test startswith(cafes.generator, "Overpass API") + @test !isempty(cafes.timestamp) + @test !isempty(cafe_ids) + @test all(n -> n["amenity"] == "cafe", nodes(cafes)) + @test all(n -> Extents.coveredby(Extents.extent(n), LIVE_EXT), nodes(cafes)) + end + + @testset "bbox keyword prepends [out:json][bbox:...]" begin + r = live_query("node[amenity=cafe]; out;", bbox=LIVE_EXT) + @test Set(n.id for n in nodes(r)) == cafe_ids + end + + @testset "bbox keyword merges into an existing settings statement" begin + r = live_query("[timeout:25]; node[amenity=cafe]; out;", bbox=LIVE_EXT) + @test Set(n.id for n in nodes(r)) == cafe_ids + end + + @testset "QLStatement query" begin + r = live_query(OQL.node[amenity="cafe"], bbox=LIVE_EXT) + @test Set(n.id for n in nodes(r)) == cafe_ids + end + + @testset "ways with out geom" begin + r = live_query(OQL.way["building"], bbox=LIVE_EXT) + ws = ways(r) + @test !isempty(ws) + @test all(w -> haskey(w, "building"), ws) + @test all(w -> length(w.geometry) == length(w.node_ids), ws) + @test all(w -> GeoInterface.geomtrait(w) == GeoInterface.LineStringTrait(), ws) + @test all(w -> Extents.intersects(Extents.extent(w), LIVE_EXT), ws) + end + + @testset "relations with out geom" begin + r = live_query(OQL.rel[type="multipolygon"], bbox=LIVE_EXT) + rs = relations(r) + @test !isempty(rs) + @test all(rel -> rel["type"] == "multipolygon", rs) + @test all(rel -> !isempty(rel.members), rs) + @test all(rel -> all(m -> m.type in ("node", "way", "relation"), rel.members), rs) + @test any(rel -> any(m -> !isempty(m.geometry), rel.members), rs) + end + + @testset "invalid query raises HTTP 400" begin + err = try + live_query("this is not valid;") + catch e + e + end + @test err isa ErrorException + @test occursin("HTTP 400", sprint(showerror, err)) + end +end