diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..13ef260fc --- /dev/null +++ b/.coveragerc @@ -0,0 +1,12 @@ +[run] +source = + c2corg_api +branch = True + +[report] +omit = + c2corg_api/tests/* + c2corg_api/scripts/migration/* + */venv/* + */.venv/* + */site-packages/* \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..ae757eeec --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.11-slim-trixie + +# Avoid prompts from apt +ENV DEBIAN_FRONTEND=noninteractive + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + curl \ + wget \ + make \ + gcc \ + g++ \ + libpq-dev \ + libgeos-dev \ + libproj-dev \ + postgresql-client \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /workspace + +# Upgrade pip +RUN pip install --upgrade pip setuptools wheel diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 000000000..f44738442 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,75 @@ +# Dev Container for c2corg v6_api + +This directory contains a [Dev Container](https://containers.dev/) configuration +that provides a fully configured development environment with all services +(PostgreSQL/PostGIS, Elasticsearch, Redis) running as companion containers. + +## Prerequisites + +- [Docker Desktop](https://www.docker.com/products/docker-desktop/) (or a compatible Docker engine) +- [VS Code](https://code.visualstudio.com/) with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + +## Getting started + +1. Open this repository in VS Code +2. When prompted, click **"Reopen in Container"** — or use the command palette: + `Dev Containers: Reopen in Container` +3. Wait for the container to build and the `postCreateCommand` setup script to + finish (first run takes a few minutes) +4. You're ready to go! + +## What's included + +| Service | Hostname | Port | +|----------------|-----------------|------| +| PostgreSQL | `postgresql` | 5432 | +| Elasticsearch | `elasticsearch` | 9200 | +| Redis | `redis` | 6379 | +| API (when run) | `localhost` | 6543 | + +All ports are forwarded to your host machine for convenience. + +## Common commands + +```bash +# Run the test suite +pytest + +# Run a specific test file +pytest c2corg_api/tests/models/test_book.py + +# Run the linter +ruff check c2corg_api es_migration + +# Start the API server (with auto-reload) +pserve development.ini --reload + +# Or use make targets +make test +make serve +make lint +``` + +## Rebuilding + +If dependencies change, rebuild the container: + +- Command palette → `Dev Containers: Rebuild Container` + +## Troubleshooting + +- **Database errors**: The setup script creates both databases automatically. If you need to re-run: + + ```bash + PGPASSWORD=test PGUSER=postgres PGHOST=postgresql \ + bash .devcontainer/setup.sh + ``` + +- **Elasticsearch not ready**: The setup script waits for ES, but if tests + fail with connection errors, check: `curl http://elasticsearch:9200` + +- **Config out of date**: Re-generate config files: + + ```bash + bash .devcontainer/setup.sh + ``` diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..2a46429b3 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,62 @@ +{ + "name": "c2corg v6_api", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/workspace", + + // VS Code settings + "customizations": { + "vscode": { + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + + "python.testing.pytestEnabled": true, + "python.testing.pytestArgs": ["c2corg_api/tests"], + + "python.languageServer": "Pylance", + "python.analysis.typeCheckingMode": "basic", + "python.analysis.diagnosticMode": "openFilesOnly", + + // ---- Formatting ---- + "editor.formatOnSave": true, + "editor.defaultFormatter": "charliermarsh.ruff", + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true + }, + // ---- Disable legacy Python linters ---- + "python.linting.enabled": false, + "python.linting.flake8Enabled": false, + "python.linting.pycodestyleEnabled": false, + "python.linting.pylintEnabled": false, + // ---- Ruff ---- + "ruff.enable": true, + // ---- Fixes & imports on save ---- + "editor.codeActionsOnSave": { + "source.fixAll.ruff": true, + "source.organizeImports.ruff": true + } + }, + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "charliermarsh.ruff" + ] + } + }, + + // Forward ports for local access + "forwardPorts": [6543, 9200, 6379, 5432], + "portsAttributes": { + "6543": { "label": "API Server" }, + "9200": { "label": "Elasticsearch" }, + "6379": { "label": "Redis" }, + "5432": { "label": "PostgreSQL" } + }, + + // Run after the container is created (one-time setup) + "postCreateCommand": "bash .devcontainer/setup.sh", + + // Run every time the container starts + "postStartCommand": "echo 'Dev container ready. Run: make test'" +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..41b3d6f4b --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,55 @@ +services: + app: + build: + context: . + dockerfile: Dockerfile + volumes: + - ..:/workspace:cached + command: sleep infinity + depends_on: + postgresql: + condition: service_healthy + elasticsearch: + condition: service_started + redis: + condition: service_healthy + environment: + # Point services to their container hostnames + PGHOST: postgresql + PGUSER: postgres + PGPASSWORD: test + + postgresql: + image: postgis/postgis:16-3.4 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: test + volumes: + - postgres_data:/var/lib/postgresql/data + - ..:/v6_api + - ../docker/conf/postgresql.conf:/etc/postgresql.conf + command: ["postgres", "-c", "config_file=/etc/postgresql.conf"] + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 10s + timeout: 5s + retries: 5 + + elasticsearch: + image: "docker.io/c2corg/c2corg_es:anon-2018-11-02" + command: -Des.index.number_of_replicas=0 -Des.path.data=/c2corg_anon -Des.script.inline=true + ulimits: + nofile: + soft: 65536 + hard: 65536 + + redis: + image: redis:7.2 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + postgres_data: diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100644 index 000000000..d26fccef0 --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,94 @@ +#!/bin/bash +set -e + +echo "==> Installing Python dependencies..." +pip install -e ".[dev]" + +echo "==> Copying env.local.sample to env.local (if not present)..." +if [ ! -f config/env.local ]; then + cp config/env.local.sample config/env.local +fi + +# Override hosts to point to docker-compose service names +# (inside the devcontainer network, services are reachable by name) +cat > config/env.devcontainer <<'EOF' +#!/bin/sh +version=0.0.0dev0 +db_host=postgresql +tests_db_host=postgresql +elasticsearch_host=elasticsearch +redis_url=redis://redis:6379/ +EOF + +echo "==> Generating config files from templates..." +./scripts/env_replace config/env.default config/env.dev config/env.devcontainer < alembic.ini.in > alembic.ini +./scripts/env_replace config/env.default config/env.dev config/env.devcontainer < common.ini.in > common.ini +./scripts/env_replace config/env.default config/env.dev config/env.devcontainer < development.ini.in > development.ini +./scripts/env_replace config/env.default config/env.dev config/env.devcontainer < test.ini.in > test.ini + +echo "==> Waiting for PostgreSQL to be ready..." +until pg_isready -h postgresql -U postgres; do + echo " PostgreSQL not ready yet, retrying in 2s..." + sleep 2 +done + +echo "==> Initializing dev database..." +export PGPASSWORD=test +export PGUSER=postgres +export PGHOST=postgresql +export POSTGRES_USER=postgres + +# Create dev database (adapted from scripts/database/create_schema.sh) +DBNAME="c2corg" +if psql -tAc "SELECT 1 FROM pg_database WHERE datname='${DBNAME}'" | grep -q 1; then + echo " Dev database already exists: ${DBNAME}" +else + echo " Creating dev database: ${DBNAME}" + psql -v ON_ERROR_STOP=1 --username "$PGUSER" < Initializing test database..." +DBNAME="c2corg_tests" +if psql -tAc "SELECT 1 FROM pg_database WHERE datname='${DBNAME}'" | grep -q 1; then + echo " Test database already exists: ${DBNAME}" +else + echo " Creating test database: ${DBNAME}" + psql -v ON_ERROR_STOP=1 --username "$PGUSER" < Waiting for Elasticsearch to be ready..." +until curl -s http://elasticsearch:9200 > /dev/null 2>&1; do + echo " Elasticsearch not ready yet, retrying in 5s..." + sleep 5 +done + +echo "==> Initializing Elasticsearch indexes..." +fill_es_index development.ini || true + +echo "" +echo "=============================================" +echo " Dev container setup complete!" +echo "" +echo " Run the API: pserve development.ini --reload" +echo " Run tests: pytest" +echo " Run linter: ruff check c2corg_api es_migration" +echo " Or use make: make test / make serve / make lint" +echo "=============================================" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19de6750c..d3a328798 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,16 +41,16 @@ jobs: - 5432:5432 steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: - python-version: 3.9 + python-version: "3.11" - name: Cache pip - uses: actions/cache@v4.3.0 + uses: actions/cache@v5 with: path: ~/.cache/pip # This path is specific to Ubuntu - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ hashFiles('dev-requirements.txt') }}-${{ hashFiles('requirements.txt') }} + # Look to see if there is a cache hit for the corresponding pyproject.toml file + key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- @@ -59,7 +59,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install .[dev] - name: Lint - run: flake8 c2corg_api es_migration + run: ruff check c2corg_api es_migration - name: Configure postgres run: | scripts/database/create_test_schema.sh @@ -71,7 +71,7 @@ jobs: - name: Create config from templates run: make CONFIG=config/env.test load-env - name: Install java 8 for (old) elasticsearch - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: "adopt" java-version: 8 @@ -100,18 +100,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker meta id: docker_meta - uses: docker/metadata-action@v5.7.0 + uses: docker/metadata-action@v6 with: images: ghcr.io/c2corg/v6_api - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v6 - run: git archive --format=tar --output project.tar "$GITHUB_SHA" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 53a8071d2..eb05af74b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -14,11 +14,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: python @@ -34,4 +34,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index cfc5cded6..719cc2762 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -12,7 +12,7 @@ jobs: contents: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Create or update release diff --git a/HOWTO-DEV.md b/HOWTO-DEV.md index 579e71567..277e608eb 100644 --- a/HOWTO-DEV.md +++ b/HOWTO-DEV.md @@ -1,41 +1,123 @@ # [camptocamp.org](https://www.camptocamp.org) API developer manual -## Requirements +There are two ways to set up your development environment: + +1. **[Dev Container](#option-1-dev-container-recommended)** — the recommended approach; everything runs inside a pre-configured container, no local Python install required. +2. **[Manual setup](#option-2-manual-setup)** — set up Python, services, and configuration by hand on your host machine. + +> **Note:** For details on the "Mobilité douce" (public transport) features, see [PUBLIC-TRANSPORTATION.md](PUBLIC-TRANSPORTATION.md). + +--- + +## Option 1: Dev Container (recommended) + +A [Dev Container](https://containers.dev/) configuration is provided in the `.devcontainer/` directory. It gives you a fully configured development environment with all companion services (PostgreSQL/PostGIS, Elasticsearch, Redis) running as side containers. + +### Prerequisites + +- [Docker Desktop](https://www.docker.com/products/docker-desktop/) (or a compatible Docker engine with Docker Compose) +- [VS Code](https://code.visualstudio.com/) with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + +### Getting started + +1. Clone the repository and open it in VS Code: + + ```sh + git clone https://github.com/c2corg/v6_api + cd v6_api + code . + ``` + +2. When prompted, click **"Reopen in Container"** — or open the command palette (`Ctrl+Shift+P` / `Cmd+Shift+P`) and select: + + ``` + Dev Containers: Reopen in Container + ``` + +3. Wait for the container to build and the `postCreateCommand` setup script to finish. On the first run this takes a few minutes — it will: + - Install all Python dependencies + - Generate the configuration files + - Create and initialize both the dev and test databases + - Populate the Elasticsearch indexes + +4. You're ready to go! See [Running the API](#running-the-api), [Running tests](#running-tests), and [Linting](#linting) below. + +### Services available in the dev container + +| Service | Hostname | Port | +|----------------|-----------------|------| +| PostgreSQL | `postgresql` | 5432 | +| Elasticsearch | `elasticsearch` | 9200 | +| Redis | `redis` | 6379 | +| API (when run) | `localhost` | 6543 | + +All ports are forwarded to your host machine automatically. + +### Rebuilding the dev container + +If dependencies or the Dockerfile change, rebuild via the command palette: + +``` +Dev Containers: Rebuild Container +``` + +Or to re-run the setup script without rebuilding: + +```bash +bash .devcontainer/setup.sh +``` + +For more details, see [`.devcontainer/README.md`](.devcontainer/README.md). + +--- + +## Option 2: Manual setup + +### Requirements - A computer running Linux or Windows (on Windows, take a look at [WSL](https://learn.microsoft.com/fr-fr/windows/wsl/install), it will make your developer life easier). Maybe MacOs can work, but it was not tested. -- Python 3.9.x installed with pip and the virtualenv package (currently, it will not work with python 3.10+) - - On Ubuntu, you can install it from the deadsnakes repository if your embedded python version is too recent (https://askubuntu.com/questions/1318846/how-do-i-install-python-3-9) - - On Windows, use Microsoft Store to install it (https://apps.microsoft.com/detail/9p7qfqmjrfp7) +- Python 3.11.x installed with pip and the virtualenv package + - On Ubuntu, you can install it from the deadsnakes repository if your embedded python version is too recent () + - On Windows, use Microsoft Store to install it () - A docker environment with docker compose plugin (either [docker desktop](https://www.docker.com/products/docker-desktop/) or a manual installation) - A development IDE is not mandatory, but highly recommended (pycharm, vscode...) - If you have GNU make installed, you can use it to run all the commands below to initialize and run your development environment (run `make help` to see what can be done) instead of running them "by hand" -## Installing the development environment +### Installing the development environment - Clone the git repository - Cd inside your cloned repo - Create a virtual environment in it: + ```shell -python3.9 -m venv venv +python3.11 -m venv venv ``` + - Activate the virtual environment (⚠️ Do not forget to activate it in all terminal instances you intend to use) + ```shell # Linux (or WSL) source venv/bin/activate # Or Windows Powershell .\venv\Scripts\Activate.ps1 ``` + - Install the API and its python dependencies in the virtual environment with the following command: + ```shell pip install -e ".[dev]" ``` + - Start the necessary tools (postgres database, redis, and elasticsearch) in docker containers: + ```shell docker compose up -d ``` + - Go to the config directory and copy the `env.local.sample` to `env.local` - You should not need to modify it as it is already filled with the necessary values to be compatible with the docker compose defaults but if necessary you can - Update the `*.ini` file: + ```shell # Linux or WSL ./scripts/env_replace config/env.default config/env.dev config/env.local < alembic.ini.in > alembic.ini @@ -46,44 +128,60 @@ Get-Content alembic.ini.in | cmd /c "python .\scripts\env_replace config/env.def Get-Content common.ini.in | cmd /c "python .\scripts\env_replace config/env.default config/env.dev config/env.local > common.ini" Get-Content development.ini.in | cmd /c "python .\scripts\env_replace config/env.default config/env.dev config/env.local > development.ini" ``` + - Initialize database: + ```shell docker compose exec -u postgres -T postgresql /v6_api/scripts/database/create_schema.sh initialize_c2corg_api_db development.ini ``` + - Initialize test database: + ```shell docker compose exec -u postgres -T postgresql /v6_api/scripts/database/create_test_schema.sh ``` + - Initialize elastic search indexes + ```shell fill_es_index development.ini ``` + ## Running the API - Make sure the postgres redis and elasticsearch docker containers are running + ```shell docker ps ``` + - If you do not see them running, restart them (without the initialization steps that should have been done just once when setting up the environment): + ```shell docker compose up -d ``` + - Run the API (this command will not exit so you should run it in a separate terminal): + ```shell pserve development.ini --reload ``` + - Run background jobs (run each of these commands in a separate terminal, as they will not exit): + ```shell python c2corg_api/scripts/es/syncer.py development.ini # Does not work in Windows, as it uses signal.pause() which is only available in unix based OSes (Did I already tell you that you should really look at WSL ?) python c2corg_api/scripts/jobs/scheduler.py development.ini ``` -- To check that everything is OK, go to http://localhost:6543/health in your browser + +- To check that everything is OK, go to in your browser ## Running tests - Update the test.ini file: + ```shell # Linux or WSL ./scripts/env_replace config/env.default config/env.dev config/env.local < test.ini.in > test.ini @@ -91,22 +189,27 @@ python c2corg_api/scripts/jobs/scheduler.py development.ini Get-Content test.ini.in | cmd /c "python .\scripts\env_replace config/env.default config/env.dev config/env.local > test.ini" ``` + - To run tests , you can either run them directly with pytest (Again, it looks like some tests will not pass in Windows): + ```shell pytest ``` + - Or directly in your favorite development IDE (pycharm, vscode...) ## Linting - Run the linter: + ```shell -flake8 c2corg_api es_migration +ruff check c2corg_api es_migration ``` ## Other - Flush redis cache: + ```shell -python c2corg_api/scripts/redis-flushdb.py development.ini +python c2corg_api/scripts/redis_flushdb.py development.ini ``` diff --git a/Makefile b/Makefile index cc8c255d0..e958829dc 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ help: @echo "- run-background-jobs" Run the background jobs @echo @echo "- test Run the unit tests" - @echo "- lint Run flake8 checker on the Python code" + @echo "- lint Run ruff checker on the Python code" @echo @echo "Secondary targets:" @echo @@ -62,7 +62,7 @@ serve: pserve development.ini --reload lint: - flake8 $(SRC_DIRS) + ruff check $(SRC_DIRS) @echo "Wonderful, python style is Ok!" test: @@ -88,7 +88,7 @@ run-background-jobs: python c2corg_api/scripts/jobs/scheduler.py development.ini flush-redis: - python c2corg_api/scripts/redis-flushdb.py development.ini + python c2corg_api/scripts/redis_flushdb.py development.ini load-env: $(TEMPLATE_FILES) diff --git a/Makefile.prod b/Makefile.prod index 74c8b980a..a59e3763e 100644 --- a/Makefile.prod +++ b/Makefile.prod @@ -10,7 +10,6 @@ export site_packages = $(SITE_PACKAGES) PYTHON = python PYTEST = pytest -FLAKE = flake8 .PHONY: help help: @@ -35,7 +34,7 @@ run-background-jobs: install production.ini $(PYTHON) c2corg_api/scripts/jobs/scheduler.py production.ini flush-redis: install production.ini - $(PYTHON) c2corg_api/scripts/redis-flushdb.py production.ini + $(PYTHON) c2corg_api/scripts/redis_flushdb.py production.ini load-env: $(TEMPLATE_FILES) diff --git a/SMART-ORIGIN-README.md b/PUBLIC-TRANSPORTATION.md similarity index 91% rename from SMART-ORIGIN-README.md rename to PUBLIC-TRANSPORTATION.md index 38c6c7242..6cc718f07 100644 --- a/SMART-ORIGIN-README.md +++ b/PUBLIC-TRANSPORTATION.md @@ -22,7 +22,7 @@ This section is used to search for public transport stops around 'access' waypoi The public transport data comes from an external API called Navitia, which is stored in the CamptoCamp database. **IF YOU DON'T HAVE ANY RESULT ON THE "Public Transports Access" section** : this means that your imported database does not contain Navitia data. -To have visible results, you must launch a `get_public_transports_*.sh` in the backend (see backend documentation). +To have visible results, you must launch a `scripts/public-transportation/get_public_transports_*.sh` in the backend (see backend documentation). There are different versions of these scripts: - whose name is ending with "_France.sh", "_Rhone.sh", or "_Isere.sh": Navitia fetching scripts for France, Rhone, and Isere regions (resp.) that should be launched *outside containers* (from the host); - whose name is ending with "_France.bm.sh", "_Rhone.bm.sh", or "_Isere.bm.sh": Same but those should be launched *within containers*; @@ -35,13 +35,13 @@ The result will always be fetched for the whole France area, and preferably usin Warning ⚠️ : it takes a while (~3h, see other option below) ``` (on api_v6/ ) -sh get_public_transports_from_France_distinct.bm.sh +sh scripts/public-transportation/get_public_transports_from_France_distinct.bm.sh ``` If you just want to work with the Isère department for local dev, you can run this script instead (~18 minutes): ``` (on api_v6/ ) -sh get_public_transports_from_Isere.sh +sh scripts/public-transportation/get_public_transports_from_Isere.sh ``` ### Files created / used for this section : @@ -87,9 +87,9 @@ BACK-END : This section is used to plan a trip by calling the Navitia API. Unlike the previous section, we don't store the results in the database; we query Navitia directly by launching a query from the backend. -This section uses the calculated_duration attribute, **which is calculated with the `calcul_duration_for_routes.bm.sh` script in the backend (see backend documentation)** +This section uses the calculated_duration attribute, **which is calculated with the `scripts/public-transportation/calcul_duration_for_routes.bm.sh` script in the backend (see backend documentation)** -Note that the `calcul_duration_for_routes.bm.sh` is intended to run from within the container. A variant named `calcul_duration_for_routes.sh` can be used to launch the script from the host. +Note that the `scripts/public-transportation/calcul_duration_for_routes.bm.sh` is intended to run from within the container. A variant named `scripts/public-transportation/calcul_duration_for_routes.sh` can be used to launch the script from the host. If you need to update the calculated duration of itineraries, you can run this : @@ -103,7 +103,7 @@ LIMIT_MAX = 100000 2) Run the script ``` (on api_v6/ ) -sh calcul_duration_for_routes.bm.sh +sh scripts/public-transportation/calcul_duration_for_routes.bm.sh ``` 3) Put the limit back to 100 ```python @@ -190,7 +190,7 @@ BACK-END : - create **Coverage** table in database *** -Script `update_navitia_coverage.sh` : +Script `scripts/public-transportation/update_navitia_coverage.sh` : **When to run** diff --git a/README.md b/README.md index 3734e13f3..bcb5de8b0 100644 --- a/README.md +++ b/README.md @@ -6,67 +6,19 @@ [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/323754cf688042688899e6028fdfeff7)](https://app.codacy.com/gh/c2corg/v6_api/dashboard) [![Known Vulnerabilities](https://snyk.io/test/github/c2corg/v6_api/badge.svg)](https://snyk.io/test/github/c2corg/v6_api) -## Development environment - -On any OS, install [git](https://git-scm.com/) and [docker](https://docs.docker.com/install/). Then : - -### Install +## Getting started ```sh -# Download camptocamp.org source code : git clone https://github.com/c2corg/v6_api cd v6_api ``` -### Run - -```sh -# the very first call may be quite long, (15 minutes, depending of your bandwith) -# time to make a coffee -docker-compose up -``` - -:heart: :heart: - -Press CTRL+C to terminate it. - -### Run the background jobs and syncer scripts - -In distinct terminals: - -```sh -docker-compose exec api make -f config/docker-dev run-background-jobs -docker-compose exec api make -f config/docker-dev run-syncer -``` - -### Check code quality - -In another terminal (`docker-compose up` must be running) : - -```sh -./scripts/lint.sh -``` - -### Run test suite - -In another terminal (`docker-compose up` must be running) : - -```sh -# full tests, take a while -./scripts/test.sh - -# If you need to test a specific point: -./scripts/test.sh c2corg_api/tests/models/test_book.py - -# or: -./scripts/test.sh c2corg_api/tests/models/test_book.py::TestBook - -# or even: -./scripts/test.sh c2corg_api/tests/models/test_book.py::TestBook::test_to_archive -``` +The recommended way to develop is to use the **Dev Container** — see [HOWTO-DEV.md](HOWTO-DEV.md) for full instructions (dev container setup, manual setup, running the API, tests, and linting). -Note: if you're using MinGW on Windows, be sure to prefix the command with `MSYS_NO_PATHCONV=1` +For documentation on the "Mobilité douce" (public transport) features, see [PUBLIC-TRANSPORTATION.md](PUBLIC-TRANSPORTATION.md). -## Useful links in [wiki](https://github.com/c2corg/v6_api/wiki) +## Useful links -[Full info about development environment](https://github.com/c2corg/v6_api/wiki/Development-environment-on-Linux) +- [Developer manual (HOWTO-DEV.md)](HOWTO-DEV.md) +- [Dev Container details (.devcontainer/README.md)](.devcontainer/README.md) +- [Wiki — Development environment](https://github.com/c2corg/v6_api/wiki/Development-environment-on-Linux) diff --git a/alembic_migration/env.py b/alembic_migration/env.py index 268d21ac8..f6091ca28 100644 --- a/alembic_migration/env.py +++ b/alembic_migration/env.py @@ -64,6 +64,7 @@ def run_migrations_online(): with context.begin_transaction(): context.run_migrations() + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/alembic_migration/versions/06d2a35e39c8_improve_serac_database.py b/alembic_migration/versions/06d2a35e39c8_improve_serac_database.py index ffa860b48..d6def89ad 100644 --- a/alembic_migration/versions/06d2a35e39c8_improve_serac_database.py +++ b/alembic_migration/versions/06d2a35e39c8_improve_serac_database.py @@ -5,11 +5,14 @@ Create Date: 2019-12-09 13:24:15.936753 """ -from alembic import op + +import csv + import sqlalchemy as sa -from sqlalchemy.sql.schema import Table, MetaData +from alembic import op +from sqlalchemy import MetaData, Table + from alembic_migration.extensions import drop_enum -import csv from c2corg_api.models.utils import ArrayOfEnum # revision identifiers, used by Alembic. @@ -30,11 +33,10 @@ b'autre': 'other', b'effondrement cascade': 'ice_cornice_collapse', b'l\xc3\xa9sion sans chute ': 'injury_without_fall', - b'personne bloqu\xc3\xa9e': 'blocked_person' + b'personne bloqu\xc3\xa9e': 'blocked_person', } - def upgrade(): # convert activities @@ -52,45 +54,81 @@ def upgrade(): ('ice_climbing', 'ice_climbing'), ] old_activity_type = ArrayOfEnum( - sa.Enum('skitouring', 'snow_ice_mixed', 'mountain_climbing', - 'rock_climbing', 'ice_climbing', 'hiking', 'snowshoeing', - 'paragliding', 'mountain_biking', 'via_ferrata', 'slacklining', - name='activity_type', schema='guidebook') - ) - new_activity_type = sa.Enum('sport_climbing', 'multipitch_climbing', - 'alpine_climbing', 'snow_ice_mixed', - 'ice_climbing', 'skitouring', 'other', - name='event_activity_type', schema='guidebook') + sa.Enum( + 'skitouring', + 'snow_ice_mixed', + 'mountain_climbing', + 'rock_climbing', + 'ice_climbing', + 'hiking', + 'snowshoeing', + 'paragliding', + 'mountain_biking', + 'via_ferrata', + 'slacklining', + name='activity_type', + schema='guidebook', + ) + ) + new_activity_type = sa.Enum( + 'sport_climbing', + 'multipitch_climbing', + 'alpine_climbing', + 'snow_ice_mixed', + 'ice_climbing', + 'skitouring', + 'other', + name='event_activity_type', + schema='guidebook', + ) new_activity_type.create(op.get_bind()) - op.add_column('xreports', - sa.Column('event_activity', new_activity_type), - schema='guidebook') - op.add_column('xreports_archives', - sa.Column('event_activity', new_activity_type), - schema='guidebook') - - xr = Table('xreports', MetaData(), - sa.Column('activities', old_activity_type), - sa.Column('event_activity', new_activity_type, nullable=True), - schema='guidebook') - xra = Table('xreports_archives', MetaData(), - sa.Column('activities', old_activity_type), - sa.Column('event_activity', new_activity_type, nullable=True), - schema='guidebook') - for (old_value, new_value) in activity_conversions: - op.execute(xr.update() - .where(xr.c.activities.contains(sa.literal([old_value]) - .cast(old_activity_type))) - .values(event_activity=op.inline_literal(new_value))) - op.execute(xra.update() - .where(xra.c.activities.contains(sa.literal([old_value]) - .cast(old_activity_type))) - .values(event_activity=op.inline_literal(new_value))) - - op.alter_column('xreports', 'event_activity', - nullable=False, schema='guidebook') - op.alter_column('xreports_archives', 'event_activity', - nullable=False, schema='guidebook') + op.add_column( + 'xreports', sa.Column('event_activity', new_activity_type), schema='guidebook' + ) + op.add_column( + 'xreports_archives', + sa.Column('event_activity', new_activity_type), + schema='guidebook', + ) + + xr = Table( + 'xreports', + MetaData(), + sa.Column('activities', old_activity_type), + sa.Column('event_activity', new_activity_type, nullable=True), + schema='guidebook', + ) + xra = Table( + 'xreports_archives', + MetaData(), + sa.Column('activities', old_activity_type), + sa.Column('event_activity', new_activity_type, nullable=True), + schema='guidebook', + ) + for old_value, new_value in activity_conversions: + op.execute( + xr.update() + .where( + xr.c.activities.contains( + sa.literal([old_value]).cast(old_activity_type) + ) + ) + .values(event_activity=op.inline_literal(new_value)) + ) + op.execute( + xra.update() + .where( + xra.c.activities.contains( + sa.literal([old_value]).cast(old_activity_type) + ) + ) + .values(event_activity=op.inline_literal(new_value)) + ) + + op.alter_column('xreports', 'event_activity', nullable=False, schema='guidebook') + op.alter_column( + 'xreports_archives', 'event_activity', nullable=False, schema='guidebook' + ) op.drop_column('xreports', 'activities', schema='guidebook') op.drop_column('xreports_archives', 'activities', schema='guidebook') @@ -107,91 +145,130 @@ def upgrade(): ('roped_fall', 'person_fall'), ('physical_failure', 'physical_failure'), ('lightning', 'weather_event'), - ('other', 'other') + ('other', 'other'), ] old_event_type = ArrayOfEnum( - sa.Enum('avalanche', 'stone_fall', 'falling_ice', 'person_fall', - 'crevasse_fall', 'roped_fall', 'physical_failure', - 'lightning', 'other', - name='event_type', schema='guidebook') + sa.Enum( + 'avalanche', + 'stone_fall', + 'falling_ice', + 'person_fall', + 'crevasse_fall', + 'roped_fall', + 'physical_failure', + 'lightning', + 'other', + name='event_type', + schema='guidebook', + ) ) new_event_type = sa.Enum( - 'avalanche', 'stone_ice_fall', 'ice_cornice_collapse', - 'person_fall', 'crevasse_fall', 'physical_failure', - 'injury_without_fall', 'blocked_person', 'weather_event', - 'safety_operation', 'critical_situation', 'other', - name='event_type_', schema='guidebook' + 'avalanche', + 'stone_ice_fall', + 'ice_cornice_collapse', + 'person_fall', + 'crevasse_fall', + 'physical_failure', + 'injury_without_fall', + 'blocked_person', + 'weather_event', + 'safety_operation', + 'critical_situation', + 'other', + name='event_type_', + schema='guidebook', ) new_event_type.create(op.get_bind()) - op.add_column('xreports', - sa.Column('event_type_', new_event_type), - schema='guidebook') - op.add_column('xreports_archives', - sa.Column('event_type_', new_event_type), - schema='guidebook') - - xr = Table('xreports', MetaData(), - sa.Column('event_type', old_event_type), - sa.Column('event_type_', new_event_type), - schema='guidebook') - xra = Table('xreports_archives', MetaData(), - sa.Column('event_type', old_event_type), - sa.Column('event_type_', new_event_type), - schema='guidebook') - for (old_value, new_value) in type_conversions: - op.execute(xr.update() - .where(xr.c.event_type.contains(sa.literal([old_value]) - .cast(old_event_type))) - .values(event_type_=op.inline_literal(new_value))) - op.execute(xra.update() - .where(xra.c.event_type.contains(sa.literal([old_value]) - .cast(old_event_type))) - .values(event_type_=op.inline_literal(new_value))) - - op.alter_column('xreports', 'event_type', - nullable=False, schema='guidebook') - op.alter_column('xreports_archives', 'event_type', - nullable=False, schema='guidebook') + op.add_column( + 'xreports', sa.Column('event_type_', new_event_type), schema='guidebook' + ) + op.add_column( + 'xreports_archives', + sa.Column('event_type_', new_event_type), + schema='guidebook', + ) + + xr = Table( + 'xreports', + MetaData(), + sa.Column('event_type', old_event_type), + sa.Column('event_type_', new_event_type), + schema='guidebook', + ) + xra = Table( + 'xreports_archives', + MetaData(), + sa.Column('event_type', old_event_type), + sa.Column('event_type_', new_event_type), + schema='guidebook', + ) + for old_value, new_value in type_conversions: + op.execute( + xr.update() + .where( + xr.c.event_type.contains(sa.literal([old_value]).cast(old_event_type)) + ) + .values(event_type_=op.inline_literal(new_value)) + ) + op.execute( + xra.update() + .where( + xra.c.event_type.contains(sa.literal([old_value]).cast(old_event_type)) + ) + .values(event_type_=op.inline_literal(new_value)) + ) + + op.alter_column('xreports', 'event_type', nullable=False, schema='guidebook') + op.alter_column( + 'xreports_archives', 'event_type', nullable=False, schema='guidebook' + ) op.drop_column('xreports', 'event_type', schema='guidebook') op.drop_column('xreports_archives', 'event_type', schema='guidebook') drop_enum('event_type', schema='guidebook') op.execute('ALTER TYPE guidebook.event_type_ RENAME TO event_type') op.alter_column( - 'xreports', - 'event_type_', - new_column_name='event_type', - schema='guidebook') + 'xreports', 'event_type_', new_column_name='event_type', schema='guidebook' + ) op.alter_column( 'xreports_archives', 'event_type_', new_column_name='event_type', - schema='guidebook') + schema='guidebook', + ) - xr = Table('xreports', MetaData(), - sa.Column('document_id', sa.types.INTEGER), - sa.Column('event_type', new_event_type), - schema='guidebook') + xr = Table( + 'xreports', + MetaData(), + sa.Column('document_id', sa.types.INTEGER), + sa.Column('event_type', new_event_type), + schema='guidebook', + ) # xra = Table('xreports_archives', MetaData(), # sa.Column('document_id', sa.types.INTEGER), # sa.Column('event_type', new_event_type), # schema='guidebook') try: - with open('./alembic_migration/versions/06d2a35e39c8_improve_serac_database_data.csv') as f: + with open( + './alembic_migration/versions/06d2a35e39c8_improve_serac_database_data.csv' + ) as f: rr = csv.reader(f) header = rr.__next__() assert (header[1] == 'Document') and (header[8] == 'ENS principal') for line in rr: - print("update {} -> {}".format(line[1], - key_map[line[8].lower().encode()])) - op.execute(xr.update() - .where(xr.c.document_id == line[1]) - .values(event_type=key_map[line[8].lower().encode()])) + print( + 'update {} -> {}'.format(line[1], key_map[line[8].lower().encode()]) + ) + op.execute( + xr.update() + .where(xr.c.document_id == int(line[1])) + .values(event_type=key_map[line[8].lower().encode()]) + ) # op.execute(xra.update() # .where(xra.c.document_id == line[1]) # .values(event_type=key_map[line[8].lower()])) except Exception as e: - print("EXCEPT!!! {} {}".format(type(e), e)) + print('EXCEPT!!! {} {}'.format(type(e), e)) # end of types conversion # convert autonomy enum @@ -199,10 +276,19 @@ def upgrade(): ('non_autonomous', 'non_autonomous'), ('autonomous', 'autonomous'), ('initiator', 'autonomous'), - ('expert', 'expert') + ('expert', 'expert'), ] - old_autonomy_type = sa.Enum('non_autonomous', 'autonomous', 'initiator', 'expert', name='autonomy', schema='guidebook') - new_autonomy_type = sa.Enum('non_autonomous', 'autonomous', 'expert', name='autonomy_', schema='guidebook') + old_autonomy_type = sa.Enum( + 'non_autonomous', + 'autonomous', + 'initiator', + 'expert', + name='autonomy', + schema='guidebook', + ) + new_autonomy_type = sa.Enum( + 'non_autonomous', 'autonomous', 'expert', name='autonomy_', schema='guidebook' + ) new_autonomy_type.create(op.get_bind()) # op.alter_column('xreports', 'autonomy', # type_=new_autonomy_type, @@ -210,28 +296,42 @@ def upgrade(): # schema='guidebook') # does not allow automatic casting if table not empty - op.add_column('xreports', - sa.Column('autonomy_', new_autonomy_type, nullable=True), - schema='guidebook') - op.add_column('xreports_archives', - sa.Column('autonomy_', new_autonomy_type, nullable=True), - schema='guidebook') - - xr = Table('xreports', MetaData(), - sa.Column('autonomy', old_autonomy_type), - sa.Column('autonomy_', new_autonomy_type, nullable=True), - schema='guidebook') - xra = Table('xreports_archives', MetaData(), - sa.Column('autonomy', old_autonomy_type), - sa.Column('autonomy_', new_autonomy_type, nullable=True), - schema='guidebook') - for (old_value, new_value) in autonomy_conversions: - op.execute(xr.update() - .where(xr.c.autonomy == op.inline_literal(old_value)) - .values(autonomy_=op.inline_literal(new_value))) - op.execute(xra.update() - .where(xra.c.autonomy == op.inline_literal(old_value)) - .values(autonomy_=op.inline_literal(new_value))) + op.add_column( + 'xreports', + sa.Column('autonomy_', new_autonomy_type, nullable=True), + schema='guidebook', + ) + op.add_column( + 'xreports_archives', + sa.Column('autonomy_', new_autonomy_type, nullable=True), + schema='guidebook', + ) + + xr = Table( + 'xreports', + MetaData(), + sa.Column('autonomy', old_autonomy_type), + sa.Column('autonomy_', new_autonomy_type, nullable=True), + schema='guidebook', + ) + xra = Table( + 'xreports_archives', + MetaData(), + sa.Column('autonomy', old_autonomy_type), + sa.Column('autonomy_', new_autonomy_type, nullable=True), + schema='guidebook', + ) + for old_value, new_value in autonomy_conversions: + op.execute( + xr.update() + .where(xr.c.autonomy == op.inline_literal(old_value)) + .values(autonomy_=op.inline_literal(new_value)) + ) + op.execute( + xra.update() + .where(xra.c.autonomy == op.inline_literal(old_value)) + .values(autonomy_=op.inline_literal(new_value)) + ) op.drop_column('xreports', 'autonomy', schema='guidebook') op.drop_column('xreports_archives', 'autonomy', schema='guidebook') @@ -243,15 +343,11 @@ def upgrade(): # Rename column op.alter_column( - 'xreports', - 'autonomy_', - new_column_name='autonomy', - schema='guidebook') + 'xreports', 'autonomy_', new_column_name='autonomy', schema='guidebook' + ) op.alter_column( - 'xreports_archives', - 'autonomy_', - new_column_name='autonomy', - schema='guidebook') + 'xreports_archives', 'autonomy_', new_column_name='autonomy', schema='guidebook' + ) # end of autonomy conversion @@ -263,40 +359,64 @@ def upgrade(): ('activity_rate_20', 'activity_rate_m2'), ('activity_rate_10', 'activity_rate_y5'), ('activity_rate_5', 'activity_rate_y5'), - ('activity_rate_1', 'activity_rate_y5') + ('activity_rate_1', 'activity_rate_y5'), ] - old_activity_type = sa.Enum('activity_rate_150', 'activity_rate_50', - 'activity_rate_30', 'activity_rate_20', - 'activity_rate_10', 'activity_rate_5', - 'activity_rate_1', - name='activity_rate', schema='guidebook') - new_activity_type = sa.Enum('activity_rate_y5', 'activity_rate_m2', - 'activity_rate_w1', - name='activity_rate_', schema='guidebook') + old_activity_type = sa.Enum( + 'activity_rate_150', + 'activity_rate_50', + 'activity_rate_30', + 'activity_rate_20', + 'activity_rate_10', + 'activity_rate_5', + 'activity_rate_1', + name='activity_rate', + schema='guidebook', + ) + new_activity_type = sa.Enum( + 'activity_rate_y5', + 'activity_rate_m2', + 'activity_rate_w1', + name='activity_rate_', + schema='guidebook', + ) new_activity_type.create(op.get_bind()) - op.add_column('xreports', - sa.Column('activity_rate_', new_activity_type, nullable=True), - schema='guidebook') - op.add_column('xreports_archives', - sa.Column('activity_rate_', new_activity_type, nullable=True), - schema='guidebook') - - xr = Table('xreports', MetaData(), - sa.Column('activity_rate', old_activity_type), - sa.Column('activity_rate_', new_activity_type, nullable=True), - schema='guidebook') - xra = Table('xreports_archives', MetaData(), - sa.Column('activity_rate', old_activity_type), - sa.Column('activity_rate_', new_activity_type, nullable=True), - schema='guidebook') - for (old_value, new_value) in activity_conversions: - op.execute(xr.update() - .where(xr.c.activity_rate == op.inline_literal(old_value)) - .values(activity_rate_=op.inline_literal(new_value))) - op.execute(xra.update() - .where(xra.c.activity_rate == op.inline_literal(old_value)) - .values(activity_rate_=op.inline_literal(new_value))) + op.add_column( + 'xreports', + sa.Column('activity_rate_', new_activity_type, nullable=True), + schema='guidebook', + ) + op.add_column( + 'xreports_archives', + sa.Column('activity_rate_', new_activity_type, nullable=True), + schema='guidebook', + ) + + xr = Table( + 'xreports', + MetaData(), + sa.Column('activity_rate', old_activity_type), + sa.Column('activity_rate_', new_activity_type, nullable=True), + schema='guidebook', + ) + xra = Table( + 'xreports_archives', + MetaData(), + sa.Column('activity_rate', old_activity_type), + sa.Column('activity_rate_', new_activity_type, nullable=True), + schema='guidebook', + ) + for old_value, new_value in activity_conversions: + op.execute( + xr.update() + .where(xr.c.activity_rate == op.inline_literal(old_value)) + .values(activity_rate_=op.inline_literal(new_value)) + ) + op.execute( + xra.update() + .where(xra.c.activity_rate == op.inline_literal(old_value)) + .values(activity_rate_=op.inline_literal(new_value)) + ) op.drop_column('xreports', 'activity_rate', schema='guidebook') op.drop_column('xreports_archives', 'activity_rate', schema='guidebook') @@ -311,12 +431,14 @@ def upgrade(): 'xreports', 'activity_rate_', new_column_name='activity_rate', - schema='guidebook') + schema='guidebook', + ) op.alter_column( 'xreports_archives', 'activity_rate_', new_column_name='activity_rate', - schema='guidebook') + schema='guidebook', + ) # end of activity conversion @@ -325,23 +447,43 @@ def upgrade(): # op.drop_column('xreports_archives', 'avalanche_slope', schema='guidebook') drop_enum('nb_outings', schema='guidebook') - supervision_type = sa.Enum('no_supervision', 'federal_supervision', - 'professional_supervision', - name='supervision_type', schema='guidebook') + supervision_type = sa.Enum( + 'no_supervision', + 'federal_supervision', + 'professional_supervision', + name='supervision_type', + schema='guidebook', + ) supervision_type.create(op.get_bind()) - op.add_column('xreports', sa.Column('supervision', supervision_type, - nullable=True), schema='guidebook') - op.add_column('xreports_archives', sa.Column('supervision', supervision_type, - nullable=True), schema='guidebook') - - qualification_type = sa.Enum('federal_supervisor', 'federal_trainer', - 'professional_diploma', - name='qualification_type', schema='guidebook') + op.add_column( + 'xreports', + sa.Column('supervision', supervision_type, nullable=True), + schema='guidebook', + ) + op.add_column( + 'xreports_archives', + sa.Column('supervision', supervision_type, nullable=True), + schema='guidebook', + ) + + qualification_type = sa.Enum( + 'federal_supervisor', + 'federal_trainer', + 'professional_diploma', + name='qualification_type', + schema='guidebook', + ) qualification_type.create(op.get_bind()) - op.add_column('xreports', sa.Column('qualification', qualification_type, - nullable=True), schema='guidebook') - op.add_column('xreports_archives', sa.Column('qualification', qualification_type, - nullable=True), schema='guidebook') + op.add_column( + 'xreports', + sa.Column('qualification', qualification_type, nullable=True), + schema='guidebook', + ) + op.add_column( + 'xreports_archives', + sa.Column('qualification', qualification_type, nullable=True), + schema='guidebook', + ) def downgrade(): @@ -353,33 +495,62 @@ def downgrade(): ('activity_rate_y5', 'activity_rate_10'), ('activity_rate_y5', 'activity_rate_5'), ] - old_activity_type = sa.Enum('activity_rate_y5', 'activity_rate_m2', 'activity_rate_w1', name='activity_rate', schema='guidebook') - new_activity_type = sa.Enum('activity_rate_150', 'activity_rate_50', 'activity_rate_30', 'activity_rate_20', 'activity_rate_10', - 'activity_rate_5', 'activity_rate_1', name='activity_rate_', schema='guidebook') + old_activity_type = sa.Enum( + 'activity_rate_y5', + 'activity_rate_m2', + 'activity_rate_w1', + name='activity_rate', + schema='guidebook', + ) + new_activity_type = sa.Enum( + 'activity_rate_150', + 'activity_rate_50', + 'activity_rate_30', + 'activity_rate_20', + 'activity_rate_10', + 'activity_rate_5', + 'activity_rate_1', + name='activity_rate_', + schema='guidebook', + ) new_activity_type.create(op.get_bind()) - op.add_column('xreports', - sa.Column('activity_rate_', new_activity_type, nullable=True), - schema='guidebook') - op.add_column('xreports_archives', - sa.Column('activity_rate_', new_activity_type, nullable=True), - schema='guidebook') - - xr = Table('xreports', MetaData(), - sa.Column('activity_rate', old_activity_type), - sa.Column('activity_rate_', new_activity_type, nullable=True), - schema='guidebook') - xra = Table('xreports_archives', MetaData(), - sa.Column('activity_rate', old_activity_type), - sa.Column('activity_rate_', new_activity_type, nullable=True), - schema='guidebook') - for (old_value, new_value) in activity_conversions: - op.execute(xr.update() - .where(xr.c.activity_rate == op.inline_literal(old_value)) - .values(activity_rate_=op.inline_literal(new_value))) - op.execute(xra.update() - .where(xra.c.activity_rate == op.inline_literal(old_value)) - .values(activity_rate_=op.inline_literal(new_value))) + op.add_column( + 'xreports', + sa.Column('activity_rate_', new_activity_type, nullable=True), + schema='guidebook', + ) + op.add_column( + 'xreports_archives', + sa.Column('activity_rate_', new_activity_type, nullable=True), + schema='guidebook', + ) + + xr = Table( + 'xreports', + MetaData(), + sa.Column('activity_rate', old_activity_type), + sa.Column('activity_rate_', new_activity_type, nullable=True), + schema='guidebook', + ) + xra = Table( + 'xreports_archives', + MetaData(), + sa.Column('activity_rate', old_activity_type), + sa.Column('activity_rate_', new_activity_type, nullable=True), + schema='guidebook', + ) + for old_value, new_value in activity_conversions: + op.execute( + xr.update() + .where(xr.c.activity_rate == op.inline_literal(old_value)) + .values(activity_rate_=op.inline_literal(new_value)) + ) + op.execute( + xra.update() + .where(xra.c.activity_rate == op.inline_literal(old_value)) + .values(activity_rate_=op.inline_literal(new_value)) + ) op.drop_column('xreports', 'activity_rate', schema='guidebook') op.drop_column('xreports_archives', 'activity_rate', schema='guidebook') @@ -394,12 +565,14 @@ def downgrade(): 'xreports', 'activity_rate_', new_column_name='activity_rate', - schema='guidebook') + schema='guidebook', + ) op.alter_column( 'xreports_archives', 'activity_rate_', new_column_name='activity_rate', - schema='guidebook') + schema='guidebook', + ) # end of activity conversion @@ -407,10 +580,19 @@ def downgrade(): autonomy_conversions = [ ('non_autonomous', 'non_autonomous'), ('autonomous', 'autonomous'), - ('expert', 'expert') + ('expert', 'expert'), ] - old_autonomy_type = sa.Enum('non_autonomous', 'autonomous', 'expert', name='autonomy', schema='guidebook') - new_autonomy_type = sa.Enum('non_autonomous', 'autonomous', 'initiator', 'expert', name='autonomy_', schema='guidebook') + old_autonomy_type = sa.Enum( + 'non_autonomous', 'autonomous', 'expert', name='autonomy', schema='guidebook' + ) + new_autonomy_type = sa.Enum( + 'non_autonomous', + 'autonomous', + 'initiator', + 'expert', + name='autonomy_', + schema='guidebook', + ) new_autonomy_type.create(op.get_bind()) # op.alter_column('xreports', 'autonomy', # type_=new_autonomy_type, @@ -418,28 +600,42 @@ def downgrade(): # schema='guidebook') # does not allow automatic casting if table not empty - op.add_column('xreports', - sa.Column('autonomy_', new_autonomy_type, nullable=True), - schema='guidebook') - op.add_column('xreports_archives', - sa.Column('autonomy_', new_autonomy_type, nullable=True), - schema='guidebook') - - xr = Table('xreports', MetaData(), - sa.Column('autonomy', old_autonomy_type), - sa.Column('autonomy_', new_autonomy_type, nullable=True), - schema='guidebook') - xra = Table('xreports_archives', MetaData(), - sa.Column('autonomy', old_autonomy_type), - sa.Column('autonomy_', new_autonomy_type, nullable=True), - schema='guidebook') - for (old_value, new_value) in autonomy_conversions: - op.execute(xr.update() - .where(xr.c.autonomy == op.inline_literal(old_value)) - .values(autonomy_=op.inline_literal(new_value))) - op.execute(xra.update() - .where(xra.c.autonomy == op.inline_literal(old_value)) - .values(autonomy_=op.inline_literal(new_value))) + op.add_column( + 'xreports', + sa.Column('autonomy_', new_autonomy_type, nullable=True), + schema='guidebook', + ) + op.add_column( + 'xreports_archives', + sa.Column('autonomy_', new_autonomy_type, nullable=True), + schema='guidebook', + ) + + xr = Table( + 'xreports', + MetaData(), + sa.Column('autonomy', old_autonomy_type), + sa.Column('autonomy_', new_autonomy_type, nullable=True), + schema='guidebook', + ) + xra = Table( + 'xreports_archives', + MetaData(), + sa.Column('autonomy', old_autonomy_type), + sa.Column('autonomy_', new_autonomy_type, nullable=True), + schema='guidebook', + ) + for old_value, new_value in autonomy_conversions: + op.execute( + xr.update() + .where(xr.c.autonomy == op.inline_literal(old_value)) + .values(autonomy_=op.inline_literal(new_value)) + ) + op.execute( + xra.update() + .where(xra.c.autonomy == op.inline_literal(old_value)) + .values(autonomy_=op.inline_literal(new_value)) + ) op.drop_column('xreports', 'autonomy', schema='guidebook') op.drop_column('xreports_archives', 'autonomy', schema='guidebook') @@ -451,15 +647,11 @@ def downgrade(): # Rename column op.alter_column( - 'xreports', - 'autonomy_', - new_column_name='autonomy', - schema='guidebook') + 'xreports', 'autonomy_', new_column_name='autonomy', schema='guidebook' + ) op.alter_column( - 'xreports_archives', - 'autonomy_', - new_column_name='autonomy', - schema='guidebook') + 'xreports_archives', 'autonomy_', new_column_name='autonomy', schema='guidebook' + ) # end of autonomy conversion @@ -470,14 +662,25 @@ def downgrade(): op.drop_column('xreports_archives', 'qualification', schema='guidebook') drop_enum('qualification_type', schema='guidebook') - nb_outing_type = sa.Enum('nb_outings_4', 'nb_outings_9', 'nb_outings_14', 'nb_outings_15', name='nb_outings', schema='guidebook') + nb_outing_type = sa.Enum( + 'nb_outings_4', + 'nb_outings_9', + 'nb_outings_14', + 'nb_outings_15', + name='nb_outings', + schema='guidebook', + ) nb_outing_type.create(op.get_bind()) - op.add_column('xreports', - sa.Column('nb_outings', nb_outing_type, nullable=True), - schema='guidebook') - op.add_column('xreports_archives', - sa.Column('nb_outings', nb_outing_type, nullable=True), - schema='guidebook') + op.add_column( + 'xreports', + sa.Column('nb_outings', nb_outing_type, nullable=True), + schema='guidebook', + ) + op.add_column( + 'xreports_archives', + sa.Column('nb_outings', nb_outing_type, nullable=True), + schema='guidebook', + ) activity_conversions = [ ('other', 'hiking'), @@ -485,52 +688,83 @@ def downgrade(): ('snow_ice_mixed', 'snow_ice_mixed'), ('alpine_climbing', 'mountain_climbing'), ('sport_climbing', 'rock_climbing'), - ('ice_climbing', 'ice_climbing') + ('ice_climbing', 'ice_climbing'), ] - old_activity_type = sa.Enum('sport_climbing', 'multipitch_climbing', - 'alpine_climbing', 'snow_ice_mixed', - 'ice_climbing', 'skitouring', 'other', - name='event_activity_type', schema='guidebook') + old_activity_type = sa.Enum( + 'sport_climbing', + 'multipitch_climbing', + 'alpine_climbing', + 'snow_ice_mixed', + 'ice_climbing', + 'skitouring', + 'other', + name='event_activity_type', + schema='guidebook', + ) activities_type = ArrayOfEnum( - sa.Enum('skitouring', 'snow_ice_mixed', 'mountain_climbing', - 'rock_climbing', 'ice_climbing', 'hiking', 'snowshoeing', - 'paragliding', 'mountain_biking', 'via_ferrata', 'slacklining', - name='activity_type', schema='guidebook') - ) - op.add_column('xreports', - sa.Column('activities', activities_type, nullable=True), - schema='guidebook') - op.add_column('xreports_archives', - sa.Column('activities', activities_type, nullable=True), - schema='guidebook') - - xr = Table('xreports', MetaData(), - sa.Column('activities', activities_type, nullable=True), - sa.Column('event_activity', old_activity_type), - schema='guidebook') - xra = Table('xreports_archives', MetaData(), - sa.Column('activities', activities_type, nullable=True), - sa.Column('event_activity', old_activity_type), - schema='guidebook') - for (old_value, new_value) in activity_conversions: - op.execute(xr.update() - .where(xr.c.event_activity == op.inline_literal(old_value)) - .values(activities=sa.literal([new_value]))) - op.execute(xra.update() - .where(xra.c.event_activity == op.inline_literal(old_value)) - .values(activities=sa.literal([new_value]))) - - op.alter_column('xreports', 'activities', - nullable=False, schema='guidebook') - op.alter_column('xreports_archives', 'activities', - nullable=False, schema='guidebook') + sa.Enum( + 'skitouring', + 'snow_ice_mixed', + 'mountain_climbing', + 'rock_climbing', + 'ice_climbing', + 'hiking', + 'snowshoeing', + 'paragliding', + 'mountain_biking', + 'via_ferrata', + 'slacklining', + name='activity_type', + schema='guidebook', + ) + ) + op.add_column( + 'xreports', + sa.Column('activities', activities_type, nullable=True), + schema='guidebook', + ) + op.add_column( + 'xreports_archives', + sa.Column('activities', activities_type, nullable=True), + schema='guidebook', + ) + + xr = Table( + 'xreports', + MetaData(), + sa.Column('activities', activities_type, nullable=True), + sa.Column('event_activity', old_activity_type), + schema='guidebook', + ) + xra = Table( + 'xreports_archives', + MetaData(), + sa.Column('activities', activities_type, nullable=True), + sa.Column('event_activity', old_activity_type), + schema='guidebook', + ) + for old_value, new_value in activity_conversions: + op.execute( + xr.update() + .where(xr.c.event_activity == op.inline_literal(old_value)) + .values(activities=sa.literal([new_value])) + ) + op.execute( + xra.update() + .where(xra.c.event_activity == op.inline_literal(old_value)) + .values(activities=sa.literal([new_value])) + ) + + op.alter_column('xreports', 'activities', nullable=False, schema='guidebook') + op.alter_column( + 'xreports_archives', 'activities', nullable=False, schema='guidebook' + ) op.drop_column('xreports', 'event_activity', schema='guidebook') op.drop_column('xreports_archives', 'event_activity', schema='guidebook') drop_enum('event_activity_type', schema='guidebook') - # convert types type_conversions = [ ('avalanche', 'avalanche'), @@ -544,58 +778,87 @@ def downgrade(): ('safety_operation', 'other'), ('critical_situation', 'other'), ('weather_event', 'lightning'), - ('other', 'other') + ('other', 'other'), ] old_event_type = sa.Enum( - 'avalanche', 'stone_ice_fall', 'ice_cornice_collapse', - 'person_fall', 'crevasse_fall', 'physical_failure', - 'injury_without_fall', 'blocked_person', 'weather_event', - 'safety_operation', 'critical_situation', 'other', - name='event_type', schema='guidebook' + 'avalanche', + 'stone_ice_fall', + 'ice_cornice_collapse', + 'person_fall', + 'crevasse_fall', + 'physical_failure', + 'injury_without_fall', + 'blocked_person', + 'weather_event', + 'safety_operation', + 'critical_situation', + 'other', + name='event_type', + schema='guidebook', ) new_event_type = sa.Enum( - 'avalanche', 'stone_fall', 'falling_ice', 'person_fall', - 'crevasse_fall', 'roped_fall', 'physical_failure', - 'lightning', 'other', - name='event_type_', schema='guidebook' + 'avalanche', + 'stone_fall', + 'falling_ice', + 'person_fall', + 'crevasse_fall', + 'roped_fall', + 'physical_failure', + 'lightning', + 'other', + name='event_type_', + schema='guidebook', ) new_event_type.create(op.get_bind()) - op.add_column('xreports', - sa.Column('event_type_', ArrayOfEnum(new_event_type)), - schema='guidebook') - op.add_column('xreports_archives', - sa.Column('event_type_', ArrayOfEnum(new_event_type)), - schema='guidebook') - - xr = Table('xreports', MetaData(), - sa.Column('event_type', old_event_type), - sa.Column('event_type_', ArrayOfEnum(new_event_type)), - schema='guidebook') - xra = Table('xreports_archives', MetaData(), - sa.Column('event_type', old_event_type), - sa.Column('event_type_', ArrayOfEnum(new_event_type)), - schema='guidebook') - for (old_value, new_value) in type_conversions: - op.execute(xr.update() - .where(xr.c.event_type == op.inline_literal(old_value)) - .values(event_type_=sa.literal([new_value]))) - op.execute(xra.update() - .where(xra.c.event_type == op.inline_literal(old_value)) - .values(event_type_=sa.literal([new_value]))) + op.add_column( + 'xreports', + sa.Column('event_type_', ArrayOfEnum(new_event_type)), + schema='guidebook', + ) + op.add_column( + 'xreports_archives', + sa.Column('event_type_', ArrayOfEnum(new_event_type)), + schema='guidebook', + ) + + xr = Table( + 'xreports', + MetaData(), + sa.Column('event_type', old_event_type), + sa.Column('event_type_', ArrayOfEnum(new_event_type)), + schema='guidebook', + ) + xra = Table( + 'xreports_archives', + MetaData(), + sa.Column('event_type', old_event_type), + sa.Column('event_type_', ArrayOfEnum(new_event_type)), + schema='guidebook', + ) + for old_value, new_value in type_conversions: + op.execute( + xr.update() + .where(xr.c.event_type == op.inline_literal(old_value)) + .values(event_type_=sa.literal([new_value])) + ) + op.execute( + xra.update() + .where(xra.c.event_type == op.inline_literal(old_value)) + .values(event_type_=sa.literal([new_value])) + ) op.drop_column('xreports', 'event_type', schema='guidebook') op.drop_column('xreports_archives', 'event_type', schema='guidebook') drop_enum('event_type', schema='guidebook') op.execute('ALTER TYPE guidebook.event_type_ RENAME TO event_type') op.alter_column( - 'xreports', - 'event_type_', - new_column_name='event_type', - schema='guidebook') + 'xreports', 'event_type_', new_column_name='event_type', schema='guidebook' + ) op.alter_column( 'xreports_archives', 'event_type_', new_column_name='event_type', - schema='guidebook') + schema='guidebook', + ) # end of types conversion diff --git a/alembic_migration/versions/24f8da659c78_add_ratings_to_outings.py b/alembic_migration/versions/24f8da659c78_add_ratings_to_outings.py index 8dcd989f7..31acb537e 100644 --- a/alembic_migration/versions/24f8da659c78_add_ratings_to_outings.py +++ b/alembic_migration/versions/24f8da659c78_add_ratings_to_outings.py @@ -9,7 +9,7 @@ import codecs from alembic import op import sqlalchemy as sa -from sqlalchemy.sql import text +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -201,8 +201,8 @@ def upgrade(): # using the best route rating for each activity associated with the outing connection = op.get_bind() raw_file = join(dirname(__file__), '24f8da659c78_outings.sql') - f = codecs.open(raw_file, encoding='utf-8') - content = f.read() + with codecs.open(raw_file, encoding='utf-8') as f: + content = f.read() connection.execute(text(content)) # we loop over every existing outing diff --git a/alembic_migration/versions/2c90b2e5ca7e_add_chinese.py b/alembic_migration/versions/2c90b2e5ca7e_add_chinese.py index dba9e3f89..8a4aed181 100644 --- a/alembic_migration/versions/2c90b2e5ca7e_add_chinese.py +++ b/alembic_migration/versions/2c90b2e5ca7e_add_chinese.py @@ -5,7 +5,7 @@ Create Date: 2021-06-23 19:36:29.664725 """ -from c2corg_api.models.common.attributes import default_langs +from c2corg_api.models.common.attributes import DefaultLangs from alembic_migration.extensions import drop_enum from alembic import op import sqlalchemy as sa @@ -48,7 +48,7 @@ def upgrade(): # if there is some value in the table. If yes, it's not a test DB and we have to complete them. conn = op.get_bind() - res = conn.execute("SELECT count(1) FROM guidebook.langs").fetchall() + res = conn.execute(sa.text("SELECT count(1) FROM guidebook.langs")).fetchall() if res[0][0] != 0: op.execute("INSERT INTO guidebook.langs VALUES ('zh');") @@ -62,7 +62,7 @@ def upgrade(): op.execute("ALTER TYPE guidebook.lang RENAME TO lang_old;") # create the new type - lang_enum = sa.Enum(*default_langs, name='lang', schema='guidebook') + lang_enum = sa.Enum(*[e.value for e in DefaultLangs], name='lang', schema='guidebook') lang_enum.create(op.get_bind(), checkfirst=False) # update the columns to use the new type diff --git a/alembic_migration/versions/305b064bdf66_add_slovenian.py b/alembic_migration/versions/305b064bdf66_add_slovenian.py index 389aa62aa..f7e799253 100644 --- a/alembic_migration/versions/305b064bdf66_add_slovenian.py +++ b/alembic_migration/versions/305b064bdf66_add_slovenian.py @@ -5,7 +5,7 @@ Create Date: 2023-01-05 08:37:27.711800 """ -from c2corg_api.models.common.attributes import default_langs +from c2corg_api.models.common.attributes import DefaultLangs from alembic_migration.extensions import drop_enum from alembic import op import sqlalchemy as sa @@ -21,7 +21,9 @@ def upgrade(): conn = op.get_bind() - res = conn.execute("SELECT count(1) FROM guidebook.langs").fetchall() + res = conn.execute( + sa.text("SELECT count(1) FROM guidebook.langs") + ).fetchall() if res[0][0] != 0: op.execute("INSERT INTO guidebook.langs VALUES ('sl');") @@ -30,7 +32,7 @@ def upgrade(): op.execute("ALTER TYPE guidebook.lang RENAME TO lang_old;") # create the new type - lang_enum = sa.Enum(*default_langs, name='lang', schema='guidebook') + lang_enum = sa.Enum(*[e.value for e in DefaultLangs], name='lang', schema='guidebook') lang_enum.create(op.get_bind(), checkfirst=False) # update the columns to use the new type diff --git a/alembic_migration/versions/38df9393c9a9_init.py b/alembic_migration/versions/38df9393c9a9_init.py index 6b5907e27..fe60e53f8 100644 --- a/alembic_migration/versions/38df9393c9a9_init.py +++ b/alembic_migration/versions/38df9393c9a9_init.py @@ -14,7 +14,7 @@ from alembic_migration.extensions import drop_enum import geoalchemy2 from sqlalchemy.dialects import postgresql -from sqlalchemy.sql.sqltypes import Enum +from sqlalchemy import Enum # revision identifiers, used by Alembic. revision = '38df9393c9a9' @@ -446,7 +446,7 @@ # https://github.com/discourse/discourse/blob/master/app/models/username_validator.rb function_check_forum_username = extensions.ReplaceableObject( 'users.check_forum_username(name TEXT)', - """ + r""" RETURNS boolean AS $$ BEGIN IF name = NULL THEN @@ -702,8 +702,8 @@ def upgrade(): op.create_table('documents_geometries', sa.Column('version', sa.Integer(), nullable=False), sa.Column('document_id', sa.Integer(), nullable=False), - sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, management=True), nullable=True), - sa.Column('geom_detail', geoalchemy2.types.Geometry(srid=3857, management=True, use_typmod=False), nullable=True), + sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, spatial_index=False), nullable=True), + sa.Column('geom_detail', geoalchemy2.types.Geometry(srid=3857, spatial_index=False, use_typmod=False), nullable=True), sa.ForeignKeyConstraint(['document_id'], ['guidebook.documents.document_id'], ), sa.PrimaryKeyConstraint('document_id'), schema='guidebook' @@ -712,8 +712,8 @@ def upgrade(): sa.Column('version', sa.Integer(), nullable=False), sa.Column('id', sa.Integer(), nullable=False), sa.Column('document_id', sa.Integer(), nullable=False), - sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, spatial_index=False, management=True), nullable=True), - sa.Column('geom_detail', geoalchemy2.types.Geometry(srid=3857, spatial_index=False, management=True, use_typmod=False), nullable=True), + sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, spatial_index=False), nullable=True), + sa.Column('geom_detail', geoalchemy2.types.Geometry(srid=3857, spatial_index=False, use_typmod=False), nullable=True), sa.ForeignKeyConstraint(['document_id'], ['guidebook.documents.document_id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('version', 'document_id', name='uq_documents_geometries_archives_document_id_version_lang'), diff --git a/alembic_migration/versions/8e0f02982746_update_avalanche_slopes.py b/alembic_migration/versions/8e0f02982746_update_avalanche_slopes.py index 131c1a99a..672fecdf0 100644 --- a/alembic_migration/versions/8e0f02982746_update_avalanche_slopes.py +++ b/alembic_migration/versions/8e0f02982746_update_avalanche_slopes.py @@ -8,7 +8,7 @@ from alembic import op import sqlalchemy as sa from alembic_migration.extensions import drop_enum -from sqlalchemy.sql.schema import Table, MetaData +from sqlalchemy import Table, MetaData # revision identifiers, used by Alembic. diff --git a/alembic_migration/versions/bacd59c5806a_add_slacklining.py b/alembic_migration/versions/bacd59c5806a_add_slacklining.py index d21f80a58..b2239935d 100644 --- a/alembic_migration/versions/bacd59c5806a_add_slacklining.py +++ b/alembic_migration/versions/bacd59c5806a_add_slacklining.py @@ -9,7 +9,7 @@ import sqlalchemy as sa from c2corg_api.models import utils from alembic_migration.extensions import drop_enum -from sqlalchemy.sql.sqltypes import Enum +from sqlalchemy import Enum # revision identifiers, used by Alembic. revision = 'bacd59c5806a' diff --git a/alembic_migration/versions/bb61456d557f_create_stops_and_waypoints_stops.py b/alembic_migration/versions/bb61456d557f_create_stops_and_waypoints_stops.py index 8489b32d7..46f3f86d4 100644 --- a/alembic_migration/versions/bb61456d557f_create_stops_and_waypoints_stops.py +++ b/alembic_migration/versions/bb61456d557f_create_stops_and_waypoints_stops.py @@ -15,6 +15,7 @@ branch_labels = None depends_on = None + def upgrade(): # stopareas op.create_table('stopareas', @@ -23,7 +24,7 @@ def upgrade(): sa.Column('stoparea_name', sa.String(), nullable=False), sa.Column('line', sa.String(), nullable=False), sa.Column('operator', sa.String(), nullable=False), - sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, management=True), nullable=True), + sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, spatial_index=False), nullable=True), schema='guidebook' ) @@ -41,7 +42,6 @@ def upgrade(): ) - def downgrade(): op.drop_table('waypoints_stopareas', schema='guidebook') op.drop_table('stopareas', schema='guidebook') diff --git a/alembic_migration/versions/d4d360ef69bc_update_public_transportation_types.py b/alembic_migration/versions/d4d360ef69bc_update_public_transportation_types.py index b5761a847..2b1105c50 100644 --- a/alembic_migration/versions/d4d360ef69bc_update_public_transportation_types.py +++ b/alembic_migration/versions/d4d360ef69bc_update_public_transportation_types.py @@ -5,54 +5,58 @@ Create Date: 2017-07-10 16:17:32.295441 """ -from alembic import op + import sqlalchemy as sa +from alembic import op +from sqlalchemy import MetaData, Table, and_, any_, cast, func, text from alembic_migration.extensions import drop_enum -from sqlalchemy.sql.expression import and_, any_, cast -from sqlalchemy.sql.functions import func -from sqlalchemy.sql.schema import Table, MetaData - from c2corg_api.models.utils import ArrayOfEnum - # revision identifiers, used by Alembic. revision = 'd4d360ef69bc' down_revision = '6d64a7fbdb8b' branch_labels = None depends_on = None + def upgrade(): old_options = ('train', 'bus', 'service_on_demand', 'boat', 'cable_car') new_options = ('train', 'bus', 'service_on_demand', 'boat') old_type = sa.Enum( - *old_options, name='public_transportation_type', schema='guidebook') + *old_options, name='public_transportation_type', schema='guidebook' + ) new_type = sa.Enum( - *new_options, name='public_transportation_type_', schema='guidebook') + *new_options, name='public_transportation_type_', schema='guidebook' + ) new_type.create(op.get_bind(), checkfirst=False) # Create new column with temporary name op.add_column( 'waypoints', sa.Column('public_transportation_types_', ArrayOfEnum(new_type)), - schema='guidebook') + schema='guidebook', + ) op.add_column( 'waypoints_archives', sa.Column('public_transportation_types_', ArrayOfEnum(new_type)), - schema='guidebook') + schema='guidebook', + ) # Create temporary string col for array types converting op.add_column( 'waypoints', sa.Column('public_transportation_types_str', sa.String), - schema='guidebook') + schema='guidebook', + ) op.add_column( 'waypoints_archives', sa.Column('public_transportation_types_str', sa.String), - schema='guidebook') + schema='guidebook', + ) waypoints = Table( 'waypoints', @@ -61,41 +65,61 @@ def upgrade(): sa.Column('public_transportation_types', ArrayOfEnum(old_type)), sa.Column('public_transportation_types_', ArrayOfEnum(new_type)), sa.Column('public_transportation_types_str', sa.String), - schema='guidebook') - # For waypoints having 'cable_car' in public_transportation_types: + schema='guidebook', + ) + # For waypoints having 'cable_car' in PublicTransportationTypes: # * set the lift_access flag to true # * remove 'cable_car' from public_transportation_types op.execute( - waypoints.update(). \ - where(and_( - waypoints.c.public_transportation_types != None, - any_(waypoints.c.public_transportation_types)==op.inline_literal('cable_car') - )). \ - values({ - 'lift_access': True, - 'public_transportation_types': func.array_remove( - waypoints.c.public_transportation_types, 'cable_car') - }) + waypoints.update() + .where( + and_( + waypoints.c.public_transportation_types != None, + any_(waypoints.c.public_transportation_types) + == op.inline_literal('cable_car'), + ) + ) + .values({'lift_access': True}) + ) + op.execute( + text( + 'UPDATE guidebook.waypoints' + ' SET public_transportation_types =' + ' array_remove(' + " public_transportation_types::text[], 'cable_car'" + ' )::guidebook.public_transportation_type[]' + ' WHERE public_transportation_types IS NOT NULL' + " AND 'cable_car' = ANY(public_transportation_types)" + ) ) # Then fill the new public_transportation_types_ col with values - # from public_transportation_types. The intermediary string conversion + # from PublicTransportationTypes. The intermediary string conversion # is needed because it is not possible to directly cast from one array # type to another. op.execute( - waypoints.update(). \ - where(waypoints.c.public_transportation_types != None). \ - values({ - 'public_transportation_types_str': func.array_to_string( - waypoints.c.public_transportation_types, ',') - }) + waypoints.update() + .where(waypoints.c.public_transportation_types != None) + .values( + { + 'public_transportation_types_str': func.array_to_string( + waypoints.c.public_transportation_types, ',' + ) + } + ) ) op.execute( - waypoints.update(). \ - where(waypoints.c.public_transportation_types != None). \ - values({ - 'public_transportation_types_': cast(func.string_to_array( - waypoints.c.public_transportation_types_str, ','), ArrayOfEnum(new_type)) - }) + waypoints.update() + .where(waypoints.c.public_transportation_types != None) + .values( + { + 'public_transportation_types_': cast( + func.string_to_array( + waypoints.c.public_transportation_types_str, ',' + ), + ArrayOfEnum(new_type), + ) + } + ) ) # Same for archives data @@ -106,58 +130,86 @@ def upgrade(): sa.Column('public_transportation_types', ArrayOfEnum(old_type)), sa.Column('public_transportation_types_', ArrayOfEnum(new_type)), sa.Column('public_transportation_types_str', sa.String), - schema='guidebook') + schema='guidebook', + ) + op.execute( + archives.update() + .where( + and_( + archives.c.public_transportation_types != None, + any_(archives.c.public_transportation_types) + == op.inline_literal('cable_car'), + ) + ) + .values({'lift_access': True}) + ) op.execute( - archives.update(). \ - where(and_( - archives.c.public_transportation_types != None, - any_(archives.c.public_transportation_types)==op.inline_literal('cable_car') - )). \ - values({ - 'lift_access': True, - 'public_transportation_types': func.array_remove( - archives.c.public_transportation_types, 'cable_car') - }) + text( + 'UPDATE guidebook.waypoints_archives' + ' SET public_transportation_types =' + ' array_remove(' + " public_transportation_types::text[], 'cable_car'" + ' )::guidebook.public_transportation_type[]' + ' WHERE public_transportation_types IS NOT NULL' + " AND 'cable_car' = ANY(public_transportation_types)" + ) ) op.execute( - archives.update(). \ - where(archives.c.public_transportation_types != None). \ - values({ - 'public_transportation_types_str': func.array_to_string( - archives.c.public_transportation_types, ',') - }) + archives.update() + .where(archives.c.public_transportation_types != None) + .values( + { + 'public_transportation_types_str': func.array_to_string( + archives.c.public_transportation_types, ',' + ) + } + ) ) op.execute( - archives.update(). \ - where(archives.c.public_transportation_types != None). \ - values({ - 'public_transportation_types_': cast(func.string_to_array( - archives.c.public_transportation_types_str, ','), ArrayOfEnum(new_type)) - }) + archives.update() + .where(archives.c.public_transportation_types != None) + .values( + { + 'public_transportation_types_': cast( + func.string_to_array( + archives.c.public_transportation_types_str, ',' + ), + ArrayOfEnum(new_type), + ) + } + ) ) # Drop old column and enum op.drop_column('waypoints', 'public_transportation_types', schema='guidebook') - op.drop_column('waypoints_archives', 'public_transportation_types', schema='guidebook') + op.drop_column( + 'waypoints_archives', 'public_transportation_types', schema='guidebook' + ) op.drop_column('waypoints', 'public_transportation_types_str', schema='guidebook') - op.drop_column('waypoints_archives', 'public_transportation_types_str', schema='guidebook') + op.drop_column( + 'waypoints_archives', 'public_transportation_types_str', schema='guidebook' + ) drop_enum('public_transportation_type', schema='guidebook') # Rename enum - op.execute('ALTER TYPE guidebook.public_transportation_type_ RENAME TO public_transportation_type') + op.execute( + 'ALTER TYPE guidebook.public_transportation_type_ RENAME TO public_transportation_type' + ) # Rename column op.alter_column( 'waypoints', 'public_transportation_types_', new_column_name='public_transportation_types', - schema='guidebook') + schema='guidebook', + ) op.alter_column( 'waypoints_archives', 'public_transportation_types_', new_column_name='public_transportation_types', - schema='guidebook') + schema='guidebook', + ) def downgrade(): diff --git a/c2corg_api/__init__.py b/c2corg_api/__init__.py index 5df001aac..b88652271 100644 --- a/c2corg_api/__init__.py +++ b/c2corg_api/__init__.py @@ -1,83 +1,22 @@ import logging import os -import requests -from c2corg_api.caching import configure_caches -from c2corg_api.security.acl import ACLDefault -from pyramid.config import Configurator -from sqlalchemy import engine_from_config, exc, event +import requests +from sqlalchemy import engine_from_config, event, exc, text from sqlalchemy.pool import Pool -from sqlalchemy import text from c2corg_api.models.document import DocumentGeometry from c2corg_api.models.route import Route -from c2corg_api.models import DBSession, Base -from c2corg_api.search import configure_es_from_config, get_queue_config - -from pyramid.settings import asbool - log = logging.getLogger(__name__) -class RootFactory(ACLDefault): - __name__ = "RootFactory" - - -def main(global_config, **settings): - """This function returns a Pyramid WSGI application.""" - - # Configure SQLAlchemy - engine = engine_from_config(settings, "sqlalchemy.") - DBSession.configure(bind=engine) - Base.metadata.bind = engine - - # Configure ElasticSearch - configure_es_from_config(settings) - - config = Configurator(settings=settings) - config.include("cornice") - config.registry.queue_config = get_queue_config(settings) - - # FIXME? Make sure this tween is run after the JWT validation - # Using an explicit ordering in config files might be needed. - config.add_tween( - "c2corg_api.tweens.rate_limiting." + "rate_limiting_tween_factory", - under="pyramid_tm.tm_tween_factory", - ) - - bypass_auth = False - if "noauthorization" in settings: - bypass_auth = asbool(settings["noauthorization"]) - - if not bypass_auth: - config.include("pyramid_jwtauth") - # Intercept request handling to validate token against the database - config.add_tween( - "c2corg_api.tweens.jwt_database_validation." - + "jwt_database_validation_tween_factory" - ) - # Inject ACLs - config.set_root_factory(RootFactory) - else: - log.warning("Bypassing authorization") - - configure_caches(settings) - configure_feed(settings, config) - configure_anonymous(settings, config) - - # Scan MUST be the last call otherwise ACLs will not be set - # and the permissions would be bypassed - config.scan(ignore="c2corg_api.tests") - return config.make_wsgi_app() - - # validate db connection before using it -@event.listens_for(Pool, "checkout") +@event.listens_for(Pool, 'checkout') def ping_connection(dbapi_connection, connection_record, connection_proxy): cursor = dbapi_connection.cursor() try: - cursor.execute("SELECT 1") + cursor.execute('SELECT 1') except Exception: # raise DisconnectionError - pool will try # connecting again up to three times before raising. @@ -85,22 +24,6 @@ def ping_connection(dbapi_connection, connection_record, connection_proxy): cursor.close() -def configure_feed(settings, config): - account_id = None - - if settings.get("feed.admin_user_account"): - account_id = int(settings.get("feed.admin_user_account")) - config.registry.feed_admin_user_account_id = account_id - - -def configure_anonymous(settings, config): - account_id = None - - if settings.get("guidebook.anonymous_user_account"): - account_id = int(settings.get("guidebook.anonymous_user_account")) - config.registry.anonymous_user_id = account_id - - def delete_waypoint_stopareas(connection, waypoint_id): # Delete existing stopareas for waypoint delete_relation_query = text( @@ -110,16 +33,11 @@ def delete_waypoint_stopareas(connection, waypoint_id): """ ) - connection.execute( - delete_relation_query, - { - "waypoint_id": waypoint_id, - }, - ) + connection.execute(delete_relation_query, {'waypoint_id': waypoint_id}) -@event.listens_for(DocumentGeometry, "after_insert") -@event.listens_for(DocumentGeometry, "after_update") +@event.listens_for(DocumentGeometry, 'after_insert') +@event.listens_for(DocumentGeometry, 'after_update') def process_new_waypoint(mapper, connection, geometry): """Processes a new waypoint to find its public transports after inserting it into documents_geometries.""" @@ -133,21 +51,19 @@ def process_new_waypoint(mapper, connection, geometry): WHERE document_id = :waypoint_id """ ), - {"waypoint_id": waypoint_id}, + {'waypoint_id': waypoint_id}, ).scalar() - if document_type != "w": + if document_type != 'w': return - log.debug("Entering process_new_waypoint callback") + log.debug('Entering process_new_waypoint callback') max_distance_waypoint_to_stoparea = int( - os.getenv("MAX_DISTANCE_WAYPOINT_TO_STOPAREA") + os.getenv('MAX_DISTANCE_WAYPOINT_TO_STOPAREA') ) - walking_speed = float(os.getenv("WALKING_SPEED")) - max_stop_area_for_1_waypoint = int( - os.getenv("MAX_STOP_AREA_FOR_1_WAYPOINT") - ) - api_key = os.getenv("NAVITIA_API_KEY") + walking_speed = float(os.getenv('WALKING_SPEED')) + max_stop_area_for_1_waypoint = int(os.getenv('MAX_STOP_AREA_FOR_1_WAYPOINT')) + api_key = os.getenv('NAVITIA_API_KEY') max_duration = int(max_distance_waypoint_to_stoparea / walking_speed) # increase number of retrieved stop areas, @@ -161,13 +77,13 @@ def process_new_waypoint(mapper, connection, geometry): WHERE document_id = :waypoint_id """ ), - {"waypoint_id": waypoint_id}, + {'waypoint_id': waypoint_id}, ).scalar() - if waypoint_type != "access": + if waypoint_type != 'access': return - log.info("Waypoint Navitia processing with ID: %d", waypoint_id) + log.info('Waypoint Navitia processing with ID: %d', waypoint_id) # Get waypoint coordinates lon_lat = connection.execute( @@ -180,33 +96,35 @@ def process_new_waypoint(mapper, connection, geometry): WHERE document_id = :waypoint_id """ ), - {"waypoint_id": waypoint_id}, + {'waypoint_id': waypoint_id}, ).scalar() if not lon_lat: - log.warning("Coordinates not found for waypoint %d", waypoint_id) + log.warning('Coordinates not found for waypoint %d', waypoint_id) return - lon, lat = lon_lat.strip().split(",") + lon, lat = lon_lat.strip().split(',') # Navitia request - get more stop areas to filter - places_url = "https://api.navitia.io/v1/coord/%s;%s/places_nearby" % ( - lon, lat) + places_url = 'https://api.navitia.io/v1/coord/%s;%s/places_nearby' % (lon, lat) places_params = { - "type[]": "stop_area", - "count": max_stop_area_fetched, # Plus d'arrêts récupérés - "distance": max_distance_waypoint_to_stoparea, + 'type[]': 'stop_area', + 'count': max_stop_area_fetched, # Plus d'arrêts récupérés + 'distance': max_distance_waypoint_to_stoparea, } - navitia_headers = {"Authorization": api_key} + navitia_headers = {'Authorization': api_key} places_response = requests.get( places_url, headers=navitia_headers, params=places_params ) places_data = places_response.json() - if "places_nearby" not in places_data or not places_data["places_nearby"]: - log.info("No Navitia stops found for the waypoint %d; \ - deleting previously registered stops", waypoint_id) + if 'places_nearby' not in places_data or not places_data['places_nearby']: + log.info( + 'No Navitia stops found for the waypoint %d; \ + deleting previously registered stops', + waypoint_id, + ) delete_waypoint_stopareas(connection, waypoint_id) return @@ -216,31 +134,29 @@ def process_new_waypoint(mapper, connection, geometry): selected_count = 0 # Treat stopareas in order (already sorted by distance using Navitia) - for place in places_data["places_nearby"]: - if place.get("embedded_type") != "stop_area": + for place in places_data['places_nearby']: + if place.get('embedded_type') != 'stop_area': continue if selected_count >= max_stop_area_for_1_waypoint: break - stop_id = place["id"] + stop_id = place['id'] # Get informations of stopareas to know its transports - stop_info_url = f"https://api.navitia.io/v1/places/{stop_id}" - stop_info_response = requests.get( - stop_info_url, headers=navitia_headers - ) + stop_info_url = f'https://api.navitia.io/v1/places/{stop_id}' + stop_info_response = requests.get(stop_info_url, headers=navitia_headers) stop_info = stop_info_response.json() - if "places" not in stop_info or not stop_info["places"]: + if 'places' not in stop_info or not stop_info['places']: continue # Extract transports from its stoparea current_stop_transports = set() - for line in stop_info["places"][0]["stop_area"].get("lines", []): - mode = line.get("commercial_mode", {}).get("name", "") - code = line.get("code", "") - transport_key = "%s %s", mode, code + for line in stop_info['places'][0]['stop_area'].get('lines', []): + mode = line.get('commercial_mode', {}).get('name', '') + code = line.get('code', '') + transport_key = '%s %s', mode, code current_stop_transports.add(transport_key) # Check if this stoparea can bring new transports @@ -248,36 +164,37 @@ def process_new_waypoint(mapper, connection, geometry): # If stoparea brings at least one new transport, select it if new_transport_found: - place["stop_info"] = stop_info + place['stop_info'] = stop_info selected_stops.append(place) known_transports.update(current_stop_transports) selected_count += 1 log.info( - "Selected %d stops out of %d for waypoint %d", + 'Selected %d stops out of %d for waypoint %d', selected_count, - len(places_data['places_nearby']), waypoint_id + len(places_data['places_nearby']), + waypoint_id, ) - log.info("Deleting previously registered stops") + log.info('Deleting previously registered stops') delete_waypoint_stopareas(connection, waypoint_id) # Only treat selected stopareas for place in selected_stops: - stop_id = place["id"] - stop_name = place["name"] - lat_stop = place["stop_area"]["coord"]["lat"] - lon_stop = place["stop_area"]["coord"]["lon"] + stop_id = place['id'] + stop_name = place['name'] + lat_stop = place['stop_area']['coord']['lat'] + lon_stop = place['stop_area']['coord']['lon'] # Get the travel time by walking - use same parameters as bash script - journey_url = "https://api.navitia.io/v1/journeys" + journey_url = 'https://api.navitia.io/v1/journeys' journey_params = { - "to": "%s;%s" % (lon, lat), - "walking_speed": walking_speed, - "max_walking_direct_path_duration": max_duration, - "direct_path_mode[]": "walking", - "from": stop_id, - "direct_path": "only_with_alternatives", + 'to': '%s;%s' % (lon, lat), + 'walking_speed': walking_speed, + 'max_walking_direct_path_duration': max_duration, + 'direct_path_mode[]': 'walking', + 'from': stop_id, + 'direct_path': 'only_with_alternatives', } journey_response = requests.get( @@ -285,14 +202,14 @@ def process_new_waypoint(mapper, connection, geometry): ) journey_data = journey_response.json() - if "error" in journey_data: + if 'error' in journey_data: continue # Get the walk duration - if "journeys" not in journey_data or not journey_data["journeys"]: + if 'journeys' not in journey_data or not journey_data['journeys']: continue - duration = journey_data["journeys"][0].get("duration", 0) + duration = journey_data['journeys'][0].get('duration', 0) # Convert to distance distance_km = round((duration * walking_speed) / 1000, 2) @@ -305,17 +222,17 @@ def process_new_waypoint(mapper, connection, geometry): """ ) existing_stop_id = connection.execute( - existing_stop_query, {"stop_id": stop_id} + existing_stop_query, {'stop_id': stop_id} ).scalar() if not existing_stop_id: - stop_info = place["stop_info"] + stop_info = place['stop_info'] - for line in stop_info["places"][0]["stop_area"].get("lines", []): - line_full_name = line.get("name", "") - line_name = line.get("code", "") - operator_name = line.get("network", {}).get("name", "") - mode = line.get("commercial_mode", {}).get("name", "") + for line in stop_info['places'][0]['stop_area'].get('lines', []): + line_full_name = line.get('name', '') + line_name = line.get('code', '') + operator_name = line.get('network', {}).get('name', '') + mode = line.get('commercial_mode', {}).get('name', '') # Create a new stop and its relation with waypoint insert_stoparea_query = text( @@ -345,15 +262,14 @@ def process_new_waypoint(mapper, connection, geometry): connection.execute( insert_stoparea_query, { - "stop_id": stop_id, - "stop_name": stop_name, - "line": "%s %s - %s" % - (mode, line_name, line_full_name), - "operator": operator_name, - "lon_stop": lon_stop, - "lat_stop": lat_stop, - "waypoint_id": waypoint_id, - "distance_km": distance_km, + 'stop_id': stop_id, + 'stop_name': stop_name, + 'line': '%s %s - %s' % (mode, line_name, line_full_name), + 'operator': operator_name, + 'lon_stop': lon_stop, + 'lat_stop': lat_stop, + 'waypoint_id': waypoint_id, + 'distance_km': distance_km, }, ) else: @@ -369,19 +285,20 @@ def process_new_waypoint(mapper, connection, geometry): connection.execute( insert_relation_query, { - "stoparea_id": existing_stop_id, - "waypoint_id": waypoint_id, - "distance_km": distance_km, + 'stoparea_id': existing_stop_id, + 'waypoint_id': waypoint_id, + 'distance_km': distance_km, }, ) - log.info("Traitement terminé pour le waypoint %d", waypoint_id) + log.info('Traitement terminé pour le waypoint %d', waypoint_id) + # pylint: disable=too-complex,too-many-branches,too-many-statements -@event.listens_for(Route, "after_insert") -@event.listens_for(Route, "after_update") +@event.listens_for(Route, 'after_insert') +@event.listens_for(Route, 'after_update') def calculate_route_duration(mapper, connection, route): """ Calcule la durée estimée d'un itinéraire @@ -390,7 +307,7 @@ def calculate_route_duration(mapper, connection, route): jour du script bash. """ route_id = route.document_id - log.info("Calculating duration for route ID: %d", route_id) + log.info('Calculating duration for route ID: %d', route_id) # Récupération des activités et normalisation des dénivelés activities = route.activities if route.activities is not None else [] @@ -423,13 +340,13 @@ def _normalize_height_differences(route): def _get_climbing_activities(): """Retourne la liste des activités considérées comme grimpantes.""" return [ - "rock_climbing", - "ice_climbing", - "mountain_climbing", - "snow_ice_mixed", - "via_ferrata", - "paragliding", - "slacklining", + 'rock_climbing', + 'ice_climbing', + 'mountain_climbing', + 'snow_ice_mixed', + 'via_ferrata', + 'paragliding', + 'slacklining', ] @@ -464,13 +381,11 @@ def _calculate_climbing_duration( """ v_diff = 50.0 # Climbing ascent speed for difficulties (m/h) - h = float( - route.route_length if route.route_length is not None else 0 - ) / 1000 # km + h = float(route.route_length if route.route_length is not None else 0) / 1000 # km dp = float(height_diff_up if height_diff_up is not None else 0) # m dn = float(height_diff_down if height_diff_down is not None else 0) # m - difficulties_height = getattr(route, "height_diff_difficulties", None) + difficulties_height = getattr(route, 'height_diff_difficulties', None) # CASE 1: difficulties height is not provided if difficulties_height is None or difficulties_height <= 0: @@ -480,9 +395,11 @@ def _calculate_climbing_duration( dm = dp / v_diff log.info( - "Calculated climbing route duration for route %d \ - (activity %s, no difficulties_height): %0.2f hours", - route_id, activity, dm + 'Calculated climbing route duration for route %d \ + (activity %s, no difficulties_height): %0.2f hours', + route_id, + activity, + dm, ) return dm @@ -491,9 +408,13 @@ def _calculate_climbing_duration( # Consistency check if dp > 0 and d_diff > dp: - log.info("Route %d: Inconsistent difficulties_height (%fm) > \ - height_diff_up ( %fm). Returning NULL.", route_id, d_diff, dp - ) + log.info( + 'Route %d: Inconsistent difficulties_height (%fm) > \ + height_diff_up ( %fm). Returning NULL.', + route_id, + d_diff, + dp, + ) return None # Compute time for difficulties @@ -512,9 +433,10 @@ def _calculate_climbing_duration( # max(t\_diff, t\_app) + 0.5 * min(t\_diff, t\_app) dm = max(t_diff, t_app) + 0.5 * min(t_diff, t_app) - log.info("Calculated climbing route duration for route \ - %d (activity %s): %.2f hours (t_diff=%.2f, t_app=%.2f)" % ( - route_id, activity, dm, t_diff, t_app) + log.info( + 'Calculated climbing route duration for route \ + %d (activity %s): %.2f hours (t_diff=%.2f, t_app=%.2f)' + % (route_id, activity, dm, t_diff, t_app) ) return dm @@ -522,7 +444,7 @@ def _calculate_climbing_duration( def _calculate_approach_time(h, d_app, dn): """Calculate the approach time for climbing using the DIN 33466 formula.""" # Parameters for the approach (hiking) - v = 5.0 # km/h (horizontal speed) + v = 5.0 # km/h (horizontal speed) a = 300.0 # m/h (ascent rate) d = 500.0 # m/h (descent rate) @@ -543,10 +465,10 @@ def _calculate_approach_time(h, d_app, dn): def _get_activity_parameters(activity): """Return speed parameters for the activity.""" parameters = { - "hiking": (5.0, 300.0, 500.0), - "snowshoeing": (4.5, 250.0, 400.0), - "skitouring": (5.0, 300.0, 1500.0), - "mountain_biking": (15.0, 250.0, 1000.0), + 'hiking': (5.0, 300.0, 500.0), + 'snowshoeing': (4.5, 250.0, 400.0), + 'skitouring': (5.0, 300.0, 1500.0), + 'mountain_biking': (15.0, 250.0, 1000.0), } return parameters.get(activity, (5.0, 300.0, 500.0)) # default values @@ -555,12 +477,10 @@ def _calculate_standard_duration( activity, route, height_diff_up, height_diff_down, route_id ): """Calculate duration for standard (non-climbing) activities - according to DIN 33466.""" + according to DIN 33466.""" v, a, d = _get_activity_parameters(activity) - h = float( - route.route_length if route.route_length is not None else 0 - ) / 1000 # km + h = float(route.route_length if route.route_length is not None else 0) / 1000 # km dp = float(height_diff_up if height_diff_up is not None else 0) # m dn = float(height_diff_down if height_diff_down is not None else 0) # m @@ -574,9 +494,10 @@ def _calculate_standard_duration( dm = (dv / 2) + dh log.info( - "Calculated standard route duration for route %s \ - (activity %s): %.2f hours" - % (route_id, activity, dm)) + 'Calculated standard route duration for route %s \ + (activity %s): %.2f hours' + % (route_id, activity, dm) + ) return dm @@ -590,17 +511,14 @@ def _validate_and_convert_duration(min_duration, route_id): or min_duration < min_duration_hours or min_duration > max_duration_hours ): - if (min_duration is None): - min_duration_str = "None" + if min_duration is None: + min_duration_str = 'None' else: - min_duration_str = "%0.2f", min_duration + min_duration_str = '%0.2f', min_duration log.info( - "Route %d: Calculated duration (min_duration=%s) is out of bounds \ - (min=%fh, max=%fh) or NULL. Setting duration to NULL." % - (route_id, - min_duration_str, - min_duration_hours, - max_duration_hours) + 'Route %d: Calculated duration (min_duration=%s) is out of bounds \ + (min=%fh, max=%fh) or NULL. Setting duration to NULL.' + % (route_id, min_duration_str, min_duration_hours, max_duration_hours) ) return None @@ -617,14 +535,14 @@ def _update_route_duration(connection, route_id, calculated_duration_in_days): WHERE document_id = :route_id """ ), - {"duration": calculated_duration_in_days, "route_id": route_id}, + {'duration': calculated_duration_in_days, 'route_id': route_id}, ) - if (calculated_duration_in_days is None): - calculated_duration_in_days_str = "None" + if calculated_duration_in_days is None: + calculated_duration_in_days_str = 'None' else: - calculated_duration_in_days_str = "%f", calculated_duration_in_days + calculated_duration_in_days_str = '%f', calculated_duration_in_days log.info( - "Route %d: Database updated with calculated_duration = %s days.", + 'Route %d: Database updated with calculated_duration = %s days.', route_id, - calculated_duration_in_days_str + calculated_duration_in_days_str, ) diff --git a/c2corg_api/app.py b/c2corg_api/app.py new file mode 100644 index 000000000..b75f97e52 --- /dev/null +++ b/c2corg_api/app.py @@ -0,0 +1,414 @@ +""" +FastAPI application entry-point. + +Usage (development) +------------------- +:: + + uvicorn c2corg_api.app:get_app --factory --host 0.0.0.0 --port 6543 --reload + +Or set ``C2CORG_INI`` to point at a different ``.ini`` file. +""" + +import logging +import os +from configparser import ConfigParser, ExtendedInterpolation +from pathlib import Path + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse + +log = logging.getLogger(__name__) + + +def _load_settings(ini_file: str) -> dict[str, str]: + """Parse a Paste-Deploy ``.ini`` file and return the ``[app:main]`` + settings dict.. + """ + ini_path = Path(ini_file) + + parser = ConfigParser(interpolation=ExtendedInterpolation()) + + # Explicit inheritance instead of PasteDeploy + common_ini = ini_path.parent / 'common.ini' + parser.read([common_ini, ini_path]) + + settings = dict(parser['app:main']) + + # Explicitly drop PasteDeploy-only key + settings.pop('use', None) + + return settings + + +def create_app(*, engine=None) -> FastAPI: + """Application factory called once at startup. + + Parameters + ---------- + engine : sqlalchemy.engine.Engine, optional + Pre-existing SQLAlchemy engine to reuse (e.g. from the test + harness). When *None* a new engine is created from the + ``.ini`` settings. + """ + + # --- resolve the .ini file ------------------------------------------- + ini_file = os.environ.get('C2CORG_INI', 'development.ini') + log.info('Loading settings from %s', ini_file) + settings = _load_settings(ini_file) + + # --- FastAPI app ----------------------------------------------------- + fastapi_app = FastAPI( + title='c2corg API', + description='Camptocamp.org API – FastAPI + Pyramid transitional', + version='7.0.0-dev', + ) + + fastapi_app.add_middleware( + CORSMiddleware, + allow_origins=['*'], + allow_methods=['*'], + allow_headers=['Content-Type'], + ) + + @fastapi_app.exception_handler(HTTPException) + async def _http_exception_handler( + request: Request, exc: HTTPException + ) -> JSONResponse: + """Normalise FastAPI ``HTTPException`` responses to the Cornice + error format that the UI relies on:: + + {"status": "error", + "errors": [{"location": …, "name": …, + "description": …}]} + + ``detail`` may be: + - a dict with ``'errors'`` key → use as-is (already Cornice-shaped) + - a plain string → wrap in a single-element errors list + """ + detail = exc.detail + if isinstance(detail, dict) and 'errors' in detail: + content = {'status': 'error', 'errors': detail['errors']} + elif isinstance(detail, str): + content = { + 'status': 'error', + 'errors': [ + {'location': 'body', 'name': 'Bad Request', 'description': detail} + ], + } + else: + content = { + 'status': 'error', + 'errors': [ + { + 'location': 'body', + 'name': 'Bad Request', + 'description': str(detail), + } + ], + } + return JSONResponse(status_code=exc.status_code, content=content) + + @fastapi_app.exception_handler(Exception) + async def _unhandled_exception_handler( + request: Request, exc: Exception + ) -> JSONResponse: + """Return a proper JSON response for unhandled exceptions. + + Without this, Starlette emits a bare-bones plain-text 500 + that bypasses ``CORSMiddleware``'s response hook — meaning + the browser hides the error from JavaScript SPA clients. + + We explicitly set the ``Access-Control-Allow-Origin`` + header because Starlette's ``ServerErrorMiddleware`` + intercepts 500 responses before ``CORSMiddleware`` can + decorate them. + + """ + log.error( + 'Unhandled exception on %s %s: %s', + request.method, + request.url.path, + exc, + exc_info=True, + ) + origin = request.headers.get('origin') + headers = {'Access-Control-Allow-Origin': '*'} if origin else {} + return JSONResponse( + status_code=500, + content={'detail': 'Internal server error'}, + headers=headers, + ) + + @fastapi_app.exception_handler(RequestValidationError) + async def _validation_error_handler( + request: Request, exc: RequestValidationError + ) -> JSONResponse: + """Return 400 instead of FastAPI's default 422. + + Colander/Cornice (the previous validation layer) returned 400 + for malformed request bodies. Returning 400 preserves backward + compatibility for existing API consumers. + + The response body follows the same Cornice error format used + throughout the API. + """ + errors = [] + seen_fields = set() + for error in exc.errors(): + loc = error.get('loc', ()) + # Build a dotted field path, skipping the leading 'body' + # segment that Pydantic injects for JSON body errors. + parts = [p for p in loc if p != 'body'] + + # When a scalar item inside a list fails validation + # (e.g. an invalid enum value at activities[1]), pydantic + # reports the loc as ('activities', 1). Collapse such + # paths to just the parent field name ('activities') so + # that API consumers can match on the top-level field, + # matching the Cornice/Pyramid pydantic validator behaviour. + if len(parts) == 2 and isinstance(parts[1], int): + parts = parts[:1] + + field = '.'.join(str(p) for p in parts) if parts else 'body' + + # Avoid duplicate errors for the same collapsed field + if field in seen_fields: + continue + seen_fields.add(field) + + msg = error.get('msg', '') + if msg == 'Field required': + msg = 'Required' + if msg.startswith('Value error, '): + msg = msg[len('Value error, ') :] + + errors.append({'location': 'body', 'name': field, 'description': msg}) + + return JSONResponse( + status_code=400, content={'status': 'error', 'errors': errors} + ) + + # --- shared database engine ------------------------------------------ + if engine is None: + from sqlalchemy import engine_from_config + + engine = engine_from_config(settings, 'sqlalchemy.') + + from c2corg_api.database import configure_db + + configure_db(engine) + + # --- dogpile.cache (Redis) ------------------------------------------- + from c2corg_api.caching import configure_caches + + configure_caches(settings) + + # --- anonymous user for xreports ------------------------------------ + from c2corg_api.routers.helpers.document_crud import configure_anonymous + + configure_anonymous(settings) + + # --- FastAPI routers ------------------------------------------------- + from c2corg_api.routers.book import router as book_router + + fastapi_app.include_router(book_router) + + from c2corg_api.routers.article import router as article_router + + fastapi_app.include_router(article_router) + + from c2corg_api.routers.xreport import router as xreport_router + + fastapi_app.include_router(xreport_router) + + from c2corg_api.routers.image import router as image_router + + fastapi_app.include_router(image_router) + + from c2corg_api.routers.route import router as route_router + + fastapi_app.include_router(route_router) + + from c2corg_api.routers.area import router as area_router + + fastapi_app.include_router(area_router) + + from c2corg_api.routers.coverage import router as coverage_router + + fastapi_app.include_router(coverage_router) + + from c2corg_api.routers.outing import router as outing_router + + fastapi_app.include_router(outing_router) + + from c2corg_api.routers.topo_map import router as topo_map_router + + fastapi_app.include_router(topo_map_router) + + from c2corg_api.routers.waypoint import router as waypoint_router + + fastapi_app.include_router(waypoint_router) + + from c2corg_api.routers.user_profile import router as user_profile_router + + fastapi_app.include_router(user_profile_router) + + from c2corg_api.routers.health import configure_health + from c2corg_api.routers.health import router as health_router + + fastapi_app.include_router(health_router) + configure_health(settings) + + from c2corg_api.routers.association_history import ( + router as association_history_router, + ) + + fastapi_app.include_router(association_history_router) + + from c2corg_api.routers.association import configure_association_router + from c2corg_api.routers.association import router as association_router + + fastapi_app.include_router(association_router) + + from c2corg_api.routers.user_preferences import router as user_preferences_router + + fastapi_app.include_router(user_preferences_router) + + from c2corg_api.routers.cooker import router as cooker_router + + fastapi_app.include_router(cooker_router) + + from c2corg_api.routers.document_changes import router as document_changes_router + + fastapi_app.include_router(document_changes_router) + + from c2corg_api.routers.document_history import router as document_history_router + + fastapi_app.include_router(document_history_router) + + from c2corg_api.routers.document_protect import router as document_protect_router + + fastapi_app.include_router(document_protect_router) + + from c2corg_api.routers.document_tag import configure_tag_router + from c2corg_api.routers.document_tag import router as document_tag_router + + fastapi_app.include_router(document_tag_router) + + from c2corg_api.routers.document_version_mask import ( + router as document_version_mask_router, + ) + + fastapi_app.include_router(document_version_mask_router) + + from c2corg_api.routers.document_merge import configure_merge_router + from c2corg_api.routers.document_merge import router as document_merge_router + + fastapi_app.include_router(document_merge_router) + + from c2corg_api.routers.document_delete import configure_delete_router + from c2corg_api.routers.document_delete import router as document_delete_router + + fastapi_app.include_router(document_delete_router) + + from c2corg_api.routers.document_revert import router as document_revert_router + + fastapi_app.include_router(document_revert_router) + + from c2corg_api.routers.feed import configure_feed_router + from c2corg_api.routers.feed import router as feed_router + + fastapi_app.include_router(feed_router) + + from c2corg_api.routers.forum import configure_forum_router + from c2corg_api.routers.forum import router as forum_router + + fastapi_app.include_router(forum_router) + + from c2corg_api.routers.navitia import router as navitia_router + + fastapi_app.include_router(navitia_router) + + from c2corg_api.routers.search import router as search_router + + fastapi_app.include_router(search_router) + + from c2corg_api.routers.sitemap import router as sitemap_router + + fastapi_app.include_router(sitemap_router) + + from c2corg_api.routers.sitemap_xml import router as sitemap_xml_router + + fastapi_app.include_router(sitemap_xml_router) + + from c2corg_api.routers.sso import configure_sso_router + from c2corg_api.routers.sso import router as sso_router + + fastapi_app.include_router(sso_router) + + from c2corg_api.routers.user import configure_user_router + from c2corg_api.routers.user import router as user_router + + fastapi_app.include_router(user_router) + + from c2corg_api.routers.user_account import configure_user_account_router + from c2corg_api.routers.user_account import router as user_account_router + + fastapi_app.include_router(user_account_router) + + from c2corg_api.routers.user_block import configure_user_block_router + from c2corg_api.routers.user_block import router as user_block_router + + fastapi_app.include_router(user_block_router) + + from c2corg_api.routers.user_follow import router as user_follow_router + + fastapi_app.include_router(user_follow_router) + + from c2corg_api.routers.user_mailinglists import router as user_mailinglists_router + + fastapi_app.include_router(user_mailinglists_router) + + from c2corg_api.routers.stoparea import router as stoparea_router + + fastapi_app.include_router(stoparea_router) + + from c2corg_api.routers.waypoint_stoparea import router as waypoint_stoparea_router + + fastapi_app.include_router(waypoint_stoparea_router) + + # --- routers queue config --------------------------------- + from c2corg_api.search import get_queue_config + + queue_config = get_queue_config(settings) + configure_association_router(queue_config) + configure_tag_router(queue_config) + configure_merge_router(queue_config) + configure_delete_router(queue_config) + + configure_feed_router(settings) + configure_forum_router(settings) + configure_sso_router(settings) + configure_user_router(settings) + configure_user_account_router(settings) + configure_user_block_router(settings) + + log.info('FastAPI app ready') + return fastapi_app + + +# module-level instance so that ``uvicorn c2corg_api.app:app`` works. +# Guarded so that importing the module for tests or introspection +# does not immediately require a running database. +app: FastAPI | None = None + + +def get_app() -> FastAPI: + """Return the singleton FastAPI application, creating it on first call.""" + global app + if app is None: + app = create_app() + return app diff --git a/c2corg_api/caching.py b/c2corg_api/caching.py index 8005cdc04..0879c3dd1 100644 --- a/c2corg_api/caching.py +++ b/c2corg_api/caching.py @@ -42,7 +42,7 @@ def create_region(name): cache_document_version, cache_document_info, cache_sitemap, - cache_sitemap_xml + cache_sitemap_xml, ] @@ -61,8 +61,7 @@ def configure_caches(settings): log.debug('Cache version {0}'.format(CACHE_VERSION)) - redis_url = '{0}?db={1}'.format( - settings['redis.url'], settings['redis.db_cache']) + redis_url = '{0}?db={1}'.format(settings['redis.url'], settings['redis.db_cache']) log.debug('Cache Redis: {0}'.format(redis_url)) redis_pool = BlockingConnectionPool.from_url( @@ -70,7 +69,7 @@ def configure_caches(settings): max_connections=int(settings['redis.cache_pool']), socket_connect_timeout=float(settings['redis.socket_connect_timeout']), socket_timeout=float(settings['redis.socket_timeout']), - timeout=float(settings['redis.pool_timeout']) + timeout=float(settings['redis.pool_timeout']), ) for cache in caches: @@ -78,12 +77,12 @@ def configure_caches(settings): 'dogpile.cache.redis', arguments={ 'connection_pool': redis_pool, - "thread_local_lock": False, + 'thread_local_lock': False, 'distributed_lock': True, 'lock_timeout': 15, # 15 seconds (dogpile lock) - 'redis_expiration_time': int(settings['redis.expiration_time']) + 'redis_expiration_time': int(settings['redis.expiration_time']), }, - replace_existing_backend=True + replace_existing_backend=True, ) if settings.get('redis.cache_status_refresh_period'): @@ -99,7 +98,7 @@ def initialize_cache_status(refresh_period): def get_or_create(cache, key, creator): - """ Try to get the value for the given key from the cache. In case of + """Try to get the value for the given key from the cache. In case of errors fallback to the creator function (e.g. load from the database). """ if cache_status.is_down(): @@ -107,11 +106,10 @@ def get_or_create(cache, key, creator): return creator() try: - value = cache.get_or_create( - key, creator_wrapper(creator), expiration_time=-1) + value = cache.get_or_create(key, creator_wrapper(creator), expiration_time=-1) cache_status.request_success() return value - except CreatorException as creator_exception: + except CreatorError as creator_exception: raise creator_exception.exc except Exception: log.error('Getting value from cache failed', exc_info=True) @@ -120,7 +118,7 @@ def get_or_create(cache, key, creator): def get_or_create_multi(cache, keys, creator, should_cache_fn=None): - """ Try to get the values for the given keys from the cache. In case of + """Try to get the values for the given keys from the cache. In case of errors fallback to the creator function (e.g. load from the database). """ if cache_status.is_down(): @@ -129,11 +127,14 @@ def get_or_create_multi(cache, keys, creator, should_cache_fn=None): try: values = cache.get_or_create_multi( - keys, creator_wrapper(creator), expiration_time=-1, - should_cache_fn=should_cache_fn) + keys, + creator_wrapper(creator), + expiration_time=-1, + should_cache_fn=should_cache_fn, + ) cache_status.request_success() return values - except CreatorException as creator_exception: + except CreatorError as creator_exception: raise creator_exception.exc except Exception: log.error('Getting values from cache failed', exc_info=True) @@ -142,7 +143,7 @@ def get_or_create_multi(cache, keys, creator, should_cache_fn=None): def get(cache, key): - """ Try to get the value for the given key from the cache. In case of + """Try to get the value for the given key from the cache. In case of errors, return NO_VALUE. """ if cache_status.is_down(): @@ -160,7 +161,7 @@ def get(cache, key): def set(cache, key, value): - """ Try to set the value with the given key in the cache. In case of + """Try to set the value with the given key in the cache. In case of errors, log the error and continue. """ if cache_status.is_down(): @@ -175,29 +176,32 @@ def set(cache, key, value): cache_status.request_failure() -class CreatorException(Exception): - """ An exception happening during the execution of a cache `creator` +class CreatorError(Exception): + """An exception happening during the execution of a cache `creator` function. """ + def __init__(self, exc): self.exc = exc def creator_wrapper(creator): - """ A wrapper around `creator` functions. The purpose of this wrapper is + """A wrapper around `creator` functions. The purpose of this wrapper is to distinguish exceptions caused in creator functions from exceptions while trying to read a value from the cache. """ + def fn(*args, **kwargs): try: return creator(*args, **kwargs) except Exception as exc: - raise CreatorException(exc) + raise CreatorError(exc) + return fn class CacheStatus(object): - """ To avoid that requests are made to the cache if it is down, the status + """To avoid that requests are made to the cache if it is down, the status of the last requests is stored. If a request in the 30 seconds failed, no new request will be made. """ diff --git a/c2corg_api/database.py b/c2corg_api/database.py new file mode 100644 index 000000000..44a1b3f33 --- /dev/null +++ b/c2corg_api/database.py @@ -0,0 +1,68 @@ +""" +FastAPI database dependency. + +Provides a ``get_db`` dependency that yields a SQLAlchemy session +backed by the same ``DBSession`` factory that Pyramid uses, but +**without** ``zope.sqlalchemy`` transaction management. + +During the transitional period both stacks share the same engine; +only the session lifecycle differs: + - Pyramid → ``zope.sqlalchemy`` + ``pyramid_tm`` + - FastAPI → explicit commit / rollback in the dependency +""" + +from typing import Generator + +from sqlalchemy.orm import Session, scoped_session, sessionmaker + +# Will be configured at startup from the same engine as the Pyramid app. +_session_factory = sessionmaker() +FastAPISession: scoped_session = scoped_session(_session_factory) + + +def configure_db(engine) -> None: + """Bind the FastAPI session factory to *engine*. + + Called once at application startup. + """ + _session_factory.configure(bind=engine) + + +def get_db() -> Generator[Session, None, None]: + """Yields a SQLAlchemy session that is committed when the request + finishes successfully, or rolled back when an exception is raised. + + That way, if several database operations are performed in a single + request, they are all committed or rolled back together. + """ + db = FastAPISession() + try: + yield db + db.commit() + except Exception: + db.rollback() + raise + finally: + FastAPISession.remove() + + +# TODO: Drop scoped_session once Pyramid is removed. +# Using scoped_session gives you two benefits: + +# Same mental model as Pyramid +# Same failure modes you already understand +# Less refactoring while Pyramid still exists + +# But this is sync only +# SessionLocal = sessionmaker(bind=engine) + +# def get_db(): +# db = SessionLocal() +# try: +# yield db +# db.commit() +# except Exception: +# db.rollback() +# raise +# finally: +# db.close() diff --git a/c2corg_api/emails/email_service.py b/c2corg_api/emails/email_service.py index bcbce7e6d..65de21bc8 100644 --- a/c2corg_api/emails/email_service.py +++ b/c2corg_api/emails/email_service.py @@ -1,12 +1,13 @@ import transaction # NOQA +from typing import Optional from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart from functools import lru_cache from pyramid.settings import asbool import smtplib -from c2corg_api.models.common.attributes import default_langs +from c2corg_api.models.common.attributes import DefaultLangs import logging import os @@ -28,21 +29,17 @@ def _get_file_content(self, lang, key): return content def get_translation(self, lang, key): - if lang not in default_langs: + if lang not in DefaultLangs: raise Exception('Bad language' + lang) try: return self._get_file_content(lang, key) except Exception: - log.exception('The %s translation for %s could not be read' % ( - lang, key)) + log.exception('The %s translation for %s could not be read' % (lang, key)) return self._get_file_content('fr', key) -# See https://docs.python.org/3/library/smtplib.html#smtplib.SMTP.sendmail -# https://github.com/Pylons/pyramid_mailer -# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/i18n.html class EmailService: - instance = None + instance: Optional['EmailService'] = None def __init__(self, settings): self.mail_from = settings['mail.from'] @@ -53,9 +50,8 @@ def __init__(self, settings): localizator = EmailLocalizator() self._ = lambda lang, key: localizator.get_translation(lang, key) - def _send_email(self, to_address, subject=None, body=None): - log.debug('Sending email to %s through %s' % ( - to_address, self.mail_server)) + def _send_email(self, to_address: str, subject: str, body: str): + log.debug('Sending email to %s through %s' % (to_address, self.mail_server)) msg = MIMEMultipart() msg['From'] = self.mail_from @@ -64,15 +60,16 @@ def _send_email(self, to_address, subject=None, body=None): msg.attach(MIMEText(body, 'plain', 'utf-8')) if asbool(self.settings.get('mail.ssl', False)): - smtp = smtplib.SMTP_SSL(self.settings['mail.host'], - port=self.settings['mail.port']) + smtp = smtplib.SMTP_SSL( + self.settings['mail.host'], port=self.settings['mail.port'] + ) else: - smtp = smtplib.SMTP(self.settings['mail.host'], - port=self.settings['mail.port']) + smtp = smtplib.SMTP( + self.settings['mail.host'], port=self.settings['mail.port'] + ) if self.settings.get('mail.username', '') not in ('', 'None'): - smtp.login(self.settings['mail.username'], - self.settings['mail.password']) + smtp.login(self.settings['mail.username'], self.settings['mail.password']) if asbool(self.settings.get('mail.tls', False)): smtp.starttls() @@ -82,36 +79,39 @@ def _send_email(self, to_address, subject=None, body=None): def send_registration_confirmation(self, user, link): self._send_email( - user.email, - subject=self._(user.lang, 'registration_subject'), - body=self._(user.lang, 'registration_body') % link) + user.email, + subject=self._(user.lang, 'registration_subject'), + body=self._(user.lang, 'registration_body') % link, + ) def send_request_change_password(self, user, link): - body = self._(user.lang, 'password_change_body') % ( - link, user.username) + body = self._(user.lang, 'password_change_body') % (link, user.username) self._send_email( - user.email, - subject=self._(user.lang, 'password_change_subject'), - body=body) + user.email, subject=self._(user.lang, 'password_change_subject'), body=body + ) def send_change_email_confirmation(self, user, link): self._send_email( - user.email_to_validate, - subject=self._(user.lang, 'email_change_subject'), - body=self._(user.lang, 'email_change_body') % link) + user.email_to_validate, + subject=self._(user.lang, 'email_change_subject'), + body=self._(user.lang, 'email_change_body') % link, + ) def send_rate_limiting_alert(self, user): url = '{}/profiles/{}'.format(self.settings['ui.url'], user.id) if user.blocked: - body = self._('fr', 'rate_limiting_blocked_alert_body') % ( - user.name, url) + body = self._('fr', 'rate_limiting_blocked_alert_body') % (user.name, url) else: body = self._('fr', 'rate_limiting_alert_body') % ( - user.name, url, user.ratelimit_times) + user.name, + url, + user.ratelimit_times, + ) self._send_email( - self.settings['rate_limiting.alert_address'], - subject=self._('fr', 'rate_limiting_alert_subject'), - body=body) + self.settings['rate_limiting.alert_address'], + subject=self._('fr', 'rate_limiting_alert_subject'), + body=body, + ) def get_email_service(request): @@ -119,3 +119,15 @@ def get_email_service(request): settings = request.registry.settings EmailService.instance = EmailService(settings) return EmailService.instance + + +def get_email_service_from_settings(settings): + """Return the ``EmailService`` singleton, creating it from *settings* + if it doesn't exist yet. + + Unlike ``get_email_service`` this does not require a Pyramid request + and can be used from FastAPI code. + """ + if not EmailService.instance: + EmailService.instance = EmailService(settings) + return EmailService.instance diff --git a/c2corg_api/ext/colander_ext.py b/c2corg_api/ext/colander_ext.py deleted file mode 100644 index 767401eac..000000000 --- a/c2corg_api/ext/colander_ext.py +++ /dev/null @@ -1,111 +0,0 @@ -# Inspired from the c2cgeoform colander extension -# https://github.com/camptocamp/c2cgeoform/blob/master/c2cgeoform/ext/ - -from colander import (null, Invalid, SchemaType) - -from geoalchemy2 import WKBElement -from geomet import wkb -from geoalchemy2.compat import buffer, bytes -import geojson -from numbers import Number - - -class Geometry(SchemaType): - """ A Colander type meant to be used with GeoAlchemy 2 geometry columns. - Example usage - .. code-block:: python - geom = Column( - geoalchemy2.Geometry('POLYGON', 4326, management=True), info={ - 'colanderalchemy': { - 'typ': colander_ext.Geometry( - 'POLYGON', srid=4326, map_srid=3857), - 'widget': deform_ext.MapWidget() - }}) - **Attributes/Arguments** - geometry_type - The geometry type should match the column geometry type. - srid - The SRID of the geometry should also match the column definition. - """ - def __init__(self, geometry_types=['GEOMETRY'], srid=-1, map_srid=-1): - self.geometry_types = [t.upper() for t in geometry_types] - self.srid = int(srid) - - def serialize(self, node, appstruct): - """ - In Colander speak: Converts a Python data structure (an appstruct) into - a serialization (a cstruct). - Or: Converts a `WKBElement` into a GeoJSON string. - """ - if appstruct is null: - return null - if isinstance(appstruct, WKBElement): - return geojson_from_wkbelement(appstruct) - - raise Invalid(node, 'Unexpected value: %r' % appstruct) - - def deserialize(self, node, cstruct): - """ - In Colander speak: Converts a serialized value (a cstruct) into a - Python data structure (a appstruct). - Or: Converts a GeoJSON string into a `WKBElement`. - """ - if cstruct is null or cstruct == '': - return null - try: - data = geojson.loads(cstruct) - except Invalid as exc: - raise exc - except Exception: - raise Invalid(node, 'Invalid geometry: %s' % cstruct) - if not isinstance(data, geojson.GeoJSON): - raise Invalid(node, 'Invalid geometry: %s' % cstruct) - geom_type = data['type'].upper() - allowed_types = self.geometry_types - if geom_type in allowed_types: - if not is_valid_geometry(data): - raise Invalid(node, 'Invalid geometry: %s' % cstruct) - else: - return wkbelement_from_geojson(data, self.srid) - else: - raise Invalid( - node, 'Invalid geometry type. Expected: %s. Got: %s.' - % (allowed_types, geom_type)) - - def cstruct_children(self, node, cstruct): - return [] - - -def wkbelement_from_geojson(geojson, srid): - geometry = wkb.dumps(geojson, big_endian=False) - return from_wkb(geometry, srid) - - -def geojson_from_wkbelement(wkb_element): - geometry = wkb.loads(bytes(wkb_element.data)) - return geojson.dumps(geometry) - - -def from_wkb(wkb, srid=-1): - return WKBElement(buffer(wkb), srid=srid) - - -def is_valid_geometry(obj): - return obj.is_valid - - -def _check_point_4d(coord): - if not isinstance(coord, list): - return 'each position must be a list' - if len(coord) not in (2, 3, 4): - return 'a position must have exactly 2, 3 or 4 values' - for number in coord: - if not isinstance(number, Number): - return 'a position cannot have inner positions' - - -# geojson RFC says that the coordinates should be 2d or 3d and -# that parsers may ignore additional elements. -# geojson raises an error on 4d coordinates (2 or 3 elements expected), -# We override the function to handle 4d coordinates -geojson.geometry.check_point = _check_point_4d diff --git a/c2corg_api/ext/geometry.py b/c2corg_api/ext/geometry.py new file mode 100644 index 000000000..21aad30ec --- /dev/null +++ b/c2corg_api/ext/geometry.py @@ -0,0 +1,210 @@ +# Geometry serialisation utilities (WKB ↔ GeoJSON). +# Originally inspired from the c2cgeoform geometry extension. + +import struct +from numbers import Number + +import geojson +from geoalchemy2 import WKBElement +from geomet import wkb + +# PostGIS EWKB bitmask flags +_EWKB_Z_FLAG = 0x80000000 +_EWKB_M_FLAG = 0x40000000 +_EWKB_SRID_FLAG = 0x20000000 + + +def wkbelement_from_geojson(geojson_data, srid): + geometry = wkb.dumps(geojson_data, big_endian=False) + return from_wkb(geometry, srid) + + +def geojson_from_wkbelement(wkb_element): + data = wkb_element.data + if isinstance(data, memoryview): + data = bytes(data) + elif isinstance(data, str): + data = bytes.fromhex(data) + data = _ewkb_to_iso_wkb(data) + geometry = wkb.loads(data) + return geojson.dumps(geometry) + + +def _ewkb_to_iso_wkb(data): + """Convert PostGIS EWKB to ISO WKB that geomet can parse. + + PostGIS EWKB encodes Z/M/SRID as bitmask flags in the type integer + (0x80000000 for Z, 0x40000000 for M, 0x20000000 for SRID). + ISO WKB uses type code offsets (+1000 for Z, +2000 for M, +3000 for ZM). + + This function handles compound geometries (Multi*, GeometryCollection) + by recursively converting each sub-geometry header. + """ + result = bytearray() + _ewkb_to_iso_wkb_at(data, 0, result) + return bytes(result) + + +# Number of coordinates per point for each dimension combo +_COORDS_PER_POINT = { + (False, False): 2, # 2D + (True, False): 3, # Z + (False, True): 3, # M + (True, True): 4, # ZM +} + +# WKB base type -> whether it's a multi/collection type +_MULTI_TYPES = {4, 5, 6, 7} # MultiPoint, MultiLS, MultiPoly, GeomCollection + + +def _ewkb_to_iso_wkb_at(data, offset, result): + """Convert one geometry starting at `offset`, appending ISO WKB to result. + Returns the new offset past the consumed bytes. + """ + if offset + 5 > len(data): + # Not enough data; copy remainder as-is + result.extend(data[offset:]) + return len(data) + + byte_order = data[offset] + fmt = 'I' + type_int = struct.unpack_from(fmt, data, offset + 1)[0] + + has_srid = bool(type_int & _EWKB_SRID_FLAG) + has_z = bool(type_int & _EWKB_Z_FLAG) + has_m = bool(type_int & _EWKB_M_FLAG) + + base_type = type_int & 0x0FFFFFFF + iso_type = base_type + if has_z and has_m: + iso_type += 3000 + elif has_z: + iso_type += 1000 + elif has_m: + iso_type += 2000 + + # Write byte order + ISO type + result.append(byte_order) + result.extend(struct.pack(fmt, iso_type)) + + # Advance past header (byte_order + type + optional SRID) + hdr_size = 5 + (4 if has_srid else 0) + offset += hdr_size + + coords_per_point = _COORDS_PER_POINT[(has_z, has_m)] + point_size = coords_per_point * 8 # 8 bytes per float64 + + if base_type in _MULTI_TYPES: + # Multi/Collection: read count, then recurse for each sub-geometry + if offset + 4 > len(data): + result.extend(data[offset:]) + return len(data) + count = struct.unpack_from(fmt, data, offset)[0] + result.extend(struct.pack(fmt, count)) + offset += 4 + for _ in range(count): + offset = _ewkb_to_iso_wkb_at(data, offset, result) + elif base_type == 1: + # Point + result.extend(data[offset : offset + point_size]) + offset += point_size + elif base_type == 2: + # LineString + if offset + 4 > len(data): + result.extend(data[offset:]) + return len(data) + num_points = struct.unpack_from(fmt, data, offset)[0] + result.extend(struct.pack(fmt, num_points)) + offset += 4 + size = num_points * point_size + result.extend(data[offset : offset + size]) + offset += size + elif base_type == 3: + # Polygon + if offset + 4 > len(data): + result.extend(data[offset:]) + return len(data) + num_rings = struct.unpack_from(fmt, data, offset)[0] + result.extend(struct.pack(fmt, num_rings)) + offset += 4 + for _ in range(num_rings): + if offset + 4 > len(data): + break + num_points = struct.unpack_from(fmt, data, offset)[0] + result.extend(struct.pack(fmt, num_points)) + offset += 4 + size = num_points * point_size + result.extend(data[offset : offset + size]) + offset += size + else: + # Unknown type: copy the rest as-is (best effort) + result.extend(data[offset:]) + offset = len(data) + + return offset + + +def from_wkb(raw_wkb, srid=-1): + """Create a WKBElement from ISO WKB bytes, embedding the SRID as EWKB. + + GeoAlchemy2's bind processor for non-extended WKBElements falls back to + Shapely (which drops 4D coords). By converting to EWKB and marking the + element as extended, the raw hex is sent directly to PostGIS, preserving + all dimensions. + """ + ewkb = _iso_wkb_to_ewkb(raw_wkb, srid) + return WKBElement(memoryview(ewkb), srid=srid, extended=True) + + +def _iso_wkb_to_ewkb(data, srid=-1): + """Convert ISO WKB to PostGIS EWKB, embedding the SRID. + + ISO WKB uses type code offsets (+1000 for Z, +2000 for M, +3000 for ZM). + PostGIS EWKB uses bitmask flags (0x80000000 Z, 0x40000000 M, 0x20000000 + SRID) and inserts a 4-byte SRID after the type integer. + """ + if len(data) < 5: + return data + byte_order = data[0] + fmt = 'I' + type_int = struct.unpack_from(fmt, data, 1)[0] + + base_type = type_int % 1000 + offset = type_int // 1000 + + ewkb_type = base_type + if offset == 1: # Z + ewkb_type |= _EWKB_Z_FLAG + elif offset == 2: # M + ewkb_type |= _EWKB_M_FLAG + elif offset == 3: # ZM + ewkb_type |= _EWKB_Z_FLAG | _EWKB_M_FLAG + + if srid > 0: + ewkb_type |= _EWKB_SRID_FLAG + return ( + data[0:1] + struct.pack(fmt, ewkb_type) + struct.pack(fmt, srid) + data[5:] + ) + else: + return data[0:1] + struct.pack(fmt, ewkb_type) + data[5:] + + +def is_valid_geometry(obj): + return obj.is_valid + + +def _check_point_4d(coord): + if not isinstance(coord, list): + return 'each position must be a list' + if len(coord) not in (2, 3, 4): + return 'a position must have exactly 2, 3 or 4 values' + for number in coord: + if not isinstance(number, Number): + return 'a position cannot have inner positions' + + +# geojson RFC says that the coordinates should be 2d or 3d and +# that parsers may ignore additional elements. +# geojson raises an error on 4d coordinates (2 or 3 elements expected), +# We override the function to handle 4d coordinates +geojson.geometry.check_point = _check_point_4d diff --git a/c2corg_api/ext/sqa_view.py b/c2corg_api/ext/sqa_view.py index a24fa0f93..028a4fc8d 100644 --- a/c2corg_api/ext/sqa_view.py +++ b/c2corg_api/ext/sqa_view.py @@ -1,4 +1,4 @@ -from sqlalchemy.sql.schema import Table, Column, MetaData +from sqlalchemy import Column, MetaData, Table # Support for views in SQLAlchemy # See: https://bitbucket.org/zzzeek/sqlalchemy/wiki/UsageRecipes/Views @@ -16,7 +16,7 @@ def view(name, schema, metadata, selectable): # created t = Table(name, MetaData(), schema=schema) - for c in selectable.c: + for c in selectable.subquery().c: t.append_column(Column(c.name, c.type, primary_key=c.primary_key)) return t diff --git a/c2corg_api/jobs/__init__.py b/c2corg_api/jobs/__init__.py index 5a4f91304..dc1d4c44a 100644 --- a/c2corg_api/jobs/__init__.py +++ b/c2corg_api/jobs/__init__.py @@ -1,11 +1,12 @@ import atexit -from apscheduler.schedulers.background import BackgroundScheduler +import logging + from apscheduler.events import EVENT_JOB_ERROR +from apscheduler.schedulers.background import BackgroundScheduler -from c2corg_api.jobs.purge_non_activated_accounts import purge_account from c2corg_api.jobs.purge_expired_tokens import purge_token +from c2corg_api.jobs.purge_non_activated_accounts import purge_account -import logging log = logging.getLogger(__name__) @@ -24,7 +25,7 @@ def configure_scheduler_from_config(settings): name='Purge accounts which where not activated', trigger='cron', hour=0, - minute=0 + minute=0, ) # run `purge_token` job at 0:30 @@ -34,7 +35,7 @@ def configure_scheduler_from_config(settings): name='Purge expired tokens', trigger='cron', hour=0, - minute=30 + minute=30, ) scheduler.add_listener(exception_listener, EVENT_JOB_ERROR) diff --git a/c2corg_api/jobs/purge_expired_tokens.py b/c2corg_api/jobs/purge_expired_tokens.py index 620ecaae1..bf3bf6fdb 100644 --- a/c2corg_api/jobs/purge_expired_tokens.py +++ b/c2corg_api/jobs/purge_expired_tokens.py @@ -1,18 +1,23 @@ -from c2corg_api.models.token import Token -from datetime import datetime +import logging +from datetime import datetime, timezone + from sqlalchemy.orm import sessionmaker -import logging +from c2corg_api.models.token import Token + log = logging.getLogger(__name__) def purge_token(test_session=None): - now = datetime.utcnow() + now = datetime.now(timezone.utc) session = sessionmaker()() if not test_session else test_session try: - count = session.query(Token).filter( - Token.expire <= now).delete(synchronize_session=False) + count = ( + session.query(Token) + .filter(Token.expire <= now) + .delete(synchronize_session=False) + ) log.info('Deleting %d expired token', count) if count > 0: diff --git a/c2corg_api/jobs/purge_non_activated_accounts.py b/c2corg_api/jobs/purge_non_activated_accounts.py index 90b063aed..4da3dc33e 100644 --- a/c2corg_api/jobs/purge_non_activated_accounts.py +++ b/c2corg_api/jobs/purge_non_activated_accounts.py @@ -1,28 +1,36 @@ -from c2corg_api.models.user import User, Purpose +import logging +from datetime import datetime, timezone + +from sqlalchemy import and_ +from sqlalchemy.orm import sessionmaker + from c2corg_api.models.document import DocumentLocale from c2corg_api.models.document_history import DocumentVersion, HistoryMetaData +from c2corg_api.models.user import Purpose, User from c2corg_api.models.user_profile import UserProfile -from datetime import datetime -from sqlalchemy.sql.expression import and_ -from sqlalchemy.orm import sessionmaker -import logging log = logging.getLogger(__name__) def purge_account(test_session=None): - now = datetime.utcnow() + now = datetime.now(timezone.utc) session = sessionmaker()() if not test_session else test_session def delete(cls, attr, ids): - session.query(cls).filter(attr.in_(ids)).delete( - synchronize_session=False) + session.query(cls).filter(attr.in_(ids)).delete(synchronize_session=False) try: - ids = session.query(User.id).filter(and_( - User.email_validated.is_(False), - User.validation_nonce.like(Purpose.registration.value + '_%'), - User.validation_nonce_expire < now)).all() + ids = ( + session.query(User.id) + .filter( + and_( + User.email_validated.is_(False), + User.validation_nonce.like(Purpose.registration.value + '_%'), + User.validation_nonce_expire < now, + ) + ) + .all() + ) ids = [idwrap[0] for idwrap in ids] log.info('Deleting %d non activated users: %s', len(ids), ids) diff --git a/c2corg_api/markdown/__init__.py b/c2corg_api/markdown/__init__.py index 7714eb32e..cf70b7fa4 100644 --- a/c2corg_api/markdown/__init__.py +++ b/c2corg_api/markdown/__init__.py @@ -1,22 +1,22 @@ -import markdown -import bleach -import bleach.css_sanitizer -import secrets import logging +import secrets from threading import RLock -from .wikilinks import C2CWikiLinkExtension +import bleach +import bleach.css_sanitizer +import markdown +from markdown.extensions.nl2br import Nl2BrExtension + +from .alerts import AlertExtension +from .emojis import C2CEmojiExtension +from .header import C2CHeaderExtension from .img import C2CImageExtension -from .video import C2CVideoExtension from .ltag import C2CLTagExtension -from .header import C2CHeaderExtension +from .nbsp import C2CNbspExtension from .ptag import C2CPTagExtension -from .alerts import AlertExtension from .toc import C2CTocExtension -from .emojis import C2CEmojiExtension -from .nbsp import C2CNbspExtension -from markdown.extensions.nl2br import Nl2BrExtension - +from .video import C2CVideoExtension +from .wikilinks import C2CWikiLinkExtension logger = logging.getLogger('MARKDOWN') @@ -36,7 +36,7 @@ _markdown_parser = None _cleaner = None -_iframe_secret_tag = "iframe_" + secrets.token_hex(32) +_iframe_secret_tag = 'iframe_' + secrets.token_hex(32) """ _***_secret_tag is used as a private key to replace critical HTML node and @@ -53,55 +53,74 @@ def _get_cleaner(): global _cleaner if not _cleaner: - allowed_tags = bleach.sanitizer.ALLOWED_TAGS.union({ - # headers - "h1", "h2", "h3", "h4", "h5", "h6", - - # blocks - "div", "p", "pre", "hr", "center", - - # inline nodes - "span", "br", "sub", "sup", "s", "del", "ins", "small", - - # images - "figure", "img", "figcaption", - - _iframe_secret_tag, - - # tables - "table", "tr", "td", "th", "tbody" - }) + allowed_tags = bleach.sanitizer.ALLOWED_TAGS.union( + { + # headers + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + # blocks + 'div', + 'p', + 'pre', + 'hr', + 'center', + # inline nodes + 'span', + 'br', + 'sub', + 'sup', + 's', + 'del', + 'ins', + 'small', + # images + 'figure', + 'img', + 'figcaption', + _iframe_secret_tag, + # tables + 'table', + 'tr', + 'td', + 'th', + 'tbody', + } + ) allowed_attributes = dict(bleach.sanitizer.ALLOWED_ATTRIBUTES) allowed_extra_attributes = { - "a": [ - "c2c:role", - "c2c:document-type", - "c2c:document-id", - "c2c:lang", - "c2c:slug", - "c2c:anchor" + 'a': [ + 'c2c:role', + 'c2c:document-type', + 'c2c:document-id', + 'c2c:lang', + 'c2c:slug', + 'c2c:anchor', ], - "h1": ["id", "c2c:role"], - "h2": ["id", "c2c:role"], - "h3": ["id", "c2c:role"], - "h4": ["id", "c2c:role"], - "h5": ["id", "c2c:role"], - "h6": ["id", "c2c:role"], - "table": ["c2c:role"], - "div": ["class", "style", "c2c:role"], - "td": ["colspan"], - "span": ["class", "translate", "id", "c2c:role"], - _iframe_secret_tag: ["src"], - "figure": ["c2c:position", "c2c:role", "c2c:size"], - "img": [ - "alt", - "c2c:document-id", - "c2c:role", - "c2c:size", - "c2c:url-proxy", - "c2c:svg-name", - "c2c:emoji-db" + 'h1': ['id', 'c2c:role'], + 'h2': ['id', 'c2c:role'], + 'h3': ['id', 'c2c:role'], + 'h4': ['id', 'c2c:role'], + 'h5': ['id', 'c2c:role'], + 'h6': ['id', 'c2c:role'], + 'table': ['c2c:role'], + 'div': ['class', 'style', 'c2c:role'], + 'td': ['colspan'], + 'span': ['class', 'translate', 'id', 'c2c:role'], + _iframe_secret_tag: ['src'], + 'figure': ['c2c:position', 'c2c:role', 'c2c:size'], + 'img': [ + 'alt', + 'c2c:document-id', + 'c2c:role', + 'c2c:size', + 'c2c:url-proxy', + 'c2c:svg-name', + 'c2c:emoji-db', ], } @@ -112,9 +131,9 @@ def _get_cleaner(): allowed_attributes[key] += allowed_extra_attributes[key] css_sanitizer = bleach.css_sanitizer.CSSSanitizer( - allowed_css_properties=list( - bleach.css_sanitizer.ALLOWED_CSS_PROPERTIES) - + ['clear']) + allowed_css_properties=list(bleach.css_sanitizer.ALLOWED_CSS_PROPERTIES) + + ['clear'] + ) _cleaner = bleach.sanitizer.Cleaner( tags=allowed_tags, @@ -122,7 +141,8 @@ def _get_cleaner(): css_sanitizer=css_sanitizer, protocols=bleach.sanitizer.ALLOWED_PROTOCOLS, strip=False, - strip_comments=True) + strip_comments=True, + ) return _cleaner @@ -134,7 +154,7 @@ def _get_markdown_parser(): C2CWikiLinkExtension(), C2CImageExtension(), Nl2BrExtension(), - C2CTocExtension(marker='[toc]', baselevel=2, toc_depth="1-4"), + C2CTocExtension(marker='[toc]', baselevel=2, toc_depth='1-4'), C2CVideoExtension(iframe_secret_tag=_iframe_secret_tag), C2CLTagExtension(), C2CHeaderExtension(), @@ -143,8 +163,9 @@ def _get_markdown_parser(): C2CEmojiExtension(), C2CNbspExtension(), ] - _markdown_parser = markdown.Markdown(output_format='xhtml5', - extensions=extensions) + _markdown_parser = markdown.Markdown( + output_format='xhtml5', extensions=extensions + ) return _markdown_parser @@ -176,9 +197,9 @@ def parse_code(text): # because we are not sure of this function text = cleaner.clean(text=text) except Exception as e: - logger.exception("While parsing markdown", exc_info=e) + logger.exception('While parsing markdown', exc_info=e) text = _PARSER_EXCEPTION_MESSAGE - text = text.replace(_iframe_secret_tag, "iframe") + text = text.replace(_iframe_secret_tag, 'iframe') return text diff --git a/c2corg_api/markdown/alerts.py b/c2corg_api/markdown/alerts.py index eee71be15..ddcbac37f 100644 --- a/c2corg_api/markdown/alerts.py +++ b/c2corg_api/markdown/alerts.py @@ -1,17 +1,14 @@ -from markdown.extensions import Extension -from markdown.blockprocessors import BlockProcessor -from xml.etree import ElementTree # nosec import re +from xml.etree import ElementTree # nosec + +from markdown.blockprocessors import BlockProcessor +from markdown.extensions import Extension class AlertProcessor(BlockProcessor): RE = re.compile(r'(^|\n)[ ]{0,3}(!{2,4})(([^!]|$).*)') - roles = { - "!!": "info", - "!!!": "warning", - "!!!!": "danger", - } + roles = {'!!': 'info', '!!!': 'warning', '!!!!': 'danger'} def test(self, parent, block): return bool(self.RE.search(block)) @@ -20,13 +17,13 @@ def run(self, parent, blocks): block = blocks.pop(0) m = self.RE.search(block) level = m.group(2) - tester = re.compile("^[ ]{0,3}" + level + "([^!]|$)") + tester = re.compile('^[ ]{0,3}' + level + '([^!]|$)') - before = block[:m.start()] # Lines before blockquote + before = block[: m.start()] # Lines before blockquote # Pass lines before alert banner self.parser.parseBlocks(parent, [before]) - after = block[m.start():].split('\n') + after = block[m.start() :].split('\n') if len(after[0]) == 0: after.pop(0) @@ -44,18 +41,18 @@ def run(self, parent, blocks): block = '\n'.join([self.clean(line) for line in block]) quote = ElementTree.SubElement(parent, 'div') - quote.set("c2c:role", self.roles[level]) + quote.set('c2c:role', self.roles[level]) # Recursively parse block with div as parent. self.parser.parseChunk(quote, block) # and continue parsing next part of the block - self.parser.parseBlocks(parent, ["\n".join(after)]) + self.parser.parseBlocks(parent, ['\n'.join(after)]) def clean(self, line): - """ Remove ``!`` from beginning of a line. """ + """Remove ``!`` from beginning of a line.""" m = self.RE.match(line) - if line.strip() in ("!!", "!!!", "!!!!"): - return "" + if line.strip() in ('!!', '!!!', '!!!!'): + return '' elif m: return m.group(3) else: @@ -65,9 +62,7 @@ def clean(self, line): class AlertExtension(Extension): def extendMarkdown(self, md): # noqa: N802 md.parser.blockprocessors.register( - AlertProcessor(md.parser), - 'c2c_alert', - 10.20 + AlertProcessor(md.parser), 'c2c_alert', 10.20 ) diff --git a/c2corg_api/markdown/emoji_databases/c2c_activities.py b/c2corg_api/markdown/emoji_databases/c2c_activities.py index 2acfecf36..7b32116c9 100644 --- a/c2corg_api/markdown/emoji_databases/c2c_activities.py +++ b/c2corg_api/markdown/emoji_databases/c2c_activities.py @@ -1,65 +1,61 @@ -SVG_CDN = "/static/img/documents/activities/" +SVG_CDN = '/static/img/documents/activities/' -name = "c2c-activities" +name = 'c2c-activities' emoji = { - ":rock_climbing:": { - "category": "activitiy", - "name": "rock climbing", - "svg_name": "rock_climbing", - "unicode": "1f9d7", + ':rock_climbing:': { + 'category': 'activitiy', + 'name': 'rock climbing', + 'svg_name': 'rock_climbing', + 'unicode': '1f9d7', + }, + ':skitouring:': { + 'category': 'activitiy', + 'name': 'ski touring', + 'svg_name': 'skitouring', + 'unicode': '26f7', + }, + ':hiking:': {'category': 'activitiy', 'name': 'hiking', 'svg_name': 'hiking'}, + ':ice_climbing:': { + 'category': 'activitiy', + 'name': 'ice climbing', + 'svg_name': 'ice_climbing', + }, + ':mountain_biking:': { + 'category': 'activitiy', + 'name': 'mountain biking', + 'svg_name': 'mountain_biking', + }, + ':paragliding:': { + 'category': 'activitiy', + 'name': 'paragliding', + 'svg_name': 'paragliding', + }, + ':slacklining:': { + 'category': 'activitiy', + 'name': 'slacklining', + 'svg_name': 'slacklining', + }, + ':snow_ice_mixed:': { + 'category': 'activitiy', + 'name': 'snow ice mixed', + 'svg_name': 'snow_ice_mixed', + }, + ':snowshoeing:': { + 'category': 'activitiy', + 'name': 'snowshoeing', + 'svg_name': 'snowshoeing', + }, + ':via_ferrata:': { + 'category': 'activitiy', + 'name': 'via ferrata', + 'svg_name': 'via_ferrata', + }, + ':mountain_climbing:': { + 'category': 'activitiy', + 'name': 'mountain climbing', + 'svg_name': 'mountain_climbing', }, - ":skitouring:": { - "category": "activitiy", - "name": "ski touring", - "svg_name": "skitouring", - "unicode": "26f7" - }, - ":hiking:": { - "category": "activitiy", - "name": "hiking", - "svg_name": "hiking", - }, - ":ice_climbing:": { - "category": "activitiy", - "name": "ice climbing", - "svg_name": "ice_climbing", - }, - ":mountain_biking:": { - "category": "activitiy", - "name": "mountain biking", - "svg_name": "mountain_biking", - }, - ":paragliding:": { - "category": "activitiy", - "name": "paragliding", - "svg_name": "paragliding", - }, - ":slacklining:": { - "category": "activitiy", - "name": "slacklining", - "svg_name": "slacklining", - }, - ":snow_ice_mixed:": { - "category": "activitiy", - "name": "snow ice mixed", - "svg_name": "snow_ice_mixed", - }, - ":snowshoeing:": { - "category": "activitiy", - "name": "snowshoeing", - "svg_name": "snowshoeing", - }, - ":via_ferrata:": { - "category": "activitiy", - "name": "via ferrata", - "svg_name": "via_ferrata", - }, - ":mountain_climbing:": { - "category": "activitiy", - "name": "mountain climbing", - "svg_name": "mountain_climbing", - } } aliases = {} diff --git a/c2corg_api/markdown/emoji_databases/c2c_waypoints.py b/c2corg_api/markdown/emoji_databases/c2c_waypoints.py index e2f39e5b8..868799bda 100644 --- a/c2corg_api/markdown/emoji_databases/c2c_waypoints.py +++ b/c2corg_api/markdown/emoji_databases/c2c_waypoints.py @@ -1,168 +1,88 @@ -SVG_CDN = "/static/img/documents/waypoints/" +SVG_CDN = '/static/img/documents/waypoints/' -name = "c2c-waypoints" +name = 'c2c-waypoints' emoji = { - ":access:": { - "category": "travel", - "name": "access", - "svg_name": "access" - }, - ":base_camp:": { - "category": "travel", - "name": "base camp", - "svg_name": "base_camp" - }, - ":bergschrund:": { - "category": "travel", - "name": "bergschrund", - "svg_name": "bergschrund" - }, - ":bisse:": { - "category": "travel", - "name": "bisse", - "svg_name": "bisse" - }, - ":bivouac:": { - "category": "travel", - "name": "bivouac", - "svg_name": "bivouac" - }, - ":camp_site:": { - "category": "travel", - "name": "camp_site", - "svg_name": "camp_site", - "unicode": "1f3d5", - }, - ":canyon:": { - "category": "travel", - "name": "canyon", - "svg_name": "canyon" - }, - ":cave:": { - "category": "travel", - "name": "cave", - "svg_name": "cave" - }, - ":cliff:": { - "category": "travel", - "name": "cliff", - "svg_name": "cliff" - }, - ":climbing_indoor:": { - "category": "travel", - "name": "climbing indoor", - "svg_name": "climbing_indoor" - }, - ":climbing_outdoor:": { - "category": "travel", - "name": "climbing outdoor", - "svg_name": "climbing_outdoor" - }, - ":confluence:": { - "category": "travel", - "name": "confluence", - "svg_name": "confluence" - }, - ":gite:": { - "category": "travel", - "name": "gite", - "svg_name": "gite" - }, - ":glacier:": { - "category": "travel", - "name": "glacier", - "svg_name": "glacier" - }, - ":hut:": { - "category": "travel", - "name": "hut", - "svg_name": "hut" - }, - ":lake:": { - "category": "travel", - "name": "lake", - "svg_name": "lake" - }, - ":local_product:": { - "category": "travel", - "name": "local product", - "svg_name": "local_product" - }, - ":locality:": { - "category": "travel", - "name": "locality", - "svg_name": "locality" - }, - ":misc:": { - "category": "travel", - "name": "misc", - "svg_name": "misc" - }, - ":paragliding_landing:": { - "category": "travel", - "name": "paragliding landing", - "svg_name": "paragliding_landing" - }, - ":paragliding_takeoff:": { - "category": "travel", - "name": "paragliding takeoff", - "svg_name": "paragliding_takeoff" - }, - ":pass:": { - "category": "travel", - "name": "pass", - "svg_name": "pass" - }, - ":pit:": { - "category": "travel", - "name": "pit", - "svg_name": "pit" - }, - ":shelter:": { - "category": "travel", - "name": "shelter", - "svg_name": "shelter" - }, - ":slackline_spot:": { - "category": "travel", - "name": "slackline spot", - "svg_name": "slackline_spot" - }, - ":summit:": { - "category": "travel", - "name": "summit", - "svg_name": "summit" - }, - ":virtual:": { - "category": "travel", - "name": "virtual", - "svg_name": "virtual" - }, - ":waterfall:": { - "category": "travel", - "name": "waterfall", - "svg_name": "waterfall" - }, - ":waterpoint:": { - "category": "travel", - "name": "waterpoint", - "svg_name": "waterpoint" - }, - ":waypoints:": { - "category": "travel", - "name": "waypoints", - "svg_name": "waypoints" - }, - ":weather_station:": { - "category": "travel", - "name": "weather station", - "svg_name": "weather_station" - }, - ":webcam:": { - "category": "travel", - "name": "webcam", - "svg_name": "webcam", - "unicode": "1f4f7" + ':access:': {'category': 'travel', 'name': 'access', 'svg_name': 'access'}, + ':base_camp:': {'category': 'travel', 'name': 'base camp', 'svg_name': 'base_camp'}, + ':bergschrund:': { + 'category': 'travel', + 'name': 'bergschrund', + 'svg_name': 'bergschrund', + }, + ':bisse:': {'category': 'travel', 'name': 'bisse', 'svg_name': 'bisse'}, + ':bivouac:': {'category': 'travel', 'name': 'bivouac', 'svg_name': 'bivouac'}, + ':camp_site:': { + 'category': 'travel', + 'name': 'camp_site', + 'svg_name': 'camp_site', + 'unicode': '1f3d5', + }, + ':canyon:': {'category': 'travel', 'name': 'canyon', 'svg_name': 'canyon'}, + ':cave:': {'category': 'travel', 'name': 'cave', 'svg_name': 'cave'}, + ':cliff:': {'category': 'travel', 'name': 'cliff', 'svg_name': 'cliff'}, + ':climbing_indoor:': { + 'category': 'travel', + 'name': 'climbing indoor', + 'svg_name': 'climbing_indoor', + }, + ':climbing_outdoor:': { + 'category': 'travel', + 'name': 'climbing outdoor', + 'svg_name': 'climbing_outdoor', + }, + ':confluence:': { + 'category': 'travel', + 'name': 'confluence', + 'svg_name': 'confluence', + }, + ':gite:': {'category': 'travel', 'name': 'gite', 'svg_name': 'gite'}, + ':glacier:': {'category': 'travel', 'name': 'glacier', 'svg_name': 'glacier'}, + ':hut:': {'category': 'travel', 'name': 'hut', 'svg_name': 'hut'}, + ':lake:': {'category': 'travel', 'name': 'lake', 'svg_name': 'lake'}, + ':local_product:': { + 'category': 'travel', + 'name': 'local product', + 'svg_name': 'local_product', + }, + ':locality:': {'category': 'travel', 'name': 'locality', 'svg_name': 'locality'}, + ':misc:': {'category': 'travel', 'name': 'misc', 'svg_name': 'misc'}, + ':paragliding_landing:': { + 'category': 'travel', + 'name': 'paragliding landing', + 'svg_name': 'paragliding_landing', + }, + ':paragliding_takeoff:': { + 'category': 'travel', + 'name': 'paragliding takeoff', + 'svg_name': 'paragliding_takeoff', + }, + ':pass:': {'category': 'travel', 'name': 'pass', 'svg_name': 'pass'}, + ':pit:': {'category': 'travel', 'name': 'pit', 'svg_name': 'pit'}, + ':shelter:': {'category': 'travel', 'name': 'shelter', 'svg_name': 'shelter'}, + ':slackline_spot:': { + 'category': 'travel', + 'name': 'slackline spot', + 'svg_name': 'slackline_spot', + }, + ':summit:': {'category': 'travel', 'name': 'summit', 'svg_name': 'summit'}, + ':virtual:': {'category': 'travel', 'name': 'virtual', 'svg_name': 'virtual'}, + ':waterfall:': {'category': 'travel', 'name': 'waterfall', 'svg_name': 'waterfall'}, + ':waterpoint:': { + 'category': 'travel', + 'name': 'waterpoint', + 'svg_name': 'waterpoint', + }, + ':waypoints:': {'category': 'travel', 'name': 'waypoints', 'svg_name': 'waypoints'}, + ':weather_station:': { + 'category': 'travel', + 'name': 'weather station', + 'svg_name': 'weather_station', + }, + ':webcam:': { + 'category': 'travel', + 'name': 'webcam', + 'svg_name': 'webcam', + 'unicode': '1f4f7', }, } diff --git a/c2corg_api/markdown/emojis.py b/c2corg_api/markdown/emojis.py index 35e76c0d5..a2b29427e 100644 --- a/c2corg_api/markdown/emojis.py +++ b/c2corg_api/markdown/emojis.py @@ -3,12 +3,13 @@ https://facelessuser.github.io/pymdown-extensions/extensions/emoji/ """ -from markdown import Extension -from markdown.inlinepatterns import Pattern -from xml.etree import ElementTree # nosec import copy +from xml.etree import ElementTree # nosec +from markdown import Extension +from markdown.inlinepatterns import Pattern from pymdownx import emoji1_db + from c2corg_api.markdown.emoji_databases import c2c_activities, c2c_waypoints emoji1_db.SVG_CDN = 'https://cdn.jsdelivr.net/emojione/assets/svg/' @@ -20,15 +21,15 @@ class Emoji(object): def __init__(self, db_name, code, **kwargs): self.db_name = db_name self.code = code - self.name = kwargs.pop("name") - self.category = kwargs.pop("category") - self.SVG_CDN = kwargs.pop("SVG_CDN") - self.unicode = kwargs.pop("unicode", None) - self.svg_name = kwargs.pop("svg_name", self.unicode) - self.unicode_alt = kwargs.pop("unicode_alt", self.unicode) - self.classes = kwargs.pop("classes", "emoji") + self.name = kwargs.pop('name') + self.category = kwargs.pop('category') + self.SVG_CDN = kwargs.pop('SVG_CDN') + self.unicode = kwargs.pop('unicode', None) + self.svg_name = kwargs.pop('svg_name', self.unicode) + self.unicode_alt = kwargs.pop('unicode_alt', self.unicode) + self.classes = kwargs.pop('classes', 'emoji') - assert self.svg_name, "Please precise a SVG name" + assert self.svg_name, 'Please precise a SVG name' def get_alt(self, user_code): """Get alt form.""" @@ -36,20 +37,22 @@ def get_alt(self, user_code): if self.unicode_alt is None: alt = user_code else: - alt = ''.join( - [chr(int(c, 16)) for c in self.unicode_alt.split('-')]) + alt = ''.join([chr(int(c, 16)) for c in self.unicode_alt.split('-')]) return alt def to_svg(self, user_code): """Return svg element.""" - return ElementTree.Element("img", { - "c2c:role": "emoji", - "c2c:emoji-db": self.db_name, - "c2c:svg-name": self.svg_name, - "alt": self.get_alt(user_code), - "title": user_code, - }) + return ElementTree.Element( + 'img', + { + 'c2c:role': 'emoji', + 'c2c:emoji-db': self.db_name, + 'c2c:svg-name': self.svg_name, + 'alt': self.get_alt(user_code), + 'title': user_code, + }, + ) class EmojiPattern(Pattern): @@ -68,13 +71,9 @@ def _append_to_index(self, db): for code in db.emoji: kwargs = db.emoji[code] - kwargs["SVG_CDN"] = kwargs.get("SVG_CDN", db.SVG_CDN) + kwargs['SVG_CDN'] = kwargs.get('SVG_CDN', db.SVG_CDN) - self.emoji_index[code] = Emoji( - db_name=db.name, - code=code, - **kwargs - ) + self.emoji_index[code] = Emoji(db_name=db.name, code=code, **kwargs) for code in db.aliases: self.emoji_index[code] = self.emoji_index[db.aliases[code]] @@ -98,7 +97,7 @@ def extendMarkdown(self, md): # noqa: N802 md.ESCAPED_CHARS = escaped emj = EmojiPattern(RE_EMOJI, md) - md.inlinePatterns.register(emj, "c2c_emoji", 72) + md.inlinePatterns.register(emj, 'c2c_emoji', 72) def makeExtension(*args, **kwargs): # noqa: N802 diff --git a/c2corg_api/markdown/header.py b/c2corg_api/markdown/header.py index 785719fa0..067381cea 100644 --- a/c2corg_api/markdown/header.py +++ b/c2corg_api/markdown/header.py @@ -1,23 +1,26 @@ +import logging +import re +from xml.etree import ElementTree # nosec + from markdown import Extension from markdown.blockprocessors import BlockProcessor -from xml.etree import ElementTree # nosec -import re -import logging logger = logging.getLogger('MARKDOWN') # copied from class markdown.blockprocessors.HashHeaderProcessor class C2CHeaderProcessor(BlockProcessor): - """ Process Hash Headers. """ + """Process Hash Headers.""" # Detect a header at start of any line in block - RE = re.compile(r'(^|\n)' - r'(?P#{1,6})' - r'(?P
.*?)' - r'(?P#+[^#]*?)?' - r'(?P\{#[\w-]+\})?' - r'(\n|$)') + RE = re.compile( + r'(^|\n)' + r'(?P#{1,6})' + r'(?P
.*?)' + r'(?P#+[^#]*?)?' + r'(?P\{#[\w-]+\})?' + r'(\n|$)' + ) def test(self, parent, block): return bool(self.RE.search(block)) @@ -26,8 +29,8 @@ def run(self, parent, blocks): block = blocks.pop(0) m = self.RE.search(block) if m: - before = block[:m.start()] # All lines before header - after = block[m.end():] # All lines after header + before = block[: m.start()] # All lines before header + after = block[m.end() :] # All lines after header if before: # As the header was not the first line of the block and the # lines before the header must be parsed first, @@ -37,11 +40,11 @@ def run(self, parent, blocks): h = ElementTree.SubElement(parent, 'h%d' % len(m.group('level'))) h.text = m.group('header').strip() - if m.group("fixed_id"): - h.set('id', m.group("fixed_id")[2:-1]) + if m.group('fixed_id'): + h.set('id', m.group('fixed_id')[2:-1]) if m.group('emphasis'): - emphasis_text = m.group('emphasis').strip("# ") + emphasis_text = m.group('emphasis').strip('# ') if len(emphasis_text) != 0: emphasis = ElementTree.SubElement(h, 'span') emphasis.set('c2c:role', 'header-emphasis') @@ -58,9 +61,8 @@ def run(self, parent, blocks): class C2CHeaderExtension(Extension): def extendMarkdown(self, md): # noqa: N802 md.parser.blockprocessors.register( - C2CHeaderProcessor(md.parser), - 'c2c_header_emphasis', - 73) + C2CHeaderProcessor(md.parser), 'c2c_header_emphasis', 73 + ) def makeExtension(configs=[]): # noqa: N802 diff --git a/c2corg_api/markdown/img.py b/c2corg_api/markdown/img.py index 63360a602..207ab9434 100644 --- a/c2corg_api/markdown/img.py +++ b/c2corg_api/markdown/img.py @@ -1,17 +1,17 @@ -''' +""" c2corg img Extension for Python-Markdown ============================================== Converts tags like [img=()(/)?]()?([/img])? to advanced HTML img tags. -''' +""" -from markdown.extensions import Extension -from markdown.blockprocessors import BlockProcessor +import re from xml.etree import ElementTree # nosec -import re +from markdown.blockprocessors import BlockProcessor +from markdown.extensions import Extension # \w\W pattern is a trick for capturing all, including new spaces IMG_RE = r'(?:^|\n)\[img=(\d+)([a-z_ ]*)(/\]|\]([\w\W]*?)\[/img\])' @@ -21,10 +21,8 @@ class C2CImageExtension(Extension): def extendMarkdown(self, md): # noqa: N802 self.md = md md.parser.blockprocessors.register( - C2CImageBlock(md.parser, - self.getConfigs()), - 'c2c_imgblock', - 15) + C2CImageBlock(md.parser, self.getConfigs()), 'c2c_imgblock', 15 + ) class C2CImageBlock(BlockProcessor): @@ -41,12 +39,12 @@ def run(self, parent, blocks): block = blocks.pop(0) m = self.RE.search(block) - before = block[:m.start()] + before = block[: m.start()] self.parser.parseBlocks(parent, [before]) parent.append(self.build_element(m)) - after = block[m.end():] + after = block[m.end() :] self.parser.parseBlocks(parent, [after]) def build_element(self, m): @@ -70,12 +68,12 @@ def build_element(self, m): img = ElementTree.Element('img') - img_url = '/images/proxy/%s' % (img_id, ) + img_url = '/images/proxy/%s' % (img_id,) if img_size: img_url += '?size=' + img_size - img.set('alt', (caption or img_id).replace("\n", " ")) + img.set('alt', (caption or img_id).replace('\n', ' ')) img.set('c2c:url-proxy', img_url) img.set('c2c:role', 'embedded-image') img.set('c2c:document-id', img_id) diff --git a/c2corg_api/markdown/ltag.py b/c2corg_api/markdown/ltag.py index a9b1ad23d..fdf3009d2 100644 --- a/c2corg_api/markdown/ltag.py +++ b/c2corg_api/markdown/ltag.py @@ -7,11 +7,12 @@ See https://regex101.com/r/bVjbkp/6 """ -from markdown.extensions import Extension +import re +from xml.etree import ElementTree # nosec + from markdown.blockprocessors import BlockProcessor +from markdown.extensions import Extension from markdown.treeprocessors import Treeprocessor -from xml.etree import ElementTree # nosec -import re def _pairwise(iterable): @@ -30,34 +31,34 @@ def _get_ltag_pattern(): Please have a look on https://forum.camptocamp.org/t/question-l/207148/69 """ - p = "(?P<{}>{})".format + p = '(?P<{}>{})'.format # small patterns used more than once raw_label = r"[a-zA-Z'\"][a-zA-Z'\"\d_]*" # let's build multi pitch pattern, like L#1-2 or L#12bis-14 - multi_pitch_label = p("multi_pitch_label", raw_label) - first_offset = p("first_offset", r"\d+") - last_offset = p("last_offset", r"\d+") - first_pitch = p("first_pitch", first_offset + multi_pitch_label + "?") - last_pitch = p("last_pitch", last_offset) - multi_pitch = p("multi_pitch", first_pitch + "-" + last_pitch) + multi_pitch_label = p('multi_pitch_label', raw_label) + first_offset = p('first_offset', r'\d+') + last_offset = p('last_offset', r'\d+') + first_pitch = p('first_pitch', first_offset + multi_pitch_label + '?') + last_pitch = p('last_pitch', last_offset) + multi_pitch = p('multi_pitch', first_pitch + '-' + last_pitch) # mono pitch, like L#, L#12 or L#13bis - mono_pitch_label = p("mono_pitch_label", raw_label) - mono_pitch_value = p("mono_pitch_value", r"\d*") - mono_pitch = p("mono_pitch", mono_pitch_value + mono_pitch_label + "?") + mono_pitch_label = p('mono_pitch_label', raw_label) + mono_pitch_value = p('mono_pitch_value', r'\d*') + mono_pitch = p('mono_pitch', mono_pitch_value + mono_pitch_label + '?') - numbering = p("numbering", multi_pitch + "|" + mono_pitch) + numbering = p('numbering', multi_pitch + '|' + mono_pitch) - text_in_the_middle = p("text_in_the_middle", "~") - header = p("header", "=") + text_in_the_middle = p('text_in_the_middle', '~') + header = p('header', '=') - typ = p("type", "[LR]") + typ = p('type', '[LR]') - text = "(" + header + "|" + text_in_the_middle + "|" + numbering + ")" + text = '(' + header + '|' + text_in_the_middle + '|' + numbering + ')' - return p("ltag", typ + "#" + text) + return p('ltag', typ + '#' + text) class LTagNumbering(object): @@ -75,9 +76,7 @@ class LTagNumbering(object): PATTERN = re.compile(_get_ltag_pattern()) # helper for final formatting - FORMAT = ('' - '{type}' - '{text}').format + FORMAT = ('{type}{text}').format FORMAT_UNMATCHED = '{}'.format def __init__(self, markdown): @@ -85,7 +84,7 @@ def __init__(self, markdown): self.get_placeholder = markdown.htmlStash.store # Values for relative patterns - self.value = {"R": 0, "L": 0} + self.value = {'R': 0, 'L': 0} # One way switch self.supported = True @@ -122,20 +121,20 @@ def compute(self, markdown, row_type, is_first_cell): # must access to row_type and is_first_cell def handle_match(match): - if match.group("header") is not None: # means L#= - result = "" if is_first_cell else match.group(0) + if match.group('header') is not None: # means L#= + result = '' if is_first_cell else match.group(0) - elif match.group("text_in_the_middle") is not None: + elif match.group('text_in_the_middle') is not None: result = match.group(0) - elif match.group("multi_pitch") is not None: + elif match.group('multi_pitch') is not None: result = self.handle_multipitch(match, is_first_cell) - elif match.group("mono_pitch") is not None: + elif match.group('mono_pitch') is not None: result = self.handle_monopitch(match, row_type, is_first_cell) else: - raise NotImplementedError("Should not happen!?") + raise NotImplementedError('Should not happen!?') return self.get_placeholder(result) @@ -156,7 +155,7 @@ def compute_label(self, raw_label): self.contains_label = True else: - raw_label = "" + raw_label = '' return raw_label @@ -167,16 +166,16 @@ def handle_multipitch(self, match, is_first_cell): L#1-4 L#1bis-4 """ - label = self.compute_label(match.group("multi_pitch_label")) + label = self.compute_label(match.group('multi_pitch_label')) - typ = match.group("type") - first_offset = match.group("first_offset") - last_offset = match.group("last_offset") + typ = match.group('type') + first_offset = match.group('first_offset') + last_offset = match.group('last_offset') if is_first_cell: # first cell impacts numbering self.value[typ] = int(last_offset) - text = "".join((first_offset, label, "-", last_offset, label)) + text = ''.join((first_offset, label, '-', last_offset, label)) return self.FORMAT(type=typ, text=text) @@ -189,9 +188,9 @@ def handle_monopitch(self, match, row_type, is_first_cell): L#13bis """ - label = self.compute_label(match.group("mono_pitch_label")) - typ = match.group("type") - value = match.group("mono_pitch_value") + label = self.compute_label(match.group('mono_pitch_label')) + typ = match.group('type') + value = match.group('mono_pitch_value') if value.isdigit(): # Fixed number : L#12 @@ -204,7 +203,7 @@ def handle_monopitch(self, match, row_type, is_first_cell): elif len(value) == 0: # Simple use case : L# self.allow_labels = False - assert not self.contains_label, "Not yet supported" + assert not self.contains_label, 'Not yet supported' value = self.value[typ if is_first_cell else row_type] @@ -215,7 +214,7 @@ def handle_monopitch(self, match, row_type, is_first_cell): return self.FORMAT(type=typ, text=str(value)) else: - raise NotImplementedError("Should not happen") + raise NotImplementedError('Should not happen') class LTagProcessor(BlockProcessor): @@ -225,7 +224,7 @@ class LTagProcessor(BlockProcessor): and will be handled by TreeProcessor """ - RE_TESTER = re.compile(r"(?:^|\n) {0,3}([LR]#)") + RE_TESTER = re.compile(r'(?:^|\n) {0,3}([LR]#)') RE_PIPE_SAVER = re.compile(r'(\|)(?![^|]*\]\])') CELL_SEPARATOR = '__--|--__' @@ -242,8 +241,9 @@ def run(self, parent, blocks): self.parser.parseBlocks(parent, [before_ltag]) # Build XML elements - table = ElementTree.SubElement(parent, 'table', {'c2c:role': 'ltag', - 'class': 'ltag'}) + table = ElementTree.SubElement( + parent, 'table', {'c2c:role': 'ltag', 'class': 'ltag'} + ) tbody = ElementTree.SubElement(table, 'tbody') col_count = 0 @@ -266,16 +266,16 @@ def _build_row(self, markdown, tbody, colspan_rows): and return row object """ - row = ElementTree.SubElement(tbody, 'tr', {"tag": markdown[0]}) + row = ElementTree.SubElement(tbody, 'tr', {'tag': markdown[0]}) marker = markdown[2:3] - if marker == "~": # the pattern L#~ - cell = ElementTree.SubElement(row, "td") - cell.text = markdown[3:].strip(" \n") + if marker == '~': # the pattern L#~ + cell = ElementTree.SubElement(row, 'td') + cell.text = markdown[3:].strip(' \n') colspan_rows.append(row) else: - cell_node_name = "th" if marker == "=" else "td" # the pattern L#= + cell_node_name = 'th' if marker == '=' else 'td' # the pattern L#= # replace separator by cell_separator to protect links markdown = self.RE_PIPE_SAVER.sub(self.CELL_SEPARATOR, markdown) @@ -283,7 +283,7 @@ def _build_row(self, markdown, tbody, colspan_rows): # and split markdown for cell_markdown in markdown.split(self.CELL_SEPARATOR): cell = ElementTree.SubElement(row, cell_node_name) - cell.text = cell_markdown.strip(" \n\xa0") + cell.text = cell_markdown.strip(' \n\xa0') return row @@ -299,7 +299,7 @@ def run(self, root): numbering = LTagNumbering(self.md) def compute(node, first_cell): - if node.tag not in ("code", "pre"): + if node.tag not in ('code', 'pre'): node.text = numbering.compute(node.text, row_type, first_cell) for child in node: @@ -307,9 +307,9 @@ def compute(node, first_cell): child.tail = numbering.compute(child.tail, row_type, False) for row in root.findall("table[@class='ltag']/tbody/tr"): - is_text_in_the_middle = row[0].get("colspan") is not None + is_text_in_the_middle = row[0].get('colspan') is not None - row_type = row.get("tag") + row_type = row.get('tag') for i, cell in enumerate(row): is_first_cell = i == 0 and not is_text_in_the_middle @@ -323,16 +323,13 @@ def compute(node, first_cell): class C2CLTagExtension(Extension): - """ Add tables to Markdown. """ + """Add tables to Markdown.""" def extendMarkdown(self, md): # noqa: N802 - """ Add an instance of TableProcessor to BlockParser. """ + """Add an instance of TableProcessor to BlockParser.""" if '|' not in md.ESCAPED_CHARS: md.ESCAPED_CHARS.append('|') - md.parser.blockprocessors.register( - LTagProcessor(md.parser), - 'c2c_ltag', - 75) + md.parser.blockprocessors.register(LTagProcessor(md.parser), 'c2c_ltag', 75) md.treeprocessors.register(LtagTreeprocessor(md), 'c2c_ltag', 0) diff --git a/c2corg_api/markdown/nbsp.py b/c2corg_api/markdown/nbsp.py index ef59570dc..406391804 100644 --- a/c2corg_api/markdown/nbsp.py +++ b/c2corg_api/markdown/nbsp.py @@ -3,21 +3,20 @@ class NbspPattern(Pattern): - HTML_ENTITY = " " + HTML_ENTITY = ' ' def handleMatch(self, m): # noqa: N802 placeholder = self.md.htmlStash.store(self.HTML_ENTITY) - return m.group(2).replace(" ", placeholder) + return m.group(2).replace(' ', placeholder) class NarrowNbspPattern(NbspPattern): - HTML_ENTITY = " " + HTML_ENTITY = ' ' class C2CNbspExtension(Extension): def extendMarkdown(self, md): # noqa: N802 - """ patterns like @@ -26,15 +25,11 @@ def extendMarkdown(self, md): # noqa: N802 must have a non-breakable space instead of a space. """ - md.inlinePatterns.register( - NbspPattern(r'(\d [a-z]| :)', md), - 'c2c_nbsp', - 7) + md.inlinePatterns.register(NbspPattern(r'(\d [a-z]| :)', md), 'c2c_nbsp', 7) md.inlinePatterns.register( - NarrowNbspPattern(r'([\w\d] [;?!])', md), - 'c2c_nnbsp', - 8) + NarrowNbspPattern(r'([\w\d] [;?!])', md), 'c2c_nnbsp', 8 + ) def makeExtension(*args, **kwargs): # noqa: N802 diff --git a/c2corg_api/markdown/ptag.py b/c2corg_api/markdown/ptag.py index c5868a670..9fd160ab9 100644 --- a/c2corg_api/markdown/ptag.py +++ b/c2corg_api/markdown/ptag.py @@ -1,24 +1,22 @@ -''' +""" c2corg p Extension for Python-Markdown ============================================== Converts tags [p] to div with clear:both -''' +""" -from markdown.extensions import Extension -from markdown.blockprocessors import BlockProcessor -from xml.etree import ElementTree # nosec import re +from xml.etree import ElementTree # nosec + +from markdown.blockprocessors import BlockProcessor +from markdown.extensions import Extension P_RE = r'(?:\n|^)\[p\](?:\n|$)' class C2CPTagExtension(Extension): def extendMarkdown(self, md): # noqa: N802 - md.parser.blockprocessors.register( - C2CPTag(md.parser), - 'c2c_ptag', - 10.25) + md.parser.blockprocessors.register(C2CPTag(md.parser), 'c2c_ptag', 10.25) class C2CPTag(BlockProcessor): @@ -31,12 +29,12 @@ def run(self, parent, blocks): block = blocks.pop(0) m = self.RE.search(block) - before = block[:m.start()] + before = block[: m.start()] self.parser.parseBlocks(parent, [before]) - parent.append(ElementTree.Element('div', {"style": "clear:both"})) + parent.append(ElementTree.Element('div', {'style': 'clear:both'})) - after = block[m.end():] + after = block[m.end() :] self.parser.parseBlocks(parent, [after]) diff --git a/c2corg_api/markdown/toc.py b/c2corg_api/markdown/toc.py index 4aa806aa7..de6e21573 100644 --- a/c2corg_api/markdown/toc.py +++ b/c2corg_api/markdown/toc.py @@ -14,15 +14,22 @@ Modified for C2C : remove title emphasis, toc HTML class. Add c2c:role """ +from markdown.extensions.toc import ( + AtomicString, + TocExtension, + TocTreeprocessor, + html, + nest_toc_tokens, + run_postprocessors, + strip_tags, + unescape, + unique, +) from markdown.util import code_escape -from markdown.extensions.toc import (TocExtension, TocTreeprocessor, - stashedHTML2text, unescape, - unique, nest_toc_tokens, - AtomicString, html) def not_emphasis(elt): - return elt.attrib.get("c2c:role") != "header-emphasis" + return elt.attrib.get('c2c:role') != 'header-emphasis' def get_name(el): @@ -64,8 +71,8 @@ def run(self, doc): # Get a list of id attributes used_ids = set() for el in doc.iter(): - if "id" in el.attrib: - used_ids.add(el.attrib["id"]) + if 'id' in el.attrib: + used_ids.add(el.attrib['id']) toc_tokens = [] for el in doc.iter(): @@ -74,33 +81,46 @@ def run(self, doc): text = get_name(el) # Do not override pre-existing ids - if "id" not in el.attrib: - innertext = unescape(stashedHTML2text(text, self.md)) - el.attrib["id"] = unique(self.slugify(innertext, self.sep), used_ids) # noqa: E501 - - if int(el.tag[-1]) >= self.toc_top and int(el.tag[-1]) <= self.toc_bottom: # noqa: E501 - toc_tokens.append({ - 'level': int(el.tag[-1]), - 'id': el.attrib["id"], - 'name': unescape(stashedHTML2text( - code_escape(el.attrib.get('data-toc-label', text)), - self.md, strip_entities=False - )) - }) + if 'id' not in el.attrib: + innertext = strip_tags(run_postprocessors(unescape(text), self.md)) # noqa: E501 + el.attrib['id'] = unique( + self.slugify(innertext, self.sep), used_ids + ) # noqa: E501 + + if ( + int(el.tag[-1]) >= self.toc_top + and int(el.tag[-1]) <= self.toc_bottom + ): # noqa: E501 + toc_tokens.append( + { + 'level': int(el.tag[-1]), + 'id': el.attrib['id'], + 'name': strip_tags( + run_postprocessors( + unescape( + code_escape( + el.attrib.get('data-toc-label', text) + ) + ), + self.md, + ) + ), + } + ) # Remove the data-toc-label attribute as it is no longer needed if 'data-toc-label' in el.attrib: del el.attrib['data-toc-label'] if self.use_anchors: - self.add_anchor(el, el.attrib["id"]) + self.add_anchor(el, el.attrib['id']) if self.use_permalinks not in [False, None]: - self.add_permalink(el, el.attrib["id"]) + self.add_permalink(el, el.attrib['id']) toc_tokens = nest_toc_tokens(toc_tokens) div = self.build_toc_div(toc_tokens) - div.attrib["c2c:role"] = "toc" - del div.attrib["class"] + div.attrib['c2c:role'] = 'toc' + del div.attrib['class'] if self.marker: self.replace_marker(doc, div) diff --git a/c2corg_api/markdown/video.py b/c2corg_api/markdown/video.py index c3ab21979..9ff11ba1a 100644 --- a/c2corg_api/markdown/video.py +++ b/c2corg_api/markdown/video.py @@ -1,19 +1,20 @@ -''' +""" c2corg video Extension for Python-Markdown ============================================== Converts video tags to advanced HTML video tags. -''' +""" -from markdown.extensions import Extension -from markdown.blockprocessors import BlockProcessor -from xml.etree import ElementTree # nosec import re +from xml.etree import ElementTree # nosec + +from markdown.blockprocessors import BlockProcessor +from markdown.extensions import Extension class C2CVideoExtension(Extension): def __init__(self, *args, **kwargs): - self._iframe_secret_tag = kwargs.pop("iframe_secret_tag") + self._iframe_secret_tag = kwargs.pop('iframe_secret_tag') super(C2CVideoExtension, self).__init__(*args, **kwargs) def extendMarkdown(self, md): # noqa: N802 @@ -25,12 +26,12 @@ def extendMarkdown(self, md): # noqa: N802 (C2CYoutubeShortVideoBlock, 13), (C2CDailymotionVideoBlock, 12), (C2CDailymotionShortVideoBlock, 11), - (C2CVimeoVideoBlock, 10.5) + (C2CVimeoVideoBlock, 10.5), ): processors.register( processor(md.parser, self._iframe_secret_tag), processor.__name__, - priority + priority, ) @@ -40,9 +41,7 @@ class C2CVideoBlock(BlockProcessor): def __init__(self, parser, iframe_secret_tag): super(C2CVideoBlock, self).__init__(parser=parser) self._iframe_secret_tag = iframe_secret_tag - self.RE = re.compile(r"(^|\n)\[video\]" + - self.PATTERN + - r"\[/video\]") + self.RE = re.compile(r'(^|\n)\[video\]' + self.PATTERN + r'\[/video\]') def test(self, parent, block): return bool(self.RE.search(block)) @@ -51,12 +50,12 @@ def run(self, parent, blocks): block = blocks.pop(0) m = self.RE.search(block) - before = block[:m.start()] + before = block[: m.start()] self.parser.parseBlocks(parent, [before]) parent.append(self.build_element(m)) - after = block[m.end():] + after = block[m.end() :] self.parser.parseBlocks(parent, [after]) def build_element(self, m): @@ -72,39 +71,47 @@ def _embed(self, link): class C2CYoutubeVideoBlock(C2CVideoBlock): - PATTERN = (r"https?:\/\/(?:www\.)?youtube\.com" - r"/watch\?(?:[=&\w]+&)?v=([-\w]+)(?:&.+)?(?:\#.*)?") + PATTERN = ( + r'https?:\/\/(?:www\.)?youtube\.com' + r'/watch\?(?:[=&\w]+&)?v=([-\w]+)(?:&.+)?(?:\#.*)?' + ) def build_element(self, m): return self._embed('//www.youtube.com/embed/' + m.group(2)) class C2CYoutubeShortVideoBlock(C2CYoutubeVideoBlock): - PATTERN = r"https?:\/\/(?:www\.)?youtu\.be/([-\w]+)(?:\#.*)?" + PATTERN = r'https?:\/\/(?:www\.)?youtu\.be/([-\w]+)(?:\#.*)?' class C2CDailymotionVideoBlock(C2CVideoBlock): - PATTERN = (r"https?://(?:www\.)?dailymotion\.com" - r"/video/([\da-zA-Z]+)_[-&;\w]+(?:\#.*)?") + PATTERN = ( + r'https?://(?:www\.)?dailymotion\.com' + r'/video/([\da-zA-Z]+)_[-&;\w]+(?:\#.*)?' + ) def build_element(self, m): - return self._embed('//www.dailymotion.com/embed/video/' + - m.group(2) + - '?theme=none&wmode=transparent') + return self._embed( + '//www.dailymotion.com/embed/video/' + + m.group(2) + + '?theme=none&wmode=transparent' + ) class C2CDailymotionShortVideoBlock(C2CDailymotionVideoBlock): - PATTERN = r"https?://(?:www\.)?dai\.ly/([\da-zA-Z]+)" + PATTERN = r'https?://(?:www\.)?dai\.ly/([\da-zA-Z]+)' class C2CVimeoVideoBlock(C2CVideoBlock): PATTERN = r'https?://(?:www\.)?vimeo\.com/(\d+)(?:\#.*)?' def build_element(self, m): - return self._embed('//player.vimeo.com/video/' + - m.group(2) + - '?title=0&byline=0' + - '&portrait=0&color=ff9933') + return self._embed( + '//player.vimeo.com/video/' + + m.group(2) + + '?title=0&byline=0' + + '&portrait=0&color=ff9933' + ) def makeExtension(*args, **kwargs): # noqa: N802 diff --git a/c2corg_api/markdown/wikilinks.py b/c2corg_api/markdown/wikilinks.py index 0daafa875..2c5cf62ab 100644 --- a/c2corg_api/markdown/wikilinks.py +++ b/c2corg_api/markdown/wikilinks.py @@ -1,4 +1,4 @@ -''' +""" c2corg wikiLinks Extension for Python-Markdown ============================================== @@ -7,11 +7,12 @@ Inspired from https://github.com/waylan/Python-Markdown/blob/master /markdown/extensions/wikilinks.py -''' +""" + +from xml.etree import ElementTree # nosec from markdown.extensions import Extension from markdown.inlinepatterns import Pattern -from xml.etree import ElementTree # nosec # document_type/document_id(/lang(/slug))(#anchor) @@ -20,11 +21,7 @@ ANCHOR_RE = r'(#(?P[a-z0-9\-_]+))?' LABEL_RE = r'(?P