diff --git a/.cortex/plans/dual-os-plan.md b/.cortex/plans/dual-os-plan.md new file mode 100644 index 0000000..ae1140c --- /dev/null +++ b/.cortex/plans/dual-os-plan.md @@ -0,0 +1,46 @@ +# Plan: Dual-OS Build for avior-go (Windows + Linux) + +## Context + +avior-go currently only compiles for Windows because `encoder/encoder.go` directly uses +the Windows API (`golang.org/x/sys/windows`: `OpenProcess`, `SetPriorityClass`, +`CloseHandle`) to set the process priority of the spawned `ffmpeg` process. +Goal: The same code should optionally produce a Windows executable (`avior-go.exe`) or a +Linux binary (`avior-go`) via Go build tags / `GOOS`, so that avior-go instances can run +under Linux in Docker in the future. + +Facts from the code (verified in this session): +- Only Windows dependency in the entire repo: `encoder/encoder.go` lines 176–188 + (`windows.OpenProcess` / `windows.SetPriorityClass` / `windows.CloseHandle`), plus the + import `"golang.org/x/sys/windows"` on line 23. +- All path operations already use `filepath.Join` (app.go, config/config.go, + api/api.go) — OS-agnostic. UNC paths exist only in config JSONs (user data, not a + code problem). +- All Go dependencies in `go.mod` are pure Go (gorilla, redis, mongo-driver, glg, + godirwalk, lumberjack …) → `CGO_ENABLED=0` cross-compile is possible, no C toolchain + needed. +- CI: `.github/workflows/go.yml` currently builds only `goos: windows`, `goarch: amd64` + via `wangyoucao577/go-release-action@v1.18`. +- Go version per go.mod: `go 1.25.0`. + +End state: `GOOS=windows go build` and `GOOS=linux go build` both work; ffmpeg priority +is set per OS via build-tag files (Windows: PriorityClass, Linux: nice level); CI builds +both artifacts. +## Tasks (one MD file each) + +1. `dual-os-task-01-priority-build-tags.md` — Extract OS-specific priority setting + from `encoder/encoder.go` into build-tag files. +2. `dual-os-task-02-build-switches.md` — Build switches (Makefile + scripts) for + Windows and Linux executables. +3. `dual-os-task-03-ci-matrix.md` — Extend GitHub Actions workflow to a + Windows+Linux matrix. +4. `dual-os-task-04-verification.md` — Verification: cross-compile both targets, + tests, smoke check. +5. `dual-os-task-05-docker-compose.md` — Dockerfile + compose.yaml for Komodo deployment + (mount `/mnt/user/media` → `/media`). + +Dependencies: Task 01 first (otherwise Linux won't compile). Tasks 02 and 03 are +independent of each other, but both require Task 01. Task 05 requires Task 01 (uses the +same Linux build). Task 04 last. + + diff --git a/.cortex/plans/dual-os-task-01-priority-build-tags.md b/.cortex/plans/dual-os-task-01-priority-build-tags.md new file mode 100644 index 0000000..8a0767d --- /dev/null +++ b/.cortex/plans/dual-os-task-01-priority-build-tags.md @@ -0,0 +1,124 @@ +# Task 01: OS-specific ffmpeg Priority via Build Tags + +## Goal + +`encoder/encoder.go` won't compile under Linux (direct import of +`golang.org/x/sys/windows`). The priority setting is extracted into two build-tag files; +`encoder.go` only calls an OS-neutral helper function. + +## Edits + +### 1. New file `encoder/priority_windows.go` + +```go +//go:build windows + +package encoder + +import ( + "os/exec" + + "github.com/Spiritreader/avior-go/config" + "github.com/kpango/glg" + "golang.org/x/sys/windows" +) + +// setProcessPriority sets the Windows PriorityClass of the spawned ffmpeg process. +// Errors are only logged (as before), since the encoding itself doesn't depend on it. +func setProcessPriority(cmd *exec.Cmd, cfg *config.Data) { + hProcess, err := windows.OpenProcess(0x0400|0x0200, false, uint32(cmd.Process.Pid)) + if err != nil { + _ = glg.Warnf("could not get ffmpeg handle using pid %d, err: %s", cmd.Process.Pid, err) + return + } + defer func() { + if err := windows.CloseHandle(hProcess); err != nil { + _ = glg.Errorf("could not close handle for pid %d, err: %s", cmd.Process.Pid, err) + } + }() + if err := windows.SetPriorityClass(hProcess, config.PriorityUint32(cfg.Local.EncoderPriority)); err != nil { + _ = glg.Warnf("could not set priority %s for ffmpeg handle using pid %d, err: %s", + cfg.Local.EncoderPriority, cmd.Process.Pid, err) + } +} +``` + +Behavior change over the original: `return` after `OpenProcess` error (avoids +`SetPriorityClass` on invalid handle) and `CloseHandle` via `defer`. This is the +correct version of the existing code — no functional change in the success case. Log +strings remain identical, except the `SetPriorityClass` log outputs +`cfg.Local.EncoderPriority` instead of `cfg.Local.EncoderConfig` (fix for a copy-paste +error in the original, line 183). + +### 2. New file `encoder/priority_linux.go` + +```go +//go:build linux + +package encoder + +import ( + "os/exec" + "syscall" + + "github.com/Spiritreader/avior-go/config" + "github.com/kpango/glg" +) + +// niceLevel maps the configured Windows priority level to a Linux nice value. +// Mapping: HIGH/ABOVE_NORMAL -> -5 (higher priority, requires root/CAP_SYS_NICE), +// NORMAL -> 0, BELOW_NORMAL -> 10, IDLE -> 19. +// Unknown values fall back to 19 (idle) — analogous to the Windows fallback in +// config.PriorityUint32, which returns IDLE for unknown values. +func niceLevel(priority string) int { + switch priority { + case config.PRIORITY_HIGH.String(), config.PRIORITY_ABOVE_NORMAL.String(): + return -5 + case config.PRIORITY_NORMAL.String(): + return 0 + case config.PRIORITY_BELOW_NORMAL.String(): + return 10 + default: + return 19 + } +} + +// setProcessPriority sets the nice level of the spawned ffmpeg process. +// An error (e.g. missing permissions for negative nice values in a Docker container) +// is only logged as a warning; encoding proceeds normally. +func setProcessPriority(cmd *exec.Cmd, cfg *config.Data) { + if err := syscall.Setpriority(syscall.PRIO_PROCESS, cmd.Process.Pid, niceLevel(cfg.Local.EncoderPriority)); err != nil { + _ = glg.Warnf("could not set priority %s for ffmpeg process with pid %d, err: %s", + cfg.Local.EncoderPriority, cmd.Process.Pid, err) + } +} +``` + +Note: In Docker without `CAP_SYS_NICE`, negative nice values will fail — this is +acceptable (warning in log, encoding runs with default priority). The container docs +may later recommend `--cap-add SYS_NICE`, but that is not part of this plan. + +### 3. Adapt `encoder/encoder.go` + +- Remove import `"golang.org/x/sys/windows"` (line 23). `config` stays imported + (still used elsewhere in the file — verified: `cfg.Local.*` throughout). +- Replace the block on lines 176–188 (the three `windows.*` calls with error logs) with: + +```go + setProcessPriority(cmd, cfg) +``` + +The call goes directly after the successful `cmd.Start()`, exactly where the old block +was. + +## No further callsites + +`grep` for `x/sys` and `windows\.` in the entire repo: only `encoder/encoder.go`. +`config.PriorityUint32` remains unchanged (still used by `priority_windows.go`). +Clean cutover: no compatibility code. + +## Check + +- `GOOS=windows go build ./...` and `GOOS=linux go build ./...` from the repo root + both compile without errors. +- `go vet ./encoder` without findings. diff --git a/.cortex/plans/dual-os-task-02-build-switches.md b/.cortex/plans/dual-os-task-02-build-switches.md new file mode 100644 index 0000000..89f318e --- /dev/null +++ b/.cortex/plans/dual-os-task-02-build-switches.md @@ -0,0 +1,82 @@ +# Task 02: Build Switches for Windows and Linux Executables + +## Goal + +One command per target platform produces the final binary. Go compiles cross-platform +natively via the `GOOS` switch; all dependencies are pure Go, therefore `CGO_ENABLED=0` +(static binary, ideal for slim Docker images). + +## Edits + +### 1. New `Makefile` in repo root + +```make +BINARY := avior-go +VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +LDFLAGS := -s -w -X main.buildVersion=$(VERSION) + +.PHONY: build-windows build-linux build all + +build-windows: + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o dist/$(BINARY)-windows-amd64.exe app.go + +build-linux: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o dist/$(BINARY)-linux-amd64 app.go + +build: build-windows + +all: build-windows build-linux +``` + +Notes: +- Entry point is `app.go` (package main in the repo root; verified via + `.vscode/launch.json`, which launches `app.go` as the program). +- `-ldflags "-s -w"` strips symbols (smaller binary for containers). +- `main.buildVersion` is only set if the variable exists — unverified whether + `buildVersion` exists in app.go. If `go build` fails with "no such variable": + reduce ldflags to `-s -w` (fallback, no code requirement). + +### 2. Alternative scripts for systems without make + +`tools/build.ps1` (Windows development, reusing existing `tools/` convention): + +```powershell +param([ValidateSet("windows","linux","all")][string]$Target = "windows") +$env:CGO_ENABLED = "0" +switch ($Target) { + "windows" { $env:GOOS="windows"; $env:GOARCH="amd64"; go build -ldflags "-s -w" -o dist/avior-go-windows-amd64.exe app.go } + "linux" { $env:GOOS="linux"; $env:GOARCH="amd64"; go build -ldflags "-s -w" -o dist/avior-go-linux-amd64 app.go } + "all" { & $PSCommandPath -Target windows; & $PSCommandPath -Target linux } +} +``` + +`tools/build.sh` (Linux/macOS): + +```sh +#!/bin/sh +set -e +target="${1:-linux}" +export CGO_ENABLED=0 GOARCH=amd64 +case "$target" in + windows) GOOS=windows go build -ldflags "-s -w" -o dist/avior-go-windows-amd64.exe app.go ;; + linux) GOOS=linux go build -ldflags "-s -w" -o dist/avior-go-linux-amd64 app.go ;; + all) "$0" windows && "$0" linux ;; + *) echo "usage: $0 [windows|linux|all]" >&2; exit 1 ;; +esac +``` + +### 3. Update `.gitignore` + +Add `dist/` (the `*.exe` line already exists, but doesn't cover `dist/` on Linux where +binaries have no `.exe` extension). + +## Usage (the "switches") + +- Windows executable: `make build-windows` or `./tools/build.ps1 -Target windows` +- Linux binary: `make build-linux` or `./tools/build.sh linux` + +## Check + +From repo root: +- `make all` produces `dist/avior-go-windows-amd64.exe` AND `dist/avior-go-linux-amd64`. +- `file dist/avior-go-linux-amd64` (under WSL/Git Bash) reports `ELF 64-bit ... statically linked`. diff --git a/.cortex/plans/dual-os-task-03-ci-matrix.md b/.cortex/plans/dual-os-task-03-ci-matrix.md new file mode 100644 index 0000000..28832ed --- /dev/null +++ b/.cortex/plans/dual-os-task-03-ci-matrix.md @@ -0,0 +1,58 @@ +# Task 03: Extend CI Workflow to Windows+Linux + +## Goal + +`.github/workflows/go.yml` builds both binaries on release and attaches them to the +release. The existing `wangyoucao577/go-release-action@v1.18` step is parameterized +per OS (matrix) instead of introducing a second mechanism (preserving the existing +convention). + +## Edit: Replace `.github/workflows/go.yml` entirely with + +```yaml +name: Go + +on: + release: + types: [created, edited] + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - goos: windows + goarch: amd64 + binary_name: avior-go + - goos: linux + goarch: amd64 + binary_name: avior-go + steps: + - uses: actions/checkout@v2 + - uses: wangyoucao577/go-release-action@v1.18 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + binary_name: ${{ matrix.binary_name }} + ldflags: -s -w +``` + +Reason: matrix instead of two hard jobs: same configuration, artifacts both land on the +same release (`go-release-action` attaches one asset per `goos`; Windows asset is +named `avior-go_windows_amd64.exe.zip`, Linux `avior-go_linux_amd64.tar.gz` — +naming convention of the action, verify on first release). + +## Check + +- Validate workflow YAML locally: `python -c "import yaml,sys; yaml.safe_load(open('.github/workflows/go.yml'))"`. +- Real check only possible on next release tag — alternative verification: + simulate the matrix run locally with the same commands + (`CGO_ENABLED=0 GOOS=linux go build ./...`, see Task 04). + +## Dependency + +Task 01 must be merged, otherwise the Linux target will fail in CI. diff --git a/.cortex/plans/dual-os-task-04-verification.md b/.cortex/plans/dual-os-task-04-verification.md new file mode 100644 index 0000000..5732845 --- /dev/null +++ b/.cortex/plans/dual-os-task-04-verification.md @@ -0,0 +1,62 @@ +# Task 04: Verification of the Dual-OS Build + +## Goal + +Prove that both targets compile from the same code and that the behavior of the +changed code paths (Task 01) is unchanged or correct. + +Working directory for all commands: repo root `C:/repos/avior-go`. No env vars or +fixtures needed; ffmpeg/ffprobe are not required for the pure build checks. + +## Checks (in this order) + +1. **Both targets compile** (exercises Tasks 01 + 02): + ``` + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/avior-go-windows-amd64.exe app.go + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/avior-go-linux-amd64 app.go + ``` + Expected: both commands exit 0, both files exist. Before Task 01, the Linux command + failed with `build constraints exclude all Go files ... x/sys/windows` — exactly + this error must be gone. + +2. **Linux binary is static** (Docker suitability): + ``` + file dist/avior-go-linux-amd64 + ``` + Expected: `ELF 64-bit LSB executable, x86-64, ... statically linked`. + (`file` via Git Bash/WSL; if unavailable: skip — building with `CGO_ENABLED=0` + guarantees static linking.) + +3. **Vet + existing tests** (regression): + ``` + go vet ./... + go test ./... + ``` + Expected: no new findings; existing tests (including the `config` package) pass. + Windows-specific paths are tested on the dev Windows machine; the Linux path + (`priority_linux.go`) has no unit tests — intentional, it only wraps a syscall + with logging. + +4. **Windows smoke test (behavior unchanged)**: + Start `avior-go-windows-amd64.exe` with the existing `config_dev.json`, run an + encode job, and verify in the log (`log/main.log`) that NO new warning + `could not set priority ... for ffmpeg handle` appears — priority setting works + as before. + +5. **Linux smoke test (new behavior)**: + Start the Linux binary in a container or via WSL (`./avior-go-linux-amd64` alongside + a `config.json` with Linux paths in `MediaPaths`/`OutDirectory` and reachable + MongoDB/Redis). Once an encode job runs: the log must not contain a + `could not set priority` warning (unless the container runs without permissions + for negative nice values at `HIGH`/`ABOVE_NORMAL` — then exactly one warning is + acceptable and documented; encoding continues). + Additionally during an encode: `ps -o pid,ni,cmd -C ffmpeg` — the `NI` field must + match the configured mapping (IDLE → 19, NORMAL → 0, BELOW_NORMAL → 10). + +## Abort criteria / fallbacks + +- Check 1 fails for Linux with import error on `x/sys/windows`: Task 01 incomplete + (import not removed or build tag missing/misspelled — tag must be exactly + `//go:build windows` or `//go:build linux` as the first line). +- Check 3 fails in `encoder` package: signature + `setProcessPriority(cmd *exec.Cmd, cfg *config.Data)` must be identical in both files. diff --git a/.cortex/plans/dual-os-task-05-docker-compose.md b/.cortex/plans/dual-os-task-05-docker-compose.md new file mode 100644 index 0000000..87ad5cb --- /dev/null +++ b/.cortex/plans/dual-os-task-05-docker-compose.md @@ -0,0 +1,117 @@ +# Task 05: Dockerfile + compose.yaml für Komodo-Deployment + +## Ziel + +avior-go läuft als Docker-Container unter Linux, verwaltet über Komodo (Stack aus dem +Repo). Host-Verzeichnis `/mnt/user/media` wird als `/media` in den Container gemountet. + +## Verifizierte Rahmenbedingungen aus dem Code + +- `globalstate.ReflectionPath()` (globalstate/globalstate.go:45) liefert + `filepath.Dir(os.Executable())` — dorthin werden `config.json` gelesen/geschrieben + (`config/config.go:259,290-303`) und `log/*.log` angelegt (app.go:34-36). + Konsequenz: Das Binary muss in einem beschreibbaren, persistenten Verzeichnis liegen, + damit Config und Logs Container-Neustarts überleben. +- HTTP-API-Port: `10000 + cfg.Local.Instance` (api/api.go:119) → Default **10000**. +- Externe Abhängigkeiten: MongoDB (`DatabaseURL`, Default `mongodb://localhost:27017`) + und optional Redis (`config.go:177-182`, `Enabled: false` per Default). Beide laufen + bereits extern (configs zeigen auf `10.10.10.96`) → **keine** mongo/redis-Services + ins compose aufnehmen. Die `config.json` der Linux-Instanz muss auf die externe + MongoDB zeigen und Linux-Pfade (`/media/...`) für `MediaPaths`/`OutDirectory`/`ObsoletePath` + verwenden — das ist Benutzerkonfiguration, kein Code. +- ffmpeg/ffprobe werden per `exec.Command("ffmpeg", ...)` / `exec.Command("ffprobe", ...)` + aus dem PATH aufgerufen (encoder/encoder.go:167, tools/tools.go:118,137) → Image + braucht ffmpeg. + +## Edits + +### 1. Neue Datei `Dockerfile` im Repo-Root + +Zweistufig: Build-Stage kompiliert das Linux-Binary (ersetzt den manuellen +`make build-linux`-Schritt für Docker), Runtime-Stage ist Debian-slim mit ffmpeg. + +```dockerfile +# syntax=docker/dockerfile:1 +FROM golang:1.25-bookworm AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o /out/avior-go app.go + +FROM debian:bookworm-slim +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /out/avior-go /opt/avior-go-bin +COPY docker/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] +``` + +### 2. Neue Datei `docker/entrypoint.sh` + +Hintergrund: `ReflectionPath()` = Verzeichnis des Executable. Config/Logs müssen im +persistenten Volume `/data` liegen, also kopiert der Entrypoint das Binary dorthin und +startet es aus `/data`. Beim ersten Start wird keine config.json angelegt — die App +erzeugt sie selbst mit Defaults (`config.Instance()`/`TryMakeCopy`), die danach auf dem +Host editierbar sind. + +```sh +#!/bin/sh +set -e +# Binary ins persistente Datenverzeichnis kopieren, damit ReflectionPath() (= Dir des +# Executable) auf /data zeigt und config.json + log/ Container-Neustarts überleben. +cp /opt/avior-go-bin /data/avior-go +chmod +x /data/avior-go +cd /data +exec /data/avior-go +``` + +### 3. Neue Datei `compose.yaml` im Repo-Root (Komodo-kompatibel) + +Komodo baut den Stack direkt aus dem Repo (`build: .`), kein Registry-Push nötig. + +```yaml +services: + avior-go: + build: . + image: avior-go:latest + container_name: avior-go + restart: unless-stopped + ports: + - "10000:10000" + volumes: + - /mnt/user/media:/media + - /mnt/user/appdata/avior-go:/data + cap_add: + - SYS_NICE # erlaubt negative nice-Werte (EncoderPriority HIGH/ABOVE_NORMAL); + # ohne Capability läuft der Encode trotzdem, nur mit Warnung im Log +``` + +Entscheidungen: +- `/mnt/user/appdata/avior-go` als Host-Pfad für `/data` (Unraid-Konvention für + App-Daten; Komodo-Ziel ist laut Mount-Pfad `/mnt/user/...` ein Unraid-Host). + Falls das Verzeichnis nicht existiert, legt Docker es beim ersten Start an. +- `.dockerignore` neu anlegen mit `dist/`, `*.exe`, `.git/`, `log/`, damit der + Build-Kontext klein bleibt. + +### 4. config.json für die Linux-Instanz (Benutzeraktion nach erstem Start) + +Nach dem ersten Start liegt `/mnt/user/appdata/avior-go/config.json` auf dem Host. +Darin Pfade auf Linux-Form anpassen: `MediaPaths`, `ObsoletePath`, +`EncoderConfig.*.OutDirectory` → z. B. `/media/transcoded`, `/media/tv` +(entsprechen den bisherigen UNC-Pfaden `\\UMS\media\...`, da `/mnt/user/media` der +gemountete Share ist). `DatabaseURL` auf die erreichbare MongoDB setzen. + +## Check + +- `docker build -t avior-go:test .` aus dem Repo-Root läuft durch. +- `docker run --rm -v /tmp/avior-data:/data avior-go:test` startet; in + `/tmp/avior-data` erscheinen `avior-go`, `config.json`, `log/`. API antwortet: + `curl http://localhost:10000/` (Port gemappt per `-p 10000:10000`). +- Im Container: `docker exec ffmpeg -version` findet ffmpeg. + +## Abhängigkeit + +Setzt Task 01 voraus (Linux kompilierbar) und nutzt denselben Build-Befehl wie Task 02. diff --git a/.cortex/plans/year-aware-dupes-plan.md b/.cortex/plans/year-aware-dupes-plan.md new file mode 100644 index 0000000..15d8536 --- /dev/null +++ b/.cortex/plans/year-aware-dupes-plan.md @@ -0,0 +1,200 @@ +# Plan: Year-aware Duplicate Detection for avior-go (config-driven) + +## Context + +Two files can share the exact same title (e.g. "Die Löwin") but be different films +(release years 2020 vs 2024). Current duplicate detection (`worker/worker.go` +`traverseDir`/`traverseMemCache`) matches only the exact `OutName()+Ext`, so a +same-title/different-year film is treated as a duplicate instead of a new film. +Goal: when a new job's title exists in the library with a **different release year**, +the new film gets the year appended in parentheses ("Die Löwin (2024)") and is then +treated as its own film — if that year-suffixed name already exists encoded, the +existing duplicate modules decide replacement; otherwise it is encoded as new. + +## Verified facts (this session) + +- `media/media.go:168-172`: `OutName()` = `f.Name` if no Subtitle, else `Name + " - " + Subtitle`. +- `worker/worker.go` `traverseDir` (line ~621) and `traverseMemCache` (~615): match + `filepath.Base(path) == file.OutName()+config.Instance().Local.Ext` — exact-name only. +- Job subtitle carries the year: "Spielfilm Deutschland 2024", "Fernsehfilm Deutschland 2007" + (seen in DB jobs this session). +- `media/media.go` `File` struct already holds BOTH sources separately: + - `MetadataLog []string` — `.txt` content (current EPG format: `Title=`, `Info=`, `Description=`) + - `TunerLog []string` — `.log` content (timer/recording log) + - `legacy bool` — true when no `.txt` exists (old format, `MetadataLog` empty) + - `readLogs()` (media.go:352): reads `stem+'.txt'` into MetadataLog, `stem+'.log'` + (or legacy `.mkv.log`/`.mpg.log`) into TunerLog; sets `legacy` when `.txt` missing. +- `movie_nfo_lib` (Python) has proven patterns; verified against real subtitles AND + real log formats this session: + - Current `.txt`: `Info=Spielfilm Deutschland/Estland/Lettland 2024` → year 2024 + (`extract_txt_meta_year_and_countries`, metadata.py:488). + - Old `.log`-only: `Melodram Südafrika/2011` (meta line after time range) → year 2011 + (`_extract_log_metadata`, metadata.py:1131; meta regex + `(\S+(?:\s+\S+)?)\s+(.+?)(?:/|\s)((?:19|20)\d{2})$`). + - Old `.log` Timer Name: `Timer Name: Die Löwin - Spielfilm Deutschland/Estland/Lettland 2024` + (`TIMER_NAME_META_PATTERN`, metadata.py:132) — most reliable source when present. + - `_slice_log_lines_for_metadata` (metadata.py:908): stop parsing at + `Removed Filler Data`/`Total Size`/`Monitoring Mode:`/first timestamp line. + - `YEAR_FOUR_RE = (?:19|20)\d{2}(?!\s*er\b|\s*er-)` — 4-digit year, ignores decades. + - `normalize_title` + `similarity` (text_utils.py) for name comparison. +- Go cannot import the Python lib; the regex patterns are platform-neutral and portable. +- No existing year extraction in avior-go (`grep year|Year` in media/ worker/ — none). + +## Year source resolution (ported from extract_txt_metadata, metadata.py:1310) + +For a given `media.File`, resolve the release year in this order (mirrors the +Python dispatcher): + +1. `f.Subtitle` — job-provided subtitle (e.g. "Spielfilm Deutschland 2024"); + `metaYearRe` first, `yearFourRe` fallback. +2. `f.MetadataLog` (.txt) — `Info=` line (e.g. "Spielfilm Deutschland/Estland/Lettland 2024"); + same patterns. +3. `f.TunerLog` (.log) — `Timer Name:` line first (`TIMER_NAME_META_PATTERN`), + else the meta line directly after the `HH:MM..HH:MM` time range + (old-format `Melodram Südafrika/2011`). Parse range bounded by + `Removed Filler Data`/`Total Size`/`Monitoring Mode:`/first `HH:MM:SS` line. + +`f.legacy` already tells us `.txt` is absent — use TunerLog for legacy files. + +### 1. New file `media/year.go` — year extraction + name normalization (ported) + +```go +package media + +import ( + "regexp" + "strings" +) + +var yearFourRe = regexp.MustCompile(`(?:19|20)\d{2}(?!\s*er\b|\s*er-)`) + +// metaYearRe matches "Spielfilm Deutschland 2024" style fragments and captures the +// 4-digit year. Genre words optional; bare country+year also matches. +var metaYearRe = regexp.MustCompile(`(?i)(?:^|\s-\s|\s–\s|\s—\s)(?:(?:fernsehfilm|spielfilm|film|serie|dokumentation|komödie|komoedie|tragikomödie|tragikomoedie|drama|thriller|krimi|melodram|melodrama|animationsfilm|zeichentrickfilm)\s+)?(?:(?:[A-Za-zÄÖÜäöüß]{2,})(?:\s*(?:/|,|;|und)\s*(?:[A-Za-zÄÖÜäöüß]{2,}))*)\s+((?:19|20)\d{2})\b`) + +// oldLogMetaRe matches the old-format meta line "Melodram Südafrika/2011" +// (genre + country + year at end, after the HH:MM..HH:MM time line). +var oldLogMetaRe = regexp.MustCompile(`(?i)^\s*(?:fernsehfilm|spielfilm|film|serie|dokumentation|komödie|komoedie|tragikomödie|tragikomoedie|drama|thriller|krimi|melodram|melodrama|animationsfilm|zeichentrickfilm)\s+.*?(?:/|\s)((?:19|20)\d{2})\s*$`) + +// timerNameMetaRe matches "Timer Name: Die Löwin - Spielfilm Deutschland/Estland/Lettland 2024". +var timerNameMetaRe = regexp.MustCompile(`(?i)^\s*Timer\s+Name\s*:\s*.*?(?:(?:fernsehfilm|spielfilm|film|serie|dokumentation|komödie|komoedie|tragikomödie|tragikomoedie|drama|thriller|krimi|melodram|melodrama|animationsfilm|zeichentrickfilm)\s+)?.*?(?:19|20)\d{2}\b`) + +// logSliceEndRe bounds the log parse range (mirrors _slice_log_lines_for_metadata). +var logSliceEndRe = regexp.MustCompile(`(?i)Removed Filler Data|Total Size|Monitoring Mode:|^\s*\d{1,2}:\d{2}:\d{2}`) + +// ExtractYearFromFile resolves the release year for f using the .txt→.log +// fallback order (subtitle → MetadataLog → TunerLog). Returns "" if none. +func (f *File) ExtractYearFromFile() string + +// ExtractYear returns the first 4-digit year found in s, or "". +func ExtractYear(s string) string + +// NormalizeName mirrors movie_nfo_lib normalize_title: lowercase, strip non-word +// chars, collapse whitespace. Used for same-title comparison. +func NormalizeName(s string) string + +// HasYearSuffix reports whether name already ends in " (YYYY)" — prevents +// double-suffixing when the year was already appended. +func HasYearSuffix(name string) bool +``` + +Edge cases: empty input → "" / false. No year → ExtractYear "". "1980er" not matched +(negative lookahead). Year-suffixed input → HasYearSuffix true. Legacy file without +`.txt` → TunerLog path only. `Timer Name:` present → wins (most reliable). + +### 2. Config flag `YearAwareDupes` (default true; off = current behavior) + +`config/config.go` `Local` struct: +```go +// YearAwareDupes: when true, same-title files with different release years are +// treated as separate films (year appended in parentheses). Default true. +YearAwareDupes bool `json:"YearAwareDupes"` +``` +`InitWithDefaults`: `cfg.Local.YearAwareDupes = true`. +(No omitempty — same lesson as CacheLibScan: false must persist. With default true +this matters less, but consistency.) + +### 3. Worker: resolve the effective output name before duplicate checks + +In `worker.ProcessJob`, after `mediaFile.Update()` and before `checkForDuplicates`: + +```go +if cfg.Local.YearAwareDupes && !media.HasYearSuffix(mediaFile.Name) { + // The first encode derives the name from EPG data (Title= + Info= without + // country/year): "Die Löwin". When that exact name already exists in the + // library but the release years differ, rename to "Die Löwin (2024)" so the + // two films are treated as separate. + year := mediaFile.ExtractYearFromFile() + if year != "" { + if collision, dupeYear := findYearCollision(mediaFile, year); collision && dupeYear != year { + _ = glg.Infof("year collision: appending (%s) to %s (existing has %s)", year, mediaFile.Name, dupeYear) + mediaFile.Name = fmt.Sprintf("%s (%s)", mediaFile.Name, year) + } + } +} +``` + +`findYearCollision(file, year)` reuses the EXISTING exact-name duplicate scan +(the `checkForDuplicates` match: `filepath.Base(path) == file.OutName()+Ext`) +and, when it finds the exact same name, extracts the found duplicate's own year +(from its `.txt`/`.log` via `ExtractYearFromFile`, or from a ` (YYYY)` suffix in +its filename). Returns the found name's year. + +**NOT a normalized title comparison across the whole library.** The collision is +found through the existing exact-name duplicate match; only the year of that one +found duplicate is compared against the new film's year. + +When years differ, `mediaFile.Name = fmt.Sprintf("%s (%s)", mediaFile.Name, year)` +(OutName() then yields "... (2024)"). The existing duplicate flow then re-runs +`checkForDuplicates` with the new name: if `Die Löwin (2024).mkv` already exists, +modules decide replacement; if absent, it is encoded as new. `HasYearSuffix` +prevents re-suffixing. + +Implementation: the exact-name scan currently runs once inside +`checkForDuplicates`. For the year flow, run the scan, and when a match exists, +extract the duplicate's year (media.File{Path: match}.ExtractYearFromFile or +parse the ` (YYYY)` suffix) and compare. If no exact-name match at all, no +collision — proceed with the original name. + +### 4. Reuse existing duplicate decision path — no changes there + +The module decision logic (LogMatch, Resolution, ErrorReplace, DuplicateLengthCheck) +in `worker.go` after `checkForDuplicates` stays untouched: once the name carries the +year suffix, the standard duplicate/replace flow applies. + +## Critical files & anchors + +- `media/media.go:168-172` — `OutName()`; year suffix is appended to `f.Name`, so + OutName picks it up automatically. +- `worker/worker.go:596-650` — `checkForDuplicates`/`traverseDir`/`traverseMemCache`; + the year-collision scan mirrors this traversal. +- `config/config.go` — `Local` struct + `InitWithDefaults` for the flag. +- `movie_nfo_lib/movie_metadata/metadata.py:125-165` — patterns to port (verified). + +## Verification + +Working dir: repo root `C:/repos/avior-go`. + +1. `go build ./... && go vet ./...`. +2. Unit check (no DB): scratch `go run` calling `ExtractYear` on + "Spielfilm Deutschland 2024" → "2024"; "Ruhe in Frieden" → ""; "1980er-Jahre" → ""; + `NormalizeName("Die Löwin - Spielfilm")` == normalized compare input; + `HasYearSuffix("Die Löwin (2024)")` → true, `HasYearSuffix("Die Löwin")` → false. +3. Logic check of the suffix flow with two synthetic media.File: + file A OutName "Die Löwin" (year 2020 in subtitle), file B "Die Löwin" (year 2024) — + collision detected, B.Name becomes "Die Löwin (2024)"; re-run: no double suffix. +4. End-to-end on Unraid (feat/dual-os, instance 1): add a job whose title exists in + the library with a different year; expect log "year collision: appending (2024)" + and output file "Die Löwin (2024).mkv". Re-push the same job: duplicate modules + decide replacement (existing behavior). +5. Regression: `YearAwareDupes=false` → identical naming as before (exact match only). + +## Assumptions & contingencies + +- Year source is the Subtitle (verified format). If a subtitle lacks a year but the + filename contains one, `ExtractYear` still finds it via `yearFourRe` fallback. +- Name comparison basis is the title (mediaFile.Name), NOT the full OutName with + subtitle — the subtitle itself carries the year and would otherwise defeat the + collision. If reality shows collisions should compare OutName-with-subtitle-minus-year, + adjust the NormalizeName comparison input accordingly (single spot). +- The suffix format is fixed " (YYYY)" per the user's example ("Die Löwin (2024)"). diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..849ddff --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +dist/ diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 651f368..ebc6069 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,11 +8,22 @@ on: jobs: build: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - goos: windows + goarch: amd64 + binary_name: avior-go + - goos: linux + goarch: amd64 + binary_name: avior-go steps: - uses: actions/checkout@v2 - uses: wangyoucao577/go-release-action@v1.18 with: github_token: ${{ secrets.GITHUB_TOKEN }} - goos: windows - goarch: amd64 - binary_name: avior-go + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + binary_name: ${{ matrix.binary_name }} + ldflags: -s -w diff --git a/.gitignore b/.gitignore index f80e5e6..07ba68c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,11 @@ *.so *.dylib +# Media files +*.ts +*.mkv +*.mp4 + # app generated logs and config config.json log/ @@ -29,4 +34,4 @@ log/ .history/ # debug bin file -__debug_bin \ No newline at end of file +__debug_bindist/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..e73bc8e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,102 @@ +# avior-go - Agent Instructions + +## Project + +Go 1.25 media transcoding service. Monitors media directories, compares files via +pluggable comparator modules, and encodes/remuxes content with ffmpeg. + +## Build + +Single entrypoint: `app.go` in repo root (package `main`). + +``` +# Windows +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "-s -w" -o dist/avior-go-windows-amd64.exe app.go + +# Linux +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o dist/avior-go-linux-amd64 app.go +``` + +Or use `make build-windows` / `make build-linux` / `make all` (requires `make`). + +## Architecture + +- `app.go` — Main entry, signal handling, logger setup. +- `api/` — HTTP API on port `10000 + Instance` (gorilla/mux, gorilla/websocket). +- `config/` — Config load/save (`config.json` next to binary), priority enums. +- `encoder/` — ffmpeg encoding. OS-specific priority in `priority_*.go` (build tags). +- `comparator/` — Pluggable comparison modules (legacy, resolution, audio, etc.). +- `media/` — File/model definitions. +- `worker/` — Background workers (encode queue, file walker, mover). +- `db/` — MongoDB access. +- `redis/` — Redis pubsub for cross-instance coordination. +- `globalstate/` — Singleton shared state. `ReflectionPath()` = dir of executable. +- `tools/` — Misc helpers (ffprobe, build scripts). +- `structs/`, `consts/`, `joblog/`, `cache/`, `log/` — Support packages. + +## Key code conventions + +- Config loaded from `filepath.Join(globalstate.ReflectionPath(), "config.json")`. + The binary's directory IS the config/log root — this matters for Docker volumes. +- All path operations use `filepath.Join` (OS-agnostic). +- Only OS-specific code: `encoder/priority_windows.go` and `encoder/priority_linux.go` + (build tags `//go:build windows` / `//go:build linux`). No other platform coupling. +- Logging via `github.com/kpango/glg` — log to `glg.FileWriter(...)` and + `lumberjack.Logger` rotation. +- MongoDB via `go.mongodb.org/mongo-driver/v2`, Redis via `go-redis/v9`. +- Priority constants (`PRIORITY_IDLE`, `PRIORITY_NORMAL`, etc.) are Windows + `PriorityClass` values; mapped to nice levels on Linux in `priority_linux.go`. +- godirwalk pinned below v1.17.0 (see `go.mod` exclude) — v1.17.0 reports io.EOF + as a walk error on Windows. + +## Docker + +`compose.yaml` + `Dockerfile` in repo root for Komodo deployment. +Binary runs from `/data` inside container (entrypoint copies to persist config/logs). +Mount `/mnt/user/media:/media` for media access. +API on port `10000`. + +## Language + +All documentation, plans, and commit messages MUST be in English. +Code comments may be in either language, but public-facing and plan text is English only. + +## Subagent routing + +Project-level rules OVERRIDE the user-level global rules in +`~/.omp/agent/AGENTS.md` for this repository. + +When delegating work via subagents (`task`), route by complexity: + +- **Low complexity** (mechanical edits, config changes, file creation) → + `deepseek-v4-flash` with high effort +- **Medium complexity** (refactoring, new features with existing patterns) → + `deepseek-v4-flash` with max effort +- **High complexity** (novel design, architectural changes, debugging unknown issues) → + `kimi-k3` with low effort + + +## Skills + +### docs_agent — Technical Writer + +Expert technical writer for this project. Read code from the repo and generate or +update documentation in `docs/`. + +- **Tech stack:** Go 1.25, MongoDB, Redis, ffmpeg, Docker. +- **File structure:** `docs/` is the documentation root (create if missing). +- **Style:** Concise, specific, value-dense. Write so a new developer can + understand — don't assume audience expertise in the topic. +- **Boundaries:** + - Write new files to `docs/`. + - Ask before major changes to existing documents. + - Do not modify source code or config files. + - Do not commit secrets. + +## Never do + +- Do not import `golang.org/x/sys/windows` in cross-platform files — use build-tagged + files instead. +- Do not hardcode UNC paths (`\\UMS\...`) or drive letters in code — those are user + config data. +- Do not upgrade `github.com/karrick/godirwalk` to v1.17.0 (see go.mod). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1810e1d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.25-bookworm AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o /out/avior-go app.go + +FROM linuxserver/ffmpeg:latest +COPY --from=build /out/avior-go /opt/avior-go-bin +COPY docker/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d36189b --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +BINARY := avior-go +VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +LDFLAGS := -s -w + +.PHONY: build-windows build-linux build all + +build-windows: + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o dist/$(BINARY)-windows-amd64.exe app.go + +build-linux: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o dist/$(BINARY)-linux-amd64 app.go + +build: build-windows + +all: build-windows build-linux diff --git a/app.go b/app.go index 6483a4b..6f5bf39 100644 --- a/app.go +++ b/app.go @@ -31,7 +31,21 @@ func main() { // Set up logger //log := glg.FileWriter(filepath.Join("log", "main.log"), os.ModeAppend) - errlog := glg.FileWriter(filepath.Join(globalstate.ReflectionPath(), "log", "err.log"), os.ModeAppend) + // Create the log dir explicitly with 0755: glg.FileWriter's perm argument is + // passed BOTH to os.MkdirAll (dir) and OpenFile (file), so one value cannot be + // right for both. A dir needs the x bit (0755), a file should be 0644. The old + // os.ModeAppend (flag bit = 0 as permission) made the dir 000 on Linux. + logDir := filepath.Join(globalstate.ReflectionPath(), "log") + // MkdirAll does NOT fix permissions of an already existing directory. A legacy + // log/ dir created with mode 000 (by the old os.ModeAppend bug) would stay + // inaccessible forever, so chmod explicitly after creation. + if err := os.MkdirAll(logDir, 0755); err != nil { + _ = glg.Errorf("could not create log directory %s: %s", logDir, err) + } + if err := os.Chmod(logDir, 0755); err != nil { + _ = glg.Errorf("could not fix permissions on log directory %s: %s", logDir, err) + } + errlog := glg.FileWriter(filepath.Join(logDir, "err.log"), 0644) log := &lumberjack.Logger{ Filename: filepath.Join(globalstate.ReflectionPath(), "log", "main.log"), MaxSize: 10, // megabytes @@ -52,7 +66,7 @@ func main() { AddLevelWriter(glg.FAIL, errlog). SetLevelColor(glg.ERR, glg.Red). SetLevelColor(glg.DEBG, glg.Cyan) - _ = glg.Info("version ==>", "hey (2.0.0) codename odysseus") + _ = glg.Info("version ==>", "hey (2.0.1) codename odysseus") defer log.Close() // read cli args diff --git a/cache/cache.go b/cache/cache.go index 552116c..dd25147 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -17,6 +17,11 @@ type Library struct { Data []string LastUpdate time.Time Valid bool `json:"-"` + // ScannedPaths fingerprints the MediaPaths this cache was built from. When the + // configured MediaPaths change (config reload), the cache must be rebuilt even + // if the TTL has not expired — otherwise files in newly added paths are never + // seen as duplicates until a restart. + ScannedPaths string `json:"-"` } // Instance retrieves the current configuration file instance diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..ac93393 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,38 @@ +# Multi-instance compose: one file, parameterized per stack via environment. +# Each Komodo stack sets INSTANCE_SUFFIX, HOST_PORT, CONTAINER_PORT and DATA_DIR +# (plus optional GPU_DEVICE). CONTAINER_PORT must equal 10000 + Instance from the +# instance's config.json (the app listens on 10000+Instance). +services: + avior-go: + build: . + # Explicit, readable container name per instance (avior-go, avior-go-1, ...); + # without it Compose derives -- (avior-go-avior-go-1). + container_name: avior-go${INSTANCE_SUFFIX:-} + hostname: avior-go${INSTANCE_SUFFIX:-} # client fallback name; ClientName in config.json wins + environment: + - TZ=${TZ:-Europe/Berlin} # local time for logs (app uses time.Local) + # Unraid standard: run as nobody:users so files are owned like every + # other share file and SMB/tools work without per-user PUID config. + # Override via PUID/PGID env when a dedicated service user is wanted. + - PUID=${PUID:-99} + - PGID=${PGID:-100} + restart: unless-stopped + ports: + - "${HOST_PORT:-10000}:${CONTAINER_PORT:-10000}" + healthcheck: + # /alive/ always answers 200 while the process runs; curl is present in + # the linuxserver/ffmpeg base image (verified: /usr/bin/curl). + test: ["CMD-SHELL", "curl -fsS http://localhost:${CONTAINER_PORT:-10000}/alive/ || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + volumes: + - /mnt/user/media:/media + - /mnt/user/recording_pool:/recording_pool + - /mnt/disks/recording_spare:/recording_spare + - ${DATA_DIR:-/mnt/user/appdata/avior-go}:/data + devices: + - ${GPU_DEVICE:-/dev/dri}:/dev/dri # Intel ARC GPU for QSV hardware acceleration + cap_add: + - SYS_NICE # allows negative nice values for encoder priority diff --git a/config/config.go b/config/config.go index 2ec3d0d..50c5cbd 100644 --- a/config/config.go +++ b/config/config.go @@ -24,6 +24,11 @@ type Data struct { // Local is the main application configuration type Local struct { Instance int + // ClientName overrides the machine hostname used to register this instance in + // the DB (collection clients). In Docker the container hostname may be a + // container ID or a stale inherited value; setting ClientName makes the + // registration deterministic. Empty = use os.Hostname() (previous behavior). + ClientName string `json:"ClientName,omitempty"` DatabaseURL string Redis Redis Ext string @@ -32,7 +37,23 @@ type Local struct { Resolutions map[string]string ObsoletePath string MediaPaths []string - EstimatedLibSize int + // PathMappings maps UNC prefixes found in DB job paths to local container paths, + // e.g. "\\\\192.168.178.75\\recording_pool" -> "/recording_pool". + // Empty/nil = no translation (Windows behavior unchanged). + PathMappings map[string]string `json:"PathMappings,omitempty"` + // CacheLibScan caches the library scan (MediaPaths walk) in memory and reuses + // it for the Redis TTL (24h) instead of rescanning per duplicate check. + // Under Docker the walk is a direct local FS access and takes <1s, so caching + // only adds staleness bugs; set false to always scan fresh. + // Default true keeps the historical Windows/SMB behavior unchanged. + // NOTE: no omitempty - a false value must be persisted, otherwise Save() drops it. + CacheLibScan bool `json:"CacheLibScan"` + // YearAwareDupes: when true, same-title files with different release years are + // treated as separate films: on an exact-name duplicate match whose year differs + // from the new film's year, the new film is renamed to "Title (YYYY)" before the + // duplicate modules decide. Default true. + YearAwareDupes bool `json:"YearAwareDupes"` + EstimatedLibSize int Modules map[string]ModuleConfig EncoderConfig map[string]EncoderConfig EncoderPriority string @@ -174,6 +195,8 @@ func InitWithDefaults(cfg *Data) { cfg.Local.Resolutions = map[string]string{"hd": "1280x720", "fhd": "1920x1080"} cfg.Local.EncoderConfig = map[string]EncoderConfig{"hd": *new(EncoderConfig)} cfg.Local.EncoderPriority = PRIORITY_IDLE.String() + cfg.Local.CacheLibScan = true + cfg.Local.YearAwareDupes = true cfg.Local.Redis = Redis{ Host: "localhost:6379", Password: "", diff --git a/db/client.go b/db/client.go index ccde7d9..a3ee9cc 100644 --- a/db/client.go +++ b/db/client.go @@ -18,19 +18,30 @@ import ( ) // GetClientForMachine returns the current db client that matches this machine's hostname. -// A new client will be created if none is found in the database +// A new client will be created if none is found in the database. +// +// The lookup and the insert BOTH use strings.ToUpper(hostname): the historical +// registry is uppercase (VDR-U, PHOENIX, ...), so storing the raw hostname would +// create a duplicate lower-case entry on every restart (lookup never matches). func (ds *DataStore) GetClientForMachine() (*structs.Client, error) { cfg := config.Instance() ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - hostname, _ := os.Hostname() + // A configured ClientName wins over the hostname: in Docker the container + // hostname may be a container ID or a stale inherited value, which would + // re-register a phantom client on every restart. + hostname := cfg.Local.ClientName + if hostname == "" { + hostname, _ = os.Hostname() + } if cfg.Local.Instance > 0 { hostname = fmt.Sprintf("%s-%d", hostname, cfg.Local.Instance) } + hostname = strings.ToUpper(hostname) state := globalstate.Instance() state.HostName = hostname var thisMachine *structs.Client - err := ds.Db().Collection("clients").FindOne(ctx, bson.M{"Name": strings.ToUpper(hostname)}).Decode(&thisMachine) + err := ds.Db().Collection("clients").FindOne(ctx, bson.M{"Name": hostname}).Decode(&thisMachine) if err == mongo.ErrNoDocuments { // Create client if it doesn't exist yet thisMachine = &structs.Client{ diff --git a/dist/avior-go-linux-amd64 b/dist/avior-go-linux-amd64 new file mode 100644 index 0000000..8c30b1f Binary files /dev/null and b/dist/avior-go-linux-amd64 differ diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..58e2494 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,70 @@ +# Docker Deployment for avior-go + +## Setup + +1. Copy `docker/config.example.json` to `/mnt/user/appdata/avior-go/config.json` on your + host and adjust settings (MongoDB/Redis hosts, paths). + +2. Path convention: the compose file mounts `/mnt/user/media` to `/media` inside the + container. All `MediaPaths`, `ObsoletePath`, and `EncoderConfig.*.OutDirectory` + values must use `/media/...` paths (forward slashes, no UNC). + +3. Deploy via Komodo from this repo's compose stack, or manually: + ``` + docker compose up -d + ``` + +### Komodo stack config + +Create a Stack in Komodo pointing at the git repo and **force a rebuild on every +redeploy** — otherwise `docker compose up` reuses a cached image with the same name +and code changes stay invisible: + +```toml +[[stack]] +name = "avior-go" +[stack.config] +server = "" +run_directory = "/opt/stacks/avior-go" +file_paths = ["compose.yaml"] +repo = "Spiritreader/avior-go" +branch = "feat/dual-os" # compose/Dockerfile currently live here +# rebuild the image on every deploy (no image: tag in compose.yaml): +extra_args = "--build" +``` + +## Hardware acceleration + +The container uses the `linuxserver/ffmpeg:latest` base image which ships ffmpeg with +Intel QSV (oneVPL) support. On an Unraid host with an Intel ARC GPU: + +- The device is passed through via `devices: ["/dev/dri:/dev/dri"]` in `compose.yaml` +- `av1_qsv`, `-hwaccel qsv`, `-hwaccel_output_format qsv` work out of the box +- The example config (`docker/config.example.json`) is pre-configured for QSV + +### Software fallback + +If no GPU is available, replace the encoder in `config.json`: + +| QSV (hardware) | Software equivalent | +|-------------------------|-------------------------| +| `av1_qsv` | `libsvtav1` or `libaom-av1` | +| `-hwaccel qsv` | (remove) | +| `-hwaccel_output_format qsv` | (remove) | +| `-init_hw_device qsv=qsv` | (remove) | +| `-filter_hw_device qsv` | (remove) | + +And switch the runtime base in `Dockerfile` back to `debian:bookworm-slim` with +additional packages: +```dockerfile +RUN apt-get install -y libsvtav1enc1 +``` + + + +## Volume layout + +| Host path | Container path | Purpose | +|----------------------------------|---------------|----------------------| +| `/mnt/user/media` | `/media` | Media library | +| `/mnt/user/appdata/avior-go` | `/data` | config.json + logs | diff --git a/docker/add_jobs_from_coding_test - ARC/avior_dis.log b/docker/add_jobs_from_coding_test - ARC/avior_dis.log new file mode 100644 index 0000000..5c90bca --- /dev/null +++ b/docker/add_jobs_from_coding_test - ARC/avior_dis.log @@ -0,0 +1,112 @@ +2026-08-03 08:25:47 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\re_encode +The system cannot find the file specified. (os error 2) for: \\UMS\recording_pool\re_encode\Wissen macht Ah! - Eine Sendung für Erwachsene_2025-03-17-08-18-00-WDR HD Köln (AC3,deu).txt +pushing \\UMS\recording_pool\re_encode\Wissen macht Ah! - Eine Sendung für Erwachsene_2025-03-17-08-18-00-WDR HD Köln (AC3,deu).ts to AVIOR-GO-TEST with 0/1 job(s) and priority 12 + +2026-08-03 08:27:02 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +database already up to date + +2026-08-03 08:28:26 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Nord bei Nordwest - Haare Hartmann!_2026-07-30-20-13-04-Das Erste HD (AC3,deu).ts to AVIOR-GO-TEST with 0/1 job(s) and priority 12 +no eligible found, pushing \\UMS\recording_pool\coding_test\Quarks Mobilität ohne Frust - gelingt uns die Verkehrswende_2026-08-01-18-43-01-BR Fernsehen Süd HD (AC3,deu).ts to default client AVIOR-GO-TEST + +2026-08-03 08:51:46 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Nord bei Nordwest - Haare Hartmann!_2026-07-30-20-13-04-Das Erste HD (AC3,deu).ts to AVIOR-GO-TEST with 0/1 job(s) and priority 12 +no eligible found, pushing \\UMS\recording_pool\coding_test\Quarks Mobilität ohne Frust - gelingt uns die Verkehrswende_2026-08-01-18-43-01-BR Fernsehen Süd HD (AC3,deu).ts to default client AVIOR-GO-TEST + +2026-08-03 11:08:42 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Nord bei Nordwest - Haare Hartmann!_2026-07-30-20-13-04-Das Erste HD (AC3,deu).ts to UNRAID with 0/10 job(s) and priority 0 +pushing \\UMS\recording_pool\coding_test\Quarks Mobilität ohne Frust - gelingt uns die Verkehrswende_2026-08-01-18-43-01-BR Fernsehen Süd HD (AC3,deu).ts to UNRAID with 1/10 job(s) and priority 0 + +2026-08-03 11:11:01 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Nord bei Nordwest - Haare Hartmann!_2026-07-30-20-13-04-Das Erste HD (AC3,deu).ts to UNRAID with 0/10 job(s) and priority 0 +pushing \\UMS\recording_pool\coding_test\Quarks Mobilität ohne Frust - gelingt uns die Verkehrswende_2026-08-01-18-43-01-BR Fernsehen Süd HD (AC3,deu).ts to UNRAID with 1/10 job(s) and priority 0 + +2026-08-03 11:36:47 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +database already up to date + +2026-08-03 11:37:34 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +database already up to date + +2026-08-03 11:37:52 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Nord bei Nordwest - Haare Hartmann!_2026-07-30-20-13-04-Das Erste HD (AC3,deu).ts to MMDG-2 with 0/10 job(s) and priority 0 +pushing \\UMS\recording_pool\coding_test\Quarks Mobilität ohne Frust - gelingt uns die Verkehrswende_2026-08-01-18-43-01-BR Fernsehen Süd HD (AC3,deu).ts to UNRAID with 0/10 job(s) and priority 0 + +2026-08-03 11:53:19 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Nord bei Nordwest - Haare Hartmann!_2026-07-30-20-13-04-Das Erste HD (AC3,deu).ts to MMDG-2 with 0/10 job(s) and priority 0 +pushing \\UMS\recording_pool\coding_test\Quarks Mobilität ohne Frust - gelingt uns die Verkehrswende_2026-08-01-18-43-01-BR Fernsehen Süd HD (AC3,deu).ts to UNRAID with 0/10 job(s) and priority 0 + +2026-08-03 12:25:15 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +database already up to date + +2026-08-03 12:29:02 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +database already up to date + +2026-08-03 12:29:21 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\SOKO Stuttgart_2026-08-03-11-13-00-ZDF HD (AC3,deu).ts to UNRAID with 0/10 job(s) and priority 12 + +2026-08-03 12:57:46 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\SOKO Stuttgart_2026-08-03-11-13-00-ZDF HD (AC3,deu).ts to AVIOR-GO-TEST with 0/1 job(s) and priority 12 + +2026-08-03 12:59:22 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +database already up to date + +2026-08-08 10:04:35 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Die Löwin_2026-08-03-22-58-00-arte HD (deu).ts to UNRAID with 1/1000 job(s) and priority 1 + +2026-08-08 10:23:48 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Die Löwin_2026-08-03-22-58-00-arte HD (deu).ts to UNRAID with 0/1000 job(s) and priority 1 + +2026-08-08 10:41:55 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Die Löwin_2026-08-03-22-58-00-arte HD (deu).ts to UNRAID with 0/1000 job(s) and priority 1 + +2026-08-08 10:48:59 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +database already up to date + +2026-08-08 10:49:12 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Die Löwin_2026-08-03-22-58-00-arte HD (deu).ts to UNRAID with 0/1000 job(s) and priority 1 + +2026-08-08 11:00:21 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Die Löwin_2026-08-03-22-58-00-arte HD (deu).ts to UNRAID with 1/1000 job(s) and priority 1 + diff --git a/docker/add_jobs_from_coding_test - ARC/dis_config.toml b/docker/add_jobs_from_coding_test - ARC/dis_config.toml new file mode 100644 index 0000000..5abf224 --- /dev/null +++ b/docker/add_jobs_from_coding_test - ARC/dis_config.toml @@ -0,0 +1,11 @@ +DbUrl = 'mongodb://192.168.178.75:27017' +DbName = 'Avior' +DefaultClient = 'UNRAID' +IgnoredClients = ['AVIOR-GO-TEST', 'VDR-U', 'VDR-U-1', 'VDR-U-2', 'PHOENIX', 'VDR', 'MMDG', 'VAVA'] +Filetypes = [ + 'ts', + 'mpg', + 'mkv' +] +IgnoredFiletypes = ['INFO.log'] +MinAge = 0 diff --git a/docker/add_jobs_from_coding_test - ARC/runDIS - docker_test.bat b/docker/add_jobs_from_coding_test - ARC/runDIS - docker_test.bat new file mode 100644 index 0000000..9216e81 --- /dev/null +++ b/docker/add_jobs_from_coding_test - ARC/runDIS - docker_test.bat @@ -0,0 +1,4 @@ +@echo off +avior-dis.exe "\\UMS\recording_pool\coding_test" +pause +exit \ No newline at end of file diff --git a/docker/add_jobs_from_coding_test/avior_dis.log b/docker/add_jobs_from_coding_test/avior_dis.log new file mode 100644 index 0000000..924da4f --- /dev/null +++ b/docker/add_jobs_from_coding_test/avior_dis.log @@ -0,0 +1,23 @@ +2026-08-03 08:25:47 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\re_encode +The system cannot find the file specified. (os error 2) for: \\UMS\recording_pool\re_encode\Wissen macht Ah! - Eine Sendung für Erwachsene_2025-03-17-08-18-00-WDR HD Köln (AC3,deu).txt +pushing \\UMS\recording_pool\re_encode\Wissen macht Ah! - Eine Sendung für Erwachsene_2025-03-17-08-18-00-WDR HD Köln (AC3,deu).ts to AVIOR-GO-TEST with 0/1 job(s) and priority 12 + +2026-08-03 08:27:02 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +database already up to date + +2026-08-03 08:28:26 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Nord bei Nordwest - Haare Hartmann!_2026-07-30-20-13-04-Das Erste HD (AC3,deu).ts to AVIOR-GO-TEST with 0/1 job(s) and priority 12 +no eligible found, pushing \\UMS\recording_pool\coding_test\Quarks Mobilität ohne Frust - gelingt uns die Verkehrswende_2026-08-01-18-43-01-BR Fernsehen Süd HD (AC3,deu).ts to default client AVIOR-GO-TEST + +2026-08-03 08:51:46 +0200 +avior dis, version 0.1.2 - (dat)^2(ei) +scan directory: \\UMS\recording_pool\coding_test +pushing \\UMS\recording_pool\coding_test\Nord bei Nordwest - Haare Hartmann!_2026-07-30-20-13-04-Das Erste HD (AC3,deu).ts to AVIOR-GO-TEST with 0/1 job(s) and priority 12 +no eligible found, pushing \\UMS\recording_pool\coding_test\Quarks Mobilität ohne Frust - gelingt uns die Verkehrswende_2026-08-01-18-43-01-BR Fernsehen Süd HD (AC3,deu).ts to default client AVIOR-GO-TEST + diff --git a/docker/add_jobs_from_coding_test/dis_config.toml b/docker/add_jobs_from_coding_test/dis_config.toml new file mode 100644 index 0000000..f589c93 --- /dev/null +++ b/docker/add_jobs_from_coding_test/dis_config.toml @@ -0,0 +1,11 @@ +DbUrl = 'mongodb://192.168.178.75:27017' +DbName = 'Avior' +DefaultClient = 'AVIOR-GO-TEST' +IgnoredClients = ['VDR-U', 'VDR-U-1', 'VDR-U-2', 'PHOENIX', 'VDR', 'MMDG', 'VAVA'] +Filetypes = [ + 'ts', + 'mpg', + 'mkv' +] +IgnoredFiletypes = ['INFO.log'] +MinAge = 1 diff --git a/docker/add_jobs_from_coding_test/runDIS - docker_test.bat b/docker/add_jobs_from_coding_test/runDIS - docker_test.bat new file mode 100644 index 0000000..9216e81 --- /dev/null +++ b/docker/add_jobs_from_coding_test/runDIS - docker_test.bat @@ -0,0 +1,4 @@ +@echo off +avior-dis.exe "\\UMS\recording_pool\coding_test" +pause +exit \ No newline at end of file diff --git a/docker/avior-go als Multiinstanz unter Komodo - Einrichtung.txt b/docker/avior-go als Multiinstanz unter Komodo - Einrichtung.txt new file mode 100644 index 0000000..be4cbd9 --- /dev/null +++ b/docker/avior-go als Multiinstanz unter Komodo - Einrichtung.txt @@ -0,0 +1,194 @@ + avior-go Multi-Instanz unter Komodo — Einrichtung + + 1. Komodo-Architektur + + ┌───────────────────┬────────────────────────────────────────────────────┬────────────────────────────────────────────┐ + │ Komponente │ Wo │ Zweck │ + ├───────────────────┼────────────────────────────────────────────────────┼────────────────────────────────────────────┤ + │ Komodo Core │ Cloud-UI │ Stack-Config, Env-Vars, Deploys │ + ├───────────────────┼────────────────────────────────────────────────────┼────────────────────────────────────────────┤ + │ Periphery │ Unraid-Host │ Führt Compose aus, Git-Checkouts │ + ├───────────────────┼────────────────────────────────────────────────────┼────────────────────────────────────────────┤ + │ Stack-Verzeichnis │ /mnt/user/appdata/komodo/periphery/stacks// │ Repo-Klon (wird bei Deploy synchronisiert) │ + └───────────────────┴────────────────────────────────────────────────────┴────────────────────────────────────────────┘ + + Regel: compose.yaml nur im Repo pflegen — in der Komodo-UI nie die Datei editieren (erzeugt Konflikt-Commits "[Komodo] Write Stack File"). + + 2. compose.yaml (Repo, parametrisiert) + + Die compose hat container_name: avior-go${INSTANCE_SUFFIX:-} — damit heißen die + Container einfach avior-go, avior-go-1, avior-go-2 (statt avior-go-avior-go-1). + INSTANCE_SUFFIX steuert also Container-Name UND Hostname (Client-Fallback). + + ┌─────────────────┬────────────────────────────┬─────────────────────────────────────┐ + │ Variable │ Default │ Zweck │ + ├─────────────────┼────────────────────────────┼─────────────────────────────────────┤ + │ INSTANCE_SUFFIX │ (leer) │ Container-Name + Hostname-Suffix │ + ├─────────────────┼────────────────────────────┼─────────────────────────────────────┤ + │ HOST_PORT │ 10000 │ Host-Port │ + ├─────────────────┼────────────────────────────┼─────────────────────────────────────┤ + │ CONTAINER_PORT │ 10000 │ Container-Port (= 10000 + Instance) │ + ├─────────────────┼────────────────────────────┼─────────────────────────────────────┤ + │ DATA_DIR │ /mnt/user/appdata/avior-go │ Daten-Volume │ + ├─────────────────┼────────────────────────────┼─────────────────────────────────────┤ + │ GPU_DEVICE │ /dev/dri │ GPU-Passthrough │ + ├─────────────────┼────────────────────────────┼─────────────────────────────────────┤ + │ TZ │ Europe/Berlin │ Log-Zeitzone │ + ├─────────────────┼────────────────────────────┼─────────────────────────────────────┤ + │ PUID / PGID │ 1002 / 100 │ Datei-Ownership (mm:users) │ + └─────────────────┴────────────────────────────┴─────────────────────────────────────┘ + + 3. Instanz-Tabelle + + ┌─────────┬──────────────────────────┬────────────────────────┬─────────────────┬────────────────┬─────────────┐ + │ Instanz │ ClientName (config.json) │ Instance (config.json) │ Registriert als │ Container-Port │ Host-Port │ + ├─────────┼──────────────────────────┼────────────────────────┼─────────────────┼────────────────┼─────────────┤ + │ 1 │ UNRAID │ 0 │ UNRAID │ 10000 │ 10000:10000 │ + ├─────────┼──────────────────────────┼────────────────────────┼─────────────────┼────────────────┼─────────────┤ + │ 2 │ UNRAID │ 1 │ UNRAID-1 │ 10001 │ 10001:10001 │ + ├─────────┼──────────────────────────┼────────────────────────┼─────────────────┼────────────────┼─────────────┤ + │ 3 │ UNRAID │ 2 │ UNRAID-2 │ 10002 │ 10002:10002 │ + └─────────┴──────────────────────────┴────────────────────────┴─────────────────┴────────────────┴─────────────┘ + + Regel: CONTAINER_PORT = 10000 + Instance. Client-Trennung über Instance (Suffix automatisch, wie MMDG/MMDG-1 unter Windows). + WICHTIG: ClientName ist IMMER das Basiswort "UNRAID" — nie das Suffix mit hineinschreiben! + (Fehlerfall: ClientName "UNRAID-2" + Instance 2 → registriert als "UNRAID-2-2", doppeltes Suffix.) + + 4. Komodo-Stack-Config pro Instanz + + ### Sidebar-Ort der Felder + + ┌─────────────────┬──────────┬─────────────────────────────────────────────────────────────────────────┐ + │ Sidebar-Eintrag │ Bereich │ Was │ + ├─────────────────┼──────────┼─────────────────────────────────────────────────────────────────────────┤ + │ Source │ GENERAL │ Repo Spiritreader/avior-go, Branch feat/dual-os, Account: None (public) │ + ├─────────────────┼──────────┼─────────────────────────────────────────────────────────────────────────┤ + │ Environment │ GENERAL │ ⭐ Env-Vars (TOML) │ + ├─────────────────┼──────────┼─────────────────────────────────────────────────────────────────────────┤ + │ Extra Args │ ADVANCED │ --build │ + └─────────────────┴──────────┴─────────────────────────────────────────────────────────────────────────┘ + + HINWEIS: Project Name muss NICHT gesetzt werden — der Container-Name kommt aus + container_name: avior-go${INSTANCE_SUFFIX:-} in der compose. Falls Project Name + trotzdem gesetzt war (ältere Variante), kann er entfernt werden; er schadet nur, + wenn er mit dem container_name kollidiert. + + ### Instanz 1 — Environment (TOML) + + ```toml + # Defaults reichen: kein INSTANCE_SUFFIX, HOST_PORT 10000, CONTAINER_PORT 10000, + # DATA_DIR /mnt/user/appdata/avior-go + ``` + + ┌────────────┬──────────┐ + │ Feld │ Wert │ + ├────────────┼──────────┤ + │ Extra Args │ --build │ + └────────────┴──────────┘ + + ### Instanz 2 — Environment (TOML) + + ```toml + INSTANCE_SUFFIX = "-1" + HOST_PORT = 10001 + CONTAINER_PORT = 10001 + DATA_DIR = "/mnt/user/appdata/avior-go-1" + ``` + + ┌────────────┬──────────┐ + │ Feld │ Wert │ + ├────────────┼──────────┤ + │ Extra Args │ --build │ + └────────────┴──────────┘ + + ### Instanz 3 — Environment (TOML) + + ```toml + INSTANCE_SUFFIX = "-2" + HOST_PORT = 10002 + CONTAINER_PORT = 10002 + DATA_DIR = "/mnt/user/appdata/avior-go-2" + ``` + + ┌────────────┬──────────┐ + │ Feld │ Wert │ + ├────────────┼──────────┤ + │ Extra Args │ --build │ + └────────────┴──────────┘ + + 5. config.json pro Instanz + + ┌─────────┬────────────┬──────────┬──────────────┐ + │ Instanz │ ClientName │ Instance │ CacheLibScan │ + ├─────────┼────────────┼──────────┼──────────────┤ + │ 1 │ UNRAID │ 0 │ false │ + ├─────────┼────────────┼──────────┼──────────────┤ + │ 2 │ UNRAID │ 1 │ false │ + ├─────────┼────────────┼──────────┼──────────────┤ + │ 3 │ UNRAID │ 2 │ false │ + └─────────┴────────────┴──────────┴──────────────┘ + + Weitere Felder (alle): PathMappings (UMS + IP, beide Shares), Encoder-Profile mit + -qsv_device /dev/dri/renderD129, MediaPaths [re_encode, transcoded, tv], + ChannelPrefix: "avior" (gleich lassen für Redis-Broadcast). + + TIPP: CacheLibScan lässt sich im Frontend über Client Configuration → Export → + JSON bearbeiten ("CacheLibScan": false ergänzen) → Import setzen. Die App + normalisiert die Struktur beim Speichern selbst. + + 6. Reihenfolge Einrichtung einer neuen Instanz (z. B. Instanz N) + + ┌─────────┬───────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Schritt │ Befehl/Aktion │ + ├─────────┼───────────────────────────────────────────────────────────────────────────────────────────────┤ + │ 1 │ Daten-Volume anlegen: mkdir -p /mnt/user/appdata/avior-go-N │ + ├─────────┼───────────────────────────────────────────────────────────────────────────────────────────────┤ + │ 2 │ config.json hinein (ClientName UNRAID, Instance N, CacheLibScan false, PathMappings, Encoder) │ + ├─────────┼───────────────────────────────────────────────────────────────────────────────────────────────┤ + │ 3 │ Komodo: Stack avior-go-unraid-docker-N anlegen (Repo/Branch wie Instanz 1) │ + ├─────────┼───────────────────────────────────────────────────────────────────────────────────────────────┤ + │ 4 │ Komodo: Environment setzen: INSTANCE_SUFFIX=-N, HOST_PORT=1000N, CONTAINER_PORT=1000N, │ + │ │ DATA_DIR="/mnt/user/appdata/avior-go-N"; Extra Args: --build │ + ├─────────┼───────────────────────────────────────────────────────────────────────────────────────────────┤ + │ 5 │ Deploy auslösen │ + └─────────┴───────────────────────────────────────────────────────────────────────────────────────────────┘ + + 7. Verifikation + + ┌───────────────────┬────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Check │ Befehl │ + ├───────────────────┼────────────────────────────────────────────────────────────────────────────────────────────┤ + │ Container läuft │ docker ps | grep avior │ + ├───────────────────┼────────────────────────────────────────────────────────────────────────────────────────────┤ + │ Env durchgereicht │ docker exec env | grep -E "HOST_PORT|CONTAINER_PORT" │ + ├───────────────────┼────────────────────────────────────────────────────────────────────────────────────────────┤ + │ Registrierung │ docker exec -it MongoDB mongosh Avior --eval 'db.clients.find({Name: /UNRAID/}).toArray()' │ + ├───────────────────┼────────────────────────────────────────────────────────────────────────────────────────────┤ + │ Log-Zeit (Berlin) │ docker logs --since 2m | tail -3 │ + ├───────────────────┼────────────────────────────────────────────────────────────────────────────────────────────┤ + │ Log-Zugriff │ docker exec ls -la /data/log/ → drwxr-xr-x mm users │ + └───────────────────┴────────────────────────────────────────────────────────────────────────────────────────────┘ + + 8. Bekannte Fallen + + ┌────────────────────────────────────┬─────────────────────────────────────────────────────────┐ + │ Falle │ Lösung │ + ├────────────────────────────────────┼─────────────────────────────────────────────────────────┤ + │ Komodo "Write Stack File"-Konflikt │ compose nie in der UI editieren, nur im Repo │ + ├────────────────────────────────────┼─────────────────────────────────────────────────────────┤ + │ Port-Mapping falsch │ CONTAINER_PORT muss = 10000 + Instance sein │ + ├────────────────────────────────────┼─────────────────────────────────────────────────────────┤ + │ Doppeltes Suffix (UNRAID-2-2) │ ClientName immer Basiswort "UNRAID", Suffix kommt │ + │ │ automatisch aus Instance │ + ├────────────────────────────────────┼─────────────────────────────────────────────────────────┤ + │ Gleicher ClientName │ absichtlich gleich (UNRAID), Trennung über Instance │ + ├────────────────────────────────────┼─────────────────────────────────────────────────────────┤ + │ Cache-Staleness │ CacheLibScan: false (via Frontend Export/Import setzen) │ + ├────────────────────────────────────┼─────────────────────────────────────────────────────────┤ + │ CacheLibScan verschwindet │ Neues Binary nötig (omitempty-Fix 63cbed4); im │ + │ │ Frontend setzen statt Datei-Ende │ + ├────────────────────────────────────┼─────────────────────────────────────────────────────────┤ + │ QSV auf iGPU statt ARC │ -qsv_device /dev/dri/renderD129 (ARC), nicht renderD128 │ + ├────────────────────────────────────┼─────────────────────────────────────────────────────────┤ + │ Log-Verzeichnis 000 │ behoben (app.go: MkdirAll + Chmod 0755) │ + └────────────────────────────────────┴─────────────────────────────────────────────────────────┘ diff --git a/docker/config.example - UNRAID with ARC380.json b/docker/config.example - UNRAID with ARC380.json new file mode 100644 index 0000000..37d90b5 --- /dev/null +++ b/docker/config.example - UNRAID with ARC380.json @@ -0,0 +1,273 @@ +{ + "Instance": 0, + "ClientName": "UNRAID", + "DatabaseURL": "mongodb://192.168.178.75:27017", + "Redis": { + "Enabled": true, + "Host": "192.168.178.75:6379", + "Password": "", + "DB": 0, + "CacheTtl": 86400000000000, + "ChannelPrefix": "avior" + }, + "Ext": ".mkv", + "PauseOnEncodeError": false, + "AudioFormats": { + "StereoTags": [ + "AC3 Audio Stereo", + "[Dolby Digital 2.0]", + "[stereo]", + "MPEG Audio Stereo", + "AC3 2/0" + ], + "MultiTags": [ + "AC3 Audio 5.1", + "[Dolby Digital 5.1]", + "[AC-3]", + "AC3 Audio 5.0", + " [Dolby Digital 5.0]", + "[5.1]", + "[5.0]", + "[7.1]" + ] + }, + "Resolutions": { + "fhd": "1920x1080", + "hd": "1280x720", + "sd": "720x576", + "stv_fhd": "1920x1088" + }, + "ObsoletePath": "/media/transcoded", + "MediaPaths": [ + "/media/re_encode", + "/media/transcoded", + "/media/tv" + ], + "PathMappings": { + "\\\\UMS\\recording_pool": "/recording_pool", + "\\\\UMS\\recording_spare": "/recording_spare", + "\\\\192.168.178.75\\recording_pool": "/recording_pool", + "\\\\192.168.178.75\\recording_spare": "/recording_spare" + }, + "CacheLibScan": false, + "EstimatedLibSize": 66335, + "Modules": { + "AgeModule": { + "Enabled": false, + "Priority": 2, + "Settings": { + "MaxAge": 5 + } + }, + "AudioModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "Accuracy": "med" + } + }, + "DuplicateLengthCheckModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 3 + } + }, + "ErrorReplaceModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 0 + } + }, + "ErrorSkipModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 8 + } + }, + "LegacyModule": { + "Enabled": true, + "Priority": 4, + "Settings": null + }, + "LengthModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 20 + } + }, + "LogMatchModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "Mode": "include" + } + }, + "MaxSizeModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "MaxSize": 30 + } + }, + "ResolutionModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "MinResolution": 20 + } + }, + "SizeApproxModule": { + "Enabled": false, + "Priority": 2, + "Settings": { + "Difference": 20, + "Fraction": 10, + "SampleCount": 3 + } + } + }, + "EncoderConfig": { + "fhd": { + "OutDirectory": "/media/transcoded/HD1080", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-init_hw_device qsv=qsv", + "-filter_hw_device qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-vf yadif=0:-1:0", + "-global_quality:v 26", + "-c:a libopus" + ], + "Stash": [ + "-vf yadif=0:-1:0 hqdn3d=2:1:2:3", + "-vf hqdn3d=2:1:2:3", + "-filter:v hqdn3d=2:1:2:3", + "-filter:v scale=1280:720", + "-r 25", + "-rc constqp", + "-qp 21", + "", + "-af aformat=channel_layouts=5.1", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "hd": { + "OutDirectory": "/media/transcoded/HD720", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-hwaccel qsv", + "-hwaccel_output_format qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-global_quality:v 26", + "-c:a libopus" + ], + "Stash": [ + "-af aformat=channel_layouts=5.1", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "sd": { + "OutDirectory": "/media/transcoded/SD", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-hwaccel qsv", + "-hwaccel_output_format qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-global_quality:v 26", + "-c:a libopus" + ], + "Stash": [ + "-af aformat=channel_layouts=5.1", + "-b:a 320k", + "", + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "stv_fhd": { + "OutDirectory": "/media/transcoded/HD1080", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-init_hw_device qsv=qsv", + "-filter_hw_device qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-vf yadif=0:-1:0", + "-c:a libopus" + ], + "Stash": [ + "-vf yadif=0:-1:0 hqdn3d=2:1:2:3", + "-vf hqdn3d=2:1:2:3", + "-r 25", + "-filter:v hqdn3d=2:1:2:3", + "-rc constqp", + "-qp 21" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + } + }, + "EncoderPriority": "IDLE" +} \ No newline at end of file diff --git a/docker/config.example.json b/docker/config.example.json new file mode 100644 index 0000000..93087cd --- /dev/null +++ b/docker/config.example.json @@ -0,0 +1,273 @@ +{ + "Instance": 0, + "ClientName": "AVIOR-GO", + "DatabaseURL": "mongodb://192.168.178.75:27017", + "Redis": { + "Enabled": true, + "Host": "192.168.178.75:6379", + "Password": "", + "DB": 0, + "CacheTtl": 86400000000000, + "ChannelPrefix": "avior" + }, + "Ext": ".mkv", + "PauseOnEncodeError": false, + "AudioFormats": { + "StereoTags": [ + "AC3 Audio Stereo", + "[Dolby Digital 2.0]", + "[stereo]", + "MPEG Audio Stereo", + "AC3 2/0" + ], + "MultiTags": [ + "AC3 Audio 5.1", + "[Dolby Digital 5.1]", + "[AC-3]", + "AC3 Audio 5.0", + " [Dolby Digital 5.0]", + "[5.1]", + "[5.0]", + "[7.1]" + ] + }, + "Resolutions": { + "fhd": "1920x1080", + "hd": "1280x720", + "sd": "720x576", + "stv_fhd": "1920x1088" + }, + "ObsoletePath": "/media/transcoded", + "MediaPaths": [ + "/media/re_encode", + "/media/transcoded", + "/media/tv" + ], + "PathMappings": { + "\\\\UMS\\recording_pool": "/recording_pool", + "\\\\UMS\\recording_spare": "/recording_spare", + "\\\\192.168.178.75\\recording_pool": "/recording_pool", + "\\\\192.168.178.75\\recording_spare": "/recording_spare" + }, + "EstimatedLibSize": 66335, + "Modules": { + "AgeModule": { + "Enabled": false, + "Priority": 2, + "Settings": { + "MaxAge": 5 + } + }, + "AudioModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "Accuracy": "med" + } + }, + "DuplicateLengthCheckModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 3 + } + }, + "ErrorReplaceModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 0 + } + }, + "ErrorSkipModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 8 + } + }, + "LegacyModule": { + "Enabled": true, + "Priority": 4, + "Settings": null + }, + "LengthModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 20 + } + }, + "LogMatchModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "Mode": "include" + } + }, + "MaxSizeModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "MaxSize": 30 + } + }, + "ResolutionModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "MinResolution": 20 + } + }, + "SizeApproxModule": { + "Enabled": false, + "Priority": 2, + "Settings": { + "Difference": 20, + "Fraction": 10, + "SampleCount": 3 + } + } + }, + "EncoderConfig": { + "fhd": { + "OutDirectory": "/media/transcoded/HD1080", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-init_hw_device qsv=qsv", + "-filter_hw_device qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-vf yadif=0:-1:0", + "-global_quality:v 26", + "-c:a libopus" + ], + "Stash": [ + "-vf yadif=0:-1:0 hqdn3d=2:1:2:3", + "-vf hqdn3d=2:1:2:3", + "-filter:v hqdn3d=2:1:2:3", + "-filter:v scale=1280:720", + "-r 25", + "-rc constqp", + "-qp 21", + "", + "-af aformat=channel_layouts=5.1", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "hd": { + "OutDirectory": "/media/transcoded/HD720", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-hwaccel qsv", + "-hwaccel_output_format qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-global_quality:v 26", + "-c:a libopus" + ], + "Stash": [ + "-af aformat=channel_layouts=5.1", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "sd": { + "OutDirectory": "/media/transcoded/SD", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-hwaccel qsv", + "-hwaccel_output_format qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-global_quality:v 26", + "-c:a libopus" + ], + "Stash": [ + "-af aformat=channel_layouts=5.1", + "-b:a 320k", + "", + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "stv_fhd": { + "OutDirectory": "/media/transcoded/HD1080", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-init_hw_device qsv=qsv", + "-filter_hw_device qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-vf yadif=0:-1:0", + "-c:a libopus" + ], + "Stash": [ + "-vf yadif=0:-1:0 hqdn3d=2:1:2:3", + "-vf hqdn3d=2:1:2:3", + "-r 25", + "-filter:v hqdn3d=2:1:2:3", + "-rc constqp", + "-qp 21" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + } + }, + "EncoderPriority": "IDLE", + "CacheLibScan": false +} \ No newline at end of file diff --git a/docker/config.example_softenc.json b/docker/config.example_softenc.json new file mode 100644 index 0000000..9ea6438 --- /dev/null +++ b/docker/config.example_softenc.json @@ -0,0 +1,251 @@ +{ + "Instance": 0, + "ClientName": "AVIOR-GO-TEST", + "DatabaseURL": "mongodb://192.168.178.75:27017", + "Redis": { + "Enabled": true, + "Host": "192.168.178.75:6379", + "Password": "", + "DB": 0, + "CacheTtl": 86400000000000, + "ChannelPrefix": "avior" + }, + "Ext": ".mkv", + "PauseOnEncodeError": false, + "AudioFormats": { + "StereoTags": [ + "AC3 Audio Stereo", + "[Dolby Digital 2.0]", + "[stereo]", + "MPEG Audio Stereo", + "AC3 2/0" + ], + "MultiTags": [ + "AC3 Audio 5.1", + "[Dolby Digital 5.1]", + "[AC-3]", + "AC3 Audio 5.0", + " [Dolby Digital 5.0]", + "[5.1]", + "[5.0]", + "[7.1]" + ] + }, + "Resolutions": { + "fhd": "1920x1080", + "hd": "1280x720", + "sd": "720x576", + "stv_fhd": "1920x1088" + }, + "ObsoletePath": "/media/transcoded", + "MediaPaths": [ + "/media/re_encode", + "/media/transcoded", + "/media/tv" + ], + "PathMappings": { + "\\\\192.168.178.75\\recording_pool": "/recording_pool", + "\\\\192.168.178.75\\recording_spare": "/recording_spare", + "\\\\UMS\\recording_pool": "/recording_pool", + "\\\\UMS\\recording_spare": "/recording_spare" + }, + "CacheLibScan": false, + "EstimatedLibSize": 66493, + "Modules": { + "AgeModule": { + "Enabled": false, + "Priority": 2, + "Settings": { + "MaxAge": 5 + } + }, + "AudioModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "Accuracy": "med" + } + }, + "DuplicateLengthCheckModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 3 + } + }, + "ErrorReplaceModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 0 + } + }, + "ErrorSkipModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 8 + } + }, + "LegacyModule": { + "Enabled": true, + "Priority": 4, + "Settings": null + }, + "LengthModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 20 + } + }, + "LogMatchModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "Mode": "include" + } + }, + "MaxSizeModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "MaxSize": 30 + } + }, + "ResolutionModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "MinResolution": 20 + } + }, + "SizeApproxModule": { + "Enabled": false, + "Priority": 2, + "Settings": { + "Difference": 20, + "Fraction": 10, + "SampleCount": 3 + } + } + }, + "EncoderConfig": { + "fhd": { + "OutDirectory": "/media/transcoded/HD1080", + "PreArguments": [], + "PostArguments": [ + "-map 0", + "-c:v libsvtav1", + "-preset 8", + "-crf 26", + "-vf yadif=0:-1:0", + "-c:a libopus" + ], + "Stash": [ + "-vf yadif=0:-1:0 hqdn3d=2:1:2:3", + "-vf hqdn3d=2:1:2:3", + "-filter:v hqdn3d=2:1:2:3", + "-filter:v scale=1280:720", + "-r 25", + "-rc constqp", + "-qp 21", + "", + "-af aformat=channel_layouts=5.1", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "hd": { + "OutDirectory": "/media/transcoded/HD720", + "PreArguments": [], + "PostArguments": [ + "-map 0", + "-t 120", + "-c:v libsvtav1", + "-preset 8", + "-crf 26", + "-c:a libopus" + ], + "Stash": [ + "-af aformat=channel_layouts=5.1", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "sd": { + "OutDirectory": "/media/transcoded/SD", + "PreArguments": [], + "PostArguments": [ + "-map 0", + "-c:v libsvtav1", + "-preset 8", + "-crf 26", + "-c:a libopus" + ], + "Stash": [ + "-af aformat=channel_layouts=5.1", + "-b:a 320k", + "", + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "stv_fhd": { + "OutDirectory": "/media/transcoded/HD1080", + "PreArguments": [], + "PostArguments": [ + "-map 0", + "-c:v libsvtav1", + "-preset 8", + "-crf 26", + "-vf yadif=0:-1:0", + "-c:a libopus" + ], + "Stash": [ + "-vf yadif=0:-1:0 hqdn3d=2:1:2:3", + "-vf hqdn3d=2:1:2:3", + "-r 25", + "-filter:v hqdn3d=2:1:2:3", + "-rc constqp", + "-qp 21" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + } + }, + "EncoderPriority": "IDLE" +} \ No newline at end of file diff --git a/docker/config.test.arc.example.json b/docker/config.test.arc.example.json new file mode 100644 index 0000000..ea8fdff --- /dev/null +++ b/docker/config.test.arc.example.json @@ -0,0 +1,272 @@ +{ + "Instance": 0, + "ClientName": "AVIOR-GO", + "DatabaseURL": "mongodb://192.168.178.75:27017", + "Redis": { + "Enabled": true, + "Host": "192.168.178.75:6379", + "Password": "", + "DB": 0, + "CacheTtl": 86400000000000, + "ChannelPrefix": "avior" + }, + "Ext": ".mkv", + "PauseOnEncodeError": false, + "AudioFormats": { + "StereoTags": [ + "AC3 Audio Stereo", + "[Dolby Digital 2.0]", + "[stereo]", + "MPEG Audio Stereo", + "AC3 2/0" + ], + "MultiTags": [ + "AC3 Audio 5.1", + "[Dolby Digital 5.1]", + "[AC-3]", + "AC3 Audio 5.0", + " [Dolby Digital 5.0]", + "[5.1]", + "[5.0]", + "[7.1]" + ] + }, + "Resolutions": { + "fhd": "1920x1080", + "hd": "1280x720", + "sd": "720x576", + "stv_fhd": "1920x1088" + }, + "ObsoletePath": "/media/transcoded", + "MediaPaths": [ + "/media/re_encode" + ], + "PathMappings": { + "\\\\UMS\\recording_pool": "/recording_pool", + "\\\\UMS\\recording_spare": "/recording_spare", + "\\\\192.168.178.75\\recording_pool": "/recording_pool", + "\\\\192.168.178.75\\recording_spare": "/recording_spare" + }, + "CacheLibScan": false, + "EstimatedLibSize": 66335, + "Modules": { + "AgeModule": { + "Enabled": false, + "Priority": 2, + "Settings": { + "MaxAge": 5 + } + }, + "AudioModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "Accuracy": "med" + } + }, + "DuplicateLengthCheckModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 3 + } + }, + "ErrorReplaceModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 0 + } + }, + "ErrorSkipModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 8 + } + }, + "LegacyModule": { + "Enabled": true, + "Priority": 4, + "Settings": null + }, + "LengthModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "Threshold": 20 + } + }, + "LogMatchModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "Mode": "include" + } + }, + "MaxSizeModule": { + "Enabled": true, + "Priority": 4, + "Settings": { + "MaxSize": 30 + } + }, + "ResolutionModule": { + "Enabled": true, + "Priority": 3, + "Settings": { + "MinResolution": 20 + } + }, + "SizeApproxModule": { + "Enabled": false, + "Priority": 2, + "Settings": { + "Difference": 20, + "Fraction": 10, + "SampleCount": 3 + } + } + }, + "EncoderConfig": { + "fhd": { + "OutDirectory": "/media/transcoded/HD1080", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-init_hw_device qsv=qsv", + "-filter_hw_device qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-vf yadif=0:-1:0", + "-global_quality:v 26", + "-c:a libopus" + ], + "Stash": [ + "-vf yadif=0:-1:0 hqdn3d=2:1:2:3", + "-vf hqdn3d=2:1:2:3", + "-filter:v hqdn3d=2:1:2:3", + "-filter:v scale=1280:720", + "-r 25", + "-rc constqp", + "-qp 21", + "", + "-af aformat=channel_layouts=5.1", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "hd": { + "OutDirectory": "/media/re_encode", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-hwaccel qsv", + "-hwaccel_output_format qsv" + ], + "PostArguments": [ + "-map 0", + "-t 120", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-global_quality:v 26", + "-c:a libopus" + ], + "Stash": [ + "-af aformat=channel_layouts=5.1", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "sd": { + "OutDirectory": "/media/transcoded/SD", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-hwaccel qsv", + "-hwaccel_output_format qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-global_quality:v 26", + "-c:a libopus" + ], + "Stash": [ + "-af aformat=channel_layouts=5.1", + "-b:a 320k", + "", + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + }, + "stv_fhd": { + "OutDirectory": "/media/transcoded/HD1080", + "PreArguments": [ + "-qsv_device", + "/dev/dri/renderD129", + "-init_hw_device qsv=qsv", + "-filter_hw_device qsv" + ], + "PostArguments": [ + "-map 0", + "-c:v av1_qsv", + "-preset veryslow", + "-profile:v main", + "-vf yadif=0:-1:0", + "-c:a libopus" + ], + "Stash": [ + "-vf yadif=0:-1:0 hqdn3d=2:1:2:3", + "-vf hqdn3d=2:1:2:3", + "-r 25", + "-filter:v hqdn3d=2:1:2:3", + "-rc constqp", + "-qp 21" + ], + "StereoArguments": [ + "-af aformat=channel_layouts=stereo", + "-b:a 160k" + ], + "MultiChArguments": [ + "-af", + "pan=7.1|FL=c0|FR=c1|FC=c2|LFE=c3|SL=c4|SR=c5", + "-b:a 320k" + ] + } + }, + "EncoderPriority": "IDLE" +} \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..b6b9504 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -e +# Copy binary into persistent data directory so ReflectionPath() (= directory of the +# executable) resolves to /data and config.json + log/ survive container restarts. +# +# /data/avior-go may be the currently running binary (executed from this volume on a +# previous start). "Text file busy" blocks plain cp onto a running executable, so +# remove first, then copy. rm on a running binary is fine on Linux (inode stays alive +# until the process exits); the new file is created fresh afterwards. +rm -f /data/avior-go +cp /opt/avior-go-bin /data/avior-go +chmod +x /data/avior-go +cd /data +# Drop the restrictive umask some base images inherit (linuxserver sets 077): files +# the app creates (0600-umasked) must stay readable/writable for other users on the +# host (SMB/NFS). 002 keeps owner+group rw and lets group/other read. +umask 002 +# PUID/PGID (LinuxServer convention): run the app as the configured user/group so +# every file it creates (logs in /data/log, .INFO.log next to media, config.json) +# is owned by that UID/GID on the host. Default: root (no env set). +if [ -n "$PUID" ] && [ -n "$PGID" ]; then + # Ensure the target directories are owned by the configured user. + chown -R "$PUID:$PGID" /data 2>/dev/null || true + # setpriv (util-linux) is preinstalled on the Ubuntu-based ffmpeg image; su-exec + # is an Alpine package and NOT available here, but keep it as a fallback. + if command -v setpriv >/dev/null 2>&1; then + exec setpriv --reuid="$PUID" --regid="$PGID" --clear-groups /data/avior-go + elif command -v su-exec >/dev/null 2>&1; then + exec su-exec "$PUID:$PGID" /data/avior-go + else + echo "warning: neither setpriv nor su-exec found, running as root" >&2 + fi +fi +exec /data/avior-go diff --git a/encoder/encoder.go b/encoder/encoder.go index 56ff443..d9a2c2a 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -20,7 +20,7 @@ import ( "github.com/Spiritreader/avior-go/tools" "github.com/kpango/glg" "github.com/rs/xid" - "golang.org/x/sys/windows" + ) type Stats struct { @@ -173,19 +173,7 @@ func Encode(file media.File, start, duration int, overwrite bool, dstDir *string return Stats{false, -1, -1337, "", ""}, err } - hProcess, err := windows.OpenProcess(0x0400|0x0200, false, uint32(cmd.Process.Pid)) - if err != nil { - _ = glg.Warnf("could not get ffmpeg handle using pid %d, err: %s", cmd.Process.Pid, err) - } - err = windows.SetPriorityClass(hProcess, config.PriorityUint32(cfg.Local.EncoderPriority)) - if err != nil { - _ = glg.Warnf("could not set priority %s for ffmpeg handle using pid %d, err: %s", - cfg.Local.EncoderConfig, cmd.Process.Pid, err) - } - err = windows.CloseHandle(hProcess) - if err != nil { - _ = glg.Errorf("could not close handle for pid %d, err: %s", cmd.Process.Pid, err) - } + setProcessPriority(cmd, cfg) // scan stdout scanner := bufio.NewScanner(multiReader) @@ -221,7 +209,7 @@ func Encode(file media.File, start, duration int, overwrite bool, dstDir *string // verify file size ok, vErrify := tools.FfProbeVerfiy(outPath) if vErrify != nil && !(errors.Is(vErrify, tools.NoStreamsError) || errors.Is(vErrify, tools.ZeroDurationError)) { - glg.Warnf("could not verify file, will be assumed good: %s", err) + glg.Warnf("could not verify file, will be assumed good: %s", vErrify) } else if !ok { glg.Warnf("file verification failed, renaming: %s", outPath) timestampString := time.Now().Format("2006-01-02 150405") diff --git a/encoder/priority_linux.go b/encoder/priority_linux.go new file mode 100644 index 0000000..2095dfb --- /dev/null +++ b/encoder/priority_linux.go @@ -0,0 +1,39 @@ +//go:build linux + +package encoder + +import ( + "os/exec" + "syscall" + + "github.com/Spiritreader/avior-go/config" + "github.com/kpango/glg" +) + +// niceLevel maps the configured Windows priority level to a Linux nice value. +// Mapping: HIGH/ABOVE_NORMAL -> -5 (higher priority, requires root/CAP_SYS_NICE), +// NORMAL -> 0, BELOW_NORMAL -> 10, IDLE -> 19. +// Unknown values fall back to 19 (idle) — analogous to the Windows fallback in +// config.PriorityUint32, which returns IDLE for unknown values. +func niceLevel(priority string) int { + switch priority { + case config.PRIORITY_HIGH.String(), config.PRIORITY_ABOVE_NORMAL.String(): + return -5 + case config.PRIORITY_NORMAL.String(): + return 0 + case config.PRIORITY_BELOW_NORMAL.String(): + return 10 + default: + return 19 + } +} + +// setProcessPriority sets the nice level of the spawned ffmpeg process. +// An error (e.g. missing permissions for negative nice values in a Docker container) +// is only logged as a warning; encoding proceeds normally. +func setProcessPriority(cmd *exec.Cmd, cfg *config.Data) { + if err := syscall.Setpriority(syscall.PRIO_PROCESS, cmd.Process.Pid, niceLevel(cfg.Local.EncoderPriority)); err != nil { + _ = glg.Warnf("could not set priority %s for ffmpeg process with pid %d, err: %s", + cfg.Local.EncoderPriority, cmd.Process.Pid, err) + } +} diff --git a/encoder/priority_windows.go b/encoder/priority_windows.go new file mode 100644 index 0000000..06f293e --- /dev/null +++ b/encoder/priority_windows.go @@ -0,0 +1,30 @@ +//go:build windows + +package encoder + +import ( + "os/exec" + + "github.com/Spiritreader/avior-go/config" + "github.com/kpango/glg" + "golang.org/x/sys/windows" +) + +// setProcessPriority sets the Windows PriorityClass of the spawned ffmpeg process. +// Errors are only logged (as before), since the encoding itself doesn't depend on it. +func setProcessPriority(cmd *exec.Cmd, cfg *config.Data) { + hProcess, err := windows.OpenProcess(0x0400|0x0200, false, uint32(cmd.Process.Pid)) + if err != nil { + _ = glg.Warnf("could not get ffmpeg handle using pid %d, err: %s", cmd.Process.Pid, err) + return + } + defer func() { + if err := windows.CloseHandle(hProcess); err != nil { + _ = glg.Errorf("could not close handle for pid %d, err: %s", cmd.Process.Pid, err) + } + }() + if err := windows.SetPriorityClass(hProcess, config.PriorityUint32(cfg.Local.EncoderPriority)); err != nil { + _ = glg.Warnf("could not set priority %s for ffmpeg handle using pid %d, err: %s", + cfg.Local.EncoderPriority, cmd.Process.Pid, err) + } +} diff --git a/go.mod b/go.mod index 102ea52..fad421a 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/gorilla/handlers v1.5.2 github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.3 - github.com/karrick/godirwalk v1.17.0 + github.com/karrick/godirwalk v1.16.2 github.com/kpango/glg v1.6.15 github.com/mitchellh/mapstructure v1.5.0 github.com/redis/go-redis/v9 v9.21.0 @@ -16,6 +16,12 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) +// godirwalk v1.17.0 is broken on Windows: its directory scanner stores the io.EOF +// that ends a normal Readdir loop as the scan error, and Walk returns it without +// consulting ErrorCallback, so every successful walk reports EOF. The project is +// archived upstream, so there is no fix release. Keep it off v1.17.0. +exclude github.com/karrick/godirwalk v1.17.0 + require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect diff --git a/go.sum b/go.sum index b802ce7..3a6f207 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,10 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= -github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/karrick/godirwalk v1.16.2 h1:eY2INUWoB2ZfpF/kXasyjWJ3Ncuof6qZuNWYZFN3kAI= +github.com/karrick/godirwalk v1.16.2/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= diff --git a/media/media.go b/media/media.go index a22e0d0..8902d4d 100644 --- a/media/media.go +++ b/media/media.go @@ -10,6 +10,7 @@ import ( "sort" "strconv" "strings" + "unicode" "github.com/Spiritreader/avior-go/config" "github.com/Spiritreader/avior-go/consts" @@ -172,6 +173,18 @@ func (f *File) OutName() string { return f.Name + " - " + f.Subtitle } +// DuplicateNameKey returns a comparison key containing only lowercase +// Unicode letters and digits. It does not modify the original name. +func DuplicateNameKey(name string) string { + var key strings.Builder + for _, r := range name { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + key.WriteRune(unicode.ToLower(r)) + } + } + return key.String() +} + func (f *File) SanitizeLog() error { found, term, idx := find(f.TunerLog, []string{consts.LOG_DELIM, "VDRAvior:"}, nil) save := false diff --git a/media/media_test.go b/media/media_test.go index 286a904..54327f7 100644 --- a/media/media_test.go +++ b/media/media_test.go @@ -7,7 +7,7 @@ import ( "github.com/Spiritreader/avior-go/consts" ) -func TestSanitize (t *testing.T) { +func TestSanitize(t *testing.T) { //testFile := &File{Path: "D:\\Recording\\Monaco 110 - Madonna di Napoli.mkv"} //testFile := &File{Path: "D:\\Recording\\Neva Give üp - Der einzig wahre Japaner.mkv"} testFile := &File{Path: "D:\\Temp\\test.log.log"} @@ -19,7 +19,26 @@ func TestSanitize (t *testing.T) { fmt.Printf("contains line: %t\n", contains) } -func TestAudioParsing (t *testing.T) { +func TestAudioParsing(t *testing.T) { testFile := &File{Path: `\\UMS\recording_pool\Manual\Thomas Hengelbrock dirigiert Ravel und Franck.mkv`} testFile.Update() } + +func TestDuplicateNameKey(t *testing.T) { + want := "aktivundgesundfaszientherapiepoolkeimestanduppaddling" + for _, name := range []string{ + "aktiv und gesund - Faszientherapie - Poolkeime - Stand-up-Paddling", + "aktiv und gesund Faszientherapie Poolkeime Stand-up-Paddling", + "aktiv und gesund _ Faszientherapie _ Poolkeime _ Stand-up-Paddling", + } { + if got := DuplicateNameKey(name); got != want { + t.Errorf("DuplicateNameKey(%q) = %q, want %q", name, got, want) + } + } + if got := DuplicateNameKey("Ä Ö Ü ß 2024"); got != "äöüß2024" { + t.Errorf("DuplicateNameKey Unicode = %q", got) + } + if got := DuplicateNameKey("A/B"); got != DuplicateNameKey("AB") { + t.Errorf("punctuation should not affect key: %q != %q", got, DuplicateNameKey("AB")) + } +} diff --git a/media/realcase_test.go b/media/realcase_test.go new file mode 100644 index 0000000..57831f9 --- /dev/null +++ b/media/realcase_test.go @@ -0,0 +1,84 @@ +package media + +import "testing" + +// Reale Fälle aus der Session — neuer Film mit .log (Timer Name 2024, Start 2026) +func TestRealNewFilmLog(t *testing.T) { + f := File{ + Name: "Die Löwin", + TunerLog: []string{ + "arte HD (deu) 03/08/2026", + `\\192.168.178.75\recording_pool\recording\Die Löwin_2026-08-03-22-58-00-arte HD (deu).ts`, + "Naming Scheme: %event_%year-%date-%time-%station", + "Device: Tvheadend:9983 20d48b009f 3", + "EventID: 63174, PDC: 0x1C5C0", + "Timer Name: Die Löwin - Spielfilm Deutschland/Estland/Lettland 2024", + "Timer Start: 03/08/2026 22:58:00", + "Timer Duration: 01:44:00 (104 min. incl. 2 min. lead time, 2 min. follow-up time)", + "Timer Options: Teletext=0, Subtitles=0, All Audio Tracks=0, Adjust PAT/PMT=1, EIT EPG Data=0, Transponder Dump=0", + "Timer Source: Search:Regex Fernsehfilm|Spielfilm|Liebesfilm|Thriller|Liebes", + "Monitoring Mode: Start/stop by running status", + "22:58:00 / 00:00:00 (~ 0,00 MB) Start EPG Monitoring", + }, + } + if y := f.ExtractYearFromFile(); y != "2024" { + t.Errorf("neuer Film .log: got %q, want 2024 (Timer Name)", y) + } +} + +// Neuer Film mit .txt (Info= 2024) — der eigentliche Produktionsfall +func TestRealNewFilmTxt(t *testing.T) { + f := File{ + Name: "Die Löwin", + MetadataLog: []string{ + "[Media]", + "Created=03.08.2026 23:00:06", + "Channel=arte HD (deu)", + "[0]", + "Id=63174", + "Date=03.08.2026", + "Title=Die Löwin", + "Info=Spielfilm Deutschland/Estland/Lettland 2024", + }, + } + if y := f.ExtractYearFromFile(); y != "2024" { + t.Errorf("neuer Film .txt: got %q, want 2024 (Info=)", y) + } +} + +// Alter Film: nur .log, "Melodram Südafrika/2011" +func TestRealOldFilmLog(t *testing.T) { + f := File{ + Name: "Die Löwin", + TunerLog: []string{ + "ZDF HD 02.01.2012", + "Die Löwin", + "20:15..21:45", + "Melodram Südafrika/2011", + "20:10:02 Start", + "Total Size 9078,8 MB", + }, + } + if y := f.ExtractYearFromFile(); y != "2011" { + t.Errorf("alter Film .log: got %q, want 2011", y) + } +} + +// Der Produktionsfall: Job bringt den Subtitle mit dem Release-Jahr 2024 mit, +// die .log des Films enthält aber auch das Aufnahmejahr 2026. Der Subtitle +// muss gewinnen (wird zuerst geprüft). +func TestRealSubtitleWinsOverLogRecordingYear(t *testing.T) { + f := File{ + Name: "Die Löwin", + Subtitle: "Spielfilm Deutschland/Estland/Lettland 2024", + TunerLog: []string{ + "arte HD (deu) 03/08/2026", + "Timer Name: Die Löwin - Spielfilm Deutschland/Estland/Lettland 2024", + "Timer Start: 03/08/2026 22:58:00", + "Monitoring Mode: Start/stop by running status", + }, + } + if y := f.ExtractYearFromFile(); y != "2024" { + t.Errorf("Subtitle-Pfad: got %q, want 2024 (Subtitle gewinnt vor .log 2026)", y) + } +} diff --git a/media/year.go b/media/year.go new file mode 100644 index 0000000..7852bc8 --- /dev/null +++ b/media/year.go @@ -0,0 +1,544 @@ +package media + +import ( + "regexp" + "strings" + + "github.com/kpango/glg" +) + +// Ported from movie_nfo_lib (movie_metadata/metadata.py) — see +// .cortex/plans/year-aware-dupes-plan.md. Patterns and extraction order mirror +// extract_txt_metadata / _extract_log_metadata / extract_txt_meta_year_and_countries. +// Only the release-year result is surfaced here (avior-go's duplicate detection +// needs the year, not countries/genres). RE2 notes: no lookahead, \w is ASCII +// (use \p{L}/\p{N} for Unicode). + +const countryHintPattern = `usa|us|u\.?s\.?|uk|u\.?k\.?|gb|gbr|can|cdn|uae|ksa|prc|rok|d-a-ch|dach|ddr|brd|de|at|ch|fr|f|it|es|d|a|cz|cssr|` + + `deutschland|österreich|oesterreich|schweiz|frankreich|italien|spanien|` + + `großbritannien|grossbritannien|belgien|niederlande|luxemburg|` + + `vereinigtes\s+königreich|vereinigtes\s+königreich(?:\s+von\s+amerika)?|` + + `vereinigte\s+staaten(?:\s+von\s+amerika)?|amerika|` + + `portugal|griechenland|graechenland|irland|island|` + + `dänemark|daenemark|schweden|norwegen|finnland|` + + `polen|tschechien|tschechoslowakei|ungarn|rumänien|rumaenien|bulgarien|slowakei|slowenien|` + + `kroatien|serbien|ukraine|ukraina|litauen|lettland|estland|zypern|malta|` + + `kosovo|bosnien|montenegro|mazedonien|nordmazedonien|` + + `brasilien|argentinien|kolumbien|peru|chile|venezuela|uruguay|mexiko|` + + `kuba|cuba|` + + `japan|china|südkorea|suedkorea|nordkorea|hongkong|taiwan|thailand|` + + `vietnam|philippinen|indonesien|malaysia|singapur|kambodscha|cambodia|` + + `indien|pakistan|bangladesch|` + + `israel|iran|irak|saudi-arabien|saudiarabien|libanon|türkei|tuerkei|` + + `südafrika|suedafrika|ägypten|aegypten|marokko|algerien|tunesien|` + + `nigeria|kenia|` + + `australien|neuseeland|kanada|russland|sowjetunion|udssr|ussr|weißrussland|` + + `weissrussland|belarus|` + + `united\s+states(?:\s+of\s+america)?|united\s+kingdom|` + + `canada|mexico|brazil|argentina|` + + `germany|austria|switzerland|france|italy|spain|` + + `netherlands|belgium|portugal|greece|ireland|iceland|` + + `denmark|sweden|norway|finland|` + + `poland|czech(?:\s+republic)?|czechoslovakia|hungary|romania|bulgaria|slovakia|slovenia|` + + `croatia|serbia|ukraine|lithuania|latvia|estonia|cyprus|malta|` + + `kosovo|bosnia|montenegro|` + + `japan|china|korea|thailand|vietnam|philippines|indonesia|malaysia|` + + `singapore|india|pakistan|bangladesh|sri\s+lanka|` + + `israel|iran|turkey|lebanon|` + + `south\s+africa|egypt|morocco|algeria|tunisia|nigeria|kenya|` + + `mauritania|mauretanien|` + + `cote\s+d(?:[\'\"]?\s?ivoire)?|ivory\s+coast|ci|` + + `australia|new\s+zealand|` + + `russia|soviet\s+union|ussr|udssr|belarus|north\s+macedonia|macedonia` + +const genreWords = `(?:fernsehfilm|spielfilm|film|serie|dokumentation|komödie|komoedie|tragikomödie|tragikomoedie|drama|thriller|krimi|melodram|melodrama|animationsfilm|zeichentrickfilm)` + +// countryListRe matches a country list: one or more COUNTRY_HINT alternatives +// separated by / , ; und. No capture group around the list (RE2 chokes on a +// capturing alternation wrapped in a repeat with a following separator). +var countryListRe = regexp.MustCompile(`(?i)(?:` + countryHintPattern + `)(?:\s*(?:/|,|;|und)\s*(?:` + countryHintPattern + `))*`) + +// txtMetaRe mirrors TXT_META_PATTERN: optional genre, then country list, then year. +// Group 1 = year (country list is a non-capturing alternation). +var txtMetaRe = regexp.MustCompile(`(?i)^\s*(?:\([^)]*\)\s*)*(?:[-–—:,]\s*)*(?:` + genreWords + `\s*,?\s*)?(?:` + countryHintPattern + `)(?:\s*(?:/|,|;|und)\s*(?:` + countryHintPattern + `))*\s*(?:,|/|\s)\s*((?:19|20)\d{2})\b`) + +// logMetaLineRe mirrors LOG_META_LINE_PATTERN: genre (optional) + countries + year. +// Group 1 = year. +var logMetaLineRe = regexp.MustCompile(`(?i)(?:^|\s-\s|\s–\s|\s—\s)(?:` + genreWords + `\s+)?(?:` + countryHintPattern + `)(?:\s*(?:/|,|;|und)\s*(?:` + countryHintPattern + `))*\s+((?:19|20)\d{2})\b`) + +// logMetaLineStrictRe mirrors LOG_META_LINE_STRICT_PATTERN: full-line match. +// Group 1 = year. +var logMetaLineStrictRe = regexp.MustCompile(`(?i)^\s*(?:` + genreWords + `\s+)?(?:` + countryHintPattern + `)(?:\s*(?:/|,|;|und)\s*(?:` + countryHintPattern + `))*\s+((?:19|20)\d{2})\s*$`) + +// timerNameMetaRe mirrors TIMER_NAME_META_PATTERN: "Timer Name: ... - Spielfilm Land 2024". +// Group 1 = year. +var timerNameMetaRe = regexp.MustCompile(`(?i)(?:^|\s-\s|\s–\s|\s—\s)(?:` + genreWords + `\s+)?(?:` + countryHintPattern + `)(?:\s*/\s*(?:` + countryHintPattern + `))*\s+((?:19|20)\d{2})\b`) + +// type2GenreCountryYearRe mirrors TYPE2_GENRE_COUNTRY_YEAR_PATTERN. +// Group 1 = year. +var type2GenreCountryYearRe = regexp.MustCompile(`(?i)^\s*(?:[A-Za-zÄÖÜäöüß][\wÄÖÜäöüß-]*(?:\s+[A-Za-zÄÖÜäöüß][\wÄÖÜäöüß-]*)?\s+)?(?:` + countryHintPattern + `)(?:\s*(?:/|,|;|und)\s*(?:` + countryHintPattern + `))*\s*(?:,|/)\s*((?:19|20)\d{2})\s*$`) + +// type2CountryYearFallbackRe mirrors TYPE2_COUNTRY_YEAR_FALLBACK_PATTERN. +// Group 1 = year. +var type2CountryYearFallbackRe = regexp.MustCompile(`(?i)^\s*(?:` + countryHintPattern + `)(?:\s*/\s*(?:` + countryHintPattern + `))*\s*(?:,|/)\s*((?:19|20)\d{2})\s*$`) + +// oldLogMetaRe matches the old-format meta line "Melodram Südafrika/2011" +// (genre + country + year at end). Group 1 = year. +var oldLogMetaRe = regexp.MustCompile(`(?i)^\s*(?:` + genreWords + `)\s+(?:` + countryHintPattern + `)(?:\s*(?:/|,|;|und)\s*(?:` + countryHintPattern + `))*\s*((?:19|20)\d{2})\s*$`) + +// yearFourRe mirrors YEAR_FOUR_RE: 4-digit year. Decade forms like "1980er" +// are rejected manually (RE2 has no lookahead). Group 1 = year. +var yearFourRe = regexp.MustCompile(`((?:19|20)\d{2})`) + +// logSliceEndRe mirrors _slice_log_lines_for_metadata stop markers. +var logSliceEndRe = regexp.MustCompile(`(?i)Removed Filler Data|Total Size|Monitoring Mode:|^\s*\d{1,2}:\d{2}:\d{2}`) + +// yearSuffixRe matches a trailing " (YYYY)" in a name. +var yearSuffixRe = regexp.MustCompile(`\s+\(((?:19|20)\d{2})\)\s*$`) + +// timeRangeRe matches the old-format "20:15..21:45" airtime line. +var timeRangeRe = regexp.MustCompile(`^\d{1,2}:\d{2}\.\.\d{1,2}:\d{2}$`) + +// ExtractYearFromFile resolves the release year for f following the library's +// extract_txt_metadata order: subtitle → .txt (Info=/Description=) → .log +// (Timer Name → meta line), with .log as supplement when .txt lacks a year. +// Returns "" when no year can be determined. Each source is logged so the +// extraction path is transparent (important for diagnosing missing years). +func (f *File) ExtractYearFromFile() string { + // 1. Job subtitle (avior-dis provides it): "Spielfilm Deutschland 2024". + if y := ExtractYear(f.Subtitle); y != "" { + glg.Infof("year extraction: subtitle -> %s", y) + return y + } + // 2. .txt metadata (current format). Info= first, then Description=-derived + // meta line. Recording dates (Created=/Date=) are NEVER release years. + infoLine := "" + for _, l := range f.MetadataLog { + if strings.HasPrefix(l, "Info=") { + infoLine = strings.TrimPrefix(l, "Info=") + break + } + } + if infoLine != "" { + if y := yearFromMetaCandidate(infoLine); y != "" { + glg.Infof("year extraction: txt Info= -> %s", y) + return y + } + glg.Infof("year extraction: txt Info= line found but no year in %q", infoLine) + } + for _, l := range f.MetadataLog { + if strings.HasPrefix(l, "Description=") { + desc := strings.TrimPrefix(l, "Description=") + if seg := firstMetaLikeSegment(desc); seg != "" { + if y := yearFromMetaCandidate(seg); y != "" { + glg.Infof("year extraction: txt Description= -> %s", y) + return y + } + } + } + } + if len(f.MetadataLog) > 0 { + glg.Infof("year extraction: no year in .txt metadata (%d lines)", len(f.MetadataLog)) + } else { + glg.Infof("year extraction: no .txt metadata present") + } + // 3. .log: Timer Name first (most reliable), then the meta line. + logYear := extractYearFromLog(f.TunerLog) + if logYear != "" { + glg.Infof("year extraction: log -> %s", logYear) + return logYear + } + glg.Infof("year extraction: no year found in subtitle/.txt/.log for %s", f.Path) + return "" +} + +// ExtractYear returns the first 4-digit year found in s, or "". +// Decade forms like "1980er" are ignored. +func ExtractYear(s string) string { + if s == "" { + return "" + } + if y := yearFromMetaCandidate(s); y != "" { + return y + } + // Fallback: any 4-digit year not followed by "er" (decade form). + for _, m := range yearFourRe.FindAllStringSubmatchIndex(s, -1) { + if len(m) != 4 { + continue + } + rest := s[m[1]:] + if strings.HasPrefix(rest, "er") { + continue + } + return s[m[0]:m[1]] + } + return "" +} + +// nonCountryWords mirrors the library's _NON_COUNTRY_WORDS: narrative words that +// must not be mistaken for a country before a year ("Jahr 2022", "im 2024"). +var nonCountryWords = map[string]bool{ + "jahr": true, "year": true, "im": true, "ein": true, "eine": true, + "der": true, "die": true, "das": true, "und": true, "mit": true, + "von": true, "für": true, "aus": true, "the": true, + "in": true, "at": true, "nach": true, "auf": true, "bei": true, + "this": true, "that": true, "all": true, +} + +// narrativeMarkerRe mirrors NARRATIVE_MARKER: a non-country word directly +// followed by a digit → the "year" is actually narrative text, not a year. +var narrativeMarkerRe = regexp.MustCompile(`(?i)^\s*(?:jahr|year|im|ein|eine|der|die|das|und|mit|von|für|fuer|aus|the|in|at|nach|auf|bei|this|that|all)\s+\d`) + +// preprocessCandidate mirrors the library's candidate cleaning before matching: +// FSK strip, <> strip, leading/trailing parens, " - subtitle" truncation, +// "Min." suffix strip, comma→space, whitespace collapse, duplicate words. +func preprocessCandidate(s string) string { + s = strings.TrimSpace(s) + fsKRe := regexp.MustCompile(`(?i)^\s*FSK\s*\d+\s*`) + s = fsKRe.ReplaceAllString(s, "") + s = regexp.MustCompile(`^\s*<\s*`).ReplaceAllString(s, "") + s = regexp.MustCompile(`\s*>\s*$`).ReplaceAllString(s, "") + s = regexp.MustCompile(`^\s*\([^)]*\)\s*`).ReplaceAllString(s, "") + s = regexp.MustCompile(`\s+[-–—]\s+.*$`).ReplaceAllString(s, "") + s = regexp.MustCompile(`\s*\([^)]*\)\s*$`).ReplaceAllString(s, "") + s = regexp.MustCompile(`(?i)deutsche\s+demokratische\s+republik`).ReplaceAllString(s, "DDR") + s = regexp.MustCompile(`\([^)]*\)`).ReplaceAllString(s, " ") + s = regexp.MustCompile(`(?i)\bFSK\s*\d+\b`).ReplaceAllString(s, "") + s = regexp.MustCompile(`(?i),?\s*[A-Za-zÄÖÜäöü0-9 &]{2,30}\s*\d{1,3}\s*Min\.?$`).ReplaceAllString(s, "") + s = regexp.MustCompile(`\s*,\s*`).ReplaceAllString(s, " ") + s = regexp.MustCompile(`\s+`).ReplaceAllString(s, " ") + // Collapse duplicate words: "Deutschland Deutschland 2024" -> "Deutschland 2024" + words := strings.Fields(s) + out := make([]string, 0, len(words)) + for i, w := range words { + if i > 0 && strings.EqualFold(w, words[i-1]) { + continue + } + out = append(out, w) + } + return strings.Join(out, " ") +} + +// yearFromMetaCandidate is the full port of the library's _extract_from_candidate +// cascade (metadata.py:589-733). Group 1 = year in all patterns. Returns "" when +// no year can be determined. +func yearFromMetaCandidate(candidate string) string { + if candidate == "" { + return "" + } + s := strings.TrimSpace(candidate) + s = preprocessCandidate(s) + + // Determine the first year position to bound the match window. + mYearFirst := yearFourRe.FindStringIndex(s) + + // Truncate candidate from the first genre/country word up to the first year + // (mirrors start_match logic). + startIdx := -1 + if m := regexp.MustCompile(`(?i)\b(?:` + genreWords + `|` + countryHintPattern + `)\b`).FindStringIndex(s); m != nil { + startIdx = m[0] + } + candidateForMatch := s + if startIdx >= 0 { + if mYearFirst != nil && startIdx <= mYearFirst[1] { + candidateForMatch = s[startIdx:mYearFirst[1]] + } else if mYearFirst != nil { + // Genre/country word appears after the first year: the bounded + // window is empty, fall back to the year-prefix window. + candidateForMatch = s[:mYearFirst[1]] + } else { + candidateForMatch = s[startIdx:] + } + } else if mYearFirst != nil { + candidateForMatch = s[:mYearFirst[1]] + } + + // 1. TXT_META_PATTERN + if m := txtMetaRe.FindStringSubmatch(candidateForMatch); len(m) == 2 { + return m[1] + } + // 2. TYPE2_GENRE_COUNTRY_YEAR_PATTERN + if m := type2GenreCountryYearRe.FindStringSubmatch(candidateForMatch); len(m) == 2 { + return m[1] + } + // 3. loose_re: at end, gated by NARRATIVE_MARKER + candidateNorm := preprocessCandidate(s) + candidateNorm = regexp.MustCompile(`\s+[-–—]\s+.*$`).ReplaceAllString(candidateNorm, "") + candidateNorm = regexp.MustCompile(`\s*\([^)]*\)\s*$`).ReplaceAllString(candidateNorm, "") + if m := looseYearRe.FindStringSubmatch(candidateNorm); len(m) == 2 { + rawCountries := m[1] + if narrativeMarkerRe.MatchString(rawCountries) { + return "" + } + // Single country starting with a genre word: "Spielfilm Deutschland" + parts := splitCountryParts(rawCountries) + if len(parts) == 1 { + for _, sg := range []string{"zeichentrick", "spielfilm", "film", "serie", "dokumentation", "komödie"} { + if strings.HasPrefix(parts[0], sg+" ") { + return m[2] + } + } + } + if len(parts) > 0 { + return m[2] + } + } + // 4. short_genre_re: "Genre Land Jahr" + if m := shortGenreYearRe.FindStringSubmatch(candidate); len(m) == 4 { + g := strings.ToLower(strings.TrimSpace(m[1])) + if shortGenres[g] { + return m[3] + } + } + // 5. permissive_re: anywhere, gated by NARRATIVE_MARKER + if m := permissiveYearRe.FindStringSubmatch(s); len(m) == 3 { + combined := m[1] + " " + m[2] + if narrativeMarkerRe.MatchString(combined) { + return "" + } + parts := splitCountryParts(m[1]) + if len(parts) > 0 { + return m[2] + } + } + return "" +} + +// looseYearRe mirrors the library's loose_re: at end. +var looseYearRe = regexp.MustCompile(`(?i)([^\d\n\r]+?)\s+((?:19|20)\d{2})\s*$`) + +// shortGenreYearRe mirrors short_genre_re: Genre + Countries + Year. +var shortGenreYearRe = regexp.MustCompile(`(?i)^\s*([A-Za-zÄÖÜäöüß\-]{3,20})\s+(.+?)\s+((?:19|20)\d{2})\s*$`) + +// permissiveYearRe mirrors m_permissive: . +var permissiveYearRe = regexp.MustCompile(`(?i)([^\d\n\r|]+?)\s+((?:19|20)\d{2})\b`) + +var shortGenres = map[string]bool{ + "fernsehfilm": true, "spielfilm": true, "film": true, "serie": true, + "dokumentation": true, "komödie": true, "komoedie": true, + "tragikomödie": true, "tragikomoedie": true, "drama": true, + "thriller": true, "krimi": true, "melodram": true, "animationsfilm": true, + "zeichentrick": true, "zeichentrickfilm": true, +} + +// splitCountryParts splits a country string by / , ; und (mirrors the library). +func splitCountryParts(s string) []string { + re := regexp.MustCompile(`(?i)\s*(?:/|,|;|\band\b|\bund\b|und)\s*`) + parts := re.Split(s, -1) + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.ToLower(strings.TrimSpace(p)) + if p != "" { + out = append(out, p) + } + } + return out +} + +// firstMetaLikeSegment picks the best "Land Jahr"-style segment from a +// pipe-separated Description= line (mirrors extract_txt_meta_year_and_countries +// description handling, simplified to the year-bearing segment). +// firstMetaLikeSegment mirrors the library's Description=-segment selection with +// scoring (narrative_penalty, meta_like, short_segment): picks the best +// "Land Jahr"-style segment from a pipe-separated Description= line. +func firstMetaLikeSegment(desc string) string { + parts := []string{} + for _, p := range strings.Split(desc, "|") { + if t := strings.TrimSpace(p); t != "" { + parts = append(parts, t) + } + } + if len(parts) == 0 { + return "" + } + // Single segment: use it if it carries a year. + if len(parts) == 1 { + if ExtractYear(parts[0]) != "" { + return parts[0] + } + return "" + } + // Score each year-bearing segment like the library: prefer meta-like + // (genre+country+year), short segments, low narrative penalty. + type scored struct { + penalty int + meta int + tail int + words int + seg string + } + var candidates []scored + for _, seg := range parts { + mYear := yearFourRe.FindStringIndex(seg) + if mYear == nil { + continue + } + tailLen := len(seg) - mYear[1] + narrativePenalty := 0 + if regexp.MustCompile(`\d{4}\s*:\s*[A-Za-z]`).MatchString(seg) { + narrativePenalty = 1 + } + metaLike := false + if yearFromMetaCandidate(seg) != "" { + metaLike = true + } + wordCount := len(strings.Fields(seg)) + shortSeg := wordCount <= 10 && tailLen < 40 + if !metaLike && !shortSeg { + continue + } + metaScore := 0 + if !metaLike { + metaScore = 1 + } + candidates = append(candidates, scored{narrativePenalty, metaScore, tailLen, wordCount, seg}) + } + if len(candidates) > 0 { + // Sort: penalty asc, meta asc, tail asc (mirrors the library's sort key). + best := candidates[0] + for _, c := range candidates[1:] { + if c.penalty < best.penalty || + (c.penalty == best.penalty && c.meta < best.meta) || + (c.penalty == best.penalty && c.meta == best.meta && c.tail < best.tail) { + best = c + } + } + return best.seg + } + // Fallback: last segment carrying a year and letters (mirrors library's last + // candidate fallback). + for i := len(parts) - 1; i >= 0; i-- { + seg := parts[i] + if yearFourRe.MatchString(seg) && regexp.MustCompile(`[A-Za-zÄÖÜäöüß]`).MatchString(seg) { + return seg + } + } + return "" +} + +// extractYearFromLog mirrors _extract_log_metadata's year sources: +// Timer Name first, then the meta line after the time range. +func extractYearFromLog(tunerLog []string) string { + metaLines := sliceLogMetadata(tunerLog) + // Timer Name is the most reliable source. + for _, line := range metaLines { + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "timer name") { + v := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(strings.TrimSpace(line), "Timer Name"), "timer name")) + v = strings.TrimPrefix(v, ":") + v = strings.TrimSpace(v) + if y := yearFromMetaCandidate(v); y != "" { + return y + } + if y := ExtractYear(v); y != "" { + return y + } + } + } + // Old format: meta line directly after the HH:MM..HH:MM time line. + for i, line := range metaLines { + if timeRangeRe.MatchString(strings.TrimSpace(line)) && i+1 < len(metaLines) { + meta := strings.TrimSpace(metaLines[i+1]) + if m := oldLogMetaRe.FindStringSubmatch(meta); len(m) == 2 { + return m[1] + } + if y := yearFromMetaCandidate(meta); y != "" { + return y + } + } + } + // General meta line anywhere in the metadata section. + for _, line := range metaLines { + if y := yearFromMetaCandidate(line); y != "" { + return y + } + } + return "" +} + +// NormalizeName mirrors movie_nfo_lib normalize_title: lowercase, strip non-word +// chars, collapse whitespace. \p{L}/\p{N} for Unicode (Go \w is ASCII-only). +func NormalizeName(s string) string { + s = strings.ToLower(s) + s = regexp.MustCompile(`[^\p{L}\p{N}\s]`).ReplaceAllString(s, "") + s = regexp.MustCompile(`\s+`).ReplaceAllString(s, " ") + return strings.TrimSpace(s) +} + +// HasYearSuffix reports whether name already ends in " (YYYY)" — prevents +// double-suffixing when the year was already appended. +func HasYearSuffix(name string) bool { + return yearSuffixRe.MatchString(name) +} + +// YearFromSuffix returns the year from a trailing " (YYYY)" suffix, or "". +func YearFromSuffix(name string) string { + m := yearSuffixRe.FindStringSubmatch(name) + if len(m) == 2 { + return m[1] + } + return "" +} + +// IsProbablyEpisodeFilename mirrors is_probably_episode_filename: True for series +// episode filenames (SxxExx, (N_N), NxNN, (Staffel N, Folge N), (123)-in-parens, +// 4-digit in parens NOT at end, Folge/Episode/Kapitel N). A year-suffixed film +// like "Die Löwin (2024)" is NOT an episode (year at end is excluded). +func IsProbablyEpisodeFilename(name string) bool { + stem := strings.TrimSuffix(name, filepathExt(name)) + for _, re := range episodePatterns { + if re.MatchString(stem) { + return true + } + } + // 4-digit in parens NOT at stem end -> episode (e.g. (1188) mid-name). + // (2011) at the very end is a year, not an episode — handled by the position + // check below (RE2 has no lookahead). + if m := episodeFourDigitInParensRe.FindStringIndex(stem); m != nil { + if m[1] < len(stem) && !regexp.MustCompile(`^\s*$`).MatchString(stem[m[1]:]) { + return true + } + } + return false +} + +// filepathExt returns the extension including the dot, or "". +func filepathExt(name string) string { + idx := strings.LastIndexAny(name, "./\\") + if idx < 0 || name[idx] != '.' { + return "" + } + return name[idx:] +} + +var episodePatterns = []*regexp.Regexp{ + // SxxExx / S01_E01 + regexp.MustCompile(`(?i)\bS\d{1,2}E\d{1,2}\b|\bS\d{1,2}_E\d{1,2}\b`), + // (N_N) season_episode in parens + regexp.MustCompile(`\(\d{1,2}_\d{1,2}\)`), + // NxNN bare notation (2x07) + regexp.MustCompile(`\b\d{1,2}x\d{1,2}\b`), + // (Staffel N, Folge N) in parens + regexp.MustCompile(`(?i)[Ss]taffel\s*\d+[^)]*[Ff]olge\s*\d+`), + // 1-3 digit in parens -> episode, never year + regexp.MustCompile(`\(\d{1,3}\)`), + // Folge 5 / Episode 3 / Kapitel 16 + regexp.MustCompile(`(?i)\b(?:Folge|Episode|Ep\.?|Kapitel)\s+\d+`), +} + +// episodeFourDigitInParensRe matches "(1188)" NOT at the stem end (episode +// number). The library uses a lookahead (?!\s*$); RE2 has none, so the position +// check is done manually in IsProbablyEpisodeFilename. +var episodeFourDigitInParensRe = regexp.MustCompile(`\(\d{4}\)`) +func sliceLogMetadata(lines []string) []string { + end := len(lines) + for i, line := range lines { + if logSliceEndRe.MatchString(line) { + end = i + break + } + } + if end > len(lines) { + end = len(lines) + } + return lines[:end] +} diff --git a/media/year_test.go b/media/year_test.go new file mode 100644 index 0000000..153b9f6 --- /dev/null +++ b/media/year_test.go @@ -0,0 +1,165 @@ +package media + +import "testing" + +func TestExtractYear(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"Spielfilm Deutschland 2024", "2024"}, + {"Fernsehfilm Deutschland 2007", "2007"}, + {"Spielfilm USA 2020", "2020"}, + {"Melodram Südafrika/2011", "2011"}, + {"Spielfilm Deutschland/Estland/Lettland 2024", "2024"}, + {"Die Löwin - Spielfilm Deutschland/Estland/Lettland 2024", "2024"}, + {"Dokumentarfilm, Deutschland 2023", "2023"}, + {"Spielfilm Deutschland 2017, ZDF", "2017"}, + {"Ruhe in Frieden", ""}, + {"1980er-Jahre", ""}, + {"", ""}, + } + for _, tt := range tests { + if got := ExtractYear(tt.in); got != tt.want { + t.Errorf("ExtractYear(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestHasYearSuffix(t *testing.T) { + tests := []struct { + in string + want bool + }{ + {"Die Löwin (2024)", true}, + {"Die Löwin", false}, + {"Die Löwin (2024) Teil 2", false}, // year not at end + {"", false}, + } + for _, tt := range tests { + if got := HasYearSuffix(tt.in); got != tt.want { + t.Errorf("HasYearSuffix(%q) = %v, want %v", tt.in, got, tt.want) + } + } +} + +func TestYearFromSuffix(t *testing.T) { + if got := YearFromSuffix("Die Löwin (2024)"); got != "2024" { + t.Errorf("YearFromSuffix = %q, want 2024", got) + } + if got := YearFromSuffix("Die Löwin"); got != "" { + t.Errorf("YearFromSuffix(no suffix) = %q, want empty", got) + } +} + +func TestNormalizeName(t *testing.T) { + a := NormalizeName("Die Löwin - Spielfilm") + b := NormalizeName("Die Löwin Spielfilm") + if a != b { + t.Errorf("NormalizeName mismatch: %q vs %q", a, b) + } + if NormalizeName("Die Löwin") != "die löwin" { + t.Errorf("NormalizeName lower/trim wrong: %q", NormalizeName("Die Löwin")) + } +} + +func TestExtractYearFromFileLegacyLog(t *testing.T) { + // Old format: no .txt, log carries "Melodram Südafrika/2011" after the time line. + f := File{ + Name: "Die Löwin", + TunerLog: []string{ + "ZDF HD 02.01.2012", + "Die Löwin", + "20:15..21:45", + "Melodram Südafrika/2011", + "20:10:02 Start", + "Total Size 9078,8 MB", + }, + } + if y := f.ExtractYearFromFile(); y != "2011" { + t.Errorf("ExtractYearFromFile(legacy log) = %q, want 2011", y) + } +} + +func TestExtractYearFromFileCurrentTxt(t *testing.T) { + // Current format: .txt metadata with Info= line. + f := File{ + Name: "Die Löwin", + MetadataLog: []string{"Info=Spielfilm Deutschland/Estland/Lettland 2024"}, + } + if y := f.ExtractYearFromFile(); y != "2024" { + t.Errorf("ExtractYearFromFile(txt) = %q, want 2024", y) + } +} + +func TestExtractYearFromFileTxtIgnoresRecordingDate(t *testing.T) { + // The .txt carries the RECORDING date (Created=03.08.2026) BEFORE the Info= + // line. The recording year must NOT be used — only the Info= release year. + f := File{ + Name: "Die Löwin", + MetadataLog: []string{ + "[Media]", + "Created=03.08.2026 23:00:06", + "Channel=arte HD (deu)", + "[0]", + "Date=03.08.2026", + "Title=Die Löwin", + "Info=Spielfilm Deutschland/Estland/Lettland 2024", + }, + } + if y := f.ExtractYearFromFile(); y != "2024" { + t.Errorf("ExtractYearFromFile(txt with recording date) = %q, want 2024 (release year), got recording year", y) + } +} + +func TestExtractYearFromFileTimerName(t *testing.T) { + // Timer Name is the most reliable source in .log. + f := File{ + Name: "Die Löwin", + TunerLog: []string{ + "Timer Name: Die Löwin - Spielfilm Deutschland/Estland/Lettland 2024", + "23:00:08 Start Recording", + "Total Size 7489,5 MB", + }, + } + if y := f.ExtractYearFromFile(); y != "2024" { + t.Errorf("ExtractYearFromFile(timer name) = %q, want 2024", y) + } +} + +func TestSliceLogMetadataStopsAtNoise(t *testing.T) { + // Lines after "Removed Filler Data" must not leak into metadata parsing. + lines := []string{ + "Melodram Südafrika/2011", + "Removed Filler Data: 117,8 MB", + "22:58:00 / 00:00:00 (~ 0,00 MB) Start EPG Monitoring", + } + sliced := sliceLogMetadata(lines) + if len(sliced) != 1 { + t.Errorf("sliceLogMetadata kept %d lines, want 1 (stop at noise)", len(sliced)) + } +} + +// Regression: yearFromMetaCandidate panicked with "slice bounds out of range" +// when the first genre/country word appears AFTER the first 4-digit number in +// the candidate (startIdx > mYearFirst[1]). Seen in production on +// "Tele-Gym (5 8)" and "Good bye, Lenin!" jobs crashing the whole service. +func TestYearFromMetaCandidateGenreAfterYear(t *testing.T) { + candidates := []string{ + "2024 Spielfilm", // genre after year + "2023 Deutschland", // country after year + "1980er Jahre Spielfilm", // number-ish prefix, then genre + "PID 5126 AC3 5.1", // log-noise style: number then token + "Spielfilm Deutschland 2024", // normal order must still work + "Aerobic, Bewegung, Tanz", + "Good bye, Lenin!", + } + for _, c := range candidates { + if got := yearFromMetaCandidate(c); got != "" { + t.Logf("yearFromMetaCandidate(%q) = %q (no panic, ok)", c, got) + } + } + if y := ExtractYear("Spielfilm Deutschland 2024"); y != "2024" { + t.Errorf("ExtractYear normal case = %q, want 2024", y) + } +} diff --git a/redis/redis.go b/redis/redis.go index 0d17276..0384f79 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -138,7 +138,13 @@ func AutoManage(prevCfg config.Redis) error { cfg := config.Instance() redis := Get() - if cfg.Local.Redis.Enabled { + // CacheLibScan=false disables the library cache entirely (always fresh FS + // scan), which makes the Redis job broadcast redundant. Skip connecting so + // the instance doesn't hold a pointless Redis session. Re-enabling + // CacheLibScan (or Redis.Enabled) restores it automatically. + redisEffective := cfg.Local.Redis.Enabled && cfg.Local.CacheLibScan + + if redisEffective { if prevCfg.Enabled && (prevCfg.Host != cfg.Local.Redis.Host || prevCfg.Password != cfg.Local.Redis.Password || prevCfg.DB != cfg.Local.Redis.DB || @@ -150,6 +156,9 @@ func AutoManage(prevCfg config.Redis) error { redis.configure() redis.Handle.subscribe() } else { + if cfg.Local.Redis.Enabled && !cfg.Local.CacheLibScan { + glg.Infof("redis: disabled because CacheLibScan is false (always fresh scan)") + } redis.Handle.Close() } return nil diff --git "a/test/identical year same name/Die L\303\266win_2026-08-03-22-58-00-arte HD (deu).log" "b/test/identical year same name/Die L\303\266win_2026-08-03-22-58-00-arte HD (deu).log" new file mode 100644 index 0000000..20290c6 --- /dev/null +++ "b/test/identical year same name/Die L\303\266win_2026-08-03-22-58-00-arte HD (deu).log" @@ -0,0 +1,60 @@ +arte HD (deu) 03/08/2026 +\\192.168.178.75\recording_pool\recording\Die Löwin_2026-08-03-22-58-00-arte HD (deu).ts +Naming Scheme: %event_%year-%date-%time-%station +Device: Tvheadend:9983 20d48b009f 3 +EventID: 63174, PDC: 0x1C5C0 +Timer Name: Die Löwin - Spielfilm Deutschland/Estland/Lettland 2024 +Timer Start: 03/08/2026 22:58:00 +Timer Duration: 01:44:00 (104 min. incl. 2 min. lead time, 2 min. follow-up time) +Timer Options: Teletext=0, Subtitles=0, All Audio Tracks=0, Adjust PAT/PMT=1, EIT EPG Data=0, Transponder Dump=0 +Timer Source: Search:Regex Fernsehfilm|Spielfilm|Liebesfilm|Thriller|Liebes +Monitoring Mode: Start/stop by running status + +22:58:00 / 00:00:00 (~ 0,00 MB) Start EPG Monitoring +22:58:00 / 00:00:00 (~ 0,00 MB) Die Löwin not running | EventID: 63174 PDC: 0x1C5C0 +22:58:01 / 00:00:01 (~ 0,00 MB) Truman Capote und "Kaltblütig" running | EventID: 63173 PDC: 0x1C585 +22:59:50 / 00:01:50 (~ 0,00 MB) Truman Capote und "Kaltblütig" running | EventID: 63173 PDC: 0x1C585 +22:59:51 / 00:01:51 (~ 0,00 MB) Die Löwin starts in a few seconds | EventID: 63174 PDC: 0x1C5C0 +23:00:06 / 00:02:06 (~ 0,00 MB) Die Löwin running | EventID: 63174 PDC: 0x1C5C0 + +23:00:06 / 00:00:00 (~ 0,00 MB) Start Recording +23:00:07 / 00:00:00 (~ 0,00 MB) Planet Finance not running | EventID: 63175 PDC: 0x24028 +23:00:08 / 00:00:01 (~ 0,09 MB) PID 5111: H.264 Video, 16:9, 1280x720, 50 fps +23:00:08 / 00:00:01 (~ 0,09 MB) PID 5112: MPEG Audio Stereo, 48 khz, 192 kbps +00:24:34 / 01:24:27 (~ 6172,55 MB) Errors: 4 +00:25:26 / 01:25:20 (~ 6237,85 MB) Errors: 4 +00:38:01 / 01:37:54 (~ 7215,25 MB) Planet Finance not running | EventID: 63175 PDC: 0x24028 +00:38:01 / 01:37:55 (~ 7216,55 MB) Die Löwin running | EventID: 63174 PDC: 0x1C5C0 +00:42:25 / 01:42:19 (~ 7488,17 MB) Planet Finance running | EventID: 63175 PDC: 0x24028 +00:42:26 / 01:42:19 (~ 7489,48 MB) Stop + +Average Data Rate: 1,220 MB/s +Total Size: 7489,5 MB (7853286964 Bytes) +Removed Filler Data: 117,8 MB (1,6%) + +avior-go info +avior-go - Saturday 2026-08-08 10:31:49 +0200 CEST + +OriginalPath: /recording_pool/coding_test/Die Löwin_2026-08-03-22-58-00-arte HD (deu).ts +Recorded/Length: 102m/100m +Audio: STEREO +EncodeParams: [] + +Module Results: +LengthModule: noch nicht - ok: r:102m / l:100m (d:-2% / t:20%) +MaxSizeModule: noch nicht - ok +ErrorSkipModule: noch nicht - ok + +Dupe Module Results: +DupPath: /media/tv/FilmeHD/FilmeHD_sonst/tobecut/Die Löwin.mkv +DuplicateLengthCheckModule: noch nicht - duplicate had insufficient length data, skipping module +ErrorReplaceModule: noch nicht - ok +LegacyModule: noch nicht - ok +AudioModule: noch nicht - no action: n:STEREO vs n:STEREO +ResolutionModule: noch nicht - resolution is the same +LogMatchModule: allow replacement - include match (mode include): -c:v av1_qsv -global_quality:v + +Encoder Info: +OutputPath: /media/tv/FilmeHD/FilmeHD_sonst/tobecut/Die Löwin.mkv +Duration: 7m56.265974696s +Parameters: -n -qsv_device /dev/dri/renderD129 -hwaccel qsv -hwaccel_output_format qsv -i /recording_pool/coding_test/Die Löwin_2026-08-03-22-58-00-arte HD (deu).ts -map 0 -c:v av1_qsv -preset veryslow -profile:v main -global_quality:v 26 -c:a libopus -af aformat=channel_layouts=stereo -b:a 160k /media/tv/FilmeHD/FilmeHD_sonst/tobecut/Die Löwin.mkv diff --git "a/test/identical year same name/Die L\303\266win_2026-08-03-22-58-00-arte HD (deu).txt" "b/test/identical year same name/Die L\303\266win_2026-08-03-22-58-00-arte HD (deu).txt" new file mode 100644 index 0000000..08b52ad --- /dev/null +++ "b/test/identical year same name/Die L\303\266win_2026-08-03-22-58-00-arte HD (deu).txt" @@ -0,0 +1,26 @@ +[General] +Version=1.1 + +[Media] +Created=03.08.2026 23:00:06 +Channel=arte HD (deu) + +[0] +Id=63174 +Date=03.08.2026 +Time=23:00:00 +Duration=01:40:00 +Title=Die Löwin +Info=Spielfilm Deutschland/Estland/Lettland 2024 +Description=Sanitäterin Helena sucht verzweifelt nach ihrer 15-jährigen Tochter Stefi, die in eine gewaltbereite Jugendclique abzugleiten droht. Als das Mädchen nach einer dramatischen Nacht im Krankenhaus landet, bringt Helena sie gegen ihren Willen in ein abgelegenes Landhaus. Drama aus Estland über Fürsorge und Kontrolle.||[16:9] [SUB] [AD] [PDC 03.08. 23:00] +Charset=255 +Content=16 +MinimumAge=0 +TimerID={69137C2D-AF4D-47C5-9F94-4A70D5D7927A} + +[Stats] +Errors=9 +Size=7,31 GB (7853286964 bytes) +Avr. Datarate=1,220 MB/s +Device=Tvheadend:9983 20d48b009f 3 + diff --git "a/test/identical year same name/searchpath_existing _videos/Die L\303\266win LogMatchModule 2026-08-08 1023.log" "b/test/identical year same name/searchpath_existing _videos/Die L\303\266win LogMatchModule 2026-08-08 1023.log" new file mode 100644 index 0000000..6d4e7b6 --- /dev/null +++ "b/test/identical year same name/searchpath_existing _videos/Die L\303\266win LogMatchModule 2026-08-08 1023.log" @@ -0,0 +1,35 @@ +ZDF HD 02.01.2012 + +Die Lwin +20:15..21:45 + +Melodram Sdafrika/2011 + +Nach langen Jahren in Deutschland kehrt die junge rztin Lena nach Sdafrika zurck. Ihr Grovater Jo, auf dessen Farm sie als Kind mit ihren Eltern gelebt hat, ist sterbenskrank. Lena will den alten Growildjger berreden, sich in Berlin behandeln zu lassen. Doch Jo hat eigene Plne: Seine Enkelin soll die Leitung seiner neuen Stiftung bernehmen, die sich der Auswilderung von verwundeten Lwen widmet. +Die Konfrontation mit den majesttischen Raubtieren ruft lang verdrngte Erinnerungen in ihr wach. +Sdafrika, 2011 + +20:10:02 Start +20:10:04 Video: 16:9 / 1280x720 @1,1 MB +20:10:04 Audio: AC3 2/0 / 448 kbps / 48 khz @1,1 MB +21:55:00 Stop + +Total Size 9078,8 MB (9519807563 Bytes) + +avior-go info +VDR-U - Sunday 2023-06-25 10:15:34 +0200 CEST + +OriginalPath: \\UMS\media\tv\FilmeHD\FilmeHD_sonst\tobecut\Die Löwin.mkv +Recorded/Length: -1m/-1m +Audio: STEREO +EncodeParams: [] + +Module Results: +LengthModule: noch nicht - disabled +MaxSizeModule: noch nicht - disabled +ErrorSkipModule: noch nicht - disabled + +Encoder Info: +OutputPath: \\UMS\media\re_encode\FilmeHD_sonst\tobecut\Die Löwin.mkv +Duration: 5m58.142741s +Parameters: -n -init_hw_device d3d11va=qsv:MFX_IMPL_hw_any -hwaccel qsv -filter_hw_device qsv -hwaccel_output_format qsv -i \\UMS\media\tv\FilmeHD\FilmeHD_sonst\tobecut\Die Löwin.mkv -c:v av1_qsv -global_quality:v 27 -preset veryslow -profile:v main -c:a libopus -af aformat=channel_layouts=stereo -b:a 160k \\UMS\media\re_encode\FilmeHD_sonst\tobecut\Die Löwin.mkv diff --git a/tools/build.ps1 b/tools/build.ps1 new file mode 100644 index 0000000..730e705 --- /dev/null +++ b/tools/build.ps1 @@ -0,0 +1,7 @@ +param([ValidateSet("windows","linux","all")][string]$Target = "windows") +$env:CGO_ENABLED = "0" +switch ($Target) { + "windows" { $env:GOOS="windows"; $env:GOARCH="amd64"; go build -ldflags "-s -w" -o dist/avior-go-windows-amd64.exe app.go } + "linux" { $env:GOOS="linux"; $env:GOARCH="amd64"; go build -ldflags "-s -w" -o dist/avior-go-linux-amd64 app.go } + "all" { & $PSCommandPath -Target windows; & $PSCommandPath -Target linux } +} diff --git a/tools/build.sh b/tools/build.sh new file mode 100644 index 0000000..b4b34bb --- /dev/null +++ b/tools/build.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -e +target="${1:-linux}" +export CGO_ENABLED=0 GOARCH=amd64 +case "$target" in + windows) GOOS=windows go build -ldflags "-s -w" -o dist/avior-go-windows-amd64.exe app.go ;; + linux) GOOS=linux go build -ldflags "-s -w" -o dist/avior-go-linux-amd64 app.go ;; + all) "$0" windows && "$0" linux ;; + *) echo "usage: $0 [windows|linux|all]" >&2; exit 1 ;; +esac diff --git a/worker/worker.go b/worker/worker.go index 219c273..8569013 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -30,6 +30,40 @@ var ( previousEncoderLineOut []string ) +// translatePath maps UNC paths from DB jobs to local container paths using the +// configured mappings (longest prefix match, case-insensitive). Returns input +// unchanged when mappings is empty or no prefix matches. +func translatePath(path string, mappings map[string]string) string { + if len(mappings) == 0 || path == "" { + return path + } + // Normalize the input to backslash form so both / and \ separators compare equal. + normalized := strings.ReplaceAll(path, "/", "\\") + // Build normalized-key -> original-key index once. + normToOrig := make(map[string]string, len(mappings)) + for orig, _ := range mappings { + k := strings.TrimRight(strings.ReplaceAll(orig, "/", "\\"), "\\") + if k != "" { + normToOrig[k] = orig + } + } + bestKey := "" + bestLen := -1 + for k := range normToOrig { + if len(k) > bestLen && len(normalized) >= len(k) && + strings.EqualFold(normalized[:len(k)], k) { + bestKey = k + bestLen = len(k) + } + } + if bestKey == "" { + return path + } + value := strings.TrimRight(strings.ReplaceAll(mappings[normToOrig[bestKey]], "\\", "/"), "/") + remainder := strings.ReplaceAll(normalized[bestLen:], "\\", "/") + return value + remainder +} + func ProcessJob(dataStore *db.DataStore, client *structs.Client, job *structs.Job, resumeChan chan string) { cfg := config.Instance() state.InFile = job.Path @@ -46,7 +80,11 @@ func ProcessJob(dataStore *db.DataStore, client *structs.Client, job *structs.Jo }() //populate media file - mediaFile := &media.File{Path: job.Path, Name: job.Name, Subtitle: job.Subtitle, CustomParams: job.CustomParameters} + translatedPath := translatePath(job.Path, cfg.Local.PathMappings) + if translatedPath != job.Path { + _ = glg.Infof("translated job path %s -> %s", job.Path, translatedPath) + } + mediaFile := &media.File{Path: translatedPath, Name: job.Name, Subtitle: job.Subtitle, CustomParams: job.CustomParameters} err := mediaFile.Update() if err != nil { _ = glg.Errorf("couldn't parse media file: %s", err) @@ -56,6 +94,40 @@ func ProcessJob(dataStore *db.DataStore, client *structs.Client, job *structs.Jo _ = glg.Logf("trimmed name: %s", mediaFile.OutName()) jobLog.AddFileProperties(*mediaFile) + // Year-aware duplicates: if the exact name already exists in the library but + // the release year differs, rename to "Title (YYYY)" so the two films are + // treated as separate. The later exact-name duplicate scan then matches the + // suffixed name and the existing modules decide replacement. + // The year-aware scan doubles as the duplicate scan: checkForDuplicates + // returns the same matches the later duplicate check would find, unless a + // year collision renamed the file (then the new name needs a fresh scan). + var duplicates []media.File + if cfg.Local.YearAwareDupes && !media.HasYearSuffix(mediaFile.Name) { + year := mediaFile.ExtractYearFromFile() + if year == "" { + _ = glg.Infof("year-aware dupes: no release year for %s, skipping collision check", mediaFile.Name) + } else { + dupeYear, matches := findDuplicateYear(mediaFile, dataStore) + duplicates = matches + switch { + case dupeYear == "": + // findDuplicateYear already logged "no normalized-name + // duplicate" when nothing matched; only report a found + // duplicate whose year could not be resolved. + if len(matches) > 0 { + _ = glg.Infof("year-aware dupes: exact-name duplicate found but its year is unknown, cannot decide collision for %s", mediaFile.Name) + } + case dupeYear == year: + _ = glg.Infof("year-aware dupes: duplicate %s has same year %s, treating as same film", mediaFile.Name, year) + default: + _ = glg.Infof("year collision: appending (%s) to %s (existing file has %s)", year, mediaFile.Name, dupeYear) + mediaFile.Name = fmt.Sprintf("%s (%s)", mediaFile.Name, year) + duplicates = nil // name changed, previous matches are stale + jobLog.Add(fmt.Sprintf("Year collision: renamed to %s (existing has %s)", mediaFile.OutName(), dupeYear)) + } + } + } + // run single file modules jobLog.Add("") res := runModules(jobLog, *mediaFile) @@ -66,18 +138,22 @@ func ProcessJob(dataStore *db.DataStore, client *structs.Client, job *structs.Jo return } - // check for duplicates and run modules + // check for duplicates and run modules. + // duplicates is nil when the year-aware check did not run (disabled, name + // already suffixed, no year) or renamed the file — only then rescan. var redirectDir *string = nil var obsoleteMovedLogPaths map[string]string = nil var obsoleteMovedFilePath map[string]string = nil - duplicates, err := checkForDuplicates(mediaFile) - if err != nil { - _ = glg.Errorf("duplicate scan failed, please fix. Pausing service to prevent unwanted behavior: %s", err) - state.Paused = true - state.PauseReason = consts.PAUSE_REASON_DUPLICATE_SCAN - appendJobTemplate(*job, jobLog, false) - writeSkippedLog(mediaFile, jobLog, false) - return + if duplicates == nil { + duplicates, err = checkForDuplicates(mediaFile) + if err != nil { + _ = glg.Errorf("duplicate scan failed, please fix. Pausing service to prevent unwanted behavior: %s", err) + state.Paused = true + state.PauseReason = consts.PAUSE_REASON_DUPLICATE_SCAN + appendJobTemplate(*job, jobLog, false) + writeSkippedLog(mediaFile, jobLog, false) + return + } } if dupeLen := len(duplicates); dupeLen > 0 { _ = glg.Infof("found %d duplicates, selecting first", dupeLen) @@ -177,7 +253,7 @@ func ProcessJob(dataStore *db.DataStore, client *structs.Client, job *structs.Jo jobLog.Add("Encoder Info:") // invalidate cache in non-redis mode as it won't be recent anymore after encoding a job - if !redis.Handle.Running(){ + if !redis.Handle.Running() { cache.Instance().Library.Valid = false } @@ -206,7 +282,7 @@ func ProcessJob(dataStore *db.DataStore, client *structs.Client, job *structs.Jo if redirectDir != nil { rollbackAllDupMoves(jobLog, obsoleteMovedFilePath, obsoleteMovedLogPaths) } - if (cfg.Local.PauseOnEncodeError) { + if cfg.Local.PauseOnEncodeError { state.Paused = true state.PauseReason = consts.PAUSE_REASON_ENCODE_ERROR } @@ -228,7 +304,7 @@ func ProcessJob(dataStore *db.DataStore, client *structs.Client, job *structs.Jo if redirectDir != nil { rollbackAllDupMoves(jobLog, obsoleteMovedFilePath, obsoleteMovedLogPaths) } - if (cfg.Local.PauseOnEncodeError) { + if cfg.Local.PauseOnEncodeError { state.Paused = true state.PauseReason = consts.PAUSE_REASON_ENCODE_ERROR } @@ -261,7 +337,7 @@ func ProcessJob(dataStore *db.DataStore, client *structs.Client, job *structs.Jo } // broadcast job if redis is enabled - if (redis.Handle.Running()) { + if redis.Handle.Running() { _ = glg.Infof("redis: broadcasting job %s", stats.OutputPath) err := redis.Handle.PushMessage(stats.OutputPath) if err != nil { @@ -404,12 +480,18 @@ func runDupeModules(jobLog *joblog.Data, fileNew media.File, fileDup media.File) // If the withFfmpegOut flag is set, the ffmpeg output will be appended to the info log, but not to the skipped log. func writeSkippedLog(mediaFile *media.File, jobLog *joblog.Data, withFfmpegOut bool) { mediaFile.LogPaths = append(mediaFile.LogPaths, mediaFile.Path+".INFO.log") + infoLogPath := mediaFile.Path + ".INFO.log" if withFfmpegOut { jobLogWithFfmpeg := *jobLog appendFfmpegOutput(&jobLogWithFfmpeg, state.Encoder) - _ = jobLogWithFfmpeg.AppendTo(mediaFile.Path+".INFO.log", false, false) + _ = jobLogWithFfmpeg.AppendTo(infoLogPath, false, false) } else { - _ = jobLog.AppendTo(mediaFile.Path+".INFO.log", false, false) + _ = jobLog.AppendTo(infoLogPath, false, false) + } + // lumberjack creates files with mode 0600. On Unraid the .INFO.log must be + // readable/writable by every user (nobody:users), so fix the mode explicitly. + if err := os.Chmod(infoLogPath, 0666); err != nil { + _ = glg.Warnf("could not chmod INFO log %s: %s", infoLogPath, err) } _ = jobLog.AppendTo(filepath.Join(globalstate.ReflectionPath(), "log", "skipped.log"), false, true) } @@ -424,9 +506,9 @@ func writeSkippedLog(mediaFile *media.File, jobLog *joblog.Data, withFfmpegOut b // // In case of an error, the path will still be returned for rollback purposes, but the error will be non-nil func moveMediaFile(file media.File, dstDir string, moduleName *string) (error, map[string]string) { - _, err := os.Stat(dstDir) - if os.IsNotExist(err) { - _ = os.Mkdir(dstDir, 0777) + if err := os.MkdirAll(dstDir, 0777); err != nil { + _ = glg.Errorf("could not create destination directory %s: %s", dstDir, err) + return err, nil } fileOut := strings.TrimSuffix(filepath.Base(file.Path), filepath.Ext(file.Path)) if moduleName != nil { @@ -436,7 +518,7 @@ func moveMediaFile(file media.File, dstDir string, moduleName *string) (error, m } fileOut += filepath.Ext(file.Path) fileOut = filepath.Join(dstDir, fileOut) - err = tools.MoppyFile(file.Path, fileOut, true) + err := tools.MoppyFile(file.Path, fileOut, true) if err != nil { return err, map[string]string{file.Path: fileOut} } @@ -451,9 +533,9 @@ func moveMediaFile(file media.File, dstDir string, moduleName *string) (error, m // // In case of an error, the path of the failed move will still be returned for rollback purposes func moveLogs(file media.File, dstDir string, moduleName *string) (error, map[string]string) { - _, err := os.Stat(dstDir) - if os.IsNotExist(err) { - _ = os.Mkdir(dstDir, 0777) + if err := os.MkdirAll(dstDir, 0777); err != nil { + _ = glg.Errorf("could not create destination directory %s: %s", dstDir, err) + return err, nil } toMovePaths := make(map[string]string) for _, log := range file.LogPaths { @@ -477,9 +559,9 @@ func moveLogs(file media.File, dstDir string, moduleName *string) (error, map[st } func copyLogsToEncOut(file media.File, dstDir string) error { - _, err := os.Stat(dstDir) - if os.IsNotExist(err) { - _ = os.Mkdir(dstDir, 0777) + if err := os.MkdirAll(dstDir, 0777); err != nil { + _ = glg.Errorf("could not create destination directory %s: %s", dstDir, err) + return err } toMovePaths := make(map[string]string) for _, log := range file.LogPaths { @@ -512,14 +594,30 @@ func checkForDuplicates(file *media.File) ([]media.File, error) { matches := make([]media.File, 0) libCache := &cache.Instance().Library + pathsFingerprint := strings.Join(cfg.Local.MediaPaths, "\x00") - // if redis is enabled the cache lifetime is determined by ttl - if redis.Get().Handle.Running() && (time.Now().Add(-cfg.Local.Redis.CacheTtl)).After(libCache.LastUpdate) { - _ = glg.Infof("invalidating shared cache after %s due to ttl", cfg.Local.Redis.CacheTtl) - libCache.Valid = false - } else if !redis.Get().Handle.Running() && (time.Now().Add(-time.Minute * 5)).After(libCache.LastUpdate) { - _ = glg.Infof("auto invalidating local lib cache after 5 minutes") + // CacheLibScan=false: always scan the MediaPaths fresh. Under Docker the walk + // is a direct local FS access (<1s), so caching only adds staleness bugs. + if !cfg.Local.CacheLibScan { libCache.Valid = false + } else { + // If the configured MediaPaths changed since the cache was built, the cache is + // stale even if the TTL has not expired: files in newly added paths would never + // be seen as duplicates until a restart. Fingerprint the path list and + // invalidate on change. + if libCache.ScannedPaths != "" && libCache.ScannedPaths != pathsFingerprint { + _ = glg.Infof("invalidating shared cache because MediaPaths changed") + libCache.Valid = false + } + + // if redis is enabled the cache lifetime is determined by ttl + if redis.Get().Handle.Running() && (time.Now().Add(-cfg.Local.Redis.CacheTtl)).After(libCache.LastUpdate) { + _ = glg.Infof("invalidating shared cache after %s due to ttl", cfg.Local.Redis.CacheTtl) + libCache.Valid = false + } else if !redis.Get().Handle.Running() && (time.Now().Add(-time.Minute * 5)).After(libCache.LastUpdate) { + _ = glg.Infof("auto invalidating local lib cache after 5 minutes") + libCache.Valid = false + } } fillCache := false @@ -538,6 +636,7 @@ func checkForDuplicates(file *media.File) ([]media.File, error) { } libCache.Valid = true libCache.LastUpdate = time.Now() + libCache.ScannedPaths = pathsFingerprint cfg.Local.EstimatedLibSize = state.FileWalker.Position } else { _ = glg.Infof("scanning via memcache") @@ -550,13 +649,23 @@ func checkForDuplicates(file *media.File) ([]media.File, error) { return matches, nil } +func duplicateNameMatch(candidate, existing, extension string) bool { + candidateExt := filepath.Ext(candidate) + existingExt := filepath.Ext(existing) + if !strings.EqualFold(candidateExt, extension) || !strings.EqualFold(existingExt, extension) { + return false + } + return media.DuplicateNameKey(strings.TrimSuffix(candidate, candidateExt)) == + media.DuplicateNameKey(strings.TrimSuffix(existing, existingExt)) +} + func traverseMemCache(file *media.File, libCache *cache.Library) []media.File { matches := make([]media.File, 0) for _, path := range libCache.Data { - if filepath.Base(path) == file.OutName()+config.Instance().Local.Ext { + if duplicateNameMatch(file.OutName()+config.Instance().Local.Ext, filepath.Base(path), config.Instance().Local.Ext) { _ = glg.Infof("found duplicate: %s", path) - file := &media.File{Path: path} - matches = append(matches, *file) + duplicate := &media.File{Path: path} + matches = append(matches, *duplicate) } if state.FileWalker.Position%1000 == 0 { _ = glg.Logf("current dir: %s, position: %d/%d", @@ -575,10 +684,10 @@ func traverseDir(file *media.File, path string, fillCache bool) ([]media.File, e if de.IsDir() && strings.HasPrefix(de.Name(), ".") { return errors.New("directory ignored") } - if !de.IsDir() && de.Name() == (file.OutName()+config.Instance().Local.Ext) { - file := &media.File{Path: path} + if !de.IsDir() && duplicateNameMatch(file.OutName()+config.Instance().Local.Ext, de.Name(), config.Instance().Local.Ext) { + duplicate := &media.File{Path: path} _ = glg.Infof("found duplicate: %s", path) - matches = append(matches, *file) + matches = append(matches, *duplicate) } if !de.IsDir() && strings.HasSuffix(de.Name(), config.Instance().Local.Ext) { if state.FileWalker.Position%1000 == 0 { @@ -606,3 +715,37 @@ func traverseDir(file *media.File, path string, fillCache bool) ([]media.File, e } return matches, nil } + +// findDuplicateYear runs the normalized-name duplicate scan for file and +// returns the release year of the first found duplicate (from its .txt/.log via +// ExtractYearFromFile, or from a " (YYYY)" suffix in its filename), plus the +// full duplicate match list. The year is "" when no duplicate exists or no year +// can be determined; the matches let the caller skip a second duplicate scan. +func findDuplicateYear(file *media.File, dataStore *db.DataStore) (string, []media.File) { + duplicates, err := checkForDuplicates(file) + if err != nil { + _ = glg.Warnf("year collision scan failed for %s: %s", file.Path, err) + return "", nil + } + if len(duplicates) == 0 { + _ = glg.Infof("year-aware dupes: no normalized-name duplicate for %s", file.OutName()+config.Instance().Local.Ext) + return "", duplicates + } + dupe := duplicates[0] + _ = glg.Infof("year-aware dupes: normalized-name duplicate: candidate=%s, existing=%s", file.OutName()+config.Instance().Local.Ext, dupe.Path) + // A year suffix in the filename is the cheapest, most reliable source. + if y := media.YearFromSuffix(filepath.Base(dupe.Path)); y != "" { + _ = glg.Infof("year-aware dupes: existing duplicate %s -> year %s from filename suffix", dupe.Path, y) + return y, duplicates + } + if err := dupe.Update(); err != nil { + _ = glg.Warnf("couldn't parse duplicate log file for year: %s (existing=%s)", err, dupe.Path) + } + year := dupe.ExtractYearFromFile() + if year == "" { + _ = glg.Infof("year-aware dupes: existing duplicate %s has unknown year", dupe.Path) + } else { + _ = glg.Infof("year-aware dupes: existing duplicate %s -> year %s from metadata", dupe.Path, year) + } + return year, duplicates +} diff --git a/worker/worker_test.go b/worker/worker_test.go index 4af2742..8c8ed6a 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -1,8 +1,12 @@ package worker import ( + "os" + "path/filepath" + "runtime" "testing" + "github.com/Spiritreader/avior-go/joblog" "github.com/Spiritreader/avior-go/media" ) @@ -13,3 +17,62 @@ func TestTraverse(t *testing.T) { } traverseDir(&file, "\\\\UMS\\media\\transcoded", false) } + +// A fully readable directory must traverse without error. godirwalk v1.17.0 +// regressed here on Windows: its scanner stores the io.EOF that ends a normal +// Readdir loop as the scan error, and Walk returns it without consulting +// ErrorCallback, so every successful walk looked like a failure. +func TestTraverseDirCleanDirHasNoError(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sub"), 0o755); err != nil { + t.Fatalf("setup: %s", err) + } + for _, name := range []string{"a.mkv", filepath.Join("sub", "b.mkv")} { + if err := os.WriteFile(filepath.Join(root, name), []byte("x"), 0o644); err != nil { + t.Fatalf("setup: %s", err) + } + } + + file := media.File{Path: filepath.Join(root, "a.mkv"), Name: "a"} + if _, err := traverseDir(&file, root, false); err != nil { + t.Errorf("traverseDir() over a clean directory returned error = %v, want nil", err) + } +} + +func TestTraverseDirMatchesPunctuationVariants(t *testing.T) { + root := t.TempDir() + existing := "aktiv und gesund _ Faszientherapie _ Poolkeime _ Stand-up-Paddling.mkv" + path := filepath.Join(root, existing) + if err := os.WriteFile(path, []byte("x"), 0o644); err != nil { + t.Fatalf("write existing file: %s", err) + } + file := media.File{ + Name: "aktiv und gesund - Faszientherapie - Poolkeime - Stand-up-Paddling", + } + matches, err := traverseDir(&file, root, false) + if err != nil { + t.Fatalf("traverseDir() returned error: %s", err) + } + if len(matches) != 1 || matches[0].Path != path { + t.Fatalf("traverseDir() matches = %#v, want original path %q", matches, path) + } +} + +// The .INFO.log files must be world-readable/writable on Unraid so every +// user (nobody:users) can read/write/delete them. lumberjack creates files +// with 0600, so writeSkippedLog must fix the mode explicitly. +func TestWriteSkippedLogInfoPerms(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("unix file permissions do not exist on windows") + } + path := filepath.Join(t.TempDir(), "test.ts") + f := media.File{Path: path} + writeSkippedLog(&f, new(joblog.Data), false) + fi, err := os.Stat(path + ".INFO.log") + if err != nil { + t.Fatalf("stat INFO.log: %s", err) + } + if got := fi.Mode().Perm(); got != 0o666 { + t.Errorf("INFO.log mode = %o, want 666", got) + } +}