Skip to content

Commit d306346

Browse files
Engine 3.0: hearing (CLAP), tempo consensus, trust layer, QC, smart cutdowns, signed passport
The engine now listens to the track instead of inferring everything from DSP numbers. - hearing: LAION CLAP (clap-htsat-unfused) zero-shot genre/mood/instrument + calibrated singer detection (22 reference tracks, 96% balanced accuracy) + 512-d sonic vector - tempo: Essentia multifeature/degara + Percival + sub-frame refined librosa folded into one consensus; fixes the 23 ms frame quantisation (117.5 for a real 120) and octave errors - genre_guard: hearing-based correction on top of context/rhythm rules - trust: per-field evidence, cross-source agreement, review flags, A-D grade - qc: EBU R128 loudness + true peak, clipping, silence/ending, lossy-origin (fake lossless) test - cutdowns: bar/phrase-aligned 15/30/60 s edits with render endpoint - passport: Ed25519-signed metadata attestation (+ public key / verify endpoints) - /api/intelligence/{qc,cutdowns,cutdowns/render,hear,similar,passport/*} - docs/ENGINE_3.md, tests (test_genre_guard.py, test_engine3.py) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 3511641 commit d306346

15 files changed

Lines changed: 1583 additions & 7 deletions

‎backend/app/main.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,12 +97,13 @@ def setup_app():
9797
from app.routes.fresh_analysis import router as fresh_router
9898
from app.routes.export import router as export_router
9999
from app.routes.tools import router as tools_router
100+
from app.routes.intelligence import router as intelligence_router
100101
from app.startup import ensure_admin_user
101102

102103
app = FastAPI(
103104
title="Music Metadata Engine",
104105
description="AI-Powered Audio Analysis",
105-
version="2.1.0"
106+
version="3.0.0"
106107
)
107108

108109
# Global Exception Handler
@@ -160,6 +161,7 @@ def setup_app():
160161

161162
app.include_router(export_router, prefix="/api")
162163
app.include_router(tools_router, prefix="/api")
164+
app.include_router(intelligence_router, prefix="/api")
163165
app.include_router(ddex_router, prefix="/api")
164166
app.include_router(cwr_router, prefix="/api")
165167
app.include_router(ai_proxy_router, prefix="/api")
@@ -178,7 +180,13 @@ def setup_app():
178180
async def startup_event_handler():
179181
logger.info("🚀 Starting application...")
180182
ensure_admin_user()
181-
183+
184+
# Load the CLAP hearing model in the background so the first analysis does not pay for it
185+
# (first start downloads ~600 MB into /data/hf - the persistent volume)
186+
import threading
187+
from app.services import hearing as _hearing
188+
threading.Thread(target=_hearing.available, name="hearing-preload", daemon=True).start()
189+
182190
# Start cleanup task
183191
asyncio.create_task(cleanup_temp_files())
184192

‎backend/app/routes/analysis.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ def get_db():
7171
"acousticScore",
7272
"hasVocals",
7373
"genreCorrected",
74+
# Engine 3.0
75+
"hearing", "trust", "passport", "syncReadiness", "technical", "sonicVector", "tempoAnalysis",
7476
# Misc
7577
"analysisReasoning",
7678
"similar_artists",
@@ -256,6 +258,16 @@ def remaining() -> float:
256258
await ws_manager.send_progress(job_id, job.message, progress=20)
257259

258260
from app.services.fresh_track_analyzer import FreshTrackAnalyzer
261+
from app.services import qc as qc_mod
262+
263+
def _safe_qc(path):
264+
try:
265+
return qc_mod.analyze_file(path)
266+
except Exception as qe: # noqa: BLE001 - QC is an extra, never fail the analysis for it
267+
logger.warning(f"Job {job_id}: QC failed: {qe}")
268+
return None
269+
270+
qc_task = asyncio.get_running_loop().run_in_executor(None, _safe_qc, file_path)
259271
analyzer = FreshTrackAnalyzer()
260272
analyzer_budget = max(15, int(remaining() - 2.0))
261273

@@ -314,6 +326,22 @@ def remaining() -> float:
314326
except Exception as e:
315327
logger.warning(f"Could not read existing file tags: {e}")
316328

329+
# Technical QC (ran in parallel) + signed passport
330+
try:
331+
qc_report = await asyncio.wait_for(qc_task, timeout=120)
332+
except Exception as qe: # noqa: BLE001
333+
logger.warning(f"Job {job_id}: QC unavailable: {qe}")
334+
qc_report = None
335+
if qc_report:
336+
metadata["syncReadiness"] = qc_report.pop("syncReadiness", None)
337+
metadata["technical"] = qc_report
338+
try:
339+
from app.services import passport as passport_mod
340+
metadata["passport"] = passport_mod.build_passport(
341+
metadata, metadata.get("trust"), (metadata.get("hearing") or {}).get("model"))
342+
except Exception as pe: # noqa: BLE001
343+
logger.warning(f"Job {job_id}: passport not issued: {pe}")
344+
317345
job.message = "Sanitizing and validating metadata..."
318346
db.commit()
319347
await ws_manager.send_progress(job_id, job.message, progress=85)

‎backend/app/routes/intelligence.py‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
"""Engine 3.0 endpoints: QC, smart cutdowns, hearing, sonic similarity, signed passports."""
2+
from __future__ import annotations
3+
4+
import asyncio
5+
import os
6+
import shutil
7+
import tempfile
8+
from contextlib import contextmanager
9+
from typing import Any, Dict, List, Optional
10+
11+
import numpy as np
12+
from fastapi import APIRouter, Body, Depends, File, Form, HTTPException, UploadFile
13+
from fastapi.responses import FileResponse
14+
from starlette.background import BackgroundTask
15+
16+
from app.dependencies import get_user_and_check_quota
17+
from app.services import cutdowns, hearing, passport, qc
18+
19+
router = APIRouter(prefix="/intelligence", tags=["intelligence"])
20+
ALLOWED = {".mp3", ".wav", ".flac", ".m4a", ".aac", ".ogg"}
21+
MAX_BYTES = 200 * 1024 * 1024
22+
23+
24+
@contextmanager
25+
def _spooled(upload: UploadFile):
26+
ext = os.path.splitext(upload.filename or "")[1].lower()
27+
if ext not in ALLOWED:
28+
raise HTTPException(400, f"Unsupported file type: {ext or 'none'}")
29+
d = tempfile.mkdtemp(prefix="intel_")
30+
path = os.path.join(d, "audio" + ext)
31+
try:
32+
written = 0
33+
with open(path, "wb") as f:
34+
while chunk := upload.file.read(1024 * 1024):
35+
written += len(chunk)
36+
if written > MAX_BYTES:
37+
raise HTTPException(413, "File too large (limit 200 MB)")
38+
f.write(chunk)
39+
yield path
40+
finally:
41+
shutil.rmtree(d, ignore_errors=True)
42+
43+
44+
async def _off_loop(fn, *a):
45+
return await asyncio.get_running_loop().run_in_executor(None, fn, *a)
46+
47+
48+
@router.post("/qc")
49+
async def technical_qc(file: UploadFile = File(...), user=Depends(get_user_and_check_quota)) -> Dict[str, Any]:
50+
"""Loudness (LUFS / true peak), clipping, lossy-origin test, head/tail silence, ending style, sync-readiness score."""
51+
with _spooled(file) as path:
52+
return await _off_loop(qc.analyze_file, path)
53+
54+
55+
@router.post("/cutdowns")
56+
async def plan_cutdowns(file: UploadFile = File(...), targets: str = Form("15,30,60"), user=Depends(get_user_and_check_quota)) -> Dict[str, Any]:
57+
"""Bar/phrase-aligned 15/30/60 s edit proposals (an 'intro' cut and a 'hook' cut per length)."""
58+
try:
59+
wanted = sorted({int(t) for t in targets.split(",") if t.strip()})
60+
except ValueError:
61+
raise HTTPException(400, "targets must be comma-separated seconds, e.g. 15,30,60")
62+
if not wanted or any(t < 5 or t > 180 for t in wanted):
63+
raise HTTPException(400, "each target must be 5-180 seconds")
64+
with _spooled(file) as path:
65+
return await _off_loop(cutdowns.plan_file, path, wanted)
66+
67+
68+
@router.post("/cutdowns/render")
69+
async def render_cutdown(file: UploadFile = File(...), start: float = Form(...), end: float = Form(...), fade_out: float = Form(1.0),
70+
format: str = Form("mp3"), user=Depends(get_user_and_check_quota)):
71+
"""Render one proposed cut (start/end from /cutdowns) with a 20 ms fade-in and a musical fade-out."""
72+
if not (0 <= start < end) or end - start > 300 or format not in ("mp3", "wav"):
73+
raise HTTPException(400, "invalid cut")
74+
ext = os.path.splitext(file.filename or "")[1].lower()
75+
if ext not in ALLOWED:
76+
raise HTTPException(400, f"Unsupported file type: {ext or 'none'}")
77+
d = tempfile.mkdtemp(prefix="cut_")
78+
src, out = os.path.join(d, "src" + ext), os.path.join(d, f"cutdown.{format}")
79+
with open(src, "wb") as f:
80+
shutil.copyfileobj(file.file, f)
81+
try:
82+
await _off_loop(cutdowns.render, src, start, end, max(0.0, min(fade_out, 8.0)), out)
83+
except Exception:
84+
shutil.rmtree(d, ignore_errors=True)
85+
raise HTTPException(500, "render failed")
86+
return FileResponse(out, media_type="audio/mpeg" if format == "mp3" else "audio/wav", filename=f"cutdown_{int(end - start)}s.{format}",
87+
background=BackgroundTask(shutil.rmtree, d, True))
88+
89+
90+
@router.post("/hear")
91+
async def hear(file: UploadFile = File(...), vector: bool = Form(False), user=Depends(get_user_and_check_quota)) -> Dict[str, Any]:
92+
"""What the audio model hears: genre / mood / instruments / singer verdict (+ optional 512-d sonic vector)."""
93+
with _spooled(file) as path:
94+
res = await _off_loop(hearing.listen, path)
95+
if not res:
96+
raise HTTPException(503, "Hearing model unavailable")
97+
if not vector:
98+
res.pop("sonicVector", None)
99+
return res
100+
101+
102+
@router.post("/similar")
103+
async def similar(payload: Dict[str, Any] = Body(...)) -> Dict[str, Any]:
104+
"""Rank candidates by audio similarity. Body: {"vector": b64, "candidates": [{"id": ..., "vector": b64}], "top": 10}.
105+
Vectors come from the analysis result (``sonicVector``). Cosine >= 0.97 usually means the same recording / another master."""
106+
try:
107+
q = hearing.unpack_vector(payload["vector"])
108+
cands = payload["candidates"]
109+
except (KeyError, ValueError, TypeError):
110+
raise HTTPException(400, "vector and candidates are required")
111+
if len(cands) > 20000:
112+
raise HTTPException(413, "too many candidates")
113+
scored: List[Dict[str, Any]] = []
114+
for c in cands:
115+
try:
116+
s = hearing.cosine(q, hearing.unpack_vector(c["vector"]))
117+
except (KeyError, ValueError, TypeError):
118+
continue
119+
scored.append({"id": c.get("id"), "similarity": round(s, 4), "likelyDuplicate": s >= 0.97})
120+
scored.sort(key=lambda x: -x["similarity"])
121+
return {"results": scored[: int(payload.get("top", 10))]}
122+
123+
124+
@router.get("/passport/public-key")
125+
async def passport_public_key() -> Dict[str, Any]:
126+
return {"alg": "Ed25519", "publicKey": passport.public_key_b64(), "keyId": passport.key_id(),
127+
"engine": passport.ENGINE_NAME, "version": passport.ENGINE_VERSION}
128+
129+
130+
@router.post("/passport/verify")
131+
async def passport_verify(payload: Dict[str, Any] = Body(...)) -> Dict[str, Any]:
132+
"""Body: the ``passport`` object from an analysis result (optional ``publicKey`` to verify against another engine's key)."""
133+
p = payload.get("passport", payload)
134+
if not isinstance(p, dict) or "payload" not in p:
135+
raise HTTPException(400, "passport object expected")
136+
return passport.verify_passport(p, payload.get("publicKey"))

‎backend/app/services/cutdowns.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
"""Smart cutdowns: 15 / 30 / 60 s edits cut on bar and phrase boundaries with a musical ending.
2+
3+
Sync buyers always ask for cutdowns. Naive trimming (first N seconds) stops mid-phrase on a random beat. This planner
4+
works on the beat grid: it groups beats into bars, finds the phrase structure (4/8-bar blocks counted from where the
5+
track actually starts), and for each target length proposes
6+
* ``intro`` - from the start of the music, ending on the bar closest to the target with a one-bar fade;
7+
* ``hook`` - the most energetic phrase-aligned window of that length, which ends where the energy naturally drops.
8+
Pure planning (``plan_cutdowns``) is unit-tested with synthetic grids; ``beat_grid`` and ``render`` use librosa/ffmpeg.
9+
"""
10+
from __future__ import annotations
11+
12+
import subprocess
13+
from typing import Any, Dict, List, Optional, Sequence
14+
15+
import numpy as np
16+
17+
18+
def _bars(beat_times: np.ndarray, beat_energy: np.ndarray, per_bar: int):
19+
n = (len(beat_times) // per_bar) * per_bar
20+
starts = beat_times[:n:per_bar]
21+
energy = beat_energy[:n].reshape(-1, per_bar).mean(axis=1)
22+
return starts, energy
23+
24+
25+
def plan_cutdowns(beat_times: Sequence[float], beat_energy: Sequence[float], duration: float,
26+
targets: Sequence[int] = (15, 30, 60), per_bar: int = 4) -> Dict[str, Any]:
27+
bt, be = np.asarray(beat_times, dtype=float), np.asarray(beat_energy, dtype=float)
28+
if bt.size < per_bar * 4 or duration <= 0:
29+
return {"ok": False, "reason": "not enough beats to plan cutdowns", "cutdowns": []}
30+
beat_dur = float(np.median(np.diff(bt)))
31+
bar_dur = beat_dur * per_bar
32+
starts, energy = _bars(bt, be, per_bar)
33+
if starts.size < 4:
34+
return {"ok": False, "reason": "too short", "cutdowns": []}
35+
# end time of bar i = start of bar i+1; the last bar ends at the end of the audio
36+
ends = np.append(starts[1:], min(duration, starts[-1] + bar_dur))
37+
med = float(np.median(energy)) or 1e-9
38+
body = int(np.argmax(energy > 0.5 * med)) # first bar with real music
39+
p75 = float(np.percentile(energy, 90)) or 1e-9 # reference level of the loud sections
40+
out: List[Dict[str, Any]] = []
41+
for target in targets:
42+
n = max(2, int(round(target / bar_dur)))
43+
if n >= starts.size:
44+
continue
45+
# option 1: intro cut
46+
s0 = body
47+
e0 = min(starts.size - 1, s0 + n - 1)
48+
intro = _cut("intro", target, starts, ends, energy, s0, e0, bar_dur, body, p75, duration)
49+
# option 2: best hook window
50+
best = None
51+
for s in range(0, starts.size - n + 1):
52+
e = s + n - 1
53+
c = _cut("hook", target, starts, ends, energy, s, e, bar_dur, body, p75, duration)
54+
if best is None or c["score"] > best["score"]:
55+
best = c
56+
out.append({"target": target, "options": [x for x in (intro, best) if x]})
57+
return {"ok": True, "bar_seconds": round(bar_dur, 3), "bpm": round(60 / beat_dur, 1), "first_music_bar": body, "cutdowns": out}
58+
59+
60+
def _cut(kind: str, target: int, starts, ends, energy, s: int, e: int, bar_dur: float, body: int, p75: float, duration: float) -> Dict[str, Any]:
61+
start, end = float(starts[s]), float(ends[e])
62+
dur = end - start
63+
phrase = ((s - body) % 8 == 0) * 1.0 + (((s - body) % 4 == 0) * 0.5) # aligned to 8/4-bar phrases
64+
prev = energy[max(0, s - 2):s]
65+
if prev.size and energy[s] > 1.2 * prev.mean(): # the section itself starts here (energy jump)
66+
phrase = max(phrase, 1.0)
67+
hook = float(np.clip(energy[s:e + 1].mean() / p75, 0, 1.2))
68+
drop_at_end = e + 1 < energy.size and energy[e] > 0 and energy[e + 1] < 0.9 * energy[e]
69+
natural = bool(e >= energy.size - 1 or drop_at_end)
70+
length_fit = 1.0 - min(1.0, abs(dur - target) / target)
71+
score = round(0.45 * hook + 0.25 * phrase + 0.2 * length_fit + 0.1 * (1.0 if natural else 0.0), 3)
72+
fade = 0.0 if (natural and e >= energy.size - 1) else round(min(bar_dur, 3.0), 2)
73+
return {"kind": kind, "start": round(start, 3), "end": round(end, 3), "duration": round(dur, 3), "bars": e - s + 1,
74+
"fadeOut": fade, "naturalEnd": natural, "phraseAligned": phrase >= 1.0, "score": score,
75+
"reason": ("starts where the music starts, ends on a bar line" if kind == "intro"
76+
else "most energetic phrase-aligned window" + (", ends where energy drops" if natural else ""))}
77+
78+
79+
def align_beats_to_tempo(beats: Sequence[float], bpm: float) -> np.ndarray:
80+
"""Make the beat grid agree with the consensus tempo: a tracker locked to half-time gets midpoints inserted,
81+
one locked to double-time keeps every second beat."""
82+
b = np.asarray(beats, dtype=float)
83+
if b.size < 4 or not bpm:
84+
return b
85+
ratio = float(np.median(np.diff(b))) / (60.0 / bpm)
86+
if 1.85 <= ratio <= 2.15: # beats are twice as far apart as the tempo says
87+
mid = (b[:-1] + b[1:]) / 2
88+
return np.sort(np.concatenate([b, mid]))
89+
if 0.45 <= ratio <= 0.55: # beats are twice as dense
90+
return b[::2]
91+
return b
92+
93+
94+
# ───────────────────────────── audio side ─────────────────────────────
95+
def beat_grid(path: str, sr: int = 22050) -> Dict[str, Any]:
96+
import librosa
97+
98+
from . import tempo as tempo_mod
99+
100+
y, _ = librosa.load(path, sr=sr, mono=True)
101+
duration = y.size / sr
102+
est = tempo_mod.estimate_file(path)
103+
beat_times = np.asarray(est.get("beats") or [])
104+
if beat_times.size >= 16 and est.get("bpm"):
105+
beat_times = align_beats_to_tempo(beat_times, est["bpm"])
106+
else:
107+
_, beats = librosa.beat.beat_track(y=y, sr=sr)
108+
beat_times = librosa.frames_to_time(beats, sr=sr)
109+
rms = librosa.feature.rms(y=y, hop_length=512)[0]
110+
frame_t = librosa.frames_to_time(np.arange(rms.size), sr=sr, hop_length=512)
111+
idx = np.searchsorted(frame_t, np.append(beat_times, duration))
112+
energy = np.array([rms[idx[i]:max(idx[i] + 1, idx[i + 1])].mean() for i in range(len(beat_times))]) if len(beat_times) else np.array([])
113+
return {"beat_times": beat_times, "beat_energy": energy, "duration": duration, "tempo": est}
114+
115+
116+
def plan_file(path: str, targets: Sequence[int] = (15, 30, 60)) -> Dict[str, Any]:
117+
g = beat_grid(path)
118+
plan = plan_cutdowns(g["beat_times"], g["beat_energy"], g["duration"], targets)
119+
plan["duration_s"] = round(g["duration"], 2)
120+
t = g.get("tempo") or {}
121+
plan["tempo"] = {k: t.get(k) for k in ("bpm", "agreement", "alternatives", "octaveAmbiguous")}
122+
return plan
123+
124+
125+
def render(path: str, start: float, end: float, fade_out: float, out_path: str, bitrate: str = "320k") -> str:
126+
dur = end - start
127+
af = f"afade=t=in:st=0:d=0.02,afade=t=out:st={max(0.0, dur - fade_out):.3f}:d={fade_out:.3f}" if fade_out > 0 else "afade=t=in:st=0:d=0.02"
128+
codec = ["-c:a", "libmp3lame", "-b:a", bitrate] if out_path.lower().endswith(".mp3") else []
129+
subprocess.run(["ffmpeg", "-v", "error", "-y", "-ss", f"{start:.3f}", "-t", f"{dur:.3f}", "-i", path, "-af", af, *codec, out_path],
130+
check=True, capture_output=True, timeout=300)
131+
return out_path

0 commit comments

Comments
 (0)