diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 7f82269..e3854db 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -7,7 +7,7 @@ permissions: contents: read jobs: tests: - if: ${{ !contains(github.event.head_commit.message, '[source-upload]') }} + if: ${{ !contains(github.event.head_commit.message, '[source-upload]') && !contains(github.event.head_commit.message, '[prepare]') }} strategy: matrix: os: [ubuntu-latest, windows-latest] @@ -19,7 +19,7 @@ jobs: with: python-version: ${{ matrix.python }} cache: pip - - run: python -m pip install -r requirements-dev.txt + - run: python -m pip install -q -r requirements-dev.txt - run: python -m pytest -q --junitxml=test-results/results.xml - uses: actions/upload-artifact@v4 if: always() @@ -27,22 +27,24 @@ jobs: name: tests-${{ matrix.os }}-${{ matrix.python }} path: test-results/results.xml browser: - if: ${{ !contains(github.event.head_commit.message, '[source-upload]') }} + if: ${{ !contains(github.event.head_commit.message, '[source-upload]') && !contains(github.event.head_commit.message, '[prepare]') }} runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.12' - - run: python -m pip install -r requirements-dev.txt + - run: python -m pip install -q -r requirements-dev.txt - run: python -m playwright install --with-deps chromium - run: python scripts/ui_smoke.py - run: python scripts/ui_studio_smoke.py - run: python scripts/ui_science_smoke.py - run: python scripts/ui_graphify_smoke.py - run: python scripts/ui_polemic_smoke.py + - run: python scripts/ui_voices_smoke.py - uses: actions/upload-artifact@v4 if: always() with: - name: browser-screenshots - path: test-results/ui/ + name: browser-workflows + path: test-results/ diff --git a/.github/workflows/voice-manifest.yml b/.github/workflows/voice-manifest.yml new file mode 100644 index 0000000..8d3ac3b --- /dev/null +++ b/.github/workflows/voice-manifest.yml @@ -0,0 +1,26 @@ +name: Refresh verified voice release manifest +on: + push: + branches: ['fix/jev-voice-graphs-v0.5'] +permissions: + contents: write +jobs: + manifest: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Refresh code-only manifest; never change application source or workflows + run: | + python scripts/prepare_assets.py --manifest + git add publish-manifest.json + git diff --cached --check + if ! git diff --cached --quiet; then + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git commit -m 'Refresh source-only publishing manifest' + git push origin HEAD:fix/jev-voice-graphs-v0.5 + fi diff --git a/.github/workflows/voices-release.yml b/.github/workflows/voices-release.yml new file mode 100644 index 0000000..222fca9 --- /dev/null +++ b/.github/workflows/voices-release.yml @@ -0,0 +1,88 @@ +name: GraphPaper 0.5 hosted Windows release +on: + push: + branches: [main] + paths: ['graphpaper/__init__.py', '.github/workflows/voices-release.yml'] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: graphpaper-0.5-release + cancel-in-progress: false +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: test "$(python -c 'from graphpaper import __version__; print(__version__)')" = '0.5.0' + - run: python -m pip install -q -r requirements-dev.txt + - run: python -m pytest -q --junitxml=test-results/linux.xml + - run: python scripts/docs_tool.py check + - run: python -m playwright install --with-deps chromium + - run: python scripts/ui_smoke.py + - run: python scripts/ui_studio_smoke.py + - run: python scripts/ui_science_smoke.py + - run: python scripts/ui_graphify_smoke.py + - run: python scripts/ui_polemic_smoke.py + - run: python scripts/ui_voices_smoke.py + - uses: actions/upload-artifact@v4 + if: always() + with: + name: voices-linux-validation + path: test-results/ + windows: + needs: validate + runs-on: windows-latest + timeout-minutes: 30 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -q -r requirements-dev.txt + - run: python -m pytest -q --junitxml=test-results/windows.xml + - name: Build with public Graphify and official Codex + shell: pwsh + env: + GITHUB_TOKEN: ${{ github.token }} + run: ./scripts/build_release.ps1 + - run: python scripts/native_smoke.py --science --graphify --polemic --voices --executable dist/GraphPaper/GraphPaper.exe --out test-results/native-package + - uses: actions/upload-artifact@v4 + with: + name: GraphPaper-v0.5.0-Windows-x64 + path: | + dist/GraphPaper-v0.5.0-Windows-x64.zip + dist/SHA256SUMS.txt + - name: Publish only the successfully tested artifact + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + $ErrorActionPreference = 'Stop' + $tag = 'v0.5.0' + $package = 'GraphPaper-v0.5.0-Windows-x64.zip' + $expected = ((Get-Content dist/SHA256SUMS.txt).Split(' ')[0]).Trim().ToLower() + $actual = (Get-FileHash "dist/$package" -Algorithm SHA256).Hash.ToLower() + if ($actual -ne $expected) { throw 'Archive checksum mismatch.' } + gh release view $tag *> $null + if ($LASTEXITCODE -eq 0) { throw 'Release already exists; refusing to overwrite it.' } + gh release create $tag "dist/$package" dist/SHA256SUMS.txt --target $env:GITHUB_SHA --title 'GraphPaper 0.5.0 - Reusable voice graphs' --notes-file docs/RELEASE-0.5.0.md --draft + if ($LASTEXITCODE -ne 0) { throw 'Release upload failed.' } + $releases = gh api "repos/$env:GITHUB_REPOSITORY/releases?per_page=100" | ConvertFrom-Json + $release = $releases | Where-Object { $_.tag_name -eq $tag } + if (-not $release) { throw 'The uploaded draft release could not be found.' } + $asset = $release.assets | Where-Object { $_.name -eq $package } + if ($asset.digest -ne "sha256:$actual") { throw 'Uploaded artifact does not match the tested ZIP.' } + gh release edit $tag --draft=false --latest + if ($LASTEXITCODE -ne 0) { throw 'Release publication failed.' } + - uses: actions/upload-artifact@v4 + if: always() + with: + name: voices-windows-validation + path: test-results/ diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d466db3..14c0d8d 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -21,10 +21,11 @@ jobs: - run: python scripts/ui_science_smoke.py - run: python scripts/ui_graphify_smoke.py - run: python scripts/ui_polemic_smoke.py + - run: python scripts/ui_voices_smoke.py windows: needs: quality runs-on: windows-latest - timeout-minutes: 25 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -37,8 +38,8 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: ./scripts/build_release.ps1 - - name: Exercise the actual Windows window, native dialogs and save-on-close - run: python scripts/native_smoke.py --science --graphify --polemic --executable dist/GraphPaper/GraphPaper.exe --out test-results/native-package + - name: Exercise the actual Windows window, voices, native dialogs and save-on-close + run: python scripts/native_smoke.py --science --graphify --polemic --voices --executable dist/GraphPaper/GraphPaper.exe --out test-results/native-package - uses: actions/upload-artifact@v4 with: name: GraphPaper-Windows-x64 @@ -54,7 +55,7 @@ jobs: $version = python -c "from graphpaper import __version__; print(__version__)" if ($env:GITHUB_REF_NAME -ne "v$version") { throw 'Tag does not match application version.' } gh release view $env:GITHUB_REF_NAME *> $null - if ($LASTEXITCODE -ne 0) { gh release create $env:GITHUB_REF_NAME --verify-tag --title "GraphPaper $env:GITHUB_REF_NAME" --notes-file docs/RELEASE-0.4.0.md } + if ($LASTEXITCODE -ne 0) { gh release create $env:GITHUB_REF_NAME --verify-tag --title "GraphPaper $env:GITHUB_REF_NAME" --notes-file "docs/RELEASE-$version.md" } $packageName = "GraphPaper-v$version-Windows-x64.zip" $release = gh release view $env:GITHUB_REF_NAME --json assets | ConvertFrom-Json if ($release.assets.name -contains $packageName) { diff --git a/CHANGELOG.md b/CHANGELOG.md index b497b12..6ae5a52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.5.0 + +- Bound and automatically batch JEV requests by actual serialized UTF-8 size; remove voice-training prose from decision payloads. +- Add named, reusable voice graphs with independent training, graph editing, import/export and explicit application across projects. +- Compose namespaced style/evidence overlays without altering the research graph; current author instructions override saved style preferences. +- Keep unrepresentable optional decisions honestly unscored instead of aborting completed writing. +- Implement and validate the update using GitHub-hosted runners only. + + ## 0.4.0 - Add Polemic and a reversible writing-mode switch for existing projects. diff --git a/README.md b/README.md index c475b95..26d977b 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ Science adds a **Research** desk for scholarly search, screening and evidence ap The portable build needs Microsoft Edge WebView2 and is currently unsigned. Existing projects live separately from the executable. [Windows setup and upgrades →](docs/WINDOWS.md) +**New: reusable voice graphs.** Save a voice once, select it in any project, and keep its training pieces separate. JEV receives compact graph context with automatic request sizing—not the article pile. [Voice graphs and author control](docs/VOICE-GRAPHS.md) + ## What makes it useful - **Explore before committing.** Inspect source-linked concepts and relationships, follow paths, pin ideas and compare competing angles. Edit a suggested thesis or write your own. diff --git a/docs/README.md b/docs/README.md index 7719e38..c94bb22 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,7 +2,7 @@ **A studio for connected thought.** GraphPaper brings source material, graph exploration and a writing desk into one local-first Windows application. -These guides describe **GraphPaper 0.4.0**. The [release page](https://github.com/AronAxe/GraphPaper/releases/latest) provides the current Windows download; [CHANGELOG](../CHANGELOG.md) records version history. The same guides are published in the [GitHub wiki](https://github.com/AronAxe/GraphPaper/wiki). +These guides describe **GraphPaper 0.5.0**. The [release page](https://github.com/AronAxe/GraphPaper/releases/latest) provides the current Windows download; [CHANGELOG](../CHANGELOG.md) records version history. The same guides are published in the [GitHub wiki](https://github.com/AronAxe/GraphPaper/wiki). ## Start here @@ -46,3 +46,7 @@ These guides describe **GraphPaper 0.4.0**. The [release page](https://github.co ## Documentation maintenance These Markdown files are the maintained source. [The documentation tooling](DEVELOPMENT.md#maintaining-the-documentation) validates links and renders the wiki with consistent navigation; screenshots are shared rather than copied into every page. Historical release notes stay available, but they are not the current setup guide. + +## Reusable voices and bounded JEV decisions + +[Save a voice graph once and reuse it across projects](VOICE-GRAPHS.md). Training pieces are separate from article evidence; current author instructions override saved preferences. diff --git a/docs/RELEASE-0.5.0.md b/docs/RELEASE-0.5.0.md new file mode 100644 index 0000000..fed2e93 --- /dev/null +++ b/docs/RELEASE-0.5.0.md @@ -0,0 +1,14 @@ +# GraphPaper 0.5.0 — reusable voice graphs and bounded JEV + +- Fix oversized JEV requests with exact UTF-8 payload preflight, automatic question/candidate batching and compact voice-graph context. Raw voice-training articles are not sent to JEV. The existing 60 KB application budget is not simply raised. +- Save and name a voice once, then apply its compact graph in any project without another model call or re-uploading the articles. Legacy learned profiles can be saved without retraining. +- Separate voice training pieces from project evidence, with upload, pasted text and article-URL inputs in the voice library. +- Edit the mini graph's habits and weights, export/import it, and view a namespaced style/evidence overlay without modifying the research graph. +- Keep author control: power language is not filtered, and current project instructions override saved style habits. Applying or updating a voice does not choose the author's opinion. +- Preserve existing profiles, source documents, graphs, settings and manuscripts. Failed/oversized optional decisions cannot destroy completed writing or fabricate a passing score. + +Download the Windows ZIP, extract it fully, and run `GraphPaper.exe` with `_internal` alongside it. The package is unsigned and retains the public Graphify runtime, Codex OAuth, reasoning controls, Polemic and Science/APA features. + +For an existing voice, open **Your writing voice → Save as reusable voice**. In another project, open **My voices → Use in this project**. The [voice-graph guide](https://github.com/AronAxe/GraphPaper/blob/main/docs/VOICE-GRAPHS.md) explains the graph editor, training library, overlay, precedence and request sizing. + +Development, regression testing and Windows packaging for this update run on GitHub-hosted runners—not the user's computer. The release workflow and attached reports record the actual completed checks. Synthetic provider responses test routing and size handling; they are not a claim of live JEV quality evaluation. diff --git a/docs/VOICE-GRAPHS.md b/docs/VOICE-GRAPHS.md new file mode 100644 index 0000000..cd8a9a3 --- /dev/null +++ b/docs/VOICE-GRAPHS.md @@ -0,0 +1,39 @@ +# Reusable voice graphs and compact JEV decisions + +Create a voice once, name it, and apply it in any project. You do not need to upload or analyze the same writing repeatedly. + +## Save the voice you already made + +Open **Your writing voice**, then **Save as reusable voice**. The existing profile becomes a small node-and-edge graph without a new model call. Its training pieces are copied into the separate voice library; original project files are not deleted. Legacy voice samples are hidden from the evidence-card view, not mistaken for research. + +In **My voices**, select the saved voice and choose **Use in this project**. Only the compact profile/graph is applied. No training articles are copied into the new project and no inference is run. Each project retains a snapshot, so changing or deleting the library entry does not silently alter other manuscripts. Apply the updated voice explicitly when desired. + +## Build and edit a voice + +Choose **New voice** in the library. Add writing by file upload, pasted text or an article URL under **Training pieces**. **Learn / rebuild graph** analyzes the samples once, with the configured extraction model. A learned graph captures rhythm, diction, structure, humor, rhetorical habits, imagery and force. Training text is stored separately in the local database and is sent for learning only; it is not put into ordinary JEV decisions. + +You can instead enter style directions and convert them to nodes without a model call. Click a graph node to edit its instruction and importance. Add or remove habits, give the voice a name and save it. A saved voice can be exported/imported as a small JSON graph without the training articles or credentials. + +**The voice graph is not a censor.** Strong language, profanity, ridicule, indignation and an uncompromising conclusion are valid author choices. A saved voice is a reusable default, not a rulebook: the current article brief and explicit instructions take precedence. Factual provenance and rhetorical strength are different things. + +## Graph overlay + +**View graph overlay** composes the saved style graph and a bounded view of the project graph. IDs are namespaced and the evidence, style and overlay layers stay distinguishable. Style edges describe how to express material, not whether an empirical claim is true. The stored evidence graph is not overwritten. + +Graphify supports graph merging with namespaced IDs. This integration uses the compatible NetworkX node-link representation and graph composition already available in GraphPaper; it does not need another Graphify extraction or claim that lexical overlap proves semantic equivalence. A style directive can target a kind of project node, such as a claim, without becoming a source for that claim. + +## Why JEV no longer receives the whole heap + +JEV receives the compact style-graph view, the authorial contract and the material relevant to its typed questions. Raw training pieces are excluded. Candidate-specific decisions are automatically divided and combined according to the **serialized UTF-8 payload size**, with original question keys and candidate IDs retained. Questions can also be split independently. Repeated long strings can use lossless references instead of being duplicated. + +The 60 KB cap is a GraphPaper transport budget, not a statement about a universal JEV model limit. The complete outgoing model/state/questions envelope is checked, including Unicode. Batches are planned before spending calls; receipts record their actual size and count. This does not raise the limit or quietly remove an author's thesis. + +If a single decision genuinely cannot fit without discarding essential content, that optional decision stays **unscored**, with a job message. It does not abort the writing or pretend to approve an automatic revision. No missing result is converted into a made-up probability. API errors and invalid typed answers remain real errors, separate from local size planning. + +## Storage and compatibility + +The voice library is an additive table in the existing local SQLite database. Profiles, graph copies, legacy samples, project graphs and existing settings remain readable. Saved voices are not automatically synchronized to a remote service. Exporting a project includes its applied compact voice snapshot, not the whole global training library. Back up the closed application's local data directory for the complete library. + +The update is implemented and built on GitHub-hosted runners. It does not access or modify the user's computer during development. + +[Author voice and prose tools](VOICE-AND-PROSE.md) · [Polemic](POLEMIC.md) · [Graph and JEV](GRAPH-AND-JEV.md) diff --git a/docs/wiki-pages.json b/docs/wiki-pages.json index 6cfbb13..3df7cbe 100644 --- a/docs/wiki-pages.json +++ b/docs/wiki-pages.json @@ -1,6 +1,6 @@ { "repository": "AronAxe/GraphPaper", - "docs_version": "0.4.0", + "docs_version": "0.5.0", "pages": [ { "title": "Home", @@ -96,6 +96,11 @@ "title": "Releases-and-Packaging", "source": "docs/PUBLISHING.md", "section": "Reference" + }, + { + "title": "Reusable-Voice-Graphs", + "source": "docs/VOICE-GRAPHS.md", + "section": "Studio guides" } ] } diff --git a/graphpaper/__init__.py b/graphpaper/__init__.py index 6f0d9f5..f56785a 100644 --- a/graphpaper/__init__.py +++ b/graphpaper/__init__.py @@ -1,2 +1,2 @@ """GraphPaper: an evidence-aware, graph-native writing studio.""" -__version__ = "0.4.0" +__version__ = "0.5.0" diff --git a/graphpaper/desktop.py b/graphpaper/desktop.py index a183715..41e65c0 100644 --- a/graphpaper/desktop.py +++ b/graphpaper/desktop.py @@ -86,11 +86,21 @@ def _ask_ui_to_close(self): def save_export(self, project_id: str, kind: str): try: import webview - if kind not in {"md", "docx", "html", "json", "graph", "bib", "ris", "evidence", "search-log", "research-package", "submission"}: + if kind not in {"md", "docx", "html", "json", "graph", "bib", "ris", "evidence", "search-log", "research-package", "submission", "voice"}: return {"error": "Unknown export type"} - p = self._store.get(project_id) - data, _, ext = export(p, kind) - stem = re.sub(r'[^\w\s.-]', '', p.title).strip()[:80] or "GraphPaper" + if kind == 'voice': + from .voice_library import VoiceLibrary + import json + v,_ = VoiceLibrary(self._store).get(project_id) + profile=v.profile.model_copy(deep=True) + profile.sample_ids=[];profile.sample_hash='';profile.library_id='';profile.library_version=0 + data=json.dumps({'format':'graphpaper-voice-1','profile':profile.model_dump()},ensure_ascii=False,indent=2).encode() + ext='.json';title=v.profile.name + else: + p = self._store.get(project_id) + data, _, ext = export(p, kind) + title=p.title + stem = re.sub(r'[^\w\s.-]', '', title).strip()[:80] or "GraphPaper" result = self._window.create_file_dialog( webview.FileDialog.SAVE, save_filename=stem + ext, file_types=(f"{kind.upper()} files (*{ext})", "All files (*.*)"), diff --git a/graphpaper/editorial.py b/graphpaper/editorial.py index b80be93..b1b9beb 100644 --- a/graphpaper/editorial.py +++ b/graphpaper/editorial.py @@ -43,7 +43,7 @@ def contract(project): detail=('Lean evidence presentation: establish indispensable factual premises and directly relevant limits, without an ornamental limitations tour.' if rigor<35 else 'Focused evidence presentation: explain the basis of material claims and limitations that actually affect this argument.' if rigor<75 else 'Detailed evidence presentation: trace important factual claims to sources and address concrete contrary evidence. Greater evidence detail is NOT a request for ideological balance, softened moral judgments or repetitive caveats.') - return {'policy_version':'authorial-intent-v1','mode':project.mode,'purpose':purpose, + return {'policy_version':'authorial-intent-v2', 'voice_precedence':'The current author brief overrides saved voice habits. Power language is allowed; saved preferences do not cap force, vocabulary or profanity.','mode':project.mode,'purpose':purpose, 'thesis':brief.thesis.strip() or brief.direction,'direction':brief.direction, 'stance_policy':stance,'rhetorical_force':force,'delivery':delivery, 'evidence_detail':rigor,'evidence_instruction':detail, @@ -52,9 +52,14 @@ def contract(project): def context(project,compact=False,include_selection=True): - voice=voice_context(project,1800 if compact else 6500) - if compact and voice.get('instructions'): - voice={**voice,'instructions':voice['instructions'][:2600], 'scope':'Compact voice context for candidate evaluation; full saved profile is retained.'} + if compact: + from .style_graph import voice_decision + profile=project.voice_profile + if not profile.library_id and profile.sample_hash and profile.sample_hash!=voice_fingerprint(project): + profile=profile.model_copy(update={'instructions':'','graph':{}}) + voice=voice_decision(profile) + else: + voice=voice_context(project,6500) selected=next((a for a in project.angles if a.id==project.selected_angle),None) if include_selection else None return {'editorial_contract':contract(project),'author_voice':voice, 'selected_argument':{'title':selected.title,'thesis':selected.thesis,'hook':selected.hook} if selected else None} @@ -65,7 +70,7 @@ def system(project,stage): def fingerprint(project): - value={'mode':project.mode,'brief':project.brief.model_dump(),'voice':project.voice_profile.model_dump(),'voice_sources':voice_fingerprint(project)} + value={'mode':project.mode,'brief':project.brief.model_dump(),'voice':project.voice_profile.model_dump(),'voice_sources':('saved:'+project.voice_profile.library_id if project.voice_profile.library_id else voice_fingerprint(project))} return hashlib.sha256(json.dumps(value,ensure_ascii=False,sort_keys=True).encode()).hexdigest() diff --git a/graphpaper/jev_budget.py b/graphpaper/jev_budget.py new file mode 100644 index 0000000..b1a1282 --- /dev/null +++ b/graphpaper/jev_budget.py @@ -0,0 +1,166 @@ +"""Automatic, exact-size JEV batching. No raw voice samples and no text clipping. + +60 KB is this application's transport budget, not a claimed JEV model limit. +A decision that cannot be represented faithfully is left unscored; it cannot +abort completed writing or masquerade as a verified automatic revision gate. +""" +from __future__ import annotations +import copy +import json +import re +from dataclasses import dataclass + +MAX_BYTES=60000 +TARGET_BYTES=26000 +MAX_QUESTIONS=100 + + +def wire_bytes(payload): + return len(json.dumps(payload,ensure_ascii=False,separators=(',',':'),allow_nan=False).encode('utf-8')) + + +def compact_state(state): + if not isinstance(state,dict):return state + result=copy.deepcopy(state) + # Training prose does not belong in a decision request. Saved graph is the representation. + for key in ('author_voice','voice','voice_references'): + value=result.get(key) + if isinstance(value,dict): + graph=value.get('graph') + if graph is None and value.get('instructions'): + from .style_graph import text_graph,view + graph=view(text_graph(value['instructions'],value.get('name','Voice'))) + result[key]={k:v for k,v in value.items() if k in {'enabled','name','influence_percent','rule','profile_needs_refresh'}} + if graph:result[key]['graph']=graph + result[key]['scope']='Compact voice graph; no training articles. Follow the current author instruction over a saved habit.' + # Duplicate profile and contract text need not travel twice. These are same-state pointers. + brief=result.get('brief');contract=result.get('editorial_contract') + if isinstance(brief,dict) and isinstance(contract,dict): + brief={k:v for k,v in brief.items() if k not in {'pinned_nodes','excluded_nodes'}} + for key in ('direction','thesis'): + if key in brief and brief[key]==contract.get(key):brief.pop(key) + result['brief']=brief + result['_brief_note']='Direction and thesis shared with editorial_contract; pin/exclusion choices are already reflected in candidate selection.' + # The editor's claims are already reproduced in the deterministic citation audit. + findings=result.get('editorial_findings') + if isinstance(findings,dict) and isinstance(result.get('citation_audit'),dict): + findings.pop('claims',None) + return result + + +@dataclass +class Batch: + state:dict|str + questions:dict + + +class Unrepresentable(ValueError):pass + + +def plan(state,questions,model): + if not questions:raise ValueError('JEV needs at least one typed question.') + state=compact_state(state) + groups=[] + candidates=state.get('candidates') if isinstance(state,dict) else None + if isinstance(candidates,list) and candidates: + assigned=set() + # Original keys and candidate IDs are retained; do not renumber c3 into c0. + for index,candidate in enumerate(candidates): + qs={k:q for k,q in questions.items() if re.match(r'^c'+str(index)+r'_',k)} + if qs: + groups.append(({**state,'candidates':[candidate]},qs));assigned.update(qs) + remainder={k:q for k,q in questions.items() if k not in assigned} + if remainder:groups.append((state,remainder)) + else:groups=[(state,questions)] + output=[] + for scoped,qs in groups: + current={} + threshold=MAX_BYTES if wire_bytes({'state':scoped})>TARGET_BYTES*0.75 else TARGET_BYTES + for key,question in qs.items(): + proposed={**current,key:question} + payload={'model':model,'state':scoped,'questions':proposed} + size=wire_bytes(payload) + if current and (size>threshold or len(proposed)>MAX_QUESTIONS): + output.append(Batch(scoped,current));current={key:question} + else:current=proposed + if wire_bytes({'model':model,'state':scoped,'questions':current})>MAX_BYTES: + # Last lossless reduction: intern repeated long strings as graph-style references. + packed=intern(scoped) + if wire_bytes({'model':model,'state':packed,'questions':current})>MAX_BYTES: + raise Unrepresentable('An indivisible decision exceeds the bounded JEV view. No prose, thesis or evidence was clipped.') + scoped=packed + if current:output.append(Batch(scoped,current)) + # Adjacent candidate groups share the same brief/voice. Pack them when the exact payload fits. + combined=[] + for b in output: + if combined and isinstance(b.state,dict) and isinstance(combined[-1].state,dict): + old=combined[-1];a=dict(old.state);c=dict(b.state) + ac=a.pop('candidates',None);bc=c.pop('candidates',None) + if ac is not None and bc is not None and a==c and not set(old.questions)&set(b.questions): + nodes=ac+[v for v in bc if v not in ac] + merged=Batch({**a,'candidates':nodes},{**old.questions,**b.questions}) + if len(merged.questions)<=MAX_QUESTIONS and wire_bytes({'model':model,'state':merged.state,'questions':merged.questions})<=TARGET_BYTES: + combined[-1]=merged;continue + combined.append(b) + assert all(wire_bytes({'model':model,'state':b.state,'questions':b.questions})<=MAX_BYTES for b in combined) + assert set().union(*(set(b.questions) for b in combined))==set(questions) + return combined + + +def intern(state): + """Lossless repeated-text references, unlike destructive context truncation.""" + counts={} + def count(value): + if isinstance(value,str) and len(value)>240:counts[value]=counts.get(value,0)+1 + elif isinstance(value,dict): + for v in value.values():count(v) + elif isinstance(value,list): + for v in value:count(v) + count(state) + symbols={text:'t'+str(i) for i,(text,n) in enumerate(counts.items()) if n>1} + if not symbols:return state + def walk(value): + if isinstance(value,str) and value in symbols:return {'text_ref':symbols[value]} + if isinstance(value,dict):return {k:walk(v) for k,v in value.items()} + if isinstance(value,list):return [walk(v) for v in value] + return value + return {'state':walk(state),'texts':{ident:text for text,ident in symbols.items()}, + 'encoding':'text_ref values point to verbatim strings in texts; nothing has been shortened.'} + + +def dispatch(clients,state,questions): + from .providers import ProviderError + route=clients.jev_route() + if route=='off':return None + bare=clients.settings.jev_model.removeprefix('~typesafe/').removeprefix('typesafe/') + model=('~typesafe/jev-latest' if bare=='jev-latest' else 'typesafe/'+bare) if route=='openrouter' else bare + url='https://openrouter.ai/api/alpha/decisions' if route=='openrouter' else 'https://api.typesafe.ai/v1/systemone' + try:batches=plan(state,questions,model) + except Unrepresentable: + if clients.job:clients.job.note('JEV: left this oversized indivisible decision unscored; the author brief and evidence were not truncated. Writing and saved work are retained.') + return None + if clients.job and len(batches)>clients.settings.max_calls-clients.job.usage['calls']: + clients.job.note('JEV: remaining request budget cannot cover this decision batch. Kept it unscored without spending partial requests.') + return None + if clients.job and len(batches)>1:clients.job.note(f'JEV: automatically sized {len(batches)} compact decision requests; training articles are excluded.') + key=clients.vault.get(route) or (clients.llm_key() if route=='openrouter' and clients.settings.provider=='openrouter' else '') + output={} + for index,b in enumerate(batches): + payload={'model':model,'state':b.state,'questions':b.questions} + size=wire_bytes(payload) + if size>MAX_BYTES:raise AssertionError('JEV preflight failed') + data=clients.post(url,payload,key,'JEV') + answers=data.get('answers',{}) + if not isinstance(answers,dict):raise ProviderError('JEV returned invalid typed answers.') + import math + for name,q in b.questions.items(): + a=answers.get(name,{}) + if not isinstance(a,dict) or a.get('type')!=q['type']:raise ProviderError('JEV returned an invalid answer type for '+name+'.') + if q['type']=='noul': + v=a.get('noul') + if type(v) not in (int,float) or not math.isfinite(v) or not 0<=v<=1:raise ProviderError('JEV returned an invalid probability.') + elif q['type']=='choice' and a.get('choice') not in q['criteria']:raise ProviderError('JEV returned a choice outside the declared options.') + output[name]=a + if clients.job and clients.job.receipts: + clients.job.receipts[-1].update({'request_bytes':size,'decision_questions':len(b.questions),'decision_batch':index+1,'decision_batches':len(batches)}) + return output diff --git a/graphpaper/models.py b/graphpaper/models.py index 7bfa3e3..8288724 100644 --- a/graphpaper/models.py +++ b/graphpaper/models.py @@ -4,7 +4,7 @@ from typing import Any, Literal from uuid import uuid4 -from pydantic import BaseModel, Field, ConfigDict +from pydantic import BaseModel, Field, ConfigDict, field_validator from .science_models import ResearchWorkspace, ScholarlyMeta @@ -136,6 +136,17 @@ class Review(Model): class VoiceProfile(Model): + graph: dict[str, Any] = Field(default_factory=dict) + library_id: str = Field('', pattern=r'^(?:v_[a-f0-9]{12})?$') + library_version: int = Field(0, ge=0) + + @field_validator('graph') + @classmethod + def validate_voice_graph(cls,value): + if not value:return {} + from .style_graph import StyleGraph + return StyleGraph.model_validate(value).model_dump() + enabled: bool = True strength: int = Field(75, ge=0, le=100) name: str = Field('My writing voice', max_length=160) diff --git a/graphpaper/pipeline.py b/graphpaper/pipeline.py index 1771c41..1186007 100644 --- a/graphpaper/pipeline.py +++ b/graphpaper/pipeline.py @@ -186,7 +186,7 @@ def make_angles(p: Project, clients: Clients, job: Job): c["priority"] = sc["valuable"] * 0.4 + sc["fit"] * 0.3 + sc["grounded"] * 0.2 + (1 - sc["spurious"]) * 0.1 else: c["priority"] = c["structural_rank"] - job.note(f"{'JEV inspected' if answers else 'Structurally explored (JEV off)'} {min(offset + 4, len(candidates))}/{len(candidates)} motifs.", 15 + int(45 * (offset + len(batch)) / len(candidates))) + job.note(f"{'JEV inspected' if answers else 'Structurally explored (unscored)'} {min(offset + 4, len(candidates))}/{len(candidates)} motifs.", 15 + int(45 * (offset + len(batch)) / len(candidates))) ranked = sorted(candidates, key=lambda x: x["priority"], reverse=True) selected = [] counts = defaultdict(int) diff --git a/graphpaper/providers.py b/graphpaper/providers.py index 4a88ad4..d8db011 100644 --- a/graphpaper/providers.py +++ b/graphpaper/providers.py @@ -166,33 +166,8 @@ def jev_route(self): return route def decide(self, state: dict | str, questions: dict) -> dict | None: - route = self.jev_route() - if route == "off": - return None - if not questions or len(questions) > 100: - raise ValueError("JEV batches must have 1–100 questions.") - # This conservative character limit is not advertised as an exact token count. - if len(json.dumps(state, ensure_ascii=False)) + len(json.dumps(questions)) > 60000: - raise ProviderError("JEV state exceeds the conservative 60,000-character request limit. Reduce the batch.") - bare = self.settings.jev_model.removeprefix("~typesafe/").removeprefix("typesafe/") - if route == "openrouter": - url = "https://openrouter.ai/api/alpha/decisions" - model = "~typesafe/jev-latest" if bare == "jev-latest" else "typesafe/" + bare - else: - url, model = "https://api.typesafe.ai/v1/systemone", bare - data = self.post(url, {"model": model, "state": state, "questions": questions}, (self.vault.get(route) or (self.llm_key() if route == "openrouter" and self.settings.provider == "openrouter" else "")), "JEV") - answers = data.get("answers", {}) - for name, q in questions.items(): - answer = answers.get(name, {}) - if answer.get("type") != q["type"]: - raise ProviderError(f"JEV returned an invalid answer type for {name}.") - if q["type"] == "noul": - v = answer.get("noul") - if isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or not 0 <= v <= 1: - raise ProviderError("JEV returned an invalid probability.") - elif q["type"] == "choice" and answer.get("choice") not in q["criteria"]: - raise ProviderError("JEV returned a choice outside the declared options.") - return answers + from .jev_budget import dispatch + return dispatch(self,state,questions) def discover_models(self): if self._model_cache is not None: diff --git a/graphpaper/server.py b/graphpaper/server.py index 497aa8f..82565d7 100644 --- a/graphpaper/server.py +++ b/graphpaper/server.py @@ -220,7 +220,7 @@ def add_source(pid, title, text, kind="text", role="evidence", url="", warnings= raise ValueError("This text is already in the project.") n = max([int(s.id[1:]) for s in p.sources if re.fullmatch(r"S\d+", s.id)] + [0]) + 1 p.sources.append(Source(id=f"S{n}", title=title[:300], text=text, kind=kind, role=role, url=url, warnings=warnings or [], digest=fingerprint, author=author[:300], published=published[:100])) - if p.graph.nodes: + if p.graph.nodes and role != "voice": p.graph.warnings = list(dict.fromkeys(p.graph.warnings + ["Sources changed after the last graph build. Rebuild before relying on coverage."])) p = store.save(p, p.version) app.state.folders.original(p, p.sources[-1]) @@ -263,7 +263,8 @@ async def source_update(pid: str, sid: str, request: Request): raise KeyError(sid) new = Source.model_validate(s.model_dump() | raw) p.sources[p.sources.index(s)] = new - p.graph.warnings = list(dict.fromkeys(p.graph.warnings + ["Sources changed after the last graph build. Rebuild before relying on coverage."])) + if s.role != "voice" or new.role != "voice": + p.graph.warnings = list(dict.fromkeys(p.graph.warnings + ["Sources changed after the last graph build. Rebuild before relying on coverage."])) return store.save(p, p.version) @app.post("/api/projects/{pid}/graph/import") @@ -351,6 +352,8 @@ async def import_project(request: Request): from .studio_routes import install install(app, store, vault, runner, settings) + from .voice_routes import install as install_voices + install_voices(app, store, vault, runner, settings) from .editorial_routes import install as install_editorial install_editorial(app,store,runner) from .science_routes import install as install_science diff --git a/graphpaper/style_graph.py b/graphpaper/style_graph.py new file mode 100644 index 0000000..4283ef0 --- /dev/null +++ b/graphpaper/style_graph.py @@ -0,0 +1,115 @@ +"""Reusable style graphs, with explicit layers and no word/tone blacklist.""" +from __future__ import annotations +import hashlib +import json +import re +from pydantic import BaseModel,ConfigDict,Field,model_validator + + +def encoded(value): + return json.dumps(value,ensure_ascii=False,separators=(',',':'),allow_nan=False) + + +class StyleNode(BaseModel): + model_config=ConfigDict(extra='forbid',allow_inf_nan=False) + id:str=Field(min_length=1,max_length=100) + label:str=Field(min_length=1,max_length=100) + instruction:str=Field('',max_length=600) + kind:str=Field('style',max_length=60) + weight:float=Field(1,ge=0,le=1) + targets:list[str]=Field(default_factory=lambda:['all'],max_length=12) + + +class StyleEdge(BaseModel): + model_config=ConfigDict(extra='forbid') + source:str=Field(max_length=100) + target:str=Field(max_length=100) + relation:str=Field('guides',max_length=80) + + +class StyleGraph(BaseModel): + model_config=ConfigDict(extra='forbid',allow_inf_nan=False) + schema_version:int=1 + nodes:list[StyleNode]=Field(default_factory=list,max_length=32) + edges:list[StyleEdge]=Field(default_factory=list,max_length=64) + source_hash:str='' + @model_validator(mode='after') + def validate_graph(self): + ids=[n.id for n in self.nodes] + if len(ids)!=len(set(ids)):raise ValueError('Voice graph node IDs must be unique.') + if any(e.source not in ids or e.target not in ids for e in self.edges):raise ValueError('Voice graph contains a dangling relationship.') + if len(encoded(self.model_dump()).encode())>30000:raise ValueError('Keep this reusable voice graph below 30 KB; training articles belong in its separate sample library.') + return self + + +def text_graph(instructions,name='My voice'): + """Lossless deterministic migration of an existing <=12,000-character profile.""" + raw=instructions.strip() + if not raw:return StyleGraph() + pieces=[] + for paragraph in raw.splitlines(): + paragraph=paragraph.strip() + if not paragraph:continue + # No banned phrases, softening, summarization or rewriting of the author's words. + pieces.extend(paragraph[i:i+600] for i in range(0,len(paragraph),600)) + if len(pieces)>31: + pieces=[raw[i:i+600] for i in range(0,len(raw),600)] + root=StyleNode(id='voice',label=name[:100],kind='voice',instruction='Reusable style preferences. The current author brief takes precedence.') + nodes=[root]+[StyleNode(id='habit_'+str(i+1),label=(p.split(':',1)[0][:70] if ':' in p else 'Writing habit '+str(i+1)),instruction=p) for i,p in enumerate(pieces)] + return StyleGraph(nodes=nodes,edges=[StyleEdge(source='voice',target=n.id,relation='expresses') for n in nodes[1:]],source_hash=hashlib.sha256(raw.encode()).hexdigest()) + + +def for_profile(profile): + graph=StyleGraph.model_validate(profile.graph or {}) + raw=profile.instructions.strip() + # An edited legacy instruction box regenerates its graph deterministically. + if graph.nodes and (not graph.source_hash or graph.source_hash==hashlib.sha256(raw.encode()).hexdigest()):return graph + return text_graph(raw,profile.name) + + +def view(graph,budget=6500): + """A node/edge-closed view. Whole style directives are selected, never clipped.""" + graph=StyleGraph.model_validate(graph) + selected=[] + def material(nodes): + ids={n.id for n in nodes} + return {'nodes':[n.model_dump(exclude={'targets'}) for n in nodes], + 'edges':[e.model_dump() for e in graph.edges if e.source in ids and e.target in ids], + 'coverage':{'shown':len(nodes),'total':len(graph.nodes),'complete':len(nodes)==len(graph.nodes)}} + for node in sorted(graph.nodes,key=lambda n:(n.kind!='voice',-n.weight,n.id)): + if len(encoded(material(selected+[node])).encode())<=budget:selected.append(node) + return material(selected) + + +def voice_decision(profile): + if not profile.enabled or not profile.strength:return {'enabled':False} + return {'enabled':True,'name':profile.name,'influence_percent':profile.strength,'graph':view(for_profile(profile)), + 'rule':'Style, not factual evidence. Current project instructions override saved habits, including their force, vocabulary and use of profanity. Do not sanitize power language.'} + + +def overlay(project,maximum=100): + """Compose namespaced node-link graphs without merging style nodes into evidence.""" + import networkx as nx + from networkx.readwrite import json_graph + voice=for_profile(project.voice_profile) + evidence=nx.DiGraph();style=nx.DiGraph() + evidence.add_node('scope:'+project.id,label=project.title,layer='scope',kind='project') + for n in project.graph.nodes[:maximum]: + evidence.add_node('evidence:'+n.id,label=n.label,kind=n.kind,layer='evidence',status=n.status) + for e in project.graph.edges: + a,b='evidence:'+e.source,'evidence:'+e.target + if a in evidence and b in evidence:evidence.add_edge(a,b,relation=e.relation,layer='evidence') + if project.voice_profile.enabled and project.voice_profile.strength: + for n in voice.nodes:style.add_node('style:'+n.id,**n.model_dump(exclude={'id'}),layer='style') + for e in voice.edges:style.add_edge('style:'+e.source,'style:'+e.target,relation=e.relation,layer='style') + combined=nx.compose(evidence,style) + for n in voice.nodes: + key='style:'+n.id + if key not in combined:continue + if n.kind=='voice':combined.add_edge(key,'scope:'+project.id,relation='guides_expression',layer='overlay') + for target in project.graph.nodes[:maximum]: + if target.kind in n.targets:combined.add_edge(key,'evidence:'+target.id,relation='guides_expression',layer='overlay') + result=json_graph.node_link_data(combined,edges='edges') + result['scope']={'evidence_nodes_shown':min(maximum,len(project.graph.nodes)),'evidence_nodes_total':len(project.graph.nodes), + 'note':'Composed style/evidence layers. Style links guide expression, not proof. The stored research graph is unchanged.'} + return result diff --git a/graphpaper/voice.py b/graphpaper/voice.py index 2bad296..93d9601 100644 --- a/graphpaper/voice.py +++ b/graphpaper/voice.py @@ -32,7 +32,7 @@ def sample_pack(project, budget=24000): if not selected: return [] selected = selected[:24] - per_source = max(200, budget // len(selected)) + per_source = max(1, budget // len(selected)) result = [] for s in selected: if len(s.text) <= per_source: @@ -49,6 +49,12 @@ def voice_context(project, budget=9000): profile = project.voice_profile if not profile.enabled or profile.strength == 0: return {'enabled': False} + if profile.library_id: + from .style_graph import for_profile + return {'enabled':True,'name':profile.name,'influence_percent':profile.strength, + 'graph':for_profile(profile).model_dump(),'instructions':'','samples':[], + 'profile_needs_refresh':False, + 'rule':'Apply this saved voice graph as flexible style preferences, not factual evidence or a filter. The current brief and author instruction take precedence, including stronger vocabulary, profanity, ridicule and rhetorical force. Training articles are not needed again.'} stale = bool(profile.sample_hash and profile.sample_hash != fingerprint(project)) return {'enabled': True, 'influence_percent': profile.strength, 'instructions': profile.instructions if not stale else '', @@ -69,9 +75,9 @@ def learn_voice(project, clients, job): job.note('Learning rhythm, diction, structure and rhetorical habits from your selected writing.', 15) budget = min(28000, max(4000, clients.settings.context_chars - 9000)) raw = clients.complete(BOUNDARY, - json.dumps({'task': 'Build an editable author voice profile from these samples. Observe authorial choices, rhetorical stance, willingness to conclude, argument structure, satirical targets, analogy, comic timing and prose mechanics; do not infer personality, identity, factual beliefs or biography. Distinguish recurring choices from one-off subject matter. Distinguish purposeful empirical qualifiers from absent boilerplate. Preserve recurring bold judgments, adversarial argument, humor, force, intentional punctuation and moral framing when present; do not normalize them into neutral or generic casual prose. Produce instructions usable for both new writing and faithful editing. Note limited or mixed evidence. Do not quote or copy distinctive sample sentences.', + json.dumps({'task': 'Build an editable author voice profile from these samples. Observe authorial choices, rhetorical stance, willingness to conclude, argument structure, satirical targets, analogy, comic timing and prose mechanics; do not infer personality, identity, factual beliefs or biography. Distinguish recurring choices from one-off subject matter. Distinguish purposeful empirical qualifiers from absent boilerplate. Preserve recurring bold judgments, adversarial argument, humor, force, intentional punctuation and moral framing when present; do not normalize them into neutral or generic casual prose. Produce instructions usable for both new writing and faithful editing. Note limited or mixed evidence. Do not quote or copy distinctive sample sentences. Return a compact reusable graph of 8-20 style nodes and relationships. Preserve force, swearing, indignation and ridicule where present; do not install a vocabulary filter. Each instruction is at most 600 characters, and the current author brief overrides saved preferences.', 'samples': sample_pack(project, budget), 'measured_style': [metrics(s.text) for s in material], - 'output_schema': {'name': 'Short author voice label', 'instructions': 'Specific prose guidance with headings for rhythm, diction, structure, stance, humor, imagery, punctuation, fiction/nonfiction differences and avoidances.', 'observations': ['Short, evidence-limited observations']}}, ensure_ascii=False), + 'output_schema': {'name': 'Short author voice label', 'instructions': 'Specific prose guidance with headings for rhythm, diction, structure, stance, humor, imagery, punctuation, fiction/nonfiction differences and avoidances.', 'observations': ['Short, evidence-limited observations'], 'graph': {'nodes':[{'id':'voice','label':'Named voice','kind':'voice','instruction':'Style identity, not factual claims','weight':1,'targets':['all']},{'id':'force','label':'Rhetorical force','kind':'style','instruction':'A compact observed writing habit, including power language when present. Current author requests override the preference.','weight':1,'targets':['claim']}], 'edges':[{'source':'voice','target':'force','relation':'expresses'}]}}}, ensure_ascii=False), role='extraction', json_mode=True) if not isinstance(raw.get('instructions'), str) or len(raw['instructions'].strip()) < 30: raise ProviderError('The model did not return a usable voice profile. Existing profile preserved.') @@ -79,4 +85,10 @@ def learn_voice(project, clients, job): enabled=True, strength=project.voice_profile.strength, sample_ids=[s.id for s in material], sample_hash=fingerprint(project), learned_at=now(), metrics=metrics('\n\n'.join(s.text for s in material)), observations=[str(x)[:500] for x in raw.get('observations', [])[:12]]) - job.note('Voice profile ready. Edit it, adjust its influence, or switch it off.', 95) + from .style_graph import for_profile,StyleGraph + graph=raw.get('graph') + if isinstance(graph,dict) and graph.get('nodes'): + project.voice_profile.graph=StyleGraph.model_validate(graph).model_dump() + else: + project.voice_profile.graph=for_profile(project.voice_profile).model_dump() + job.note('Voice graph ready. Save it to My voices to reuse across projects without retraining.', 95) diff --git a/graphpaper/voice_library.py b/graphpaper/voice_library.py new file mode 100644 index 0000000..fa9f103 --- /dev/null +++ b/graphpaper/voice_library.py @@ -0,0 +1,87 @@ +"""Named, reusable voice graphs. Training text is stored apart from project evidence.""" +from __future__ import annotations +import json +from typing import Any +from pydantic import BaseModel,ConfigDict,Field +from .models import VoiceProfile,Source,now,uid +from .storage import Conflict +from .style_graph import for_profile + + +class SavedVoice(BaseModel): + model_config=ConfigDict(extra='forbid',allow_inf_nan=False) + id:str=Field(default_factory=lambda:uid('v_')) + version:int=0 + updated:str=Field(default_factory=now) + profile:VoiceProfile=Field(default_factory=VoiceProfile) + usage:dict[str,Any]=Field(default_factory=dict) + training_revision:int=0 + learned_revision:int=0 + + +def validate_samples(samples): + if len(samples)>24 or sum(len(s.text) for s in samples)>6000000: + raise ValueError('Use at most 24 selected training pieces and 6 million characters per saved voice. Original project samples remain untouched.') + if any(s.role!='voice' for s in samples):raise ValueError('Voice training records must have the voice role.') + + +class VoiceLibrary: + def __init__(self,store): + self.store=store + with store.connect() as c: + c.execute('CREATE TABLE IF NOT EXISTS voices(id TEXT PRIMARY KEY, version INTEGER NOT NULL, data TEXT NOT NULL, samples TEXT NOT NULL)') + + def get(self,vid): + with self.store.connect() as c:row=c.execute('SELECT data,samples FROM voices WHERE id=?',(vid,)).fetchone() + if not row:raise KeyError('Saved voice not found.') + return SavedVoice.model_validate_json(row['data']),[Source.model_validate(x) for x in json.loads(row['samples'])] + + def list(self): + with self.store.connect() as c:rows=c.execute('SELECT data,samples FROM voices ORDER BY id').fetchall() + result=[] + for row in rows: + v=SavedVoice.model_validate_json(row['data']);samples=json.loads(row['samples']) + result.append({'id':v.id,'version':v.version,'name':v.profile.name,'updated':v.updated, + 'nodes':len(for_profile(v.profile).nodes),'samples':len(samples),'needs_relearning':v.training_revision!=v.learned_revision}) + return result + + def create(self,profile=None,samples=None): + profile=(profile or VoiceProfile()).model_copy(deep=True) + profile.graph=for_profile(profile).model_dump() + profile.library_id='';profile.library_version=0 + samples=[s.model_copy(deep=True) for s in (samples or [])] + for i,s in enumerate(samples):s.id='S'+str(i+1);s.role='voice' + validate_samples(samples) + # Training is separate. A new voice with samples but no learned representation is stale. + v=SavedVoice(profile=profile,training_revision=1 if samples else 0, + learned_revision=1 if samples and profile.graph.get('nodes') else 0) + with self.store.lock,self.store.connect() as c: + c.execute('INSERT INTO voices VALUES(?,?,?,?)',(v.id,0,v.model_dump_json(),json.dumps([s.model_dump() for s in samples],ensure_ascii=False))) + return v + + def save(self,v,expected,samples=None): + v=SavedVoice.model_validate(v.model_dump()) + v.profile.graph=for_profile(v.profile).model_dump() + v.version=expected+1;v.updated=now() + if samples is not None:validate_samples(samples) + with self.store.lock,self.store.connect() as c: + if samples is None: + changed=c.execute('UPDATE voices SET version=?,data=? WHERE id=? AND version=?',(v.version,v.model_dump_json(),v.id,expected)) + else: + changed=c.execute('UPDATE voices SET version=?,data=?,samples=? WHERE id=? AND version=?',(v.version,v.model_dump_json(),json.dumps([s.model_dump() for s in samples],ensure_ascii=False),v.id,expected)) + if changed.rowcount!=1:raise Conflict('This voice changed. Refresh before saving; existing work was not overwritten.') + return v + + def delete(self,vid,version): + with self.store.lock,self.store.connect() as c: + changed=c.execute('DELETE FROM voices WHERE id=? AND version=?',(vid,version)) + if changed.rowcount!=1:raise Conflict('This voice changed or was removed. Refresh first.') + + def apply(self,project,voice): + profile=voice.profile.model_copy(deep=True) + profile.graph=for_profile(profile).model_dump() + if not profile.graph['nodes']:raise ValueError('Learn or define this voice graph first.') + profile.library_id=voice.id;profile.library_version=voice.version + profile.sample_ids=[];profile.sample_hash='';profile.enabled=True + project.voice_profile=profile + return project diff --git a/graphpaper/voice_routes.py b/graphpaper/voice_routes.py new file mode 100644 index 0000000..bb3d422 --- /dev/null +++ b/graphpaper/voice_routes.py @@ -0,0 +1,175 @@ +"""GUI-facing voice library, isolated training jobs and explicit project application.""" +from __future__ import annotations +import json +from pathlib import Path +from fastapi import Request,UploadFile,File +from fastapi.responses import Response +from starlette.concurrency import run_in_threadpool +from .models import Source,VoiceProfile,Project,now +from .voice_library import VoiceLibrary +from .style_graph import StyleGraph,for_profile,overlay +from .ingest import clean,digest,extract,fetch_url,MAX_BYTES +from .storage import Conflict + + +def install(app,store,vault,runner,settings): + library=VoiceLibrary(store);app.state.voices=library + def active(vid): + return any(j.project_id==vid and j.state in {'queued','running'} for j in runner.jobs.values()) + def get_editable(vid,version=None): + if active(vid):raise ValueError('Finish or cancel this voice-learning job before editing it.') + v,s=library.get(vid) + if version is not None and v.version!=version:raise Conflict('The voice changed. Refresh before saving.') + return v,s + def public(v,s): + return {**v.model_dump(),'graph':for_profile(v.profile).model_dump(), + 'samples':[{'id':x.id,'title':x.title,'words':len(x.text.split()),'url':x.url} for x in s], + 'needs_relearning':v.training_revision!=v.learned_revision} + def add_training(vid,title,text,kind='text',url='',warnings=None): + with runner.lock,store.lock: + v,s=get_editable(vid);text=clean(text) + if any(x.digest==digest(text) for x in s):raise ValueError('This writing sample is already in the voice library.') + number=max([int(x.id[1:]) for x in s]+[0])+1 + s.append(Source(id='S'+str(number),title=title[:300],text=text,role='voice',kind=kind,url=url,warnings=warnings or [],digest=digest(text))) + v.training_revision+=1 + return public(library.save(v,v.version,s),s) + + @app.get('/api/voices') + def list_voices():return library.list() + + @app.post('/api/voices') + async def create_voice(request:Request): + raw=await request.json() + if set(raw)-{'name','instructions','graph','project_id','version'}:raise ValueError('Unsupported voice property.') + with runner.lock,store.lock: + if raw.get('project_id'): + p=store.get(raw['project_id']) + if runner.active(p.id):raise ValueError('Finish the active project job first.') + if p.version!=raw.get('version'):raise Conflict('The project changed. Refresh before saving its voice.') + profile=p.voice_profile.model_copy(deep=True) + samples=[s for s in p.sources if s.role=='voice' and s.enabled] + else:profile=VoiceProfile();samples=[] + if 'name' in raw:profile.name=raw['name'] + if 'instructions' in raw:profile.instructions=raw['instructions'];profile.graph={} + if 'graph' in raw:profile.graph=StyleGraph.model_validate(raw['graph']).model_dump() + profile=VoiceProfile.model_validate(profile.model_dump()) + v=library.create(profile,samples) + return public(v,library.get(v.id)[1]) + + @app.post('/api/voices/import') + async def import_voice(request:Request): + raw=await request.json() + if raw.get('format')!='graphpaper-voice-1':raise ValueError('Choose a GraphPaper voice graph export.') + profile=VoiceProfile.model_validate(raw.get('profile',{})) + profile.graph=StyleGraph.model_validate(profile.graph).model_dump() + v=library.create(profile) + return public(v,[]) + + @app.get('/api/voices/{vid}') + def read_voice(vid:str):return public(*library.get(vid)) + + @app.patch('/api/voices/{vid}') + async def edit_voice(vid:str,request:Request): + raw=await request.json() + if set(raw)-{'version','name','instructions','graph','strength'}:raise ValueError('Unsupported saved-voice property.') + with runner.lock,store.lock: + v,s=get_editable(vid,raw.get('version',-1)) + for key in ('name','strength'): + if key in raw:setattr(v.profile,key,raw[key]) + if 'instructions' in raw and raw['instructions']!=v.profile.instructions: + v.profile.instructions=raw['instructions'];v.profile.graph={} + if 'graph' in raw: + g=StyleGraph.model_validate(raw['graph']);g.source_hash='' + v.profile.graph=g.model_dump() + v.profile=VoiceProfile.model_validate(v.profile.model_dump()) + return public(library.save(v,v.version),s) + + @app.delete('/api/voices/{vid}') + async def delete_voice(vid:str,request:Request): + raw=await request.json() + with runner.lock,store.lock: + v,_=get_editable(vid,raw.get('version',-1));library.delete(vid,v.version) + return {'deleted':True,'note':'Applied project copies remain available.'} + + @app.get('/api/voices/{vid}/export') + def export_voice(vid:str): + v,_=library.get(vid) + profile=v.profile.model_copy(deep=True);profile.sample_ids=[];profile.sample_hash='';profile.library_id='';profile.library_version=0 + content=json.dumps({'format':'graphpaper-voice-1','profile':profile.model_dump()},ensure_ascii=False,indent=2) + return Response(content,media_type='application/json',headers={'Content-Disposition':'attachment; filename="voice-graph.json"'}) + + @app.post('/api/voices/{vid}/samples/text') + async def sample_text(vid:str,request:Request): + raw=await request.json();return add_training(vid,str(raw.get('title','Writing sample')),str(raw.get('text',''))) + + @app.post('/api/voices/{vid}/samples/file') + async def sample_file(vid:str,file:UploadFile=File(...)): + data=await file.read(MAX_BYTES+1);await file.close() + title=Path(file.filename or 'sample.txt').name + text,warnings=await run_in_threadpool(extract,data,title) + return add_training(vid,title,text,'file',warnings=warnings) + + @app.post('/api/voices/{vid}/samples/url') + async def sample_url(vid:str,request:Request): + raw=await request.json() + title,text,url,warnings=await run_in_threadpool(fetch_url,str(raw.get('url',''))) + return add_training(vid,title,text,'url',url,warnings) + + @app.get('/api/voices/{vid}/samples/{sid}') + def sample_read(vid:str,sid:str): + _,samples=library.get(vid);s=next((x for x in samples if x.id==sid),None) + if s is None:raise KeyError('Training sample not found.') + return {'id':s.id,'title':s.title,'text':s.text} + + @app.delete('/api/voices/{vid}/samples/{sid}') + async def sample_delete(vid:str,sid:str,request:Request): + raw=await request.json() + with runner.lock,store.lock: + v,s=get_editable(vid,raw.get('version',-1)) + rest=[x for x in s if x.id!=sid] + if len(rest)==len(s):raise KeyError('Training sample not found.') + v.training_revision+=1 + return public(library.save(v,v.version,rest),rest) + + def learn_job(job,v,s,configuration): + from .voice import learn_voice + from .providers import Cancelled,ProviderError + job.state='running';original_version=v.version + try: + temporary=Project(title='Voice learning',sources=s,voice_profile=v.profile.model_copy(deep=True)) + clients=runner.clients_factory(configuration,vault,job) + learn_voice(temporary,clients,job);job.check() + v.profile=temporary.voice_profile;v.profile.graph=for_profile(v.profile).model_dump() + v.learned_revision=v.training_revision + for key,value in job.usage.items():v.usage[key]=v.usage.get(key,0)+value + library.save(v,original_version) + job.state='completed';job.progress=100 + except Cancelled:job.state='cancelled';job.error='Voice learning cancelled. The previous saved voice is unchanged.' + except Exception as e: + job.state='failed';job.error=str(e) if isinstance(e,(ValueError,ProviderError,Conflict)) else 'Voice learning failed. The previous saved voice is unchanged.' + finally:job.finished=now() + + @app.post('/api/voices/{vid}/learn') + async def learn(vid:str,request:Request): + from .pipeline import Job + raw=await request.json() + with runner.lock,store.lock: + v,s=get_editable(vid,raw.get('version',-1)) + if not s:raise ValueError('Add writing samples to this voice first.') + job=Job(vid,'voice-library');runner.jobs[job.id]=job + runner.pool.submit(learn_job,job,v,s,settings()) + return job.public() + + @app.post('/api/projects/{pid}/voice/apply') + async def apply(pid:str,request:Request): + raw=await request.json() + with runner.lock,store.lock: + if runner.active(pid):raise ValueError('Finish the current project job before applying a voice.') + p=store.get(pid) + if p.version!=raw.get('version'):raise Conflict('The project changed. Refresh before applying this voice.') + v,_=library.get(raw.get('voice_id','')) + library.apply(p,v) + return store.save(p,p.version) + + @app.get('/api/projects/{pid}/voice/overlay') + def get_overlay(pid:str):return overlay(store.get(pid)) diff --git a/publish-manifest.json b/publish-manifest.json index ffcb167..32cd0d9 100644 --- a/publish-manifest.json +++ b/publish-manifest.json @@ -5,17 +5,19 @@ ".github/topics.json": "341aa983b1829624dec7ab7cd2e83eed08d45bc78305033c1e860631c1227e41", ".github/workflows/docs.yml": "19c4d6229a804adf09358171b19a139b45f1c2630b751213af1e9e62baed2c40", ".github/workflows/import-build.yml": "c56a990b97242f154907f76059f5b859448ca8de7b582928d63bb3717ac0d60a", - ".github/workflows/quality.yml": "cf701794996387e3ab9f0d74cbb59d6995a91a4a5517a0f044732e3c1bb5d096", - ".github/workflows/windows.yml": "1c0c111c293f2e9e836833676fa270940f62e36e403bc8d91f98f8774762c9ad", + ".github/workflows/quality.yml": "2ffc5f11447f1cfb5a0f45798df1d217f5d5c4eaf5808270523a69f468f7d9b6", + ".github/workflows/voice-manifest.yml": "19647138870a09bbe7835df34ddab4017a91da9c1937444a32f3a35b9ef1fa88", + ".github/workflows/voices-release.yml": "a67ce85adbbb1ceafa3687119690eb5269ade57b26fc6705c3f9db8b41863361", + ".github/workflows/windows.yml": "65512954b719620063384ce93950b89c9b0aaac2ad6c7c620d505f8787b0444a", ".gitignore": "229a84e2b07c91b31c30f06df236ea17fe211eb75b217adc2eecb3f9d377f9bf", - "CHANGELOG.md": "414d7fce75f795d6962cacc4b37efd95d6e6f23bb64b64561ce0966c7db739ed", + "CHANGELOG.md": "1f8159e524f9e0138c8d706b897108ea8ebdd9a764fb5a1e2b74d04bfc297e98", "CONTRIBUTING.md": "170e7cd71886a6cd3aa82c3ab5a478cde152faa4f243900e047bca565e658001", "GraphPaper.pyw": "8fd19e9ba8caa25b11b0b458e00aa3ea4067ca5d90161f7ba2781f1d514fa094", "LICENSE": "5849844cbc9e9e199cf576224279d63d5fa56ca913f7a13285e4aa4fcb99bf80", "Open GraphPaper.vbs": "6ddf70eba29c281f9373c11b6aa7f69862c8ea7b03050b974c98d817ca11974e", "Publish GraphPaper.pyw": "a55984668dd0040d9bb5fc8c4c18ea5a201eede7e8a51ab79f8f0f095162ec8b", "Publish GraphPaper.vbs": "f9c595bcd13da5df60dcc2cfd906665243628f6e6c10a8b6a03ad1392a9698ad", - "README.md": "24551b6d604d0d76dc1941dcc2b4fcd8edfe3407f8696204f0d5b6b5dc8ae689", + "README.md": "dac0e710729e57ee730a867495db842e4fc14b597009c18f0e994e1cf8b89cb8", "SECURITY.md": "e70e5b87ea06cdc075c3111a1eb0bb6ac5f5b6f8b55f5e735ad69d84ae63cda6", "docs/ARCHITECTURE.md": "86a28806513cf8b533d6a4767322c7dd339a6a2ed112081ea7ce5fcdf4ffd89d", "docs/CONNECTIONS.md": "eec1286ddc356bbb2cd7d63692e73abae6d0f7e9159bbb2edee6377dd5b3fd72", @@ -30,12 +32,13 @@ "docs/PROJECTS-AND-SOURCES.md": "828179218c31bb9f50698e2d047a04b4a056b979800795a37cd790d29060e2dd", "docs/PUBLISHING.md": "324b83d5138b22bb29ca1737cbc0563595cc14f03177d478c478bde9109f975f", "docs/QUALITY.md": "75b998fe5bd366c955516e1671c19a43ad6f6d5449e7356218980f0073bd3250", - "docs/README.md": "b63803fe5efa31bc7bfaa01d718a8217fd4562dff46e1be5ddf7bd4bb903a009", + "docs/README.md": "393335da0650e4a657ce6deb694a8182cb8429c042551e722cbd82fa5a90d101", "docs/RELEASE-0.2.1.md": "42642432ff5c6017630d33a64726c5dca37abbbb36a476a901a59ec8887aa870", "docs/RELEASE-0.2.md": "c03a41903088590845d0da0707a562b9e6cbf4c5e1ff57d8794763f349093e44", "docs/RELEASE-0.3.0.md": "e32f30a3a9716c310be21dcf540cf9060384b0c7fdf69c8fc1c45d36c105cf29", "docs/RELEASE-0.3.1.md": "3879831312c10f64c9faceaac28066ef557825416d1ba75560729c514c16d8f8", "docs/RELEASE-0.4.0.md": "ed7d83f07cf10c447efa2e1ba9fa2634af691cad792928548871eccbee96c751", + "docs/RELEASE-0.5.0.md": "960b36e74d2044bdc05c8b26d54113ea6580c259103d6ef6017809822032b0bf", "docs/SCIENCE.md": "9357a2f4ac714b6330aff4ccfe45ca0457d18e7816a73646314024823fd0ea00", "docs/SECURITY.md": "0f818eb0cf4e8f94ffaaeb6da3cc2c2bc474674b673d185c61b86b2eb5fd3864", "docs/TROUBLESHOOTING.md": "0834f56c656c97e721226a335b24a7aa8afe9cd99b4e28e950412bd86ae3d2d6", @@ -43,6 +46,7 @@ "docs/UPDATE-0.3.1.md": "c1232dc6067211467ec713703966472bf75e22a114d62d0f5e0a40acdabff84f", "docs/UPDATE-0.3.md": "99076ec9ea12986d79b93ec25a729c90403d3432d8e51598db5903b862fba459", "docs/VOICE-AND-PROSE.md": "4e60347a8defd69d9b10276f690ea4652830ae594827d81d3ff43d5a608a8b68", + "docs/VOICE-GRAPHS.md": "ac05b2f2a401664729e8098ed121b348a6fd897b4c671c798eba7c9d395de63b", "docs/WINDOWS.md": "8b7ed8cbe01cf69d0dbc16967fa31cbce78ed28bb4475eb27eff7535737847ca", "docs/images/angles.webp": "1f9a0b5b46c9ae7e84ed1f96ea909d8ca6ba3f7daad2d86f424332c0aac1fb4e", "docs/images/apa-preview-v0.3.webp": "fc2ac3803d88c782e2a02cdfa345ce095e29a184a0ce7d0d6cdc3e7e024aabb0", @@ -118,18 +122,18 @@ "docs/validation/v0.4.0/package.json": "95434f0949b5ec5018287f8fb2b73cea28e8337c1cb4bbb2c93f0997c25114ad", "docs/validation/v0.4.0/summary.json": "3e50403b4db2359985c803f0b275aee1fb69ca7fb7a53a4cbbb2d5d6eaa52bdc", "docs/validation/v0.4.0/windows-unit.xml": "aa1a17c42d241651f9c0b91599d6683ab038644b867127f2d36b19eebdd6eb51", - "docs/wiki-pages.json": "48ab6345c318ff49b4d4245de8399401c61e57a0e31fa47b25647b771215ad41", + "docs/wiki-pages.json": "ef0b18a9821ae2200d709448a29f874ad3842cc20c25ace1d9006350c0da45d3", "examples/README.md": "0b7b83630202501a3ba6ad070708419aa23e32c8060ef9447491c6434043bbf6", "examples/fiction.json": "e3f90c0eed71102640c8a65ae86510e0258e9c90c3cc18730ed05329f632e88d", "examples/nonfiction.json": "af9196cf15dbe1d431f02c7d8ec11a720f25e1c3e30a756c1f9711b4c65a08cc", - "graphpaper/__init__.py": "105ee38fe22cff02668528d721815fe8046898f7748896e337ed3a1b32e6ac3a", + "graphpaper/__init__.py": "a3146b399d581208e7ae734d18409e209de2cbe2eb9e115ccf1a0bdcd6aa955a", "graphpaper/apa.py": "e8c5b844c0f02f14c5a73d6128e4d174d4ad655e380b4d675215beb125cd717f", "graphpaper/argument_draft.py": "a2b5ff0e2a100688a6c0e3be684b68107542a3341a307da7682e0b24e0f5ea3d", "graphpaper/author_web.py": "16ad1218ce0bf76bdd067e731176357586a28dfb55d297a393a3e6a596e126ff", "graphpaper/codex.py": "29361635a46669b5c0ce7d06b903902f05a007424c0061a0df983bf6c81af95f", "graphpaper/demo.py": "5fcee68b786da702cd4365dca17e3f5df415fa26c7c6928568d64d8b7d693006", - "graphpaper/desktop.py": "01ae73f8b863c01ebcb05575719dc44eb08dc515ae6ef64099e87cb36caee63e", - "graphpaper/editorial.py": "04ab7a70ef480c0c9e1f9f42a72bc113261168875cb671e71493c7ad8874e70a", + "graphpaper/desktop.py": "34a660766547f9a543bd21a2e9d6ecd6a2bf24f099176ea9ba06a32f02906e86", + "graphpaper/editorial.py": "ebbd847e987da10fe02b1a7d3187ef2f0628a9fc5b55f2269d4baece77f8a755", "graphpaper/editorial_routes.py": "a62e6cb5854f49fdc2fda51337d3f639bb8bfcb945c9e17c377d3d010d98bd7d", "graphpaper/export.py": "fd76bec7858c53700dfe068035c187277ed85fa7e90a46cb1f1c47b57eb1e744", "graphpaper/folders.py": "37f9b255406e3ff93b6f6886294722c14561a4c1195d5c2107fe67c8de6fbca8", @@ -140,10 +144,11 @@ "graphpaper/graphify_runtime.py": "7ae7629d7f4349ebfcba558a0147bbbab6f0f6113c6e2ad2dca3f716959d967f", "graphpaper/graphify_worker.py": "362444c55086940a394c1569bed492fb8646ea968e6dc1a542e27f182444b868", "graphpaper/ingest.py": "6cb0d9b0d1412e08f627c045f6e70db44882bdf3fd18ff03ea8922a4af83cce8", - "graphpaper/models.py": "f96af9550240df2b04f670b3cadc1421584426ed53b3e4d91b475a7e8e49c39b", - "graphpaper/pipeline.py": "30461790afef78f7042271144870bd48a3ee6d160864673c464965b817d9d4f1", + "graphpaper/jev_budget.py": "854983ddaf076604fe83bd908b35d090a088db4c61f10a9c12239608842d707c", + "graphpaper/models.py": "38ec29c13204c259434e6f58447ce41dd5de174b7bfeda1682cecbf08e3ee950", + "graphpaper/pipeline.py": "5fc0fd06499c3329e59a05e44db46aa51564d27391dd97355931cbf7f277dccd", "graphpaper/polish.py": "6284b296127e495c9ff105ba055a454b53dde7ecd31ee2e05349b9becffb88cb", - "graphpaper/providers.py": "8e63b30cc76f80a3e4a9298e12a33f45a86c23670622bbd7dd782afa5f6266a6", + "graphpaper/providers.py": "7d1848aca208a997afaef2d7184f81724baffbee2f8fe0945733d3cc8c1d9d4b", "graphpaper/reasoning.py": "62f72e3dcdbae5dd6bae6eb08916bcdea8d8a6d768e8305bc4e9acb7c45d3edc", "graphpaper/scholarly.py": "c4bb1faf7af34aedb8afa38b940c711296c92e16260a319244d3325d40c5b9d2", "graphpaper/science.py": "75df98076aa88028f8cab2528fef29c961cd33e485e6eff1c7a8ea3f377eafbb", @@ -151,17 +156,20 @@ "graphpaper/science_models.py": "986446c848927d85aa57bc2d2ba9f248f656fac9f6acc67d056e8f55776258e8", "graphpaper/science_routes.py": "cd2e5c76035dcd7a9384c74b4264306da8429072359a476b9d4e060aee86bb69", "graphpaper/secrets.py": "2d90bfed7124875d50cd977c0c4f17c4cb19e79af307445bc85af99596c3766e", - "graphpaper/server.py": "a813c98beed5c803e80f73c7c89b377608bfe3f385fc92be5dadcf17aa89f936", + "graphpaper/server.py": "49cbcb61cb2ade46f69ca13aa7f1100c37ef239c9958b0461ff52ba6280224c0", "graphpaper/storage.py": "38b0df820a0c4dae0199bf38656d529e726a4475fe7a47b66e41903e96d14eae", "graphpaper/studio_routes.py": "5597db3ee3cba50e7eb4bd8c4e36d3a74be5768bb01b75749f16ac28e10ee84f", + "graphpaper/style_graph.py": "01adedab019d8108fd8643be8b3863ddac53936e368b3f6ef6ec97e5af8b9568", "graphpaper/support.py": "2f4a3f5f7f6a725b15719a775155bdc73c32ce0df022d43b4fb53dbf7e862d8e", - "graphpaper/voice.py": "594155a938a2afdd59f65da9e6794ba709dcdf89f2c305b359f3dbbde1483065", - "pyproject.toml": "2ce6c5cee9f449b2f6b7e12f9ca846eeec6b046fc984374013a148f419eb31fd", + "graphpaper/voice.py": "74be72d1e6edb70a6b541547836db4a7c4af575314c92a253395f9ede26b7976", + "graphpaper/voice_library.py": "2db504911796b564489494b95628fde606b834f1ab890443eea0c2f9db7e670e", + "graphpaper/voice_routes.py": "725a858590e93b9f51ecb95378713ae8e8e88dad99fb1c903889f02a272e4532", + "pyproject.toml": "2559651cf1b7748205d3f71a46bba8894655e601bb5d5b1aeccea8d2108e9d9e", "requirements-desktop.txt": "c9df0bdb57d7e9586ae7c927b5dca32f6e20010e71fc71cc4a5656387a129892", "requirements-dev.txt": "40b197f9671e3c6db4d47ca73d9679eabafe662a56d4824c6f9e64255e47a5ce", "requirements-graphify.txt": "9eea3e6ea7fa3ceb1247518901d41d55a0813dc5133475172541c68c122454f1", "requirements.txt": "4e73f195c8a6d32b841b671200bf0dcaa91e5a042c585e935e3132ee5e4a02a3", - "scripts/build_release.ps1": "a4db771f5ae6e4c0e23873e0baed8cded017c10a203ed8d01b471d4f593b0c95", + "scripts/build_release.ps1": "d56a421d287431c42e7971381ebd5234fb561cb67c117f6b321b86717a751c6a", "scripts/build_windows.ps1": "44e0008ff8221097ead7355aa184a60e824d2fcbf0c77e54c74ddfc8f3a54858", "scripts/bundle_codex.py": "651d08e26949dbda47bca14dbc368d97b82e11a28759792cb6096adcea174efa", "scripts/check_codex.py": "2fff4a6cc80188d54914086ab3f3d7d3cf1ec02001541bad6bc8a426fc1bbc81", @@ -173,7 +181,7 @@ "scripts/docs_tool.py": "7fd2a681bae5fd4a49ae394c74639dee8e833ad08e8a156406c292af6db46572", "scripts/graphify_ui_checks.py": "6dc1e45478a10fa8e3d17190864b2145cd973a5a6e23c7500f0d99ddfb824dc6", "scripts/hooks/hook-graphify.py": "5659b57f3b3e11f9327b2da6e00c4bd82a523f7f978e3b9506811c2caf007ff3", - "scripts/native_smoke.py": "d36060750e1369ed7ab52f76c32ef3fac65f3984105a67f70f04dd81e29ef6fa", + "scripts/native_smoke.py": "e628c22f4ee5cd20bcc662fdd8573f1d759bc52e9e1a7e6c439536bd722853e3", "scripts/package_windows.py": "8b56c4235ca2bbac39657905fa4633d78273bc4df6c9e67558a7e893176af8b3", "scripts/prepare_assets.py": "179b65a0662d329fadecd9a41c134a249ed1f3cc3123a814095ee231636d819d", "scripts/prepare_graphify.py": "c81d4146c3b4a137b93bb61d16308c483a3e3a453501426569242d227bb58e03", @@ -183,6 +191,8 @@ "scripts/ui_science_smoke.py": "fa744ee223db49c0e2325e94fe7deec62470f1d6186b28f2b2dacccb1264f2fb", "scripts/ui_smoke.py": "f1920dc00f8f97a1f5523c375ce573198ac32f177889b5584cd30921a0aa66b2", "scripts/ui_studio_smoke.py": "cf3d51f68362ed9dce39ccfa76729772971470548997b379c122cd4979d5dce3", + "scripts/ui_voices_smoke.py": "d72ad1ed4fdffedb6d35905216524b352dad7d766c3b5180885d106ab3c4cbb1", + "scripts/voice_ui_checks.py": "120c54529db6205d883f3011e6e50549468f35f4dc91a3b185de8b788c46e4bf", "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "tests/conftest.py": "fb734bdc191b96cc60a20d83a4fbeea7b032a35e1b882a09d1cb80167e9bb1a4", "tests/graphify_fixtures.py": "6961e0ceca4a78fca57baa9d9ab9f748e587e8d73adb7fc2d25b791716202e00", @@ -200,6 +210,7 @@ "tests/test_graphify_gateway.py": "f64249c44525b8337f7aa517c5c149b6cddc8b76f42f67a2b5e1a4930a88ebe5", "tests/test_graphify_provenance.py": "ce91f39b3a60bd88c52bd57f70361c92daca7e5e261f9f30868dddfde00384d3", "tests/test_graphify_validation.py": "6aa87f59e9e6be66d8dde810a2967dbffae254dcd253ed46d1c3fc73ad6ec1f1", + "tests/test_jev_budget.py": "2b5e0dbdf064f4128807b547b81693564a61e0a1d03c03e80e4a8a8e8d0611f3", "tests/test_native_bridge.py": "5b611fb9f94733d97d87ac6a45cecd8575e708c0ab687acf59468a76c213251a", "tests/test_pipeline.py": "28373af3ea4739c252de747710923f6945f06a3397b9e59344263c117f48487a", "tests/test_providers.py": "83737b8773b8a11936675fba26388848cf6d95f40ef4b6db25baf40baa9d79cb", @@ -209,18 +220,22 @@ "tests/test_scholarly_research.py": "e3392f5b0957009c555620958a2ead17372dae292cf87b9fb5b5ce812b5dca96", "tests/test_science_integrity.py": "06cb517992a0744e5d73cc475945cad59813fce1d60e59378c6d260d631748e6", "tests/test_studio_update.py": "d01b6c74cd0afc300f4468755213d9d21a1f58986efe61eca9d58c3f677c9653", + "tests/test_voice_integration.py": "63c9e4d4dadba187d36169593868af44a4df94896ae1dc06b8b4f451ac0f8eab", + "tests/test_voice_library.py": "12071c049d7ccca7d10d531879ed67c19eee97178c0a5c3bc34559aed77417e7", "tests/test_windowless.py": "421620004c87cf24dec771966602b0666e75ecca67fec05071cf40e4b6887f44", - "ui/app.js": "0aaf5cbde87d59b57d0f519d3162a6f97bbfe5c2d59efec44d497d2b839c50e5", + "ui/app.js": "09e96444fe3dc7b25db957616f8b60655e6625ff878e9cf8049fa2eecc1c10e4", "ui/graphify.js": "1bad42f86764de73c5e9411cca64d0c86d3bdfecaa8682be3e0728663b85d18c", "ui/graphpaper.ico": "f94c4061c53e8e1a1f2be96ef1a8c18b20c19a984bc35f6337f1037c1025a493", "ui/icon.svg": "9f97b80a90020b682f96756d1245dfa3cc9f19ed989b0751f802220a3934be0f", - "ui/index.html": "257eb6d9c7839b61fa146c7d0e907c6ea0470dc4be6728848f62dc77b39ad954", + "ui/index.html": "b157036ea077e65718e3b4e641051b1b959814077f078fad5618c2854b67fbb3", "ui/polemic.css": "08e5b07f2b8cda29c79794714456502c058f5baa65ad7e47ec758e2a524cc80b", "ui/polemic.js": "3e759fd397e3dac234a81101675e0ef5d0b7dcc3a217fc0fe35d1187ac8224d3", "ui/science.css": "0234fa0a9542afc269ea5b3914a80e0d574843d45f7033394d9e8edaa40ed91c", "ui/science.js": "0ea82fc5cf4f31ea929e3a145b3534507b33ddcf0e0c2d3b0b76046abfdfe30b", "ui/studio.css": "c143a880c50ed59b9d315afaacd20377d6e47c54a701ea105f5c0757cde43a24", - "ui/studio.js": "35ec4ed3ac5e691855680103b96fcc184eca9c61089753ebc7e0cfcf79c8ee87", - "ui/styles.css": "99fb8fb724a74d2d8cc31ea1e7e6b4422f87dfff7015d26313129e999d8631c2" + "ui/studio.js": "daaac1223a29eda59cc2d42af92dc4538dbd1f547d20eb50550563107aff11f8", + "ui/styles.css": "99fb8fb724a74d2d8cc31ea1e7e6b4422f87dfff7015d26313129e999d8631c2", + "ui/voices.css": "c6d90d8271288ceecc4498a8f1fc8be069f4d6d6e296f81c89b86b8043950038", + "ui/voices.js": "7531334f26ba0aea86cda2157890c22e2debe92d1faed6a1b87c32948c53d1cc" } } diff --git a/pyproject.toml b/pyproject.toml index b49b7e7..b930d83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphpaper-studio" -version = "0.4.0" +version = "0.5.0" description = "A local-first graph-guided writing studio for nonfiction and fiction" readme = "README.md" requires-python = ">=3.11" @@ -24,7 +24,7 @@ pythonpath = ["."] addopts = "-ra" [tool.setuptools.data-files] -"share/graphpaper/ui" = ["ui/index.html", "ui/styles.css", "ui/app.js", "ui/studio.js", "ui/studio.css", "ui/science.js", "ui/science.css", "ui/graphify.js", "ui/polemic.js", "ui/polemic.css", "ui/icon.svg", "ui/graphpaper.ico"] +"share/graphpaper/ui" = ["ui/index.html", "ui/styles.css", "ui/app.js", "ui/studio.js", "ui/studio.css", "ui/science.js", "ui/science.css", "ui/graphify.js", "ui/polemic.js", "ui/polemic.css", "ui/voices.js", "ui/voices.css", "ui/icon.svg", "ui/graphpaper.ico"] [project.optional-dependencies] desktop = ["pywebview>=6.0,<7", "graphifyy[openai]==0.9.80"] diff --git a/scripts/build_release.ps1 b/scripts/build_release.ps1 index 1935b64..ef8a7a9 100644 --- a/scripts/build_release.ps1 +++ b/scripts/build_release.ps1 @@ -11,7 +11,7 @@ if ($LASTEXITCODE -ne 0) { throw 'Public Graphify runtime verification failed.' python -m PyInstaller --noconfirm --clean --windowed --onedir --name GraphPaper --paths . --additional-hooks-dir scripts/hooks --icon ui/graphpaper.ico --add-data 'ui;ui' --add-data 'vendor;vendor' --collect-all webview scripts/desktop_entry.py if ($LASTEXITCODE -ne 0) { throw 'Windows desktop build failed.' } Copy-Item README.md, LICENSE -Destination dist/GraphPaper/ -Copy-Item docs/RELEASE-0.4.0.md -Destination dist/GraphPaper/WHATS-NEW.md +Copy-Item docs/RELEASE-0.5.0.md -Destination dist/GraphPaper/WHATS-NEW.md python scripts/check_package.py dist/GraphPaper if ($LASTEXITCODE -ne 0) { throw 'Compiled application self-test failed.' } python scripts/check_graphify.py --executable dist/GraphPaper/GraphPaper.exe --out test-results/graphify-package.json diff --git a/scripts/native_smoke.py b/scripts/native_smoke.py index 095a024..712e13b 100644 --- a/scripts/native_smoke.py +++ b/scripts/native_smoke.py @@ -22,6 +22,7 @@ def main(): parser.add_argument('--science',action='store_true',help='Exercise the Science workspace in the actual native shell') parser.add_argument('--graphify',action='store_true',help='Verify the actual external Graphify action using a synthetic loopback model') parser.add_argument('--polemic',action='store_true',help='Verify mode conversion and authorial controls in the native shell') + parser.add_argument('--voices',action='store_true',help='Exercise saved voice graphs without model calls') parser.add_argument('--out',type=Path,default=ROOT/'test-results/native') args=parser.parse_args() if sys.platform!='win32':raise SystemExit('This regression requires a native Windows desktop.') @@ -146,6 +147,10 @@ def cdp_ready(): assert page.evaluate('() => state.p.brief.rhetorical_force')==95 check('Native Polemic thesis and rhetorical force controls save correctly') page.screenshot(path=str(out/'polemic.png'),full_page=True) + if args.voices: + from voice_ui_checks import exercise + report['voice_library']=exercise(page,check,out,native=True) + responsive(hwnd) page.locator('[data-action="settings"]').click();page.wait_for_selector('#s-provider') expect(page.locator('#s-reasoning_effort')).to_have_count(1) page.get_by_role('button',name='Close dialog',exact=True).click();responsive(hwnd) diff --git a/scripts/ui_voices_smoke.py b/scripts/ui_voices_smoke.py new file mode 100644 index 0000000..b7a5d89 --- /dev/null +++ b/scripts/ui_voices_smoke.py @@ -0,0 +1,44 @@ +"""Real HTTP graphical voice-library workflow. No model calls.""" +from pathlib import Path +import argparse,json,socket,sys,tempfile,threading,time,traceback +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT)) +from graphpaper.server import create_app +from playwright.sync_api import sync_playwright +from voice_ui_checks import exercise +import uvicorn + + +def main(): + parser=argparse.ArgumentParser();parser.add_argument('--out',type=Path,default=ROOT/'test-results/voices-ui') + out=parser.parse_args().out;out.mkdir(parents=True,exist_ok=True) + report={'checks':[],'page_errors':[],'live_models':False,'ok':False} + def check(label):report['checks'].append(label);print('PASS',label,flush=True) + with tempfile.TemporaryDirectory(prefix='graphpaper-voices-') as temp: + app=create_app(Path(temp));sock=socket.socket();sock.bind(('127.0.0.1',0));port=sock.getsockname()[1] + server=uvicorn.Server(uvicorn.Config(app,log_level='error')) + thread=threading.Thread(target=lambda:server.run(sockets=[sock]),daemon=True);thread.start() + for _ in range(300): + if server.started:break + time.sleep(.05) + assert server.started + with sync_playwright() as pw: + browser=pw.chromium.launch(headless=True);page=browser.new_page(viewport={'width':1440,'height':1000},accept_downloads=True) + page.set_default_timeout(20000);page.on('pageerror',lambda e:report['page_errors'].append(str(e))) + try: + page.goto(f'http://127.0.0.1:{port}/');page.wait_for_selector('.welcome') + page.get_by_role('button',name='New project',exact=True).click();page.locator('#project-title').fill('First isolated article') + page.get_by_role('button',name='Create project',exact=True).click();page.wait_for_selector('#dropzone') + report['library']=exercise(page,check,out) + page.set_viewport_size({'width':1000,'height':800});page.wait_for_timeout(100) + assert page.evaluate('() => document.documentElement.scrollWidth')<=1000 + assert not report['page_errors'],report['page_errors'] + report['ok']=True + except Exception: + report['error']=traceback.format_exc();page.screenshot(path=str(out/'failure.png'));raise + finally: + (out/'report.json').write_text(json.dumps(report,indent=2)+'\n',encoding='utf-8') + browser.close();server.should_exit=True;thread.join(timeout=5);sock.close() + app.state.runner.pool.shutdown(wait=True,cancel_futures=True) + print(json.dumps(report,indent=2)) + +if __name__=='__main__':main() diff --git a/scripts/voice_ui_checks.py b/scripts/voice_ui_checks.py new file mode 100644 index 0000000..f872c8c --- /dev/null +++ b/scripts/voice_ui_checks.py @@ -0,0 +1,51 @@ +"""Reusable real-interface checks. Synthetic voice; no paid model or private material.""" +from playwright.sync_api import expect + + +def exercise(page,check,out,native=False): + original_id=page.evaluate('() => state.p.id') + if page.locator('.step[data-tab="sources"]').count():page.locator('.step[data-tab="sources"]').click() + page.get_by_role('button',name='Your writing voice',exact=True).click() + page.locator('#voice-name').fill('Power and wit') + page.locator('#voice-instructions').fill('Keep the fucking point. Use satire and anger when requested. End on the verdict. The current author brief overrides these defaults.') + page.get_by_role('button',name='Save as reusable voice',exact=True).click() + page.wait_for_selector('.voice-graph') + expect(page.locator('#saved-voice-name')).to_have_value('Power and wit') + assert 'fucking' in page.evaluate('() => JSON.stringify(voicesUI.current.graph)') + vid=page.evaluate('() => voicesUI.current.id') + check('An existing voice is saved as a named graph without a model call or tone filter') + before=page.evaluate('() => ({sources:state.p.sources,graph:state.p.graph,draft:state.p.draft})') + page.get_by_role('button',name='Use in this project',exact=True).click() + expect(page.locator('.modal')).to_have_count(0) + after=page.evaluate('() => ({sources:state.p.sources,graph:state.p.graph,draft:state.p.draft})') + assert after==before + assert page.evaluate('() => state.p.voice_profile.library_id')==vid + check('Applying a graph preserves the project evidence, manuscript and graph') + page.get_by_role('button',name='New project',exact=True).click() + page.locator('#project-title').fill('Another isolated article') + page.get_by_role('button',name='Create project',exact=True).click() + page.wait_for_selector('#dropzone') + page.get_by_role('button',name='Your writing voice',exact=True).click() + page.get_by_role('button',name='Choose a saved voice',exact=True).click() + page.locator('.voice-item').filter(has_text='Power and wit').click() + page.wait_for_selector('.voice-graph') + page.screenshot(path=str(out/'reusable-voice.png')) + if not native: + with page.expect_download() as download:page.get_by_role('button',name='Export voice',exact=True).click() + download.value.save_as(str(out/'synthetic-voice.json')) + page.get_by_role('button',name='Use in this project',exact=True).click() + expect(page.locator('.modal')).to_have_count(0) + assert page.evaluate('() => state.p.voice_profile.library_id')==vid + assert page.evaluate('() => state.p.sources.length')==0 + assert page.evaluate('() => state.p.usage.calls')==0 + check('A second project reuses the voice without importing training pieces or spending tokens') + page.get_by_role('button',name='Your writing voice',exact=True).click() + page.get_by_role('button',name='View graph overlay',exact=True).click() + expect(page.locator('svg[aria-label="Voice and evidence overlay"]')).to_be_visible() + page.get_by_role('button',name='Close dialog',exact=True).click() + check('Namespaced style/project overlay renders without changing stored evidence') + if native: + # The caller's remaining native save/close assertions refer to this project. + page.evaluate('async id => await openProject(id)',original_id) + assert page.evaluate('() => state.p.id')==original_id + return {'model_calls':0,'raw_samples_copied':0,'power_language_preserved':True} diff --git a/tests/test_jev_budget.py b/tests/test_jev_budget.py new file mode 100644 index 0000000..5b19927 --- /dev/null +++ b/tests/test_jev_budget.py @@ -0,0 +1,103 @@ +import copy +import json +import httpx +import pytest +from graphpaper.jev_budget import plan,compact_state,wire_bytes,intern,MAX_BYTES +from graphpaper.providers import Clients,ProviderError +from graphpaper.models import Settings +from graphpaper.pipeline import Job +from graphpaper.secrets import Vault + + +def questions(n=4): + return {f'c{i}_fit':{'type':'noul','instructions':f'Evaluate candidate candidate_{i} only.'} for i in range(n)} + + +def test_raw_voice_pieces_are_not_in_decisions(): + original={'author_voice':{'enabled':True,'instructions':'Use strong language. Keep the sarcasm.','samples':[{'excerpts':['PRIVATE_TRAINING_ARTICLE'*20000]}]},'brief':{'direction':'Preserve my forceful thesis'},'candidates':[{'id':'candidate_0','description':'An argument'}]} + before=copy.deepcopy(original) + batches=plan(original,questions(1),'jev-latest') + assert 'PRIVATE_TRAINING_ARTICLE' not in json.dumps([b.state for b in batches]) + assert batches[0].state['author_voice']['graph']['nodes'] + assert 'strong language' in json.dumps(batches[0].state) + assert original==before + + +def test_unicode_candidate_batches_keep_keys_ids_and_complete_text(): + candidates=[{'id':f'candidate_{i}','description':('猫'+str(i))*3500} for i in range(8)] + qs=questions(8) + batches=plan({'candidates':candidates,'editorial_contract':{'thesis':'A categorical moral judgment'}},qs,'jev-latest') + assert len(batches)>1 + recovered={} + for b in batches: + assert wire_bytes({'model':'jev-latest','state':b.state,'questions':b.questions})<=MAX_BYTES + for k in b.questions: + index=int(k.split('_')[0][1:]);match=next(x for x in b.state['candidates'] if x['id']==f'candidate_{index}') + assert match==candidates[index];recovered[k]=True + assert set(recovered)==set(qs) + + +def test_question_count_and_serialized_envelope_are_bounded(): + qs={f'q{i}':{'type':'noul','instructions':'Read the same supplied material.'} for i in range(225)} + batches=plan({'text':'material'},qs,'jev-latest') + assert len(batches)==3 + assert sum(len(b.questions) for b in batches)==225 + assert all(len(b.questions)<=100 for b in batches) + + +def test_repeated_text_uses_lossless_references_not_excerpts(): + value='Do not remove the negation. '*3000 + original={'a':value,'b':value,'c':value} + packed=intern(original) + assert len(packed['texts'])==1 + assert packed['texts'][packed['state']['a']['text_ref']]==value + assert original['a']==value + + +@pytest.mark.parametrize('route',['typesafe','openrouter']) +def test_actual_transport_combines_typed_answers_and_records_batch_cost(tmp_path,route): + vault=Vault(tmp_path);vault.set(route,'synthetic-key',False);seen=[] + def handler(request): + data=json.loads(request.content);seen.append(data) + assert len(request.content)<=MAX_BYTES + return httpx.Response(200,json={'answers':{k:{'type':'noul','noul':.81} for k in data['questions']},'usage':{'input_tokens':10,'output_tokens':3,'cost':.0001}}) + job=Job('p','angles') + c=Clients(Settings(model='fixture',allow_cloud=True,jev_provider=route,max_calls=20),vault,job,httpx.MockTransport(handler)) + state={'candidates':[{'id':f'candidate_{i}','description':str(i)*14000} for i in range(8)]} + answers=c.decide(state,questions(8)) + assert len(answers)==8 and len(seen)>1 + assert job.usage['calls']==len(seen) + assert all(r['request_bytes']<=MAX_BYTES for r in job.receipts) + assert all(r['decision_batches']==len(seen) for r in job.receipts) + assert job.usage['reported_cost']==pytest.approx(.0001*len(seen)) + + +def test_indivisible_oversize_preserves_work_and_makes_no_request(tmp_path): + vault=Vault(tmp_path);vault.set('typesafe','synthetic-key',False) + called=[];job=Job('p','review') + c=Clients(Settings(model='fixture',allow_cloud=True,jev_provider='typesafe'),vault,job,httpx.MockTransport(lambda r:called.append(r))) + assert c.decide({'draft':'Unique manuscript '*10000},{'prefer_revision':{'type':'noul','instructions':'Compare the whole draft'}}) is None + assert not called and job.usage['calls']==0 + assert 'unscored' in job.messages[-1]['text'] + + +def test_budget_preflight_does_not_spend_a_partial_batch(tmp_path): + vault=Vault(tmp_path);vault.set('typesafe','synthetic-key',False) + job=Job('p','angles');job.usage['calls']=4 + c=Clients(Settings(model='x',allow_cloud=True,jev_provider='typesafe',max_calls=5),vault,job) + assert c.decide({'candidates':[{'id':f'candidate_{i}','text':str(i)*30000} for i in range(4)]},questions()) is None + assert job.usage['calls']==4 + + +def test_off_remains_off_and_fabricated_answers_are_rejected(tmp_path): + vault=Vault(tmp_path) + assert Clients(Settings(jev_provider='off'),vault).decide('x',{}) is None + vault.set('typesafe','x',False) + c=Clients(Settings(allow_cloud=True,jev_provider='typesafe'),vault,transport=httpx.MockTransport(lambda r:httpx.Response(200,json={'answers':{'q':{'type':'noul','noul':True}}}))) + with pytest.raises(ProviderError):c.decide('x',{'q':{'type':'noul','instructions':'Check'}}) + + +def test_choice_vocabulary_and_missing_answer_are_not_relaxed(tmp_path): + vault=Vault(tmp_path);vault.set('typesafe','x',False) + c=Clients(Settings(allow_cloud=True,jev_provider='typesafe'),vault,transport=httpx.MockTransport(lambda r:httpx.Response(200,json={'answers':{'q':{'type':'choice','choice':'invented'}}}))) + with pytest.raises(ProviderError):c.decide('x',{'q':{'type':'choice','instructions':'Choose','criteria':{'a':'A','b':'B'}}}) diff --git a/tests/test_voice_integration.py b/tests/test_voice_integration.py new file mode 100644 index 0000000..9cb11c0 --- /dev/null +++ b/tests/test_voice_integration.py @@ -0,0 +1,48 @@ +import copy +import json +from types import SimpleNamespace +from unittest.mock import Mock +import pytest +from graphpaper.models import Project,VoiceProfile,Source +from graphpaper.voice_library import VoiceLibrary +from graphpaper.storage import Store +from graphpaper.desktop import DesktopBridge + + +def test_native_voice_export_uses_existing_explicit_save_bridge(tmp_path,monkeypatch): + import sys + store=Store(tmp_path);library=VoiceLibrary(store) + v=library.create(VoiceProfile(name='Power voice',instructions='Keep the fucking point and comic timing.'), + [Source(id='S1',title='Private training',role='voice',text='SECRET_TRAINING_TEXT')]) + target=tmp_path/'voice.json' + monkeypatch.setitem(sys.modules,'webview',SimpleNamespace(FileDialog=SimpleNamespace(SAVE=1))) + bridge=DesktopBridge(store);bridge._window=Mock() + bridge._window.create_file_dialog.return_value=str(target) + assert bridge.save_export(v.id,'voice')=={'saved':True} + value=json.loads(target.read_text()) + assert value['format']=='graphpaper-voice-1' + assert 'fucking' in target.read_text() + assert 'SECRET_TRAINING_TEXT' not in target.read_text() + assert not value['profile']['sample_ids'] + assert set(name for name in dir(bridge) if callable(getattr(bridge,name)) and not name.startswith('_'))=={'notify_ready','request_close','cancel_close','save_export'} + + +def test_training_bounds_apply_before_creating_a_library_entry(tmp_path): + library=VoiceLibrary(Store(tmp_path)) + samples=[Source(id=f'S{i}',title='Training',role='voice',text='Some writing.') for i in range(25)] + with pytest.raises(ValueError,match='24'):library.create(samples=samples) + assert library.list()==[] + + +def test_manual_project_override_replaces_the_derived_graph(client,app): + p=Project(voice_profile=VoiceProfile(instructions='Understated by default.')) + app.state.store.create(p) + library=app.state.voices;v=library.create(p.voice_profile);library.apply(p,v);app.state.store.save(p,0) + p=app.state.store.get(p.id);before=copy.deepcopy(p.voice_profile.graph) + profile=p.voice_profile.model_dump();profile['instructions']='Use uncompromising power language for this piece.';profile['graph']={} + r=client.patch('/api/projects/'+p.id,json={'version':p.version,'voice_profile':profile}) + assert r.status_code==200 + from graphpaper.voice import voice_context + context=voice_context(app.state.store.get(p.id)) + assert 'power language' in str(context['graph']) + assert library.get(v.id)[0].profile.graph==before diff --git a/tests/test_voice_library.py b/tests/test_voice_library.py new file mode 100644 index 0000000..81f66ab --- /dev/null +++ b/tests/test_voice_library.py @@ -0,0 +1,128 @@ +import copy +import json +import time +import pytest +from graphpaper.models import Project,Source,VoiceProfile,Graph,Node,Edge,Settings +from graphpaper.style_graph import text_graph,view,overlay,for_profile,StyleGraph +from graphpaper.voice import voice_context,sample_pack +from graphpaper.editorial import context,contract +from graphpaper.voice_library import VoiceLibrary +from .conftest import ScriptedClients + + +def voice_project(): + return Project(title='First article',voice_profile=VoiceProfile(name='Unapologetic',instructions='Keep the sarcasm. Use fucking strong language when requested. Finish with the verdict.'),sources=[Source(id='S1',title='Evidence',text='Documented observation.'),Source(id='S2',title='My writing',text='TRAINING_TEXT_ONLY '+('Original prose with wit and rhythm. '*30),role='voice')],graph=Graph(nodes=[Node(id='voice',label='An evidence concept',kind='claim')]),draft='Current writing.') + + +def test_graph_roundtrip_and_power_language_are_not_sanitized(): + text='Use fucking strong language.\nKeep anger, ridicule and jokes.\nThe author can change the tone.' + graph=text_graph(text,'My voice') + assert StyleGraph.model_validate_json(graph.model_dump_json())==graph + assert 'fucking' in graph.model_dump_json() + assert all(e.target in {n.id for n in graph.nodes} for e in graph.edges) + + +def test_small_graph_view_is_edge_closed_without_clipping_habits(): + graph=text_graph('\n'.join('Habit '+str(i)+': '+str(i)*450 for i in range(24))) + result=view(graph,2500);ids={n['id'] for n in result['nodes']} + assert len(json.dumps(result,ensure_ascii=False,separators=(',',':')).encode())<=2500 + assert all(e['source'] in ids and e['target'] in ids for e in result['edges']) + assert all(n['instruction']==next(x.instruction for x in graph.nodes if x.id==n['id']) for n in result['nodes']) + assert not result['coverage']['complete'] + + +def test_save_apply_does_not_copy_training_or_mutate_other_projects(tmp_path): + from graphpaper.storage import Store + store=Store(tmp_path);library=VoiceLibrary(store);original=voice_project();store.create(original) + saved=library.create(original.voice_profile,[s for s in original.sources if s.role=='voice']) + another=Project(title='Second article');before=another.sources.copy();library.apply(another,saved) + assert another.sources==before and not another.voice_profile.sample_ids + result=voice_context(another) + assert not result['samples'] and result['graph']['nodes'] + assert 'TRAINING_TEXT_ONLY' not in json.dumps(result) + assert library.list()[0]['samples']==1 + assert store.get(original.id)==original + library.delete(saved.id,saved.version) + assert voice_context(another)==result + + +def test_overlay_namespaces_colliding_ids_and_does_not_change_evidence(): + p=voice_project();p.voice_profile.graph=for_profile(p.voice_profile).model_dump() + before=p.graph.model_dump();result=overlay(p) + ids={n['id'] for n in result['nodes']} + assert 'style:voice' in ids and 'evidence:voice' in ids + assert any(e['layer']=='overlay' for e in result['edges']) + assert p.graph.model_dump()==before + + +def test_graph_preferences_do_not_cap_current_article_force(): + p=voice_project();p.brief.rhetorical_force=100;p.brief.direction='Use forceful language and do not soften the conclusion.' + assert 'current author brief' in contract(p)['voice_precedence'].lower() + value=context(p,compact=True) + assert value['author_voice']['graph']['nodes'] + assert 'samples' not in value['author_voice'] + assert p.brief.direction in str(value) + + +def test_many_training_pieces_respect_the_actual_excerpt_budget(): + p=Project(sources=[Source(id=f'S{i}',title='Writing',text='word '*1000,role='voice') for i in range(24)]) + samples=sample_pack(p,1800) + assert sum(len(t) for s in samples for t in s['excerpts'])<=1800 + + +def test_library_api_reuse_export_and_conflict(client,app): + p=voice_project();app.state.store.create(p) + response=client.post('/api/voices',json={'project_id':p.id,'version':0,'name':'My voice'}) + assert response.status_code==200,response.text + v=response.json();assert v['graph']['nodes'] + other=client.post('/api/projects',json={'title':'Second'}).json() + applied=client.post('/api/projects/'+other['id']+'/voice/apply',json={'version':0,'voice_id':v['id']}) + assert applied.status_code==200 and not applied.json()['sources'] + assert applied.json()['voice_profile']['library_id']==v['id'] + assert client.post('/api/projects/'+other['id']+'/voice/apply',json={'version':0,'voice_id':v['id']}).status_code==409 + exported=client.get('/api/voices/'+v['id']+'/export').json() + assert 'TRAINING_TEXT_ONLY' not in json.dumps(exported) + restored=client.post('/api/voices/import',json=exported) + assert restored.status_code==200 and restored.json()['id']!=v['id'] + assert not restored.json()['samples'] + assert app.state.store.get(p.id)==p + + +def test_new_training_is_not_project_evidence_and_learns_once(client,app): + class VoiceClients(ScriptedClients): + def complete(self,system,user,**kwargs): + data=json.loads(user) + if data.get('task','').startswith('Build an editable author voice profile'): + self._record('extraction') + assert 'samples' in data and 'Original writing' in str(data['samples']) + return {'name':'Reusable fixture','instructions':'Preserve incisive phrasing, varied rhythm and the author\'s forceful conclusion.','observations':['Synthetic material only.']} + return super().complete(system,user,**kwargs) + app.state.runner.clients_factory=VoiceClients + app.state.store.set_settings(Settings(model='fixture',allow_cloud=True).model_dump()) + v=client.post('/api/voices',json={'name':'Reusable'}).json() + v=client.post('/api/voices/'+v['id']+'/samples/text',json={'title':'Training','text':'Original writing with rhythm and wit. '*40}).json() + assert not client.get('/api/bootstrap').json()['projects'] + j=client.post('/api/voices/'+v['id']+'/learn',json={'version':v['version']}).json() + for _ in range(200): + status=client.get('/api/jobs/'+j['id']).json() + if status['state'] not in {'queued','running'}:break + time.sleep(.02) + assert status['state']=='completed',status.get('error',status) + saved=client.get('/api/voices/'+v['id']).json() + assert saved['graph']['nodes'] and status['usage']['calls']==1 + assert not client.get('/api/bootstrap').json()['projects'] + + +def test_invalid_graph_and_outdated_library_save_fail_without_mutation(client): + v=client.post('/api/voices',json={'instructions':'My direct style.'}).json() + assert client.patch('/api/voices/'+v['id'],json={'version':999,'name':'Overwrite'}).status_code==409 + invalid={'nodes':[{'id':'a','label':'A'}],'edges':[{'source':'a','target':'missing'}]} + assert client.patch('/api/voices/'+v['id'],json={'version':v['version'],'graph':invalid}).status_code==400 + assert client.get('/api/voices/'+v['id']).json()['profile']==v['profile'] + + +def test_existing_project_schema_still_loads_and_voice_graph_is_optional(): + p=Project.model_validate({'title':'Old work','voice_profile':{'instructions':'A direct style.'}}) + assert not p.voice_profile.library_id + assert for_profile(p.voice_profile).nodes + assert p.sources==[] diff --git a/ui/app.js b/ui/app.js index 236c3ed..1f6dbb4 100644 --- a/ui/app.js +++ b/ui/app.js @@ -65,7 +65,7 @@ function progressDone(t){const p=state.p;return t==='sources'?p.sources.length>0 function render(){ const p=state.p,s=state.settings; document.documentElement.dataset.theme=s.theme||'dark'; - $('#app').innerHTML=`
${''}
${icon('check')}${esc(state.save)}${p?ibtn('Project options','project-options','settings'):''}${btn(p?'Export':'Import project',p?'export':'import-project',p?'download':'upload','small')}
${p?projectHTML():welcomeHTML()}
`; + $('#app').innerHTML=`
${''}
${icon('check')}${esc(state.save)}${p?ibtn('Project options','project-options','settings'):''}${btn(p?'Export':'Import project',p?'export':'import-project',p?'download':'upload','small')}
${p?projectHTML():welcomeHTML()}
`; if(p&&state.tab==='graph'&&p.graph.nodes.length)requestAnimationFrame(drawGraph); setupDrops();paintJob(); } diff --git a/ui/index.html b/ui/index.html index 4820166..6e13acb 100644 --- a/ui/index.html +++ b/ui/index.html @@ -11,6 +11,8 @@ + +

Opening your writing studio…

diff --git a/ui/studio.js b/ui/studio.js index 373866d..12f5ea7 100644 --- a/ui/studio.js +++ b/ui/studio.js @@ -25,7 +25,7 @@ handleAction=async function(el){const action=el.dataset.action;if(!action.starts case 'studio-discover':el.disabled=true;try{$('#modal-status').textContent='Finding linked articles…';const r=await api('/voice/discover',{url:$('#author-url').value});studio.discovery=r.pages;$('#author-results').innerHTML=r.pages.map((p,i)=>``).join('');$('#modal-status').textContent=r.note;$('[data-action="studio-import-authors"]').disabled=false;}finally{el.disabled=false;}break; case 'studio-import-authors':{const pages=$$('.author-page:checked').map(x=>studio.discovery[Number(x.value)]);if(!pages.length||pages.length>12)throw new Error('Select between 1 and 12 writing samples.');el.disabled=true;await flush();const errors=[];try{for(let i=0;i${btn('My reusable voices','voices-library','graph','small')}Build a voice once. Select it in any project.`;}; +const voicesPriorDialog=voiceDialog; +voiceDialog=function(){voicesPriorDialog();$('.modal-body').insertAdjacentHTML('afterbegin',`
${esc(state.p.voice_profile.library_id?'Saved voice: '+state.p.voice_profile.name:'Keep a voice, not a pile of articles')}

Saved voice graphs work across projects without retraining. The current brief can override any habit, including force and vocabulary.

${btn('Choose a saved voice','voices-library','graph','small')}${btn('Save as reusable voice','voices-save-project','plus','primary small')}${state.p.voice_profile.library_id?btn('View graph overlay','voices-overlay','branch','small'):''}
`);}; +const voicesPriorSources=sourcesHTML; +sourcesHTML=function(){const p=state.p,original=p.sources,count=original.filter(s=>s.role==='voice').length;try{p.sources=original.filter(s=>s.role!=='voice');return (count?`
${count} legacy voice sample(s) kept outside this evidence view

No originals were deleted. Save the learned voice once and reuse its graph.

${btn('Manage writing voice','studio-voice','graph','small')}
`:'')+voicesPriorSources();}finally{p.sources=original;}}; +function voiceSVG(graph,overlayMode=false){const nodes=graph.nodes||[],edges=graph.edges||[];if(!nodes.length)return '

No style graph yet. Add a habit, paste instructions or learn from training pieces.

';const width=680,height=420,positions={};nodes.forEach((n,i)=>{const angle=2*Math.PI*i/Math.max(1,nodes.length);positions[n.id]=[width/2+Math.cos(angle)*250,height/2+Math.sin(angle)*145];});return `${edges.map(e=>{const a=positions[e.source],b=positions[e.target];return a&&b?`${esc(e.relation)}`:'';}).join('')}${nodes.map(n=>{const [x,y]=positions[n.id];return `${esc(n.label.length>24?n.label.slice(0,22)+'…':n.label)}${esc(n.instruction||n.label)}`;}).join('')}`;} +async function showVoiceLibrary(id=null){if(state.p)await flush();voicesUI.list=await api('/voices');if(id)voicesUI.current=await api('/voices/'+id);else voicesUI.current=null;paintVoices();} +function paintVoices(){const v=voicesUI.current;showModal('My voices','Reusable graphs of how you write. Learn once; apply wherever you choose.',`
${v?voiceEditor(v):`

Create once, reuse.

Build a mini graph from your writing or your own style directions. Training articles stay here; applying a voice copies only its compact graph.

${state.p?btn('Save this project’s voice','voices-save-project','plus','primary mt20'):''}`}
`,v?btn('Save graph','voices-save','check','primary small')+(state.p?btn('Use in this project','voices-apply','arrow','small'):'')+btn('Export voice','voices-export','download','small'):'',true);} +function voiceEditor(v){const g=v.graph,n=g.nodes.find(x=>x.id===voicesUI.node)||g.nodes[0];voicesUI.node=n?.id||null;return `
${voiceSVG(g)}
${btn('Add writing habit','voices-add-node','plus','small')}${n?btn('Remove selected habit','voices-remove-node','trash','small ghost'):''}
${n?`
`:''}
Build graph from written style directions${btn('Convert directions to graph','voices-from-text','graph','small mt12')}
Training pieces (${v.samples.length}) · separate from article evidence
${btn('Upload writing','voices-upload','upload','small')}${btn('Paste writing','voices-paste','file','small')}${btn('Article URL','voices-url','link','small')}${btn('Learn / rebuild graph','voices-learn','spark','primary small')}

${v.needs_relearning?'Training changed; the previous saved graph remains usable until you explicitly relearn.':'Applying this voice needs no model call and no re-upload.'}

${v.samples.map(s=>`
${s.words} words${ibtn('Delete this training piece','voices-delete-sample','trash',`data-id="${esc(s.id)}"`)}
`).join('')}

Your current article instructions override saved style preferences. This graph is not a vocabulary filter. Updating the library does not silently rewrite voices already applied to other projects.

`;} +function collectVoiceNode(){const v=voicesUI.current;if(!v)return;if($('#saved-voice-name'))v.profile.name=$('#saved-voice-name').value;if($('#saved-voice-strength'))v.profile.strength=Number($('#saved-voice-strength').value);const n=v.graph.nodes.find(x=>x.id===voicesUI.node);if(n&&$('#voice-node-instruction')){n.label=$('#voice-node-label').value;n.instruction=$('#voice-node-instruction').value;n.weight=Number($('#voice-node-weight').value);}} +async function saveVoiceGraph(){collectVoiceNode();const v=voicesUI.current;voicesUI.current=await api('/voices/'+v.id,{version:v.version,name:v.profile.name,strength:v.profile.strength,graph:v.graph},'PATCH');voicesUI.list=await api('/voices');return voicesUI.current;} +async function voiceDownload(path,name){if(window.pywebview?.api?.save_export){const r=await window.pywebview.api.save_export(voicesUI.current.id,'voice');if(r.error)throw new Error(r.error);if(r.saved)toast('Voice graph exported.');return;}const r=await fetch('/api'+path,{headers:{'X-GraphPaper':'1'},credentials:'same-origin'});if(!r.ok)throw new Error('Voice export failed.');const url=URL.createObjectURL(await r.blob());const a=document.createElement('a');a.href=url;a.download=name;document.body.append(a);a.click();a.remove();setTimeout(()=>URL.revokeObjectURL(url),60000);} +async function pollVoiceJob(){const id=voicesUI.job;if(!id)return;try{const j=await api('/jobs/'+id);if($('#voices-job-status'))$('#voices-job-status').textContent=j.state+' · '+j.usage.calls+' model calls'+(j.error?' · '+j.error:'');if(['queued','running'].includes(j.state)){voicesUI.poll=setTimeout(pollVoiceJob,900);return;}voicesUI.job=null;if(j.state==='completed'){const vid=voicesUI.current?.id;await showVoiceLibrary(vid);toast('Voice graph learned and saved. Ready to reuse.');}else toast(j.error||'Voice learning did not complete.',true);}catch(e){voicesUI.job=null;toast(e.message,true);}} +const voicesPriorAction=handleAction; +handleAction=async function(el){const a=el.dataset.action;if(!a.startsWith('voices-'))return voicesPriorAction(el);switch(a){ +case 'voices-library':return showVoiceLibrary(); +case 'voices-open':voicesUI.node=null;return showVoiceLibrary(el.dataset.id); +case 'voices-new':{const v=await api('/voices',{name:'My new voice'});return showVoiceLibrary(v.id);} +case 'voices-save-project':{if(!state.p||!editable())return;await flush();if($('#voice-name')){const profile={...state.p.voice_profile,name:$('#voice-name').value,instructions:$('#voice-instructions').value,strength:Number($('#voice-strength').value),enabled:$('#voice-enabled').checked};if(profile.instructions!==state.p.voice_profile.instructions)profile.graph={};queueSave({voice_profile:profile});await flush();}const v=await api('/voices',{project_id:state.p.id,version:state.p.version});return showVoiceLibrary(v.id);} +case 'voices-node':collectVoiceNode();voicesUI.node=el.dataset.id;paintVoices();break; +case 'voices-add-node':collectVoiceNode();{const g=voicesUI.current.graph;if(g.nodes.length>=32)throw new Error('Use at most 32 compact writing habits.');const id=uid();g.nodes.push({id,label:'New writing habit',instruction:'',kind:'style',weight:1,targets:['all']});if(g.nodes.length>1)g.edges.push({source:g.nodes[0].id,target:id,relation:'expresses'});voicesUI.node=id;paintVoices();}break; +case 'voices-remove-node':collectVoiceNode();{const g=voicesUI.current.graph,id=voicesUI.node;g.nodes=g.nodes.filter(n=>n.id!==id);g.edges=g.edges.filter(e=>e.source!==id&&e.target!==id);voicesUI.node=null;paintVoices();}break; +case 'voices-save':await saveVoiceGraph();paintVoices();toast('Voice graph saved.');break; +case 'voices-from-text':{collectVoiceNode();const v=voicesUI.current;voicesUI.current=await api('/voices/'+v.id,{version:v.version,name:v.profile.name,instructions:$('#saved-voice-instructions').value},'PATCH');paintVoices();break;} +case 'voices-apply':{if(!state.p||!editable())return;const v=await saveVoiceGraph();await flush();state.p=await api('/projects/'+state.p.id+'/voice/apply',{version:state.p.version,voice_id:v.id});closeModal();syncSummary();render();toast('Voice applied. No training articles copied and no model call made.');break;} +case 'voices-export':await saveVoiceGraph();return voiceDownload('/voices/'+voicesUI.current.id+'/export','voice-graph.json'); +case 'voices-import':chooseFile('.json',false,async files=>{const file=files[0];if(file.size>100000)throw new Error('Choose a compact voice export, not the training corpus.');const v=await api('/voices/import',JSON.parse(await file.text()));await showVoiceLibrary(v.id);});break; +case 'voices-upload':{const vid=voicesUI.current.id;chooseFile('.pdf,.docx,.txt,.md,.html,.htm',true,async files=>{for(const f of files){const body=new FormData();body.append('file',f);await api('/voices/'+vid+'/samples/file',body);}await showVoiceLibrary(vid);});break;} +case 'voices-paste':case 'voices-url':showModal(a==='voices-url'?'Add writing from an article URL':'Add a writing sample','Training stays in the voice library, outside every project’s evidence.',``,btn('Add training piece',a==='voices-url'?'voices-submit-url':'voices-submit-text','plus','primary'));break; +case 'voices-submit-text':case 'voices-submit-url':{const vid=voicesUI.current.id;const value=$('#voice-sample-text').value;await api('/voices/'+vid+'/samples/'+(a==='voices-submit-url'?'url':'text'),a==='voices-submit-url'?{url:value}:{title:$('#voice-sample-title').value,text:value});return showVoiceLibrary(vid);} +case 'voices-learn':{const v=await saveVoiceGraph();const j=await api('/voices/'+v.id+'/learn',{version:v.version});voicesUI.job=j.id;paintVoices();pollVoiceJob();break;} +case 'voices-read-sample':{const s=await api('/voices/'+voicesUI.current.id+'/samples/'+el.dataset.id);showModal(s.title,'Voice training only.',`
${esc(s.text)}
`,btn('Back to voice','voices-open','back','small',`data-id="${esc(voicesUI.current.id)}"`),true);break;} +case 'voices-delete-sample':if(confirm('Delete this training piece from the saved voice? Applied voices and original project files remain intact.')){const v=voicesUI.current;await api('/voices/'+v.id+'/samples/'+el.dataset.id,{version:v.version},'DELETE');return showVoiceLibrary(v.id);}break; +case 'voices-delete':if(confirm('Delete this saved voice? Projects already using it keep their own compact copies.')){await api('/voices/'+voicesUI.current.id,{version:voicesUI.current.version},'DELETE');return showVoiceLibrary();}break; +case 'voices-overlay':{await flush();const g=await api('/projects/'+state.p.id+'/voice/overlay');showModal('Voice + project graph','Namespaced layers guide expression without altering the research graph.',voiceSVG(g,true)+`

${esc(g.scope.note)}

`,'',true);break;} +}}; +document.addEventListener('keydown',e=>{const node=e.target.closest?.('.voice-graph-node[data-action]');if(node&&['Enter',' '].includes(e.key)){e.preventDefault();handleAction(node).catch(err=>toast(err.message,true));}});