Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions docs/anima-lora-parameters-sdxl.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,15 +97,31 @@ Anima 不像 SDXL 只选一个 ckpt,页面上有三项**必填**:
|------|------|
| `prefer_json_caption` | **建议开启**,优先读同名 `.json` 标签 |
| `caption_extension` | 没有 JSON 时,回退读 `.txt` |
| `shuffle_caption` | 训练时打乱标签顺序;JSON 模式下会分组打乱 |
| `shuffle_caption` | 训练时打乱标签顺序;JSON 标签解析后使用相同规则 |
| `keep_tokens` | 打乱时保留前几个词不动(触发词建议放前面) |

JSON 标签推荐字段顺序(表单里也有提示):
JSON 标签按以下固定字段顺序展开(表单里也有提示):

```text
quality / count / character / series / artist / appearance[] / tags[] / environment[] / nl
```

例如 `image1.png` 对应 `image1.json`:

```json
{
"quality": "masterpiece",
"count": 1,
"character": "character name",
"appearance": ["blue eyes", "red dress"],
"tags": ["smile"],
"environment": ["indoors"],
"nl": "a natural language description"
}
```

字段可写字符串、数字或对应数组,空值会跳过。JSON 没有可用内容时回退读取文本标签;JSON 语法错误或字段类型不受支持时会明确报错,不会静默按无标签训练。

你以前用「一行逗号 tag」完全可以继续用;如果愿意,把角色名、外观、环境分开写进 JSON,Anima 往往吃得更好。触发词放在 `character` 或 `tags` 前面,并配合 `keep_tokens`,效果更稳。

**注意**:如果开了「缓存文本编码器输出」(`cache_text_encoder_outputs`),需要**关掉** `shuffle_caption`,否则两者会打架。
Expand Down
100 changes: 89 additions & 11 deletions mikazuki/engines/kohya/extension_state.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,27 @@
import json
import hashlib
import json
import os
import threading
import time
import uuid

from .environment import TRAINING_DEPS, TORCH_PINS, TORCH_PINS_AARCH64, audit_python, compute_drift
from .environment import (
TORCH_PINS,
TORCH_PINS_AARCH64,
TRAINING_DEPS,
audit_python,
compute_drift,
)
from .settings import main_env_python_candidates

FINGERPRINT_SCHEME = "content-v1"
FINGERPRINT_MAX_HASH_BYTES = 8 * 1024 * 1024
_recovery_lock = threading.Lock()


def write_state(runtime, state, facts=None, reason=""):
runtime.root.mkdir(parents=True, exist_ok=True)
temporary = runtime.state_file.with_suffix(".tmp")
temporary = runtime.state_file.with_suffix(f".{uuid.uuid4().hex}.tmp")
temporary.write_text(json.dumps({"state": state, "facts": facts or {}, "reason": reason}, ensure_ascii=False, indent=2), encoding="utf-8")
temporary.replace(runtime.state_file)

Expand All @@ -18,17 +31,32 @@ def fingerprint(runtime):
# only the venv content and the pinned dependency set gate readiness.
files = [runtime.python]
files += sorted((runtime.root / ".venv").glob("**/*.dist-info/METADATA"))
stats = []
entries = []
for path in files:
if not path.is_file():
if not path.is_file() and not path.is_symlink():
continue
try:
identity = path.relative_to(runtime.root).as_posix()
except ValueError:
identity = path.name
stats.append((identity, path.stat().st_size, path.stat().st_mtime_ns))
if path.is_symlink():
entries.append((identity, path.lstat().st_size, "symlink", os.readlink(path)))
continue
stat = path.stat()
if stat.st_size > FINGERPRINT_MAX_HASH_BYTES:
entries.append((identity, stat.st_size, "size", ""))
continue
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
entries.append((identity, stat.st_size, "sha256", digest.hexdigest()))
pins = [*TRAINING_DEPS, *TORCH_PINS, *TORCH_PINS_AARCH64]
return hashlib.sha256(json.dumps([pins, stats]).encode()).hexdigest()
return hashlib.sha256(json.dumps([FINGERPRINT_SCHEME, pins, entries]).encode()).hexdigest()


def fingerprint_facts(runtime):
return {"fingerprint": fingerprint(runtime), "fingerprint_scheme": FINGERPRINT_SCHEME}


# Main-env audits spawn a python subprocess importing torch (seconds), so
Expand Down Expand Up @@ -155,7 +183,7 @@ def _main_env_upgrade_status(runtime):
global _ACTIVE_MAIN_ENV_UPGRADE
if not _ACTIVE_MAIN_ENV_UPGRADE:
return None
from mikazuki.tasks import tm, TaskStatus
from mikazuki.tasks import TaskStatus, tm
task = tm.tasks.get(_ACTIVE_MAIN_ENV_UPGRADE["task_id"])
if task is not None and task.status not in {TaskStatus.FAILED, TaskStatus.TERMINATED, TaskStatus.FINISHED}:
return {
Expand All @@ -170,19 +198,69 @@ def _main_env_upgrade_status(runtime):
return None


def read_status(runtime):
def _recover_missing_state(runtime, audit_fn):
if not any(path.exists() for path in (runtime.root / ".venv", runtime.python_install_dir)):
return {"state": "not_installed", "facts": {}}
if not runtime.python.is_file():
return {"state": "broken", "facts": {}, "reason": "独立 Python 缺失,请修复环境。"}
audit = audit_fn(runtime)
facts = {"recovery": "missing_install_state", "audit": audit}
if not audit.get("ok"):
reason = "; ".join(audit.get("errors") or ["Kohya 环境审计失败。"])
write_state(runtime, "broken", facts, reason)
return {"state": "broken", "facts": facts, "reason": reason}
facts.update(fingerprint_facts(runtime))
reason = "状态文件缺失,已根据实体环境自动恢复。"
write_state(runtime, "ready", facts, reason)
return {"state": "ready", "facts": facts, "reason": reason}


def _migrate_legacy_state(runtime, data, audit_fn):
with _recovery_lock:
if runtime.state_file.exists():
latest = json.loads(runtime.state_file.read_text(encoding="utf-8"))
if latest.get("state") != "ready" or latest.get("facts", {}).get("fingerprint_scheme") == FINGERPRINT_SCHEME:
return latest
data = latest
from .environment import audit_environment
audit = (audit_fn or audit_environment)(runtime)
data["facts"]["audit"] = audit
if not audit.get("ok"):
data.update(state="broken", reason="; ".join(audit.get("errors") or ["Kohya 环境审计失败。"]))
else:
data["facts"].update(fingerprint_facts(runtime))
write_state(runtime, "ready", data["facts"], "旧版环境状态已通过审计并迁移。")
return data


def read_status(runtime, *, audit_fn=None):
upgrading = _main_env_upgrade_status(runtime)
if upgrading is not None:
return upgrading
data = json.loads(runtime.state_file.read_text(encoding="utf-8")) if runtime.state_file.exists() else {"state": "not_installed", "facts": {}}
if runtime.state_file.exists():
data = json.loads(runtime.state_file.read_text(encoding="utf-8"))
else:
with _recovery_lock:
if runtime.state_file.exists():
data = json.loads(runtime.state_file.read_text(encoding="utf-8"))
else:
from .environment import audit_environment
data = _recover_missing_state(runtime, audit_fn or audit_environment)
if data["state"] in {"installing", "auditing"}:
from mikazuki.tasks import tm, TaskStatus
from mikazuki.tasks import TaskStatus, tm
task = tm.tasks.get(data["facts"].get("task_id"))
if task is None or task.status in {TaskStatus.FAILED, TaskStatus.TERMINATED, TaskStatus.FINISHED}:
invalidate_main_env_audit_cache()
data.update(state="broken", reason="安装中断,请修复 kohya 环境。")
if data["state"] == "ready" and not runtime.python.is_file():
data.update(state="broken", reason="独立 Python 缺失,请修复环境。")
if data["state"] == "ready":
if data["facts"].get("fingerprint_scheme") != FINGERPRINT_SCHEME:
data = _migrate_legacy_state(runtime, data, audit_fn)
if data["state"] == "ready" and data["facts"].get("fingerprint") != fingerprint(runtime):
data.update(state="broken", reason="环境已变化,请修复后重新检查。")
elif data["facts"].get("fingerprint") != fingerprint(runtime):
data.update(state="broken", reason="环境已变化,请修复后重新检查。")
if data["state"] == "ready":
# Pin drift never blocks: the env was audited ready at install time,
# so a mismatch with the current pins only means an upgrade is advised.
Expand Down
4 changes: 2 additions & 2 deletions mikazuki/engines/kohya/install_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
from mikazuki.engines.kohya.environment import install_env, audit_environment, audit_python
from mikazuki.engines.kohya.extension_state import write_state, fingerprint
from mikazuki.engines.kohya.extension_state import write_state, fingerprint_facts
from mikazuki.engines.kohya.resource import environment_lock
from mikazuki.engines.kohya.settings import Runtime

Expand Down Expand Up @@ -60,7 +60,7 @@ def main():
print(json.dumps(audit, ensure_ascii=False), flush=True)
if not audit['ok']:
raise RuntimeError('; '.join(audit['errors']))
write_state(rt, 'ready', {**facts, 'audit': audit, 'fingerprint': fingerprint(rt)})
write_state(rt, 'ready', {**facts, 'audit': audit, **fingerprint_facts(rt)})
print('[mikazuki-progress] ' + json.dumps({'percent': 100, 'message': '环境检查完成'}, ensure_ascii=False), flush=True)
except Exception as exc:
if target == 'standalone':
Expand Down
2 changes: 1 addition & 1 deletion mikazuki/schema/sd3-lora.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ Schema.intersect([
Schema.intersect([
Schema.object(UpdateSchema(SHARED_SCHEMAS.RAW.CAPTION_SETTINGS, {
caption_extension: Schema.string().default(".txt").description("回退读取的 Tag 文件扩展名"),
shuffle_caption: Schema.boolean().default(false).description("训练时随机打乱 tokens;JSON 模式下会对 appearance / tags / environment 分组打乱"),
shuffle_caption: Schema.boolean().default(false).description("训练时随机打乱 tokens;JSON 标签解析后使用相同的打乱规则"),
keep_tokens: Schema.number().min(0).max(255).step(1).default(0).description("随机打乱时保留前 N 个 token 不变"),
caption_tag_dropout_rate: Schema.number().min(0).step(0.01).description("按标签随机丢弃 tag 的概率"),
prefer_json_caption: Schema.boolean().default(true).description("优先读取同名 JSON 标签文件;适合 Anima 结构化标签流程"),
Expand Down
12 changes: 12 additions & 0 deletions tests/test_anima_backend_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,18 @@ def test_adapter_keeps_supported_anima_fields(self):
self.assertNotIn("enable_preview", adapted)
self.assertEqual(warnings, [])

def test_adapter_keeps_json_caption_preference_for_lora_and_finetune(self):
for finetune in (False, True):
with self.subTest(finetune=finetune):
adapted, warnings = adapt_anima_config(
{"prefer_json_caption": True, "caption_extension": ".txt"},
finetune=finetune,
)

self.assertIs(adapted["prefer_json_caption"], True)
self.assertEqual(adapted["caption_extension"], ".txt")
self.assertEqual(warnings, [])

def test_adapter_warns_for_unsupported_debug_fields(self):
adapted, warnings = adapt_anima_config(
{
Expand Down
Loading
Loading