Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ Fixed cameras win only with exact poses or 0.5% translation noise. In the
other seven noisy conditions Naive + warm-up is best in six and 0.09 dB behind
CamP-style + warm-up in the seventh (2° rotation).

The recovered cameras show why. Joint optimization pulls the noisy training
cameras back to within 0.025 scene units on average (from 0.16); removing one
global similarity transform brings that to 0.016.

![Training-camera center errors before and after joint pose optimization, magnified 10x](docs/assets/camera_correction.png)

### 2. Most of the "drop from perfect poses" is coordinate drift

![Ground truth, fixed-camera, naive raw and naive aligned renders with per-pixel error maps](docs/assets/hero_drift.png)
Expand Down
4 changes: 4 additions & 0 deletions README.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ flowchart LR

固定相机只在位姿完全正确或只有 0.5% 平移扰动时最好。其余 7 种扰动条件里,Naive + warm-up 在 6 种中最好,剩下的 2° 旋转条件下只比 CamP-style + warm-up 低 0.09 dB。

从恢复出的相机位置可以看出原因:联合优化把带噪声的训练相机拉回到平均距 GT 0.025 个场景单位以内(原始扰动为 0.16);再去掉一个全局相似变换,误差降到 0.016。

![联合位姿优化前后的训练相机中心误差(放大 10 倍显示)](docs/assets/camera_correction.png)

### 2. "从正确位姿出发反而变差",主要是坐标漂移

![GT、固定相机、Naive 原坐标与 Naive 对齐后的渲染及逐像素误差图](docs/assets/hero_drift.png)
Expand Down
5 changes: 4 additions & 1 deletion docs/RUNNING.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,10 @@ With that directory available locally:
uv run python scripts/showcase/import_delivery.py path/to/delivery_20261014

# Rebuild every figure in docs/assets/ (the GIF needs ffmpeg on PATH).
uv run python scripts/showcase/make_figures.py --delivery path/to/delivery_20261014
# --gt-transforms is the public NeRF-Synthetic lego/transforms_train.json; the T5
# perturbation is regenerated from it with seed 42 and checked against the archive.
uv run python scripts/showcase/make_figures.py --delivery path/to/delivery_20261014 \
--gt-transforms path/to/nerf_synthetic/lego/transforms_train.json
```

## 8. Tests and help
Expand Down
Binary file added docs/assets/camera_correction.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
59 changes: 58 additions & 1 deletion scripts/showcase/make_figures.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,19 @@

Metric figures read artifacts/ only. Render figures and the GIF also need the
extracted delivery_<date>/ directory (800x800 stills, 400x400 video); pass it with
--delivery or they are skipped. The GIF needs ffmpeg on PATH.
--delivery or they are skipped. The GIF needs ffmpeg on PATH. The camera figure
needs the public NeRF-Synthetic lego/transforms_train.json (--gt-transforms).
"""
import argparse
import csv
import json
from pathlib import Path
import shutil
import subprocess
import sys

ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
ARTIFACTS = ROOT/'artifacts/analysis'
CONDITIONS = ['SV100-GT', 'SV100-T0.5', 'SV100-T1', 'SV100-T2', 'SV100-T5',
'SV100-R0.5', 'SV100-R1', 'SV100-R2', 'SV100-R5']
Expand Down Expand Up @@ -175,6 +178,57 @@ def perturbation_grid(plt, stills, output, view=175):
fig.savefig(output/'perturbation_grid.png', dpi=100); plt.close(fig)


def perturbed_poses(gt_transforms, condition_args):
"""Re-create a perturbed training split with the original script and seed."""
import tempfile
with tempfile.TemporaryDirectory() as directory:
source = Path(directory)/'lego'
source.mkdir()
shutil.copy2(gt_transforms, source/'transforms_train.json')
subprocess.run([sys.executable, str(ROOT/'scripts/data/perturb_poses.py'), '--src', str(source),
'--dst', str(Path(directory)/'noisy'), *condition_args, '--seed', '42'],
check=True, capture_output=True)
frames = json.loads((Path(directory)/'noisy/transforms_train.json').read_text())['frames']
return [frame['transform_matrix'] for frame in frames]


def camera_correction(plt, gt_transforms, output, magnify=10):
"""T5 training-camera centers: injected noise vs. what Naive + warm-up leaves after optimization."""
import numpy as np
from poseguard.alignment import fit_sim3, transform_poses
gt = np.array([f['transform_matrix'] for f in json.loads(Path(gt_transforms).read_text())['frames']])[:, :3]
noisy = np.array(perturbed_poses(gt_transforms, ['--translation-ratio', '0.05']))[:, :3]
record = json.loads((ROOT/'artifacts/experiments/warmup/naive/SV100-T5/optimized_poses.json').read_text())
optimized = np.array(record['poses'])[:, :3]
stored = record['initial_errors'].get('raw', record['initial_errors'])['translation_radius_pct_mean']
radius = np.linalg.norm(gt[:, :, 3], axis=1).mean()
error = lambda poses: np.linalg.norm(poses[:, :, 3]-gt[:, :, 3], axis=1).mean()
if abs(100*error(noisy)/radius-stored) > 1e-6:
raise ValueError('Regenerated perturbation does not match the archived run')
aligned = transform_poses(optimized, fit_sim3(optimized[:, :, 3], gt[:, :, 3]))
panels = [(noisy, '#52514e', 'Injected noise (5%% translation)\nmean camera-center error %.3f' % error(noisy)),
(optimized, BLUE, 'After Naive + warm-up, 30k steps\nmean error %.3f, %.3f after one global Sim(3)'
% (error(optimized), error(aligned)))]
fig = plt.figure(figsize=(11, 4.4))
for index, (poses, color, title) in enumerate(panels, 1):
ax = fig.add_subplot(1, 2, index, projection='3d')
centers, offsets = gt[:, :, 3], (poses[:, :, 3]-gt[:, :, 3])*magnify
for start, offset in zip(centers, offsets):
ax.plot(*np.stack((start, start+offset)).T, color=color, linewidth=1.6, solid_capstyle='round')
ax.scatter(*centers.T, s=12, color='#b5b4ae', depthshade=False)
ax.scatter([0], [0], [0], s=60, marker='s', color=INK)
ax.text(0, 0, -.7, 'scene', ha='center', fontsize=9, color=MUTED)
ax.set_title(title, fontsize=10.5, loc='left', y=1.0, pad=0)
ax.view_init(elev=24, azim=-60)
ax.set_box_aspect((1, 1, .55), zoom=1.2)
ax.set_xlim(-5, 5); ax.set_ylim(-5, 5); ax.set_zlim(-.8, 5.2)
ax.set_axis_off()
fig.subplots_adjust(left=0, right=1, bottom=-.08, top=.86, wspace=0)
fig.text(.5, .015, 'Lego training cameras (gray, %.2f units from the scene). Segments: camera-center error, '
'drawn ×%d.' % (radius, magnify), ha='center', fontsize=9.5, color=MUTED)
fig.savefig(output/'camera_correction.png', dpi=130); plt.close(fig)


def side_by_side_gif(delivery, output):
ffmpeg = shutil.which('ffmpeg')
if not ffmpeg:
Expand All @@ -188,6 +242,7 @@ def side_by_side_gif(delivery, output):
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--delivery', type=Path, help='extracted delivery_<date>/ directory (renders, videos)')
parser.add_argument('--gt-transforms', type=Path, help='NeRF-Synthetic lego/transforms_train.json')
parser.add_argument('--output', type=Path, default=ROOT/'docs/assets')
args = parser.parse_args()
import matplotlib
Expand All @@ -201,6 +256,8 @@ def main():
hero(plt, args.delivery/'stills', args.output)
perturbation_grid(plt, args.delivery/'stills', args.output)
side_by_side_gif(args.delivery, args.output)
if args.gt_transforms:
camera_correction(plt, args.gt_transforms, args.output)
for path in sorted(args.output.iterdir()):
print('%-28s %7.0f KB' % (path.name, path.stat().st_size/1024))

Expand Down
Loading