File size: 4,521 Bytes
8e02a28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | import json, math, sys, time
from pathlib import Path
import numpy as np
import torch
from PIL import Image
from plyfile import PlyData, PlyElement
sys.path.insert(0, '/home/dev/polymorphic/vendor/AnySplat')
from src.model.model.anysplat import AnySplat
from src.utils.image import process_image
ROOT = Path('/home/dev/polymorphic')
OUT = ROOT / 'fourier_scene_apt'
FRAMES = sorted((ROOT / 'exp_apt' / 'frames').glob('*.jpg'))
OUT.mkdir(parents=True, exist_ok=True)
(OUT / 'images').mkdir(exist_ok=True)
assert len(FRAMES) == 24, len(FRAMES)
device = 'cuda'
imgs = torch.stack([process_image(str(f)) for f in FRAMES]).unsqueeze(0).to(device)
model = AnySplat.from_pretrained('lhjiang/anysplat').to(device).eval()
for p in model.parameters(): p.requires_grad_(False)
ctx_i = list(range(0, len(FRAMES), 2))
tgt_i = list(range(1, len(FRAMES), 2))
with torch.inference_mode():
g_ctx, p_ctx = model.inference((imgs[:, ctx_i] + 1) * 0.5)
g_all, p_all = model.inference((imgs + 1) * 0.5)
# Align the all-frame predicted target cameras into the context-only reconstruction frame,
# matching the held-out AnySplat evaluation protocol.
src = p_all['extrinsic'][0, ctx_i, :3, 3].double().cpu().numpy()
dst = p_ctx['extrinsic'][0, :, :3, 3].double().cpu().numpy()
mu_s, mu_d = src.mean(0), dst.mean(0)
xs, xd = src-mu_s, dst-mu_d
cov = xd.T @ xs / len(src)
U,S,Vt = np.linalg.svd(cov)
D = np.eye(3); D[2,2] = np.sign(np.linalg.det(U)*np.linalg.det(Vt))
R = U @ D @ Vt
scale = (S*np.diag(D)).sum() / (xs**2).sum(axis=1).mean()
trans = mu_d - scale * R @ mu_s
H, W = imgs.shape[-2:]
fovxs = []
def blender_matrix(c2w):
# FourierSplatting's Blender loader flips camera Y/Z. Pre-flip so that
# loading recovers the original AnySplat camera-to-world matrix.
return (c2w @ np.diag([1., -1., -1., 1.])).tolist()
train_frames, test_frames = [], []
for i, f in enumerate(FRAMES):
im = Image.open(f).convert('RGB')
w, h = im.size
side = min(w, h)
left, top = (w-side)//2, (h-side)//2
im = im.crop((left, top, left+side, top+side)).resize((W,H), Image.Resampling.LANCZOS)
name = f'{i+1:04d}.png'
im.save(OUT / 'images' / name)
if i in ctx_i:
j = ctx_i.index(i)
c2w = p_ctx['extrinsic'][0,j].float().cpu().numpy()
intr = p_ctx['intrinsic'][0,j].float().cpu().numpy()
rec = {'file_path':f'images/{name[:-4]}','transform_matrix':blender_matrix(c2w),'index':i}
train_frames.append(rec)
else:
c2w0 = p_all['extrinsic'][0,i].float().cpu().numpy()
c2w = np.eye(4, dtype=np.float32)
c2w[:3,:3] = R @ c2w0[:3,:3]
c2w[:3,3] = scale * R @ c2w0[:3,3] + trans
intr = p_all['intrinsic'][0,i].float().cpu().numpy()
rec = {'file_path':f'images/{name[:-4]}','transform_matrix':blender_matrix(c2w),'index':i}
test_frames.append(rec)
fovxs.append(2.0 * math.atan(1.0 / (2.0 * float(intr[0,0]))))
fovx = float(np.median(fovxs))
(OUT/'transforms_train.json').write_text(json.dumps({'camera_angle_x':fovx,'frames':train_frames},indent=2))
(OUT/'transforms_test.json').write_text(json.dumps({'camera_angle_x':fovx,'frames':test_frames},indent=2))
# Seed from context-only AnySplat primitives; no target-frame geometry enters initialization.
means = g_ctx.means[0].float()
scales = g_ctx.scales[0].float()
opacity = g_ctx.opacities[0].float().reshape(-1)
harm = g_ctx.harmonics[0].float()
score = opacity * scales.max(dim=1).values.clamp_min(1e-7)
n = min(120_000, len(score))
idx = torch.topk(score, n, sorted=False).indices
xyz = means[idx].cpu().numpy().astype(np.float32)
rgb = (0.5 + 0.28209479177387814 * harm[idx, :, 0]).clamp(0,1).cpu().numpy()
verts = np.empty(n,dtype=[('x','f4'),('y','f4'),('z','f4'),('nx','f4'),('ny','f4'),('nz','f4'),('red','u1'),('green','u1'),('blue','u1')])
verts['x'],verts['y'],verts['z'] = xyz.T
verts['nx'],verts['ny'],verts['nz'] = 0,0,0
verts['red'],verts['green'],verts['blue'] = (rgb*255).astype(np.uint8).T
PlyData([PlyElement.describe(verts,'vertex')]).write(str(OUT/'points3d.ply'))
meta={'source':'AnySplat context-only geometry initialization','total_video_frames':len(FRAMES),'context_views':len(train_frames),'heldout_views':len(test_frames),'seed_primitives':n,'image_hw':[H,W],'camera_fov_x':fovx,'pose_alignment_scale':float(scale),'pose_alignment_translation_norm':float(np.linalg.norm(trans)),'heldout_geometry_used_for_initialization':False}
(OUT/'seed_metadata.json').write_text(json.dumps(meta,indent=2))
print(json.dumps(meta,indent=2),flush=True)
|