FAR release bundles

README.md

@@ -54,19 +54,24 @@ or `huggingface-cli download 1202kbs/FAR-Checkpoints --local-dir models`.
 | AI2-THOR-dyn | `ai2thor_dyn/far_multicue` | FAR -- Multi-Cue | 500k | 0.56 GB |
 | AI2-THOR-dyn | `ai2thor_dyn/temporal` | Temporal | 500k | 0.49 GB |
 | AI2-THOR-dyn | `ai2thor_dyn/worldmem` | WorldMem | 500k | 0.49 GB |
+| AI2-THOR-dyn | `ai2thor_dyn/retriever_object` | retriever (object cue), init of the FAR arms | 12.5k | 0.06 GB |
 | AI2-THOR v3 | `ai2thor_v3/far_multicue` | FAR -- Multi-Cue | 700k | 0.55 GB |
 | AI2-THOR v3 | `ai2thor_v3/temporal` | Temporal | 700k | 0.49 GB |
 | AI2-THOR v3 | `ai2thor_v3/worldmem` | WorldMem | 700k | 0.49 GB |
+| AI2-THOR v3 | `ai2thor_v3/retriever_jepa` | retriever (visual), init of the FAR arm | 400k | 0.06 GB |
 | LoopNav | `loopnav/far_meta` | FAR -- Meta | 700k | 0.55 GB |
 | LoopNav | `loopnav/far_multicue` | FAR -- Multi-Cue | 700k | 0.55 GB |
 | LoopNav | `loopnav/far_visual` | FAR -- Visual | 700k | 0.55 GB |
 | LoopNav | `loopnav/longlive_rag` | LongLive-RAG | 700k | 0.55 GB |
 | LoopNav | `loopnav/temporal` | Temporal | 700k | 0.49 GB |
 | LoopNav | `loopnav/worldmem` | WorldMem | 700k | 0.49 GB |
+| LoopNav | `loopnav/retriever_jepa` | retriever (visual), init of the FAR arms | 400k | 0.06 GB |
+| LoopNav | `loopnav/retriever_longlive` | retriever (content query), init of LongLive-RAG | 400k | 0.06 GB |
 | SoundSpaces v1 | `soundspaces_v1/far_meta` | FAR -- Meta | 700k | 0.55 GB |
 | SoundSpaces v1 | `soundspaces_v1/far_multicue` | FAR -- Multi-Cue | 700k | 0.55 GB |
 | SoundSpaces v1 | `soundspaces_v1/temporal` | Temporal | 700k | 0.49 GB |
 | SoundSpaces v1 | `soundspaces_v1/worldmem` | WorldMem | 700k | 0.49 GB |
+| SoundSpaces v1 | `soundspaces_v1/retriever_audio` | retriever (audio), init of the SoundSpaces v1 and v2 FAR arms | 400k | 0.06 GB |
 | SoundSpaces v2 | `soundspaces_v2/far_meta` | FAR -- Meta | 700k | 0.55 GB |
 | SoundSpaces v2 | `soundspaces_v2/far_multicue` | FAR -- Multi-Cue | 700k | 0.55 GB |
 | SoundSpaces v2 | `soundspaces_v2/temporal` | Temporal | 700k | 0.49 GB |
@@ -74,7 +79,9 @@ or `huggingface-cli download 1202kbs/FAR-Checkpoints --local-dir models`.
 
 "Temporal" and "WorldMem" are the recency and field-of-view retrieval
 baselines, "LongLive-RAG" the content-query retrieval baseline; the FAR arms
-differ in the cues the retriever fuses (metadata, visual, multi-cue).
+differ in the cues the retriever fuses (metadata, visual, multi-cue). The retriever bundles are
+the contrastively pre-trained encoders the FAR launchers start from; evaluation does not need
+them (each FAR bundle already carries its trained retriever).
 
 ## License
 

ai2thor_dyn/retriever_object/checkpoints/0012500.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d3d3e13be18d6e5d75bfb6ae59ef5e4965b11f48ae22133db917a99a44222f5
+size 66429235

ai2thor_dyn/retriever_object/config.yaml

@@ -0,0 +1,948 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+ai2thor_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor/ai2thor_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor/ai2thor_latent_train.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal}
+      transform: ${ai2thor_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor/ai2thor_latent_test.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal_test}
+      transform: ${ai2thor_latent.shared.transform}
+ai2thor_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_v2/ai2thor_v2_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal}
+      transform: ${ai2thor_v2_latent.shared.transform}
+      event_index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_test.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v2_latent.shared.transform}
+ai2thor_v3_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_v3/ai2thor_v3_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v3_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v3/ai2thor_v3_latent_train.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal}
+      transform: ${ai2thor_v3_latent.shared.transform}
+      event_index_path: results/indices/ai2thor_v3/ai2thor_v3_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v3/ai2thor_v3_latent_test.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v3_latent.shared.transform}
+ai2thor_dyn_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_dyn/ai2thor_dyn_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: .agent.npy
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_dyn_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_dyn/ai2thor_dyn_latent_train.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+      event_index_path: null
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_dyn/ai2thor_dyn_latent_test.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal_test}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+model:
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 4
+    softmax_temperature: 1.0
+    scoring: visual
+    metadata_hidden: 128
+    metadata_layers: 2
+    local_pos_stats: null
+    stride_to_seconds: 1.0
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_source: frame
+    query_conditioning: action
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 32
+      img_width: 32
+      patch_size: 2
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+      interact_dim: 4
+wandb:
+  project: PMem-THOR
+  name: null
+  entity: null
+  mode: online
+  group: ai2thor_dyn_latent
+  tags:
+  - ai2thor_dyn_latent
+  - retriever_pretrain
+  - object_centric
+run_name: retriever_thordyn_object_ft
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 64
+goals_per_obs: 1
+lr: 5.0e-05
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 50000
+warmup_steps: 2000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 2500
+eval_every: 2500
+bfloat16: false
+batch_size: 16
+num_workers: 12
+datasets:
+- ai2thor_dyn_latent
+contrastive:
+  chunk_size: ${model.memory.chunk_size}
+  positive_pair_type:
+  - jepa
+  num_positives: 8
+  num_negatives: 24
+  cl_temperature: 0.1
+  cl_pose_radius: 0.5
+  cl_pose_fov_half_h: 180.0
+  cl_pose_fov_half_v: 180.0
+  cl_pose_num_samples: 1024
+  local_pos_stats: ${ai2thor_dyn_latent.spec.local_pos_stats}
+  init_from: results/20260818_150341__pretrain_retriever__retriever_thor_jepa_pretrain_ltp64__bs16__ai2thor_latent/checkpoints/0400000.pth.tar
+  obj_weight: 1.0
+  obj_num_positives: 8
+  obj_pulse_gap_s: 2.0
+  obj_min_frac: 0.0
+  obj_last_only: true
+  num_hard_negatives: 0
+  hard_neg_weight: 4.0
+  hard_pos_tol: 0.05
+  hard_yaw_tol_deg: 1.0
+  hard_pitch_tol_deg: 1.0
+  hard_tau: 0.05
+  hard_min_tokens: 8
+  hard_sym_weight: 1.0
+  stride_clip: null
+  max_frame_offset: 10.0
+  fps: 10.0

ai2thor_v3/retriever_jepa/checkpoints/0400000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32ecb108654a7df2b3404c9508f4ada791c8c802b48558ab978eae5d9830119d
+size 66429235

ai2thor_v3/retriever_jepa/config.yaml

@@ -0,0 +1,611 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+ai2thor_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor/ai2thor_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor/ai2thor_latent_train.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal}
+      transform: ${ai2thor_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor/ai2thor_latent_test.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal_test}
+      transform: ${ai2thor_latent.shared.transform}
+model:
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 4
+    softmax_temperature: 1.0
+    scoring: visual
+    metadata_hidden: 128
+    local_pos_stats: null
+    stride_to_seconds: 1.0
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_source: frame
+    query_conditioning: action
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 32
+      img_width: 32
+      patch_size: 2
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+      interact_dim: 4
+wandb:
+  project: PMem-THOR
+  name: null
+  entity: null
+  mode: online
+  group: ai2thor_latent
+  tags:
+  - ai2thor_latent
+  - retriever_pretrain
+  - visual
+run_name: retriever_thor_jepa_pretrain_ltp64
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 64
+goals_per_obs: 1
+lr: 0.0002
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 400000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 25000
+eval_every: 5000
+bfloat16: false
+batch_size: 16
+num_workers: 12
+datasets:
+- ai2thor_latent
+contrastive:
+  chunk_size: ${model.memory.chunk_size}
+  positive_pair_type:
+  - jepa
+  num_positives: 8
+  num_negatives: 24
+  cl_temperature: 0.1
+  cl_pose_radius: 0.5
+  cl_pose_fov_half_h: 180.0
+  cl_pose_fov_half_v: 180.0
+  cl_pose_num_samples: 1024
+  local_pos_stats: ${ai2thor_latent.spec.local_pos_stats}

bundles.json

@@ -33,6 +33,20 @@
     "core_buffers"
    ]
   },
+  {
+   "corpus": "ai2thor_dyn",
+   "arm": "retriever_object",
+   "step": "0012500",
+   "role": "retriever",
+   "release": true,
+   "path": "ai2thor_dyn/retriever_object/checkpoints/0012500.pth.tar",
+   "size_bytes": 66429235,
+   "sha256": "6d3d3e13be18d6e5d75bfb6ae59ef5e4965b11f48ae22133db917a99a44222f5",
+   "keys": [
+    "memory",
+    "pretrain_steps"
+   ]
+  },
   {
    "corpus": "ai2thor_dyn",
    "arm": "temporal",
@@ -81,6 +95,20 @@
     "core_buffers"
    ]
   },
+  {
+   "corpus": "ai2thor_v3",
+   "arm": "retriever_jepa",
+   "step": "0400000",
+   "role": "retriever",
+   "release": true,
+   "path": "ai2thor_v3/retriever_jepa/checkpoints/0400000.pth.tar",
+   "size_bytes": 66429235,
+   "sha256": "32ecb108654a7df2b3404c9508f4ada791c8c802b48558ab978eae5d9830119d",
+   "keys": [
+    "memory",
+    "pretrain_steps"
+   ]
+  },
   {
    "corpus": "ai2thor_v3",
    "arm": "temporal",
@@ -177,6 +205,34 @@
     "core_buffers"
    ]
   },
+  {
+   "corpus": "loopnav",
+   "arm": "retriever_jepa",
+   "step": "0400000",
+   "role": "retriever",
+   "release": true,
+   "path": "loopnav/retriever_jepa/checkpoints/0400000.pth.tar",
+   "size_bytes": 65900649,
+   "sha256": "41cf8aa4b4a274e9849d211b16a066b4d973f8763d26ed525483c90fe4306723",
+   "keys": [
+    "memory",
+    "pretrain_steps"
+   ]
+  },
+  {
+   "corpus": "loopnav",
+   "arm": "retriever_longlive",
+   "step": "0400000",
+   "role": "retriever",
+   "release": true,
+   "path": "loopnav/retriever_longlive/checkpoints/0400000.pth.tar",
+   "size_bytes": 65893225,
+   "sha256": "cb511f4b56853c7e06495151809a34d023c2db9f181f9d48c5d3e9f7d1a57af4",
+   "keys": [
+    "memory",
+    "pretrain_steps"
+   ]
+  },
   {
    "corpus": "loopnav",
    "arm": "temporal",
@@ -241,6 +297,20 @@
     "core_buffers"
    ]
   },
+  {
+   "corpus": "soundspaces_v1",
+   "arm": "retriever_audio",
+   "step": "0400000",
+   "role": "retriever",
+   "release": true,
+   "path": "soundspaces_v1/retriever_audio/checkpoints/0400000.pth.tar",
+   "size_bytes": 65900649,
+   "sha256": "0e792c5ee5d6f852f15e2d731d981b86e35bc055c65c92e9800c27ee063bf727",
+   "keys": [
+    "memory",
+    "pretrain_steps"
+   ]
+  },
   {
    "corpus": "soundspaces_v1",
    "arm": "temporal",

loopnav/retriever_jepa/checkpoints/0400000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41cf8aa4b4a274e9849d211b16a066b4d973f8763d26ed525483c90fe4306723
+size 65900649

loopnav/retriever_jepa/config.yaml

@@ -0,0 +1,334 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      goal_chunk_size: null
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+model:
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 7
+    num_sets: 4
+    beam_size: 64
+    lambda_div: 0.0
+    softmax_temperature: 1.0
+    random_replace_prob: 0.0
+    random_replace_warmup_prob: 0.0
+    random_replace_warmup_steps: 50000
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 18
+      img_width: 32
+      patch_size: 2
+      in_chans: 16
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+      query_adapter_hidden: null
+wandb:
+  project: PMem-World
+  name: null
+  entity: null
+  mode: online
+  group: loopnav_latent
+  tags:
+  - loopnav_latent
+  - retriever_pretrain
+run_name: retriever_jepa_pretrain
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 360
+image_width: 640
+len_traj_pred: 128
+goals_per_obs: 1
+lr: 0.0002
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 400000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 50000
+eval_every: 5000
+bfloat16: false
+batch_size: 64
+num_workers: 12
+datasets:
+- loopnav_latent
+contrastive:
+  chunk_size: ${model.memory.chunk_size}
+  positive_pair_type:
+  - jepa
+  num_positives: 8
+  num_negatives: 24
+  cl_temperature: 0.1
+  cl_pose_radius: 30.0
+  cl_pose_fov_half_h: 52.5
+  cl_pose_fov_half_v: 37.5
+  cl_pose_num_samples: 1024
+  local_pos_stats: ${loopnav_latent.spec.local_pos_stats}

loopnav/retriever_longlive/checkpoints/0400000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb511f4b56853c7e06495151809a34d023c2db9f181f9d48c5d3e9f7d1a57af4
+size 65893225

loopnav/retriever_longlive/config.yaml

@@ -0,0 +1,440 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: null
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 8
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+model:
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 20
+    num_sets: 4
+    softmax_temperature: 1.0
+    scoring: visual
+    metadata_hidden: 128
+    local_pos_stats: null
+    stride_to_seconds: 1.0
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_source: frame
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 18
+      img_width: 32
+      patch_size: 2
+      in_chans: 16
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+wandb:
+  project: PMem-World
+  name: null
+  entity: null
+  mode: online
+  group: loopnav_latent
+  tags:
+  - loopnav_latent
+  - retriever_pretrain
+  - longlive
+run_name: retriever_longlive_pretrain
+context_size: 4
+action_dim: 5
+pose_dim: 3
+image_height: 224
+image_width: 224
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0003
+weight_decay: 0.0001
+grad_clip_val: 1.0
+iterations: 400000
+warmup_steps: 0
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 25000
+eval_every: 5000
+bfloat16: false
+batch_size: 32
+num_workers: 12
+datasets:
+- loopnav_latent
+contrastive:
+  chunk_size: ${model.memory.chunk_size}
+  positive_pair_type:
+  - temporal
+  - pose
+  - jepa
+  num_positives: 4
+  num_negatives: 16
+  cl_temperature: 0.1
+  cl_pose_radius: 30.0
+  cl_pose_fov_half_h: 52.5
+  cl_pose_fov_half_v: 37.5
+  cl_pose_num_samples: 1024
+  local_pos_stats: null
+longlive:
+  hidden_dims:
+  - 64
+  - 128
+  - 256
+  delta_weight: 1.0
+  delta_margin: 0.85
+  delta_window: 3
+  smooth_weight: 1.0

soundspaces_v1/retriever_audio/checkpoints/0400000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e792c5ee5d6f852f15e2d731d981b86e35bc055c65c92e9800c27ee063bf727
+size 65900649

soundspaces_v1/retriever_audio/config.yaml

@@ -0,0 +1,429 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: frames
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+model:
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 4
+    softmax_temperature: 1.0
+    scoring: visual
+    metadata_hidden: 128
+    local_pos_stats: null
+    stride_to_seconds: 1.0
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_source: frame
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 64
+      img_width: 48
+      patch_size: 4
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+wandb:
+  project: PMem-SS
+  name: null
+  entity: null
+  mode: online
+  group: soundspaces_latent
+  tags:
+  - soundspaces_latent
+  - retriever_pretrain
+  - audio
+run_name: retriever_audio_jepa_pretrain_ltp64
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 64
+goals_per_obs: 1
+lr: 0.0002
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 400000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 25000
+eval_every: 5000
+bfloat16: false
+batch_size: 16
+num_workers: 12
+datasets:
+- soundspaces_latent
+contrastive:
+  chunk_size: ${model.memory.chunk_size}
+  positive_pair_type:
+  - jepa
+  num_positives: 8
+  num_negatives: 24
+  cl_temperature: 0.1
+  cl_pose_radius: 0.5
+  cl_pose_fov_half_h: 180.0
+  cl_pose_fov_half_v: 180.0
+  cl_pose_num_samples: 1024
+  local_pos_stats: ${soundspaces_latent.spec.local_pos_stats}