init

.gitignore

@@ -0,0 +1,28 @@
+.vscode/
+__pycache__/
+*.pyc
+*.DS_Store
+*.swp
+*.pth
+tmp.*
+*/.ipynb_checkpoints/*
+
+logs/
+weights/
+dump/
+demo/*.mp4
+demo/demo_images/
+src/loftr/utils/superglue.py
+demo/utils.py
+
+notebooks/QccDayNight.ipynb
+notebooks/westlake.ipynb
+assets/westlake
+assets/qcc_pairs.txt
+configs/.petrel*
+tools/draw_QccDayNights.py
+
+scripts/slurm/
+scripts/sbatch_submit.sh
+src/utils/client.py
+

README.md

@@ -0,0 +1,137 @@
+This repository contains codes for ["NeuMap: Neural Coordinate Mapping by Auto-Transdecoder for Camera Localization"](https://arxiv.org/abs/2211.11177).
+
+![demo_vid](teaser.png)
+
+If you find this project useful, please cite:
+```
+@inproceedings{Tang2022NeuMap,
+  title={NeuMap: Neural Coordinate Mapping by Auto-Transdecoder for Camera Localization},
+  author={Shitao Tang, Sicong Tang, Andrea Tagliasacchi, Ping Tan and Yasutaka Furukawa},
+  journal={arXiv preprint arXiv:2211.11177},
+  year={2022}
+}
+```
+
+The codes are based on [LoFTR](https://github.com/zju3dv/LoFTR).
+
+## Installation
+```shell
+# For full pytorch-lightning trainer features (recommended)
+conda create --name neumap python=3.8
+conda activate neumap
+pip install -r requirement.txt
+```
+
+## Testing
+- We provide testing codes and pretrained models for 7scenes, Cambridge, Aachen, NAVER LABS and ScanNet. Download the pretrained models in [Google Drive](https://drive.google.com/drive/folders/1LpkymcohcHb0FeL6ZfBtiNc2zTin-zOo?usp=share_link)/[Dropbox](https://www.dropbox.com/sh/o3htftym3e649cr/AABKn16WUN9VaZ3njqbpBEPZa?dl=0).
+
+### Data
+- The file structure should be like the following:
+```
+data
+├── kapture
+│   ├── train_list
+│   ├── aachen1.0  
+│   ├── HyundaiDepartmentStore
+│   ├── GreatCourt
+│   ├── chess
+│   ...
+├── scannet
+│   ├── train_list
+│   ├── train
+```
+
+### Aachen
+We provide the processed data in [Google Drive](https://drive.google.com/file/d/1a-am7fk54EzcL-fGo6d7hFDV-lRA-e4K/view?usp=share_link)/[Dropbox](https://www.dropbox.com/s/qjjgfy2pcs3ka2x/aachen.tar?dl=0) and the label files in [Google Drive](https://drive.google.com/file/d/106uvy1b4hZK10rvUfoM6n5p8hNL2HVKw/view?usp=share_link)/[Dropbox](https://www.dropbox.com/s/7mszrb7iz44rg8z/aachen_label.tar?dl=0).
+```
+bash scripts/reproduce_test/aachen_v10.sh
+```
+
+### 7scenes
+We provide the processed data in [Google Drive](https://drive.google.com/drive/folders/1YBunSGt62OA-koBiFWSMoXENMyvYpYVA?usp=sharing)/[Dropbox](https://www.dropbox.com/scl/fo/xwxoj8f1ohihvz9l8crxs/h?dl=0&rlkey=vq1a6su2jwf9ox87ioiu16qvo) and the label files in [Google Drive](https://drive.google.com/file/d/1rU7NvFkUzfiNPT7zEPYBmlNLHavUBj4i/view?usp=sharing)/[Dropbox](https://www.dropbox.com/s/wth9vqx7xd52evj/7scenes_label.tar?dl=0).
+```
+bash scripts/reproduce_test/7scenes.sh
+```
+
+### Cambridge
+We provide the processed data in [Google Drive](https://drive.google.com/drive/folders/1LMg5euGQIhP36wLL5tk4E879BjnA3BuX?usp=sharing)/[Dropbox](https://www.dropbox.com/scl/fo/l819h2hxrm2sk450gpx6x/h?dl=0&rlkey=fwziaglxv590ms2ph0d7lhnrh) and the label files in [Google Drive](https://drive.google.com/file/d/1wIbHjyWVNzQ9bPjQk828rH-5J0RdbaOA/view?usp=sharing)/[Dropbox](https://www.dropbox.com/s/mbmtgwrksdaymjc/cambridge_label.tar?dl=0).
+```
+bash scripts/reproduce_test/cambridge.sh
+```
+
+### NAVER LAB
+We provide the processed data in [Google Drive](https://drive.google.com/drive/folders/1nznfL2bkZhBE31-sbVjcMLFGcXwqX5L1?usp=sharing)/[Dropbox](https://www.dropbox.com/scl/fo/30kh1oqun813e4uh933xa/h?dl=0&rlkey=5kwn8535koivmn3gpzh4nb0fv) and the label files in [Google Drive](https://drive.google.com/file/d/1XbQYQEuwsmKCNRui2JljS4A0UNkn2WLh/view?usp=sharing)/[Dropbox](https://www.dropbox.com/s/vh8ly50gfa8o0vd/store_label.tar?dl=0).
+```
+bash scripts/reproduce_test/store.sh
+```
+
+### ScanNet
+We provide the testing data in [Google Drive](https://drive.google.com/file/d/1gFhHhw8nXpCvbm8KYnd59HprVpeeiWxh/view?usp=sharing)/[Dropbox](https://www.dropbox.com/s/r6vhr4fbzm3d5mh/scannet.tar?dl=0) and the label files in [Google Drive](https://drive.google.com/file/d/1W_ogXynFoKhL6qmGtEtjXCijwe_URQ-G/view?usp=sharing)/[Dropbox](https://www.dropbox.com/s/adh6bj0387n48ww/scannet_label.tar?dl=0).
+```
+bash scripts/reproduce_test/scannet.sh
+```
+
+## Training
+
+### Aachen
+First stage training
+```
+bash scripts/reproduce_train/aachen_v10_stage1.sh
+```
+Second stage training
+```
+bash scripts/reproduce_train/aachen_v10_stage2.sh
+```
+
+### 7scenes
+First stage training
+```
+bash scripts/reproduce_train/7scenes_stage1.sh
+```
+Second stage training
+```
+bash scripts/reproduce_train/7scenes_stage2.sh
+```
+
+### Cambridge
+First stage training
+```
+bash scripts/reproduce_train/cambridge_stage1.sh
+```
+Second stage training
+```
+bash scripts/reproduce_train/cambridge_stage2.sh
+```
+
+### NAVER LAB
+```
+bash scripts/reproduce_train/store.sh
+```
+
+### ScanNet
+Train network
+```
+bash scripts/reproduce_train/scannet.sh
+```
+Finetune code for new scenes
+```
+bash scripts/reproduce_train/scannet_code_finetune.sh
+```
+
+## Data size
+To get data size, please refer to `tools/get_params.py` and tools/`get_params_cambridge.py`.
+
+## Process data
+We provide example codes to divide scenes to voxels and generate label files in `tools/get_label_file.py` and `tools/get_query_file.py`. 
+```
+python tools/get_label_file.py --root data/kapture -i aachen1.0/mapping -s 15 -o aachen_debug
+```
+Generate fake query images
+```
+python tools/get_query_file.py \
+    -o query_regular_15 \
+    --retrieval_path data/kapture/aachen1.0/pairs-query-netvlad30.txt \
+    --root data/kapture \
+    --train_list_path data/kapture/train_list/aachen_debug.txt
+```
+To run custom dataset, please first use r2d2 to generate key-points and triangulate 3D points with COLMAP. Then, convert the COLMAP output files to the same format as `data/kapture/aachen1.0/mapping/points`. 

configs/data/7scenes.py

@@ -0,0 +1,17 @@
+from configs.data.base import cfg
+
+TEST_BASE_PATH = "data/kapture"
+TRAIN_BASE_PATH = "data/kapture"
+
+cfg.DATASET.TRAINVAL_DATA_SOURCE = "kapture"
+cfg.DATASET.TRAIN_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.TRAIN_LIST_PATH = f"{TRAIN_BASE_PATH}/train_list/train_7scenes_v3.txt"
+cfg.DATASET.TRAIN_SUBDIR='mapping'
+
+cfg.DATASET.TEST_DATA_SOURCE = "kapture"
+cfg.DATASET.VAL_DATA_ROOT= cfg.DATASET.TEST_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.VAL_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_heads_v3.txt"
+cfg.DATASET.VAL_SUBDIR='query'
+
+cfg.DATASET.TEST_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_heads_v3.txt"   
+cfg.DATASET.TEST_SUBDIR='query'

configs/data/aachen_v10.py

@@ -0,0 +1,15 @@
+from configs.data.base import cfg
+
+TEST_BASE_PATH = "data/kapture"
+TRAIN_BASE_PATH = "data/kapture"
+
+cfg.DATASET.TRAINVAL_DATA_SOURCE = "kapture"
+cfg.DATASET.TRAIN_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.TRAIN_LIST_PATH = f"{TRAIN_BASE_PATH}/train_list/train_aachen_v10.txt"
+
+cfg.DATASET.TEST_DATA_SOURCE = "kapture"
+cfg.DATASET.VAL_DATA_ROOT= cfg.DATASET.TEST_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.VAL_LIST_PATH = f"{TEST_BASE_PATH}/train_list/val_aachen_v10.txt"
+
+cfg.DATASET.TEST_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_aachen_v10.txt"   
+

configs/data/aachen_v8.py

@@ -0,0 +1,16 @@
+from configs.data.base import cfg
+
+TEST_BASE_PATH = "data/kapture"
+TRAIN_BASE_PATH = "data/kapture"
+
+cfg.DATASET.TRAINVAL_DATA_SOURCE = "kapture"
+cfg.DATASET.TRAIN_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.TRAIN_LIST_PATH = f"{TRAIN_BASE_PATH}/train_list/train_aachen_v8.txt"
+
+cfg.DATASET.TEST_DATA_SOURCE = "kapture"
+cfg.DATASET.VAL_DATA_ROOT= cfg.DATASET.TEST_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.VAL_LIST_PATH = f"{TEST_BASE_PATH}/train_list/val_aachen_v8.txt"
+
+cfg.DATASET.TEST_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_aachen_v8.txt"   
+
+

configs/data/base.py

@@ -0,0 +1,24 @@
+"""
+The data config will be the last one merged into the main config.
+Setups in data configs will override all existed setups!
+"""
+
+from yacs.config import CfgNode as CN
+_CN = CN()
+_CN.DATASET = CN()
+_CN.TRAINER = CN()
+
+# training data config
+_CN.DATASET.TRAIN_DATA_ROOT = None
+_CN.DATASET.TRAIN_LIST_PATH = None
+# validation set config
+_CN.DATASET.VAL_DATA_ROOT = None
+_CN.DATASET.VAL_LIST_PATH = None
+
+# testing data config
+_CN.DATASET.TEST_DATA_ROOT = None
+_CN.DATASET.TEST_LIST_PATH = None
+
+
+
+cfg = _CN

configs/data/cambridge.py

@@ -0,0 +1,19 @@
+from configs.data.base import cfg
+
+TEST_BASE_PATH = "data/kapture"
+TRAIN_BASE_PATH = "data/kapture"
+
+cfg.DATASET.TRAINVAL_DATA_SOURCE = "kapture"
+cfg.DATASET.TRAIN_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.TRAIN_LIST_PATH = f"{TRAIN_BASE_PATH}/train_list/train_cambridge.txt"
+cfg.DATASET.TRAIN_SUBDIR='mapping'
+
+cfg.DATASET.TEST_DATA_SOURCE = "kapture"
+cfg.DATASET.VAL_DATA_ROOT= cfg.DATASET.TEST_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+
+cfg.DATASET.VAL_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_ShopFacade.txt"
+cfg.DATASET.VAL_SUBDIR='query'
+
+cfg.DATASET.TEST_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_ShopFacade.txt"   
+cfg.DATASET.TEST_SUBDIR='query'
+

configs/data/scannet.py

@@ -0,0 +1,15 @@
+from configs.data.base import cfg
+
+TEST_BASE_PATH = "data/scannet"
+TRAIN_BASE_PATH = "data/scannet"
+
+cfg.DATASET.TRAINVAL_DATA_SOURCE = "scannet"
+cfg.DATASET.TRAIN_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.TRAIN_LIST_PATH = f"{TRAIN_BASE_PATH}/train_list/train_scannet_80.txt"
+
+cfg.DATASET.TEST_DATA_SOURCE = "scannet"
+cfg.DATASET.VAL_DATA_ROOT= cfg.DATASET.TEST_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.VAL_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_scannet_1.txt"
+
+cfg.DATASET.TEST_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_scannet_1.txt"   
+

configs/data/scannet_code_finetune.py

@@ -0,0 +1,15 @@
+from configs.data.base import cfg
+
+TEST_BASE_PATH = "data/scannet"
+TRAIN_BASE_PATH = "data/scannet"
+
+cfg.DATASET.TRAINVAL_DATA_SOURCE = "scannet"
+cfg.DATASET.TRAIN_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.TRAIN_LIST_PATH = f"{TRAIN_BASE_PATH}/train_list/train_code_finetune.txt"
+
+cfg.DATASET.TEST_DATA_SOURCE = "scannet"
+cfg.DATASET.VAL_DATA_ROOT= cfg.DATASET.TEST_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.VAL_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_code_finetune.txt"
+
+cfg.DATASET.TEST_LIST_PATH = f"{TEST_BASE_PATH}/train_list/query_code_finetune.txt"   
+

configs/data/store1f.py

@@ -0,0 +1,18 @@
+from configs.data.base import cfg
+
+TEST_BASE_PATH = "data/kapture"
+TRAIN_BASE_PATH = "data/kapture"
+
+cfg.DATASET.TRAINVAL_DATA_SOURCE = "kapture"
+cfg.DATASET.TRAIN_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.TRAIN_LIST_PATH = f"{TRAIN_BASE_PATH}/train_list/train_regular_4_Store_1F.txt"
+cfg.DATASET.TRAIN_INTRINSIC_PATH = f"{TRAIN_BASE_PATH}/intrinsics.npz"
+cfg.DATASET.TRAIN_SUBDIR='mapping'
+
+cfg.DATASET.TEST_DATA_SOURCE = "kapture"
+cfg.DATASET.VAL_DATA_ROOT= cfg.DATASET.TEST_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.VAL_LIST_PATH = f"{TEST_BASE_PATH}/train_list/val_regular_4_Store_1F.txt"
+cfg.DATASET.VAL_SUBDIR='validation'
+
+cfg.DATASET.TEST_LIST_PATH = f"{TEST_BASE_PATH}/train_list/val_regular_4_Store_1F.txt"   
+cfg.DATASET.TEST_SUBDIR='validation'

configs/data/store4f.py

@@ -0,0 +1,18 @@
+from configs.data.base import cfg
+
+TEST_BASE_PATH = "data/kapture"
+TRAIN_BASE_PATH = "data/kapture"
+
+cfg.DATASET.TRAINVAL_DATA_SOURCE = "kapture"
+cfg.DATASET.TRAIN_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.TRAIN_LIST_PATH = f"{TRAIN_BASE_PATH}/train_list/train_store4f_v4.txt"
+cfg.DATASET.TRAIN_SUBDIR='mapping'
+
+cfg.DATASET.TEST_DATA_SOURCE = "kapture"
+cfg.DATASET.VAL_DATA_ROOT= cfg.DATASET.TEST_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.VAL_LIST_PATH = f"{TEST_BASE_PATH}/train_list/val_store4f_v4.txt"
+cfg.DATASET.VAL_SUBDIR='validation'
+
+cfg.DATASET.TEST_LIST_PATH = f"{TEST_BASE_PATH}/train_list/val_store4f_v4.txt"   
+cfg.DATASET.TEST_SUBDIR='validation'
+

configs/data/storeb1.py

@@ -0,0 +1,19 @@
+from configs.data.base import cfg
+
+TEST_BASE_PATH = "data/kapture"
+TRAIN_BASE_PATH = "data/kapture"
+
+cfg.DATASET.TRAINVAL_DATA_SOURCE = "kapture"
+cfg.DATASET.TRAIN_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.TRAIN_LIST_PATH = f"{TRAIN_BASE_PATH}/train_list/train_regular_3_Store_B1.txt"
+cfg.DATASET.TRAIN_INTRINSIC_PATH = f"{TRAIN_BASE_PATH}/intrinsics.npz"
+cfg.DATASET.TRAIN_SUBDIR='mapping'
+
+cfg.DATASET.TEST_DATA_SOURCE = "kapture"
+cfg.DATASET.VAL_DATA_ROOT= cfg.DATASET.TEST_DATA_ROOT = f"{TRAIN_BASE_PATH}"
+cfg.DATASET.VAL_LIST_PATH = f"{TEST_BASE_PATH}/train_list/val_regular_3_Store_B1_sampled.txt"
+cfg.DATASET.VAL_SUBDIR='validation'
+
+cfg.DATASET.TEST_LIST_PATH = f"{TEST_BASE_PATH}/train_list/val_regular_3_Store_B1.txt"   
+cfg.DATASET.TEST_SUBDIR='validation'
+

configs/neumap/7scenes_stage1.py

@@ -0,0 +1,19 @@
+
+
+from src.config.default import _CN as cfg
+
+cfg.TRAINER.MSLR_MILESTONES = [ 40, 80, 120, 160, 200]
+
+cfg.TRAINER.TRUE_LR=2e-3
+cfg.TRAINER.DATA_SAMPLER = 'scene_balance' 
+cfg.TRAINER.N_SAMPLES_PER_SUBSET = 1024
+
+cfg.MODEL.CODE_NUM=100
+cfg.MODEL.D_MODEL=256
+cfg.MODEL.RESNETFPN.BLOCK_DIMS = [128, 196, 256] 
+cfg.MODEL.RESNETFPN.INITIAL_DIM = 128
+cfg.MODEL.TRANS_BLOCK_NUM=6
+cfg.MODEL.BACKBONE_FREEZE=True
+
+cfg.DATASET.RANDOM_CROP=False
+cfg.DATASET.RESOLUTION=480