diff --git a/README.md b/README.md index ac88ed3..6074948 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # 4D Gaussian Splatting for Real-Time Dynamic Scene Rendering -## arXiv Preprint +## ArXiv Preprint ### [Project Page](https://guanjunwu.github.io/4dgs/index.html)| [arXiv Paper](https://arxiv.org/abs/2310.08528) @@ -89,6 +89,16 @@ You can just run the following script to evaluate the model. ``` python metrics.py --model_path "output/dnerf/bouncingballs/" ``` +## Custom Datasets +Install nerfstudio and follow their colmap pipeline. + +``` +pip install nerfstudio +ns-process-data images --data data/your-data --output-dir data/your-ns-data +python train.py -s data/your-ns-data --port 6017 --expname "custom" --configs arguments/hypernerf/default.py + +``` + ## Scripts There are some helpful scripts in , please feel free to use them. diff --git a/run.sh b/run.sh index 95f2d2b..393152b 100644 --- a/run.sh +++ b/run.sh @@ -24,4 +24,7 @@ # bash scripts/train_ablation.sh dnerf_tv_2_1 -bash scripts/train_ablation.sh dnerf_tv_2_1_nodrds \ No newline at end of file +bash scripts/process_dnerf.sh dnerf +bash scripts/train_dynerf.sh dynerf +bash scripts/train_hyper_interp.sh hypernerf +bash scripts/train_hyper_virg.sh hypernerf \ No newline at end of file diff --git a/scene/dataset_readers.py b/scene/dataset_readers.py index 718aac7..32c23bf 100644 --- a/scene/dataset_readers.py +++ b/scene/dataset_readers.py @@ -640,10 +640,6 @@ sceneLoadTypeCallbacks = { "Blender" : readNerfSyntheticInfo, "dynerf" : readdynerfInfo, "nerfies": readHyperDataInfos, # NeRFies & HyperNeRF dataset proposed by [https://github.com/google/hypernerf/releases/tag/v0.1] -<<<<<<< HEAD "PanopticSports" : readPanopticSportsinfos } -======= -} ->>>>>>> 6433420eb1f0463278e6800c0c604398f9245208 diff --git a/scripts/process_dnerf.sh b/scripts/process_dnerf.sh index b204592..e417d07 100644 --- a/scripts/process_dnerf.sh +++ b/scripts/process_dnerf.sh @@ -2,41 +2,41 @@ exp_name1=$1 -export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dnerf/jumpingjacks --port 7169 --expname "$exp_name1/jumpingjacks" --configs arguments/$exp_name1/jumpingjacks.py & +export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/jumpingjacks --port 7169 --expname "$exp_name1/jumpingjacks" --configs arguments/$exp_name1/jumpingjacks.py & export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/trex --port 7170 --expname "$exp_name1/trex" --configs arguments/$exp_name1/trex.py export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/jumpingjacks/" --skip_train --configs arguments/$exp_name1/jumpingjacks.py & -export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/trex/" --skip_train --configs arguments/$exp_name1/trex.py +export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/trex/" --skip_train --configs arguments/$exp_name1/trex.py wait -export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name1/jumpingjacks/" & +export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/jumpingjacks/" & export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/trex/" wait export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/mutant --port 7168 --expname "$exp_name1/mutant" --configs arguments/$exp_name1/mutant.py & -export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dnerf/standup --port 7166 --expname "$exp_name1/standup" --configs arguments/$exp_name1/standup.py +export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/standup --port 7166 --expname "$exp_name1/standup" --configs arguments/$exp_name1/standup.py export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/mutant/" --skip_train --configs arguments/$exp_name1/mutant.py & -export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/standup/" --skip_train --configs arguments/$exp_name1/standup.py +export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/standup/" --skip_train --configs arguments/$exp_name1/standup.py wait -export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name1/mutant/" & +export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/mutant/" & export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/standup/" wait export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/hook --port 7369 --expname "$exp_name1/hook" --configs arguments/$exp_name1/hook.py & -export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dnerf/hellwarrior --port 7370 --expname "$exp_name1/hellwarrior" --configs arguments/$exp_name1/hellwarrior.py +export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/hellwarrior --port 7370 --expname "$exp_name1/hellwarrior" --configs arguments/$exp_name1/hellwarrior.py wait export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/hellwarrior/" --skip_train --configs arguments/$exp_name1/hellwarrior.py & -export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/hook/" --skip_train --configs arguments/$exp_name1/hook.py +export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/hook/" --skip_train --configs arguments/$exp_name1/hook.py wait export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/hellwarrior/" & -export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name1/hook/" +export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/hook/" wait export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/lego --port 7168 --expname "$exp_name1/lego" --configs arguments/$exp_name1/lego.py & -export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dnerf/bouncingballs --port 7166 --expname "$exp_name1/bouncingballs" --configs arguments/$exp_name1/bouncingballs.py +export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/bouncingballs --port 7166 --expname "$exp_name1/bouncingballs" --configs arguments/$exp_name1/bouncingballs.py wait export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/bouncingballs/" --skip_train --configs arguments/$exp_name1/bouncingballs.py & -export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/lego/" --skip_train --configs arguments/$exp_name1/lego.py +export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/lego/" --skip_train --configs arguments/$exp_name1/lego.py wait export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/bouncingballs/" & -export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name1/lego/" +export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/lego/" wait echo "Done" diff --git a/scripts/train_dynerf.sh b/scripts/train_dynerf.sh index 69238ed..1621414 100644 --- a/scripts/train_dynerf.sh +++ b/scripts/train_dynerf.sh @@ -1,14 +1,14 @@ exp_name=$1 -# export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/dynerf/flame_salmon_1 --port 6468 --expname "$exp_name/flame_salmon_1" --configs arguments/$exp_name/flame_salmon_1.py & -# export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dynerf/coffee_martini --port 6472 --expname "$exp_name/coffee_martini" --configs arguments/$exp_name/coffee_martini.py & -# -# export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dynerf/cook_spinach --port 6436 --expname "$exp_name/cook_spinach" --configs arguments/$exp_name/cook_spinach.py & -# wait -# export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dynerf/cut_roasted_beef --port 6470 --expname "$exp_name/cut_roasted_beef" --configs arguments/$exp_name/cut_roasted_beef.py -# -# export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/dynerf/flame_steak --port 6471 --expname "$exp_name/flame_steak" --configs arguments/$exp_name/flame_steak.py & -# export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dynerf/sear_steak --port 6569 --expname "$exp_name/sear_steak" --configs arguments/$exp_name/sear_steak.py +export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/dynerf/flame_salmon_1 --port 6468 --expname "$exp_name/flame_salmon_1" --configs arguments/$exp_name/flame_salmon_1.py & +export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dynerf/coffee_martini --port 6472 --expname "$exp_name/coffee_martini" --configs arguments/$exp_name/coffee_martini.py & +wait +export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dynerf/cook_spinach --port 6436 --expname "$exp_name/cook_spinach" --configs arguments/$exp_name/cook_spinach.py & # wait +export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dynerf/cut_roasted_beef --port 6470 --expname "$exp_name/cut_roasted_beef" --configs arguments/$exp_name/cut_roasted_beef.py +wait +export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/dynerf/flame_steak --port 6471 --expname "$exp_name/flame_steak" --configs arguments/$exp_name/flame_steak.py & +export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dynerf/sear_steak --port 6569 --expname "$exp_name/sear_steak" --configs arguments/$exp_name/sear_steak.py +wait export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/cut_roasted_beef --configs arguments/$exp_name/cut_roasted_beef.py --skip_train & export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/sear_steak --configs arguments/$exp_name/sear_steak.py --skip_train @@ -19,12 +19,12 @@ wait export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/cook_spinach --configs arguments/$exp_name/cook_spinach.py --skip_train & export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/coffee_martini --configs arguments/$exp_name/coffee_martini.py --skip_train & wait -# export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/cut_roasted_beef/" & -# export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/cook_spinach/" -# wait -# export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/sear_steak/" & -# export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/flame_salmon_1/" -# wait -# export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/flame_steak/" & -# export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/coffee_martini/" +export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/cut_roasted_beef/" & +export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/cook_spinach/" +wait +export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/sear_steak/" & +export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/flame_salmon_1/" +wait +export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/flame_steak/" & +export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/coffee_martini/" echo "Done" \ No newline at end of file diff --git a/scripts/train_hyper_interp.sh b/scripts/train_hyper_interp.sh index 20a8d43..70daa09 100644 --- a/scripts/train_hyper_interp.sh +++ b/scripts/train_hyper_interp.sh @@ -1,27 +1,27 @@ exp_name=$1 -export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/interp/aleks-teapot --port 6568 --expname "$exp_name/interp/aleks-teapot" --configs arguments/$exp_name/default.py & +export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/interp/aleks-teapot --port 6568 --expname "$exp_name/interp/aleks-teapot" --configs arguments/$exp_name/default.py & export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/interp/slice-banana --port 6566 --expname "$exp_name/interp/slice-banana" --configs arguments/$exp_name/default.py & export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/interp/chickchicken --port 6569 --expname "$exp_name/interp/interp-chicken" --configs arguments/$exp_name/default.py & wait -export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/interp/cut-lemon1 --port 6670 --expname $exp_name/interp/cut-lemon1 --configs arguments/$exp_name/default.py & -export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/interp/hand1-dense-v2 --port 6671 --expname $exp_name/interp/hand1-dense-v2 --configs arguments/$exp_name/default.py & +export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/interp/cut-lemon1 --port 6670 --expname $exp_name/interp/cut-lemon1 --configs arguments/$exp_name/default.py & +export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/interp/hand1-dense-v2 --port 6671 --expname $exp_name/interp/hand1-dense-v2 --configs arguments/$exp_name/default.py & export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/interp/torchocolate --port 6672 --expname $exp_name/interp/torchocolate --configs arguments/$exp_name/default.py & wait export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/interp/aleks-teapot --configs arguments/$exp_name/default.py --skip_train & export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/interp/slice-banana --configs arguments/$exp_name/default.py --skip_train & -export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/interp/interp-chicken --configs arguments/$exp_name/default.py --skip_train & +export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/interp/interp-chicken --configs arguments/$exp_name/default.py --skip_train & wait export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/interp/cut-lemon1 --configs arguments/$exp_name/default.py --skip_train & export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/interp/hand1-dense-v2 --configs arguments/$exp_name/default.py --skip_train& -export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/interp/torchocolate --configs arguments/$exp_name/default.py --skip_train & +export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/interp/torchocolate --configs arguments/$exp_name/default.py --skip_train & wait export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/interp/aleks-teapot/" & export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/interp/slice-banana/" & -export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/interp/interp-chicken/" +export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/interp/interp-chicken/" export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/interp/cut-lemon1/" & export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/interp/hand1-dense-v2/" & -export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/interp/torchocolate/" +export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/interp/torchocolate/" wait echo "Done" \ No newline at end of file diff --git a/scripts/train_hyper_virg.sh b/scripts/train_hyper_virg.sh index cb7b209..4075d9c 100644 --- a/scripts/train_hyper_virg.sh +++ b/scripts/train_hyper_virg.sh @@ -1,17 +1,17 @@ exp_name=$1 export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/virg/broom2 --port 6068 --expname "$exp_name/broom2" --configs arguments/$exp_name/broom2.py & -export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/hypernerf/virg/vrig-3dprinter --port 6066 --expname "$exp_name/3dprinter" --configs arguments/$exp_name/3dprinter.py & +export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/virg/vrig-3dprinter --port 6066 --expname "$exp_name/3dprinter" --configs arguments/$exp_name/3dprinter.py & export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/virg/peel-banana --port 6069 --expname "$exp_name/peel-banana" --configs arguments/$exp_name/banana.py & -export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/virg/vrig-chicken --port 6070 --expname "$exp_name/vrig-chicken" --configs arguments/$exp_name/chicken.py +export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/virg/vrig-chicken --port 6070 --expname "$exp_name/vrig-chicken" --configs arguments/$exp_name/chicken.py wait export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/broom2 --configs arguments/$exp_name/broom2.py --skip_train --skip_test & -export CUDA_VISIBLE_DEVICES=1&&python render.py --model_path output/$exp_name/3dprinter --configs arguments/$exp_name/3dprinter.py --skip_train --skip_test & +export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/3dprinter --configs arguments/$exp_name/3dprinter.py --skip_train --skip_test & export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/peel-banana --configs arguments/$exp_name/banana.py --skip_train --skip_test & -export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/vrig-chicken --configs arguments/$exp_name/chicken.py --skip_train --skip_test & +export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/vrig-chicken --configs arguments/$exp_name/chicken.py --skip_train --skip_test & wait export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/broom2/" & -export CUDA_VISIBLE_DEVICES=1&&python metrics.py --model_path "output/$exp_name/3dprinter/" & +export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/3dprinter/" & export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/peel-banana/" & -export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/vrig-chicken/" & +export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/vrig-chicken/" & wait echo "Done" \ No newline at end of file