BiohazardVFX/4DGaussians
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #47 from hustvl/latest

Browse Source 
Latest
This commit is contained in:
Geralt_of_Rivia 2023-12-09 12:54:38 +08:00 committed by GitHub
commit 78c2053598
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
84 changed files with 265706 additions and 653 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -8,5 +8,3 @@ tensorboard_3d
screenshots
data/
data
argument/
scripts/

162
.ipynb_checkpoints/mask_psnr-checkpoint.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

255
.ipynb_checkpoints/test-checkpoint.ipynb Normal file
View File

@ -0,0 +1,255 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import torch\n",
"import os\n",
"import imageio"
]
},
{
"cell_type": "code",
"execution_count": 48,
"metadata": {},
"outputs": [],
"source": [
"# path = \"/data3/guanjunwu/project_scp/TiNeuVox/logs/interp_data/interp/chicken/render_test_fine_last\"\n",
"path = \"output/hypernerf4/interp/americano/test/ours_14000/renders\"\n",
"# \n",
"# path = \"output/dynamic3dgs/dynamic3dgs/basketball/test/ours_30000/renders\"\n",
"image_list = os.listdir(path)\n",
"len_image = len(image_list)\n",
"tile = image_list[0].split('.')[-1]"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [],
"source": [
"import re\n",
"def sort_numeric_filenames(filenames):\n",
" \"\"\"\n",
" Sort a list of filenames based on the numeric part of the filename.\n",
" Assumes filenames have a format like '0000.png', '0001.png', etc.\n",
" \"\"\"\n",
" def extract_number(filename):\n",
" # 使用正则表达式提取文件名中的数字\n",
" match = re.search(r'\\d+', filename)\n",
" return int(match.group()) if match else 0\n",
"\n",
" # 使用提取的数字进行排序\n",
" return sorted(filenames, key=extract_number)\n",
"\n",
"# 示例文件名列表\n",
"filenames = image_list\n",
"\n",
"# 进行排序\n",
"sorted_filenames = sort_numeric_filenames(filenames)\n",
"sorted_filenames = [i for i in sorted_filenames if 'png' in i]"
]
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['000.png',\n",
" '001.png',\n",
" '002.png',\n",
" '003.png',\n",
" '004.png',\n",
" '005.png',\n",
" '006.png',\n",
" '007.png',\n",
" '008.png',\n",
" '009.png',\n",
" '010.png',\n",
" '011.png',\n",
" '012.png',\n",
" '013.png',\n",
" '014.png',\n",
" '015.png',\n",
" '016.png',\n",
" '017.png',\n",
" '018.png',\n",
" '019.png',\n",
" '020.png',\n",
" '021.png',\n",
" '022.png',\n",
" '023.png',\n",
" '024.png',\n",
" '025.png',\n",
" '026.png',\n",
" '027.png',\n",
" '028.png',\n",
" '029.png',\n",
" '030.png',\n",
" '031.png',\n",
" '032.png',\n",
" '033.png',\n",
" '034.png',\n",
" '035.png',\n",
" '036.png',\n",
" '037.png',\n",
" '038.png',\n",
" '039.png',\n",
" '040.png',\n",
" '041.png',\n",
" '042.png',\n",
" '043.png',\n",
" '044.png',\n",
" '045.png',\n",
" '046.png',\n",
" '047.png',\n",
" '048.png',\n",
" '049.png',\n",
" '050.png',\n",
" '051.png',\n",
" '052.png',\n",
" '053.png',\n",
" '054.png',\n",
" '055.png',\n",
" '056.png',\n",
" '057.png',\n",
" '058.png',\n",
" '059.png',\n",
" '060.png',\n",
" '061.png',\n",
" '062.png',\n",
" '063.png',\n",
" '064.png',\n",
" '065.png',\n",
" '066.png',\n",
" '067.png',\n",
" '068.png',\n",
" '069.png',\n",
" '070.png',\n",
" '071.png',\n",
" '072.png',\n",
" '073.png',\n",
" '074.png',\n",
" '075.png',\n",
" '076.png',\n",
" '077.png',\n",
" '078.png',\n",
" '079.png',\n",
" '080.png',\n",
" '081.png',\n",
" '082.png',\n",
" '083.png',\n",
" '084.png',\n",
" '085.png',\n",
" '086.png',\n",
" '087.png',\n",
" '088.png',\n",
" '089.png',\n",
" '090.png',\n",
" '091.png',\n",
" '092.png',\n",
" '093.png',\n",
" '094.png',\n",
" '095.png',\n",
" '096.png',\n",
" '097.png',\n",
" '098.png',\n",
" '099.png',\n",
" '100.png',\n",
" '101.png',\n",
" '102.png',\n",
" '103.png',\n",
" '104.png',\n",
" '105.png',\n",
" '106.png',\n",
" '107.png',\n",
" '108.png',\n",
" '109.png',\n",
" '110.png',\n",
" '111.png',\n",
" '112.png']"
]
},
"execution_count": 50,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sorted_filenames"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/data/guanjunwu/disk2/miniconda3/envs/Gaussians4D/lib/python3.7/site-packages/ipykernel_launcher.py:6: DeprecationWarning: Starting with ImageIO v3 the behavior of this function will switch to that of iio.v3.imread. To keep the current behavior (and make this warning disappear) use `import imageio.v2 as imageio` or call `imageio.v2.imread` directly.\n",
" \n",
"IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (536, 960) to (544, 960) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).\n",
"[swscaler @ 0x67a2580] Warning: data is not aligned! This can lead to a speed loss\n"
]
}
],
"source": [
"writer = imageio.get_writer(os.path.join(path,\"video111.mp4\"),fps=10)\n",
"video_num = 1\n",
"video_list = [[] for i in range(video_num)]\n",
"for i, image in enumerate(sorted_filenames):\n",
" if i % video_num == 0:\n",
" image = imageio.imread(os.path.join(path,image))\n",
" writer.append_data(image)\n",
"writer.close()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!pip install imageio[ffmpeg]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.16"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

261654
.ipynb_checkpoints/weight_visualization-checkpoint.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

65
README.md
View File

@ -1,6 +1,6 @@
# 4D Gaussian Splatting for Real-Time Dynamic Scene Rendering
## arXiv Preprint
## ArXiv Preprint
### [Project Page](https://guanjunwu.github.io/4dgs/index.html)| [arXiv Paper](https://arxiv.org/abs/2310.08528)
@ -14,20 +14,11 @@
---------------------------------------------------
---
![block](assets/teaserfig.png)
![block](assets/teaserfig.jpg)
Our method converges very quickly and achieves real-time rendering speed.
Colab demo:[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hustvl/4DGaussians/blob/master/4DGaussians.ipynb) (Thanks [camenduru](https://github.com/camenduru/4DGaussians-colab).)
<video width="320" height="240" controls>
<source src="assets/teaservideo.mp4" type="video/mp4">
</video>
<video width="320" height="240" controls>
<source src="assets/cut_roasted_beef_time.mp4" type="video/mp4">
</video>
## Environmental Setups
@ -98,9 +89,57 @@ You can just run the following script to evaluate the model.
```
python metrics.py --model_path "output/dnerf/bouncingballs/"
```
## Custom Datasets
Install nerfstudio and follow their colmap pipeline.
```
pip install nerfstudio
ns-process-data images --data data/your-data --output-dir data/your-ns-data
python train.py -s data/your-ns-data --port 6017 --expname "custom" --configs arguments/hypernerf/default.py
```
## Scripts
There are some helpful scripts in `scripts/`, please feel free to use them.
There are some helpful scripts in , please feel free to use them.
`vis_point.py`:
get all points clouds at each timestamps.
usage:
```python
export exp_name="hypernerf"
python vis_point.py --model_path output/$exp_name/interp/aleks-teapot --configs arguments/$exp_name/default.py
```
`weight_visualization.ipynb`:
visualize the weight of Multi-resolution HexPlane module.
`merge_many_4dgs.py`:
merge your trained 4dgs.
usage:
```python
export exp_name="dynerf"
python merge_many_4dgs.py --model_path output/$exp_name/flame_salmon_1
```
`colmap.sh`:
generate point clouds from input data
```bash
bash colmap.sh data/hypernerf/virg/vrig-chicken hypernerf
bash colmap.sh data/dynerf/flame_salmon_1 llff
```
**Blender** format seems doesn't work. Welcome to raise a pull request to fix it.
`downsample_point.py` :downsample generated point clouds by sfm.
```python
python scripts/downsample_point.py data/dynerf/sear_steak/points3D_downsample.ply data/dynerf/sear_steak/points3D_downsample2.ply
```
In my paper, I always use `colmap.sh` to generate dense point clouds and downsample it to less than 40000 points.
Here are some codes maybe useful but never adopted in my paper, you can also try it.
---
## Contributions
@ -108,7 +147,7 @@ There are some helpful scripts in `scripts/`, please feel free to use them.
**This project is still under development. Please feel free to raise issues or submit pull requests to contribute to our codebase.**
---
Some source code of ours is borrowed from [3DGS](https://github.com/graphdeco-inria/gaussian-splatting), [k-planes](https://github.com/Giodiro/kplanes_nerfstudio),[HexPlane](https://github.com/Caoang327/HexPlane), [TiNeuVox](https://github.com/hustvl/TiNeuVox). We sincerely appreciate the excellent works of these authors.
Some source code of ours is borrowed from [3DGS](https://github.com/graphdeco-inria/gaussian-splatting), [k-planes](https://github.com/Giodiro/kplanes_nerfstudio),[HexPlane](https://github.com/Caoang327/HexPlane), [TiNeuVox](https://github.com/hustvl/TiNeuVox). We sincerely appreciate the excellent works of these authors.
## Acknowledgement

BIN
alex.pth Normal file
View File

Binary file not shown.

16
arguments/__init__.py
View File

@ -55,6 +55,9 @@ class ModelParams(ParamGroup):
self.data_device = "cuda"
self.eval = True
self.render_process=False
self.add_points=False
self.extension=".png"
self.llffhold=8
super().__init__(parser, "Loading Parameters", sentinel)
def extract(self, args):
@ -66,7 +69,7 @@ class PipelineParams(ParamGroup):
def __init__(self, parser):
self.convert_SHs_python = False
self.compute_cov3D_python = False
self.debug = False
self.debug = True
super().__init__(parser, "Pipeline Parameters")
class ModelHiddenParams(ParamGroup):
def __init__(self, parser):
@ -89,10 +92,16 @@ class ModelHiddenParams(ParamGroup):
'resolution': [64, 64, 64, 25]
}
self.multires = [1, 2, 4, 8]
self.no_dx=False
self.no_grid=False
self.no_ds=False
self.no_dr=False
self.no_do=True
self.no_dshs=True
self.empty_voxel=False
self.grid_pe=0
self.static_mlp=False
self.apply_rotation=False
super().__init__(parser, "ModelHiddenParams")
@ -100,6 +109,8 @@ class ModelHiddenParams(ParamGroup):
class OptimizationParams(ParamGroup):
def __init__(self, parser):
self.dataloader=False
self.zerostamp_init=False
self.custom_sampler=None
self.iterations = 30_000
self.coarse_iterations = 3000
self.position_lr_init = 0.00016
@ -134,7 +145,8 @@ class OptimizationParams(ParamGroup):
self.opacity_threshold_coarse = 0.005
self.opacity_threshold_fine_init = 0.005
self.opacity_threshold_fine_after = 0.005
self.batch_size=1
self.add_point=False
super().__init__(parser, "Optimization Parameters")
def get_combined_args(parser : ArgumentParser):

12
arguments/dnerf/dnerf_default.py
View File

@ -11,18 +11,22 @@ OptimizationParams = dict(
iterations = 20000,
pruning_interval = 8000,
percent_dense = 0.01,
render_process=True,
# no_do=False,
# no_dshs=False
# opacity_reset_interval=30000
)
ModelHiddenParams = dict(
multires = [1, 2, 4, 8 ],
multires = [1, 2],
defor_depth = 0,
net_width = 64,
plane_tv_weight = 0,
time_smoothness_weight = 0,
l1_time_planes = 0,
plane_tv_weight = 0.0001,
time_smoothness_weight = 0.01,
l1_time_planes = 0.0001,
weight_decay_iteration=0,
bounds=1.6
)

10
arguments/dnerf_tv/bouncingballs.py
View File

@ -1,10 +0,0 @@
_base_ = './dnerf_default.py'
ModelHiddenParams = dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 32,
'resolution': [64, 64, 64, 75]
}
)

28
arguments/dnerf_tv/dnerf_default.py
View File

@ -1,28 +0,0 @@
OptimizationParams = dict(
coarse_iterations = 3000,
deformation_lr_init = 0.00016,
deformation_lr_final = 0.0000016,
deformation_lr_delay_mult = 0.01,
grid_lr_init = 0.0016,
grid_lr_final = 0.000016,
iterations = 20000,
pruning_interval = 8000,
percent_dense = 0.01,
# opacity_reset_interval=30000
)
ModelHiddenParams = dict(
multires = [1, 2, 4, 8 ],
defor_depth = 0,
net_width = 64,
plane_tv_weight = 0.0002,
time_smoothness_weight = 0.001,
l1_time_planes = 0.001,
weight_decay_iteration=0,
bounds=1.6
)

10
arguments/dnerf_tv/hellwarrior.py
View File

@ -1,10 +0,0 @@
_base_ = './dnerf_default.py'
ModelHiddenParams = dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 32,
'resolution': [64, 64, 64, 50]
}
)

10
arguments/dnerf_tv/hook.py
View File

@ -1,10 +0,0 @@
_base_ = './dnerf_default.py'
ModelHiddenParams = dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 32,
'resolution': [64, 64, 64, 50]
}
)

16
arguments/dnerf_tv/lego.py
View File

@ -1,16 +0,0 @@
_base_ = './dnerf_default.py'
ModelHiddenParams = dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 32,
'resolution': [64, 64, 64, 25]
},
# deformation_lr_init = 0.001,
# deformation_lr_final = 0.001,
# deformation_lr_delay_mult = 0.01,
# grid_lr_init = 0.001,
# grid_lr_final = 0.001,
)

10
arguments/dnerf_tv/mutant.py
View File

@ -1,10 +0,0 @@
_base_ = './dnerf_default.py'
ModelHiddenParams = dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 32,
'resolution': [64, 64, 64, 75]
}
)

10
arguments/dnerf_tv/standup.py
View File

@ -1,10 +0,0 @@
_base_ = './dnerf_default.py'
ModelHiddenParams = dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 32,
'resolution': [64, 64, 64, 75]
}
)

4
arguments/dynerf/coffee_martini.py Normal file
View File

@ -0,0 +1,4 @@
_base_ = './default.py'
OptimizationParams = dict(
)

4
arguments/dynerf/cook_spinach.py Normal file
View File

@ -0,0 +1,4 @@
_base_ = './default.py'
OptimizationParams = dict(
)

4
arguments/dynerf/cut_roasted_beef.py Normal file
View File

@ -0,0 +1,4 @@
_base_ = './default.py'
OptimizationParams = dict(
)

22
arguments/dynerf/default.py
View File

@ -5,21 +5,27 @@ ModelHiddenParams = dict(
'output_coordinate_dim': 16,
'resolution': [64, 64, 64, 150]
},
multires = [1,2,4,8],
defor_depth = 2,
net_width = 256,
multires = [1,2],
defor_depth = 0,
net_width = 128,
plane_tv_weight = 0.0002,
time_smoothness_weight = 0.001,
l1_time_planes = 0.001,
l1_time_planes = 0.0001,
no_do=False,
no_dshs=False,
no_ds=False,
empty_voxel=False,
render_process=False,
static_mlp=False
)
OptimizationParams = dict(
dataloader=True,
iterations = 30_000,
iterations = 15000,
batch_size=1,
coarse_iterations = 3000,
densify_until_iter = 15_000,
opacity_reset_interval = 6000,
densify_until_iter = 10_000,
# opacity_reset_interval = 60000,
opacity_threshold_coarse = 0.005,
opacity_threshold_fine_init = 0.005,
opacity_threshold_fine_after = 0.005,

4
arguments/dynerf/flame_salmon_1.py Normal file
View File

@ -0,0 +1,4 @@
_base_ = './default.py'
OptimizationParams = dict(
)

4
arguments/dynerf/flame_steak.py Normal file
View File

@ -0,0 +1,4 @@
_base_ = './default.py'
OptimizationParams = dict(
)

4
arguments/dynerf/sear_steak.py Normal file
View File

@ -0,0 +1,4 @@
_base_ = './default.py'
OptimizationParams = dict(
)

11
arguments/dnerf_tv/trex.py → arguments/hypernerf/3dprinter.py
View File

@ -1,10 +1,11 @@
_base_ = './dnerf_default.py'
ModelHiddenParams = dict(
_base_="default.py"
ModelParams=dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 32,
'output_coordinate_dim': 16,
'resolution': [64, 64, 64, 100]
}
},
)
OptimizationParams=dict(
)

11
arguments/hypernerf/banana.py Normal file
View File

@ -0,0 +1,11 @@
_base_="default.py"
ModelParams=dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 16,
'resolution': [64, 64, 64, 250]
},
)
OptimizationParams=dict(
)

11
arguments/dnerf_tv/jumpingjacks.py → arguments/hypernerf/broom2.py
View File

@ -1,10 +1,11 @@
_base_ = './dnerf_default.py'
ModelHiddenParams = dict(
_base_="default.py"
ModelParams=dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 32,
'output_coordinate_dim': 16,
'resolution': [64, 64, 64, 100]
}
},
)
OptimizationParams=dict(
)

11
arguments/hypernerf/chicken.py Normal file
View File

@ -0,0 +1,11 @@
_base_="default.py"
ModelParams=dict(
kplanes_config = {
'grid_dimensions': 2,
'input_coordinate_dim': 4,
'output_coordinate_dim': 16,
'resolution': [64, 64, 64, 80]
},
)
OptimizationParams=dict(
)

37
arguments/hypernerf/default.py
View File

@ -5,32 +5,25 @@ ModelHiddenParams = dict(
'output_coordinate_dim': 16,
'resolution': [64, 64, 64, 150]
},
multires = [1,2,4,8],
defor_depth = 2,
net_width = 256,
multires = [1,2,4],
defor_depth = 1,
net_width = 128,
plane_tv_weight = 0.0002,
time_smoothness_weight = 0.001,
l1_time_planes = 0.001,
l1_time_planes = 0.0001,
render_process=True
)
OptimizationParams = dict(
dataloader=False,
iterations = 60_000,
# dataloader=True,
iterations = 14_000,
batch_size=2,
coarse_iterations = 3000,
densify_until_iter = 45_000,
opacity_reset_interval = 6000,
# position_lr_init = 0.00016,
# position_lr_final = 0.0000016,
# position_lr_delay_mult = 0.01,
# position_lr_max_steps = 60_000,
deformation_lr_init = 0.0016,
deformation_lr_final = 0.00016,
deformation_lr_delay_mult = 0.01,
grid_lr_init = 0.016,
grid_lr_final = 0.0016,
# densify_until_iter = 50_000,
opacity_threshold_coarse = 0.005,
opacity_threshold_fine_init = 0.005,
opacity_threshold_fine_after = 0.005,
densify_until_iter = 10_000,
opacity_reset_interval = 300000,
# grid_lr_init = 0.0016,
# grid_lr_final = 16,
# opacity_threshold_coarse = 0.005,
# opacity_threshold_fine_init = 0.005,
# opacity_threshold_fine_after = 0.005,
# pruning_interval = 2000
)

BIN
assets/teaserfig.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 632 KiB

BIN
assets/teaserfig.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 893 KiB

25
colmap.sh Normal file
View File

@ -0,0 +1,25 @@
workdir=$1
datatype=$2 # blender, hypernerf, llff
export CUDA_VISIBLE_DEVICES=1
rm -rf $workdir/sparse_
rm -rf $workdir/image_colmap
python scripts/"$datatype"2colmap.py $workdir
rm -rf $workdir/colmap
rm -rf $workdir/colmap/sparse/0
mkdir $workdir/colmap
cp -r $workdir/image_colmap $workdir/colmap/images
cp -r $workdir/sparse_ $workdir/colmap/sparse_custom
colmap feature_extractor --database_path $workdir/colmap/database.db --image_path $workdir/colmap/images --SiftExtraction.max_image_size 4096 --SiftExtraction.max_num_features 16384 --SiftExtraction.estimate_affine_shape 1 --SiftExtraction.domain_size_pooling 1
python database.py --database_path $workdir/colmap/database.db --txt_path $workdir/colmap/sparse_custom/cameras.txt
colmap exhaustive_matcher --database_path $workdir/colmap/database.db
mkdir -p $workdir/colmap/sparse/0
colmap point_triangulator --database_path $workdir/colmap/database.db --image_path $workdir/colmap/images --input_path $workdir/colmap/sparse_custom --output_path $workdir/colmap/sparse/0 --clear_points 1
mkdir -p $workdir/colmap/dense/workspace
colmap image_undistorter --image_path $workdir/colmap/images --input_path $workdir/colmap/sparse/0 --output_path $workdir/colmap/dense/workspace
colmap patch_match_stereo --workspace_path $workdir/colmap/dense/workspace
colmap stereo_fusion --workspace_path $workdir/colmap/dense/workspace --output_path $workdir/colmap/dense/workspace/fused.ply

92
gaussian_renderer/__init__.py
View File

@ -14,8 +14,8 @@ import math
from diff_gaussian_rasterization import GaussianRasterizationSettings, GaussianRasterizer
from scene.gaussian_model import GaussianModel
from utils.sh_utils import eval_sh
def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None, stage="fine"):
from time import time as get_time
def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None, stage="fine", cam_type=None):
"""
Render the scene.
@ -31,33 +31,40 @@ def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor,
# Set up rasterization configuration
tanfovx = math.tan(viewpoint_camera.FoVx * 0.5)
tanfovy = math.tan(viewpoint_camera.FoVy * 0.5)
means3D = pc.get_xyz
if cam_type != "PanopticSports":
tanfovx = math.tan(viewpoint_camera.FoVx * 0.5)
tanfovy = math.tan(viewpoint_camera.FoVy * 0.5)
raster_settings = GaussianRasterizationSettings(
image_height=int(viewpoint_camera.image_height),
image_width=int(viewpoint_camera.image_width),
tanfovx=tanfovx,
tanfovy=tanfovy,
bg=bg_color,
scale_modifier=scaling_modifier,
viewmatrix=viewpoint_camera.world_view_transform.cuda(),
projmatrix=viewpoint_camera.full_proj_transform.cuda(),
sh_degree=pc.active_sh_degree,
campos=viewpoint_camera.camera_center.cuda(),
prefiltered=False,
debug=pipe.debug
)
time = torch.tensor(viewpoint_camera.time).to(means3D.device).repeat(means3D.shape[0],1)
else:
raster_settings = viewpoint_camera['camera']
time=torch.tensor(viewpoint_camera['time']).to(means3D.device).repeat(means3D.shape[0],1)
raster_settings = GaussianRasterizationSettings(
image_height=int(viewpoint_camera.image_height),
image_width=int(viewpoint_camera.image_width),
tanfovx=tanfovx,
tanfovy=tanfovy,
bg=bg_color,
scale_modifier=scaling_modifier,
viewmatrix=viewpoint_camera.world_view_transform.cuda(),
projmatrix=viewpoint_camera.full_proj_transform.cuda(),
sh_degree=pc.active_sh_degree,
campos=viewpoint_camera.camera_center.cuda(),
prefiltered=False,
debug=pipe.debug
)
rasterizer = GaussianRasterizer(raster_settings=raster_settings)
# means3D = pc.get_xyz
# add deformation to each points
# deformation = pc.get_deformation
means3D = pc.get_xyz
time = torch.tensor(viewpoint_camera.time).to(means3D.device).repeat(means3D.shape[0],1)
means2D = screenspace_points
opacity = pc._opacity
shs = pc.get_features
# If precomputed 3d covariance is provided, use it. If not, then it will be computed from
# scaling / rotation by the rasterizer.
@ -71,35 +78,30 @@ def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor,
rotations = pc._rotation
deformation_point = pc._deformation_table
if stage == "coarse" :
means3D_deform, scales_deform, rotations_deform, opacity_deform = means3D, scales, rotations, opacity
means3D_final, scales_final, rotations_final, opacity_final, shs_final = means3D, scales, rotations, opacity, shs
else:
means3D_deform, scales_deform, rotations_deform, opacity_deform = pc._deformation(means3D[deformation_point], scales[deformation_point],
rotations[deformation_point], opacity[deformation_point],
time[deformation_point])
# time0 = get_time()
# means3D_deform, scales_deform, rotations_deform, opacity_deform = pc._deformation(means3D[deformation_point], scales[deformation_point],
# rotations[deformation_point], opacity[deformation_point],
# time[deformation_point])
means3D_final, scales_final, rotations_final, opacity_final, shs_final = pc._deformation(means3D, scales,
rotations, opacity, shs,
time)
# time1 = get_time()
# print("deformation forward:",time1-time0)
# print(time.max())
with torch.no_grad():
pc._deformation_accum[deformation_point] += torch.abs(means3D_deform-means3D[deformation_point])
means3D_final = torch.zeros_like(means3D)
rotations_final = torch.zeros_like(rotations)
scales_final = torch.zeros_like(scales)
opacity_final = torch.zeros_like(opacity)
means3D_final[deformation_point] = means3D_deform
rotations_final[deformation_point] = rotations_deform
scales_final[deformation_point] = scales_deform
opacity_final[deformation_point] = opacity_deform
means3D_final[~deformation_point] = means3D[~deformation_point]
rotations_final[~deformation_point] = rotations[~deformation_point]
scales_final[~deformation_point] = scales[~deformation_point]
opacity_final[~deformation_point] = opacity[~deformation_point]
# time2 = get_time()
# print("asset value:",time2-time1)
scales_final = pc.scaling_activation(scales_final)
rotations_final = pc.rotation_activation(rotations_final)
opacity = pc.opacity_activation(opacity)
opacity = pc.opacity_activation(opacity_final)
# print(opacity.max())
# If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors
# from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer.
shs = None
# shs = None
colors_precomp = None
if override_color is None:
if pipe.convert_SHs_python:
@ -109,21 +111,25 @@ def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor,
sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized)
colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0)
else:
shs = pc.get_features
pass
# shs =
else:
colors_precomp = override_color
# Rasterize visible Gaussians to image, obtain their radii (on screen).
# time3 = get_time()
rendered_image, radii, depth = rasterizer(
means3D = means3D_final,
means2D = means2D,
shs = shs,
shs = shs_final,
colors_precomp = colors_precomp,
opacities = opacity,
scales = scales_final,
rotations = rotations_final,
cov3D_precomp = cov3D_precomp)
# time4 = get_time()
# print("rasterization:",time4-time3)
# breakpoint()
# Those Gaussians that were frustum culled or had a radius of 0 were not visible.
# They will be excluded from value updates used in the splitting criteria.
return {"render": rendered_image,

230
merge_many_4dgs.py Normal file
View File

@ -0,0 +1,230 @@
import imageio
import numpy as np
import torch
from scene import Scene
import os
import cv2
from tqdm import tqdm
from os import makedirs
from gaussian_renderer import render
import torchvision
from utils.general_utils import safe_state
from argparse import ArgumentParser
from arguments import ModelParams, PipelineParams, get_combined_args, ModelHiddenParams
from gaussian_renderer import GaussianModel
from time import time
import open3d as o3d
# import torch.multiprocessing as mp
import threading
import concurrent.futures
from copy import deepcopy
#
# Copyright (C) 2023, Inria
# GRAPHDECO research group, https://team.inria.fr/graphdeco
# All rights reserved.
#
# This software is free for non-commercial, research and evaluation use
# under the terms of the LICENSE.md file.
#
# For inquiries contact george.drettakis@inria.fr
#
import torch
import math
from diff_gaussian_rasterization import GaussianRasterizationSettings, GaussianRasterizer
from scene.gaussian_model import GaussianModel
from utils.render_utils import get_state_at_time
from tqdm import tqdm
def rotate_point_cloud(point_cloud, displacement, rotation_angles, scales_bias):
theta, phi = rotation_angles
rotation_matrix_z = torch.tensor([
[torch.cos(theta), -torch.sin(theta), 0],
[torch.sin(theta), torch.cos(theta), 0],
[0, 0, 1]
]).to(point_cloud)
rotation_matrix_x = torch.tensor([
[1, 0, 0],
[0, torch.cos(phi), -torch.sin(phi)],
[0, torch.sin(phi), torch.cos(phi)]
]).to(point_cloud)
rotation_matrix = torch.matmul(rotation_matrix_z, rotation_matrix_x)
# print(rotation_matrix)
point_cloud = point_cloud*scales_bias
rotated_point_cloud = torch.matmul(point_cloud, rotation_matrix.t())
displaced_point_cloud = rotated_point_cloud + displacement
return displaced_point_cloud
@torch.no_grad()
def render(viewpoint_camera, gaussians, bg_color : torch.Tensor, scaling_modifier = 1.0, motion_bias = [torch.tensor([0,0,0])], rotation_bias = [torch.tensor([0,0])],
scales_bias=[1,1]):
"""
Render the scene.
Background tensor (bg_color) must be on GPU!
"""
# Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means
# Set up rasterization configuration
tanfovx = math.tan(viewpoint_camera.FoVx * 0.5)
tanfovy = math.tan(viewpoint_camera.FoVy * 0.5)
screenspace_points = None
for pc in gaussians:
if screenspace_points is None:
screenspace_points = torch.zeros_like(pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda") + 0
else:
screenspace_points1 = torch.zeros_like(pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda") + 0
screenspace_points = torch.cat([screenspace_points,screenspace_points1],dim=0)
try:
screenspace_points.retain_grad()
except:
pass
raster_settings = GaussianRasterizationSettings(
image_height=int(viewpoint_camera.image_height),
image_width=int(viewpoint_camera.image_width),
tanfovx=tanfovx,
tanfovy=tanfovy,
bg=bg_color,
scale_modifier=scaling_modifier,
viewmatrix=viewpoint_camera.world_view_transform.cuda(),
projmatrix=viewpoint_camera.full_proj_transform.cuda(),
sh_degree=gaussians[0].active_sh_degree,
campos=viewpoint_camera.camera_center.cuda(),
prefiltered=False,
debug=False
)
rasterizer = GaussianRasterizer(raster_settings=raster_settings)
# means3D = pc.get_xyz
# add deformation to each points
# deformation = pc.get_deformation
means3D_final, scales_final, rotations_final, opacity_final, shs_final = None, None, None, None, None
for index, pc in enumerate(gaussians):
means3D_final1, scales_final1, rotations_final1, opacity_final1, shs_final1 = get_state_at_time(pc, viewpoint_camera)
if index == 0:
means3D_final, scales_final, rotations_final, opacity_final, shs_final = means3D_final1, scales_final1, rotations_final1, opacity_final1, shs_final1
else:
motion_bias_t = motion_bias[index-1].to(means3D_final)
rotation_bias_t = rotation_bias[index-1].to(means3D_final)
means3D_final1 = rotate_point_cloud(means3D_final1,motion_bias_t,rotation_bias_t,scales_bias[index-1])
# breakpoint()
scales_final1 = scales_final1*scales_bias[index-1]
means3D_final = torch.cat([means3D_final,means3D_final1],dim=0)
scales_final = torch.cat([scales_final,scales_final1],dim=0)
rotations_final = torch.cat([rotations_final,rotations_final1],dim=0)
opacity_final = torch.cat([opacity_final,opacity_final1],dim=0)
shs_final = torch.cat([shs_final,shs_final1],dim=0)
colors_precomp = None
cov3D_precomp = None
rendered_image, radii, depth = rasterizer(
means3D = means3D_final,
means2D = screenspace_points,
shs = shs_final,
colors_precomp = colors_precomp,
opacities = opacity_final,
scales = scales_final,
rotations = rotations_final,
cov3D_precomp = cov3D_precomp)
return {"render": rendered_image,
"viewspace_points": screenspace_points,
"visibility_filter" : radii > 0,
"radii": radii,
"depth":depth}
def init_gaussians(dataset : ModelParams, hyperparam, iteration : int, pipeline : PipelineParams, skip_train : bool, skip_test : bool, skip_video: bool):
with torch.no_grad():
gaussians = GaussianModel(dataset.sh_degree, hyperparam)
scene = Scene(dataset, gaussians, load_iteration=iteration, shuffle=False)
bg_color = [1,1,1] if dataset.white_background else [0, 0, 0]
background = torch.tensor(bg_color, dtype=torch.float32, device="cuda")
print("hello!!")
return gaussians, scene, background
def save_point_cloud(points, model_path, timestamp):
output_path = os.path.join(model_path,"point_pertimestamp")
if not os.path.exists(output_path):
os.makedirs(output_path,exist_ok=True)
points = points.detach().cpu().numpy()
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(points)
ply_path = os.path.join(output_path,f"points_{timestamp}.ply")
o3d.io.write_point_cloud(ply_path, pcd)
parser = ArgumentParser(description="Testing script parameters")
model = ModelParams(parser, sentinel=True)
pipeline = PipelineParams(parser)
hyperparam = ModelHiddenParams(parser)
parser.add_argument("--iteration", default=-1, type=int)
parser.add_argument("--skip_train", action="store_true")
parser.add_argument("--skip_test", action="store_true")
parser.add_argument("--quiet", action="store_true")
parser.add_argument("--skip_video", action="store_true")
parser.add_argument("--configs1", type=str, default="arguments/dynerf_9/flame_salmon_1.py")
parser.add_argument("--configs2", type=str, default="arguments/dnerf_tv_2/hellwarrior.py")
parser.add_argument("--modelpath2", type=str, default="output/dnerf_tv_2/hellwarrior")
parser.add_argument("--configs3", type=str, default="arguments/dnerf_tv_2/mutant.py")
parser.add_argument("--modelpath3", type=str, default="output/dnerf_tv_2/mutant")
render_path = "output/editing_render_flame_salmon"
args = get_combined_args(parser)
print("Rendering " , args.model_path)
args2 = deepcopy(args)
args3 = deepcopy(args)
if args.configs1:
import mmcv
from utils.params_utils import merge_hparams
config = mmcv.Config.fromfile(args.configs1)
args1 = merge_hparams(args, config)
# breakpoint()
if args2.configs2:
import mmcv
from utils.params_utils import merge_hparams
config = mmcv.Config.fromfile(args2.configs2)
args2 = merge_hparams(args2, config)
args2.model_path = args2.modelpath2
if args3.configs3:
import mmcv
from utils.params_utils import merge_hparams
config = mmcv.Config.fromfile(args3.configs3)
args3 = merge_hparams(args3, config)
args3.model_path = args3.modelpath3
safe_state(args.quiet)
gaussians1, scene1, background = init_gaussians(model.extract(args1), hyperparam.extract(args1), args1.iteration, pipeline.extract(args1), args1.skip_train, args1.skip_test, args1.skip_video)
gaussians2, scene2, background = init_gaussians(model.extract(args2), hyperparam.extract(args2), args2.iteration, pipeline.extract(args2), args2.skip_train, args2.skip_test, args2.skip_video)
gaussians3, scene3, background = init_gaussians(model.extract(args3), hyperparam.extract(args3), args3.iteration, pipeline.extract(args3), args3.skip_train, args3.skip_test, args3.skip_video)
gaussians = [gaussians1,gaussians2,gaussians3]
# breakpoint()
to8b = lambda x : (255*np.clip(x.cpu().numpy(),0,1)).astype(np.uint8)
render_images=[]
if not os.path.exists(render_path):
os.makedirs(render_path,exist_ok=True)
for index, viewpoint in tqdm(enumerate(scene1.getVideoCameras())):
result = render(viewpoint, gaussians,
bg_color=background,
motion_bias=[
torch.tensor([4,4,12]),
torch.tensor([-2,4,12])
]
,rotation_bias=[
torch.tensor([0,1.9*np.pi/4]),
torch.tensor([0,1.9*np.pi/4])
],
scales_bias = [1,1])
render_images.append(to8b(result["render"]).transpose(1,2,0))
torchvision.utils.save_image(result["render"],os.path.join(render_path,f"output_image{index}.png"))
imageio.mimwrite(os.path.join(render_path, 'video_rgb.mp4'), render_images, fps=30, codec='libx265')
# points = get_state_at_time(gaussians, viewpoint)
# save_point_cloud(points, args.model_path, index)

34
metrics.py
View File

@ -20,7 +20,7 @@ import json
from tqdm import tqdm
from utils.image_utils import psnr
from argparse import ArgumentParser
from pytorch_msssim import ms_ssim
def readImages(renders_dir, gt_dir):
renders = []
gts = []
@ -67,30 +67,50 @@ def evaluate(model_paths):
ssims = []
psnrs = []
lpipss = []
lpipsa = []
ms_ssims = []
Dssims = []
for idx in tqdm(range(len(renders)), desc="Metric evaluation progress"):
ssims.append(ssim(renders[idx], gts[idx]))
psnrs.append(psnr(renders[idx], gts[idx]))
lpipss.append(lpips(renders[idx], gts[idx], net_type='vgg'))
ms_ssims.append(ms_ssim(renders[idx], gts[idx],data_range=1, size_average=True ))
lpipsa.append(lpips(renders[idx], gts[idx], net_type='alex'))
Dssims.append((1-ms_ssims[-1])/2)
print("Scene: ", scene_dir, "SSIM : {:>12.7f}".format(torch.tensor(ssims).mean(), ".5"))
print("Scene: ", scene_dir, "PSNR : {:>12.7f}".format(torch.tensor(psnrs).mean(), ".5"))
print("Scene: ", scene_dir, "LPIPS: {:>12.7f}".format(torch.tensor(lpipss).mean(), ".5"))
print("")
print("Scene: ", scene_dir, "LPIPS-vgg: {:>12.7f}".format(torch.tensor(lpipss).mean(), ".5"))
print("Scene: ", scene_dir, "LPIPS-alex: {:>12.7f}".format(torch.tensor(lpipsa).mean(), ".5"))
print("Scene: ", scene_dir, "MS-SSIM: {:>12.7f}".format(torch.tensor(ms_ssims).mean(), ".5"))
print("Scene: ", scene_dir, "D-SSIM: {:>12.7f}".format(torch.tensor(Dssims).mean(), ".5"))
full_dict[scene_dir][method].update({"SSIM": torch.tensor(ssims).mean().item(),
"PSNR": torch.tensor(psnrs).mean().item(),
"LPIPS": torch.tensor(lpipss).mean().item()})
"LPIPS-vgg": torch.tensor(lpipss).mean().item(),
"LPIPS-alex": torch.tensor(lpipsa).mean().item(),
"MS-SSIM": torch.tensor(ms_ssims).mean().item(),
"D-SSIM": torch.tensor(Dssims).mean().item()},
)
per_view_dict[scene_dir][method].update({"SSIM": {name: ssim for ssim, name in zip(torch.tensor(ssims).tolist(), image_names)},
"PSNR": {name: psnr for psnr, name in zip(torch.tensor(psnrs).tolist(), image_names)},
"LPIPS": {name: lp for lp, name in zip(torch.tensor(lpipss).tolist(), image_names)}})
"LPIPS-vgg": {name: lp for lp, name in zip(torch.tensor(lpipss).tolist(), image_names)},
"LPIPS-alex": {name: lp for lp, name in zip(torch.tensor(lpipsa).tolist(), image_names)},
"MS-SSIM": {name: lp for lp, name in zip(torch.tensor(ms_ssims).tolist(), image_names)},
"D-SSIM": {name: lp for lp, name in zip(torch.tensor(Dssims).tolist(), image_names)},
}
)
with open(scene_dir + "/results.json", 'w') as fp:
json.dump(full_dict[scene_dir], fp, indent=True)
with open(scene_dir + "/per_view.json", 'w') as fp:
json.dump(per_view_dict[scene_dir], fp, indent=True)
except:
except Exception as e:
print("Unable to compute metrics for model", scene_dir)
raise e
if __name__ == "__main__":
device = torch.device("cuda:0")

65
render.py
View File

@ -23,8 +23,28 @@ from argparse import ArgumentParser
from arguments import ModelParams, PipelineParams, get_combined_args, ModelHiddenParams
from gaussian_renderer import GaussianModel
from time import time
# import torch.multiprocessing as mp
import threading
import concurrent.futures
def multithread_write(image_list, path):
executor = concurrent.futures.ThreadPoolExecutor(max_workers=None)
def write_image(image, count, path):
try:
torchvision.utils.save_image(image, os.path.join(path, '{0:05d}'.format(count) + ".png"))
return count, True
except:
return count, False
tasks = []
for index, image in enumerate(image_list):
tasks.append(executor.submit(write_image, image, index, path))
executor.shutdown()
for index, status in enumerate(tasks):
if status == False:
write_image(image_list[index], index, path)
to8b = lambda x : (255*np.clip(x.cpu().numpy(),0,1)).astype(np.uint8)
def render_set(model_path, name, iteration, views, gaussians, pipeline, background):
def render_set(model_path, name, iteration, views, gaussians, pipeline, background, cam_type):
render_path = os.path.join(model_path, name, "ours_{}".format(iteration), "renders")
gts_path = os.path.join(model_path, name, "ours_{}".format(iteration), "gt")
@ -33,49 +53,52 @@ def render_set(model_path, name, iteration, views, gaussians, pipeline, backgrou
render_images = []
gt_list = []
render_list = []
# breakpoint()
print("point nums:",gaussians._xyz.shape[0])
for idx, view in enumerate(tqdm(views, desc="Rendering progress")):
if idx == 0:time1 = time()
rendering = render(view, gaussians, pipeline, background)["render"]
# breakpoint()
rendering = render(view, gaussians, pipeline, background,cam_type=cam_type)["render"]
# torchvision.utils.save_image(rendering, os.path.join(render_path, '{0:05d}'.format(idx) + ".png"))
render_images.append(to8b(rendering).transpose(1,2,0))
# print(to8b(rendering).shape)
render_list.append(rendering)
if name in ["train", "test"]:
gt = view.original_image[0:3, :, :]
if cam_type != "PanopticSports":
gt = view.original_image[0:3, :, :]
else:
gt = view['image'].cuda()
# torchvision.utils.save_image(gt, os.path.join(gts_path, '{0:05d}'.format(idx) + ".png"))
gt_list.append(gt)
# if idx >= 10:
# break
time2=time()
print("FPS:",(len(views)-1)/(time2-time1))
count = 0
print("writing training images.")
if len(gt_list) != 0:
for image in tqdm(gt_list):
torchvision.utils.save_image(image, os.path.join(gts_path, '{0:05d}'.format(count) + ".png"))
count+=1
count = 0
print("writing rendering images.")
if len(render_list) != 0:
for image in tqdm(render_list):
torchvision.utils.save_image(image, os.path.join(render_path, '{0:05d}'.format(count) + ".png"))
count +=1
# print("writing training images.")
multithread_write(gt_list, gts_path)
# print("writing rendering images.")
multithread_write(render_list, render_path)
imageio.mimwrite(os.path.join(model_path, name, "ours_{}".format(iteration), 'video_rgb.mp4'), render_images, fps=30, quality=8)
imageio.mimwrite(os.path.join(model_path, name, "ours_{}".format(iteration), 'video_rgb.mp4'), render_images, fps=30)
def render_sets(dataset : ModelParams, hyperparam, iteration : int, pipeline : PipelineParams, skip_train : bool, skip_test : bool, skip_video: bool):
with torch.no_grad():
gaussians = GaussianModel(dataset.sh_degree, hyperparam)
scene = Scene(dataset, gaussians, load_iteration=iteration, shuffle=False)
cam_type=scene.dataset_type
bg_color = [1,1,1] if dataset.white_background else [0, 0, 0]
background = torch.tensor(bg_color, dtype=torch.float32, device="cuda")
if not skip_train:
render_set(dataset.model_path, "train", scene.loaded_iter, scene.getTrainCameras(), gaussians, pipeline, background)
render_set(dataset.model_path, "train", scene.loaded_iter, scene.getTrainCameras(), gaussians, pipeline, background,cam_type)
if not skip_test:
render_set(dataset.model_path, "test", scene.loaded_iter, scene.getTestCameras(), gaussians, pipeline, background)
render_set(dataset.model_path, "test", scene.loaded_iter, scene.getTestCameras(), gaussians, pipeline, background,cam_type)
if not skip_video:
render_set(dataset.model_path,"video",scene.loaded_iter,scene.getVideoCameras(),gaussians,pipeline,background)
render_set(dataset.model_path,"video",scene.loaded_iter,scene.getVideoCameras(),gaussians,pipeline,background,cam_type)
if __name__ == "__main__":
# Set up command line argument parser
parser = ArgumentParser(description="Testing script parameters")

3
requirements.txt
View File

@ -6,4 +6,5 @@ matplotlib
argparse
lpips
plyfile
imageio-ffmpeg
pytorch_msssim
open3d

30
run.sh Normal file
View File

@ -0,0 +1,30 @@
# bash scripts/process_dnerf.sh dnerf_ab/dnerf_tv_30000
# bash scripts/process_dnerf.sh dnerf_ab/dnerf_tv_depth2
# bash scripts/process_dnerf_2.sh dnerf_tv_dshs
# bash scripts/process_dnerf_2.sh dnerf_tv_do
# bash scripts/process_dnerf_2.sh dnerf_tv_2
# bash scripts/process_dnerf_2.sh dnerf_tv_8
# bash scripts/process_dnerf_2.sh dnerf_tv_deepmlp
# bash scripts/process_dnerf_2.sh dnerf_tv_nods
# bash scripts/process_dnerf.sh dnerf_ab/dnerf_tv
# exp_name1="hypernerf_3dgs"
# export CUDA_VISIBLE_DEVICES=2&&python render2.py --model_path "output/$exp_name1/3dprinter/" --skip_train --configs arguments/$exp_name1/3dprinter.py &
# export CUDA_VISIBLE_DEVICES=3&&python render2.py --model_path "output/$exp_name1/broom2/" --skip_train --configs arguments/$exp_name1/broom2.py &
# #
# wait
# export CUDA_VISIBLE_DEVICES=2&&python render2.py --model_path "output/$exp_name1/peel-banana/" --skip_train --configs arguments/$exp_name1/banana.py &
# export CUDA_VISIBLE_DEVICES=3&&python render2.py --model_path "output/$exp_name1/vrig-chicken/" --skip_train --configs arguments/$exp_name1/chicken.py &
# wait
# exp_name="hypernerf_3dgs"
# export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/vrig-chicken/" &
# export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/peel-banana/" &
# export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/broom2/" &
# export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/3dprinter/" &
# bash scripts/train_ablation.sh dnerf_tv_2_1
bash scripts/process_dnerf.sh dnerf
bash scripts/train_dynerf.sh dynerf
bash scripts/train_hyper_interp.sh hypernerf
bash scripts/train_hyper_virg.sh hypernerf

65
scene/__init__.py
View File

@ -19,7 +19,7 @@ from scene.dataset import FourDGSdataset
from arguments import ModelParams
from utils.camera_utils import cameraList_from_camInfos, camera_to_JSON
from torch.utils.data import Dataset
from scene.dataset_readers import add_points
class Scene:
gaussians : GaussianModel
@ -43,54 +43,41 @@ class Scene:
self.test_cameras = {}
self.video_cameras = {}
if os.path.exists(os.path.join(args.source_path, "sparse")):
scene_info = sceneLoadTypeCallbacks["Colmap"](args.source_path, args.images, args.eval)
scene_info = sceneLoadTypeCallbacks["Colmap"](args.source_path, args.images, args.eval, args.llffhold)
dataset_type="colmap"
elif os.path.exists(os.path.join(args.source_path, "transforms_train.json")):
print("Found transforms_train.json file, assuming Blender data set!")
scene_info = sceneLoadTypeCallbacks["Blender"](args.source_path, args.white_background, args.eval)
scene_info = sceneLoadTypeCallbacks["Blender"](args.source_path, args.white_background, args.eval, args.extension)
dataset_type="blender"
elif os.path.exists(os.path.join(args.source_path, "poses_bounds.npy")):
scene_info = sceneLoadTypeCallbacks["dynerf"](args.source_path, args.white_background, args.eval)
dataset_type="dynerf"
elif os.path.exists(os.path.join(args.source_path,"dataset.json")):
scene_info = sceneLoadTypeCallbacks["nerfies"](args.source_path, False, args.eval)
dataset_type="nerfies"
elif os.path.exists(os.path.join(args.source_path,"train_meta.json")):
scene_info = sceneLoadTypeCallbacks["PanopticSports"](args.source_path)
dataset_type="PanopticSports"
else:
assert False, "Could not recognize scene type!"
self.maxtime = scene_info.maxtime
# if not self.loaded_iter:
# with open(scene_info.ply_path, 'rb') as src_file, open(os.path.join(self.model_path, "input.ply") , 'wb') as dest_file:
# dest_file.write(src_file.read())
# json_cams = []
# camlist = []
# if scene_info.test_cameras:
# camlist.extend(scene_info.test_cameras)
# if scene_info.train_cameras:
# camlist.extend(scene_info.train_cameras)
# for id, cam in enumerate(camlist):
# json_cams.append(camera_to_JSON(id, cam))
# with open(os.path.join(self.model_path, "cameras.json"), 'w') as file:
# json.dump(json_cams, file)
# if shuffle:
# random.shuffle(scene_info.train_cameras) # Multi-res consistent random shuffling
# random.shuffle(scene_info.test_cameras) # Multi-res consistent random shuffling
self.dataset_type = dataset_type
self.cameras_extent = scene_info.nerf_normalization["radius"]
# for resolution_scale in resolution_scales:
# print("Loading Training Cameras")
# self.train_cameras[resolution_scale] = cameraList_from_camInfos(scene_info.train_cameras, resolution_scale, args)
# print("Loading Test Cameras")
# self.test_cameras[resolution_scale] = cameraList_from_camInfos(scene_info.test_cameras, resolution_scale, args)
# print("Loading Video Cameras")
# self.video_cameras[resolution_scale] = cameraList_from_camInfos(scene_info.video_cameras, resolution_scale, args)
print("Loading Training Cameras")
self.train_camera = FourDGSdataset(scene_info.train_cameras, args)
self.train_camera = FourDGSdataset(scene_info.train_cameras, args, dataset_type)
print("Loading Test Cameras")
self.test_camera = FourDGSdataset(scene_info.test_cameras, args)
self.test_camera = FourDGSdataset(scene_info.test_cameras, args, dataset_type)
print("Loading Video Cameras")
self.video_camera = cameraList_from_camInfos(scene_info.video_cameras,-1,args)
self.video_camera = FourDGSdataset(scene_info.video_cameras, args, dataset_type)
# self.video_camera = cameraList_from_camInfos(scene_info.video_cameras,-1,args)
xyz_max = scene_info.point_cloud.points.max(axis=0)
xyz_min = scene_info.point_cloud.points.min(axis=0)
self.gaussians._deformation.deformation_net.grid.set_aabb(xyz_max,xyz_min)
if args.add_points:
print("add points.")
# breakpoint()
scene_info = scene_info._replace(point_cloud=add_points(scene_info.point_cloud, xyz_max=xyz_max, xyz_min=xyz_min))
self.gaussians._deformation.deformation_net.set_aabb(xyz_max,xyz_min)
if self.loaded_iter:
self.gaussians.load_ply(os.path.join(self.model_path,
"point_cloud",
@ -100,16 +87,6 @@ class Scene:
"point_cloud",
"iteration_" + str(self.loaded_iter),
))
# elif load_coarse:
# self.gaussians.load_ply(os.path.join(self.model_path,
# "point_cloud",
# "coarse_iteration_" + str(load_coarse),
# "point_cloud.ply"))
# self.gaussians.load_model(os.path.join(self.model_path,
# "point_cloud",
# "coarse_iteration_" + str(load_coarse),
# ))
# print("load coarse stage gaussians")
else:
self.gaussians.create_from_pcd(scene_info.point_cloud, self.cameras_extent, self.maxtime)

307
scene/camera.py Normal file
View File

@ -0,0 +1,307 @@
import numpy as np
import os,sys,time
import torch
import torch.nn.functional as torch_F
import collections
from easydict import EasyDict as edict
import util
from util import log,debug
class Pose():
"""
A class of operations on camera poses (PyTorch tensors with shape [...,3,4])
each [3,4] camera pose takes the form of [R|t]
"""
def __call__(self,R=None,t=None):
# construct a camera pose from the given R and/or t
assert(R is not None or t is not None)
if R is None:
if not isinstance(t,torch.Tensor): t = torch.tensor(t)
R = torch.eye(3,device=t.device).repeat(*t.shape[:-1],1,1)
elif t is None:
if not isinstance(R,torch.Tensor): R = torch.tensor(R)
t = torch.zeros(R.shape[:-1],device=R.device)
else:
if not isinstance(R,torch.Tensor): R = torch.tensor(R)
if not isinstance(t,torch.Tensor): t = torch.tensor(t)
assert(R.shape[:-1]==t.shape and R.shape[-2:]==(3,3))
R = R.float()
t = t.float()
pose = torch.cat([R,t[...,None]],dim=-1) # [...,3,4]
assert(pose.shape[-2:]==(3,4))
return pose
def invert(self,pose,use_inverse=False):
# invert a camera pose
R,t = pose[...,:3],pose[...,3:]
R_inv = R.inverse() if use_inverse else R.transpose(-1,-2)
t_inv = (-R_inv@t)[...,0]
pose_inv = self(R=R_inv,t=t_inv)
return pose_inv
def compose(self,pose_list):
# compose a sequence of poses together
# pose_new(x) = poseN o ... o pose2 o pose1(x)
pose_new = pose_list[0]
for pose in pose_list[1:]:
pose_new = self.compose_pair(pose_new,pose)
return pose_new
def compose_pair(self,pose_a,pose_b):
# pose_new(x) = pose_b o pose_a(x)
R_a,t_a = pose_a[...,:3],pose_a[...,3:]
R_b,t_b = pose_b[...,:3],pose_b[...,3:]
R_new = R_b@R_a
t_new = (R_b@t_a+t_b)[...,0]
pose_new = self(R=R_new,t=t_new)
return pose_new
class Lie():
"""
Lie algebra for SO(3) and SE(3) operations in PyTorch
"""
def so3_to_SO3(self,w): # [...,3]
wx = self.skew_symmetric(w)
theta = w.norm(dim=-1)[...,None,None]
I = torch.eye(3,device=w.device,dtype=torch.float32)
A = self.taylor_A(theta)
B = self.taylor_B(theta)
R = I+A*wx+B*wx@wx
return R
def SO3_to_so3(self,R,eps=1e-7): # [...,3,3]
trace = R[...,0,0]+R[...,1,1]+R[...,2,2]
theta = ((trace-1)/2).clamp(-1+eps,1-eps).acos_()[...,None,None]%np.pi # ln(R) will explode if theta==pi
lnR = 1/(2*self.taylor_A(theta)+1e-8)*(R-R.transpose(-2,-1)) # FIXME: wei-chiu finds it weird
w0,w1,w2 = lnR[...,2,1],lnR[...,0,2],lnR[...,1,0]
w = torch.stack([w0,w1,w2],dim=-1)
return w
def se3_to_SE3(self,wu): # [...,3]
w,u = wu.split([3,3],dim=-1)
wx = self.skew_symmetric(w)
theta = w.norm(dim=-1)[...,None,None]
I = torch.eye(3,device=w.device,dtype=torch.float32)
A = self.taylor_A(theta)
B = self.taylor_B(theta)
C = self.taylor_C(theta)
R = I+A*wx+B*wx@wx
V = I+B*wx+C*wx@wx
Rt = torch.cat([R,(V@u[...,None])],dim=-1)
return Rt
def SE3_to_se3(self,Rt,eps=1e-8): # [...,3,4]
R,t = Rt.split([3,1],dim=-1)
w = self.SO3_to_so3(R)
wx = self.skew_symmetric(w)
theta = w.norm(dim=-1)[...,None,None]
I = torch.eye(3,device=w.device,dtype=torch.float32)
A = self.taylor_A(theta)
B = self.taylor_B(theta)
invV = I-0.5*wx+(1-A/(2*B))/(theta**2+eps)*wx@wx
u = (invV@t)[...,0]
wu = torch.cat([w,u],dim=-1)
return wu
def skew_symmetric(self,w):
w0,w1,w2 = w.unbind(dim=-1)
O = torch.zeros_like(w0)
wx = torch.stack([torch.stack([O,-w2,w1],dim=-1),
torch.stack([w2,O,-w0],dim=-1),
torch.stack([-w1,w0,O],dim=-1)],dim=-2)
return wx
def taylor_A(self,x,nth=10):
# Taylor expansion of sin(x)/x
ans = torch.zeros_like(x)
denom = 1.
for i in range(nth+1):
if i>0: denom *= (2*i)*(2*i+1)
ans = ans+(-1)**i*x**(2*i)/denom
return ans
def taylor_B(self,x,nth=10):
# Taylor expansion of (1-cos(x))/x**2
ans = torch.zeros_like(x)
denom = 1.
for i in range(nth+1):
denom *= (2*i+1)*(2*i+2)
ans = ans+(-1)**i*x**(2*i)/denom
return ans
def taylor_C(self,x,nth=10):
# Taylor expansion of (x-sin(x))/x**3
ans = torch.zeros_like(x)
denom = 1.
for i in range(nth+1):
denom *= (2*i+2)*(2*i+3)
ans = ans+(-1)**i*x**(2*i)/denom
return ans
class Quaternion():
def q_to_R(self,q):
# https://en.wikipedia.org/wiki/Rotation_matrix#Quaternion
qa,qb,qc,qd = q.unbind(dim=-1)
R = torch.stack([torch.stack([1-2*(qc**2+qd**2),2*(qb*qc-qa*qd),2*(qa*qc+qb*qd)],dim=-1),
torch.stack([2*(qb*qc+qa*qd),1-2*(qb**2+qd**2),2*(qc*qd-qa*qb)],dim=-1),
torch.stack([2*(qb*qd-qa*qc),2*(qa*qb+qc*qd),1-2*(qb**2+qc**2)],dim=-1)],dim=-2)
return R
def R_to_q(self,R,eps=1e-8): # [B,3,3]
# https://en.wikipedia.org/wiki/Rotation_matrix#Quaternion
# FIXME: this function seems a bit problematic, need to double-check
row0,row1,row2 = R.unbind(dim=-2)
R00,R01,R02 = row0.unbind(dim=-1)
R10,R11,R12 = row1.unbind(dim=-1)
R20,R21,R22 = row2.unbind(dim=-1)
t = R[...,0,0]+R[...,1,1]+R[...,2,2]
r = (1+t+eps).sqrt()
qa = 0.5*r
qb = (R21-R12).sign()*0.5*(1+R00-R11-R22+eps).sqrt()
qc = (R02-R20).sign()*0.5*(1-R00+R11-R22+eps).sqrt()
qd = (R10-R01).sign()*0.5*(1-R00-R11+R22+eps).sqrt()
q = torch.stack([qa,qb,qc,qd],dim=-1)
for i,qi in enumerate(q):
if torch.isnan(qi).any():
K = torch.stack([torch.stack([R00-R11-R22,R10+R01,R20+R02,R12-R21],dim=-1),
torch.stack([R10+R01,R11-R00-R22,R21+R12,R20-R02],dim=-1),
torch.stack([R20+R02,R21+R12,R22-R00-R11,R01-R10],dim=-1),
torch.stack([R12-R21,R20-R02,R01-R10,R00+R11+R22],dim=-1)],dim=-2)/3.0
K = K[i]
eigval,eigvec = torch.linalg.eigh(K)
V = eigvec[:,eigval.argmax()]
q[i] = torch.stack([V[3],V[0],V[1],V[2]])
return q
def invert(self,q):
qa,qb,qc,qd = q.unbind(dim=-1)
norm = q.norm(dim=-1,keepdim=True)
q_inv = torch.stack([qa,-qb,-qc,-qd],dim=-1)/norm**2
return q_inv
def product(self,q1,q2): # [B,4]
q1a,q1b,q1c,q1d = q1.unbind(dim=-1)
q2a,q2b,q2c,q2d = q2.unbind(dim=-1)
hamil_prod = torch.stack([q1a*q2a-q1b*q2b-q1c*q2c-q1d*q2d,
q1a*q2b+q1b*q2a+q1c*q2d-q1d*q2c,
q1a*q2c-q1b*q2d+q1c*q2a+q1d*q2b,
q1a*q2d+q1b*q2c-q1c*q2b+q1d*q2a],dim=-1)
return hamil_prod
pose = Pose()
lie = Lie()
quaternion = Quaternion()
def to_hom(X):
# get homogeneous coordinates of the input
X_hom = torch.cat([X,torch.ones_like(X[...,:1])],dim=-1)
return X_hom
# basic operations of transforming 3D points between world/camera/image coordinates
def world2cam(X,pose): # [B,N,3]
X_hom = to_hom(X)
return X_hom@pose.transpose(-1,-2)
def cam2img(X,cam_intr):
return X@cam_intr.transpose(-1,-2)
def img2cam(X,cam_intr):
return X@cam_intr.inverse().transpose(-1,-2)
def cam2world(X,pose):
X_hom = to_hom(X)
pose_inv = Pose().invert(pose)
return X_hom@pose_inv.transpose(-1,-2)
def angle_to_rotation_matrix(a,axis):
# get the rotation matrix from Euler angle around specific axis
roll = dict(X=1,Y=2,Z=0)[axis]
O = torch.zeros_like(a)
I = torch.ones_like(a)
M = torch.stack([torch.stack([a.cos(),-a.sin(),O],dim=-1),
torch.stack([a.sin(),a.cos(),O],dim=-1),
torch.stack([O,O,I],dim=-1)],dim=-2)
M = M.roll((roll,roll),dims=(-2,-1))
return M
def get_center_and_ray(opt,pose,intr=None): # [HW,2]
# given the intrinsic/extrinsic matrices, get the camera center and ray directions]
assert(opt.camera.model=="perspective")
with torch.no_grad():
# compute image coordinate grid
y_range = torch.arange(opt.H,dtype=torch.float32,device=opt.device).add_(0.5)
x_range = torch.arange(opt.W,dtype=torch.float32,device=opt.device).add_(0.5)
Y,X = torch.meshgrid(y_range,x_range) # [H,W]
xy_grid = torch.stack([X,Y],dim=-1).view(-1,2) # [HW,2]
# compute center and ray
batch_size = len(pose)
xy_grid = xy_grid.repeat(batch_size,1,1) # [B,HW,2]
grid_3D = img2cam(to_hom(xy_grid),intr) # [B,HW,3]
center_3D = torch.zeros_like(grid_3D) # [B,HW,3]
# transform from camera to world coordinates
grid_3D = cam2world(grid_3D,pose) # [B,HW,3]
center_3D = cam2world(center_3D,pose) # [B,HW,3]
ray = grid_3D-center_3D # [B,HW,3]
return center_3D,ray
def get_3D_points_from_depth(opt,center,ray,depth,multi_samples=False):
if multi_samples: center,ray = center[:,:,None],ray[:,:,None]
# x = c+dv
points_3D = center+ray*depth # [B,HW,3]/[B,HW,N,3]/[N,3]
return points_3D
def convert_NDC(opt,center,ray,intr,near=1):
# shift camera center (ray origins) to near plane (z=1)
# (unlike conventional NDC, we assume the cameras are facing towards the +z direction)
center = center+(near-center[...,2:])/ray[...,2:]*ray
# projection
cx,cy,cz = center.unbind(dim=-1) # [B,HW]
rx,ry,rz = ray.unbind(dim=-1) # [B,HW]
scale_x = intr[:,0,0]/intr[:,0,2] # [B]
scale_y = intr[:,1,1]/intr[:,1,2] # [B]
cnx = scale_x[:,None]*(cx/cz)
cny = scale_y[:,None]*(cy/cz)
cnz = 1-2*near/cz
rnx = scale_x[:,None]*(rx/rz-cx/cz)
rny = scale_y[:,None]*(ry/rz-cy/cz)
rnz = 2*near/cz
center_ndc = torch.stack([cnx,cny,cnz],dim=-1) # [B,HW,3]
ray_ndc = torch.stack([rnx,rny,rnz],dim=-1) # [B,HW,3]
return center_ndc,ray_ndc
def rotation_distance(R1,R2,eps=1e-7):
# http://www.boris-belousov.net/2016/12/01/quat-dist/
R_diff = R1@R2.transpose(-2,-1)
trace = R_diff[...,0,0]+R_diff[...,1,1]+R_diff[...,2,2]
angle = ((trace-1)/2).clamp(-1+eps,1-eps).acos_() # numerical stability near -1/+1
return angle
def procrustes_analysis(X0,X1): # [N,3]
# translation
t0 = X0.mean(dim=0,keepdim=True)
t1 = X1.mean(dim=0,keepdim=True)
X0c = X0-t0
X1c = X1-t1
# scale
s0 = (X0c**2).sum(dim=-1).mean().sqrt()
s1 = (X1c**2).sum(dim=-1).mean().sqrt()
X0cs = X0c/s0
X1cs = X1c/s1
# rotation (use double for SVD, float loses precision)
U,S,V = (X0cs.t()@X1cs).double().svd(some=True)
R = (U@V.t()).float()
if R.det()<0: R[2] *= -1
# align X1 to X0: X1to0 = (X1-t1)/s1@R.t()*s0+t0
sim3 = edict(t0=t0[0],t1=t1[0],s0=s0,s1=s1,R=R)
return sim3
def get_novel_view_poses(opt,pose_anchor,N=60,scale=1):
# create circular viewpoints (small oscillations)
theta = torch.arange(N)/N*2*np.pi
R_x = angle_to_rotation_matrix((theta.sin()*0.05).asin(),"X")
R_y = angle_to_rotation_matrix((theta.cos()*0.05).asin(),"Y")
pose_rot = pose(R=R_y@R_x)
pose_shift = pose(t=[0,0,-4*scale])
pose_shift2 = pose(t=[0,0,3.8*scale])
pose_oscil = pose.compose([pose_shift,pose_rot,pose_shift2])
pose_novel = pose.compose([pose_oscil,pose_anchor.cpu()[None]])
return pose_novel

10
scene/cameras.py
View File

@ -17,7 +17,8 @@ from utils.graphics_utils import getWorld2View2, getProjectionMatrix
class Camera(nn.Module):
def __init__(self, colmap_id, R, T, FoVx, FoVy, image, gt_alpha_mask,
image_name, uid,
trans=np.array([0.0, 0.0, 0.0]), scale=1.0, data_device = "cuda", time = 0
trans=np.array([0.0, 0.0, 0.0]), scale=1.0, data_device = "cuda", time = 0,
mask = None, depth=None
):
super(Camera, self).__init__()
@ -35,7 +36,8 @@ class Camera(nn.Module):
print(e)
print(f"[Warning] Custom device {data_device} failed, fallback to default cuda device" )
self.data_device = torch.device("cuda")
self.original_image = image.clamp(0.0, 1.0)
self.original_image = image.clamp(0.0, 1.0)[:3,:,:]
# breakpoint()
# .to(self.data_device)
self.image_width = self.original_image.shape[2]
self.image_height = self.original_image.shape[1]
@ -46,8 +48,8 @@ class Camera(nn.Module):
else:
self.original_image *= torch.ones((1, self.image_height, self.image_width))
# , device=self.data_device)
self.depth = depth
self.mask = mask
self.zfar = 100.0
self.znear = 0.01

42
scene/dataset.py
View File

@ -10,27 +10,37 @@ class FourDGSdataset(Dataset):
def __init__(
self,
dataset,
args
args,
dataset_type
):
self.dataset = dataset
self.args = args
self.dataset_type=dataset_type
def __getitem__(self, index):
# breakpoint()
try:
image, w2c, time = self.dataset[index]
R,T = w2c
FovX = focal2fov(self.dataset.focal[0], image.shape[2])
FovY = focal2fov(self.dataset.focal[0], image.shape[1])
except:
caminfo = self.dataset[index]
image = caminfo.image
R = caminfo.R
T = caminfo.T
FovX = caminfo.FovX
FovY = caminfo.FovY
time = caminfo.time
return Camera(colmap_id=index,R=R,T=T,FoVx=FovX,FoVy=FovY,image=image,gt_alpha_mask=None,
image_name=f"{index}",uid=index,data_device=torch.device("cuda"),time=time)
if self.dataset_type != "PanopticSports":
try:
image, w2c, time = self.dataset[index]
R,T = w2c
FovX = focal2fov(self.dataset.focal[0], image.shape[2])
FovY = focal2fov(self.dataset.focal[0], image.shape[1])
mask=None
except:
caminfo = self.dataset[index]
image = caminfo.image
R = caminfo.R
T = caminfo.T
FovX = caminfo.FovX
FovY = caminfo.FovY
time = caminfo.time
mask = caminfo.mask
return Camera(colmap_id=index,R=R,T=T,FoVx=FovX,FoVy=FovY,image=image,gt_alpha_mask=None,
image_name=f"{index}",uid=index,data_device=torch.device("cuda"),time=time,
mask=mask)
else:
return self.dataset[index]
def __len__(self):
return len(self.dataset)

278
scene/dataset_readers.py
View File

@ -12,6 +12,8 @@
import os
import sys
from PIL import Image
from scene.cameras import Camera
from typing import NamedTuple
from scene.colmap_loader import read_extrinsics_text, read_intrinsics_text, qvec2rotmat, \
read_extrinsics_binary, read_intrinsics_binary, read_points3D_binary, read_points3D_text
@ -40,6 +42,7 @@ class CameraInfo(NamedTuple):
width: int
height: int
time : float
mask: np.array
class SceneInfo(NamedTuple):
point_cloud: BasicPointCloud
@ -70,7 +73,7 @@ def getNerfppNorm(cam_info):
radius = diagonal * 1.1
translate = -center
# breakpoint()
return {"translate": translate, "radius": radius}
def readColmapCameras(cam_extrinsics, cam_intrinsics, images_folder):
@ -113,7 +116,7 @@ def readColmapCameras(cam_extrinsics, cam_intrinsics, images_folder):
image = PILtoTorch(image,None)
cam_info = CameraInfo(uid=uid, R=R, T=T, FovY=FovY, FovX=FovX, image=image,
image_path=image_path, image_name=image_name, width=width, height=height,
time = 0)
time = 0, mask=None)
cam_infos.append(cam_info)
sys.stdout.write('\n')
return cam_infos
@ -130,11 +133,12 @@ def storePly(path, xyz, rgb):
# Define the dtype for the structured array
dtype = [('x', 'f4'), ('y', 'f4'), ('z', 'f4'),
('nx', 'f4'), ('ny', 'f4'), ('nz', 'f4'),
('red', 'u1'), ('green', 'u1'), ('blue', 'u1')]
('red', 'f4'), ('green', 'f4'), ('blue', 'f4')]
normals = np.zeros_like(xyz)
elements = np.empty(xyz.shape[0], dtype=dtype)
# breakpoint()
attributes = np.concatenate((xyz, normals, rgb), axis=1)
elements[:] = list(map(tuple, attributes))
@ -158,7 +162,7 @@ def readColmapSceneInfo(path, images, eval, llffhold=8):
reading_dir = "images" if images == None else images
cam_infos_unsorted = readColmapCameras(cam_extrinsics=cam_extrinsics, cam_intrinsics=cam_intrinsics, images_folder=os.path.join(path, reading_dir))
cam_infos = sorted(cam_infos_unsorted.copy(), key = lambda x : x.image_name)
# breakpoint()
if eval:
train_cam_infos = [c for idx, c in enumerate(cam_infos) if idx % llffhold != 0]
test_cam_infos = [c for idx, c in enumerate(cam_infos) if idx % llffhold == 0]
@ -184,7 +188,7 @@ def readColmapSceneInfo(path, images, eval, llffhold=8):
except:
pcd = None
scene_info = SceneInfo(point_cloud=pcd,
train_cameras=train_cam_infos,
test_cameras=test_cam_infos,
@ -219,11 +223,16 @@ def generateCamerasFromTransforms(path, template_transformsfile, extension, maxt
return c2w
cam_infos = []
# generate render poses and times
render_poses = torch.stack([pose_spherical(angle, -30.0, 4.0) for angle in np.linspace(-180,180,40+1)[:-1]], 0)
render_poses = torch.stack([pose_spherical(angle, -30.0, 4.0) for angle in np.linspace(-180,180,160+1)[:-1]], 0)
render_times = torch.linspace(0,maxtime,render_poses.shape[0])
with open(os.path.join(path, template_transformsfile)) as json_file:
template_json = json.load(json_file)
fovx = template_json["camera_angle_x"]
try:
fovx = template_json["camera_angle_x"]
except:
fovx = focal2fov(template_json["fl_x"], template_json['w'])
print("hello!!!!")
# breakpoint()
# load a single image to get image info.
for idx, frame in enumerate(template_json["frames"]):
cam_name = os.path.join(path, frame["file_path"] + extension)
@ -245,15 +254,17 @@ def generateCamerasFromTransforms(path, template_transformsfile, extension, maxt
FovX = fovx
cam_infos.append(CameraInfo(uid=idx, R=R, T=T, FovY=FovY, FovX=FovX, image=image,
image_path=None, image_name=None, width=image.shape[1], height=image.shape[2],
time = time))
time = time, mask=None))
return cam_infos
def readCamerasFromTransforms(path, transformsfile, white_background, extension=".png", mapper = {}):
cam_infos = []
with open(os.path.join(path, transformsfile)) as json_file:
contents = json.load(json_file)
fovx = contents["camera_angle_x"]
try:
fovx = contents["camera_angle_x"]
except:
fovx = focal2fov(contents['fl_x'],contents['w'])
frames = contents["frames"]
for idx, frame in enumerate(frames):
cam_name = os.path.join(path, frame["file_path"] + extension)
@ -281,7 +292,7 @@ def readCamerasFromTransforms(path, transformsfile, white_background, extension=
cam_infos.append(CameraInfo(uid=idx, R=R, T=T, FovY=FovY, FovX=FovX, image=image,
image_path=image_path, image_name=image_name, width=image.shape[1], height=image.shape[2],
time = time))
time = time, mask=None))
return cam_infos
def read_timeline(path):
@ -314,20 +325,23 @@ def readNerfSyntheticInfo(path, white_background, eval, extension=".png"):
nerf_normalization = getNerfppNorm(train_cam_infos)
ply_path = os.path.join(path, "points3d.ply")
# Since this data set has no colmap data, we start with random points
num_pts = 2000
print(f"Generating random point cloud ({num_pts})...")
# We create random points inside the bounds of the synthetic Blender scenes
xyz = np.random.random((num_pts, 3)) * 2.6 - 1.3
shs = np.random.random((num_pts, 3)) / 255.0
pcd = BasicPointCloud(points=xyz, colors=SH2RGB(shs), normals=np.zeros((num_pts, 3)))
storePly(ply_path, xyz, SH2RGB(shs) * 255)
try:
ply_path = os.path.join(path, "fused.ply")
if not os.path.exists(ply_path):
# Since this data set has no colmap data, we start with random points
num_pts = 2000
print(f"Generating random point cloud ({num_pts})...")
# We create random points inside the bounds of the synthetic Blender scenes
# xyz = np.random.random((num_pts, 3)) * 2.6 - 1.3
xyz = np.random.random((num_pts, 3)) * 0.5 - 0.25
shs = np.random.random((num_pts, 3)) / 255.0
pcd = BasicPointCloud(points=xyz, colors=SH2RGB(shs), normals=np.zeros((num_pts, 3)))
# storePly(ply_path, xyz, SH2RGB(shs) * 255)
else:
pcd = fetchPly(ply_path)
except:
pcd = None
# xyz = -np.array(pcd.points)
# pcd = pcd._replace(points=xyz)
scene_info = SceneInfo(point_cloud=pcd,
train_cameras=train_cam_infos,
@ -353,7 +367,7 @@ def format_infos(dataset,split):
FovY = focal2fov(dataset.focal[0], image.shape[2])
cameras.append(CameraInfo(uid=idx, R=R, T=T, FovY=FovY, FovX=FovX, image=image,
image_path=image_path, image_name=image_name, width=image.shape[2], height=image.shape[1],
time = time))
time = time, mask=None))
return cameras
@ -361,24 +375,30 @@ def format_infos(dataset,split):
def readHyperDataInfos(datadir,use_bg_points,eval):
train_cam_infos = Load_hyper_data(datadir,0.5,use_bg_points,split ="train")
test_cam_infos = Load_hyper_data(datadir,0.5,use_bg_points,split="test")
print("load finished")
train_cam = format_hyper_data(train_cam_infos,"train")
print("format finished")
max_time = train_cam_infos.max_time
video_cam_infos = copy.deepcopy(test_cam_infos)
video_cam_infos.split="video"
ply_path = os.path.join(datadir, "points.npy")
xyz = np.load(ply_path,allow_pickle=True)
xyz -= train_cam_infos.scene_center
xyz *= train_cam_infos.coord_scale
xyz = xyz.astype(np.float32)
shs = np.random.random((xyz.shape[0], 3)) / 255.0
pcd = BasicPointCloud(points=xyz, colors=SH2RGB(shs), normals=np.zeros((xyz.shape[0], 3)))
# ply_path = os.path.join(datadir, "points.npy")
# xyz = np.load(ply_path,allow_pickle=True)
# xyz -= train_cam_infos.scene_center
# xyz *= train_cam_infos.coord_scale
# xyz = xyz.astype(np.float32)
# shs = np.random.random((xyz.shape[0], 3)) / 255.0
# pcd = BasicPointCloud(points=xyz, colors=SH2RGB(shs), normals=np.zeros((xyz.shape[0], 3)))
ply_path = os.path.join(datadir, "points3D_downsample.ply")
# ply_path = os.path.join(datadir, "points3D.ply")
pcd = fetchPly(ply_path)
xyz = np.array(pcd.points)
# xyz -= train_cam_infos.scene_center
# xyz *= train_cam_infos.coord_scale
pcd = pcd._replace(points=xyz)
nerf_normalization = getNerfppNorm(train_cam)
plot_camera_orientations(train_cam_infos, pcd.points)
scene_info = SceneInfo(point_cloud=pcd,
train_cameras=train_cam_infos,
test_cameras=test_cam_infos,
@ -411,14 +431,33 @@ def format_render_poses(poses,data_infos):
FovY = focal2fov(data_infos.focal[0], image.shape[1])
cameras.append(CameraInfo(uid=idx, R=R, T=T, FovY=FovY, FovX=FovX, image=image,
image_path=image_path, image_name=image_name, width=image.shape[2], height=image.shape[1],
time = time))
time = time, mask=None))
return cameras
# plydata = PlyData.read(path)
# vertices = plydata['vertex']
# positions = np.vstack([vertices['x'], vertices['y'], vertices['z']]).T
# colors = np.vstack([vertices['red'], vertices['green'], vertices['blue']]).T / 255.0
# normals = np.vstack([vertices['nx'], vertices['ny'], vertices['nz']]).T
# return BasicPointCloud(points=positions, colors=colors, normals=normals)
def add_points(pointsclouds, xyz_min, xyz_max):
add_points = (np.random.random((100000, 3)))* (xyz_max-xyz_min) + xyz_min
add_points = add_points.astype(np.float32)
addcolors = np.random.random((100000, 3)).astype(np.float32)
addnormals = np.random.random((100000, 3)).astype(np.float32)
# breakpoint()
new_points = np.vstack([pointsclouds.points,add_points])
new_colors = np.vstack([pointsclouds.colors,addcolors])
new_normals = np.vstack([pointsclouds.normals,addnormals])
pointsclouds=pointsclouds._replace(points=new_points)
pointsclouds=pointsclouds._replace(colors=new_colors)
pointsclouds=pointsclouds._replace(normals=new_normals)
return pointsclouds
# breakpoint()
# new_
def readdynerfInfo(datadir,use_bg_points,eval):
# loading all the data follow hexplane format
ply_path = os.path.join(datadir, "points3d.ply")
# ply_path = os.path.join(datadir, "points3D_dense.ply")
ply_path = os.path.join(datadir, "points3D_downsample2.ply")
from scene.neural_3D_dataset_NDC import Neural3D_NDC_Dataset
train_dataset = Neural3D_NDC_Dataset(
datadir,
@ -446,24 +485,26 @@ def readdynerfInfo(datadir,use_bg_points,eval):
# create pcd
# if not os.path.exists(ply_path):
# Since this data set has no colmap data, we start with random points
num_pts = 2000
print(f"Generating random point cloud ({num_pts})...")
threshold = 3
# num_pts = 2000
# print(f"Generating random point cloud ({num_pts})...")
# threshold = 3
# xyz_max = np.array([1.5*threshold, 1.5*threshold, 1.5*threshold])
# xyz_min = np.array([-1.5*threshold, -1.5*threshold, -3*threshold])
xyz_max = np.array([1.5*threshold, 1.5*threshold, 1.5*threshold])
xyz_min = np.array([-1.5*threshold, -1.5*threshold, -1.5*threshold])
# xyz_max = np.array([1.5*threshold, 1.5*threshold, 1.5*threshold])
# xyz_min = np.array([-1.5*threshold, -1.5*threshold, -1.5*threshold])
# We create random points inside the bounds of the synthetic Blender scenes
xyz = (np.random.random((num_pts, 3)))* (xyz_max-xyz_min) + xyz_min
print("point cloud initialization:",xyz.max(axis=0),xyz.min(axis=0))
shs = np.random.random((num_pts, 3)) / 255.0
pcd = BasicPointCloud(points=xyz, colors=SH2RGB(shs), normals=np.zeros((num_pts, 3)))
storePly(ply_path, xyz, SH2RGB(shs) * 255)
try:
# xyz = np.load
pcd = fetchPly(ply_path)
except:
pcd = None
# xyz = (np.random.random((num_pts, 3)))* (xyz_max-xyz_min) + xyz_min
# print("point cloud initialization:",xyz.max(axis=0),xyz.min(axis=0))
# shs = np.random.random((num_pts, 3)) / 255.0
# pcd = BasicPointCloud(points=xyz, colors=SH2RGB(shs), normals=np.zeros((num_pts, 3)))
# storePly(ply_path, xyz, SH2RGB(shs) * 255)
# xyz = np.load
pcd = fetchPly(ply_path)
print("origin points,",pcd.points.shape[0])
print("after points,",pcd.points.shape[0])
scene_info = SceneInfo(point_cloud=pcd,
train_cameras=train_dataset,
test_cameras=test_dataset,
@ -473,9 +514,132 @@ def readdynerfInfo(datadir,use_bg_points,eval):
maxtime=300
)
return scene_info
def setup_camera(w, h, k, w2c, near=0.01, far=100):
from diff_gaussian_rasterization import GaussianRasterizationSettings as Camera
fx, fy, cx, cy = k[0][0], k[1][1], k[0][2], k[1][2]
w2c = torch.tensor(w2c).cuda().float()
cam_center = torch.inverse(w2c)[:3, 3]
w2c = w2c.unsqueeze(0).transpose(1, 2)
opengl_proj = torch.tensor([[2 * fx / w, 0.0, -(w - 2 * cx) / w, 0.0],
[0.0, 2 * fy / h, -(h - 2 * cy) / h, 0.0],
[0.0, 0.0, far / (far - near), -(far * near) / (far - near)],
[0.0, 0.0, 1.0, 0.0]]).cuda().float().unsqueeze(0).transpose(1, 2)
full_proj = w2c.bmm(opengl_proj)
cam = Camera(
image_height=h,
image_width=w,
tanfovx=w / (2 * fx),
tanfovy=h / (2 * fy),
bg=torch.tensor([0, 0, 0], dtype=torch.float32, device="cuda"),
scale_modifier=1.0,
viewmatrix=w2c,
projmatrix=full_proj,
sh_degree=0,
campos=cam_center,
prefiltered=False,
debug=True
)
return cam
def plot_camera_orientations(cam_list, xyz):
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
# ax2 = fig.add_subplot(122, projection='3d')
# xyz = xyz[xyz[:,0]<1]
threshold=2
xyz = xyz[(xyz[:, 0] >= -threshold) & (xyz[:, 0] <= threshold) &
(xyz[:, 1] >= -threshold) & (xyz[:, 1] <= threshold) &
(xyz[:, 2] >= -threshold) & (xyz[:, 2] <= threshold)]
ax.scatter(xyz[:,0],xyz[:,1],xyz[:,2],c='r',s=0.1)
for cam in tqdm(cam_list):
# 提取 R 和 T
R = cam.R
T = cam.T
# print(R,T)
# breakpoint()
# 计算相机朝向（一个单位向量）
direction = R @ np.array([0, 0, 1])
# 绘制相机位置和朝向
ax.quiver(T[0], T[1], T[2], direction[0], direction[1], direction[2], length=1)
ax.set_xlabel('X Axis')
ax.set_ylabel('Y Axis')
ax.set_zlabel('Z Axis')
plt.savefig("output.png")
# breakpoint()
def readPanopticmeta(datadir, json_path):
with open(os.path.join(datadir,json_path)) as f:
test_meta = json.load(f)
w = test_meta['w']
h = test_meta['h']
max_time = len(test_meta['fn'])
cam_infos = []
for index in range(len(test_meta['fn'])):
focals = test_meta['k'][index]
w2cs = test_meta['w2c'][index]
fns = test_meta['fn'][index]
cam_ids = test_meta['cam_id'][index]
time = index / len(test_meta['fn'])
# breakpoint()
for focal, w2c, fn, cam in zip(focals, w2cs, fns, cam_ids):
image_path = os.path.join(datadir,"ims")
image_name=fn
# breakpoint()
image = Image.open(os.path.join(datadir,"ims",fn))
im_data = np.array(image.convert("RGBA"))
# breakpoint()
im_data = PILtoTorch(im_data,None)[:3,:,:]
# breakpoint()
# print(w2c,focal,image_name)
camera = setup_camera(w, h, focal, w2c)
cam_infos.append({
"camera":camera,
"time":time,
"image":im_data})
cam_centers = np.linalg.inv(test_meta['w2c'][0])[:, :3, 3] # Get scene radius
scene_radius = 1.1 * np.max(np.linalg.norm(cam_centers - np.mean(cam_centers, 0)[None], axis=-1))
# breakpoint()
return cam_infos, max_time, scene_radius
def readPanopticSportsinfos(datadir):
train_cam_infos, max_time, scene_radius = readPanopticmeta(datadir, "train_meta.json")
test_cam_infos,_, _ = readPanopticmeta(datadir, "test_meta.json")
nerf_normalization = {
"radius":scene_radius,
"translate":torch.tensor([0,0,0])
}
ply_path = os.path.join(datadir, "pointd3D.ply")
# Since this data set has no colmap data, we start with random points
plz_path = os.path.join(datadir, "init_pt_cld.npz")
data = np.load(plz_path)["data"]
xyz = data[:,:3]
rgb = data[:,3:6]
num_pts = xyz.shape[0]
pcd = BasicPointCloud(points=xyz, colors=rgb, normals=np.ones((num_pts, 3)))
storePly(ply_path, xyz, rgb)
# pcd = fetchPly(ply_path)
# breakpoint()
scene_info = SceneInfo(point_cloud=pcd,
train_cameras=train_cam_infos,
test_cameras=test_cam_infos,
video_cameras=test_cam_infos,
nerf_normalization=nerf_normalization,
ply_path=ply_path,
maxtime=max_time,
)
return scene_info
sceneLoadTypeCallbacks = {
"Colmap": readColmapSceneInfo,
"Blender" : readNerfSyntheticInfo,
"dynerf" : readdynerfInfo,
"nerfies": readHyperDataInfos, # NeRFies & HyperNeRF dataset proposed by [https://github.com/google/hypernerf/releases/tag/v0.1]
"PanopticSports" : readPanopticSportsinfos
}

179
scene/deformation.py
View File

@ -8,88 +8,144 @@ import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.cpp_extension import load
import torch.nn.init as init
from utils.graphics_utils import apply_rotation, batch_quaternion_multiply
from scene.hexplane import HexPlaneField
from scene.grid import DenseGrid
# from scene.grid import HashHexPlane
class Deformation(nn.Module):
def __init__(self, D=8, W=256, input_ch=27, input_ch_time=9, skips=[], args=None):
def __init__(self, D=8, W=256, input_ch=27, input_ch_time=9, grid_pe=0, skips=[], args=None):
super(Deformation, self).__init__()
self.D = D
self.W = W
self.input_ch = input_ch
self.input_ch_time = input_ch_time
self.skips = skips
self.grid_pe = grid_pe
self.no_grid = args.no_grid
self.grid = HexPlaneField(args.bounds, args.kplanes_config, args.multires)
self.pos_deform, self.scales_deform, self.rotations_deform, self.opacity_deform = self.create_net()
self.args = args
def create_net(self):
self.args = args
# self.args.empty_voxel=True
if self.args.empty_voxel:
self.empty_voxel = DenseGrid(channels=1, world_size=[64,64,64])
if self.args.static_mlp:
self.static_mlp = nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 1))
self.ratio=0
self.create_net()
@property
def get_aabb(self):
return self.grid.get_aabb
def set_aabb(self, xyz_max, xyz_min):
print("Deformation Net Set aabb",xyz_max, xyz_min)
self.grid.set_aabb(xyz_max, xyz_min)
if self.args.empty_voxel:
self.empty_voxel.set_aabb(xyz_max, xyz_min)
def create_net(self):
mlp_out_dim = 0
if self.grid_pe !=0:
grid_out_dim = self.grid.feat_dim+(self.grid.feat_dim)*2
else:
grid_out_dim = self.grid.feat_dim
if self.no_grid:
self.feature_out = [nn.Linear(4,self.W)]
else:
self.feature_out = [nn.Linear(mlp_out_dim + self.grid.feat_dim ,self.W)]
self.feature_out = [nn.Linear(mlp_out_dim + grid_out_dim ,self.W)]
for i in range(self.D-1):
self.feature_out.append(nn.ReLU())
self.feature_out.append(nn.Linear(self.W,self.W))
self.feature_out = nn.Sequential(*self.feature_out)
output_dim = self.W
return \
nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 3)),\
nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 3)),\
nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 4)), \
nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 1))
def query_time(self, rays_pts_emb, scales_emb, rotations_emb, time_emb):
self.pos_deform = nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 3))
self.scales_deform = nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 3))
self.rotations_deform = nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 4))
self.opacity_deform = nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 1))
self.shs_deform = nn.Sequential(nn.ReLU(),nn.Linear(self.W,self.W),nn.ReLU(),nn.Linear(self.W, 16*3))
def query_time(self, rays_pts_emb, scales_emb, rotations_emb, time_feature, time_emb):
if self.no_grid:
h = torch.cat([rays_pts_emb[:,:3],time_emb[:,:1]],-1)
else:
grid_feature = self.grid(rays_pts_emb[:,:3], time_emb[:,:1])
h = grid_feature
# breakpoint()
if self.grid_pe > 1:
grid_feature = poc_fre(grid_feature,self.grid_pe)
hidden = torch.cat([grid_feature],-1)
h = self.feature_out(h)
return h
hidden = self.feature_out(hidden)
def forward(self, rays_pts_emb, scales_emb=None, rotations_emb=None, opacity = None, time_emb=None):
return hidden
@property
def get_empty_ratio(self):
return self.ratio
def forward(self, rays_pts_emb, scales_emb=None, rotations_emb=None, opacity = None,shs_emb=None, time_feature=None, time_emb=None):
if time_emb is None:
return self.forward_static(rays_pts_emb[:,:3])
else:
return self.forward_dynamic(rays_pts_emb, scales_emb, rotations_emb, opacity, time_emb)
return self.forward_dynamic(rays_pts_emb, scales_emb, rotations_emb, opacity, shs_emb, time_feature, time_emb)
def forward_static(self, rays_pts_emb):
grid_feature = self.grid(rays_pts_emb[:,:3])
dx = self.static_mlp(grid_feature)
return rays_pts_emb[:, :3] + dx
def forward_dynamic(self,rays_pts_emb, scales_emb, rotations_emb, opacity_emb, time_emb):
hidden = self.query_time(rays_pts_emb, scales_emb, rotations_emb, time_emb).float()
dx = self.pos_deform(hidden)
pts = rays_pts_emb[:, :3] + dx
if self.args.no_ds:
def forward_dynamic(self,rays_pts_emb, scales_emb, rotations_emb, opacity_emb, shs_emb, time_feature, time_emb):
hidden = self.query_time(rays_pts_emb, scales_emb, rotations_emb, time_feature, time_emb)
if self.args.static_mlp:
mask = self.static_mlp(hidden)
elif self.args.empty_voxel:
mask = self.empty_voxel(rays_pts_emb[:,:3])
else:
mask = torch.ones_like(opacity_emb[:,0]).unsqueeze(-1)
# breakpoint()
if self.args.no_dx:
pts = rays_pts_emb[:,:3]
else:
dx = self.pos_deform(hidden)
pts = torch.zeros_like(rays_pts_emb[:,:3])
pts = rays_pts_emb[:,:3]*mask + dx
if self.args.no_ds :
scales = scales_emb[:,:3]
else:
ds = self.scales_deform(hidden)
scales = scales_emb[:,:3] + ds
if self.args.no_dr:
scales = torch.zeros_like(scales_emb[:,:3])
scales = scales_emb[:,:3]*mask + ds
if self.args.no_dr :
rotations = rotations_emb[:,:4]
else:
dr = self.rotations_deform(hidden)
rotations = rotations_emb[:,:4] + dr
if self.args.no_do:
rotations = torch.zeros_like(rotations_emb[:,:4])
if self.args.apply_rotation:
rotations = batch_quaternion_multiply(rotations_emb, dr)
else:
rotations = rotations_emb[:,:4] + dr
if self.args.no_do :
opacity = opacity_emb[:,:1]
else:
do = self.opacity_deform(hidden)
opacity = opacity_emb[:,:1] + do
# + do
# print("deformation value:","pts:",torch.abs(dx).mean(),"rotation:",torch.abs(dr).mean())
opacity = torch.zeros_like(opacity_emb[:,:1])
opacity = opacity_emb[:,:1]*mask + do
if self.args.no_dshs:
shs = shs_emb
else:
dshs = self.shs_deform(hidden).reshape([shs_emb.shape[0],16,3])
return pts, scales, rotations, opacity
shs = torch.zeros_like(shs_emb)
# breakpoint()
shs = shs_emb*mask.unsqueeze(-1) + dshs
return pts, scales, rotations, opacity, shs
def get_mlp_parameters(self):
parameter_list = []
for name, param in self.named_parameters():
@ -97,8 +153,11 @@ class Deformation(nn.Module):
parameter_list.append(param)
return parameter_list
def get_grid_parameters(self):
return list(self.grid.parameters() )
# + list(self.timegrid.parameters())
parameter_list = []
for name, param in self.named_parameters():
if "grid" in name:
parameter_list.append(param)
return parameter_list
class deform_network(nn.Module):
def __init__(self, args) :
super(deform_network, self).__init__()
@ -110,11 +169,12 @@ class deform_network(nn.Module):
opacity_pe = args.opacity_pe
timenet_width = args.timenet_width
timenet_output = args.timenet_output
grid_pe = args.grid_pe
times_ch = 2*timebase_pe+1
self.timenet = nn.Sequential(
nn.Linear(times_ch, timenet_width), nn.ReLU(),
nn.Linear(timenet_width, timenet_output))
self.deformation_net = Deformation(W=net_width, D=defor_depth, input_ch=(4+3)+((4+3)*scale_rotation_pe)*2, input_ch_time=timenet_output, args=args)
self.deformation_net = Deformation(W=net_width, D=defor_depth, input_ch=(3)+(3*(posbase_pe))*2, grid_pe=grid_pe, input_ch_time=timenet_output, args=args)
self.register_buffer('time_poc', torch.FloatTensor([(2**i) for i in range(timebase_pe)]))
self.register_buffer('pos_poc', torch.FloatTensor([(2**i) for i in range(posbase_pe)]))
self.register_buffer('rotation_scaling_poc', torch.FloatTensor([(2**i) for i in range(scale_rotation_pe)]))
@ -122,26 +182,34 @@ class deform_network(nn.Module):
self.apply(initialize_weights)
# print(self)
def forward(self, point, scales=None, rotations=None, opacity=None, times_sel=None):
if times_sel is not None:
return self.forward_dynamic(point, scales, rotations, opacity, times_sel)
else:
return self.forward_static(point)
def forward(self, point, scales=None, rotations=None, opacity=None, shs=None, times_sel=None):
return self.forward_dynamic(point, scales, rotations, opacity, shs, times_sel)
@property
def get_aabb(self):
return self.deformation_net.get_aabb
@property
def get_empty_ratio(self):
return self.deformation_net.get_empty_ratio
def forward_static(self, points):
points = self.deformation_net(points)
return points
def forward_dynamic(self, point, scales=None, rotations=None, opacity=None, times_sel=None):
def forward_dynamic(self, point, scales=None, rotations=None, opacity=None, shs=None, times_sel=None):
# times_emb = poc_fre(times_sel, self.time_poc)
means3D, scales, rotations, opacity = self.deformation_net( point,
scales,
rotations,
point_emb = poc_fre(point,self.pos_poc)
scales_emb = poc_fre(scales,self.rotation_scaling_poc)
rotations_emb = poc_fre(rotations,self.rotation_scaling_poc)
# time_emb = poc_fre(times_sel, self.time_poc)
# times_feature = self.timenet(time_emb)
means3D, scales, rotations, opacity, shs = self.deformation_net( point_emb,
scales_emb,
rotations_emb,
opacity,
# times_feature,
shs,
None,
times_sel)
return means3D, scales, rotations, opacity
return means3D, scales, rotations, opacity, shs
def get_mlp_parameters(self):
return self.deformation_net.get_mlp_parameters() + list(self.timenet.parameters())
def get_grid_parameters(self):
@ -154,3 +222,10 @@ def initialize_weights(m):
if m.bias is not None:
init.xavier_uniform_(m.weight,gain=1)
# init.constant_(m.bias, 0)
def poc_fre(input_data,poc_buf):
input_data_emb = (input_data.unsqueeze(-1) * poc_buf).flatten(-2)
input_data_sin = input_data_emb.sin()
input_data_cos = input_data_emb.cos()
input_data_emb = torch.cat([input_data, input_data_sin,input_data_cos], -1)
return input_data_emb

146
scene/gaussian_model.py
View File

@ -14,6 +14,7 @@ import numpy as np
from utils.general_utils import inverse_sigmoid, get_expon_lr_func, build_rotation
from torch import nn
import os
import open3d as o3d
from utils.system_utils import mkdir_p
from plyfile import PlyData, PlyElement
from random import randint
@ -21,6 +22,7 @@ from utils.sh_utils import RGB2SH
from simple_knn._C import distCUDA2
from utils.graphics_utils import BasicPointCloud
from utils.general_utils import strip_symmetric, build_scaling_rotation
# from utils.point_utils import addpoint, combine_pointcloud, downsample_point_cloud_open3d, find_indices_in_A
from scene.deformation import deform_network
from scene.regulation import compute_plane_smoothness
class GaussianModel:
@ -135,6 +137,7 @@ class GaussianModel:
def create_from_pcd(self, pcd : BasicPointCloud, spatial_lr_scale : float, time_line: int):
self.spatial_lr_scale = spatial_lr_scale
# breakpoint()
fused_point_cloud = torch.tensor(np.asarray(pcd.points)).float().cuda()
fused_color = RGB2SH(torch.tensor(np.asarray(pcd.colors)).float().cuda())
features = torch.zeros((fused_color.shape[0], 3, (self.max_sh_degree + 1) ** 2)).float().cuda()
@ -418,6 +421,8 @@ class GaussianModel:
padded_grad = torch.zeros((n_init_points), device="cuda")
padded_grad[:grads.shape[0]] = grads.squeeze()
selected_pts_mask = torch.where(padded_grad >= grad_threshold, True, False)
# breakpoint()
selected_pts_mask = torch.logical_and(selected_pts_mask,
torch.max(self.get_scaling, dim=1).values > self.percent_dense*scene_extent)
if not selected_pts_mask.any():
@ -438,12 +443,33 @@ class GaussianModel:
prune_filter = torch.cat((selected_pts_mask, torch.zeros(N * selected_pts_mask.sum(), device="cuda", dtype=bool)))
self.prune_points(prune_filter)
def densify_and_clone(self, grads, grad_threshold, scene_extent):
# Extract points that satisfy the gradient condition
selected_pts_mask = torch.where(torch.norm(grads, dim=-1) >= grad_threshold, True, False)
selected_pts_mask = torch.logical_and(selected_pts_mask,
torch.max(self.get_scaling, dim=1).values <= self.percent_dense*scene_extent)
def densify_and_clone(self, grads, grad_threshold, scene_extent, density_threshold=20, displacement_scale=20, model_path=None, iteration=None, stage=None):
grads_accum_mask = torch.where(torch.norm(grads, dim=-1) >= grad_threshold, True, False)
# 主动增加稀疏点云
# if not hasattr(self,"voxel_size"):
# self.voxel_size = 8
# if not hasattr(self,"density_threshold"):
# self.density_threshold = density_threshold
# if not hasattr(self,"displacement_scale"):
# self.displacement_scale = displacement_scale
# point_cloud = self.get_xyz.detach().cpu()
# sparse_point_mask = self.downsample_point(point_cloud)
# _, low_density_points, new_points, low_density_index = addpoint(point_cloud[sparse_point_mask],density_threshold=self.density_threshold,displacement_scale=self.displacement_scale,iter_pass=0)
# sparse_point_mask = sparse_point_mask.to(grads_accum_mask)
# low_density_index = low_density_index.to(grads_accum_mask)
# if new_points.shape[0] < 100 :
# self.density_threshold /= 2
# self.displacement_scale /= 2
# print("reduce diplacement_scale to: ",self.displacement_scale)
# global_mask = torch.zeros((point_cloud.shape[0]), dtype=torch.bool).to(grads_accum_mask)
# global_mask[sparse_point_mask] = low_density_index
# selected_pts_mask_grow = torch.logical_and(global_mask, grads_accum_mask)
# print("降采样点云:",sparse_point_mask.sum(),"选中的稀疏点云：",global_mask.sum(),"梯度累计点云：",grads_accum_mask.sum(),"选中增长点云：",selected_pts_mask_grow.sum())
# Extract points that satisfy the gradient condition
selected_pts_mask = torch.logical_and(grads_accum_mask,
torch.max(self.get_scaling, dim=1).values <= self.percent_dense*scene_extent)
# breakpoint()
new_xyz = self._xyz[selected_pts_mask]
# - 0.001 * self._xyz.grad[selected_pts_mask]
new_features_dc = self._features_dc[selected_pts_mask]
@ -452,15 +478,111 @@ class GaussianModel:
new_scaling = self._scaling[selected_pts_mask]
new_rotation = self._rotation[selected_pts_mask]
new_deformation_table = self._deformation_table[selected_pts_mask]
# if opt.add_point:
# selected_xyz, grow_xyz = self.add_point_by_mask(selected_pts_mask_grow.to(self.get_xyz.device), self.displacement_scale)
self.densification_postfix(new_xyz, new_features_dc, new_features_rest, new_opacities, new_scaling, new_rotation, new_deformation_table)
# print("被动增加点云：",selected_xyz.shape[0])
# print("主动增加点云：",selected_pts_mask.sum())
# if model_path is not None and iteration is not None:
# point = combine_pointcloud(self.get_xyz.detach().cpu().numpy(), new_xyz.detach().cpu().numpy(), selected_xyz.detach().cpu().numpy())
# write_path = os.path.join(model_path,"add_point_cloud")
# os.makedirs(write_path,exist_ok=True)
# o3d.io.write_point_cloud(os.path.join(write_path,f"iteration_{stage}{iteration}.ply"),point)
# print("write output.")
@property
def get_aabb(self):
return self._deformation.get_aabb
def get_displayment(self,selected_point, point, perturb):
xyz_max, xyz_min = self.get_aabb
displacements = torch.randn(selected_point.shape[0], 3).to(selected_point) * perturb
final_point = selected_point + displacements
mask_a = final_point<xyz_max
mask_b = final_point>xyz_min
mask_c = mask_a & mask_b
mask_d = mask_c.all(dim=1)
final_point = final_point[mask_d]
# while (mask_d.sum()/final_point.shape[0])<0.5:
# perturb/=2
# displacements = torch.randn(selected_point.shape[0], 3).to(selected_point) * perturb
# final_point = selected_point + displacements
# mask_a = final_point<xyz_max
# mask_b = final_point>xyz_min
# mask_c = mask_a & mask_b
# mask_d = mask_c.all(dim=1)
# final_point = final_point[mask_d]
return final_point, mask_d
def add_point_by_mask(self, selected_pts_mask, perturb=0):
selected_xyz = self._xyz[selected_pts_mask]
new_xyz, mask = self.get_displayment(selected_xyz, self.get_xyz.detach(),perturb)
# displacements = torch.randn(selected_xyz.shape[0], 3).to(self._xyz) * perturb
# new_xyz = selected_xyz + displacements
# - 0.001 * self._xyz.grad[selected_pts_mask]
new_features_dc = self._features_dc[selected_pts_mask][mask]
new_features_rest = self._features_rest[selected_pts_mask][mask]
new_opacities = self._opacity[selected_pts_mask][mask]
new_scaling = self._scaling[selected_pts_mask][mask]
new_rotation = self._rotation[selected_pts_mask][mask]
new_deformation_table = self._deformation_table[selected_pts_mask][mask]
self.densification_postfix(new_xyz, new_features_dc, new_features_rest, new_opacities, new_scaling, new_rotation, new_deformation_table)
return selected_xyz, new_xyz
def downsample_point(self, point_cloud):
if not hasattr(self,"voxel_size"):
self.voxel_size = 8
point_downsample = point_cloud
flag = False
while point_downsample.shape[0]>1000:
if flag:
self.voxel_size+=8
point_downsample = downsample_point_cloud_open3d(point_cloud,voxel_size=self.voxel_size)
flag = True
print("point size:",point_downsample.shape[0])
# downsampled_point_mask = torch.eq(point_downsample.view(1,-1,3), point_cloud.view(-1,1,3)).all(dim=1)
downsampled_point_index = find_indices_in_A(point_cloud, point_downsample)
downsampled_point_mask = torch.zeros((point_cloud.shape[0]), dtype=torch.bool).to(point_downsample.device)
downsampled_point_mask[downsampled_point_index]=True
return downsampled_point_mask
def grow(self, density_threshold=20, displacement_scale=20, model_path=None, iteration=None, stage=None):
if not hasattr(self,"voxel_size"):
self.voxel_size = 8
if not hasattr(self,"density_threshold"):
self.density_threshold = density_threshold
if not hasattr(self,"displacement_scale"):
self.displacement_scale = displacement_scale
flag = False
point_cloud = self.get_xyz.detach().cpu()
point_downsample = point_cloud.detach()
downsampled_point_index = self.downsample_point(point_downsample)
_, low_density_points, new_points, low_density_index = addpoint(point_cloud[downsampled_point_index],density_threshold=self.density_threshold,displacement_scale=self.displacement_scale,iter_pass=0)
if new_points.shape[0] < 100 :
self.density_threshold /= 2
self.displacement_scale /= 2
print("reduce diplacement_scale to: ",self.displacement_scale)
elif new_points.shape[0] == 0:
print("no point added")
return
global_mask = torch.zeros((point_cloud.shape[0]), dtype=torch.bool)
global_mask[downsampled_point_index] = low_density_index
global_mask
selected_xyz, new_xyz = self.add_point_by_mask(global_mask.to(self.get_xyz.device), self.displacement_scale)
print("point growing,add point num:",global_mask.sum())
if model_path is not None and iteration is not None:
point = combine_pointcloud(point_cloud, selected_xyz.detach().cpu().numpy(), new_xyz.detach().cpu().numpy())
write_path = os.path.join(model_path,"add_point_cloud")
os.makedirs(write_path,exist_ok=True)
o3d.io.write_point_cloud(os.path.join(write_path,f"iteration_{stage}{iteration}.ply"),point)
return
def prune(self, max_grad, min_opacity, extent, max_screen_size):
prune_mask = (self.get_opacity < min_opacity).squeeze()
# prune_mask_2 = torch.logical_and(self.get_opacity <= inverse_sigmoid(0.101 , dtype=torch.float, device="cuda"), self.get_opacity >= inverse_sigmoid(0.999 , dtype=torch.float, device="cuda"))
# prune_mask = torch.logical_or(prune_mask, prune_mask_2)
# deformation_sum = abs(self._deformation).sum(dim=-1).mean(dim=-1)
# deformation_mask = (deformation_sum < torch.quantile(deformation_sum, torch.tensor([0.5]).to("cuda")))
# prune_mask = prune_mask & deformation_mask
if max_screen_size:
big_points_vs = self.max_radii2D > max_screen_size
big_points_ws = self.get_scaling.max(dim=1).values > 0.1 * extent
@ -470,11 +592,11 @@ class GaussianModel:
self.prune_points(prune_mask)
torch.cuda.empty_cache()
def densify(self, max_grad, min_opacity, extent, max_screen_size):
def densify(self, max_grad, min_opacity, extent, max_screen_size, density_threshold, displacement_scale, model_path=None, iteration=None, stage=None):
grads = self.xyz_gradient_accum / self.denom
grads[grads.isnan()] = 0.0
self.densify_and_clone(grads, max_grad, extent)
self.densify_and_clone(grads, max_grad, extent, density_threshold, displacement_scale, model_path, iteration, stage)
self.densify_and_split(grads, max_grad, extent)
def standard_constaint(self):

91
scene/grid.py Normal file
View File

@ -0,0 +1,91 @@
import os
import time
import functools
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
# import tinycudann as tcnn
parent_dir = os.path.dirname(os.path.abspath(__file__))
''' Dense 3D grid
'''
class DenseGrid(nn.Module):
def __init__(self, channels, world_size, **kwargs):
super(DenseGrid, self).__init__()
self.channels = channels
self.world_size = world_size
# self.xyz_max = xyz_max
# self.xyz_min = xyz_min
# self.register_buffer('xyz_min', torch.Tensor(xyz_min))
# self.register_buffer('xyz_max', torch.Tensor(xyz_max))
self.grid = nn.Parameter(torch.ones([1, channels, *world_size]))
def forward(self, xyz):
'''
xyz: global coordinates to query
'''
shape = xyz.shape[:-1]
xyz = xyz.reshape(1,1,1,-1,3)
ind_norm = ((xyz - self.xyz_min) / (self.xyz_max - self.xyz_min)).flip((-1,)) * 2 - 1
out = F.grid_sample(self.grid, ind_norm, mode='bilinear', align_corners=True)
out = out.reshape(self.channels,-1).T.reshape(*shape,self.channels)
# if self.channels == 1:
# out = out.squeeze(-1)
return out
def scale_volume_grid(self, new_world_size):
if self.channels == 0:
self.grid = nn.Parameter(torch.ones([1, self.channels, *new_world_size]))
else:
self.grid = nn.Parameter(
F.interpolate(self.grid.data, size=tuple(new_world_size), mode='trilinear', align_corners=True))
def set_aabb(self, xyz_max, xyz_min):
self.register_buffer('xyz_min', torch.Tensor(xyz_min))
self.register_buffer('xyz_max', torch.Tensor(xyz_max))
def get_dense_grid(self):
return self.grid
@torch.no_grad()
def __isub__(self, val):
self.grid.data -= val
return self
def extra_repr(self):
return f'channels={self.channels}, world_size={self.world_size}'
# class HashHexPlane(nn.Module):
# def __init__(self,hparams,
# desired_resolution=1024,
# base_solution=128,
# n_levels=4,
# ):
# super(HashHexPlane, self).__init__()
# per_level_scale = np.exp2(np.log2(desired_resolution / base_solution) / (int(n_levels) - 1))
# encoding_2d_config = {
# "otype": "Grid",
# "type": "Hash",
# "n_levels": n_levels,
# "n_features_per_level": 2,
# "base_resolution": base_solution,
# "per_level_scale":per_level_scale,
# }
# self.xy = tcnn.Encoding(n_input_dims=2, encoding_config=encoding_2d_config)
# self.yz = tcnn.Encoding(n_input_dims=2, encoding_config=encoding_2d_config)
# self.xz = tcnn.Encoding(n_input_dims=2, encoding_config=encoding_2d_config)
# self.xt = tcnn.Encoding(n_input_dims=2, encoding_config=encoding_2d_config)
# self.yt = tcnn.Encoding(n_input_dims=2, encoding_config=encoding_2d_config)
# self.zt = tcnn.Encoding(n_input_dims=2, encoding_config=encoding_2d_config)
# self.feat_dim = n_levels * 2 *3
# def forward(self, x, bound):
# x = (x + bound) / (2 * bound) # zyq: map to [0, 1]
# xy_feat = self.xy(x[:, [0, 1]])
# yz_feat = self.yz(x[:, [0, 2]])
# xz_feat = self.xz(x[:, [1, 2]])
# xt_feat = self.xt(x[:, []])
# return torch.cat([xy_feat, yz_feat, xz_feat], dim=-1)

11
scene/hexplane.py
View File

@ -146,19 +146,20 @@ class HexPlaneField(nn.Module):
self.grids.append(gp)
# print(f"Initialized model grids: {self.grids}")
print("feature_dim:",self.feat_dim)
@property
def get_aabb(self):
return self.aabb[0], self.aabb[1]
def set_aabb(self,xyz_max, xyz_min):
aabb = torch.tensor([
xyz_max,
xyz_min
])
self.aabb = nn.Parameter(aabb,requires_grad=True)
],dtype=torch.float32)
self.aabb = nn.Parameter(aabb,requires_grad=False)
print("Voxel Plane: set aabb=",self.aabb)
def get_density(self, pts: torch.Tensor, timestamps: Optional[torch.Tensor] = None):
"""Computes and returns the densities."""
# breakpoint()
pts = normalize_aabb(pts, self.aabb)
pts = torch.cat((pts, timestamps), dim=-1) # [n_rays, n_samples, 4]

71
scene/hyper_loader.py
View File

@ -16,8 +16,9 @@ from typing import NamedTuple
from torch.utils.data import Dataset
from utils.general_utils import PILtoTorch
# from scene.dataset_readers import
import torch.nn.functional as F
from utils.graphics_utils import getWorld2View2, focal2fov, fov2focal
import copy
from utils.pose_utils import smooth_camera_poses
class CameraInfo(NamedTuple):
uid: int
R: np.array
@ -30,6 +31,7 @@ class CameraInfo(NamedTuple):
width: int
height: int
time : float
mask: np.array
class Load_hyper_data(Dataset):
@ -72,7 +74,6 @@ class Load_hyper_data(Dataset):
self.i_test.append(i)
if id in self.train_id:
self.i_train.append(i)
self.all_cam = [meta_json[i]['camera_id'] for i in self.all_img]
self.all_time = [meta_json[i]['warp_id'] for i in self.all_img]
@ -84,21 +85,34 @@ class Load_hyper_data(Dataset):
self.min_time = min(self.all_time)
self.i_video = [i for i in range(len(self.all_img))]
self.i_video.sort()
# all poses
self.all_cam_params = []
for im in self.all_img:
camera = Camera.from_json(f'{datadir}/camera/{im}.json')
camera = camera.scale(ratio)
camera.position -= self.scene_center
camera.position *= self.coord_scale
self.all_cam_params.append(camera)
self.all_img_origin = self.all_img
self.all_depth = [f'{datadir}/depth/{int(1/ratio)}x/{i}.npy' for i in self.all_img]
self.all_img = [f'{datadir}/rgb/{int(1/ratio)}x/{i}.png' for i in self.all_img]
self.h, self.w = self.all_cam_params[0].image_shape
self.map = {}
self.image_one = Image.open(self.all_img[0])
self.image_one_torch = PILtoTorch(self.image_one,None).to(torch.float32)
if os.path.exists(os.path.join(datadir,"covisible")):
self.image_mask = [f'{datadir}/covisible/{int(2)}x/val/{i}.png' for i in self.all_img_origin]
else:
self.image_mask = None
self.generate_video_path()
def generate_video_path(self):
self.select_video_cams = [item for i, item in enumerate(self.all_cam_params) if i % 1 == 0 ]
self.video_path, self.video_time = smooth_camera_poses(self.select_video_cams,10)
# breakpoint()
self.video_path = self.video_path[:500]
self.video_time = self.video_time[:500]
# breakpoint()
def __getitem__(self, index):
if self.split == "train":
return self.load_raw(self.i_train[index])
@ -106,24 +120,26 @@ class Load_hyper_data(Dataset):
elif self.split == "test":
return self.load_raw(self.i_test[index])
elif self.split == "video":
return self.load_video(self.i_video[index])
return self.load_video(index)
def __len__(self):
if self.split == "train":
return len(self.i_train)
elif self.split == "test":
return len(self.i_test)
elif self.split == "video":
# return len(self.i_video)
return len(self.video_v2)
return len(self.video_path)
# return len(self.video_v2)
def load_video(self, idx):
if idx in self.map.keys():
return self.map[idx]
camera = self.all_cam_params[idx]
# camera = self.video_path[idx]
w = self.image_one.size[0]
h = self.image_one.size[1]
# image = PILtoTorch(image,None)
# image = image.to(torch.float32)
time = self.all_time[idx]
time = self.video_time[idx]
# .astype(np.float32)
R = camera.orientation.T
T = - camera.position @ R
FovY = focal2fov(camera.focal_length, self.h)
@ -131,7 +147,7 @@ class Load_hyper_data(Dataset):
image_path = "/".join(self.all_img[idx].split("/")[:-1])
image_name = self.all_img[idx].split("/")[-1]
caminfo = CameraInfo(uid=idx, R=R, T=T, FovY=FovY, FovX=FovX, image=self.image_one_torch,
image_path=image_path, image_name=image_name, width=w, height=h, time=time,
image_path=image_path, image_name=image_name, width=w, height=h, time=time, mask=None
)
self.map[idx] = caminfo
return caminfo
@ -143,7 +159,7 @@ class Load_hyper_data(Dataset):
w = image.size[0]
h = image.size[1]
image = PILtoTorch(image,None)
image = image.to(torch.float32)
image = image.to(torch.float32)[:3,:,:]
time = self.all_time[idx]
R = camera.orientation.T
T = - camera.position @ R
@ -151,8 +167,18 @@ class Load_hyper_data(Dataset):
FovX = focal2fov(camera.focal_length, self.w)
image_path = "/".join(self.all_img[idx].split("/")[:-1])
image_name = self.all_img[idx].split("/")[-1]
if self.image_mask is not None and self.split == "test":
mask = Image.open(self.image_mask[idx])
mask = PILtoTorch(mask,None)
mask = mask.to(torch.float32)[0:1,:,:]
mask = F.interpolate(mask.unsqueeze(0), size=[self.h, self.w], mode='bilinear', align_corners=False).squeeze(0)
else:
mask = None
caminfo = CameraInfo(uid=idx, R=R, T=T, FovY=FovY, FovX=FovX, image=image,
image_path=image_path, image_name=image_name, width=w, height=h, time=time,
image_path=image_path, image_name=image_name, width=w, height=h, time=time, mask=mask
)
self.map[idx] = caminfo
return caminfo
@ -177,12 +203,19 @@ def format_hyper_data(data_class, split):
FovX = focal2fov(camera.focal_length, data_class.w)
image_path = "/".join(data_class.all_img[index].split("/")[:-1])
image_name = data_class.all_img[index].split("/")[-1]
if data_class.image_mask is not None and data_class.split == "test":
mask = Image.open(data_class.image_mask[index])
mask = PILtoTorch(mask,None)
mask = mask.to(torch.float32)[0:1,:,:]
else:
mask = None
cam_info = CameraInfo(uid=uid, R=R, T=T, FovY=FovY, FovX=FovX, image=None,
image_path=image_path, image_name=image_name, width=int(data_class.w), height=int(data_class.h), time=time,
image_path=image_path, image_name=image_name, width=int(data_class.w),
height=int(data_class.h), time=time, mask=mask
)
cam_infos.append(cam_info)
return cam_infos
# matrix = np.linalg.inv(np.array(poses))
# R = -np.transpose(matrix[:3,:3])
# R[:,0] = -R[:,0]
# T = -matrix[:3, 3]
return cam_infos

21
scene/neural_3D_dataset_NDC.py
View File

@ -264,25 +264,26 @@ class Neural3D_NDC_Dataset(Dataset):
self.near_fars = poses_arr[:, -2:]
videos = glob.glob(os.path.join(self.root_dir, "cam*"))
videos = sorted(videos)
# breakpoint()
assert len(videos) == poses_arr.shape[0]
H, W, focal = poses[0, :, -1]
focal = focal / self.downsample
self.focal = [focal, focal]
poses = np.concatenate([poses[..., 1:2], -poses[..., :1], poses[..., 2:4]], -1)
poses, _ = center_poses(
poses, self.blender2opencv
) # Re-center poses so that the average is near the center.
# poses, _ = center_poses(
# poses, self.blender2opencv
# ) # Re-center poses so that the average is near the center.
near_original = self.near_fars.min()
scale_factor = near_original * 0.75
self.near_fars /= (
scale_factor # rescale nearest plane so that it is at z = 4/3.
)
poses[..., 3] /= scale_factor
# near_original = self.near_fars.min()
# scale_factor = near_original * 0.75
# self.near_fars /= (
# scale_factor # rescale nearest plane so that it is at z = 4/3.
# )
# poses[..., 3] /= scale_factor
# Sample N_views poses for validation - NeRF-like camera trajectory.
N_views = 120
N_views = 300
self.val_poses = get_spiral(poses, self.near_fars, N_views=N_views)
# self.val_poses = self.directions
W, H = self.img_wh

95
script.sh Normal file
View File

@ -0,0 +1,95 @@
# bash colmap.sh data/hypernerf/interp/aleks-teapot
# bash colmap.sh data/hypernerf/interp/chickchicken
# bash colmap.sh data/hypernerf/interp/cut-lemon1
# bash colmap.sh data/hypernerf/interp/hand1-dense-v2
# bash colmap.sh data/hypernerf/interp/slice-banana
# bash colmap.sh data/hypernerf/interp/torchocolate
# bash colmap.sh data/hypernerf/virg/broom2
# bash colmap.sh data/hypernerf/virg/peel-banana
# bash colmap.sh data/hypernerf/virg/vrig-3dprinter
# bash colmap.sh data/hypernerf/virg/vrig-chicken
# python scripts/downsample_point.py data/dynerf/coffee_martini/points3D_downsample.ply data/dynerf/coffee_martini/points3D_downsample2.ply
# python scripts/downsample_point.py data/dynerf/flame_salmon_1/points3D_downsample.ply data/dynerf/flame_salmon_1/points3D_downsample2.ply
# python scripts/downsample_point.py data/dynerf/cut_roasted_beef/points3D_downsample.ply data/dynerf/cut_roasted_beef/points3D_downsample2.ply
# python scripts/downsample_point.py data/dynerf/cook_spinach/points3D_downsample.ply data/dynerf/cook_spinach/points3D_downsample2.ply
# python scripts/downsample_point.py data/dynerf/flame_steak/points3D_downsample.ply data/dynerf/flame_steak/points3D_downsample2.ply
# python scripts/downsample_point.py data/dynerf/sear_steak/points3D_downsample.ply data/dynerf/sear_steak/points3D_downsample2.ply
# python scripts/downsample_point.py data/hypernerf/virg/broom2/dense.ply data/hypernerf/virg/broom2/dense_downsample.ply
# python scripts/downsample_point.py data/hypernerf/virg/peel-banana/dense.ply data/hypernerf/virg/peel-banana/dense_downsample.ply
# python scripts/downsample_point.py data/hypernerf/virg/vrig-chicken/dense.ply data/hypernerf/virg/vrig-chicken/dense_downsample.ply
# python scripts/downsample_point.py data/hypernerf/virg/vrig-3dprinter/dense.ply data/hypernerf/virg/vrig-3dprinter/dense_downsample.ply
# bash colmap.sh data/dycheck/sriracha-tree
# bash colmap.sh data/dycheck/apple
# bash colmap.sh data/dycheck/space-out
# bash colmap.sh data/dycheck/teddy
# bash colmap.sh data/dycheck/wheel
# bash colmap.sh data/dycheck/spin
# bash colmap.sh data/dnerf/hook/
# bash colmap.sh data/dnerf/mutant
# bash colmap.sh data/dnerf/standup
# bash colmap.sh data/dnerf/lego
# bash colmap.sh data/dnerf/trex
# bash colmap.sh data/dnerf/bouncingballs
# bash colmap.sh data/dnerf/hellwarrior
# bash colmap.sh data/nerf_synthetic/chair
# bash colmap.sh data/nerf_synthetic/drums
# bash colmap.sh data/nerf_synthetic/ficus
# bash colmap.sh data/nerf_synthetic/hotdog
# bash colmap.sh data/nerf_synthetic/lego
# bash colmap.sh data/nerf_synthetic/materials
# bash colmap.sh data/nerf_synthetic/mic
# bash colmap.sh data/nerf_synthetic/ship
# bash scripts/metric_dynerf.sh dynerf_batch4_do
# wait
# bash scripts/metric_hyper_one.sh hypernerf2
# wait
# bash scripts/metric_hyper_one.sh hypernerf_emptyvoxel2
# wait
# bash scripts/metric_hyper_one.sh hypernerf_emptyvoxel
# wait
# bash scripts/metric_dynerf.sh dynerf_batch1_do
# wait
# bash scripts/metric_dynerf.sh dynerf_res124
# wait
# bash scripts/metric_dynerf.sh dynerf_emptyvoxel1
# wait
# bash scripts/metric_dynerf.sh dynerf_emptyvoxel2
# wait
# exp_name="dynerf_static"
# export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dynerf/flame_salmon_1/colmap/dense/workspace --port 6368 --expname "$exp_name/flame_salmon_1" --configs arguments/$exp_name/default.py &
# export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dynerf/coffee_martini/colmap/dense/workspace --port 6369 --expname "$exp_name/coffee_martini" --configs arguments/$exp_name/default.py
# exp_name="dynerf_4_batch1"
# bash scripts/train_dynerf_ab1.sh dynerf_4_batch1_2 &
# bash scripts/train_dynerf_ab2.sh dynerf_4_batch4_2
# wait
# bash scripts/train_hyper_virg.sh hypernerf3
# bash scripts/train_hyper_interp.sh hypernerf4
# bash scripts/train_hyper_virg.sh hypernerf_3dgs
# exp_name="hypernerf4"
# export CUDA_VISIBLE_DEVICES=0&&python vis_point.py --model_path output/$exp_name/broom2 --configs arguments/$exp_name/broom2.py &
# export CUDA_VISIBLE_DEVICES=2&&python vis_point.py --model_path output/$exp_name/3dprinter --configs arguments/$exp_name/3dprinter.py &
# export CUDA_VISIBLE_DEVICES=2&&python vis_point.py --model_path output/$exp_name/peel-banana --configs arguments/$exp_name/banana.py&
# export CUDA_VISIBLE_DEVICES=3&&python vis_point.py --model_path output/$exp_name/vrig-chicken --configs arguments/$exp_name/chicken.py &
# wait
# exp_name="dnerf_tv_2"
# export CUDA_VISIBLE_DEVICES=3&&python editing.py --model_path output/$exp_name/lego
# exp_name="dnerf_tv_2_1"
# export CUDA_VISIBLE_DEVICES=3&&python vis_point.py --model_path output/ablation/$exp_name/hook --configs arguments/$exp_name/hook.py
# export CUDA_VISIBLE_DEVICES=3&&python vis_point.py --model_path output/ablation/$exp_name/hellwarrior --configs arguments/$exp_name/hellwarrior.py
# export CUDA_VISIBLE_DEVICES=3&&python vis_point.py --model_path output/ablation/$exp_name/jumpingjacks --configs arguments/$exp_name/jumpingjacks.py
# export CUDA_VISIBLE_DEVICES=3&&python vis_point.py --model_path output/ablation/$exp_name/standup --configs arguments/$exp_name/standup.py
exp_name1="medical"
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/medicaldata/images --port 6068 --expname "medical/$exp_name1/" --configs arguments/$exp_name1/bouncingballs.py

88
scripts/blender2colmap.py Normal file
View File

@ -0,0 +1,88 @@
import os
import numpy as np
import glob
import sys
import json
from PIL import Image
from tqdm import tqdm
import shutil
import math
def fov2focal(fov, pixels):
return pixels / (2 * math.tan(fov / 2))
def rotmat2qvec(R):
Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat
K = np.array([
[Rxx - Ryy - Rzz, 0, 0, 0],
[Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0],
[Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0],
[Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz]]) / 3.0
eigvals, eigvecs = np.linalg.eigh(K)
qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)]
if qvec[0] < 0:
qvec *= -1
return qvec
root_dir = sys.argv[1]
colmap_dir = os.path.join(root_dir,"sparse_")
if not os.path.exists(colmap_dir):
os.makedirs(colmap_dir)
imagecolmap_dir = os.path.join(root_dir,"image_colmap")
if not os.path.exists(imagecolmap_dir):
os.makedirs(imagecolmap_dir)
image_dir = os.path.join(root_dir)
images = os.listdir(image_dir)
images.sort()
camera_json = os.path.join(root_dir,"transforms_train.json")
with open (camera_json) as f:
meta = json.load(f)
try:
image_size = meta['w'], meta['h']
focal = [meta['fl_x'],meta['fl_y']]
except:
try:
image_size = meta['frames'][0]['w'], meta['frames'][0]['h']
focal = [meta['frames'][0]['fl_x'],meta['frames'][0]['fl_y']]
except:
image_size = 800,800
focal = fov2focal(meta['camera_angle_x'], 800)
focal = [focal,focal]
# size = image.size
# breakpoint()
object_images_file = open(os.path.join(colmap_dir,"images.txt"),"w")
object_cameras_file = open(os.path.join(colmap_dir,"cameras.txt"),"w")
idx=0
sizes=1
cnt=0
while len(meta['frames'])//sizes > 200:
sizes += 1
for frame in meta['frames']:
cnt+=1
if cnt % sizes != 0:
continue
matrix = np.linalg.inv(np.array(frame["transform_matrix"]))
R = -np.transpose(matrix[:3,:3])
R[:,0] = -R[:,0]
T = -matrix[:3, 3]
T = -np.matmul(R,T)
T = [str(i) for i in T]
qevc = [str(i) for i in rotmat2qvec(np.transpose(R))]
print(idx+1," ".join(qevc)," ".join(T),1,frame['file_path'].split('/')[-1]+".png","\n",file=object_images_file)
print(idx,"SIMPLE_PINHOLE",image_size[0],image_size[1],focal[0],image_size[0]/2,image_size[1]/2,file=object_cameras_file)
idx+=1
# breakpoint()
print(os.path.join(image_dir,frame['file_path']),os.path.join(imagecolmap_dir,frame['file_path'].split('/')[-1]+".png"))
shutil.copy(os.path.join(image_dir,frame['file_path']+".png"),os.path.join(imagecolmap_dir,frame['file_path'].split('/')[-1]+".png"))
# write camera infomation.
# print(1,"SIMPLE_PINHOLE",image_size[0],image_size[1],focal[0],image_sizep0/2,image_size[1]/2,file=object_cameras_file)
object_point_file = open(os.path.join(colmap_dir,"points3D.txt"),"w")
object_cameras_file.close()
object_images_file.close()
object_point_file.close()

472
scripts/colmap_converter.py Normal file
View File

@ -0,0 +1,472 @@
import os
import collections
import numpy as np
import struct
import argparse
CameraModel = collections.namedtuple(
"CameraModel", ["model_id", "model_name", "num_params"])
Camera = collections.namedtuple(
"Camera", ["id", "model", "width", "height", "params"])
BaseImage = collections.namedtuple(
"Image", ["id", "qvec", "tvec", "camera_id", "name", "xys", "point3D_ids"])
Point3D = collections.namedtuple(
"Point3D", ["id", "xyz", "rgb", "error", "image_ids", "point2D_idxs"])
class Image(BaseImage):
def qvec2rotmat(self):
return qvec2rotmat(self.qvec)
CAMERA_MODELS = {
CameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3),
CameraModel(model_id=1, model_name="PINHOLE", num_params=4),
CameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4),
CameraModel(model_id=3, model_name="RADIAL", num_params=5),
CameraModel(model_id=4, model_name="OPENCV", num_params=8),
CameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8),
CameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12),
CameraModel(model_id=7, model_name="FOV", num_params=5),
CameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4),
CameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5),
CameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12)
}
CAMERA_MODEL_IDS = dict([(camera_model.model_id, camera_model)
for camera_model in CAMERA_MODELS])
CAMERA_MODEL_NAMES = dict([(camera_model.model_name, camera_model)
for camera_model in CAMERA_MODELS])
def read_next_bytes(fid, num_bytes, format_char_sequence, endian_character="<"):
"""Read and unpack the next bytes from a binary file.
:param fid:
:param num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc.
:param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}.
:param endian_character: Any of {@, =, <, >, !}
:return: Tuple of read and unpacked values.
"""
data = fid.read(num_bytes)
return struct.unpack(endian_character + format_char_sequence, data)
def write_next_bytes(fid, data, format_char_sequence, endian_character="<"):
"""pack and write to a binary file.
:param fid:
:param data: data to send, if multiple elements are sent at the same time,
they should be encapsuled either in a list or a tuple
:param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}.
should be the same length as the data list or tuple
:param endian_character: Any of {@, =, <, >, !}
"""
if isinstance(data, (list, tuple)):
bytes = struct.pack(endian_character + format_char_sequence, *data)
else:
bytes = struct.pack(endian_character + format_char_sequence, data)
fid.write(bytes)
def read_cameras_text(path):
"""
see: src/base/reconstruction.cc
void Reconstruction::WriteCamerasText(const std::string& path)
void Reconstruction::ReadCamerasText(const std::string& path)
"""
cameras = {}
with open(path, "r") as fid:
while True:
line = fid.readline()
if not line:
break
line = line.strip()
if len(line) > 0 and line[0] != "#":
elems = line.split()
camera_id = int(elems[0])
model = elems[1]
width = int(elems[2])
height = int(elems[3])
params = np.array(tuple(map(float, elems[4:])))
cameras[camera_id] = Camera(id=camera_id, model=model,
width=width, height=height,
params=params)
return cameras
def read_cameras_binary(path_to_model_file):
"""
see: src/base/reconstruction.cc
void Reconstruction::WriteCamerasBinary(const std::string& path)
void Reconstruction::ReadCamerasBinary(const std::string& path)
"""
cameras = {}
with open(path_to_model_file, "rb") as fid:
num_cameras = read_next_bytes(fid, 8, "Q")[0]
for _ in range(num_cameras):
camera_properties = read_next_bytes(
fid, num_bytes=24, format_char_sequence="iiQQ")
camera_id = camera_properties[0]
model_id = camera_properties[1]
model_name = CAMERA_MODEL_IDS[camera_properties[1]].model_name
width = camera_properties[2]
height = camera_properties[3]
num_params = CAMERA_MODEL_IDS[model_id].num_params
params = read_next_bytes(fid, num_bytes=8*num_params,
format_char_sequence="d"*num_params)
cameras[camera_id] = Camera(id=camera_id,
model=model_name,
width=width,
height=height,
params=np.array(params))
assert len(cameras) == num_cameras
return cameras
def write_cameras_text(cameras, path):
"""
see: src/base/reconstruction.cc
void Reconstruction::WriteCamerasText(const std::string& path)
void Reconstruction::ReadCamerasText(const std::string& path)
"""
HEADER = "# Camera list with one line of data per camera:\n" + \
"# CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n" + \
"# Number of cameras: {}\n".format(len(cameras))
with open(path, "w") as fid:
fid.write(HEADER)
for _, cam in cameras.items():
to_write = [cam.id, cam.model, cam.width, cam.height, *cam.params]
line = " ".join([str(elem) for elem in to_write])
fid.write(line + "\n")
def write_cameras_binary(cameras, path_to_model_file):
"""
see: src/base/reconstruction.cc
void Reconstruction::WriteCamerasBinary(const std::string& path)
void Reconstruction::ReadCamerasBinary(const std::string& path)
"""
with open(path_to_model_file, "wb") as fid:
write_next_bytes(fid, len(cameras), "Q")
for _, cam in cameras.items():
model_id = CAMERA_MODEL_NAMES[cam.model].model_id
camera_properties = [cam.id,
model_id,
cam.width,
cam.height]
write_next_bytes(fid, camera_properties, "iiQQ")
for p in cam.params:
write_next_bytes(fid, float(p), "d")
return cameras
def read_images_text(path):
"""
see: src/base/reconstruction.cc
void Reconstruction::ReadImagesText(const std::string& path)
void Reconstruction::WriteImagesText(const std::string& path)
"""
images = {}
with open(path, "r") as fid:
while True:
line = fid.readline()
if not line:
break
line = line.strip()
if len(line) > 0 and line[0] != "#":
elems = line.split()
image_id = int(elems[0])
qvec = np.array(tuple(map(float, elems[1:5])))
tvec = np.array(tuple(map(float, elems[5:8])))
camera_id = int(elems[8])
image_name = elems[9]
elems = fid.readline().split()
xys = np.column_stack([tuple(map(float, elems[0::3])),
tuple(map(float, elems[1::3]))])
point3D_ids = np.array(tuple(map(int, elems[2::3])))
images[image_id] = Image(
id=image_id, qvec=qvec, tvec=tvec,
camera_id=camera_id, name=image_name,
xys=xys, point3D_ids=point3D_ids)
return images
def read_images_binary(path_to_model_file):
"""
see: src/base/reconstruction.cc
void Reconstruction::ReadImagesBinary(const std::string& path)
void Reconstruction::WriteImagesBinary(const std::string& path)
"""
images = {}
with open(path_to_model_file, "rb") as fid:
num_reg_images = read_next_bytes(fid, 8, "Q")[0]
for _ in range(num_reg_images):
binary_image_properties = read_next_bytes(
fid, num_bytes=64, format_char_sequence="idddddddi")
image_id = binary_image_properties[0]
qvec = np.array(binary_image_properties[1:5])
tvec = np.array(binary_image_properties[5:8])
camera_id = binary_image_properties[8]
image_name = ""
current_char = read_next_bytes(fid, 1, "c")[0]
while current_char != b"\x00": # look for the ASCII 0 entry
image_name += current_char.decode("utf-8")
current_char = read_next_bytes(fid, 1, "c")[0]
num_points2D = read_next_bytes(fid, num_bytes=8,
format_char_sequence="Q")[0]
x_y_id_s = read_next_bytes(fid, num_bytes=24*num_points2D,
format_char_sequence="ddq"*num_points2D)
xys = np.column_stack([tuple(map(float, x_y_id_s[0::3])),
tuple(map(float, x_y_id_s[1::3]))])
point3D_ids = np.array(tuple(map(int, x_y_id_s[2::3])))
images[image_id] = Image(
id=image_id, qvec=qvec, tvec=tvec,
camera_id=camera_id, name=image_name,
xys=xys, point3D_ids=point3D_ids)
return images
def write_images_text(images, path):
"""
see: src/base/reconstruction.cc
void Reconstruction::ReadImagesText(const std::string& path)
void Reconstruction::WriteImagesText(const std::string& path)
"""
if len(images) == 0:
mean_observations = 0
else:
mean_observations = sum((len(img.point3D_ids) for _, img in images.items()))/len(images)
HEADER = "# Image list with two lines of data per image:\n" + \
"# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n" + \
"# POINTS2D[] as (X, Y, POINT3D_ID)\n" + \
"# Number of images: {}, mean observations per image: {}\n".format(len(images), mean_observations)
with open(path, "w") as fid:
fid.write(HEADER)
for _, img in images.items():
image_header = [img.id, *img.qvec, *img.tvec, img.camera_id, img.name]
first_line = " ".join(map(str, image_header))
fid.write(first_line + "\n")
points_strings = []
for xy, point3D_id in zip(img.xys, img.point3D_ids):
points_strings.append(" ".join(map(str, [*xy, point3D_id])))
fid.write(" ".join(points_strings) + "\n")
def write_images_binary(images, path_to_model_file):
"""
see: src/base/reconstruction.cc
void Reconstruction::ReadImagesBinary(const std::string& path)
void Reconstruction::WriteImagesBinary(const std::string& path)
"""
with open(path_to_model_file, "wb") as fid:
write_next_bytes(fid, len(images), "Q")
for _, img in images.items():
write_next_bytes(fid, img.id, "i")
write_next_bytes(fid, img.qvec.tolist(), "dddd")
write_next_bytes(fid, img.tvec.tolist(), "ddd")
write_next_bytes(fid, img.camera_id, "i")
for char in img.name:
write_next_bytes(fid, char.encode("utf-8"), "c")
write_next_bytes(fid, b"\x00", "c")
write_next_bytes(fid, len(img.point3D_ids), "Q")
for xy, p3d_id in zip(img.xys, img.point3D_ids):
write_next_bytes(fid, [*xy, p3d_id], "ddq")
def read_points3D_text(path):
"""
see: src/base/reconstruction.cc
void Reconstruction::ReadPoints3DText(const std::string& path)
void Reconstruction::WritePoints3DText(const std::string& path)
"""
points3D = {}
with open(path, "r") as fid:
while True:
line = fid.readline()
if not line:
break
line = line.strip()
if len(line) > 0 and line[0] != "#":
elems = line.split()
point3D_id = int(elems[0])
xyz = np.array(tuple(map(float, elems[1:4])))
rgb = np.array(tuple(map(int, elems[4:7])))
error = float(elems[7])
image_ids = np.array(tuple(map(int, elems[8::2])))
point2D_idxs = np.array(tuple(map(int, elems[9::2])))
points3D[point3D_id] = Point3D(id=point3D_id, xyz=xyz, rgb=rgb,
error=error, image_ids=image_ids,
point2D_idxs=point2D_idxs)
return points3D
def read_points3D_binary(path_to_model_file):
"""
see: src/base/reconstruction.cc
void Reconstruction::ReadPoints3DBinary(const std::string& path)
void Reconstruction::WritePoints3DBinary(const std::string& path)
"""
points3D = {}
with open(path_to_model_file, "rb") as fid:
num_points = read_next_bytes(fid, 8, "Q")[0]
for _ in range(num_points):
binary_point_line_properties = read_next_bytes(
fid, num_bytes=43, format_char_sequence="QdddBBBd")
point3D_id = binary_point_line_properties[0]
xyz = np.array(binary_point_line_properties[1:4])
rgb = np.array(binary_point_line_properties[4:7])
error = np.array(binary_point_line_properties[7])
track_length = read_next_bytes(
fid, num_bytes=8, format_char_sequence="Q")[0]
track_elems = read_next_bytes(
fid, num_bytes=8*track_length,
format_char_sequence="ii"*track_length)
image_ids = np.array(tuple(map(int, track_elems[0::2])))
point2D_idxs = np.array(tuple(map(int, track_elems[1::2])))
points3D[point3D_id] = Point3D(
id=point3D_id, xyz=xyz, rgb=rgb,
error=error, image_ids=image_ids,
point2D_idxs=point2D_idxs)
return points3D
def write_points3D_text(points3D, path):
"""
see: src/base/reconstruction.cc
void Reconstruction::ReadPoints3DText(const std::string& path)
void Reconstruction::WritePoints3DText(const std::string& path)
"""
if len(points3D) == 0:
mean_track_length = 0
else:
mean_track_length = sum((len(pt.image_ids) for _, pt in points3D.items()))/len(points3D)
HEADER = "# 3D point list with one line of data per point:\n" + \
"# POINT3D_ID, X, Y, Z, R, G, B, ERROR, TRACK[] as (IMAGE_ID, POINT2D_IDX)\n" + \
"# Number of points: {}, mean track length: {}\n".format(len(points3D), mean_track_length)
with open(path, "w") as fid:
fid.write(HEADER)
for _, pt in points3D.items():
point_header = [pt.id, *pt.xyz, *pt.rgb, pt.error]
fid.write(" ".join(map(str, point_header)) + " ")
track_strings = []
for image_id, point2D in zip(pt.image_ids, pt.point2D_idxs):
track_strings.append(" ".join(map(str, [image_id, point2D])))
fid.write(" ".join(track_strings) + "\n")
def write_points3D_binary(points3D, path_to_model_file):
"""
see: src/base/reconstruction.cc
void Reconstruction::ReadPoints3DBinary(const std::string& path)
void Reconstruction::WritePoints3DBinary(const std::string& path)
"""
with open(path_to_model_file, "wb") as fid:
write_next_bytes(fid, len(points3D), "Q")
for _, pt in points3D.items():
write_next_bytes(fid, pt.id, "Q")
write_next_bytes(fid, pt.xyz.tolist(), "ddd")
write_next_bytes(fid, pt.rgb.tolist(), "BBB")
write_next_bytes(fid, pt.error, "d")
track_length = pt.image_ids.shape[0]
write_next_bytes(fid, track_length, "Q")
for image_id, point2D_id in zip(pt.image_ids, pt.point2D_idxs):
write_next_bytes(fid, [image_id, point2D_id], "ii")
def detect_model_format(path, ext):
if os.path.isfile(os.path.join(path, "cameras" + ext)) and \
os.path.isfile(os.path.join(path, "images" + ext)) and \
os.path.isfile(os.path.join(path, "points3D" + ext)):
print("Detected model format: '" + ext + "'")
return True
return False
def read_model(path, ext=""):
# try to detect the extension automatically
if ext == "":
if detect_model_format(path, ".bin"):
ext = ".bin"
elif detect_model_format(path, ".txt"):
ext = ".txt"
else:
print("Provide model format: '.bin' or '.txt'")
return
if ext == ".txt":
cameras = read_cameras_text(os.path.join(path, "cameras" + ext))
images = read_images_text(os.path.join(path, "images" + ext))
points3D = read_points3D_text(os.path.join(path, "points3D") + ext)
else:
cameras = read_cameras_binary(os.path.join(path, "cameras" + ext))
images = read_images_binary(os.path.join(path, "images" + ext))
points3D = read_points3D_binary(os.path.join(path, "points3D") + ext)
return cameras, images, points3D
def write_model(cameras, images, points3D, path, ext=".bin"):
if ext == ".txt":
write_cameras_text(cameras, os.path.join(path, "cameras" + ext))
write_images_text(images, os.path.join(path, "images" + ext))
write_points3D_text(points3D, os.path.join(path, "points3D") + ext)
else:
write_cameras_binary(cameras, os.path.join(path, "cameras" + ext))
write_images_binary(images, os.path.join(path, "images" + ext))
write_points3D_binary(points3D, os.path.join(path, "points3D") + ext)
return cameras, images, points3D
def qvec2rotmat(qvec):
return np.array([
[1 - 2 * qvec[2]**2 - 2 * qvec[3]**2,
2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3],
2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2]],
[2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3],
1 - 2 * qvec[1]**2 - 2 * qvec[3]**2,
2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1]],
[2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2],
2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1],
1 - 2 * qvec[1]**2 - 2 * qvec[2]**2]])
def rotmat2qvec(R):
Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat
K = np.array([
[Rxx - Ryy - Rzz, 0, 0, 0],
[Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0],
[Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0],
[Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz]]) / 3.0
eigvals, eigvecs = np.linalg.eigh(K)
qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)]
if qvec[0] < 0:
qvec *= -1
return qvec
def main():
parser = argparse.ArgumentParser(description="Read and write COLMAP binary and text models")
parser.add_argument("--input_model", help="path to input model folder")
parser.add_argument("--input_format", choices=[".bin", ".txt"],
help="input model format", default="")
parser.add_argument("--output_model",
help="path to output model folder")
parser.add_argument("--output_format", choices=[".bin", ".txt"],
help="outut model format", default=".txt")
args = parser.parse_args()
cameras, images, points3D = read_model(path=args.input_model, ext=args.input_format)
print("num_cameras:", len(cameras))
print("num_images:", len(images))
print("num_points3D:", len(points3D))
if args.output_model is not None:
write_model(cameras, images, points3D, path=args.output_model, ext=args.output_format)
if __name__ == "__main__":
main()

19
scripts/downsample_point.py Normal file
View File

@ -0,0 +1,19 @@
import open3d as o3d
import sys
def process_ply_file(input_file, output_file):
# 读取输入的ply文件
pcd = o3d.io.read_point_cloud(input_file)
print(f"Total points: {len(pcd.points)}")
# 通过点云下采样将输入的点云减少
voxel_size=0.02
while len(pcd.points) > 40000:
pcd = pcd.voxel_down_sample(voxel_size=voxel_size)
print(f"Downsampled points: {len(pcd.points)}")
voxel_size+=0.01
# 将结果保存到输入的路径中
o3d.io.write_point_cloud(output_file, pcd)
# 使用函数
process_ply_file(sys.argv[1], sys.argv[2])

40
scripts/fliter_point.py Normal file
View File

@ -0,0 +1,40 @@
import open3d as o3d
import os
# 指定根目录路径
root_path = "data/dynerf/sear_steak/"
# 文件名
input_file = "points3D.ply"
output_file = "points3d_filtered.ply"
# 读取点云数据
point_cloud_before = o3d.io.read_point_cloud(os.path.join(root_path, input_file))
# 计算过滤前的点的数量
num_points_before = len(point_cloud_before.points)
# 计算过滤前的点云的边界框大小
bbox_before = point_cloud_before.get_axis_aligned_bounding_box()
bbox_size_before = bbox_before.get_max_bound() - bbox_before.get_min_bound()
# 进行离群点滤波
cl, ind = point_cloud_before.remove_statistical_outlier(nb_neighbors=20, std_ratio=2.0)
# 创建一个新的点云对象，包含滤波后的点
filtered_point_cloud = point_cloud_before.select_by_index(ind)
# 保存滤波后的点云到新文件
o3d.io.write_point_cloud(os.path.join(root_path, output_file), filtered_point_cloud)
# 计算过滤后的点的数量
num_points_after = len(filtered_point_cloud.points)
# 计算边界框的大小
bbox = filtered_point_cloud.get_axis_aligned_bounding_box()
bbox_size = bbox.get_max_bound() - bbox.get_min_bound()
print(f"过滤前的点数: {num_points_before}")
print(f"过滤前的点云边界框大小: {bbox_size_before}")
print(f"过滤后的点数: {num_points_after}")
print(f"过滤后的点云边界框大小: {bbox_size}")
print(f"离群点过滤完成，结果已保存到 {output_file}")

25
scripts/grow_point.py Normal file
View File

@ -0,0 +1,25 @@
import open3d as o3d
import numpy as np
def grow_sparse_regions(input_file, output_file):
# 读取输入的ply文件
pcd = o3d.io.read_point_cloud(input_file)
# 计算点云的密度
densities = o3d.geometry.PointCloud.compute_nearest_neighbor_distance(pcd)
avg_density = np.average(densities)
print(f"Average density: {avg_density}")
# 找到稀疏部分
sparse_indices = np.where(densities > avg_density * 1.2)[0] # 这里我们假设稀疏部分的密度大于平均密度的1.2倍
sparse_points = np.asarray(pcd.points)[sparse_indices]
breakpoint()
# 复制并增长稀疏部分
# for _ in range(5): # 这里我们假设每个稀疏点复制5次
# pcd.points.extend(sparse_points)
# 将结果保存到输入的路径中
o3d.io.write_point_cloud(output_file, pcd)
# 使用函数
grow_sparse_regions("data/hypernerf/vrig/chickchicken/dense_downsample.ply", "data/hypernerf/interp/chickchicken/dense_downsample.ply")

81
scripts/hypernerf2colmap.py Normal file
View File

@ -0,0 +1,81 @@
import os
import numpy as np
import glob
import sys
import json
from PIL import Image
from tqdm import tqdm
import shutil
def rotmat2qvec(R):
Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat
K = np.array([
[Rxx - Ryy - Rzz, 0, 0, 0],
[Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0],
[Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0],
[Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz]]) / 3.0
eigvals, eigvecs = np.linalg.eigh(K)
qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)]
if qvec[0] < 0:
qvec *= -1
return qvec
root_dir = sys.argv[1]
colmap_dir = os.path.join(root_dir,"sparse_")
if not os.path.exists(colmap_dir):
os.makedirs(colmap_dir)
imagecolmap_dir = os.path.join(root_dir,"image_colmap")
if not os.path.exists(imagecolmap_dir):
os.makedirs(imagecolmap_dir)
image_dir = os.path.join(root_dir,"rgb","2x")
images = os.listdir(image_dir)
images.sort()
camera_dir = os.path.join(root_dir,"camera")
cameras = os.listdir(camera_dir)
cameras.sort()
cams = []
for jsonfile in tqdm(cameras):
with open (os.path.join(camera_dir,jsonfile)) as f:
cams.append(json.load(f))
image_size = cams[0]['image_size']
image = Image.open(os.path.join(image_dir,images[0]))
size = image.size
# breakpoint()
object_images_file = open(os.path.join(colmap_dir,"images.txt"),"w")
object_cameras_file = open(os.path.join(colmap_dir,"cameras.txt"),"w")
idx=0
cnt=0
sizes=2
while len(cams)//sizes > 200:
sizes += 1
# breakpoint()
for cam, image in zip(cams, images):
cnt+=1
# print(image)
# breakpoint()
if cnt % sizes != 0:
continue
# print("begin to write")
R = np.array(cam['orientation']).T
# breakpoint()
T = -np.array(cam['position'])@R
# T = -np.matmul(R,T)
T = [str(i) for i in T]
qevc = [str(i) for i in rotmat2qvec(R.T)]
print(idx+1," ".join(qevc)," ".join(T),1,image,"\n",file=object_images_file)
print(idx,"SIMPLE_PINHOLE",image_size[0]/2,image_size[1]/2,cam['focal_length']/2,cam['principal_point'][0]/2,cam['principal_point'][1]/2,file=object_cameras_file)
idx+=1
shutil.copy(os.path.join(image_dir,image),os.path.join(imagecolmap_dir,image))
print(idx)
# write camera infomation.
# print(1,"SIMPLE_PINHOLE",image_size[0],image_size[1],focal[0],image_sizep0/2,image_size[1]/2,file=object_cameras_file)
object_point_file = open(os.path.join(colmap_dir,"points3D.txt"),"w")
object_cameras_file.close()
object_images_file.close()
object_point_file.close()

163
scripts/llff2colmap.py Normal file
View File

@ -0,0 +1,163 @@
import os
import numpy as np
import glob
import sys
def rotmat2qvec(R):
Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat
K = np.array([
[Rxx - Ryy - Rzz, 0, 0, 0],
[Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0],
[Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0],
[Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz]]) / 3.0
eigvals, eigvecs = np.linalg.eigh(K)
qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)]
if qvec[0] < 0:
qvec *= -1
return qvec
def normalize(v):
"""Normalize a vector."""
return v / np.linalg.norm(v)
def average_poses(poses):
"""
Calculate the average pose, which is then used to center all poses
using @center_poses. Its computation is as follows:
1. Compute the center: the average of pose centers.
2. Compute the z axis: the normalized average z axis.
3. Compute axis y': the average y axis.
4. Compute x' = y' cross product z, then normalize it as the x axis.
5. Compute the y axis: z cross product x.
Note that at step 3, we cannot directly use y' as y axis since it's
not necessarily orthogonal to z axis. We need to pass from x to y.
Inputs:
poses: (N_images, 3, 4)
Outputs:
pose_avg: (3, 4) the average pose
"""
# 1. Compute the center
center = poses[..., 3].mean(0) # (3)
# 2. Compute the z axis
z = normalize(poses[..., 2].mean(0)) # (3)
# 3. Compute axis y' (no need to normalize as it's not the final output)
y_ = poses[..., 1].mean(0) # (3)
# 4. Compute the x axis
x = normalize(np.cross(z, y_)) # (3)
# 5. Compute the y axis (as z and x are normalized, y is already of norm 1)
y = np.cross(x, z) # (3)
pose_avg = np.stack([x, y, z, center], 1) # (3, 4)
return pose_avg
blender2opencv = np.eye(4)
def center_poses(poses, blender2opencv):
"""
Center the poses so that we can use NDC.
See https://github.com/bmild/nerf/issues/34
Inputs:
poses: (N_images, 3, 4)
Outputs:
poses_centered: (N_images, 3, 4) the centered poses
pose_avg: (3, 4) the average pose
"""
poses = poses @ blender2opencv
pose_avg = average_poses(poses) # (3, 4)
pose_avg_homo = np.eye(4)
pose_avg_homo[
:3
] = pose_avg # convert to homogeneous coordinate for faster computation
pose_avg_homo = pose_avg_homo
# by simply adding 0, 0, 0, 1 as the last row
last_row = np.tile(np.array([0, 0, 0, 1]), (len(poses), 1, 1)) # (N_images, 1, 4)
poses_homo = np.concatenate(
[poses, last_row], 1
) # (N_images, 4, 4) homogeneous coordinate
poses_centered = np.linalg.inv(pose_avg_homo) @ poses_homo # (N_images, 4, 4)
# poses_centered = poses_centered @ blender2opencv
poses_centered = poses_centered[:, :3] # (N_images, 3, 4)
return poses_centered, pose_avg_homo
root_dir = sys.argv[1]
colmap_dir = os.path.join(root_dir,"sparse_")
if not os.path.exists(colmap_dir):
os.makedirs(colmap_dir)
poses_arr = np.load(os.path.join(root_dir, "poses_bounds.npy"))
poses = poses_arr[:, :-2].reshape([-1, 3, 5]) # (N_cams, 3, 5)
near_fars = poses_arr[:, -2:]
videos = glob.glob(os.path.join(root_dir, "cam*"))
videos = sorted(videos)
assert len(videos) == poses_arr.shape[0]
H, W, focal = poses[0, :, -1]
focal = focal/2
focal = [focal, focal]
poses = np.concatenate([poses[..., 1:2], -poses[..., :1], poses[..., 2:4]], -1)
# poses, _ = center_poses(
# poses, blender2opencv
# ) # Re-center poses so that the average is near the center.
# near_original = near_fars.min()
# scale_factor = near_original * 0.75
# near_fars /= (
# scale_factor # rescale nearest plane so that it is at z = 4/3.
# )
# poses[..., 3] /= scale_factor
# Sample N_views poses for validation - NeRF-like camera trajectory.
# val_poses = directions
videos = glob.glob(os.path.join(root_dir, "cam*"))
videos = sorted(videos)
image_paths = []
for index, video_path in enumerate(videos):
image_path = os.path.join(video_path,"images","0000.png")
image_paths.append(image_path)
print(image_paths)
goal_dir = os.path.join(root_dir,"image_colmap")
if not os.path.exists(goal_dir):
os.makedirs(goal_dir)
import shutil
image_name_list =[]
for index, image in enumerate(image_paths):
image_name = image.split("/")[-1].split('.')
image_name[0] = "r_%03d" % index
print(image_name)
# breakpoint()
image_name = ".".join(image_name)
image_name_list.append(image_name)
goal_path = os.path.join(goal_dir,image_name)
shutil.copy(image,goal_path)
print(poses)
# breakpoint()
# write image information.
object_images_file = open(os.path.join(colmap_dir,"images.txt"),"w")
for idx, pose in enumerate(poses):
# pose_44 = np.eye(4)
R = pose[:3,:3]
R = -R
R[:,0] = -R[:,0]
T = pose[:3,3]
R = np.linalg.inv(R)
T = -np.matmul(R,T)
T = [str(i) for i in T]
# T = ["%.3f"%i for i in pose[:3,3]]
qevc = [str(i) for i in rotmat2qvec(R)]
# breakpoint()
print(idx+1," ".join(qevc)," ".join(T),1,image_name_list[idx],"\n",file=object_images_file)
# breakpoint()
# write camera infomation.
object_cameras_file = open(os.path.join(colmap_dir,"cameras.txt"),"w")
print(1,"SIMPLE_PINHOLE",1352,1014,focal[0],1352/2,1014/2,file=object_cameras_file)
object_point_file = open(os.path.join(colmap_dir,"points3D.txt"),"w")
object_cameras_file.close()
object_images_file.close()
object_point_file.close()

23
scripts/merge_point.py Normal file
View File

@ -0,0 +1,23 @@
import open3d as o3d
import os
from tqdm import tqdm
def merge_point_clouds(directory, output_file):
# 初始化一个空的点云
merged_pcd = o3d.geometry.PointCloud()
# 遍历文件夹下的所有文件
for filename in tqdm(os.listdir(directory)):
if filename.endswith('.ply'):
# 读取点云文件
pcd = o3d.io.read_point_cloud(os.path.join(directory, filename))
# 将点云合并
merged_pcd += pcd
# 移除位置相同的点
merged_pcd = merged_pcd.remove_duplicate_points()
# 将合并后的点云输出到一个文件中
o3d.io.write_point_cloud(output_file, merged_pcd)
# 使用函数
merge_point_clouds("point_clouds_directory", "merged.ply")

13
scripts/metric_dnerf.sh
View File

@ -1,13 +0,0 @@
exp_name1=$1
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/standup/" &
export CUDA_VISIBLE_DEVICES=1&&python metrics.py --model_path "output/$exp_name1/jumpingjacks/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/bouncingballs/" &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name1/lego/"
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/hellwarrior/" &
export CUDA_VISIBLE_DEVICES=1&&python metrics.py --model_path "output/$exp_name1/hook/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/trex/" &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name1/mutant/" &
wait
echo "Done"

61
scripts/process_dnerf.sh
View File

@ -1,43 +1,42 @@
exp_name1=$1
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/lego --port 6068 --expname "$exp_name1/lego" --configs arguments/$exp_name1/lego.py &
export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/dnerf/bouncingballs --port 6066 --expname "$exp_name1/bouncingballs" --configs arguments/$exp_name1/bouncingballs.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/jumpingjacks --port 6069 --expname "$exp_name1/jumpingjacks" --configs arguments/$exp_name1/jumpingjacks.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dnerf/trex --port 6070 --expname "$exp_name1/trex" --configs arguments/$exp_name1/trex.py &
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/jumpingjacks --port 7169 --expname "$exp_name1/jumpingjacks" --configs arguments/$exp_name1/jumpingjacks.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/trex --port 7170 --expname "$exp_name1/trex" --configs arguments/$exp_name1/trex.py
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/jumpingjacks/" --skip_train --configs arguments/$exp_name1/jumpingjacks.py &
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/trex/" --skip_train --configs arguments/$exp_name1/trex.py
wait
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/mutant --port 6068 --expname "$exp_name1/mutant" --configs arguments/$exp_name1/mutant.py &
export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/dnerf/standup --port 6066 --expname "$exp_name1/standup" --configs arguments/$exp_name1/standup.py &
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/jumpingjacks/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/trex/"
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/hook --port 6069 --expname "$exp_name1/hook" --configs arguments/$exp_name1/hook.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dnerf/hellwarrior --port 6070 --expname "$exp_name1/hellwarrior" --configs arguments/$exp_name1/hellwarrior.py &
wait
echo "Done"
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/mutant --port 7168 --expname "$exp_name1/mutant" --configs arguments/$exp_name1/mutant.py &
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/standup --port 7166 --expname "$exp_name1/standup" --configs arguments/$exp_name1/standup.py
exp_name1=$1
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/standup/" --skip_train --configs arguments/$exp_name1/standup.py &
export CUDA_VISIBLE_DEVICES=1&&python render.py --model_path "output/$exp_name1/jumpingjacks/" --skip_train --configs arguments/$exp_name1/jumpingjacks.py &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/mutant/" --skip_train --configs arguments/$exp_name1/mutant.py &
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/standup/" --skip_train --configs arguments/$exp_name1/standup.py
wait
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/mutant/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/standup/"
wait
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/hook --port 7369 --expname "$exp_name1/hook" --configs arguments/$exp_name1/hook.py &
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/hellwarrior --port 7370 --expname "$exp_name1/hellwarrior" --configs arguments/$exp_name1/hellwarrior.py
wait
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/hellwarrior/" --skip_train --configs arguments/$exp_name1/hellwarrior.py &
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/hook/" --skip_train --configs arguments/$exp_name1/hook.py
wait
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/hellwarrior/" &
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/hook/"
wait
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/lego --port 7168 --expname "$exp_name1/lego" --configs arguments/$exp_name1/lego.py &
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dnerf/bouncingballs --port 7166 --expname "$exp_name1/bouncingballs" --configs arguments/$exp_name1/bouncingballs.py
wait
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/bouncingballs/" --skip_train --configs arguments/$exp_name1/bouncingballs.py &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/lego/" --skip_train --configs arguments/$exp_name1/lego.py &
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/lego/" --skip_train --configs arguments/$exp_name1/lego.py
wait
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path "output/$exp_name1/hellwarrior/" --skip_train --configs arguments/$exp_name1/hellwarrior.py &
export CUDA_VISIBLE_DEVICES=1&&python render.py --model_path "output/$exp_name1/hook/" --skip_train --configs arguments/$exp_name1/hook.py &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/trex/" --skip_train --configs arguments/$exp_name1/trex.py &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/mutant/" --skip_train --configs arguments/$exp_name1/mutant.py &
# wait
echo "Done"
exp_name1=$1
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/standup/" &
export CUDA_VISIBLE_DEVICES=1&&python metrics.py --model_path "output/$exp_name1/jumpingjacks/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/bouncingballs/" &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name1/lego/"
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/hellwarrior/" &
export CUDA_VISIBLE_DEVICES=1&&python metrics.py --model_path "output/$exp_name1/hook/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name1/trex/" &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name1/mutant/" &
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name1/lego/"
wait
echo "Done"

25
scripts/read_all_metrics.py
View File

@ -1,9 +1,15 @@
import json
import os
exp_name = ["dnerf_tv_test"]
scene_name = ["bouncingballs","jumpingjacks","lego","standup","hook","mutant","hellwarrior","trex"]
# exp_name = ["dnerf_tv_nodx","dnerf_tv_nodr","dnerf_tv_nods","dnerf_tv","dnerf_tv_dshs","dnerf_tv_do",
# "dnerf_tv_2","dnerf_tv_8","dnerf_tv_deepmlp"]
# exp_name= ["dnerf_tv_2_slim"]
exp_name=["dynerf_default","dynerf_9"]
# exp_name = ["hypernerf_3dgs"]
scene_name = ["coffee_martini", "cook_spinach", "cut_roasted_beef", "flame_salmon_1", "flame_steak", "sear_steak"]
# scene_name = ["bouncingballs","jumpingjacks","lego","standup","hook","mutant","hellwarrior","trex"]
# scene_name = ["3dprinter","broom2","peel-banana","vrig-chicken"]
json_name = "results.json"
result_json = {"SSIM":0,"PSNR":0,"LPIPS":0}
result_json = {"PSNR":0,"SSIM":0,"MS-SSIM":0,"D-SSIM":0,"LPIPS-vgg":0,"LPIPS-alex":0,"LPIPS":0}
exp_json = {}
for exps in exp_name:
exp_json[exps] = result_json.copy()
@ -14,10 +20,12 @@ for scene in scene_name:
js = json.load(f)
# print(js)
# print(scene, experiment, js["ours_20000"])
for res in ["ours_30000","ours_20000","ours_14000","ours_7000","ours_3000"]:
for res in ["ours_30000","ours_20000","ours_14000","ours_10000","ours_7000","ours_3000"]:
if res in js.keys():
for key, item in js[res].items():
exp_json[experiment][key] += item
if key in exp_json[experiment].keys():
exp_json[experiment][key] += item
print(scene, key, item)
break
# for scene in scene_name:
@ -25,8 +33,11 @@ for scene in scene_name:
for experiment in exp_name:
print(exp_json[experiment])
for key, item in exp_json[experiment].items():
exp_json[experiment][key] /= 8
exp_json[experiment][key] /= len(scene_name)
for key,item in exp_json.items():
print(key)
print("%.4f"%item["PSNR"],"&","%.4f"%item["SSIM"],"&","%.4f"%item["LPIPS"],)
print("PSNR,SSIM,D-SSIM,MS-SSIM,LPIPS-alex,LPIPS-vgg","LPIPS")
print("%.4f"%item["PSNR"],"&","%.4f"%item["SSIM"],"%.4f"%item["D-SSIM"],
"%.4f"%item["MS-SSIM"],"&","%.4f"%item["LPIPS-alex"],"%.4f"%item["LPIPS-vgg"],
"%.4f"%item["LPIPS"])
# break

13
scripts/render_dnerf.sh
View File

@ -1,13 +0,0 @@
exp_name1=$1
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/standup/" --skip_train --configs arguments/$exp_name1/standup.py &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/jumpingjacks/" --skip_train --configs arguments/$exp_name1/jumpingjacks.py &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/bouncingballs/" --skip_train --configs arguments/$exp_name1/bouncingballs.py &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/lego/" --skip_train --configs arguments/$exp_name1/lego.py &
wait
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/hellwarrior/" --skip_train --configs arguments/$exp_name1/hellwarrior.py &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/hook/" --skip_train --configs arguments/$exp_name1/hook.py &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path "output/$exp_name1/trex/" --skip_train --configs arguments/$exp_name1/trex.py &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path "output/$exp_name1/mutant/" --skip_train --configs arguments/$exp_name1/mutant.py &
# wait
echo "Done"

2
scripts/train_dnerf.sh
View File

@ -1,7 +1,7 @@
exp_name1=$1
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/lego --port 6068 --expname "$exp_name1/lego" --configs arguments/$exp_name1/lego.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dnerf/bouncingballs --port 6066 --expname "$exp_name1/bouncingballs" --configs arguments/$exp_name1/bouncingballs.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dnerf/bouncingballs --port 6266 --expname "$exp_name1/bouncingballs" --configs arguments/$exp_name1/bouncingballs.py &
wait
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dnerf/jumpingjacks --port 6069 --expname "$exp_name1/jumpingjacks" --configs arguments/$exp_name1/jumpingjacks.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dnerf/trex --port 6070 --expname "$exp_name1/trex" --configs arguments/$exp_name1/trex.py &

21
scripts/train_dnerf_all.sh
View File

@ -1,21 +0,0 @@
bash scripts/process_dnerf.sh dnerf_tv_test
wait
# bash scripts/train_ablation.sh dnerf_3dgs
# wait
# bash scripts/train_ablation.sh dnerf_gridlarge
# wait
# bash scripts/train_ablation.sh dnerf_gridsmall
# wait
# bash scripts/train_ablation.sh dnerf_gridsmaller
# wait
# bash scripts/train_ablation.sh dnerf_mlplarge
# wait
# bash scripts/train_ablation.sh dnerf_mlplarger
# wait
# bash scripts/train_ablation.sh dnerf_nocoarse
# wait
# bash scripts/train_ablation.sh dnerf_slim
# wait
# bash scripts/train_ablation.sh dnerf_notv
# wait
# bash scripts/train_ablation.sh dnerf_imageloss

21
scripts/train_dycheck.sh Normal file
View File

@ -0,0 +1,21 @@
exp_name1=$1
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dycheck/spin --port 6084 --expname $exp_name1/spin/ --configs arguments/$exp_name1/default.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dycheck/space-out --port 6083 --expname $exp_name1/space-out/ --configs arguments/$exp_name1/default.py &
wait
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name1/space-out/ --configs arguments/$exp_name1/default.py &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name1/spin/ --configs arguments/$exp_name1/default.py
wait
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dycheck/teddy/ --port 6081 --expname $exp_name1/teddy/ --configs arguments/$exp_name1/default.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dycheck/apple/ --port 6082 --expname $exp_name1/apple/ --configs arguments/$exp_name1/default.py
wait
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name1/teddy/ --skip_train --configs arguments/$exp_name1/default.py &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name1/apple/ --skip_train --configs arguments/$exp_name1/default.py
wait
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path output/$exp_name1/apple/ &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path output/$exp_name1/teddy/ &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path output/$exp_name1/space-out/ &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path output/$exp_name1/spin/
echo "Done"

25
scripts/train_dynamic3dgs.sh Normal file
View File

@ -0,0 +1,25 @@
exp_name1=$1
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dynamic3dgs/data/basketball --port 6068 --expname "$exp_name1/dynamic3dgs/basketball" --configs arguments/$exp_name1/default.py
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dynamic3dgs/data/boxes --port 6069 --expname "$exp_name1/dynamic3dgs/boxes" --configs arguments/$exp_name1/default.py
wait
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dynamic3dgs/data/football --port 6068 --expname "$exp_name1/dynamic3dgs/football" --configs arguments/$exp_name1/default.py
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dynamic3dgs/data/juggle --port 6069 --expname "$exp_name1/dynamic3dgs/juggle" --configs arguments/$exp_name1/default.py
wait
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dynamic3dgs/data/softball --port 6068 --expname "$exp_name1/dynamic3dgs/softball" --configs arguments/$exp_name1/default.py
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/dynamic3dgs/data/tennis --port 6069 --expname "$exp_name1/dynamic3dgs/tennis" --configs arguments/$exp_name1/default.py
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name1/dynamic3dgs/basketball --configs arguments/$exp_name1/default.py --skip_train
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name1/dynamic3dgs/boxes --configs arguments/$exp_name1/default.py --skip_train
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name1/dynamic3dgs/football --configs arguments/$exp_name1/default.py --skip_train
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name1/dynamic3dgs/juggle --configs arguments/$exp_name1/default.py --skip_train
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name1/dynamic3dgs/softball --configs arguments/$exp_name1/default.py --skip_train
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name1/dynamic3dgs/tennis --configs arguments/$exp_name1/default.py --skip_train
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path output/$exp_name/dynamic3dgs/basketball
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path output/$exp_name/dynamic3dgs/boxes
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path output/$exp_name/dynamic3dgs/football
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path output/$exp_name/dynamic3dgs/juggle
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path output/$exp_name/dynamic3dgs/softball
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path output/$exp_name/dynamic3dgs/tennis

38
scripts/train_dynerf.sh
View File

@ -1,18 +1,30 @@
exp_name=$1
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/misc/split-cookie --port 6068 --expname "$exp_name/split-cookie" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/hypernerf/virg/vrig-3dprinter --port 6066 --expname "$exp_name/3dprinter" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/interp/chickchicken --port 6069 --expname "$exp_name/interp-chicken" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/interp/cut-lemon1 --port 6070 --expname "$exp_name/cut-lemon1" --configs arguments/$exp_name/cut-lemon1.py &
# export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/interp/hand1-dense-v2 --port 6071 --expname "$exp_name/hand1-dense-v2" --configs arguments/$exp_name/hand1-dense-v2.py
export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/dynerf/flame_salmon_1 --port 6468 --expname "$exp_name/flame_salmon_1" --configs arguments/$exp_name/flame_salmon_1.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dynerf/coffee_martini --port 6472 --expname "$exp_name/coffee_martini" --configs arguments/$exp_name/coffee_martini.py &
wait
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/cut_roasted_beef --configs arguments/$exp_name/default.py --skip_train --skip_test&
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/cook_spinach --configs arguments/$exp_name/default.py --skip_train --skip_test &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/sear_steak --configs arguments/$exp_name/default.py --skip_train --skip_test&
# export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/hand1-dense-v2 --configs arguments/$exp_name/hand1-dense-v2.py --skip_train
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dynerf/cook_spinach --port 6436 --expname "$exp_name/cook_spinach" --configs arguments/$exp_name/cook_spinach.py &
# wait
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/dynerf/cut_roasted_beef --port 6470 --expname "$exp_name/cut_roasted_beef" --configs arguments/$exp_name/cut_roasted_beef.py
wait
export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/dynerf/flame_steak --port 6471 --expname "$exp_name/flame_steak" --configs arguments/$exp_name/flame_steak.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/dynerf/sear_steak --port 6569 --expname "$exp_name/sear_steak" --configs arguments/$exp_name/sear_steak.py
wait
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/cut_roasted_beefe/" &
export CUDA_VISIBLE_DEVICES=1&&python metrics.py --model_path "output/$exp_name/cook_spinach/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/sear_steak/" &
# export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/hand1-dense-v2/"
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/cut_roasted_beef --configs arguments/$exp_name/cut_roasted_beef.py --skip_train &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/sear_steak --configs arguments/$exp_name/sear_steak.py --skip_train
wait
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/flame_steak --configs arguments/$exp_name/flame_steak.py --skip_train &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/flame_salmon_1 --configs arguments/$exp_name/flame_salmon_1.py --skip_train
wait
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/cook_spinach --configs arguments/$exp_name/cook_spinach.py --skip_train &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/coffee_martini --configs arguments/$exp_name/coffee_martini.py --skip_train &
wait
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/cut_roasted_beef/" &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/cook_spinach/"
wait
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/sear_steak/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/flame_salmon_1/"
wait
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/flame_steak/" &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/coffee_martini/"
echo "Done"

20
scripts/train_hyper.sh
View File

@ -1,20 +0,0 @@
exp_name=$1
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/misc/split-cookie --port 6068 --expname "$exp_name/split-cookie" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/hypernerf/virg/vrig-3dprinter --port 6066 --expname "$exp_name/3dprinter" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/interp/chickchicken --port 6069 --expname "$exp_name/interp-chicken" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/interp/cut-lemon1 --port 6070 --expname "$exp_name/cut-lemon1" --configs arguments/$exp_name/cut-lemon1.py &
# export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/interp/hand1-dense-v2 --port 6071 --expname "$exp_name/hand1-dense-v2" --configs arguments/$exp_name/hand1-dense-v2.py
wait
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/split-cookie --configs arguments/$exp_name/default.py --skip_train --skip_test &
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/3dprinter --configs arguments/$exp_name/default.py --skip_train --skip_test &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/interp-chicken --configs arguments/$exp_name/default.py --skip_train --skip_test&
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/cut-lemon1 --configs arguments/$exp_name/cut-lemon1.py --skip_train --skip_test&
# export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/hand1-dense-v2 --configs arguments/$exp_name/hand1-dense-v2.py --skip_train
wait
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/split-cookie/" &
export CUDA_VISIBLE_DEVICES=1&&python metrics.py --model_path "output/$exp_name/3dprinter/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/interp-chicken/" &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/cut-lemon1/" &
# export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/hand1-dense-v2/"
wait
echo "Done"

13
scripts/train_hyper_all.sh
View File

@ -1,13 +0,0 @@
# bash scripts/train_3dvideo.sh
# wait
bash scripts/train_hyper_one.sh hypernerf_format2_virg2
wait
bash scripts/train_hyper_one.sh hypernerf_format2_virg3
wait
# bash scripts/train_hyper.sh hypernerf_format2_lr2
# wait
# bash scripts/train_hyper.sh hypernerf_format2_prune
# wait
# wait
# bash scripts/train_ablation.sh dnerf_imageloss

27
scripts/train_hyper_interp.sh Normal file
View File

@ -0,0 +1,27 @@
exp_name=$1
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/interp/aleks-teapot --port 6568 --expname "$exp_name/interp/aleks-teapot" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/interp/slice-banana --port 6566 --expname "$exp_name/interp/slice-banana" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/interp/chickchicken --port 6569 --expname "$exp_name/interp/interp-chicken" --configs arguments/$exp_name/default.py &
wait
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/interp/cut-lemon1 --port 6670 --expname $exp_name/interp/cut-lemon1 --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/interp/hand1-dense-v2 --port 6671 --expname $exp_name/interp/hand1-dense-v2 --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/interp/torchocolate --port 6672 --expname $exp_name/interp/torchocolate --configs arguments/$exp_name/default.py &
wait
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/interp/aleks-teapot --configs arguments/$exp_name/default.py --skip_train &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/interp/slice-banana --configs arguments/$exp_name/default.py --skip_train &
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/interp/interp-chicken --configs arguments/$exp_name/default.py --skip_train &
wait
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/interp/cut-lemon1 --configs arguments/$exp_name/default.py --skip_train &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/interp/hand1-dense-v2 --configs arguments/$exp_name/default.py --skip_train&
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/interp/torchocolate --configs arguments/$exp_name/default.py --skip_train &
wait
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/interp/aleks-teapot/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/interp/slice-banana/" &
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/interp/interp-chicken/"
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/interp/cut-lemon1/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/interp/hand1-dense-v2/" &
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/interp/torchocolate/"
wait
echo "Done"

20
scripts/train_hyper_one.sh
View File

@ -1,20 +0,0 @@
exp_name=$1
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/virg/broom2 --port 6068 --expname "$exp_name/broom2" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=1&&python train.py -s data/hypernerf/virg/vrig-3dprinter --port 6066 --expname "$exp_name/3dprinter" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/virg/peel-banana --port 6069 --expname "$exp_name/peel-banana" --configs arguments/$exp_name/default.py &
export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/virg/vrig-chicken --port 6070 --expname "$exp_name/vrig-chicken" --configs arguments/$exp_name/default.py &
# export CUDA_VISIBLE_DEVICES=3&&python train.py -s data/hypernerf/interp/hand1-dense-v2 --port 6071 --expname "$exp_name/hand1-dense-v2" --configs arguments/$exp_name/hand1-dense-v2.py
wait
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/broom2 --configs arguments/$exp_name/default.py --skip_train &
export CUDA_VISIBLE_DEVICES=1&&python render.py --model_path output/$exp_name/3dprinter --configs arguments/$exp_name/default.py --skip_train &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/peel-banana --configs arguments/$exp_name/default.py --skip_train&
export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/vrig-chicken --configs arguments/$exp_name/default.py --skip_train&
# export CUDA_VISIBLE_DEVICES=3&&python render.py --model_path output/$exp_name/hand1-dense-v2 --configs arguments/$exp_name/hand1-dense-v2.py --skip_train
wait
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/broom2/" &
export CUDA_VISIBLE_DEVICES=1&&python metrics.py --model_path "output/$exp_name/3dprinter/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/peel-banana/" &
export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/vrig-chicken/" &
# export CUDA_VISIBLE_DEVICES=3&&python metrics.py --model_path "output/$exp_name/hand1-dense-v2/"
wait
echo "Done"

17
scripts/train_hyper_virg.sh Normal file
View File

@ -0,0 +1,17 @@
exp_name=$1
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/virg/broom2 --port 6068 --expname "$exp_name/broom2" --configs arguments/$exp_name/broom2.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/virg/vrig-3dprinter --port 6066 --expname "$exp_name/3dprinter" --configs arguments/$exp_name/3dprinter.py &
export CUDA_VISIBLE_DEVICES=2&&python train.py -s data/hypernerf/virg/peel-banana --port 6069 --expname "$exp_name/peel-banana" --configs arguments/$exp_name/banana.py &
export CUDA_VISIBLE_DEVICES=0&&python train.py -s data/hypernerf/virg/vrig-chicken --port 6070 --expname "$exp_name/vrig-chicken" --configs arguments/$exp_name/chicken.py
wait
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/broom2 --configs arguments/$exp_name/broom2.py --skip_train --skip_test &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/3dprinter --configs arguments/$exp_name/3dprinter.py --skip_train --skip_test &
export CUDA_VISIBLE_DEVICES=2&&python render.py --model_path output/$exp_name/peel-banana --configs arguments/$exp_name/banana.py --skip_train --skip_test &
export CUDA_VISIBLE_DEVICES=0&&python render.py --model_path output/$exp_name/vrig-chicken --configs arguments/$exp_name/chicken.py --skip_train --skip_test &
wait
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/broom2/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/3dprinter/" &
export CUDA_VISIBLE_DEVICES=2&&python metrics.py --model_path "output/$exp_name/peel-banana/" &
export CUDA_VISIBLE_DEVICES=0&&python metrics.py --model_path "output/$exp_name/vrig-chicken/" &
wait
echo "Done"

70
scripts/train_test_split.py Normal file
View File

@ -0,0 +1,70 @@
import numpy as np
import cv2
import os
import shutil
from tqdm import tqdm
def resort(frames):
newframes = {}
min_frameid = 10000000
for frame in frames:
frameid = int(frame["file_path"].split('/')[1].split('.')[0])
# print()
if frameid < min_frameid:min_frameid = frameid
newframes[frameid] = frame
return [newframes[i+min_frameid] for i in range(len(frames))]
inputpath = "data/custom/wave-ns/"
outputpath = "data/custom/wave-train/"
testskip = 10
if not os.path.exists(outputpath):
os.makedirs(outputpath)
image_path = os.listdir(os.path.join(inputpath,"images"))
import json
with open(os.path.join(inputpath,"transforms.json"),"r") as f:
meta = json.load(f)
cnt = 0
train_json = {
"w": meta["w"],
"h": meta["h"],
"fl_x": meta["fl_x"],
"fl_y": meta["fl_y"],
"cx": meta["cx"],
"cy": meta["cy"],
"camera_model" : meta["camera_model"],
"frames":[]
}
test_json = {
"w": meta["w"],
"h": meta["h"],
"fl_x": meta["fl_x"],
"fl_y": meta["fl_y"],
"cx": meta["cx"],
"cy": meta["cy"],
"camera_model" : meta["camera_model"],
"frames":[]
}
train_image_path = os.path.join(outputpath,"train")
os.makedirs(train_image_path)
test_image_path = os.path.join(outputpath,"test")
os.makedirs(test_image_path)
# meta["frames"] = resort(meta["frames"])
totallen = len(meta["frames"])
for index, frame in tqdm(enumerate(meta["frames"])):
image_path = os.path.join(inputpath,frame["file_path"])
frame["time"] = index/totallen
if index % testskip == 0:
frame["file_path"] = "test/" + frame["file_path"].split("/")[-1]
test_json["frames"].append(frame)
shutil.copy(image_path, test_image_path)
else:
frame["file_path"] = "train/" + frame["file_path"].split("/")[-1]
train_json["frames"].append(frame)
shutil.copy(image_path, train_image_path)
with open(os.path.join(outputpath,"transforms_train.json"),"w") as f:
json.dump(train_json, f)
with open(os.path.join(outputpath,"transforms_test.json"),"w") as f:
json.dump(test_json, f)
print("done")

2
submodules/depth-diff-gaussian-rasterization

@ -1 +1 @@
Subproject commit f2d8fa9921ea9a6cb9ac1c33a34ebd1b11510657
Subproject commit 2eb32ea251d3b339dab3af8b6fd78d7dec3caf8e

39
test.py Normal file
View File

@ -0,0 +1,39 @@
import cv2
import os
import re
def sorted_alphanumeric(data):
"""
对给定的数据进行字母数字排序（考虑数字的数值大小）
"""
convert = lambda text: int(text) if text.isdigit() else text.lower()
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
return sorted(data, key=alphanum_key)
def create_video_from_images(folder_path, output_file, frame_rate=30, img_size=None):
images = [img for img in os.listdir(folder_path) if img.endswith(".jpg") or img.endswith(".png")]
images = sorted_alphanumeric(images) # 使用自定义的排序函数
# 获取第一张图片的尺寸
frame = cv2.imread(os.path.join(folder_path, images[0]))
height, width, layers = frame.shape
# 如果指定了img_size，则调整尺寸
if img_size is not None:
width, height = img_size
# 定义视频编码和创建VideoWriter对象
fourcc = cv2.VideoWriter_fourcc(*'mp4v') # 可以更改为其他编码器
video = cv2.VideoWriter(output_file, fourcc, frame_rate, (width, height))
for image in images:
img = cv2.imread(os.path.join(folder_path, image))
if img_size is not None:
img = cv2.resize(img, img_size)
video.write(img)
cv2.destroyAllWindows()
video.release()
# 使用示例
folder_path = 'output/editing_render' # 替换为您的图片文件夹路径
output_file = 'output_video.mp4' # 输出视频文件名
create_video_from_images(folder_path, output_file)

157
train.py
View File

@ -10,7 +10,7 @@
#
import numpy as np
import random
import os
import os, sys
import torch
from random import randint
from utils.loss_utils import l1_loss, ssim, l2_loss, lpips_loss
@ -25,10 +25,12 @@ from argparse import ArgumentParser, Namespace
from arguments import ModelParams, PipelineParams, OptimizationParams, ModelHiddenParams
from torch.utils.data import DataLoader
from utils.timer import Timer
from utils.loader_utils import FineSampler, get_stamp_list
import lpips
from utils.scene_utils import render_training_image
from time import time
import copy
to8b = lambda x : (255*np.clip(x.cpu().numpy(),0,1)).astype(np.uint8)
try:
@ -60,8 +62,41 @@ def scene_reconstruction(dataset, opt, hyper, pipe, testing_iterations, saving_i
progress_bar = tqdm(range(first_iter, final_iter), desc="Training progress")
first_iter += 1
lpips_model = lpips.LPIPS(net="alex").cuda()
# lpips_model = lpips.LPIPS(net="alex").cuda()
video_cams = scene.getVideoCameras()
test_cams = scene.getTestCameras()
train_cams = scene.getTrainCameras()
if not viewpoint_stack and not opt.dataloader:
# dnerf's branch
viewpoint_stack = [i for i in train_cams]
temp_list = copy.deepcopy(viewpoint_stack)
#
batch_size = opt.batch_size
print("data loading done")
if opt.dataloader:
viewpoint_stack = scene.getTrainCameras()
if opt.custom_sampler is not None:
sampler = FineSampler(viewpoint_stack)
viewpoint_stack_loader = DataLoader(viewpoint_stack, batch_size=batch_size,sampler=sampler,num_workers=32,collate_fn=list)
random_loader = False
else:
viewpoint_stack_loader = DataLoader(viewpoint_stack, batch_size=batch_size,shuffle=True,num_workers=32,collate_fn=list)
random_loader = True
loader = iter(viewpoint_stack_loader)
# dynerf, zerostamp_init
# breakpoint()
if stage == "coarse" and opt.zerostamp_init:
load_in_memory = True
# batch_size = 4
temp_list = get_stamp_list(viewpoint_stack,0)
viewpoint_stack = temp_list.copy()
else:
load_in_memory = False
for iteration in range(first_iter, final_iter+1):
if network_gui.conn == None:
network_gui.try_connect()
@ -70,7 +105,7 @@ def scene_reconstruction(dataset, opt, hyper, pipe, testing_iterations, saving_i
net_image_bytes = None
custom_cam, do_training, pipe.convert_SHs_python, pipe.compute_cov3D_python, keep_alive, scaling_modifer, ts = network_gui.receive()
if custom_cam != None:
net_image = render(custom_cam, gaussians, pipe, background, scaling_modifer, stage="stage")["render"]
net_image = render(custom_cam, gaussians, pipe, background, scaling_modifer, stage=stage, cam_type=scene.dataset_type)["render"]
net_image_bytes = memoryview((torch.clamp(net_image, min=0, max=1.0) * 255).byte().permute(1, 2, 0).contiguous().cpu().numpy())
network_gui.send(net_image_bytes, dataset.source_path)
if do_training and ((iteration < int(opt.iterations)) or not keep_alive):
@ -87,22 +122,33 @@ def scene_reconstruction(dataset, opt, hyper, pipe, testing_iterations, saving_i
gaussians.oneupSHdegree()
# Pick a random Camera
if not viewpoint_stack:
viewpoint_stack = scene.getTrainCameras()
batch_size = 1
viewpoint_stack_loader = DataLoader(viewpoint_stack, batch_size=batch_size,shuffle=True,num_workers=32,collate_fn=list)
loader = iter(viewpoint_stack_loader)
if opt.dataloader:
# dynerf's branch
if opt.dataloader and not load_in_memory:
try:
viewpoint_cams = next(loader)
except StopIteration:
print("reset dataloader")
batch_size = 1
print("reset dataloader into random dataloader.")
if not random_loader:
viewpoint_stack_loader = DataLoader(viewpoint_stack, batch_size=opt.batch_size,shuffle=True,num_workers=32,collate_fn=list)
random_loader = True
loader = iter(viewpoint_stack_loader)
else:
idx = randint(0, len(viewpoint_stack)-1)
viewpoint_cams = [viewpoint_stack[idx]]
else:
idx = 0
viewpoint_cams = []
while idx < batch_size :
viewpoint_cam = viewpoint_stack.pop(randint(0,len(viewpoint_stack)-1))
if not viewpoint_stack :
viewpoint_stack = temp_list.copy()
viewpoint_cams.append(viewpoint_cam)
idx +=1
if len(viewpoint_cams) == 0:
continue
# print(len(viewpoint_cams))
# breakpoint()
# Render
if (iteration - 1) == debug_from:
pipe.debug = True
@ -112,10 +158,14 @@ def scene_reconstruction(dataset, opt, hyper, pipe, testing_iterations, saving_i
visibility_filter_list = []
viewspace_point_tensor_list = []
for viewpoint_cam in viewpoint_cams:
render_pkg = render(viewpoint_cam, gaussians, pipe, background, stage=stage)
render_pkg = render(viewpoint_cam, gaussians, pipe, background, stage=stage,cam_type=scene.dataset_type)
image, viewspace_point_tensor, visibility_filter, radii = render_pkg["render"], render_pkg["viewspace_points"], render_pkg["visibility_filter"], render_pkg["radii"]
images.append(image.unsqueeze(0))
gt_image = viewpoint_cam.original_image.cuda()
if scene.dataset_type!="PanopticSports":
gt_image = viewpoint_cam.original_image.cuda()
else:
gt_image = viewpoint_cam['image'].cuda()
gt_images.append(gt_image.unsqueeze(0))
radii_list.append(radii.unsqueeze(0))
visibility_filter_list.append(visibility_filter.unsqueeze(0))
@ -127,8 +177,8 @@ def scene_reconstruction(dataset, opt, hyper, pipe, testing_iterations, saving_i
image_tensor = torch.cat(images,0)
gt_image_tensor = torch.cat(gt_images,0)
# Loss
Ll1 = l1_loss(image_tensor, gt_image_tensor)
# Ll1 = l2_loss(image, gt_image)
# breakpoint()
Ll1 = l1_loss(image_tensor, gt_image_tensor[:,:3,:,:])
psnr_ = psnr(image_tensor, gt_image_tensor).mean().double()
# norm
@ -137,16 +187,19 @@ def scene_reconstruction(dataset, opt, hyper, pipe, testing_iterations, saving_i
loss = Ll1
if stage == "fine" and hyper.time_smoothness_weight != 0:
# tv_loss = 0
tv_loss = gaussians.compute_regulation(hyper.time_smoothness_weight, hyper.plane_tv_weight, hyper.l1_time_planes)
tv_loss = gaussians.compute_regulation(hyper.time_smoothness_weight, hyper.l1_time_planes, hyper.plane_tv_weight)
loss += tv_loss
if opt.lambda_dssim != 0:
ssim_loss = ssim(image_tensor,gt_image_tensor)
loss += opt.lambda_dssim * (1.0-ssim_loss)
if opt.lambda_lpips !=0:
lpipsloss = lpips_loss(image_tensor,gt_image_tensor,lpips_model)
loss += opt.lambda_lpips * lpipsloss
# if opt.lambda_lpips !=0:
# lpipsloss = lpips_loss(image_tensor,gt_image_tensor,lpips_model)
# loss += opt.lambda_lpips * lpipsloss
loss.backward()
if torch.isnan(loss).any():
print("loss is nan,end training, reexecv program now.")
os.execv(sys.executable, [sys.executable] + sys.argv)
viewspace_point_tensor_grad = torch.zeros_like(viewspace_point_tensor)
for idx in range(0, len(viewspace_point_tensor_list)):
viewspace_point_tensor_grad = viewspace_point_tensor_grad + viewspace_point_tensor_list[idx].grad
@ -167,17 +220,19 @@ def scene_reconstruction(dataset, opt, hyper, pipe, testing_iterations, saving_i
# Log and save
timer.pause()
training_report(tb_writer, iteration, Ll1, loss, l1_loss, iter_start.elapsed_time(iter_end), testing_iterations, scene, render, [pipe, background], stage)
training_report(tb_writer, iteration, Ll1, loss, l1_loss, iter_start.elapsed_time(iter_end), testing_iterations, scene, render, [pipe, background], stage, scene.dataset_type)
if (iteration in saving_iterations):
print("\n[ITER {}] Saving Gaussians".format(iteration))
scene.save(iteration, stage)
if dataset.render_process:
if (iteration < 1000 and iteration % 10 == 1) \
or (iteration < 3000 and iteration % 50 == 1) \
or (iteration < 10000 and iteration % 100 == 1) \
or (iteration < 60000 and iteration % 100 ==1):
if (iteration < 1000 and iteration % 10 == 9) \
or (iteration < 3000 and iteration % 50 == 49) \
or (iteration < 60000 and iteration % 100 == 99) :
# breakpoint()
render_training_image(scene, gaussians, [test_cams[iteration%len(test_cams)]], render, pipe, background, stage+"test", iteration,timer.get_elapsed_time(),scene.dataset_type)
render_training_image(scene, gaussians, [train_cams[iteration%len(train_cams)]], render, pipe, background, stage+"train", iteration,timer.get_elapsed_time(),scene.dataset_type)
# render_training_image(scene, gaussians, train_cams, render, pipe, background, stage+"train", iteration,timer.get_elapsed_time(),scene.dataset_type)
render_training_image(scene, gaussians, video_cams, render, pipe, background, stage, iteration-1,timer.get_elapsed_time())
# total_images.append(to8b(temp_image).transpose(1,2,0))
timer.start()
# Densification
@ -192,20 +247,24 @@ def scene_reconstruction(dataset, opt, hyper, pipe, testing_iterations, saving_i
else:
opacity_threshold = opt.opacity_threshold_fine_init - iteration*(opt.opacity_threshold_fine_init - opt.opacity_threshold_fine_after)/(opt.densify_until_iter)
densify_threshold = opt.densify_grad_threshold_fine_init - iteration*(opt.densify_grad_threshold_fine_init - opt.densify_grad_threshold_after)/(opt.densify_until_iter )
if iteration > opt.densify_from_iter and iteration % opt.densification_interval == 0 :
if iteration > opt.densify_from_iter and iteration % opt.densification_interval == 0 and gaussians.get_xyz.shape[0]<360000:
size_threshold = 20 if iteration > opt.opacity_reset_interval else None
gaussians.densify(densify_threshold, opacity_threshold, scene.cameras_extent, size_threshold)
if iteration > opt.pruning_from_iter and iteration % opt.pruning_interval == 0:
gaussians.densify(densify_threshold, opacity_threshold, scene.cameras_extent, size_threshold, 5, 5, scene.model_path, iteration, stage)
if iteration > opt.pruning_from_iter and iteration % opt.pruning_interval == 0 and gaussians.get_xyz.shape[0]>200000:
size_threshold = 20 if iteration > opt.opacity_reset_interval else None
gaussians.prune(densify_threshold, opacity_threshold, scene.cameras_extent, size_threshold)
if iteration % opt.opacity_reset_interval == 0 or (dataset.white_background and iteration == opt.densify_from_iter):
# if iteration > opt.densify_from_iter and iteration % opt.densification_interval == 0 :
if iteration % opt.densification_interval == 0 and gaussians.get_xyz.shape[0]<360000 and opt.add_point:
gaussians.grow(5,5,scene.model_path,iteration,stage)
# torch.cuda.empty_cache()
if iteration % opt.opacity_reset_interval == 0:
print("reset opacity")
gaussians.reset_opacity()
# Optimizer step
if iteration < opt.iterations:
@ -253,7 +312,7 @@ def prepare_output_and_logger(expname):
print("Tensorboard not available: not logging progress")
return tb_writer
def training_report(tb_writer, iteration, Ll1, loss, l1_loss, elapsed, testing_iterations, scene : Scene, renderFunc, renderArgs, stage):
def training_report(tb_writer, iteration, Ll1, loss, l1_loss, elapsed, testing_iterations, scene : Scene, renderFunc, renderArgs, stage, dataset_type):
if tb_writer:
tb_writer.add_scalar(f'{stage}/train_loss_patches/l1_loss', Ll1.item(), iteration)
tb_writer.add_scalar(f'{stage}/train_loss_patchestotal_loss', loss.item(), iteration)
@ -263,6 +322,7 @@ def training_report(tb_writer, iteration, Ll1, loss, l1_loss, elapsed, testing_i
# Report test and samples of training set
if iteration in testing_iterations:
torch.cuda.empty_cache()
#
validation_configs = ({'name': 'test', 'cameras' : [scene.getTestCameras()[idx % len(scene.getTestCameras())] for idx in range(10, 5000, 299)]},
{'name': 'train', 'cameras' : [scene.getTrainCameras()[idx % len(scene.getTrainCameras())] for idx in range(10, 5000, 299)]})
@ -271,17 +331,26 @@ def training_report(tb_writer, iteration, Ll1, loss, l1_loss, elapsed, testing_i
l1_test = 0.0
psnr_test = 0.0
for idx, viewpoint in enumerate(config['cameras']):
image = torch.clamp(renderFunc(viewpoint, scene.gaussians,stage=stage, *renderArgs)["render"], 0.0, 1.0)
gt_image = torch.clamp(viewpoint.original_image.to("cuda"), 0.0, 1.0)
if tb_writer and (idx < 5):
tb_writer.add_images(stage + "/"+config['name'] + "_view_{}/render".format(viewpoint.image_name), image[None], global_step=iteration)
if iteration == testing_iterations[0]:
tb_writer.add_images(stage + "/"+config['name'] + "_view_{}/ground_truth".format(viewpoint.image_name), gt_image[None], global_step=iteration)
image = torch.clamp(renderFunc(viewpoint, scene.gaussians,stage=stage, cam_type=dataset_type, *renderArgs)["render"], 0.0, 1.0)
if dataset_type == "PanopticSports":
gt_image = torch.clamp(viewpoint["image"].to("cuda"), 0.0, 1.0)
else:
gt_image = torch.clamp(viewpoint.original_image.to("cuda"), 0.0, 1.0)
try:
if tb_writer and (idx < 5):
tb_writer.add_images(stage + "/"+config['name'] + "_view_{}/render".format(viewpoint.image_name), image[None], global_step=iteration)
if iteration == testing_iterations[0]:
tb_writer.add_images(stage + "/"+config['name'] + "_view_{}/ground_truth".format(viewpoint.image_name), gt_image[None], global_step=iteration)
except:
pass
l1_test += l1_loss(image, gt_image).mean().double()
psnr_test += psnr(image, gt_image).mean().double()
# mask=viewpoint.mask
psnr_test += psnr(image, gt_image, mask=None).mean().double()
psnr_test /= len(config['cameras'])
l1_test /= len(config['cameras'])
print("\n[ITER {}] Evaluating {}: L1 {} PSNR {}".format(iteration, config['name'], l1_test, psnr_test))
# print("sh feature",scene.gaussians.get_features.shape)
if tb_writer:
tb_writer.add_scalar(stage + "/"+config['name'] + '/loss_viewpoint - l1_loss', l1_test, iteration)
tb_writer.add_scalar(stage+"/"+config['name'] + '/loss_viewpoint - psnr', psnr_test, iteration)
@ -314,8 +383,8 @@ if __name__ == "__main__":
parser.add_argument('--port', type=int, default=6009)
parser.add_argument('--debug_from', type=int, default=-1)
parser.add_argument('--detect_anomaly', action='store_true', default=False)
parser.add_argument("--test_iterations", nargs="+", type=int, default=[i*500 for i in range(0,120)])
parser.add_argument("--save_iterations", nargs="+", type=int, default=[2000, 3000, 7_000, 8000, 9000, 14000, 20000, 30_000,45000,60000])
parser.add_argument("--test_iterations", nargs="+", type=int, default=[500*i for i in range(100)])
parser.add_argument("--save_iterations", nargs="+", type=int, default=[1000, 3000, 4000, 5000, 6000, 7_000, 9000, 10000, 12000, 14000, 20000, 30_000, 45000, 60000])
parser.add_argument("--quiet", action="store_true")
parser.add_argument("--checkpoint_iterations", nargs="+", type=int, default=[])
parser.add_argument("--start_checkpoint", type=str, default = None)

5
utils/general_utils.py
View File

@ -23,7 +23,10 @@ def PILtoTorch(pil_image, resolution):
resized_image_PIL = pil_image.resize(resolution)
else:
resized_image_PIL = pil_image
resized_image = torch.from_numpy(np.array(resized_image_PIL)) / 255.0
if np.array(resized_image_PIL).max()!=1:
resized_image = torch.from_numpy(np.array(resized_image_PIL)) / 255.0
else:
resized_image = torch.from_numpy(np.array(resized_image_PIL))
if len(resized_image.shape) == 3:
return resized_image.permute(2, 0, 1)
else:

57
utils/graphics_utils.py
View File

@ -74,4 +74,59 @@ def fov2focal(fov, pixels):
return pixels / (2 * math.tan(fov / 2))
def focal2fov(focal, pixels):
return 2*math.atan(pixels/(2*focal))
return 2*math.atan(pixels/(2*focal))
def apply_rotation(q1, q2):
"""
Applies a rotation to a quaternion.
Parameters:
q1 (Tensor): The original quaternion.
q2 (Tensor): The rotation quaternion to be applied.
Returns:
Tensor: The resulting quaternion after applying the rotation.
"""
# Extract components for readability
w1, x1, y1, z1 = q1
w2, x2, y2, z2 = q2
# Compute the product of the two quaternions
w3 = w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2
x3 = w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2
y3 = w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2
z3 = w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2
# Combine the components into a new quaternion tensor
q3 = torch.tensor([w3, x3, y3, z3])
# Normalize the resulting quaternion
q3_normalized = q3 / torch.norm(q3)
return q3_normalized
def batch_quaternion_multiply(q1, q2):
"""
Multiply batches of quaternions.
Args:
- q1 (torch.Tensor): A tensor of shape [N, 4] representing the first batch of quaternions.
- q2 (torch.Tensor): A tensor of shape [N, 4] representing the second batch of quaternions.
Returns:
- torch.Tensor: The resulting batch of quaternions after applying the rotation.
"""
# Calculate the product of each quaternion in the batch
w = q1[:, 0] * q2[:, 0] - q1[:, 1] * q2[:, 1] - q1[:, 2] * q2[:, 2] - q1[:, 3] * q2[:, 3]
x = q1[:, 0] * q2[:, 1] + q1[:, 1] * q2[:, 0] + q1[:, 2] * q2[:, 3] - q1[:, 3] * q2[:, 2]
y = q1[:, 0] * q2[:, 2] - q1[:, 1] * q2[:, 3] + q1[:, 2] * q2[:, 0] + q1[:, 3] * q2[:, 1]
z = q1[:, 0] * q2[:, 3] + q1[:, 1] * q2[:, 2] - q1[:, 2] * q2[:, 1] + q1[:, 3] * q2[:, 0]
# Combine into new quaternions
q3 = torch.stack((w, x, y, z), dim=1)
# Normalize the quaternions
norm_q3 = q3 / torch.norm(q3, dim=1, keepdim=True)
return norm_q3

25
utils/image_utils.py
View File

@ -14,6 +14,25 @@ import torch
def mse(img1, img2):
return (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
@torch.no_grad()
def psnr(img1, img2):
mse = (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
return 20 * torch.log10(1.0 / torch.sqrt(mse))
def psnr(img1, img2, mask=None):
if mask is not None:
img1 = img1.flatten(1)
img2 = img2.flatten(1)
mask = mask.flatten(1).repeat(3,1)
mask = torch.where(mask!=0,True,False)
img1 = img1[mask]
img2 = img2[mask]
mse = (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
else:
mse = (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
psnr = 20 * torch.log10(1.0 / torch.sqrt(mse.float()))
if mask is not None:
if torch.isinf(psnr).any():
print(mse.mean(),psnr.mean())
psnr = 20 * torch.log10(1.0 / torch.sqrt(mse.float()))
psnr = psnr[~torch.isinf(psnr)]
return psnr

52
utils/loader_utils.py Normal file
View File

@ -0,0 +1,52 @@
import os
import cv2
import random
import numpy as np
from PIL import Image
import torch
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import Sampler
from torchvision import transforms, utils
import random
def get_stamp_list(dataset, timestamp):
frame_length = int(len(dataset)/len(dataset.dataset.poses))
# print(frame_length)
if timestamp > frame_length:
raise IndexError("input timestamp bigger than total timestamp.")
print("select index:",[i*frame_length+timestamp for i in range(len(dataset.dataset.poses))])
return [dataset[i*frame_length+timestamp] for i in range(len(dataset.dataset.poses))]
class FineSampler(Sampler):
def __init__(self, dataset):
self.len_dataset = len(dataset)
self.len_pose = len(dataset.dataset.poses)
self.frame_length = int(self.len_dataset/ self.len_pose)
sample_list = []
for i in range(self.frame_length):
for j in range(4):
idx = torch.randperm(self.len_pose) *self.frame_length + i
# print(idx)
# breakpoint()
now_list = []
cnt = 0
for item in idx.tolist():
now_list.append(item)
cnt+=1
if cnt % 2 == 0 and len(sample_list)>2:
select_element = [x for x in random.sample(sample_list,2)]
now_list += select_element
sample_list += now_list
self.sample_list = sample_list
# print(self.sample_list)
# breakpoint()
print("one epoch containing:",len(self.sample_list))
def __iter__(self):
return iter(self.sample_list)
def __len__(self):
return len(self.sample_list)

196
utils/point_utils.py Normal file
View File

@ -0,0 +1,196 @@
import torch
import open3d as o3d
from torch.utils.data import TensorDataset, random_split
from tqdm import tqdm
import open3d as o3d
import numpy as np
from torch_cluster import grid_cluster
def voxel_down_sample_custom(points, voxel_size):
# 将点云归一化到体素网格
voxel_grid = torch.floor(points / voxel_size)
# 找到唯一的体素，并获取它们在原始体素网格中的索引
unique_voxels, inverse_indices = torch.unique(voxel_grid, dim=0, return_inverse=True)
# 创建一个新的点云，其中每个点是其对应体素中所有点的平均值
new_points = torch.zeros_like(unique_voxels)
new_points_count = torch.zeros(unique_voxels.size(0), dtype=torch.long)
# for i in tqdm(range(points.size(0))):
new_points[inverse_indices] = points
# new_points_count[inverse_indices[i]] += 1
# new_points /= new_points_count.unsqueeze(-1)
return new_points, inverse_indices
def downsample_point_cloud(points, ratio):
# 创建一个TensorDataset
dataset = TensorDataset(points)
# 计算下采样后的点的数量
num_points = len(dataset)
num_downsampled_points = int(num_points * ratio)
# 使用random_split进行下采样
downsampled_dataset, _ = random_split(dataset, [num_downsampled_points, num_points - num_downsampled_points])
# 获取下采样后的点的index和点云矩阵
indices = torch.tensor([i for i, _ in enumerate(downsampled_dataset)])
downsampled_points = torch.stack([x for x, in downsampled_dataset])
return indices, downsampled_points
def downsample_point_cloud_open3d(points, voxel_size):
# 创建一个点云对象
downsampled_pcd, inverse_indices = voxel_down_sample_custom(points, voxel_size)
downsampled_points = downsampled_pcd
# 获取下采样后的点云矩阵
return torch.tensor(downsampled_points)
def downsample_point_cloud_cluster(points, voxel_size):
# 创建一个点云对象
cluster = grid_cluster(points, size=torch.tensor([1,1,1]))
# 获取下采样后的点云矩阵
# downsampled_points = np.asarray(downsampled_pcd.points)
return cluster, points
import torch
from sklearn.neighbors import NearestNeighbors
def upsample_point_cloud(points, density_threshold, displacement_scale, iter_pass):
# 计算每个点的密度
# breakpoint()
try:
nbrs = NearestNeighbors(n_neighbors=2+iter_pass, algorithm='ball_tree').fit(points)
distances, indices = nbrs.kneighbors(points)
except:
print("no point added")
return points, torch.tensor([]), torch.tensor([]), torch.zeros((points.shape[0]), dtype=torch.bool)
# 找出密度低的点
low_density_points = points[distances[:,1] > density_threshold]
low_density_index = distances[:,1] > density_threshold
low_density_index = torch.from_numpy(low_density_index)
# 复制这些点并添加随机位移
num_points = low_density_points.shape[0]
displacements = torch.randn(num_points, 3) * displacement_scale
new_points = low_density_points + displacements
# 返回新的点云矩阵
return points, low_density_points, new_points, low_density_index
def visualize_point_cloud(points, low_density_points, new_points):
# 创建一个点云对象
pcd = o3d.geometry.PointCloud()
# 给被选中的点云添加一个小的偏移量
low_density_points += 0.01
# 将所有的点合并到一起
all_points = np.concatenate([points, low_density_points, new_points], axis=0)
pcd.points = o3d.utility.Vector3dVector(all_points)
# 创建颜色数组
colors = np.zeros((all_points.shape[0], 3))
colors[:points.shape[0]] = [0, 0, 0] # 黑色表示初始化的点云
colors[points.shape[0]:points.shape[0]+low_density_points.shape[0]] = [1, 0, 0] # 红色表示被选中的点云
colors[points.shape[0]+low_density_points.shape[0]:] = [0, 1, 0] # 绿色表示增长的点云
pcd.colors = o3d.utility.Vector3dVector(colors)
# 显示点云
o3d.visualization.draw_geometries([pcd])
def combine_pointcloud(points, low_density_points, new_points):
pcd = o3d.geometry.PointCloud()
# 给被选中的点云添加一个小的偏移量
low_density_points += 0.01
new_points -= 0.01
# 将所有的点合并到一起
all_points = np.concatenate([points, low_density_points, new_points], axis=0)
pcd.points = o3d.utility.Vector3dVector(all_points)
# 创建颜色数组
colors = np.zeros((all_points.shape[0], 3))
colors[:points.shape[0]] = [0, 0, 0] # 黑色表示初始化的点云
colors[points.shape[0]:points.shape[0]+low_density_points.shape[0]] = [1, 0, 0] # 红色表示被选中的点云
colors[points.shape[0]+low_density_points.shape[0]:] = [0, 1, 0] # 绿色表示增长的点云
pcd.colors = o3d.utility.Vector3dVector(colors)
return pcd
def addpoint(point_cloud,density_threshold,displacement_scale, iter_pass,):
# density_threshold: 密度的阈值，越大能筛选出越稀疏的点。
# displacement_scale: 在以displacement_scale的圆心内随机生成点
points, low_density_points, new_points, low_density_index = upsample_point_cloud(point_cloud,density_threshold,displacement_scale, iter_pass)
# breakpoint()
# breakpoint()
print("low_density_points",low_density_points.shape[0])
return point_cloud, low_density_points, new_points, low_density_index
def find_point_indices(origin_point, goal_point):
indices = torch.nonzero((origin_point[:, None] == goal_point).all(-1), as_tuple=True)[0]
return indices
def find_indices_in_A(A, B):
"""
找出子集矩阵 B 中每个点在点云矩阵 A 中的索引 u。
参数:
A (torch.Tensor): 点云矩阵 A，大小为 [N, 3]。
B (torch.Tensor): 子集矩阵 B，大小为 [M, 3]。
返回:
torch.Tensor: 包含 B 中每个点在 A 中的索引 u 的张量，形状为 (M,)。
"""
is_equal = torch.eq(B.view(1, -1, 3), A.view(-1, 1, 3))
u_indices = torch.nonzero(is_equal, as_tuple=False)[:, 0]
return torch.unique(u_indices)
if __name__ =="__main__":
#
from time import time
pass_=0
# filename=f"pointcloud/pass_{pass_}.ply"
filename = "point_cloud.ply"
pcd = o3d.io.read_point_cloud(filename)
point_cloud = torch.tensor(pcd.points)
voxel_size = 8
density_threshold=20
displacement_scale=5
for i in range(pass_+1, 50):
print("pass ",i)
time0 = time()
point_downsample = point_cloud
flag = False
while point_downsample.shape[0]>1000:
if flag:
voxel_size+=8
print("point size:",point_downsample.shape[0])
point_downsample = downsample_point_cloud_open3d(point_cloud,voxel_size=voxel_size)
flag = True
print("point size:",point_downsample.shape[0])
# downsampled_point_index = find_point_indices(point_cloud, point_downsample)
downsampled_point_index = find_indices_in_A(point_cloud, point_downsample)
print("selected_num",point_cloud[downsampled_point_index].shape[0])
_, low_density_points, new_points, low_density_index = addpoint(point_cloud[downsampled_point_index],density_threshold=density_threshold,displacement_scale=displacement_scale,iter_pass=0)
if new_points.shape[0] < 100:
density_threshold /= 2
displacement_scale /= 2
print("reduce diplacement_scale to: ",displacement_scale)
global_mask = torch.zeros((point_cloud.shape[0]), dtype=torch.bool)
global_mask[downsampled_point_index] = low_density_index
time1 = time()
print("time cost:",time1-time0,"new_points:",new_points.shape[0])
if low_density_points.shape[0] == 0:
print("no more points.")
continue
# breakpoint()
point = combine_pointcloud(point_cloud, low_density_points, new_points)
point_cloud = torch.tensor(point.points)
o3d.io.write_point_cloud(f"pointcloud/pass_{i}.ply",point)
# visualize_qpoint_cloud( point_cloud, low_density_points, new_points)

91
utils/pose_utils.py Normal file
View File

@ -0,0 +1,91 @@
import numpy as np
from scipy.spatial.transform import Rotation as R
from scene.utils import Camera
from copy import deepcopy
def rotation_matrix_to_quaternion(rotation_matrix):
"""将旋转矩阵转换为四元数"""
return R.from_matrix(rotation_matrix).as_quat()
def quaternion_to_rotation_matrix(quat):
"""将四元数转换为旋转矩阵"""
return R.from_quat(quat).as_matrix()
def quaternion_slerp(q1, q2, t):
"""在两个四元数之间进行球面线性插值（SLERP）"""
# 计算两个四元数之间的点积
dot = np.dot(q1, q2)
# 如果点积为负，取反一个四元数以保证最短路径插值
if dot < 0.0:
q1 = -q1
dot = -dot
# 防止数值误差导致的问题
dot = np.clip(dot, -1.0, 1.0)
# 计算插值参数
theta = np.arccos(dot) * t
q3 = q2 - q1 * dot
q3 = q3 / np.linalg.norm(q3)
# 计算插值结果
return np.cos(theta) * q1 + np.sin(theta) * q3
def bezier_interpolation(p1, p2, t):
"""在两点之间使用贝塞尔曲线进行插值"""
return (1 - t) * p1 + t * p2
def linear_interpolation(v1, v2, t):
"""线性插值"""
return (1 - t) * v1 + t * v2
def smooth_camera_poses(cameras, num_interpolations=5):
"""对一系列相机位姿进行平滑处理，通过在每对位姿之间插入额外的位姿"""
smoothed_cameras = []
smoothed_times = []
total_poses = len(cameras) - 1 + (len(cameras) - 1) * num_interpolations
time_increment = 10 / total_poses
for i in range(len(cameras) - 1):
cam1 = cameras[i]
cam2 = cameras[i + 1]
# 将旋转矩阵转换为四元数
quat1 = rotation_matrix_to_quaternion(cam1.orientation)
quat2 = rotation_matrix_to_quaternion(cam2.orientation)
for j in range(num_interpolations + 1):
t = j / (num_interpolations + 1)
# 插值方向
interp_orientation_quat = quaternion_slerp(quat1, quat2, t)
interp_orientation_matrix = quaternion_to_rotation_matrix(interp_orientation_quat)
# 插值位置
interp_position = linear_interpolation(cam1.position, cam2.position, t)
# 计算插值时间戳
interp_time = i*10 / (len(cameras) - 1) + time_increment * j
# 添加新的相机位姿和时间戳
newcam = deepcopy(cam1)
newcam.orientation = interp_orientation_matrix
newcam.position = interp_position
smoothed_cameras.append(newcam)
smoothed_times.append(interp_time)
# 添加最后一个原始位姿和时间戳
smoothed_cameras.append(cameras[-1])
smoothed_times.append(1.0)
print(smoothed_times)
return smoothed_cameras, smoothed_times
# # 示例：使用两个相机位姿
# cam1 = Camera(np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]), np.array([0, 0, 0]))
# cam2 = Camera(np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]]), np.array([1, 1, 1]))
# # 应用平滑处理
# smoothed_cameras = smooth_camera_poses([cam1, cam2], num_interpolations=5)
# # 打印结果
# for cam in smoothed_cameras:
# print("Orientation:\n", cam.orientation)
# print("Position:", cam.position)

25
utils/render_utils.py Normal file
View File

@ -0,0 +1,25 @@
import torch
@torch.no_grad()
def get_state_at_time(pc,viewpoint_camera):
means3D = pc.get_xyz
time = torch.tensor(viewpoint_camera.time).to(means3D.device).repeat(means3D.shape[0],1)
opacity = pc._opacity
shs = pc.get_features
# If precomputed 3d covariance is provided, use it. If not, then it will be computed from
# scaling / rotation by the rasterizer.
scales = pc._scaling
rotations = pc._rotation
cov3D_precomp = None
# time0 = get_time()
# means3D_deform, scales_deform, rotations_deform, opacity_deform = pc._deformation(means3D[deformation_point], scales[deformation_point],
# rotations[deformation_point], opacity[deformation_point],
# time[deformation_point])
means3D_final, scales_final, rotations_final, opacity_final, shs_final = pc._deformation(means3D, scales,
rotations, opacity, shs,
time)
scales_final = pc.scaling_activation(scales_final)
rotations_final = pc.rotation_activation(rotations_final)
opacity = pc.opacity_activation(opacity_final)
return means3D_final, scales_final, rotations_final, opacity, shs

14
utils/scene_utils.py
View File

@ -8,10 +8,10 @@ import numpy as np
import copy
@torch.no_grad()
def render_training_image(scene, gaussians, viewpoints, render_func, pipe, background, stage, iteration, time_now):
def render(gaussians, viewpoint, path, scaling):
def render_training_image(scene, gaussians, viewpoints, render_func, pipe, background, stage, iteration, time_now, dataset_type):
def render(gaussians, viewpoint, path, scaling, cam_type):
# scaling_copy = gaussians._scaling
render_pkg = render_func(viewpoint, gaussians, pipe, background, stage=stage)
render_pkg = render_func(viewpoint, gaussians, pipe, background, stage=stage, cam_type=cam_type)
label1 = f"stage:{stage},iter:{iteration}"
times = time_now/60
if times < 1:
@ -21,11 +21,15 @@ def render_training_image(scene, gaussians, viewpoints, render_func, pipe, backg
label2 = "time:%.2f" % times + end
image = render_pkg["render"]
depth = render_pkg["depth"]
if dataset_type == "PanopticSports":
gt_np = viewpoint['image'].permute(1,2,0).cpu().numpy()
else:
gt_np = viewpoint.original_image.permute(1,2,0).cpu().numpy()
image_np = image.permute(1, 2, 0).cpu().numpy() # 转换通道顺序为 (H, W, 3)
depth_np = depth.permute(1, 2, 0).cpu().numpy()
depth_np /= depth_np.max()
depth_np = np.repeat(depth_np, 3, axis=2)
image_np = np.concatenate((image_np, depth_np), axis=1)
image_np = np.concatenate((gt_np, image_np, depth_np), axis=1)
image_with_labels = Image.fromarray((np.clip(image_np,0,1) * 255).astype('uint8')) # 转换为8位图像
# 创建PIL图像对象的副本以绘制标签
draw1 = ImageDraw.Draw(image_with_labels)
@ -59,7 +63,7 @@ def render_training_image(scene, gaussians, viewpoints, render_func, pipe, backg
# point_save_path = os.path.join(point_cloud_path,f"{iteration}.jpg")
for idx in range(len(viewpoints)):
image_save_path = os.path.join(image_path,f"{iteration}_{idx}.jpg")
render(gaussians,viewpoints[idx],image_save_path,scaling = 1)
render(gaussians,viewpoints[idx],image_save_path,scaling = 1,cam_type=dataset_type)
# render(gaussians,point_save_path,scaling = 0.1)
# 保存带有标签的图像

62
vis_point.py Normal file
View File

@ -0,0 +1,62 @@
import imageio
import numpy as np
import torch
from scene import Scene
import os
import cv2
from tqdm import tqdm
from os import makedirs
from gaussian_renderer import render
import torchvision
from utils.general_utils import safe_state
from argparse import ArgumentParser
from arguments import ModelParams, PipelineParams, get_combined_args, ModelHiddenParams
from gaussian_renderer import GaussianModel
from time import time
import open3d as o3d
# import torch.multiprocessing as mp
import threading
from utils.render_utils import get_state_at_time
import concurrent.futures
def render_sets(dataset : ModelParams, hyperparam, iteration : int, pipeline : PipelineParams, skip_train : bool, skip_test : bool, skip_video: bool):
with torch.no_grad():
gaussians = GaussianModel(dataset.sh_degree, hyperparam)
scene = Scene(dataset, gaussians, load_iteration=iteration, shuffle=False)
bg_color = [1,1,1] if dataset.white_background else [0, 0, 0]
background = torch.tensor(bg_color, dtype=torch.float32, device="cuda")
return gaussians, scene
def save_point_cloud(points, model_path, timestamp):
output_path = os.path.join(model_path,"point_pertimestamp")
if not os.path.exists(output_path):
os.makedirs(output_path,exist_ok=True)
points = points.detach().cpu().numpy()
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(points)
ply_path = os.path.join(output_path,f"points_{timestamp}.ply")
o3d.io.write_point_cloud(ply_path, pcd)
parser = ArgumentParser(description="Testing script parameters")
model = ModelParams(parser, sentinel=True)
pipeline = PipelineParams(parser)
hyperparam = ModelHiddenParams(parser)
parser.add_argument("--iteration", default=-1, type=int)
parser.add_argument("--skip_train", action="store_true")
parser.add_argument("--skip_test", action="store_true")
parser.add_argument("--quiet", action="store_true")
parser.add_argument("--skip_video", action="store_true")
parser.add_argument("--configs", type=str)
args = get_combined_args(parser)
print("Rendering " , args.model_path)
if args.configs:
import mmcv
from utils.params_utils import merge_hparams
config = mmcv.Config.fromfile(args.configs)
args = merge_hparams(args, config)
# Initialize system state (RNG)
safe_state(args.quiet)
gaussians, scene = render_sets(model.extract(args), hyperparam.extract(args), args.iteration, pipeline.extract(args), args.skip_train, args.skip_test, args.skip_video)
for index, viewpoint in enumerate(scene.getVideoCameras()):
points, scales_final, rotations_final, opacity_final, shs_final = get_state_at_time(gaussians, viewpoint)
save_point_cloud(points, args.model_path, index)

155
weight_visualization.ipynb Normal file
View File

File diff suppressed because one or more lines are too long