diff --git a/4DGaussians.ipynb b/4DGaussians.ipynb new file mode 100644 index 0000000..a6bbc44 --- /dev/null +++ b/4DGaussians.ipynb @@ -0,0 +1,585 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "VjYy0F2gZIPR", + "outputId": "129c4176-455a-42b0-e06a-b30ef7792aea", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + } + }, + "outputs": [ + { + "metadata": { + "tags": null + }, + "name": "stdout", + "output_type": "stream", + "text": [ + "/content\n", + "fatal: destination path '4DGaussians' already exists and is not an empty directory.\n", + "/content/4DGaussians\n", + "Submodule 'submodules/depth-diff-gaussian-rasterization' (https://github.com/ingra14m/depth-diff-gaussian-rasterization) registered for path 'submodules/depth-diff-gaussian-rasterization'\n", + "Submodule 'submodules/simple-knn' (https://gitlab.inria.fr/bkerbl/simple-knn.git) registered for path 'submodules/simple-knn'\n", + "Cloning into '/content/4DGaussians/submodules/depth-diff-gaussian-rasterization'...\n", + "Cloning into '/content/4DGaussians/submodules/simple-knn'...\n", + "Submodule path 'submodules/depth-diff-gaussian-rasterization': checked out 'f2d8fa9921ea9a6cb9ac1c33a34ebd1b11510657'\n", + "Submodule 'third_party/glm' (https://github.com/g-truc/glm.git) registered for path 'submodules/depth-diff-gaussian-rasterization/third_party/glm'\n", + "Cloning into '/content/4DGaussians/submodules/depth-diff-gaussian-rasterization/third_party/glm'...\n", + "Submodule path 'submodules/depth-diff-gaussian-rasterization/third_party/glm': checked out '5c46b9c07008ae65cb81ab79cd677ecc1934b903'\n", + "Submodule path 'submodules/simple-knn': checked out '44f764299fa305faf6ec5ebd99939e0508331503'\n", + "/bin/bash: line 1: conda: command not found\n", + "/bin/bash: line 1: conda: command not found\n", + "Collecting torch==1.13.1 (from -r requirements.txt (line 1))\n", + " Downloading torch-1.13.1-cp310-cp310-manylinux1_x86_64.whl (887.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m887.5/887.5 MB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting torchvision==0.14.1 (from -r requirements.txt (line 2))\n", + " Downloading torchvision-0.14.1-cp310-cp310-manylinux1_x86_64.whl (24.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.2/24.2 MB\u001b[0m \u001b[31m71.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting torchaudio==0.13.1 (from -r requirements.txt (line 3))\n", + " Downloading torchaudio-0.13.1-cp310-cp310-manylinux1_x86_64.whl (4.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.2/4.2 MB\u001b[0m \u001b[31m105.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting mmcv==1.6.0 (from -r requirements.txt (line 4))\n", + " Downloading mmcv-1.6.0.tar.gz (554 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m554.9/554.9 kB\u001b[0m \u001b[31m50.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (3.7.1)\n", + "Collecting argparse (from -r requirements.txt (line 6))\n", + " Downloading argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Collecting lpips (from -r requirements.txt (line 7))\n", + " Downloading lpips-0.1.4-py3-none-any.whl (53 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.8/53.8 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting plyfile (from -r requirements.txt (line 8))\n", + " Downloading plyfile-1.0.1-py3-none-any.whl (23 kB)\n", + "Requirement already satisfied: imageio-ffmpeg in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 9)) (0.4.9)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch==1.13.1->-r requirements.txt (line 1)) (4.5.0)\n", + "Collecting nvidia-cuda-runtime-cu11==11.7.99 (from torch==1.13.1->-r requirements.txt (line 1))\n", + " Downloading nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m849.3/849.3 kB\u001b[0m \u001b[31m56.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cudnn-cu11==8.5.0.96 (from torch==1.13.1->-r requirements.txt (line 1))\n", + " Downloading nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m557.1/557.1 MB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cublas-cu11==11.10.3.66 (from torch==1.13.1->-r requirements.txt (line 1))\n", + " Downloading nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.1/317.1 MB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cuda-nvrtc-cu11==11.7.99 (from torch==1.13.1->-r requirements.txt (line 1))\n", + " Downloading nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m75.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision==0.14.1->-r requirements.txt (line 2)) (1.23.5)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torchvision==0.14.1->-r requirements.txt (line 2)) (2.31.0)\n", + "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision==0.14.1->-r requirements.txt (line 2)) (9.4.0)\n", + "Collecting addict (from mmcv==1.6.0->-r requirements.txt (line 4))\n", + " Downloading addict-2.4.0-py3-none-any.whl (3.8 kB)\n", + "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from mmcv==1.6.0->-r requirements.txt (line 4)) (23.2)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from mmcv==1.6.0->-r requirements.txt (line 4)) (6.0.1)\n", + "Collecting yapf (from mmcv==1.6.0->-r requirements.txt (line 4))\n", + " Downloading yapf-0.40.2-py3-none-any.whl (254 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m254.7/254.7 kB\u001b[0m \u001b[31m30.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch==1.13.1->-r requirements.txt (line 1)) (67.7.2)\n", + "Requirement already satisfied: wheel in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch==1.13.1->-r requirements.txt (line 1)) (0.41.2)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 5)) (1.1.1)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 5)) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 5)) (4.43.1)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 5)) (1.4.5)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 5)) (3.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 5)) (2.8.2)\n", + "Requirement already satisfied: scipy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from lpips->-r requirements.txt (line 7)) (1.11.3)\n", + "Requirement already satisfied: tqdm>=4.28.1 in /usr/local/lib/python3.10/dist-packages (from lpips->-r requirements.txt (line 7)) (4.66.1)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->-r requirements.txt (line 5)) (1.16.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision==0.14.1->-r requirements.txt (line 2)) (3.3.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision==0.14.1->-r requirements.txt (line 2)) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision==0.14.1->-r requirements.txt (line 2)) (2.0.6)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision==0.14.1->-r requirements.txt (line 2)) (2023.7.22)\n", + "Requirement already satisfied: importlib-metadata>=6.6.0 in /usr/local/lib/python3.10/dist-packages (from yapf->mmcv==1.6.0->-r requirements.txt (line 4)) (6.8.0)\n", + "Requirement already satisfied: platformdirs>=3.5.1 in /usr/local/lib/python3.10/dist-packages (from yapf->mmcv==1.6.0->-r requirements.txt (line 4)) (3.11.0)\n", + "Requirement already satisfied: tomli>=2.0.1 in /usr/local/lib/python3.10/dist-packages (from yapf->mmcv==1.6.0->-r requirements.txt (line 4)) (2.0.1)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata>=6.6.0->yapf->mmcv==1.6.0->-r requirements.txt (line 4)) (3.17.0)\n", + "Building wheels for collected packages: mmcv\n", + " Building wheel for mmcv (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for mmcv: filename=mmcv-1.6.0-py2.py3-none-any.whl size=847986 sha256=4aa22bff8e71e96eb0c412dea29087e675ea7ff37a484c5be7ca685662aa958c\n", + " Stored in directory: /root/.cache/pip/wheels/02/e2/7c/97f72e34ee40d71cdd28b94c9fdfec7bcc453651ad6e65c96d\n", + "Successfully built mmcv\n", + "Installing collected packages: argparse, addict, plyfile, nvidia-cuda-runtime-cu11, nvidia-cuda-nvrtc-cu11, nvidia-cublas-cu11, yapf, nvidia-cudnn-cu11, torch, mmcv, torchvision, torchaudio, lpips\n", + " Attempting uninstall: torch\n", + " Found existing installation: torch 2.0.1+cu118\n", + " Uninstalling torch-2.0.1+cu118:\n", + " Successfully uninstalled torch-2.0.1+cu118\n", + " Attempting uninstall: torchvision\n", + " Found existing installation: torchvision 0.15.2+cu118\n", + " Uninstalling torchvision-0.15.2+cu118:\n", + " Successfully uninstalled torchvision-0.15.2+cu118\n", + " Attempting uninstall: torchaudio\n", + " Found existing installation: torchaudio 2.0.2+cu118\n", + " Uninstalling torchaudio-2.0.2+cu118:\n", + " Successfully uninstalled torchaudio-2.0.2+cu118\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "torchdata 0.6.1 requires torch==2.0.1, but you have torch 1.13.1 which is incompatible.\n", + "torchtext 0.15.2 requires torch==2.0.1, but you have torch 1.13.1 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0mSuccessfully installed addict-2.4.0 argparse-1.4.0 lpips-0.1.4 mmcv-1.6.0 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 plyfile-1.0.1 torch-1.13.1 torchaudio-0.13.1 torchvision-0.14.1 yapf-0.40.2\n" + ] + }, + { + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "argparse" + ] + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Obtaining file:///content/4DGaussians/submodules/depth-diff-gaussian-rasterization\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Installing collected packages: diff-gaussian-rasterization\n", + " Running setup.py develop for diff-gaussian-rasterization\n", + "Successfully installed diff-gaussian-rasterization-0.0.0\n", + "Obtaining file:///content/4DGaussians/submodules/simple-knn\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Installing collected packages: simple-knn\n", + " Running setup.py develop for simple-knn\n", + "Successfully installed simple-knn-0.0.0\n" + ] + } + ], + "source": [ + "%cd /content\n", + "!git clone https://github.com/hustvl/4DGaussians\n", + "%cd 4DGaussians\n", + "!git submodule update --init --recursive\n", + "\n", + "!pip install -r requirements.txt\n", + "!pip install -e submodules/depth-diff-gaussian-rasterization\n", + "!pip install -e submodules/simple-knn\n" + ] + }, + { + "cell_type": "code", + "source": [ + "!mkdir /content/test\n", + "%cd /content/test\n", + "!wget https://huggingface.co/camenduru/4DGaussians/resolve/main/data/data.zip\n", + "!unzip data.zip" + ], + "metadata": { + "id": "wpqmK97Koq36" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [], + "metadata": { + "id": "IBB-voY0oqpy" + } + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "8hk10A8Al7_V", + "outputId": "e6df1ec7-fe5c-4b08-bee5-bcf077759c2a", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content/4DGaussians\n", + "Optimizing \n", + "Output folder: ./output/dnerf/bouncingballs [18/10 12:15:58]\n", + "2023-10-18 12:15:59.094660: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2023-10-18 12:16:00.723098: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "feature_dim: 128 [18/10 12:16:03]\n", + "Found transforms_train.json file, assuming Blender data set! [18/10 12:16:03]\n", + "Reading Training Transforms [18/10 12:16:03]\n", + "Reading Test Transforms [18/10 12:16:16]\n", + "Generating Video Transforms [18/10 12:16:18]\n", + "Generating random point cloud (2000)... [18/10 12:16:19]\n", + "Loading Training Cameras [18/10 12:16:19]\n", + "Loading Test Cameras [18/10 12:16:19]\n", + "Loading Video Cameras [18/10 12:16:19]\n", + "Voxel Plane: set aabb= Parameter containing:\n", + "tensor([[ 1.2998, 1.2999, 1.2999],\n", + " [-1.2998, -1.2998, -1.2987]], requires_grad=True) [18/10 12:16:20]\n", + "Number of points at initialisation : 2000 [18/10 12:16:24]\n", + "Training progress: 0% 0/3000 [00:00\n", + " \n", + " \n", + " \"\"\" % data_url)" + ], + "metadata": { + "id": "oLyN3bAw0KCI" + }, + "execution_count": 3, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "save_dir = '/content/4DGaussians/output/dnerf/bouncingballs/video/ours_20000/video_rgb.mp4'\n", + "\n", + "import os\n", + "import glob\n", + "# video_path = glob.glob(os.path.join(save_dir, \"*-test.mp4\"))[0]\n", + "display_video(save_dir)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "nCTKtptS0MrA", + "outputId": "ad2bdbac-ddeb-4d27-900c-24c96009bc44" + }, + "execution_count": 6, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " " + ] + }, + "metadata": {}, + "execution_count": 6 + } + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/README.md b/README.md index b03acd7..8a392ec 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ ![block](assets/teaserfig.png) Our method converges very quickly and achieves real-time rendering speed. -Colab demo:[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/4DGaussians-colab/blob/main/4DGaussians_colab.ipynb) (Thanks [camenduru](https://github.com/camenduru/4DGaussians-colab).) +Colab demo:[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hustvl/4DGaussians/4DGaussians.ipynb) (Thanks [camenduru](https://github.com/camenduru/4DGaussians-colab).)