Init

smartroboticslab · Mar 20, 2023 · 567a16e · 567a16e
commit 567a16e
Show file tree

Hide file tree

Showing 803 changed files with 231,999 additions and 0 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -0,0 +1,14 @@
+{
+    "name": "vimid Dev Container",
+    "dockerComposeFile": "docker-compose.yml",
+    "service": "vimid",
+    "workspaceFolder": "/volume",
+    "shutdownAction": "stopCompose",
+    "postStartCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
+    "extensions": [
+		"ms-vscode.cpptools",
+		"ms-vscode.cmake-tools",
+		"mutantdino.resourcemonitor",
+		"ms-python.python"
+	]
+}
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -0,0 +1,22 @@
+version: "3.8"
+
+services:
+  vimid:
+    image: vimid:latest
+    # user: "${UID}:${GID}"
+    build:
+      context: ..
+      dockerfile: ./Dockerfile
+    stdin_open: true
+    tty: true
+    network_mode: host
+    environment:
+      NVIDIA_DRIVER_CAPABILITIES: compute,utility,graphics
+      DISPLAY: $DISPLAY
+    volumes:
+      - ../:/volume
+      - /tmp/.X11-unix:/tmp/.X11-unix
+      - /home/bxu/data/datasets:/data
+    working_dir: /volume
+    command: /bin/bash
+
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,18 @@
+apps/kfusion/build/
+apps/kfusion/Release/
+apps/kfusion/cmake-build-debug/
+apps/kfusion/TooN/
+apps/kfusion/scripts/results/
+apps/kfusion/results/*
+apps/kfusion/scripts/*.pyc
+apps/kfusion/test/tracking/CMakeFiles/
+apps/kfusion/test/preprocessing/CMakeFiles/
+apps/kfusion/test/tracking/Makefile
+./apps/kfusion/indoor/*
+.vscode/settings.json
+.idea/
+apps/kfusion/vimid_test/cmake-build-debug/
+apps/kfusion/vimid_test/build/
+apps/kfusion/scripts/drift_analysis/
+apps/kfusion/Vocabulary/
+build/
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "third-party/okvis_public"]
+	path = third-party/okvis_public
+	url = git@github.com:binbin-xu/okvis_vimid.git
+[submodule "third-party/librealsense"]
+	path = third-party/librealsense
+	url = git@github.com:binbin-xu/realsense1_legacy.git
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,103 @@
+# SPDX-FileCopyrightText: 2023 Binbin Xu
+# SPDX-License-Identifier: CC0-1.0
+
+FROM ubuntu:18.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# system
+RUN echo "Installing apt packages..." \
+	&& export DEBIAN_FRONTEND=noninteractive \
+	&& apt -y update --no-install-recommends \
+	&& apt -y install --no-install-recommends \
+	git \
+	wget \
+    libeigen3-dev \
+    freeglut3-dev \
+	sudo \
+	&& apt autoremove -y \
+	&& apt clean -y \
+	&& export DEBIAN_FRONTEND=dialog
+
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
+
+# install OKVIS dependencies
+RUN echo "Installing apt packages..." \
+    && export DEBIAN_FRONTEND=noninteractive \
+    && apt -y update --no-install-recommends \
+    && apt -y install --no-install-recommends \
+    cmake \
+    libgoogle-glog-dev \
+    libatlas-base-dev \
+    libeigen3-dev \
+    libsuitesparse-dev \
+    libboost-all-dev \
+    libopencv-dev \
+    libceres-dev \
+    ca-certificates \
+    && apt autoremove -y \
+    && apt clean -y \
+    && export DEBIAN_FRONTEND=dialog
+
+# install librealsense
+RUN echo "Installing librealsense..." \
+    && export DEBIAN_FRONTEND=noninteractive \
+    && apt -y update --no-install-recommends \
+    && apt -y install --no-install-recommends \
+    libusb-1.0-0-dev \
+    libglfw3-dev \
+    pkg-config \
+    libgtk-3-dev \
+    gcc-6 g++-6 \
+    build-essential \
+    gnupg2 lsb-release \
+    && apt autoremove -y \
+    && apt clean -y \
+    && export DEBIAN_FRONTEND=dialog
+
+# install legacy realsense 1
+COPY ./third-party/librealsense /opt/librealsense
+RUN cd /opt/librealsense \
+    && mkdir build \
+    && cd build \
+    && cmake -DCMAKE_BUILD_TYPE=Release .. \
+    && make -j8 \
+    && make install
+
+
+# install geographiclib
+RUN echo "Installing geographiclib..." 
+RUN git clone git://git.code.sourceforge.net/p/geographiclib/code /opt/geographiclib \
+    && cd /opt/geographiclib \
+    && mkdir build \
+    && cd build \
+    && cmake -DCMAKE_BUILD_TYPE=Release .. \
+    && make -j8 \
+    && make install
+
+
+# OKVIS public
+COPY ./third-party/okvis_public /opt/okvis
+RUN cd /opt/okvis \
+    && mkdir build \
+    && cd build \
+    && cmake -D CMAKE_C_COMPILER=gcc-6 -D CMAKE_CXX_COMPILER=g++-6 -DCMAKE_BUILD_TYPE=Release .. \
+    && make -j8 \
+    && make install
+
+# install GUI stuff
+RUN echo "Installing GUI stuff..." \
+    && export DEBIAN_FRONTEND=noninteractive \
+    && apt -y update --no-install-recommends \
+    && apt -y install --no-install-recommends \
+    language-pack-en-base \
+    gdb \
+    libcanberra-gtk-module libcanberra-gtk3-module \
+    && apt autoremove -y \
+    && apt clean -y \
+    && export DEBIAN_FRONTEND=dialog
+RUN dpkg-reconfigure locales && \
+    locale-gen en_US.UTF-8 && \
+    /usr/sbin/update-locale LANG=en_US.UTF-8
+
+ENV DISPLAY :1
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,30 @@
+BSD 3-Clause License
+
+Copyright © 2017-2023 Smart Robotics Lab, Imperial College London
+Copyright © 2021-2023 Yifei Ren
+Copyright © 2017-2023 Binbin Xu
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/README.md b/README.md
@@ -0,0 +1,109 @@
+# VI-MID
+
+## [Project Page](https://mlr.in.tum.de/research/projects/vimid) | [Video](https://youtu.be/6GY5cBwvuJE) | [Paper](https://arxiv.org/pdf/2208.04274.pdf)
+
+
+This repository contains VI-MID: a new multi-instance dynamic RGBD-Inertial SLAM system using an object-level octree-based volumetric representation.
+
+![VI-MID](teaser/VI-MID.gif)
+
+
+We present a tightly-coupled visual-inertial object-level multi-instance dynamic SLAM system. Even in extremely dynamic scenes, it can robustly optimise for the camera pose, velocity, IMU biases and build a dense 3D reconstruction object-level map of the environment. Our system can robustly track and reconstruct the geometries of arbitrary objects, their semantics and motion by incrementally fusing associated colour, depth, semantic, and foreground object probabilities into each object model thanks to its robust sensor and object tracking. In addition, when an object is lost or moved outside the camera field of view, our system can reliably recover its pose upon re-observation. We demonstrate the robustness and accuracy of our method by quantitatively and qualitatively testing it in real-world data sequences.
+
+
+## TODO:
+
+- [x] Create custom dataset
+- [ ] Evaluation examples
+- [x] Documentation
+- [x] License
+
+## How to use our software
+### Dependencies
+Check `Dockerfile` and `.devcontainer/docker-compose.yml` for the required dependencies. We provided a [vscode devcontainer](https://code.visualstudio.com/docs/devcontainers/create-dev-container) for easy development.
+`xhost +` is required to run the container with GUI support.
+If you are using remote computer for developmnt without a monitor and meet an issue to get GUI: try a hack `export DISPLAY=$LOCAL_DISPLAY$`
+
+### Installation:
+Get our code and submodules:
+```
+git clone git@github.com:binbin-xu/vimid.git
+git submodule update --init --recursive
+```
+Install the custom OKVIS and realsense library in the `third-party` folder. 
+
+Then go into the ***apps/kfusion*** folder, simply run the following command to build the software and the dependencies:
+
+```
+make
+```
+If any error occurs, please check the dockerfile for any possible dependencies.
+
+### Uninstall:
+
+```
+make clean
+```
+
+<!-- ### Dependency
+If you meet any dependency issue in compiling, please refer to [github action file](https://github.com/binbin-xu/mid-fusion/blob/master/.github/workflows/main.yml) or [report an issue](https://github.com/binbin-xu/mid-fusion/issues).
+ -->
+
+### Demo:
+
+We provide some usage samples. Please see the bash files in the ***apps/kfusion/demo***  folder.
+
+The data used to run those bash can be downloaded via [this link](https://drive.google.com/drive/folders/1Hn0M1assEKimjLWulQBTiWJARa3fdrUn?usp=sharing). Remember to modify the datasets address in the bash files accordingly.
+
+### Customised settings:
+
+RGB-D sequences need to be given as the demo format. Specifically, RGB images in ***/cam0*** folder, depth images in ***/depth0*** folder, mask-RCNN outputs in ***/mask_RCNN*** folder, IMU data in ***/imu0*** folder (***/cam1***, ***/cam0_ori*** can be ignored). If there's no alignment between RGB and depth image, you may need to associate them (check [this link](https://vision.in.tum.de/data/datasets/rgbd-dataset/tools) for details). \
+The input images are defined as in the TUM RGBD datasets, where the input images are in the resolution of 640 X 480 and depth is scaled by 5000. Images are named in the recorded timestamples (nanoseconds).
+
+You may need to tune some hyper-paramters defined in the [file](https://github.com/binbin-xu/vimid_alpha/blob/main/apps/kfusion/include/default_parameters.h) and parse them as arguments for your own sequences. \
+Make sure to complie and run in debug mode first to expose bugs that were hidden in unoptimized code.
+```
+make debug
+```
+
+Then you can run our modified Mask RCNN script (check [demo/vimid-mask.py](https://github.com/binbin-xu/detectron2_for_vimid/blob/master/demo/vimid-mask.py) in [this repo](https://github.com/binbin-xu/detectron2_for_vimid)) to generate masks, classes, and semantic probability in cnpy format. Here we provide a [detectron2](https://github.com/binbin-xu/detectron2_for_vimid) version for usage. We did not finetune the pretrained coco models, and the results would be much improved with a better/more suited segmentation mask. Therefore if you want to increase performance in your specific domain, please **consider training a network on your data**. 
+
+Those tunable parameters can be found in `apps/kfusion/include/default_parameters.h`.
+
+
+<!-- ### Difference with supereight implementation:
+
+  This is an official implementation of MID-Fusion system. The system is implemented based on [supereight](https://github.com/emanuelev/supereight), an octree-based volumetric representation. However, MID-Fusion was developed in parallel with supereight system. Therefore you may notice some structure and contents differences between the two systems. We are trying to merge the new updates from supereight into our MID-Fusion implementation. -->
+
+
+
+## Citations
+
+Please consider citing this project in your publications if it helps your work. The following is a BibTeX reference. The BibTeX entry requires the `url` LaTeX package.
+
+```
+@inproceedings{Ren:Xu:etal:IROS2022,
+  title={Visual-Inertial Multi-Instance Dynamic SLAM with Object-level Relocalisation},
+  author={Ren, Yifei and Xu, Binbin and Choi, Christopher L and Leutenegger, Stefan},
+  booktitle={2022 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},
+  pages={11055--11062},
+  year={2022},
+}
+
+@inproceedings{Xu:etal:ICRA2019,
+author = {Binbin Xu and Wenbin Li and Dimos Tzoumanikas and Michael Bloesch and Andrew Davison and Stefan Leutenegger},
+booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
++ title = {{MID-Fusion}: Octree-based Object-Level Multi-Instance Dynamic SLAM},
+year = {2019},
+}
+```
+
+
+## License
+Copyright © 2017-2023 Smart Robotics Lab, Imperial College London \
+Copyright © 2021-2023 Yifei Ren \
+Copyright © 2017-2023 Binbin Xu
+
+Distributed under the [BSD 3-clause license](LICENSE).
+
+