diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..b7626fad975bf4ba49b0540746974ba60773b8c3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +*.zip +.git +datasets/linemod/Linemod_preprocessed +datasets/ycb/YCB_Video_Dataset +*.pth +YCB_Video_toolbox/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..15cbdcf48201095e96c3363f65063a8375d75858 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +datasets/linemod/Linemod_preprocessed +datasets/ycb/YCB_Video_Dataset +*.zip +*__pycache__ +experiments/logs/* +*.pyc +lib/knn/knn_pytorch/__pycache__/* +__init__.* +*.pth +YCB_Video_toolbox diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..4c7ed8bd8380058c9ae8ef95564ed53255a369af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM nvidia/cudagl:9.0-devel-ubuntu16.04 + +ARG DEBIAN_FRONTEND=noninteractive + +# Essentials: developer tools, build tools, OpenBLAS +RUN apt-get update && apt-get install -y --no-install-recommends \ + apt-utils git curl vim unzip openssh-client wget \ + build-essential cmake \ + libopenblas-dev \ + libglib2.0-0 \ + libsm6 \ + libxext6 \ + libxrender-dev + +# Python 3.5 +RUN apt-get update && apt-get install -y --no-install-recommends python3.5 python3.5-dev python3-pip python3-tk && \ + pip3 install --no-cache-dir --upgrade pip setuptools && \ + echo "alias python='python3'" >> /root/.bash_aliases && \ + echo "alias pip='pip3'" >> /root/.bash_aliases + +# Science libraries and other common packages +RUN pip3 --no-cache-dir install \ + numpy scipy pyyaml cffi pyyaml matplotlib Cython requests opencv-python "pillow<7" + +# Tensorflow +RUN pip3 install https://download.pytorch.org/whl/cu90/torch-0.4.1-cp35-cp35m-linux_x86_64.whl && \ + pip3 install torchvision==0.2.2.post3 + +# Expose port for TensorBoard +EXPOSE 6006 + +# cd to home on login +RUN echo "cd /root/dense_fusion" >> /root/.bashrc diff --git a/README.md b/README.md index 79c9e6d476ffc34f6815294d7edc65ad6e54d5b4..e8cdcfff683d267d223203e9d35afb66d13881d1 100644 --- a/README.md +++ b/README.md @@ -128,19 +128,19 @@ To make the best use of the training set, several data augementation techniques (2) A random pose translation noise is added to the training set of the pose estimator, where we set the range of the translation noise to 3cm for both datasets. -(3) For the YCB_Video dataset, since the synthetic data do not contain background. We randomly select the real training data as the background. In each frame, we also randomly select two instances segmentation clips from another synthetic training image to mask at the front of the input RGB-D image, so that more occlusion situations can be generated. +(3) For the YCB_Video dataset, since the synthetic data is not contain background. We randomly select the real training data as the background. In each frame, we also randomly select two instances segmentation clips from another synthetic training image to mask at the front of the input RGB-D image, so that more occlusion situations can be generated. ## Evaluation ### Evaluation on YCB_Video Dataset -For fair comparsion, we use the same segmentation results of [PoseCNN](https://rse-lab.cs.washington.edu/projects/posecnn/) and compare with their results after ICP refinement. +For fair comparison, we use the same segmentation results of [PoseCNN](https://rse-lab.cs.washington.edu/projects/posecnn/) and compare with their results after ICP refinement. Please run: ``` ./experiments/scripts/eval_ycb.sh ``` This script will first download the `YCB_Video_toolbox` to the root folder of this repo and test the selected DenseFusion and Iterative Refinement models on the 2949 keyframes of the 10 testing video in YCB_Video Dataset with the same segmentation result of PoseCNN. The result without refinement is stored in `experiments/eval_result/ycb/Densefusion_wo_refine_result` and the refined result is in `experiments/eval_result/ycb/Densefusion_iterative_result`. -After that, you can add the path of `experiments/eval_result/ycb/Densefusion_wo_refine_result/` and `experiments/eval_result/ycb/Densefusion_iterative_result/` to the code `YCB_Video_toolbox/evaluate_poses_keyframe.m` and run it with [MATLAB](https://www.mathworks.com/products/matlab.html). The code `YCB_Video_toolbox/plot_accuracy_keyframe.m` can show you the comparsion plot result. You can easily make it by copying the adapted codes from the `replace_ycb_toolbox/` folder and replace them in the `YCB_Video_toolbox/` folder. But you might still need to change the path of your `YCB_Video Dataset/` in the `globals.m` and copy two result folders(`Densefusion_wo_refine_result/` and `Densefusion_iterative_result/`) to the `YCB_Video_toolbox/` folder. +After that, you can add the path of `experiments/eval_result/ycb/Densefusion_wo_refine_result/` and `experiments/eval_result/ycb/Densefusion_iterative_result/` to the code `YCB_Video_toolbox/evaluate_poses_keyframe.m` and run it with [MATLAB](https://www.mathworks.com/products/matlab.html). The code `YCB_Video_toolbox/plot_accuracy_keyframe.m` can show you the comparsion plot result. You can easily make it by copying the adapted codes from the `replace_ycb_toolbox/` folder and replace them in the `YCB_Video_toolbox/` folder. But you might still need to change the path of your `YCB_Video Dataset/` in the `globals.m` and copy two result folders(`Densefusion_wo_refine_result/` and `Densefusion_iterative_result/`) to the `YCB_Video_toolbox/` folder. ### Evaluation on LineMOD Dataset @@ -181,7 +181,7 @@ The qualitative result on the YCB_Video dataset. You can download the trained DenseFusion and Iterative Refinement checkpoints of both datasets from [Link](https://drive.google.com/drive/folders/19ivHpaKm9dOrr12fzC8IDFczWRPFxho7). ## Tips for your own dataset -As you can see in this repo, the network code and the hyperparameters(lr and w) remain the same for both datasets. Which means you might not need to adjust too much on the network structure and hyperparameters when you use this repo on your own dataset. Please make sure that the distance metric in your dataset should be converted to meter, otherwise the hyperparameter w need to be adjusted. Several useful tools including [LabelFusion](https://github.com/RobotLocomotion/LabelFusion) and [sixd_toolkit](https://github.com/thodan/sixd_toolkit) has been tested to work well. (Please make sure to turn on the depth image collection in LabelFusion when you use it.) +As you can see in this repo, the network code and the hyperparameters (lr and w) remain the same for both datasets. Which means you might not need to adjust too much on the network structure and hyperparameters when you use this repo on your own dataset. Please make sure that the distance metric in your dataset should be converted to meter, otherwise the hyperparameter w need to be adjusted. Several useful tools including [LabelFusion](https://github.com/RobotLocomotion/LabelFusion) and [sixd_toolkit](https://github.com/thodan/sixd_toolkit) has been tested to work well. (Please make sure to turn on the depth image collection in LabelFusion when you use it.) ## Citations diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..e7d85044f7562a1a6381eb017fe0a886a3978117 --- /dev/null +++ b/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +#docker build --no-cache -t hri -f Dockerfile . +docker build -t dense_fusion -f Dockerfile . diff --git a/datasets/linemod/dataset.py b/datasets/linemod/dataset.py index b887a81c60ef019a3eed778ebbaf5120f117f845..0a3a852ca5417c4790f68dd50383ca2c861dab97 100755 --- a/datasets/linemod/dataset.py +++ b/datasets/linemod/dataset.py @@ -215,7 +215,9 @@ img_length = 640 def mask_to_bbox(mask): mask = mask.astype(np.uint8) - _, contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + x = 0 y = 0 w = 0 diff --git a/download.sh b/download.sh old mode 100644 new mode 100755 diff --git a/experiments/scripts/eval_linemod.sh b/experiments/scripts/eval_linemod.sh old mode 100644 new mode 100755 diff --git a/experiments/scripts/eval_ycb.sh b/experiments/scripts/eval_ycb.sh old mode 100644 new mode 100755 diff --git a/experiments/scripts/train_linemod.sh b/experiments/scripts/train_linemod.sh old mode 100644 new mode 100755 diff --git a/experiments/scripts/train_ycb.sh b/experiments/scripts/train_ycb.sh old mode 100644 new mode 100755 diff --git a/run.sh b/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..b7081c85d1feb8133420aba375dc79be3e1af7ed --- /dev/null +++ b/run.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# Runs a docker container with the image created by build.sh +until nvidia-docker ps +do + echo "Waiting for docker server" + sleep 1 +done + +XSOCK=/tmp/.X11-unix + +XAUTH=/root/.Xauthority + +SRC_CONTAINER=/root/dense_fusion +SRC_HOST="$(pwd)" + +xhost local:root + +nvidia-docker run \ + --name dense_fusion \ + -it --rm \ + --volume=$XSOCK:$XSOCK:rw \ + --volume=$XAUTH:$XAUTH:rw \ + --volume=$SRC_HOST:$SRC_CONTAINER:rw \ + --env="XAUTHORITY=${XAUTH}" \ + --env="DISPLAY=${DISPLAY}" \ + --privileged -v /dev/bus/usb:/dev/bus/usb \ + --net=host \ + dense_fusion