Build workflow (#2)

benjaminum · web-flow · commit 581a4a1d68f6 · 2021-12-12T09:41:24.000+01:00
Adds a workflow for building python wheels and the app image
diff --git a/.github/actions/docker-action/action.yml b/.github/actions/docker-action/action.yml
@@ -0,0 +1,6 @@
+# action.yml
+name: 'Build and run docker container'
+description: 'Builds and runs the docker container for the Dockerfile in the root'
+runs:
+  using: 'docker'
+  image: '../../../Dockerfile'
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -0,0 +1,75 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Build artifacts
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the main branch
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-18.04
+
+    strategy:
+      matrix:
+        configuration: [py36, py37, py38, py39, appimage]
+        include:
+          - configuration: py36
+            python_version: 3.6
+            build_python: ON
+            build_appimage: OFF
+          - configuration: py37
+            python_version: 3.7
+            build_python: ON
+            build_appimage: OFF
+          - configuration: py38
+            python_version: 3.8
+            build_python: ON
+            build_appimage: OFF
+          - configuration: py39
+            python_version: 3.9
+            build_python: ON
+            build_appimage: OFF
+          - configuration: appimage
+            python_version: 3.9
+            build_python: OFF
+            build_appimage: ON
+
+    env:
+      PYTHON_VERSION: ${{ matrix.python_version }}
+      BUILD_PYTHON_MODULE: ${{ matrix.build_python }}
+      BUILD_APPIMAGE: ${{ matrix.build_appimage }}
+      MAKE_FLAGS: -j2
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - name: Download model files
+        run: |
+          wget -nv https://github.com/isl-org/adaptive-surface-reconstruction/releases/download/models/models.zip
+          unzip models.zip
+          rm models.zip
+
+      - name: Build inside container
+        uses: ./.github/actions/docker-action
+      
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: artifact_${{ matrix.configuration }}
+          path: |
+            build/a*.whl
+            build/asr*.AppImage
+
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
 build
 __pycache__
+datasets/t10k
+dummy_resource_file
 *.ply
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -3,7 +3,7 @@ cmake_minimum_required( VERSION 3.18 )
 project(adaptivesurfacereconstruction VERSION 0.1.0)
 
 option(BUILD_PYTHON_MODULE  "Build the python module" ON)
-option(ENABLE_APPIMAGE  "Enables the target 'appimage' for building the AppImage" OFF)
+option(BUILD_APPIMAGE  "Build the AppImage" OFF)
 
 if( NOT CMAKE_BUILD_TYPE )
         set( CMAKE_BUILD_TYPE "Release" CACHE STRING "Build configuration 'Release' or 'Debug'." FORCE )
@@ -23,7 +23,7 @@ if( BUILD_PYTHON_MODULE )
         add_subdirectory( cpp/pybind pybind ) 
         add_subdirectory( python ) 
 endif()
-if( ENABLE_APPIMAGE )
+if( BUILD_APPIMAGE )
         add_subdirectory( appimage ) 
 endif()
 
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,28 @@
+FROM ubuntu:18.04
+
+# Miniconda requires bash as the default shell.
+SHELL ["/bin/bash", "-c"]
+
+
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  file \
+  git \
+  libglu1-mesa-dev \
+  libtbb-dev \
+  patchelf \
+  wget \
+  xorg-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+# Miniconda
+ENV PATH="/root/miniconda3/bin:${PATH}"
+RUN wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+ && bash Miniconda3-latest-Linux-x86_64.sh -b \
+ && rm Miniconda3-latest-Linux-x86_64.sh \
+ && conda --version
+
+
+COPY docker_entrypoint.sh /docker_entrypoint.sh
+
+ENTRYPOINT ["/docker_entrypoint.sh"]
diff --git a/README.md b/README.md
@@ -25,13 +25,13 @@ If you find this repository useful please cite our [paper](https://openaccess.th
 ## Dependencies
 
 ## Packages for building the library
-- Pytorch 1.8.1 (can be installed with `python -m pip install torch==1.8.1+cpu -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html`)
+- Pytorch 1.8.2 (can be installed with `python -m pip install torch==1.8.2+cpu -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html`)
 - On Ubuntu the following packages are required: patchelf, xorg-dev, libglu1-mesa-dev, python3-dev
   These can be installed with `apt install patchelf xorg-dev libglu1-mesa-dev python3-dev`
 
 ## Packages required for training the network
 - Tensorflow 2.6.0
-- Open3D 0.14 or later with ML module (https://github.com/intel-isl/Open3D/)
+- Open3D 0.14 or later with ML module (https://github.com/isl-org/Open3D/)
 - Tensorpack DataFlow (for reading data, ```pip install --upgrade git+https://github.com/tensorpack/dataflow.git```)
 - python-prctl (needed by Tensorpack DataFlow; depends on libcap-dev, install with ```apt install libcap-dev``` )
 - msgpack (```pip install msgpack``` )
diff --git a/appimage/CMakeLists.txt b/appimage/CMakeLists.txt
@@ -1,20 +1,21 @@
 
 set( appdir "${CMAKE_CURRENT_BINARY_DIR}/AppDir" )
-file( GLOB_RECURSE asr_model_files ${PROJECT_SOURCE_DIR}/models/v*/model*.pt )
-file( GLOB_RECURSE dataset_attribution_files ${PROJECT_SOURCE_DIR}/datasets/*/*_attribution.txt )
-list( APPEND asr_model_files dummy_resource_file )
+file( GLOB_RECURSE asr_resources_files 
+        ${PROJECT_SOURCE_DIR}/models/v*/model*.pt 
+        ${PROJECT_SOURCE_DIR}/datasets/*/*_attribution.txt
+        )
+list( APPEND asr_resources_files dummy_resource_file )
 
 find_program(linuxdeploy_binary linuxdeploy-x86_64.AppImage )
 if( NOT linuxdeploy_binary )
 message( FATAL_ERROR "Building the appimage requires 'linuxdeploy-x86_64.AppImage' from https://github.com/linuxdeploy/linuxdeploy/releases/tag/continuous")
 endif()
 
-add_custom_target( appimage
-        COMMAND ${CMAKE_COMMAND} -E rm -r "${appdir}"
+add_custom_target( appimage ALL
+        COMMAND ${CMAKE_COMMAND} -E rm -rf "${appdir}"
         COMMAND ${CMAKE_COMMAND} -E touch dummy_resource_file
         COMMAND ${CMAKE_COMMAND} -E make_directory "${appdir}/usr/lib/asr_resources"
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${asr_model_files} "${appdir}/usr/lib/asr_resources"
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${dataset_attribution_files} "${appdir}/usr/lib/asr_resources"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${asr_resources_files} "${appdir}/usr/lib/asr_resources"
         COMMAND LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}/../_deps/libtorch-src/lib VERSION=${PROJECT_VERSION} ${linuxdeploy_binary} --create-desktop-file --appdir "${appdir}" -e $<TARGET_FILE:asrtool> -i ${CMAKE_CURRENT_SOURCE_DIR}/asrtool.png -o appimage
         DEPENDS asrtool
 )
diff --git a/build_with_docker.sh b/build_with_docker.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+function printhelp {
+    usage="
+Usage: $(basename $0) [OPTIONS]
+
+Example: 
+    ./$(basename $0) --wheel 3.9 --appimage
+
+Options:
+    --wheel <python version>  Build a python wheel with the specified version
+    --appimage                Build the AppImage
+    --help, -h                Print this help  
+"
+    echo "$usage"
+    exit 0
+}
+
+PYTHON_VERSION=3.9
+BUILD_PYTHON_MODULE=OFF
+BUILD_APPIMAGE=OFF
+
+if [ $# -eq 0 ]; then
+    printhelp
+fi
+
+parse_iter=0
+while [ $parse_iter -lt 100 ] ; do
+    parse_iter=$((parse_iter+1))
+    case "$1" in
+        --wheel) PYTHON_VERSION="$2"; BUILD_PYTHON_MODULE=ON ; shift 2 ;;
+        --appimage) BUILD_APPIMAGE=ON ; shift ;;
+        --help | -h) printhelp ; shift ;;
+        *) break ;;
+    esac
+done
+set -euo pipefail
+
+REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+docker build -t adaptive-surface-reconstruction .
+
+docker run \
+    --rm \
+    --workdir /workspace \
+    -e PYTHON_VERSION="$PYTHON_VERSION" \
+    -e BUILD_PYTHON_MODULE="$BUILD_PYTHON_MODULE" \
+    -e BUILD_APPIMAGE="$BUILD_APPIMAGE" \
+    -e MAKE_FLAGS="-j" \
+    -v "$REPO_ROOT":/workspace \
+    adaptive-surface-reconstruction
diff --git a/cmake/external_deps.cmake b/cmake/external_deps.cmake
@@ -3,8 +3,12 @@ include(ExternalProject)
 
 find_package( Python COMPONENTS Interpreter )
 
-# show downloads during configure step
-set( FETCHCONTENT_QUIET OFF CACHE BOOL "" FORCE )
+# show downloads during configure step but not on github
+if(DEFINED ENV{GITHUB_WORKSPACE})
+        set( FETCHCONTENT_QUIET ON CACHE BOOL "" FORCE )
+else()
+        set( FETCHCONTENT_QUIET OFF CACHE BOOL "" FORCE )
+endif()
 
 find_program( patchelf_binary patchelf REQUIRED )
 
diff --git a/datasets/create_t10k_msgpacks.py b/datasets/create_t10k_msgpacks.py
@@ -30,7 +30,8 @@ def read_compressed_msgpack(path, decompressor=None):
 
 def select_good_meshes(info_dict, data_dir):
     # select only good meshes
-    raw_meshes_dir = os.path.join(data_dir,'raw_meshes')
+    if data_dir:
+        raw_meshes_dir = os.path.join(data_dir,'raw_meshes')
     selected_meshes = []
     attribution = []
     selection = {
@@ -63,7 +64,8 @@ def select_good_meshes(info_dict, data_dir):
                 break;
         if selected and info['License'] in licenses:
             attribution.append('"{}"({}) by {} is licensed under {}'.format(info['title'].strip(), info['Thing ID'], info['author'], info['License']))
-            selected_meshes.append(glob(os.path.join(raw_meshes_dir,key+'.*'))[0])
+            if data_dir:
+                selected_meshes.append(glob(os.path.join(raw_meshes_dir,key+'.*'))[0])
 
     return selected_meshes, attribution
 
@@ -105,11 +107,14 @@ def create_data(mesh_paths, output_path):
 def main():
     parser = argparse.ArgumentParser(description="Create data files for training",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument("--data_dir", type=str, required=True, help="The path to the Thingi10k dataset root.")
+    parser.add_argument("--data_dir", type=str, default=None, help="The path to the Thingi10k dataset root.")
     parser.add_argument("--output_dir", type=str, default=os.path.join(os.path.dirname(__file__), 't10k'), help="The path to the output dir")
     parser.add_argument("--attribution_file_only", action="store_true", help="Create only the attribution file")
 
     args = parser.parse_args()
+    if not args.attribution_file_only and not args.data_dir:
+        print("Please specify the path to the Thingi10K root with '--data-dir'")
+        return
 
     info_dict = read_compressed_msgpack(os.path.join(os.path.dirname(__file__),'thingi10k_info.msgpack.zst'))
 
diff --git a/docker_entrypoint.sh b/docker_entrypoint.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "================================================================================"
+echo " PYTHON_VERSION         $PYTHON_VERSION"
+echo " BUILD_PYTHON_MODULE    $BUILD_PYTHON_MODULE"
+echo " BUILD_APPIMAGE         $BUILD_APPIMAGE"
+echo " MAKE_FLAGS             $MAKE_FLAGS"
+echo "================================================================================"
+
+conda create -y -n asr python=$PYTHON_VERSION
+source activate asr
+conda install -y cmake
+
+python -m pip install torch==1.8.2+cpu -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+python -m pip install open3d==0.14.1 zstandard msgpack msgpack-numpy
+
+pushd datasets
+python create_t10k_msgpacks.py --attribution_file_only
+popd
+if [ -d build ]; then
+    rm -rf build
+fi
+mkdir build
+pushd build
+
+if [ "$BUILD_APPIMAGE" = "ON" ]; then
+    mkdir linuxdeploy
+    pushd linuxdeploy
+    wget https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage
+    chmod u+x linuxdeploy-x86_64.AppImage
+
+    # FUSE does not work inside docker -> extract app image
+    ./linuxdeploy-x86_64.AppImage --appimage-extract
+    rm linuxdeploy-x86_64.AppImage 
+    popd
+fi
+linuxdeploy="$(pwd)/linuxdeploy/squashfs-root/AppRun"
+
+cmake \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DBUILD_PYTHON_MODULE=$BUILD_PYTHON_MODULE \
+    -DBUILD_APPIMAGE=$BUILD_APPIMAGE \
+    -Dlinuxdeploy_binary="$linuxdeploy" \
+    ..
+
+df -h
+make $MAKE_FLAGS
+
+mv python/python_package/dist/a*.whl appimage/asr*.AppImage . || true
+popd
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
@@ -1,19 +1,20 @@
 
 set( python_package_out_dir "${CMAKE_CURRENT_BINARY_DIR}/python_package" )
-file( GLOB_RECURSE asr_model_files ${PROJECT_SOURCE_DIR}/models/v*/model*.pt )
-file( GLOB_RECURSE dataset_attribution_files ${PROJECT_SOURCE_DIR}/datasets/*/*_attribution.txt )
-list( APPEND asr_model_files dummy_resource_file )
+file( GLOB_RECURSE asr_resources_files 
+        ${PROJECT_SOURCE_DIR}/models/v*/model*.pt 
+        ${PROJECT_SOURCE_DIR}/datasets/*/*_attribution.txt
+        )
+list( APPEND asr_resources_files dummy_resource_file )
 
 
 # delete old wheels to make sure that there is only one
-add_custom_target( python_wheel 
+add_custom_target( python_wheel ALL
         COMMAND ${CMAKE_COMMAND} -E rm -rf "${python_package_out_dir}"
         COMMAND ${CMAKE_COMMAND} -E make_directory "${python_package_out_dir}"
         COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}/adaptivesurfacereconstruction" "${python_package_out_dir}/adaptivesurfacereconstruction"
         COMMAND ${CMAKE_COMMAND} -E make_directory "${python_package_out_dir}/adaptivesurfacereconstruction/asr_resources"
         COMMAND ${CMAKE_COMMAND} -E touch dummy_resource_file
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${asr_model_files} "${python_package_out_dir}/adaptivesurfacereconstruction/asr_resources"
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${dataset_attribution_files} "${python_package_out_dir}/adaptivesurfacereconstruction/asr_resources"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${asr_resources_files} "${python_package_out_dir}/adaptivesurfacereconstruction/asr_resources"
         COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" "${python_package_out_dir}"
         COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_CURRENT_SOURCE_DIR}/MANIFEST.in" "${python_package_out_dir}"
         COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:asrpybind> "${python_package_out_dir}/adaptivesurfacereconstruction/"
