Merge pull request #362 from astro-informatics/cg_hdf5_interface

HDF5 interface and support for stochastic reading

Author: mmcleod89

.github/workflows/ci.yml

@@ -124,6 +124,15 @@ jobs:
         # Fix bug in FFT3 (cf. https://github.com/FFTW/fftw3/issues/332)
         sed -i -e 's/^.*FFTW3LibraryDepends.cmake.*$//1' ${{github.workspace}}/local/lib*/cmake/*/FFTW3Config.cmake
 
+    - name: Install HDF5
+      run: |
+        wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.3/src/hdf5-1.14.3.tar.gz -O- | tar --no-same-owner -xz;
+        cd hdf5-1.14.3/
+        mkdir build && cd build
+        #CC=`which mpicc` CXX=`which mpicxx` ./configure --prefix=${{github.workspace}}/local --enable-parallel --enable-shared --enable-cxx
+        CC=`which mpicc` CXX=`which mpicxx` cmake .. -DBUILD_TESTING=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -DHDF5_ENABLE_PARALLEL:BOOL=ON -DALLOW_UNSUPPORTED:BOOL=ON -DHDF5_BUILD_FORTRAN:BOOL=OFF -DHDF5_BUILD_CPP_LIB:BOOL=ON -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF
+        make -j2 install
+
     - name: Checkout SOPT
       uses: actions/checkout@v4
       with:
@@ -214,9 +223,10 @@ jobs:
       # Build your program with the given configuration
       run: |
         export CMAKE_PREFIX_PATH=${{github.workspace}}/local:$CMAKE_PREFIX_PATH
+        #export PATH=${{github.workspace}}/local:$PATH
         mkdir -p ${{github.workspace}}/build
         cd ${{github.workspace}}/build
-        cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Donnxrt=ON -Ddocasa=OFF -Ddompi=${{matrix.mpi}} -Dopenmp=${{matrix.omp}} -Dtests=ON
+        cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Donnxrt=ON -Dhdf5=ON -Ddocasa=OFF -Ddompi=${{matrix.mpi}} -Dopenmp=${{matrix.omp}} -Dtests=ON
         make -j$(nproc --ignore 1) install
 
     - name: Test
@@ -278,7 +288,7 @@ jobs:
         export CMAKE_PREFIX_PATH=${{github.workspace}}/local:$CMAKE_PREFIX_PATH
         mkdir -p ${{github.workspace}}/build
         cd ${{github.workspace}}/build
-        cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Donnxrt=ON -Ddompi=OFF -Dopenmp=OFF -Ddocs=ON
+        cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Donnxrt=ON -Dhdf5=ON -Ddompi=OFF -Dopenmp=OFF -Ddocs=ON
         make -j$(nproc --ignore 1) install
 
     - name: Deploy to GH pages

CMakeLists.txt

@@ -16,6 +16,7 @@ option(docasa         "Enable CASA"                                     off)
 option(docs           "Build documentation"                             off)
 option(coverage       "Build coverage"                                  off)
 option(onnxrt         "Build with ONNXruntime interface"                off)
+option(hdf5           "Build with HDF5 interface"                       off)
 
 if(NOT CMAKE_BUILD_TYPE)
   message(STATUS "Setting build type to 'Release' as none was specified.")

cmake_files/LookUpHighFive.cmake

@@ -0,0 +1,40 @@
+# Installs BlueBrain/HighFive into build directory
+#
+# - GIT_REPOSITORY: defaults to origin BlueBrain/HighFive repo on github
+# - GIT_TAG: defaults to master
+include(ExternalProject)
+if(HighFive_ARGUMENTS)
+  cmake_parse_arguments(HighFive "" "GIT_REPOSITORY;GIT_TAG" ""
+    ${HighFive_ARGUMENTS})
+endif()
+if(NOT HighFive_GIT_REPOSITORY)
+  set(HighFive_GIT_REPOSITORY https://github.com/BlueBrain/HighFive)
+endif()
+if(NOT HighFive_GIT_TAG)
+  set(HighFive_GIT_TAG master)
+endif()
+
+set(HighFive_DIR "${CMAKE_INSTALL_PREFIX}/external")
+ExternalProject_Add(
+  Lookup-HighFive
+  GIT_REPOSITORY ${HighFive_GIT_REPOSITORY}
+  GIT_TAG ${HighFive_GIT_TAG}
+  PREFIX "${CMAKE_BINARY_DIR}/external"
+  #INSTALL_DIR ${EXTERNAL_ROOT}
+  CMAKE_ARGS
+  -DCMAKE_INSTALL_PREFIX=${HighFive_DIR}
+  -DCMAKE_INSTALL_LIBDIR=${CMAKE_SHARED_LIBRARY_PREFIX}
+  -DHIGHFIVE_EXAMPLES=OFF
+  -DHIGHFIVE_USE_BOOST=OFF
+  -DHIGHFIVE_UNIT_TESTS=OFF
+  # Wrap download, configure and build steps in a script to log output
+  UPDATE_COMMAND ""
+  LOG_DOWNLOAD ON
+  LOG_CONFIGURE ON
+  LOG_BUILD ON
+  LOG_INSTALL ON
+)
+set(HighFive_INCLUDE_DIR "${HighFive_DIR}/include")
+#set(HighFive_LIBRARY_DIR "${HighFive_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}")
+#set(HighFive_LIBRARIES "HighFive")
+

cmake_files/dependencies.cmake

@@ -94,7 +94,11 @@ find_package(fftw3 NAMES FFTW3 REQUIRED)
 set(PURIFY_MPI FALSE)
 if(dompi)
   find_package(MPI REQUIRED)
-  set(PURIFY_MPI TRUE)
+  if (${sopt_HAS_MPI})
+    set(PURIFY_MPI TRUE)
+  else()
+    message(FATAL_ERROR "SOPT built without MPI support")
+  endif()
 endif()
 find_package(TIFF REQUIRED)
 
@@ -122,5 +126,23 @@ if(docasa)
   set(PURIFY_CASACORE TRUE)
 endif()
 
+set(PURIFY_H5 FALSE)
+if(hdf5)
+  find_package(HDF5 COMPONENTS CXX REQUIRED)
+  message(STATUS "Found HDF5 include dir: ${HDF5_INCLUDE_DIR}")
+  message(STATUS "Found HDF5 CXX library: ${HDF5_CXX_LIBRARIES}")
+  message(STATUS "HDF5 parallel: ${HDF5_IS_PARALLEL}")
+  if (PURIFY_MPI AND NOT ${HDF5_IS_PARALLEL})
+    message(FATAL_ERROR "HDF5 built without MPI support")
+  endif()
+  find_package(HighFive QUIET)
+  if(NOT HighFive_FOUND)
+    message(STATUS "HighFive not found. Attempt to install...")
+    include(LookUpHighFive)
+    set(PURIFY_HIGHFIVE_LOOKUP TRUE)
+  endif()
+  set(PURIFY_H5 TRUE)
+endif()
+
 # Add script to execute to make sure libraries in the build tree can be found
 add_to_ld_path("${EXTERNAL_ROOT}/lib")

cpp/CMakeLists.txt

@@ -23,6 +23,8 @@ add_include_dir(
   ${yaml-cpp_INCLUDE_DIR}
   ${Cubature_INCLUDE_DIR}
   ${CImg_INCLUDE_DIR}
+  ${HDF5_INCLUDE_DIR}
+  ${HighFive_INCLUDE_DIR}
 )
 
 

cpp/purify/CMakeLists.txt

@@ -16,8 +16,8 @@ endfunction()
 configure_file(config.in.h "${PROJECT_BINARY_DIR}/include/purify/config.h")
 
 set(HEADERS
-  logging.h kernels.h pfitsio.h  types.h
-  IndexMapping.h utilities.h operators.h  wproj_utilities.h
+  logging.h kernels.h pfitsio.h types.h
+  IndexMapping.h utilities.h operators.h wproj_utilities.h
   cimg.h uvfits.h convolution.h measurement_operator_factory.h wavelet_operator_factory.h distribute.h
   update_factory.h
   convergence_factory.h
@@ -42,6 +42,10 @@ if(PURIFY_CASACORE)
   list(APPEND HEADERS casacore.h)
 endif()
 
+if(PURIFY_H5)
+  list(APPEND HEADERS h5reader.h)
+endif()
+
 if(PURIFY_MPI)
   list(APPEND HEADERS mpi_utilities.h distribute.h DistributeSparseVector.h
   random_update_factory.h
@@ -70,6 +74,11 @@ if(PURIFY_ARRAYFIRE)
   target_link_libraries(libpurify ${ArrayFire_LIBRARIES})
 endif()
 
+if(PURIFY_H5)
+  target_link_libraries(libpurify ${HDF5_CXX_LIBRARIES})
+  target_include_directories(libpurify SYSTEM PUBLIC ${HDF5_INCLUDE_DIR} ${HighFive_INCLUDE_DIR})
+endif()
+
 target_include_directories(libpurify PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>
   $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/include>
@@ -111,6 +120,10 @@ if(PURIFY_CASACORE_LOOKUP)
   add_dependencies(libpurify Lookup-CasaCore)
 endif()
 
+if(PURIFY_HIGHFIVE_LOOKUP)
+  add_dependencies(libpurify Lookup-HighFive)
+endif()
+
 install(FILES ${HEADERS} DESTINATION include/purify)
 install(TARGETS libpurify
   EXPORT PurifyTargets

cpp/purify/config.in.h

@@ -28,6 +28,9 @@
 //! Whether PURIFY is using (and SOPT was built with) onnxrt support
 #cmakedefine PURIFY_ONNXRT
 
+//! Whether PURIFY is using HDF5 support
+#cmakedefine PURIFY_H5
+
 #include <cstdint>
 #include <string>
 #include <tuple>

cpp/purify/distribute.cc

@@ -1,16 +1,15 @@
 #include "purify/distribute.h"
 #include "purify/wide_field_utilities.h"
 
-namespace purify {
-namespace distribute {
+namespace purify::distribute {
 
 std::vector<t_int> distribute_measurements(Vector<t_real> const &u, Vector<t_real> const &v,
                                            Vector<t_real> const &w, t_int const number_of_nodes,
                                            distribute::plan const distribution_plan,
                                            t_int const &grid_size) {
   // distrubte visibilities from a measurement
   Vector<t_int> index = Vector<t_int>::LinSpaced(u.size(), 0, u.size());
-  t_int const patition_size =
+  t_int const partition_size =
       std::ceil(static_cast<t_real>(u.size()) / static_cast<t_real>(number_of_nodes));
   // return a vector of vectors of indicies for each node
   std::string plan_name = "";
@@ -41,23 +40,25 @@ std::vector<t_int> distribute_measurements(Vector<t_real> const &u, Vector<t_rea
   }
   PURIFY_DEBUG(
       "Using {} to make {} partitions from {} visibilities, with {} visibilities per a node.",
-      plan_name, number_of_nodes, index.size(), patition_size);
-  std::vector<t_int> patitions(u.size());
-  // creating patitions
-  for (t_int i = 0; i < index.size(); i++) {
-    if (std::floor(static_cast<t_real>(i) / static_cast<t_real>(patition_size)) >
-        number_of_nodes - 1) {
-      PURIFY_ERROR("Error: Probably a bug in distribution plan.");
-      throw std::runtime_error("Distributing data into too many nodes");
-    }
-    patitions[index(i)] = std::floor(static_cast<t_real>(i) / static_cast<t_real>(patition_size));
+      plan_name, number_of_nodes, index.size(), partition_size);
+  std::vector<t_int> partitions(u.size());
+  if (std::floor(static_cast<t_real>(index.size() - 1) / static_cast<t_real>(partition_size)) >
+      number_of_nodes - 1) {
+    PURIFY_ERROR("Error: Probably a bug in distribution plan.");
+    throw std::runtime_error("Distributing data into too many nodes");
+  }
+  // creating partitions
+  for (t_int i = 0; i < index.size(); ++i) {
+    partitions[index(i)] = std::floor(static_cast<t_real>(i) / static_cast<t_real>(partition_size));
   }
-  return patitions;
+  return partitions;
 }
+
 Vector<t_int> w_distribution(const Vector<t_real> &u, const Vector<t_real> &v,
                              Vector<t_real> const &w) {
   return w_distribution(w);
 }
+
 Vector<t_int> w_distribution(Vector<t_real> const &w) {
   // sort visibilities by w from  w_max to w_min
   Vector<t_int> index = Vector<t_int>::LinSpaced(w.size(), 0, w.size());
@@ -98,6 +99,7 @@ Vector<t_int> equal_distribution(Vector<t_real> const &u, Vector<t_real> const &
             });
   return index;
 }
+
 std::tuple<std::vector<t_int>, std::vector<t_real>> kmeans_algo(
     const Vector<t_real> &w, const t_int number_of_nodes, const t_int iters,
     const std::function<t_real(t_real)> &cost, const t_real rel_diff) {
@@ -147,6 +149,7 @@ std::tuple<std::vector<t_int>, std::vector<t_real>> kmeans_algo(
 
   return std::make_tuple(w_node, w_centre);
 }
+
 #ifdef PURIFY_MPI
 std::tuple<std::vector<t_int>, std::vector<t_real>> kmeans_algo(
     const Vector<t_real> &w, const t_int number_of_nodes, const t_int iters,
@@ -164,7 +167,7 @@ std::tuple<std::vector<t_int>, std::vector<t_real>> kmeans_algo(
   for (int i = 0; i < w_centre.size(); i++)
     w_centre[i] =
         (static_cast<t_real>(i) * (wmax - wmin) / static_cast<t_real>(number_of_nodes) + wmin);
-  // lopp through even nodes to reduces w-term
+  // loop through even nodes to reduce w-term
   for (int n = 0; n < iters; n++) {
     if (comm.is_root()) PURIFY_DEBUG("clustering iteration {}", n);
     for (int i = 0; i < w.size(); i++) {
@@ -216,8 +219,9 @@ std::vector<t_int> w_support(Vector<t_real> const &w, const std::vector<t_int> &
                                         min_support, max_support);
   const t_real coeff_average =
       comm.all_sum_all<t_real>(coeff_total) / static_cast<t_real>(comm.size());
-  if (comm.is_root())
+  if (comm.is_root()) {
     PURIFY_DEBUG("Each node should have on average {} coefficients.", coeff_average);
+  }
   t_real coeff_sum = 0;
   t_int group = 0;
   std::vector<t_int> groups(w.size(), comm.rank());
@@ -242,22 +246,23 @@ std::vector<t_int> w_support(Vector<t_real> const &w, const std::vector<t_int> &
         groups[i] = group;
       }
     }
-    if (group > comm.size() - 1)
+    if (group > comm.size() - 1) {
       throw std::runtime_error(
           "Error distributing visibilites to even computational load for wide field imaging. Group "
           "number out of bounds.");
+    }
     coeff_sum = comm.broadcast(coeff_sum, rank);
     group = comm.broadcast(group, rank);
 
-    if (total != coeff_total and comm.rank() == rank)
+    if (total != coeff_total and comm.rank() == rank) {
       throw std::runtime_error(
           "Total number of coefficients calculated is not the same, loop might be broken. " +
           std::to_string(total) + " != " + std::to_string(coeff_total));
+    }
   }
   if (comm.is_root()) PURIFY_DEBUG("{} node should have {} coefficients.", group, coeff_sum);
   return groups;
 }
 #endif
 
-}  // namespace distribute
-}  // namespace purify
+}  // namespace purify::distribute