From 43bea69f0e9b8cea8e7a7d7352297da3965e98a2 Mon Sep 17 00:00:00 2001 From: Christian Gutschow Date: Mon, 17 Feb 2025 12:58:08 +0000 Subject: [PATCH 1/6] add a simple H5 writer --- cpp/purify/h5reader.h | 46 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/cpp/purify/h5reader.h b/cpp/purify/h5reader.h index 965a636b9..9089f8931 100644 --- a/cpp/purify/h5reader.h +++ b/cpp/purify/h5reader.h @@ -160,7 +160,7 @@ class H5Handler { size_t _datalen, _slicepos, _slicelen, _batchpos; }; -/// @brief Reads an HDF5 file with u,v visibilities, constructs a vis_params objects and returns it. +/// @brief Reads an HDF5 file with u,v visibilities, constructs a vis_params object and returns it. /// /// @note vis_name: name of input HDF5 file containing [u, v, real(V), imag(V)]. utilities::vis_params read_visibility(const std::string& vis_name, const bool w_term) { @@ -240,6 +240,50 @@ utilities::vis_params stochread_visibility(H5Handler& file, const size_t N, cons return uv_vis; } + +/// @brief Write an HDF5 file with u,v visibilities from a vis_params object. +void write_visibility(const utilities::vis_params& uv_vis, const std::string& h5name, + const bool w_term, cons int chunksize = -1) { + // Set up HDF5 file + HighFive::File h5file(h5name, HighFive::File::OpenOrCreate | HighFive::File::Truncate); + // Set up file properies, such as chunking and compression + // Note: the I/O is minimised if compression is disabeld (obvs) + // If using decompressed data is not an option, then the + // I/O performance can be optiomised by chunking the data + // in such a way that each MPI rank only has to decompressi + // its allocated segment (or a subset thereof) + HighFive::DataSetCreateProps props; + if (uv_vis.u.size()) { + if (chunkize > 0) { + props.add(HighFive::Chunking(std::vector{(size_t)chunksize})); + } + else { + props.add(HighFive::Chunking(std::vector{uv_vis.u.size()})); + } + props.add(HighFive::Deflate(9)); // maximal compression + } + // Create the H5 datasets + h5file.createDataSet("u", std::vector(uv_vis.u.data(), uv_vis.u.data() + uv_vis.u.size()), props); + h5file.createDataSet("v", std::vector(uv_vis.v.data(), uv_vis.v.data() + uv_vis.v.size()), props); + if (w_term) { + h5file.createDataSet("w", std::vector(uv_vis.w.data(), uv_vis.w.data() + uv_vis.w.size()), props); + } + + vector redata, imdata, sigma; + redata.reserve(uv_vis.vis.size()); + imdata.reserve(uv_vis.vis.size()); + sigma.reserve(uv_vis.weights.size()); + for (size_t i = 0; i < uv_vis.vis.size(); ++i) { + redata.push_back(uv_vis.vis(i).real()); + imdata.push_back(uv_vis.vis(i).imag()); + sigma.push_back(1 / uv_vis.weights(i)); + } + h5file.createDataSet("re", std::move(redata), props); + h5file.createDataSet("im", std::move(imdata), props); + h5file.createDataSet("sigma", std::move(imdata), props); +} + + } // namespace purify::H5 #endif From 7efe905e22dbf61aa11965c71f83f086d7b75f2a Mon Sep 17 00:00:00 2001 From: Christian Gutschow Date: Mon, 17 Feb 2025 13:05:29 +0000 Subject: [PATCH 2/6] stupid linting --- cpp/purify/h5reader.h | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cpp/purify/h5reader.h b/cpp/purify/h5reader.h index 9089f8931..5ad40a8cb 100644 --- a/cpp/purify/h5reader.h +++ b/cpp/purify/h5reader.h @@ -240,33 +240,34 @@ utilities::vis_params stochread_visibility(H5Handler& file, const size_t N, cons return uv_vis; } - /// @brief Write an HDF5 file with u,v visibilities from a vis_params object. void write_visibility(const utilities::vis_params& uv_vis, const std::string& h5name, - const bool w_term, cons int chunksize = -1) { + const bool w_term, const size_t chunksize = 0) { // Set up HDF5 file HighFive::File h5file(h5name, HighFive::File::OpenOrCreate | HighFive::File::Truncate); - // Set up file properies, such as chunking and compression - // Note: the I/O is minimised if compression is disabeld (obvs) + // Set up file properties, such as chunking and compression + // Note: the I/O is minimised if compression is disabled (obvs) // If using decompressed data is not an option, then the - // I/O performance can be optiomised by chunking the data - // in such a way that each MPI rank only has to decompressi + // I/O performance can be optimised by chunking the dataset + // in such a way that each MPI rank only has to decompress // its allocated segment (or a subset thereof) HighFive::DataSetCreateProps props; if (uv_vis.u.size()) { if (chunkize > 0) { - props.add(HighFive::Chunking(std::vector{(size_t)chunksize})); - } - else { + props.add(HighFive::Chunking(std::vector{chunksize})); + } else { props.add(HighFive::Chunking(std::vector{uv_vis.u.size()})); } - props.add(HighFive::Deflate(9)); // maximal compression + props.add(HighFive::Deflate(9)); // maximal compression } // Create the H5 datasets - h5file.createDataSet("u", std::vector(uv_vis.u.data(), uv_vis.u.data() + uv_vis.u.size()), props); - h5file.createDataSet("v", std::vector(uv_vis.v.data(), uv_vis.v.data() + uv_vis.v.size()), props); + h5file.createDataSet("u", std::vector(uv_vis.u.data(), uv_vis.u.data() + uv_vis.u.size()), + props); + h5file.createDataSet("v", std::vector(uv_vis.v.data(), uv_vis.v.data() + uv_vis.v.size()), + props); if (w_term) { - h5file.createDataSet("w", std::vector(uv_vis.w.data(), uv_vis.w.data() + uv_vis.w.size()), props); + h5file.createDataSet( + "w", std::vector(uv_vis.w.data(), uv_vis.w.data() + uv_vis.w.size()), props); } vector redata, imdata, sigma; @@ -283,7 +284,6 @@ void write_visibility(const utilities::vis_params& uv_vis, const std::string& h5 h5file.createDataSet("sigma", std::move(imdata), props); } - } // namespace purify::H5 #endif From 523a4b69b1da427e7553f4f1c16805dfebd910d6 Mon Sep 17 00:00:00 2001 From: Christian Gutschow Date: Mon, 17 Feb 2025 13:17:15 +0000 Subject: [PATCH 3/6] fix typos and namespaces --- cpp/purify/h5reader.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/purify/h5reader.h b/cpp/purify/h5reader.h index 5ad40a8cb..be56357f3 100644 --- a/cpp/purify/h5reader.h +++ b/cpp/purify/h5reader.h @@ -253,10 +253,10 @@ void write_visibility(const utilities::vis_params& uv_vis, const std::string& h5 // its allocated segment (or a subset thereof) HighFive::DataSetCreateProps props; if (uv_vis.u.size()) { - if (chunkize > 0) { + if (chunksize > 0) { props.add(HighFive::Chunking(std::vector{chunksize})); } else { - props.add(HighFive::Chunking(std::vector{uv_vis.u.size()})); + props.add(HighFive::Chunking(std::vector{static_cast(uv_vis.u.size())})); } props.add(HighFive::Deflate(9)); // maximal compression } @@ -270,14 +270,14 @@ void write_visibility(const utilities::vis_params& uv_vis, const std::string& h5 "w", std::vector(uv_vis.w.data(), uv_vis.w.data() + uv_vis.w.size()), props); } - vector redata, imdata, sigma; + std::vector redata, imdata, sigma; redata.reserve(uv_vis.vis.size()); imdata.reserve(uv_vis.vis.size()); sigma.reserve(uv_vis.weights.size()); for (size_t i = 0; i < uv_vis.vis.size(); ++i) { redata.push_back(uv_vis.vis(i).real()); imdata.push_back(uv_vis.vis(i).imag()); - sigma.push_back(1 / uv_vis.weights(i)); + sigma.push_back(1.0 / uv_vis.weights(i).real()); } h5file.createDataSet("re", std::move(redata), props); h5file.createDataSet("im", std::move(imdata), props); From 8648d62a1fc08833901a3fa53ea02c121a2da607 Mon Sep 17 00:00:00 2001 From: Christian Gutschow Date: Mon, 17 Feb 2025 14:05:52 +0000 Subject: [PATCH 4/6] add unit test --- cpp/tests/purify_h5.cc | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/cpp/tests/purify_h5.cc b/cpp/tests/purify_h5.cc index 50bd42e01..41d50025f 100644 --- a/cpp/tests/purify_h5.cc +++ b/cpp/tests/purify_h5.cc @@ -3,6 +3,7 @@ #include "purify/config.h" #include "purify/types.h" #include "purify/logging.h" +#include "purify/read_measurements.h" #include "purify/directories.h" #include "purify/h5reader.h" @@ -11,7 +12,7 @@ using namespace purify; -TEST_CASE("Purify H5", "[HDF5]") { +TEST_CASE("Purify H5 reader", "[HDF5]") { H5::H5Handler f(atca_filename("0332-391.h5")); const std::vector u = f.read("u"); @@ -32,3 +33,31 @@ TEST_CASE("Purify H5", "[HDF5]") { u.size() == re.size() && u.size() == im.size() && u.size() == sigma.size(); CHECK(pass); } + +TEST_CASE("Purify H5 writer", "[HDF5]") { + + const auto uvfits = read_measurements::read_measurements(atca_filename("0332-391.uvfits")); + + H5::write_visibility(uvfits, "test-h5.h5", false); + + H5::H5Handler f("test-h5.h5"); + + const std::vector u = f.read("u"); + const std::vector v = f.read("v"); + //const std::vector w = f.read("w"); + const std::vector re = f.read("re"); + const std::vector im = f.read("im"); + const std::vector sigma = f.read("sigma"); + + CAPTURE(u.size()); + CAPTURE(v.size()); + //CAPTURE(w.size()); + CAPTURE(re.size()); + CAPTURE(im.size()); + CAPTURE(sigma.size()); + + const bool pass = u.size() == uvfits.u.size() && u.size() == v.size() && /*u.size() == w.size() &&*/ + u.size() == re.size() && u.size() == im.size() && u.size() == sigma.size(); + + CHECK(pass); +} From 158e4eb55a59d5b2e0108cc983b371897c9e416d Mon Sep 17 00:00:00 2001 From: Christian Gutschow Date: Mon, 17 Feb 2025 14:07:23 +0000 Subject: [PATCH 5/6] more stoopid linting --- cpp/tests/purify_h5.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/tests/purify_h5.cc b/cpp/tests/purify_h5.cc index 41d50025f..750a64698 100644 --- a/cpp/tests/purify_h5.cc +++ b/cpp/tests/purify_h5.cc @@ -44,19 +44,20 @@ TEST_CASE("Purify H5 writer", "[HDF5]") { const std::vector u = f.read("u"); const std::vector v = f.read("v"); - //const std::vector w = f.read("w"); + // const std::vector w = f.read("w"); const std::vector re = f.read("re"); const std::vector im = f.read("im"); const std::vector sigma = f.read("sigma"); CAPTURE(u.size()); CAPTURE(v.size()); - //CAPTURE(w.size()); + // CAPTURE(w.size()); CAPTURE(re.size()); CAPTURE(im.size()); CAPTURE(sigma.size()); - const bool pass = u.size() == uvfits.u.size() && u.size() == v.size() && /*u.size() == w.size() &&*/ + const bool pass = u.size() == uvfits.u.size() && + u.size() == v.size() && /*u.size() == w.size() &&*/ u.size() == re.size() && u.size() == im.size() && u.size() == sigma.size(); CHECK(pass); From 9b79c81724a2f4acc175d2942aa70f65f8c5141c Mon Sep 17 00:00:00 2001 From: Christian Gutschow Date: Mon, 17 Feb 2025 14:10:27 +0000 Subject: [PATCH 6/6] absolute joke this --- cpp/tests/purify_h5.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/tests/purify_h5.cc b/cpp/tests/purify_h5.cc index 750a64698..fd0bed453 100644 --- a/cpp/tests/purify_h5.cc +++ b/cpp/tests/purify_h5.cc @@ -35,7 +35,6 @@ TEST_CASE("Purify H5 reader", "[HDF5]") { } TEST_CASE("Purify H5 writer", "[HDF5]") { - const auto uvfits = read_measurements::read_measurements(atca_filename("0332-391.uvfits")); H5::write_visibility(uvfits, "test-h5.h5", false);