Skip to content

Commit e846563

Browse files
tkoskelatkoskela
committed
Merge branch 'development' into mm_primal_dual_test
2 parents fbde1c2 + 7779be2 commit e846563

File tree

90 files changed

+3056
-1152
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

90 files changed

+3056
-1152
lines changed

‎.github/workflows/ci.yml

Lines changed: 117 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ env:
1919
OMP_NUM_THREADS: 2
2020

2121
jobs:
22-
build:
22+
dependencies:
2323
# Skip CI if PR is a draft
2424
if: github.event.pull_request.draft == false
25-
name: build (${{matrix.os}}-${{matrix.cxx}}-mpi:${{matrix.mpi}}-openmp:${{matrix.omp}})
25+
name: dependencies (${{matrix.os}}-${{matrix.cxx}}-mpi:${{matrix.mpi}}-openmp:${{matrix.omp}})
2626
# The CMake configure and build commands are platform agnostic and should work equally
2727
# well on Windows or Mac. You can convert this to a matrix build if you need
2828
# cross-platform coverage.
@@ -63,6 +63,11 @@ jobs:
6363
steps:
6464
- uses: actions/checkout@v4
6565

66+
- name: Setup cmake
67+
uses: jwlawson/actions-setup-cmake@v2
68+
with:
69+
cmake-version: '3.31.6'
70+
6671
# Enable tmate debugging of manually-triggered workflows if the input option was provided
6772
- name: Setup tmate session
6873
uses: mxschmitt/action-tmate@v3
@@ -97,7 +102,7 @@ jobs:
97102
- name: Install Dependencies on MacOS
98103
if: ${{ contains(matrix.os, 'macos') }}
99104
run: |
100-
brew install open-mpi libomp yaml-cpp eigen cfitsio ccache
105+
brew install open-mpi libomp yaml-cpp eigen cfitsio ccache onnxruntime
101106
echo "CMAKE_PREFIX_PATH=/opt/homebrew/opt/libomp" >> $GITHUB_ENV
102107
echo "/opt/homebrew/opt/ccache/libexec" >> $GITHUB_PATH
103108
@@ -124,6 +129,15 @@ jobs:
124129
# Fix bug in FFT3 (cf. https://github.com/FFTW/fftw3/issues/332)
125130
sed -i -e 's/^.*FFTW3LibraryDepends.cmake.*$//1' ${{github.workspace}}/local/lib*/cmake/*/FFTW3Config.cmake
126131
132+
- name: Install HDF5
133+
run: |
134+
wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.3/src/hdf5-1.14.3.tar.gz -O- | tar --no-same-owner -xz;
135+
cd hdf5-1.14.3/
136+
mkdir build && cd build
137+
#CC=`which mpicc` CXX=`which mpicxx` ./configure --prefix=${{github.workspace}}/local --enable-parallel --enable-shared --enable-cxx
138+
CC=`which mpicc` CXX=`which mpicxx` cmake .. -DBUILD_TESTING=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -DHDF5_ENABLE_PARALLEL:BOOL=ON -DALLOW_UNSUPPORTED:BOOL=ON -DHDF5_BUILD_FORTRAN:BOOL=OFF -DHDF5_BUILD_CPP_LIB:BOOL=ON -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF
139+
make -j2 install
140+
127141
- name: Checkout SOPT
128142
uses: actions/checkout@v4
129143
with:
@@ -136,14 +150,14 @@ jobs:
136150
export CMAKE_PREFIX_PATH=${{github.workspace}}/local:$CMAKE_PREFIX_PATH
137151
mkdir -p ${{github.workspace}}/sopt/build
138152
cd ${{github.workspace}}/sopt/build
139-
cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Ddompi=${{matrix.mpi}} -Dopenmp=${{matrix.omp}} -Dtests=OFF -Dexamples=OFF
153+
cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Donnxrt=ON -Ddompi=${{matrix.mpi}} -Dopenmp=${{matrix.omp}} -Dtests=OFF -Dexamples=OFF
140154
make -j$(nproc --ignore 1) install
141155
142156
- name: Pack dependencies
143157
run: |
144158
cd ${{github.workspace}}
145159
tar cfv dependencies.tar local
146-
160+
147161
- uses: actions/upload-artifact@v4
148162
with:
149163
name: dependencies-${{ matrix.os }}-${{ matrix.cxx }}-${{ matrix.mpi }}-${{ matrix.omp }}
@@ -152,7 +166,7 @@ jobs:
152166

153167
test:
154168
needs:
155-
build
169+
dependencies
156170
name: test (${{matrix.os}}-${{matrix.cxx}}-mpi:${{matrix.mpi}}-openmp:${{matrix.omp}})
157171
runs-on: ${{matrix.os}}
158172
env:
@@ -194,6 +208,11 @@ jobs:
194208
with:
195209
name: dependencies-${{ matrix.os }}-${{ matrix.cxx }}-${{ matrix.mpi }}-${{ matrix.omp }}
196210

211+
- name: Setup cmake
212+
uses: jwlawson/actions-setup-cmake@v2
213+
with:
214+
cmake-version: '3.31.6'
215+
197216
- name: Unpack dependencies
198217
run: tar xfv dependencies.tar
199218

@@ -206,17 +225,18 @@ jobs:
206225
- name: Install Dependencies on MacOS
207226
if: ${{ contains(matrix.os, 'macos') }}
208227
run: |
209-
brew install open-mpi libomp eigen ccache cfitsio boost yaml-cpp
228+
brew install open-mpi libomp eigen ccache cfitsio boost yaml-cpp onnxruntime
210229
echo "CMAKE_PREFIX_PATH=/opt/homebrew/opt/libomp" >> $GITHUB_ENV
211230
echo "/opt/homebrew/opt/ccache/libexec" >> $GITHUB_PATH
212231
213232
- name: Build tests
214233
# Build your program with the given configuration
215234
run: |
216235
export CMAKE_PREFIX_PATH=${{github.workspace}}/local:$CMAKE_PREFIX_PATH
236+
#export PATH=${{github.workspace}}/local:$PATH
217237
mkdir -p ${{github.workspace}}/build
218238
cd ${{github.workspace}}/build
219-
cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Ddocasa=OFF -Ddompi=${{matrix.mpi}} -Dopenmp=${{matrix.omp}} -Dtests=ON
239+
cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Donnxrt=ON -Dhdf5=ON -Ddocasa=OFF -Ddompi=${{matrix.mpi}} -Dopenmp=${{matrix.omp}} -Dtests=ON
220240
make -j$(nproc --ignore 1) install
221241
222242
- name: Test
@@ -225,11 +245,92 @@ jobs:
225245
# See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
226246
run: |
227247
export LD_LIBRARY_PATH=${{github.workspace}}/local/lib:${{github.workspace}}/local/external/lib:${LD_LIBRARY_PATH}
248+
export DYLD_LIBRARY_PATH=${{github.workspace}}/local/lib:${{github.workspace}}/local/external/lib:${DYLD_LIBRARY_PATH}
249+
export DYLD_LIBRARY_PATH=$(brew --prefix)/lib:${DYLD_LIBRARY_PATH}
228250
ctest -C ${{env.BUILD_TYPE}} --output-on-failure
229251
252+
examples_benchmarks:
253+
needs:
254+
dependencies
255+
name: examples_benchmarks (${{matrix.os}}-${{matrix.cxx}}-mpi:${{matrix.mpi}}-openmp:${{matrix.omp}})
256+
runs-on: ${{matrix.os}}
257+
env:
258+
CC: ${{ matrix.cc }}
259+
CXX: ${{ matrix.cxx }}
260+
strategy:
261+
fail-fast: false
262+
matrix:
263+
os:
264+
- ubuntu-22.04
265+
- macos-14
266+
cc:
267+
- gcc-12
268+
- clang
269+
cxx:
270+
- g++-12
271+
- clang++
272+
mpi:
273+
- "ON"
274+
- "OFF"
275+
omp:
276+
- "ON"
277+
exclude:
278+
- os: ubuntu-22.04
279+
cc: clang
280+
- os: ubuntu-22.04
281+
cxx: clang++
282+
- os: macos-14
283+
cc: gcc-12
284+
- os: macos-14
285+
cxx: g++-12
286+
- os: macos-14
287+
mpi: "ON"
288+
- os: ubuntu-22.04
289+
mpi: "OFF"
290+
steps:
291+
- uses: actions/checkout@v4
292+
293+
- uses: actions/download-artifact@v4
294+
with:
295+
name: dependencies-${{ matrix.os }}-${{ matrix.cxx }}-${{ matrix.mpi }}-${{ matrix.omp }}
296+
297+
- name: Setup cmake
298+
uses: jwlawson/actions-setup-cmake@v2
299+
with:
300+
cmake-version: '3.31.6'
301+
302+
- name: Unpack dependencies
303+
run: tar xfv dependencies.tar
304+
- name: Install Dependencies on Ubunutu
305+
if: ${{ contains(matrix.os, 'ubuntu') }}
306+
run: |
307+
sudo apt update
308+
sudo apt install openmpi-bin libopenmpi-dev ccache graphviz libeigen3-dev libtiff-dev libcfitsio-dev libboost-all-dev libyaml-cpp-dev
309+
wget https://github.com/google/benchmark/archive/refs/tags/v1.8.5.tar.gz -O- | tar --no-same-owner -xz;
310+
cd benchmark-1.8.5
311+
cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -S . -B "build"
312+
cmake --build "build" --config Release --target install
313+
314+
- name: Install Dependencies on MacOS
315+
if: ${{ contains(matrix.os, 'macos') }}
316+
run: |
317+
brew install open-mpi libomp eigen ccache cfitsio boost yaml-cpp onnxruntime google-benchmark
318+
echo "CMAKE_PREFIX_PATH=/opt/homebrew/opt/libomp" >> $GITHUB_ENV
319+
echo "/opt/homebrew/opt/ccache/libexec" >> $GITHUB_PATH
320+
321+
- name: Build examples
322+
# Build your program with the given configuration
323+
run: |
324+
export CMAKE_PREFIX_PATH=${{github.workspace}}/local:$CMAKE_PREFIX_PATH
325+
mkdir -p ${{github.workspace}}/build
326+
cd ${{github.workspace}}/build
327+
cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Donnxrt=ON -Dhdf5=ON -Ddocasa=OFF -Ddompi=${{matrix.mpi}} -Dopenmp=${{matrix.omp}} -Dtests=OFF -Dexamples=ON -Dbenchmarks=ON
328+
make -j$(nproc --ignore 1) install
329+
330+
230331
doc:
231332
needs:
232-
build
333+
dependencies
233334
name: doc (${{matrix.os}}-${{matrix.cxx}}-mpi:${{matrix.mpi}}-openmp:${{matrix.omp}})
234335
runs-on: ${{matrix.os}}
235336
env:
@@ -255,6 +356,11 @@ jobs:
255356
with:
256357
name: dependencies-${{ matrix.os }}-${{ matrix.cxx }}-${{ matrix.mpi }}-${{ matrix.omp }}
257358

359+
- name: Setup cmake
360+
uses: jwlawson/actions-setup-cmake@v2
361+
with:
362+
cmake-version: '3.31.6'
363+
258364
- name: Unpack dependencies
259365
run: tar xfv dependencies.tar
260366

@@ -267,7 +373,7 @@ jobs:
267373
- name: Install Dependencies on MacOS
268374
if: ${{ contains(matrix.os, 'macos') }}
269375
run: |
270-
brew install gcc libtiff eigen libyaml ccache cfitsio boost yaml-cpp
376+
brew install gcc libtiff eigen libyaml ccache cfitsio boost yaml-cpp onnxruntime
271377
echo "CMAKE_PREFIX_PATH=/opt/homebrew/opt/libomp" >> $GITHUB_ENV
272378
echo "/opt/homebrew/opt/ccache/libexec" >> $GITHUB_PATH
273379
@@ -276,7 +382,7 @@ jobs:
276382
export CMAKE_PREFIX_PATH=${{github.workspace}}/local:$CMAKE_PREFIX_PATH
277383
mkdir -p ${{github.workspace}}/build
278384
cd ${{github.workspace}}/build
279-
cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Ddompi=OFF -Dopenmp=OFF -Ddocs=ON
385+
cmake .. --fresh -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/local -Donnxrt=ON -Dhdf5=ON -Ddompi=OFF -Dopenmp=OFF -Ddocs=ON
280386
make -j$(nproc --ignore 1) install
281387
282388
- name: Deploy to GH pages

‎CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ option(docasa "Enable CASA" off)
1616
option(docs "Build documentation" off)
1717
option(coverage "Build coverage" off)
1818
option(onnxrt "Build with ONNXruntime interface" off)
19+
option(hdf5 "Build with HDF5 interface" off)
1920

2021
if(NOT CMAKE_BUILD_TYPE)
2122
message(STATUS "Setting build type to 'Release' as none was specified.")
@@ -63,6 +64,5 @@ endif()
6364

6465
add_subdirectory(cpp)
6566

66-
6767
# Exports Purify so other packages can access it
6868
include(export_purify)

‎README.md

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,34 @@ When `purify` runs a directory will be created, and the output images will be
9191
saved and time-stamped. Additionally, a config file with the settings used will
9292
be saved and time-stamped, helping for reproducibility and book-keeping.
9393

94+
Uncertainty Quantification
95+
--------------------------
96+
97+
Bayesian hypothesis testing may be performed with the `purify_UQ` application, which can be located in the `build` directory.
98+
99+
This application takes a config yaml file with the following parameters:
100+
- `confidence_interval` or `alpha`. (alpha = 1 - confidence interval.))
101+
- `measurements_path`: path to measurements data (.vis file)
102+
- `reference_image_path`: path to reference image i.e. output from purify run. (.fits files)
103+
- `surrogate_image_path`: path to surrogate image i.e. doctored image with blurring or structural change that you want to test. (.fits file)
104+
- `sigma`: standard deviation for Gaussian likelihood.
105+
- `gamma`: multiplicative factor for prior.
106+
- `purify_config`: path to purify config used to generate the reference image. **This should be used if available in order to ensure consistency of things like measurement and wavelet operators.**
107+
108+
You can then run the uncertainty quantification with the command:
109+
```
110+
purify_UQ <path to UQ_config yaml>
111+
```
112+
113+
The application will report the value of the objective function for each image, the threshold value calculated from the reference image, and whether the surrogate image is ruled out or not.
114+
115+
Presently this is designed to work for the unconstrained problem where:
116+
- The negative log-likelihood is a scaled L2-norm i.e. $ \frac{1}{2 \sigma^2} \sum (y_i - \Phi x_i)$ for some data $y$, image $x$, and measurement operator $\Phi$. (Equivalent to indepdendent multivariate Gaussian likelihood.)
117+
- The negative log prior is a scaled L1-norm in _wavelet space_ i.e. $\gamma \sum (\Psi^\dag x)_i$ for some image $x$ and wavelet operator $\Psi$.
118+
- The objective function is the sum of these two terms.
119+
120+
Docker
121+
-------
94122
## Debugging the CI workflow with tmate
95123

96124
The CI workflow has a manual dispatch trigger which allows you to log into the job while it's running. You can trigger it in

‎cmake_files/LookUpHighFive.cmake

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Installs BlueBrain/HighFive into build directory
2+
#
3+
# - GIT_REPOSITORY: defaults to origin BlueBrain/HighFive repo on github
4+
# - GIT_TAG: defaults to master
5+
include(ExternalProject)
6+
if(HighFive_ARGUMENTS)
7+
cmake_parse_arguments(HighFive "" "GIT_REPOSITORY;GIT_TAG" ""
8+
${HighFive_ARGUMENTS})
9+
endif()
10+
if(NOT HighFive_GIT_REPOSITORY)
11+
set(HighFive_GIT_REPOSITORY https://github.com/BlueBrain/HighFive)
12+
endif()
13+
if(NOT HighFive_GIT_TAG)
14+
set(HighFive_GIT_TAG master)
15+
endif()
16+
17+
set(HighFive_DIR "${CMAKE_INSTALL_PREFIX}/external")
18+
ExternalProject_Add(
19+
Lookup-HighFive
20+
GIT_REPOSITORY ${HighFive_GIT_REPOSITORY}
21+
GIT_TAG ${HighFive_GIT_TAG}
22+
PREFIX "${CMAKE_BINARY_DIR}/external"
23+
#INSTALL_DIR ${EXTERNAL_ROOT}
24+
CMAKE_ARGS
25+
-DCMAKE_INSTALL_PREFIX=${HighFive_DIR}
26+
-DCMAKE_INSTALL_LIBDIR=${CMAKE_SHARED_LIBRARY_PREFIX}
27+
-DHIGHFIVE_EXAMPLES=OFF
28+
-DHIGHFIVE_USE_BOOST=OFF
29+
-DHIGHFIVE_UNIT_TESTS=OFF
30+
# Wrap download, configure and build steps in a script to log output
31+
UPDATE_COMMAND ""
32+
LOG_DOWNLOAD ON
33+
LOG_CONFIGURE ON
34+
LOG_BUILD ON
35+
LOG_INSTALL ON
36+
)
37+
set(HighFive_INCLUDE_DIR "${HighFive_DIR}/include")
38+
#set(HighFive_LIBRARY_DIR "${HighFive_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}")
39+
#set(HighFive_LIBRARIES "HighFive")
40+

0 commit comments

Comments
 (0)