From 30b00c572ec860de0206d74e0a18424b72798167 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Apr 2025 18:44:04 +0200 Subject: [PATCH 01/36] add the runbenchmarks --- benchmark/runbenchmarks.jl | 120 +++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 benchmark/runbenchmarks.jl diff --git a/benchmark/runbenchmarks.jl b/benchmark/runbenchmarks.jl new file mode 100644 index 0000000000..fe76746e78 --- /dev/null +++ b/benchmark/runbenchmarks.jl @@ -0,0 +1,120 @@ +using BenchmarkTools +using Oceananigans +using Oceananigans.TurbulenceClosures.TKEBasedVerticalDiffusivities: CATKEVerticalDiffusivity +using SeawaterPolynomials.TEOS10 +using Random +Random.seed!(1234) + +function hydrostatic_model(grid, + free_surface, + momentum_advection, + tracer_advection, + closure) + + buoyancy = SeawaterBuoyancy(equation_of_state=TEOS10EquationOfState()) + + model = HydrostaticFreeSurfaceModel(; grid, + free_surface, + momentum_advection, + tracers = (:T, :S, :e), + buoyancy, + tracer_advection, + closure) + + set!(model, T=20, S=35) + + # Warm up + time_step!(model, 0.00001) + time_step!(model, 0.00001) + + return model +end + +function nonhydrostatic_model(grid, advection) + + model = NonhydrostaticModel(; grid, advection) + + # Warm up + time_step!(model, 0.00001) + time_step!(model, 0.00001) + + return model +end + +function run_model_benchmark(model) + for i in 1:10 + time_step!(model, 0.00001) + end +end + +suite = BenchmarkGroup() + +Nx = 20 +Ny = 20 +Nz = 20 + +rgrid = RectilinearGrid(size=(Nx, Ny, Nz), extent=(1, 1, 1), halo=(7, 7, 7)) +lgrid = LatitudeLongitudeGrid(size=(Nx, Ny, Nz), latitude=(-10, 10), longitude=(0, 360), z=(-1, 0), halo=(7, 7, 7)) +tgrid = TripolarGrid(size=(Nx, Ny, Nz), z=(-1, 0), halo=(7, 7, 7)) + +bottom = 0.5 .* rand(Nx, Ny) .- 1 + +rigrid = ImmersedBoundaryGrid(rgrid, GridFittedBottom(bottom); active_cells_map=true) +ligrid = ImmersedBoundaryGrid(lgrid, GridFittedBottom(bottom); active_cells_map=true) +tigrid = ImmersedBoundaryGrid(tgrid, GridFittedBottom(bottom); active_cells_map=true) + +# All grids we test +grids = [ + rgrid, + lgrid, + tgrid, + rigrid, + ligrid, + tigrid +] + +nonhydrostatic_grids = [rgrid, rigrid] + +free_surfaces = [ + ExplicitFreeSurface(), + SplitExplicitFreeSurface(substeps=10), +] + +momentum_advections = [ + WENOVectorInvariant() +] + +tracer_advections = [ + Centered(order=4), + WENO(order=9) +] + +hydrostatic_closures = [ + nothing, + CATKEVerticalDiffusivity() +] + +suite["hydrostatic"] = BenchmarkGroup(["grid", "free_surface", "momentum_advection", "tracer_advection", "closure"]) +suite["nonhydrostatic"] = BenchmarkGroup(["grid", "advection"]) + +for grid in grids, + free_surface in free_surfaces, + momentum_advection in momentum_advections, + tracer_advection in tracer_advections, + closure in hydrostatic_closures + + model = hydrostatic_model(grid, free_surface, momentum_advection, tracer_advection, closure) + suite["hydrostatic"][grid, free_surface, momentum_advection, tracer_advection, closure] = @benchmarkable run_model_benchmark(model) +end + +for grid in nonhydrostatic_grids, + advection in tracer_advections + + model = nonhydrostatic_model(grid, advection) + suite["nonhydrostatic"][grid, advection] = @benchmarkable run_model_benchmark(model) +end + +tune!(suite) +results = run(suite, verbose = true) + +BenchmarkTools.save("output.json", median(results)) \ No newline at end of file From f41f048f619fc1f3d0713bfa122cf306ace36f85 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Apr 2025 19:59:46 +0200 Subject: [PATCH 02/36] let's ty it out --- .github/workflows/benchmarks.yml | 81 ++++++++++++++++++++++++++++++++ benchmark/runbenchmarks.jl | 25 +++++++++- 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/benchmarks.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..75e4cb44a2 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,81 @@ +name: Oceananigans Benchmarks +on: + push: + branches: + - main + - ss/performance-github-actions + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Run Oceananigans benchmarks + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.10' + - '1.11' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v4 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: runner.os−test−env.cache−name−{{ hashFiles('**/Project.toml') }} + restore-keys: | + runner.os−test− + ${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - name: Run benchmark + run: | + cd benchmark/ + julia --project --color=yes -e ' + using Pkg; + Pkg.instantiate(); + include("runbenchmarks.jl")' + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Oceananigans benchmark result + tool: 'julia' + output-file-path: benchmark/output.json + # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@glwagner, @simone-silvestri, @navidcy' + + # Not sure if this is needed + # - name: Store benchmark result - separate results repo + # uses: benchmark-action/github-action-benchmark@v1 + # with: + # name: Oceananigans benchmark result + # tool: 'julia' + # output-file-path: examples/julia/output.json + # # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 + # github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} + # auto-push: true + # # Show alert with commit comment on detecting possible performance regression + # alert-threshold: '200%' + # comment-on-alert: true + # fail-on-alert: true + # alert-comment-cc-users: '@ktrz,@findmyway' + # gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' \ No newline at end of file diff --git a/benchmark/runbenchmarks.jl b/benchmark/runbenchmarks.jl index fe76746e78..1bfda92926 100644 --- a/benchmark/runbenchmarks.jl +++ b/benchmark/runbenchmarks.jl @@ -97,6 +97,27 @@ hydrostatic_closures = [ suite["hydrostatic"] = BenchmarkGroup(["grid", "free_surface", "momentum_advection", "tracer_advection", "closure"]) suite["nonhydrostatic"] = BenchmarkGroup(["grid", "advection"]) +params(model::HydrostaticFreeSurfaceModel) = (params(model.grid), + params(model.free_surface), + params(model.advection.momentum), + params(model.advection.T), + params(model.closure)) + +params(model::NonhydrostaticModel) = (params(model.grid), + params(model.advection)) + +params(::Nothing) = "no closure" +params(::WENO) = "WENO" +params(::Centered) = "Centered" +params(::CATKEVerticalDiffusivity) = "CATKE" +params(::SplitExplicitFreeSurface) = "SplitExplicitFreeSurface" +params(::ExplicitFreeSurface) = "ExplicitFreeSurface" +params(::VectorInvariant) = "VectorInvariant" +params(i::ImmersedBoundaryGrid) = "ImmersedBoundaryGrid on a " * params(i.underlying_grid) +params(::RectilinearGrid) = "RectilinearGrid" +params(::LatitudeLongitudeGrid) = "LatitudeLongitudeGrid" +params(::TripolarGrid) = "TripolarGrid" + for grid in grids, free_surface in free_surfaces, momentum_advection in momentum_advections, @@ -104,14 +125,14 @@ for grid in grids, closure in hydrostatic_closures model = hydrostatic_model(grid, free_surface, momentum_advection, tracer_advection, closure) - suite["hydrostatic"][grid, free_surface, momentum_advection, tracer_advection, closure] = @benchmarkable run_model_benchmark(model) + suite["hydrostatic"][params(model)...] = @benchmarkable run_model_benchmark($model) end for grid in nonhydrostatic_grids, advection in tracer_advections model = nonhydrostatic_model(grid, advection) - suite["nonhydrostatic"][grid, advection] = @benchmarkable run_model_benchmark(model) + suite["nonhydrostatic"][params(model)...] = @benchmarkable run_model_benchmark($model) end tune!(suite) From e839d0ecca8340f92e456431b4589516106efcca Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Apr 2025 20:06:37 +0200 Subject: [PATCH 03/36] trigger tests --- .github/workflows/benchmarks.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 75e4cb44a2..4128e57c88 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -63,19 +63,19 @@ jobs: fail-on-alert: true alert-comment-cc-users: '@glwagner, @simone-silvestri, @navidcy' - # Not sure if this is needed - # - name: Store benchmark result - separate results repo - # uses: benchmark-action/github-action-benchmark@v1 - # with: - # name: Oceananigans benchmark result - # tool: 'julia' - # output-file-path: examples/julia/output.json - # # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 - # github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - # auto-push: true - # # Show alert with commit comment on detecting possible performance regression - # alert-threshold: '200%' - # comment-on-alert: true - # fail-on-alert: true - # alert-comment-cc-users: '@ktrz,@findmyway' - # gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' \ No newline at end of file + # Not sure if this is needed + # - name: Store benchmark result - separate results repo + # uses: benchmark-action/github-action-benchmark@v1 + # with: + # name: Oceananigans benchmark result + # tool: 'julia' + # output-file-path: examples/julia/output.json + # # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 + # github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} + # auto-push: true + # # Show alert with commit comment on detecting possible performance regression + # alert-threshold: '200%' + # comment-on-alert: true + # fail-on-alert: true + # alert-comment-cc-users: '@ktrz,@findmyway' + # gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' \ No newline at end of file From 7952af8e25b69ea3f108745df416d1cfe1e47d9f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Apr 2025 20:36:16 +0200 Subject: [PATCH 04/36] on everything --- .github/workflows/benchmarks.yml | 34 +++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 75e4cb44a2..fc042c5620 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,13 +1,33 @@ name: Oceananigans Benchmarks -on: - push: - branches: - - main - - ss/performance-github-actions +on: + pull_request: + paths: + - '.github/workflows/ci.yml' + - 'ext/**' + - 'src/**' + - 'test/**' + - 'Project.toml' + push: + branches: + - main + tags: '*' + paths: + - '.github/workflows/ci.yml' + - 'ext/**' + - 'src/**' + - 'test/**' + - 'Project.toml' + permissions: - contents: write - deployments: write + contents: write + deployments: write + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: benchmark: From bd16eae09575fad714260e1fc4821a71a4a358e7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Apr 2025 20:37:14 +0200 Subject: [PATCH 05/36] run the benchmarks --- .github/workflows/benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index ab87ccaabe..44cd4a227f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -3,7 +3,7 @@ name: Oceananigans Benchmarks on: pull_request: paths: - - '.github/workflows/ci.yml' + - '.github/workflows/benchmarks.yml' - 'ext/**' - 'src/**' - 'test/**' @@ -13,7 +13,7 @@ on: - main tags: '*' paths: - - '.github/workflows/ci.yml' + - '.github/workflows/benchmarks.yml' - 'ext/**' - 'src/**' - 'test/**' From 299c8e44b0f6932dc3c67100463dd7468936026a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Apr 2025 21:19:24 +0200 Subject: [PATCH 06/36] maybe this works --- .github/workflows/benchmarks.yml | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 44cd4a227f..18efd3404a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,23 +1,23 @@ name: Oceananigans Benchmarks on: - pull_request: - paths: - - '.github/workflows/benchmarks.yml' - - 'ext/**' - - 'src/**' - - 'test/**' - - 'Project.toml' - push: - branches: - - main - tags: '*' - paths: - - '.github/workflows/benchmarks.yml' - - 'ext/**' - - 'src/**' - - 'test/**' - - 'Project.toml' + pull_request: + paths: + - '.github/workflows/benchmarks.yml' + - 'ext/**' + - 'src/**' + - 'test/**' + - 'Project.toml' + push: + branches: + - main + tags: '*' + paths: + - '.github/workflows/benchmarks.yml' + - 'ext/**' + - 'src/**' + - 'test/**' + - 'Project.toml' permissions: contents: write From 2155099685fe707b1a7f18d1716e26a65df27e27 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Apr 2025 21:22:25 +0200 Subject: [PATCH 07/36] above project --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 18efd3404a..f10b7afb00 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -63,7 +63,7 @@ jobs: - name: Run benchmark run: | cd benchmark/ - julia --project --color=yes -e ' + julia --project=../ --color=yes -e ' using Pkg; Pkg.instantiate(); include("runbenchmarks.jl")' From 88ea9e77221216f872ce01e2a57f1831d9a73847 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Apr 2025 21:27:26 +0200 Subject: [PATCH 08/36] adding benchmarktools --- .github/workflows/benchmarks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f10b7afb00..ccfe95e8c5 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -65,6 +65,7 @@ jobs: cd benchmark/ julia --project=../ --color=yes -e ' using Pkg; + Pkg.add("BenchmarkTools"); Pkg.instantiate(); include("runbenchmarks.jl")' From 3bb144d84884e18c167dc5dab35f9d27fcdbe523 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Apr 2025 13:34:33 +0200 Subject: [PATCH 09/36] try pushing on docs build --- .github/workflows/benchmarks.yml | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index ccfe95e8c5..8080a63b95 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -69,12 +69,12 @@ jobs: Pkg.instantiate(); include("runbenchmarks.jl")' - - name: Store benchmark result + - name: Store benchmark result - separate results repo uses: benchmark-action/github-action-benchmark@v1 with: name: Oceananigans benchmark result tool: 'julia' - output-file-path: benchmark/output.json + output-file-path: examples/julia/output.json # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true @@ -82,21 +82,5 @@ jobs: alert-threshold: '200%' comment-on-alert: true fail-on-alert: true - alert-comment-cc-users: '@glwagner, @simone-silvestri, @navidcy' - - # Not sure if this is needed - # - name: Store benchmark result - separate results repo - # uses: benchmark-action/github-action-benchmark@v1 - # with: - # name: Oceananigans benchmark result - # tool: 'julia' - # output-file-path: examples/julia/output.json - # # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 - # github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - # auto-push: true - # # Show alert with commit comment on detecting possible performance regression - # alert-threshold: '200%' - # comment-on-alert: true - # fail-on-alert: true - # alert-comment-cc-users: '@ktrz,@findmyway' - # gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' \ No newline at end of file + alert-comment-cc-users: '@simone-silvestri, @glwagner, @navidcy' + gh-repository: 'github.com/CliMA/OceananigansDocumentation' \ No newline at end of file From e38dc9bf4f75fd3dced0bbdb22aa3751616db5ab Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 10:30:15 +0200 Subject: [PATCH 10/36] try this once --- .buildkite/pipeline-benchmarks.yml | 66 +++++++++++++++++++++ test/benchmark_tests.jl | 95 ++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 .buildkite/pipeline-benchmarks.yml create mode 100644 test/benchmark_tests.jl diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml new file mode 100644 index 0000000000..3decd0f697 --- /dev/null +++ b/.buildkite/pipeline-benchmarks.yml @@ -0,0 +1,66 @@ +env: + JULIA_VERSION: "1.10.9" + JULIA_MINOR_VERSION: "1.10" + TARTARUS_HOME: "/storage5/buildkite-agent" + JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" + JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager + JULIA_NUM_PRECOMPILE_TASKS: 24 + JULIA_NUM_THREADS: 8 + NSYS: "/storage6/simone/new_nsys/bin/nsys" + CUDA_VISIBLE_DEVICES: "0" # Tartarus device for GPU Benchmarking + TMPDIR: "$TARTARUS_HOME/tmp" + +agents: + queue: "Oceananigans benchmarks" + +steps: + - label: "🏕️ initialize tartarus environment" + key: "init" + env: + JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" + TEST_GROUP: "init" + JULIA_BINDIR: "$TARTARUS_HOME/julia-$JULIA_VERSION/bin" + command: | + # Download julia binaries + wget -N -P $TARTARUS_HOME https://julialang-s3.julialang.org/bin/linux/x64/$JULIA_MINOR_VERSION/julia-$JULIA_VERSION-linux-x86_64.tar.gz + tar xf $TARTARUS_HOME/julia-$JULIA_VERSION-linux-x86_64.tar.gz -C $TARTARUS_HOME + $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.instantiate()' + agents: + queue: "Oceananigans benchmarks" + retry: + automatic: + - exit_status: 1 + limit: 1 + + - wait + + - label: "{{ matrix.architecture }} - {{ matrix.group }} tests" + key: "tests" + agents: + queue: "Oceananigans" + command: | + + # Strip emoji for environment variable + group="{{ matrix.group }}" + export BENCHMARK_GROUP="\${group#* }" + echo $BENCHMARK_GROUP + + OUTPUT=output_$BENCHMARK_GROUP + + # Run tests + $NSYS profile $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl + $NSYS stats $OUTPUT.nsys-rep + matrix: + setup: + architecture: + - "CPU" + - "GPU" + group: + - "🍩 periodic" + - "📦 bounded" + - "🕊 periodic_cheap_vertical_advection" + - "🦖 bounded_cheap_vertical_advection" + - "🌷 immersed" + soft_fail: + - exit_status: 3 + diff --git a/test/benchmark_tests.jl b/test/benchmark_tests.jl new file mode 100644 index 0000000000..41fcf8029c --- /dev/null +++ b/test/benchmark_tests.jl @@ -0,0 +1,95 @@ +using Oceananigans +using Oceananigans.Units +using Oceananigans.Architectures: on_architecture +using Oceananigans.TurbulenceClosures.TKEBasedVerticalDiffusivities: CATKEVerticalDiffusivity +using SeawaterPolynomials.TEOS10 +using Random + +function ocean_benchmark(arch, Nx, Ny, Nz, topology, immersed, tracer_advection=WENO(order=7)) + + z_faces = collect(range(-6000, 0, length=Nz+1)) + + grid = RectilinearGrid(arch; size=(Nx, Ny, Nz), + halo=(7, 7, 7), + z=z_faces, + x=(-1000kilometers, 1000kilometers), + y=(-1000kilometers, 1000kilometers), + topology) + + grid = if immersed + Random.seed!(1234) + bottom = Oceananigans.Architectures.on_architecture(arch, - 5000 .* rand(Nx, Ny) .- 1000) + ImmersedBoundaryGrid(grid, GridFittedBottom(bottom); active_cells_map=true) + else + grid + end + + @info "Grid is built" + momentum_advection = WENOVectorInvariant() + buoyancy = SeawaterBuoyancy(equation_of_state=TEOS10EquationOfState()) + free_surface = SplitExplicitFreeSurface(grid; substeps=70) + closure = CATKEVerticalDiffusivity() + + model = HydrostaticFreeSurfaceModel(; grid, + momentum_advection, + tracer_advection, + buoyancy, + closure, + free_surface, + tracers = (:T, :S, :e)) + + @info "Model is built" + + R = rand(size(model.grid)) + + # initialize variables with randomish values + Tᵢ = 0.0001 .* R .+ 20 + Sᵢ = 0.0001 .* R .+ 35 + uᵢ = 0.0001 .* R + vᵢ = 0.0001 .* R + + set!(model, T=Tᵢ, S=Sᵢ, e=1e-6, u=uᵢ, v=vᵢ) + + return model +end + +function run_benchmark(model) + for _ in 1:15 + time_step!(model, 1.0) + end +end + +group = get(ENV, "BENCHMARK_GROUP", "all") |> Symbol + +const Nx = 1000 +const Ny = 500 +const Nz = 60 + +arch = GPU() + +cheap_advection = (WENO(order=7), WENO(order=7), Centered()) + +if group == :periodic + model = ocean_benchmark(arch, Nx, Ny, Nz, (Periodic, Periodic, Bounded), false) + run_benchmark(model) +end + +if group == :bounded + model = ocean_benchmark(arch, Nx, Ny, Nz, (Bounded, Bounded, Bounded), false) + run_benchmark(model) +end + +if group == :periodic_cheap_vertical_advection + model = ocean_benchmark(arch, Nx, Ny, Nz, (Periodic, Periodic, Bounded), false, cheap_advection) + run_benchmark(model) +end + +if group == :bounded_cheap_vertical_advection + model = ocean_benchmark(arch, Nx, Ny, Nz, (Bounded, Bounded, Bounded), false, cheap_advection) + run_benchmark(model) +end + +if group == :immersed + model = ocean_benchmark(arch, Nx, Ny, Nz, (Periodic, Periodic, Bounded), true) + run_benchmark(model) +end \ No newline at end of file From 1564c5f586f8f105d4782f81346da20755fe1030 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 10:33:59 +0200 Subject: [PATCH 11/36] correct queue --- .buildkite/pipeline-benchmarks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 3decd0f697..08654b50b3 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -11,7 +11,7 @@ env: TMPDIR: "$TARTARUS_HOME/tmp" agents: - queue: "Oceananigans benchmarks" + queue: "Oceananigans-benchmarks" steps: - label: "🏕️ initialize tartarus environment" @@ -26,7 +26,7 @@ steps: tar xf $TARTARUS_HOME/julia-$JULIA_VERSION-linux-x86_64.tar.gz -C $TARTARUS_HOME $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.instantiate()' agents: - queue: "Oceananigans benchmarks" + queue: "Oceananigans-benchmarks" retry: automatic: - exit_status: 1 @@ -37,7 +37,7 @@ steps: - label: "{{ matrix.architecture }} - {{ matrix.group }} tests" key: "tests" agents: - queue: "Oceananigans" + queue: "Oceananigans-benchmarks" command: | # Strip emoji for environment variable From 38b68857a410cca4fb47764f345b6dff94b8ba34 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 10:38:42 +0200 Subject: [PATCH 12/36] try out the pipeline --- .buildkite/pipeline-benchmarks.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 08654b50b3..b4ad3b2c89 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -39,17 +39,21 @@ steps: agents: queue: "Oceananigans-benchmarks" command: | - + # Strip emoji for environment variable group="{{ matrix.group }}" export BENCHMARK_GROUP="\${group#* }" echo $BENCHMARK_GROUP + # Save output to a file OUTPUT=output_$BENCHMARK_GROUP # Run tests $NSYS profile $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl $NSYS stats $OUTPUT.nsys-rep + + # Remove generated output files + rm $OUTPUT.* matrix: setup: architecture: From c11b8f3313249af7b35f3f19d01eb7fcb7b78c20 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 10:39:30 +0200 Subject: [PATCH 13/36] no need for an architecture --- .buildkite/pipeline-benchmarks.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index b4ad3b2c89..786f067fdf 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -56,9 +56,6 @@ steps: rm $OUTPUT.* matrix: setup: - architecture: - - "CPU" - - "GPU" group: - "🍩 periodic" - "📦 bounded" From 4ce978e1145198172eb7e7365d71b9ac0a65bb6d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 10:51:43 +0200 Subject: [PATCH 14/36] use the oceananigans queue --- .buildkite/pipeline-benchmarks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 786f067fdf..d6b37663e9 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -11,7 +11,7 @@ env: TMPDIR: "$TARTARUS_HOME/tmp" agents: - queue: "Oceananigans-benchmarks" + queue: "Oceananigans" steps: - label: "🏕️ initialize tartarus environment" @@ -26,7 +26,7 @@ steps: tar xf $TARTARUS_HOME/julia-$JULIA_VERSION-linux-x86_64.tar.gz -C $TARTARUS_HOME $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.instantiate()' agents: - queue: "Oceananigans-benchmarks" + queue: "Oceananigans" retry: automatic: - exit_status: 1 @@ -37,7 +37,7 @@ steps: - label: "{{ matrix.architecture }} - {{ matrix.group }} tests" key: "tests" agents: - queue: "Oceananigans-benchmarks" + queue: "Oceananigans" command: | # Strip emoji for environment variable From 65a13b5cbba04a8b4804ca3eb668a266bad435d4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 11:01:02 +0200 Subject: [PATCH 15/36] add a trace --- .buildkite/pipeline-benchmarks.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index d6b37663e9..3ec16e4274 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -6,7 +6,7 @@ env: JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager JULIA_NUM_PRECOMPILE_TASKS: 24 JULIA_NUM_THREADS: 8 - NSYS: "/storage6/simone/new_nsys/bin/nsys" + NSYS: "/storage6/simone/new_nsight/bin/nsys" CUDA_VISIBLE_DEVICES: "0" # Tartarus device for GPU Benchmarking TMPDIR: "$TARTARUS_HOME/tmp" @@ -46,10 +46,11 @@ steps: echo $BENCHMARK_GROUP # Save output to a file - OUTPUT=output_$BENCHMARK_GROUP + export OUTPUT=output_${BENCHMARK_GROUP} + echo $OUTPUT # Run tests - $NSYS profile $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl + $NSYS profile --output=$OUTPUT --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl $NSYS stats $OUTPUT.nsys-rep # Remove generated output files From a4b41bfa4fb4c9a496d1cedaadce93b456431b11 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 11:13:17 +0200 Subject: [PATCH 16/36] change output per group --- .buildkite/pipeline-benchmarks.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 3ec16e4274..f317af5bcd 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -45,16 +45,12 @@ steps: export BENCHMARK_GROUP="\${group#* }" echo $BENCHMARK_GROUP - # Save output to a file - export OUTPUT=output_${BENCHMARK_GROUP} - echo $OUTPUT - - # Run tests - $NSYS profile --output=$OUTPUT --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl + # Run benchmarks + $NSYS profile --output=nsys_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl $NSYS stats $OUTPUT.nsys-rep # Remove generated output files - rm $OUTPUT.* + rm nsys_output.* matrix: setup: group: From a4e021a99ad0e4ff0acd22c87eaabd219c6916ae Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 11:13:48 +0200 Subject: [PATCH 17/36] use GPU1 for the moment --- .buildkite/pipeline-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index f317af5bcd..215ca0c632 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -7,7 +7,7 @@ env: JULIA_NUM_PRECOMPILE_TASKS: 24 JULIA_NUM_THREADS: 8 NSYS: "/storage6/simone/new_nsight/bin/nsys" - CUDA_VISIBLE_DEVICES: "0" # Tartarus device for GPU Benchmarking + CUDA_VISIBLE_DEVICES: "1" # Tartarus device for GPU Benchmarking TMPDIR: "$TARTARUS_HOME/tmp" agents: From ef8a7f466d72bd483c5a24615483938d651162b2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 11:21:00 +0200 Subject: [PATCH 18/36] instantiate inside the test --- .buildkite/pipeline-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 215ca0c632..08e603a74d 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -24,7 +24,6 @@ steps: # Download julia binaries wget -N -P $TARTARUS_HOME https://julialang-s3.julialang.org/bin/linux/x64/$JULIA_MINOR_VERSION/julia-$JULIA_VERSION-linux-x86_64.tar.gz tar xf $TARTARUS_HOME/julia-$JULIA_VERSION-linux-x86_64.tar.gz -C $TARTARUS_HOME - $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.instantiate()' agents: queue: "Oceananigans" retry: @@ -46,6 +45,7 @@ steps: echo $BENCHMARK_GROUP # Run benchmarks + $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no -e 'using Pkg; Pkg.instantiate()' $NSYS profile --output=nsys_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl $NSYS stats $OUTPUT.nsys-rep From ed012b9b65baad1b6423acb96d2a436eb0b16d60 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 11:23:07 +0200 Subject: [PATCH 19/36] instantiate first --- .buildkite/pipeline-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 08e603a74d..884e8a6bf1 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -24,6 +24,7 @@ steps: # Download julia binaries wget -N -P $TARTARUS_HOME https://julialang-s3.julialang.org/bin/linux/x64/$JULIA_MINOR_VERSION/julia-$JULIA_VERSION-linux-x86_64.tar.gz tar xf $TARTARUS_HOME/julia-$JULIA_VERSION-linux-x86_64.tar.gz -C $TARTARUS_HOME + $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no -e 'using Pkg; Pkg.instantiate()' agents: queue: "Oceananigans" retry: @@ -45,7 +46,6 @@ steps: echo $BENCHMARK_GROUP # Run benchmarks - $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no -e 'using Pkg; Pkg.instantiate()' $NSYS profile --output=nsys_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl $NSYS stats $OUTPUT.nsys-rep From 72bdc1e51a1c30a9ea3028c754b7f5490406106d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 11:31:58 +0200 Subject: [PATCH 20/36] try uploading an artifact --- .buildkite/pipeline-benchmarks.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 884e8a6bf1..cc367e0b53 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -24,7 +24,6 @@ steps: # Download julia binaries wget -N -P $TARTARUS_HOME https://julialang-s3.julialang.org/bin/linux/x64/$JULIA_MINOR_VERSION/julia-$JULIA_VERSION-linux-x86_64.tar.gz tar xf $TARTARUS_HOME/julia-$JULIA_VERSION-linux-x86_64.tar.gz -C $TARTARUS_HOME - $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no -e 'using Pkg; Pkg.instantiate()' agents: queue: "Oceananigans" retry: @@ -45,20 +44,24 @@ steps: export BENCHMARK_GROUP="\${group#* }" echo $BENCHMARK_GROUP + # Instantiate + $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no -e 'using Pkg; Pkg.instantiate()' + # Run benchmarks $NSYS profile --output=nsys_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl - $NSYS stats $OUTPUT.nsys-rep + $NSYS stats nsy_output.nsys-rep > nsys_output.txt # Remove generated output files - rm nsys_output.* + rm nsys_output.nsys-rep + rm nsys_output.sqlite matrix: setup: group: - "🍩 periodic" - - "📦 bounded" - - "🕊 periodic_cheap_vertical_advection" - - "🦖 bounded_cheap_vertical_advection" - - "🌷 immersed" + # - "📦 bounded" + # - "🕊 periodic_cheap_vertical_advection" + # - "🦖 bounded_cheap_vertical_advection" + # - "🌷 immersed" soft_fail: - exit_status: 3 From 115f46639d3739930d4dec09128bd2ee4941dc11 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 11:32:42 +0200 Subject: [PATCH 21/36] add an artifact --- .buildkite/pipeline-benchmarks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index cc367e0b53..aaca69aaf8 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -54,6 +54,8 @@ steps: # Remove generated output files rm nsys_output.nsys-rep rm nsys_output.sqlite + artifact_paths: + - "nsys_output.txt" matrix: setup: group: From 8c68e3262abb3282bc2eed3d267d27d4935c2966 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 11:44:11 +0200 Subject: [PATCH 22/36] should work all in one pipeline --- .buildkite/pipeline-benchmarks.yml | 62 +++++++++++++++++++++++------- test/benchmark_tests.jl | 4 +- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index aaca69aaf8..0bb9c4497d 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -41,29 +41,65 @@ steps: # Strip emoji for environment variable group="{{ matrix.group }}" - export BENCHMARK_GROUP="\${group#* }" - echo $BENCHMARK_GROUP # Instantiate $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no -e 'using Pkg; Pkg.instantiate()' - # Run benchmarks - $NSYS profile --output=nsys_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl - $NSYS stats nsy_output.nsys-rep > nsys_output.txt + # Run Periodic benchmarks + export BENCHMARK_GROUP="periodic" + $NSYS profile --output=periodic_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl + $NSYS stats periodic_output.nsys-rep > periodic_output.txt # Remove generated output files - rm nsys_output.nsys-rep - rm nsys_output.sqlite + rm periodic_output.nsys-rep + rm periodic_output.sqlite + + # Run Bounded benchmarks + export BENCHMARK_GROUP="bounded" + $NSYS profile --output=bounded_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl + $NSYS stats bounded_output.nsys-rep > bounded_output.txt + + # Remove generated output files + rm bounded_output.nsys-rep + rm bounded_output.sqlite + + # Run Periodic cheap advection benchmarks + export BENCHMARK_GROUP="periodic_cheap_advection" + $NSYS profile --output=periodic_cheap_advection_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl + $NSYS stats pperiodic_cheap_advection_output.nsys-rep > periodic_cheap_advection_output.txt + + # Remove generated output files + rm periodic_cheap_advection.nsys-rep + rm periodic_cheap_advection.sqlite + + # Run Bounded cheap advection benchmarks + export BENCHMARK_GROUP="bounded_cheap_advection" + $NSYS profile --output=bounded_cheap_advection_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl + $NSYS stats bounded_cheap_advection_output.nsys-rep > bounded_cheap_advection_output.txt + + # Remove generated output files + rm bounded_cheap_advection_output.nsys-rep + rm bounded_cheap_advection_output.sqlite + + # Run Immersed benchmarks + export BENCHMARK_GROUP="immersed" + $NSYS profile --output=immersed_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl + $NSYS stats immersed_output.nsys-rep > immersed_output.txt + + # Remove generated output files + rm immersed_output.nsys-rep + rm immersed_output.sqlite + artifact_paths: - - "nsys_output.txt" + - "periodic_output.txt" + - "bounded_output.txt" + - "periodic_cheap_advection_output.txt" + - "bounded_cheap_advection_output.txt" + - "immersed_output.txt" matrix: setup: group: - - "🍩 periodic" - # - "📦 bounded" - # - "🕊 periodic_cheap_vertical_advection" - # - "🦖 bounded_cheap_vertical_advection" - # - "🌷 immersed" + - "🚀 benchmark tests" soft_fail: - exit_status: 3 diff --git a/test/benchmark_tests.jl b/test/benchmark_tests.jl index 41fcf8029c..fc3d904bb3 100644 --- a/test/benchmark_tests.jl +++ b/test/benchmark_tests.jl @@ -61,8 +61,8 @@ end group = get(ENV, "BENCHMARK_GROUP", "all") |> Symbol -const Nx = 1000 -const Ny = 500 +const Nx = 500 +const Ny = 200 const Nz = 60 arch = GPU() From 2d423e056cc467f78bac491bb7fcd228a170c1b0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 11:59:30 +0200 Subject: [PATCH 23/36] some bugfixes --- .buildkite/pipeline-benchmarks.yml | 7 +- .github/workflows/benchmarks.yml | 86 ------------------ benchmark/runbenchmarks.jl | 141 ----------------------------- 3 files changed, 3 insertions(+), 231 deletions(-) delete mode 100644 .github/workflows/benchmarks.yml delete mode 100644 benchmark/runbenchmarks.jl diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 0bb9c4497d..001f572346 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -34,11 +34,10 @@ steps: - wait - label: "{{ matrix.architecture }} - {{ matrix.group }} tests" - key: "tests" + key: "benchmarks" agents: queue: "Oceananigans" command: | - # Strip emoji for environment variable group="{{ matrix.group }}" @@ -66,7 +65,7 @@ steps: # Run Periodic cheap advection benchmarks export BENCHMARK_GROUP="periodic_cheap_advection" $NSYS profile --output=periodic_cheap_advection_output --trace=cuda $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no test/benchmark_tests.jl - $NSYS stats pperiodic_cheap_advection_output.nsys-rep > periodic_cheap_advection_output.txt + $NSYS stats periodic_cheap_advection_output.nsys-rep > periodic_cheap_advection_output.txt # Remove generated output files rm periodic_cheap_advection.nsys-rep @@ -99,7 +98,7 @@ steps: matrix: setup: group: - - "🚀 benchmark tests" + - "🚀 Oceananigans GPU benchmarks" soft_fail: - exit_status: 3 diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml deleted file mode 100644 index 8080a63b95..0000000000 --- a/.github/workflows/benchmarks.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Oceananigans Benchmarks - -on: - pull_request: - paths: - - '.github/workflows/benchmarks.yml' - - 'ext/**' - - 'src/**' - - 'test/**' - - 'Project.toml' - push: - branches: - - main - tags: '*' - paths: - - '.github/workflows/benchmarks.yml' - - 'ext/**' - - 'src/**' - - 'test/**' - - 'Project.toml' - -permissions: - contents: write - deployments: write - -concurrency: - # Skip intermediate builds: always. - # Cancel intermediate builds: only if it is a pull request build. - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -jobs: - benchmark: - name: Run Oceananigans benchmarks - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - version: - - '1.10' - - '1.11' - os: - - ubuntu-latest - arch: - - x64 - steps: - - uses: actions/checkout@v4 - - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.version }} - arch: ${{ matrix.arch }} - - uses: actions/cache@v4 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: runner.os−test−env.cache−name−{{ hashFiles('**/Project.toml') }} - restore-keys: | - runner.os−test− - ${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- - - name: Run benchmark - run: | - cd benchmark/ - julia --project=../ --color=yes -e ' - using Pkg; - Pkg.add("BenchmarkTools"); - Pkg.instantiate(); - include("runbenchmarks.jl")' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Oceananigans benchmark result - tool: 'julia' - output-file-path: examples/julia/output.json - # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@simone-silvestri, @glwagner, @navidcy' - gh-repository: 'github.com/CliMA/OceananigansDocumentation' \ No newline at end of file diff --git a/benchmark/runbenchmarks.jl b/benchmark/runbenchmarks.jl deleted file mode 100644 index 1bfda92926..0000000000 --- a/benchmark/runbenchmarks.jl +++ /dev/null @@ -1,141 +0,0 @@ -using BenchmarkTools -using Oceananigans -using Oceananigans.TurbulenceClosures.TKEBasedVerticalDiffusivities: CATKEVerticalDiffusivity -using SeawaterPolynomials.TEOS10 -using Random -Random.seed!(1234) - -function hydrostatic_model(grid, - free_surface, - momentum_advection, - tracer_advection, - closure) - - buoyancy = SeawaterBuoyancy(equation_of_state=TEOS10EquationOfState()) - - model = HydrostaticFreeSurfaceModel(; grid, - free_surface, - momentum_advection, - tracers = (:T, :S, :e), - buoyancy, - tracer_advection, - closure) - - set!(model, T=20, S=35) - - # Warm up - time_step!(model, 0.00001) - time_step!(model, 0.00001) - - return model -end - -function nonhydrostatic_model(grid, advection) - - model = NonhydrostaticModel(; grid, advection) - - # Warm up - time_step!(model, 0.00001) - time_step!(model, 0.00001) - - return model -end - -function run_model_benchmark(model) - for i in 1:10 - time_step!(model, 0.00001) - end -end - -suite = BenchmarkGroup() - -Nx = 20 -Ny = 20 -Nz = 20 - -rgrid = RectilinearGrid(size=(Nx, Ny, Nz), extent=(1, 1, 1), halo=(7, 7, 7)) -lgrid = LatitudeLongitudeGrid(size=(Nx, Ny, Nz), latitude=(-10, 10), longitude=(0, 360), z=(-1, 0), halo=(7, 7, 7)) -tgrid = TripolarGrid(size=(Nx, Ny, Nz), z=(-1, 0), halo=(7, 7, 7)) - -bottom = 0.5 .* rand(Nx, Ny) .- 1 - -rigrid = ImmersedBoundaryGrid(rgrid, GridFittedBottom(bottom); active_cells_map=true) -ligrid = ImmersedBoundaryGrid(lgrid, GridFittedBottom(bottom); active_cells_map=true) -tigrid = ImmersedBoundaryGrid(tgrid, GridFittedBottom(bottom); active_cells_map=true) - -# All grids we test -grids = [ - rgrid, - lgrid, - tgrid, - rigrid, - ligrid, - tigrid -] - -nonhydrostatic_grids = [rgrid, rigrid] - -free_surfaces = [ - ExplicitFreeSurface(), - SplitExplicitFreeSurface(substeps=10), -] - -momentum_advections = [ - WENOVectorInvariant() -] - -tracer_advections = [ - Centered(order=4), - WENO(order=9) -] - -hydrostatic_closures = [ - nothing, - CATKEVerticalDiffusivity() -] - -suite["hydrostatic"] = BenchmarkGroup(["grid", "free_surface", "momentum_advection", "tracer_advection", "closure"]) -suite["nonhydrostatic"] = BenchmarkGroup(["grid", "advection"]) - -params(model::HydrostaticFreeSurfaceModel) = (params(model.grid), - params(model.free_surface), - params(model.advection.momentum), - params(model.advection.T), - params(model.closure)) - -params(model::NonhydrostaticModel) = (params(model.grid), - params(model.advection)) - -params(::Nothing) = "no closure" -params(::WENO) = "WENO" -params(::Centered) = "Centered" -params(::CATKEVerticalDiffusivity) = "CATKE" -params(::SplitExplicitFreeSurface) = "SplitExplicitFreeSurface" -params(::ExplicitFreeSurface) = "ExplicitFreeSurface" -params(::VectorInvariant) = "VectorInvariant" -params(i::ImmersedBoundaryGrid) = "ImmersedBoundaryGrid on a " * params(i.underlying_grid) -params(::RectilinearGrid) = "RectilinearGrid" -params(::LatitudeLongitudeGrid) = "LatitudeLongitudeGrid" -params(::TripolarGrid) = "TripolarGrid" - -for grid in grids, - free_surface in free_surfaces, - momentum_advection in momentum_advections, - tracer_advection in tracer_advections, - closure in hydrostatic_closures - - model = hydrostatic_model(grid, free_surface, momentum_advection, tracer_advection, closure) - suite["hydrostatic"][params(model)...] = @benchmarkable run_model_benchmark($model) -end - -for grid in nonhydrostatic_grids, - advection in tracer_advections - - model = nonhydrostatic_model(grid, advection) - suite["nonhydrostatic"][params(model)...] = @benchmarkable run_model_benchmark($model) -end - -tune!(suite) -results = run(suite, verbose = true) - -BenchmarkTools.save("output.json", median(results)) \ No newline at end of file From a6b7fba9e0206897f39d8e171c20c54846500f5b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 12:13:13 +0200 Subject: [PATCH 24/36] runb the complete benchmark test suite --- .buildkite/pipeline-benchmarks.yml | 4 ++-- test/benchmark_tests.jl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 001f572346..02477364d0 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -68,8 +68,8 @@ steps: $NSYS stats periodic_cheap_advection_output.nsys-rep > periodic_cheap_advection_output.txt # Remove generated output files - rm periodic_cheap_advection.nsys-rep - rm periodic_cheap_advection.sqlite + rm periodic_cheap_advection_output.nsys-rep + rm periodic_cheap_advection_output.sqlite # Run Bounded cheap advection benchmarks export BENCHMARK_GROUP="bounded_cheap_advection" diff --git a/test/benchmark_tests.jl b/test/benchmark_tests.jl index fc3d904bb3..ec5fdf139d 100644 --- a/test/benchmark_tests.jl +++ b/test/benchmark_tests.jl @@ -79,12 +79,12 @@ if group == :bounded run_benchmark(model) end -if group == :periodic_cheap_vertical_advection +if group == :periodic_cheap_advection model = ocean_benchmark(arch, Nx, Ny, Nz, (Periodic, Periodic, Bounded), false, cheap_advection) run_benchmark(model) end -if group == :bounded_cheap_vertical_advection +if group == :bounded_cheap_advection model = ocean_benchmark(arch, Nx, Ny, Nz, (Bounded, Bounded, Bounded), false, cheap_advection) run_benchmark(model) end From aba63efcd684e2d5ce4171beedc3cce268dd0c21 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 12:28:40 +0200 Subject: [PATCH 25/36] load cheap advection --- .buildkite/pipeline-benchmarks.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 02477364d0..b375aed8b5 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -33,14 +33,11 @@ steps: - wait - - label: "{{ matrix.architecture }} - {{ matrix.group }} tests" + - label: "🚀 Oceananigans GPU benchmarks" key: "benchmarks" agents: queue: "Oceananigans" command: | - # Strip emoji for environment variable - group="{{ matrix.group }}" - # Instantiate $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no -e 'using Pkg; Pkg.instantiate()' @@ -95,10 +92,6 @@ steps: - "periodic_cheap_advection_output.txt" - "bounded_cheap_advection_output.txt" - "immersed_output.txt" - matrix: - setup: - group: - - "🚀 Oceananigans GPU benchmarks" soft_fail: - exit_status: 3 From 9c96189300dbb8bcba7e822b607600002fe9fabf Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 12:28:49 +0200 Subject: [PATCH 26/36] fix cheap advection --- test/benchmark_tests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark_tests.jl b/test/benchmark_tests.jl index ec5fdf139d..99a5a87b34 100644 --- a/test/benchmark_tests.jl +++ b/test/benchmark_tests.jl @@ -67,7 +67,7 @@ const Nz = 60 arch = GPU() -cheap_advection = (WENO(order=7), WENO(order=7), Centered()) +cheap_advection = FluxFormAdvection(WENO(order=7), WENO(order=7), Centered()) if group == :periodic model = ocean_benchmark(arch, Nx, Ny, Nz, (Periodic, Periodic, Bounded), false) From 227766b1b6278df474c57faa86efd12eba68d718 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 15:04:51 +0200 Subject: [PATCH 27/36] Update .buildkite/pipeline-benchmarks.yml Co-authored-by: Gregory L. Wagner --- .buildkite/pipeline-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index b375aed8b5..60b42e6732 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -6,7 +6,7 @@ env: JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager JULIA_NUM_PRECOMPILE_TASKS: 24 JULIA_NUM_THREADS: 8 - NSYS: "/storage6/simone/new_nsight/bin/nsys" + NSYS: "/storage6/nsight/bin/nsys" CUDA_VISIBLE_DEVICES: "1" # Tartarus device for GPU Benchmarking TMPDIR: "$TARTARUS_HOME/tmp" From 78e277a75718d3a4af41099c093fd45d4d0fbace Mon Sep 17 00:00:00 2001 From: Gregory Wagner Date: Wed, 23 Apr 2025 09:37:41 -0400 Subject: [PATCH 28/36] switch queue --- .buildkite/pipeline-benchmarks.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 60b42e6732..0309c61262 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -10,9 +10,6 @@ env: CUDA_VISIBLE_DEVICES: "1" # Tartarus device for GPU Benchmarking TMPDIR: "$TARTARUS_HOME/tmp" -agents: - queue: "Oceananigans" - steps: - label: "🏕️ initialize tartarus environment" key: "init" @@ -25,7 +22,7 @@ steps: wget -N -P $TARTARUS_HOME https://julialang-s3.julialang.org/bin/linux/x64/$JULIA_MINOR_VERSION/julia-$JULIA_VERSION-linux-x86_64.tar.gz tar xf $TARTARUS_HOME/julia-$JULIA_VERSION-linux-x86_64.tar.gz -C $TARTARUS_HOME agents: - queue: "Oceananigans" + queue: "Oceananigans-benchmarks" retry: automatic: - exit_status: 1 @@ -36,7 +33,7 @@ steps: - label: "🚀 Oceananigans GPU benchmarks" key: "benchmarks" agents: - queue: "Oceananigans" + queue: "Oceananigans-benchmarks" command: | # Instantiate $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no -e 'using Pkg; Pkg.instantiate()' From cddc36f2e4c087280d055e44a564efca08884013 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 15:41:08 +0200 Subject: [PATCH 29/36] add login to the benchmark pipeline --- .buildkite/pipeline-benchmarks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 0309c61262..d4360a75dd 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -12,6 +12,7 @@ env: steps: - label: "🏕️ initialize tartarus environment" + if: "build.pull_request.labels includes \"benchmark performance\" || build.tag != null || build.branch == 'main'" key: "init" env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" @@ -31,6 +32,7 @@ steps: - wait - label: "🚀 Oceananigans GPU benchmarks" + if: "build.pull_request.labels includes \"benchmark performance\" || build.tag != null || build.branch == 'main'" key: "benchmarks" agents: queue: "Oceananigans-benchmarks" From 6349407773a623705b23fe6cab38832f0d412df9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 15:52:25 +0200 Subject: [PATCH 30/36] add login to the benchmark pipeline --- .buildkite/pipeline-benchmarks.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index d4360a75dd..0309c61262 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -12,7 +12,6 @@ env: steps: - label: "🏕️ initialize tartarus environment" - if: "build.pull_request.labels includes \"benchmark performance\" || build.tag != null || build.branch == 'main'" key: "init" env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" @@ -32,7 +31,6 @@ steps: - wait - label: "🚀 Oceananigans GPU benchmarks" - if: "build.pull_request.labels includes \"benchmark performance\" || build.tag != null || build.branch == 'main'" key: "benchmarks" agents: queue: "Oceananigans-benchmarks" From d0c9b02a981551b20132fed1770048c9bda3ba13 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 16:17:25 +0200 Subject: [PATCH 31/36] same the main build name --- .buildkite/pipeline-benchmarks.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 0309c61262..0c940535cd 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -11,6 +11,14 @@ env: TMPDIR: "$TARTARUS_HOME/tmp" steps: + - label: "save main ID build" # This is required to later retrieve the latest main build ID and compare artifacts + if: "build.branch == 'main'" + command: | + echo "Setting metadata with build ID: $BUILDKITE_BUILD_ID" + buildkite-agent meta-data set "latest-main-build" "$BUILDKITE_BUILD_ID" + agents: + queue: Oceananigans-benchmarks + - label: "🏕️ initialize tartarus environment" key: "init" env: @@ -34,6 +42,7 @@ steps: key: "benchmarks" agents: queue: "Oceananigans-benchmarks" + command: | # Instantiate $TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia --color=yes --project --check-bounds=no -e 'using Pkg; Pkg.instantiate()' From 694c41703c59aa81acfe9b1c3d430f6de33cad29 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 16:26:51 +0200 Subject: [PATCH 32/36] add a condiitonal block --- .buildkite/pipeline-benchmarks.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 0c940535cd..3154c90336 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -11,11 +11,14 @@ env: TMPDIR: "$TARTARUS_HOME/tmp" steps: - - label: "save main ID build" # This is required to later retrieve the latest main build ID and compare artifacts - if: "build.branch == 'main'" + - label: "conditional metadata save" command: | - echo "Setting metadata with build ID: $BUILDKITE_BUILD_ID" - buildkite-agent meta-data set "latest-main-build" "$BUILDKITE_BUILD_ID" + if [ "$BUILDKITE_BRANCH" = "main" ]; then + echo "Saving build ID for main..." + buildkite-agent meta-data set "latest-main-build" "$BUILDKITE_BUILD_ID" + else + echo "Not on main, skipping metadata save." + fi agents: queue: Oceananigans-benchmarks From a28249654533963c0d99549832eb8b7a7f0eba78 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 16:29:56 +0200 Subject: [PATCH 33/36] add a conditional save --- .buildkite/pipeline-benchmarks.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 3154c90336..1ff2caf6cf 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -11,17 +11,6 @@ env: TMPDIR: "$TARTARUS_HOME/tmp" steps: - - label: "conditional metadata save" - command: | - if [ "$BUILDKITE_BRANCH" = "main" ]; then - echo "Saving build ID for main..." - buildkite-agent meta-data set "latest-main-build" "$BUILDKITE_BUILD_ID" - else - echo "Not on main, skipping metadata save." - fi - agents: - queue: Oceananigans-benchmarks - - label: "🏕️ initialize tartarus environment" key: "init" env: @@ -41,6 +30,19 @@ steps: - wait + - label: "💾 conditional metadata save" + key: "save-metadata" + agents: + queue: Oceananigans-benchmarks + + command: | + if [ "$BUILDKITE_BRANCH" = "main" ]; then + echo "Saving build ID for main..." + buildkite-agent meta-data set "latest-main-build" "$BUILDKITE_BUILD_ID" + else + echo "Not on main, skipping metadata save." + fi + - label: "🚀 Oceananigans GPU benchmarks" key: "benchmarks" agents: From d4808f7a1af976233731fa901cc5f6e940434503 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 16:31:32 +0200 Subject: [PATCH 34/36] not working? --- .buildkite/pipeline-benchmarks.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 1ff2caf6cf..23671ff516 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -30,18 +30,18 @@ steps: - wait - - label: "💾 conditional metadata save" - key: "save-metadata" - agents: - queue: Oceananigans-benchmarks + # - label: "💾 conditional metadata save" + # key: "save-metadata" + # agents: + # queue: Oceananigans-benchmarks - command: | - if [ "$BUILDKITE_BRANCH" = "main" ]; then - echo "Saving build ID for main..." - buildkite-agent meta-data set "latest-main-build" "$BUILDKITE_BUILD_ID" - else - echo "Not on main, skipping metadata save." - fi + # command: | + # if [ "$BUILDKITE_BRANCH" = "main" ]; then + # echo "Saving build ID for main..." + # buildkite-agent meta-data set "latest-main-build" "$BUILDKITE_BUILD_ID" + # else + # echo "Not on main, skipping metadata save." + # fi - label: "🚀 Oceananigans GPU benchmarks" key: "benchmarks" From 268c8cc365690b0d646779bb55b41f51d7b42f31 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 16:32:27 +0200 Subject: [PATCH 35/36] conditional metadata save once again --- .buildkite/pipeline-benchmarks.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 23671ff516..1ff2caf6cf 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -30,18 +30,18 @@ steps: - wait - # - label: "💾 conditional metadata save" - # key: "save-metadata" - # agents: - # queue: Oceananigans-benchmarks + - label: "💾 conditional metadata save" + key: "save-metadata" + agents: + queue: Oceananigans-benchmarks - # command: | - # if [ "$BUILDKITE_BRANCH" = "main" ]; then - # echo "Saving build ID for main..." - # buildkite-agent meta-data set "latest-main-build" "$BUILDKITE_BUILD_ID" - # else - # echo "Not on main, skipping metadata save." - # fi + command: | + if [ "$BUILDKITE_BRANCH" = "main" ]; then + echo "Saving build ID for main..." + buildkite-agent meta-data set "latest-main-build" "$BUILDKITE_BUILD_ID" + else + echo "Not on main, skipping metadata save." + fi - label: "🚀 Oceananigans GPU benchmarks" key: "benchmarks" From aaafd2d23a95fb7d0f538c5ddfae15782b76370a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Apr 2025 16:34:34 +0200 Subject: [PATCH 36/36] add a comment --- .buildkite/pipeline-benchmarks.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline-benchmarks.yml b/.buildkite/pipeline-benchmarks.yml index 1ff2caf6cf..f20e4b7281 100644 --- a/.buildkite/pipeline-benchmarks.yml +++ b/.buildkite/pipeline-benchmarks.yml @@ -30,7 +30,9 @@ steps: - wait - - label: "💾 conditional metadata save" + # Saving the latest build ID of the main branch for + # later comparison of the artifacts in PRs + - label: "💾 conditional metadata save" key: "save-metadata" agents: queue: Oceananigans-benchmarks