Merge branch 'develop' into 2652-gpu-data-compaction

Author: m09526

.devcontainer/Sleeper/devcontainer.json

@@ -48,6 +48,7 @@
     "AWS_SECRET_ACCESS_KEY": "${localEnv:AWS_SECRET_ACCESS_KEY}",
     "AWS_SESSION_TOKEN": "${localEnv:AWS_SESSION_TOKEN}"
   },
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.aws ; mkdir -p ${localEnv:HOME}/.m2 ; mkdir -p ${localEnv:HOME}/.ssh",
   "postStartCommand": "head -1 /proc/self/cgroup | cut -d/ -f3 | echo \"HOSTNAME=$(cat -)\" >> /home/sleeper/.bashrc",
   "remoteUser": "sleeper",
   "updateRemoteUserUID": true

.github/config/chunks.yaml

@@ -28,11 +28,13 @@ chunks:
       - build
       - common/dynamodb-tools
       - common/dynamodb-test
+      - common/localstack-test
       - statestore
       - statestore-committer-core
       - statestore-lambda
       - statestore
       - metrics
+      - example-iterators
   compaction:
     name: Compaction
     workflow: chunk-compaction.yaml
@@ -41,7 +43,7 @@ chunks:
       - compaction/compaction-task-creation
       - compaction/compaction-job-creation
       - compaction/compaction-job-creation-lambda
-      - compaction/compaction-status-store
+      - compaction/compaction-tracker
       - compaction/compaction-core
       - splitter/splitter-core
       - splitter/splitter-lambda
@@ -64,7 +66,7 @@ chunks:
     workflow: chunk-ingest.yaml
     modules:
       - ingest/ingest-core
-      - ingest/ingest-status-store
+      - ingest/ingest-tracker
       - ingest/ingest-starter
       - ingest/ingest-batcher-core
       - ingest/ingest-batcher-store

.github/workflows/chunk-clients-cdk.yaml

@@ -22,31 +22,33 @@ on:
       - 'java/system-test/system-test-suite/**'
       - 'java/bulk-import/bulk-import-starter/**'
       - 'java/ingest/ingest-batcher-submitter/**'
+      - 'java/compaction/compaction-job-creation/**'
       - 'java/common/common-task/**'
+      - 'java/query/query-runner/**'
       - 'java/ingest/ingest-batcher-store/**'
       - 'java/bulk-import/bulk-import-core/**'
       - 'java/splitter/splitter-core/**'
-      - 'java/compaction/compaction-job-creation/**'
       - 'java/compaction/compaction-job-execution/**'
       - 'java/statestore-committer-core/**'
       - 'java/garbage-collector/**'
       - 'java/ingest/ingest-batcher-core/**'
-      - 'java/query/query-runner/**'
+      - 'java/query/query-core/**'
       - 'java/compaction/compaction-rust/**'
-      - 'java/compaction/compaction-status-store/**'
+      - 'java/compaction/compaction-tracker/**'
       - 'java/common/common-invoke-tables/**'
-      - 'java/query/query-core/**'
       - 'java/ingest/ingest-runner/**'
       - 'java/compaction/compaction-core/**'
       - 'java/statestore/**'
       - 'java/common/common-job/**'
-      - 'java/ingest/ingest-status-store/**'
+      - 'java/ingest/ingest-tracker/**'
       - 'java/sketches/**'
       - 'java/parquet/**'
-      - 'java/ingest/ingest-core/**'
       - 'java/configuration/**'
+      - 'java/ingest/ingest-core/**'
       - 'java/common/dynamodb-tools/**'
+      - 'java/common/localstack-test/**'
       - 'java/core/**'
+      - 'java/example-iterators/**'
       - 'java/common/dynamodb-test/**'
 
 jobs:

.github/workflows/chunk-common.yaml

@@ -10,23 +10,26 @@ on:
       - 'java/pom.xml'
       - 'java/common/pom.xml'
       - 'java/core/**'
+      - 'java/example-iterators/**'
       - 'java/configuration/**'
       - 'java/sketches/**'
       - 'java/parquet/**'
-      - 'java/statestore/**'
-      - 'java/statestore-committer-core/**'
-      - 'java/statestore-lambda/**'
-      - 'java/metrics/**'
-      - 'java/compaction/compaction-status-store/**'
-      - 'java/compaction/compaction-core/**'
-      - 'java/ingest/ingest-status-store/**'
-      - 'java/ingest/ingest-core/**'
       - 'java/common/common-job/**'
       - 'java/common/common-task/**'
       - 'java/common/common-invoke-tables/**'
       - 'java/build/**'
       - 'java/common/dynamodb-tools/**'
       - 'java/common/dynamodb-test/**'
+      - 'java/common/localstack-test/**'
+      - 'java/statestore/**'
+      - 'java/statestore-committer-core/**'
+      - 'java/statestore-lambda/**'
+      - 'java/statestore/**'
+      - 'java/metrics/**'
+      - 'java/compaction/compaction-tracker/**'
+      - 'java/compaction/compaction-core/**'
+      - 'java/ingest/ingest-tracker/**'
+      - 'java/ingest/ingest-core/**'
 
 jobs:
   chunk-workflow:

.github/workflows/chunk-compaction.yaml

@@ -10,35 +10,37 @@ on:
       - 'java/pom.xml'
       - 'java/compaction/pom.xml'
       - 'java/splitter/pom.xml'
+      - 'java/bulk-export/pom.xml'
       - 'java/compaction/compaction-job-execution/**'
       - 'java/compaction/compaction-task-creation/**'
       - 'java/compaction/compaction-rust/**'
       - 'java/compaction/compaction-gpu/**'
       - 'java/compaction/compaction-job-creation/**'
       - 'java/compaction/compaction-job-creation-lambda/**'
-      - 'java/compaction/compaction-status-store/**'
+      - 'java/compaction/compaction-tracker/**'
       - 'java/compaction/compaction-core/**'
       - 'java/splitter/splitter-core/**'
       - 'java/splitter/splitter-lambda/**'
-      - 'java/compaction/compaction-rust/**'
       - 'java/garbage-collector/**'
+      - 'java/bulk-export/bulk-export-core/**'
+      - 'java/bulk-export/bulk-export-runner/**'
+      - 'java/bulk-export/bulk-export-starter/**'
+      - 'java/compaction/compaction-rust/**'
       - 'java/common/common-task/**'
-      - 'java/common/common-invoke-tables/**'
       - 'java/ingest/ingest-runner/**'
       - 'java/common/common-job/**'
-      - 'java/ingest/ingest-status-store/**'
-      - 'java/ingest/ingest-core/**'
+      - 'java/ingest/ingest-tracker/**'
       - 'java/sketches/**'
+      - 'java/ingest/ingest-core/**'
       - 'java/statestore/**'
-      - 'java/parquet/**'
+      - 'java/common/common-invoke-tables/**'
       - 'java/configuration/**'
       - 'java/common/dynamodb-tools/**'
-      - 'java/core/**'
       - 'java/common/dynamodb-test/**'
-      - 'java/bulk-export/pom.xml'
-      - 'java/bulk-export/bulk-export-core/**'
-      - 'java/bulk-export/bulk-export-runner/**'
-      - 'java/bulk-export/bulk-export-starter/**'
+      - 'java/parquet/**'
+      - 'java/common/localstack-test/**'
+      - 'java/example-iterators/**'
+      - 'java/core/**'
 
 jobs:
   chunk-workflow:

.github/workflows/chunk-ingest.yaml

@@ -9,27 +9,29 @@ on:
       - 'code-style/spotbugs*.xml'
       - 'java/pom.xml'
       - 'java/ingest/pom.xml'
+      - 'java/bulk-import/pom.xml'
       - 'java/ingest/ingest-core/**'
-      - 'java/ingest/ingest-status-store/**'
+      - 'java/ingest/ingest-tracker/**'
       - 'java/ingest/ingest-starter/**'
       - 'java/ingest/ingest-batcher-core/**'
       - 'java/ingest/ingest-batcher-store/**'
       - 'java/ingest/ingest-batcher-submitter/**'
       - 'java/ingest/ingest-batcher-job-creator/**'
-      - 'java/ingest/ingest-runner/**'
-      - 'java/bulk-import/pom.xml'
       - 'java/bulk-import/bulk-import-core/**'
       - 'java/bulk-import/bulk-import-starter/**'
       - 'java/bulk-import/bulk-import-runner/**'
+      - 'java/ingest/ingest-runner/**'
       - 'java/common/common-task/**'
-      - 'java/common/common-job/**'
-      - 'java/sketches/**'
       - 'java/statestore/**'
-      - 'java/parquet/**'
+      - 'java/common/common-job/**'
       - 'java/configuration/**'
       - 'java/common/dynamodb-tools/**'
-      - 'java/core/**'
       - 'java/common/dynamodb-test/**'
+      - 'java/parquet/**'
+      - 'java/common/localstack-test/**'
+      - 'java/sketches/**'
+      - 'java/example-iterators/**'
+      - 'java/core/**'
 
 jobs:
   chunk-workflow:

.github/workflows/chunk-query.yaml

@@ -8,24 +8,26 @@ on:
       - 'code-style/checkstyle*.xml'
       - 'code-style/spotbugs*.xml'
       - 'java/pom.xml'
-      - 'java/athena/**'
-      - 'java/trino/**'
       - 'java/query/pom.xml'
       - 'java/query/query-core/**'
       - 'java/query/query-runner/**'
       - 'java/query/query-lambda/**'
+      - 'java/athena/**'
+      - 'java/trino/**'
       - 'java/splitter/splitter-core/**'
       - 'java/ingest/ingest-runner/**'
+      - 'java/statestore/**'
       - 'java/common/common-job/**'
-      - 'java/ingest/ingest-status-store/**'
+      - 'java/ingest/ingest-tracker/**'
       - 'java/sketches/**'
-      - 'java/statestore/**'
-      - 'java/ingest/ingest-core/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
+      - 'java/ingest/ingest-core/**'
       - 'java/common/dynamodb-tools/**'
+      - 'java/example-iterators/**'
       - 'java/core/**'
       - 'java/common/dynamodb-test/**'
+      - 'java/common/localstack-test/**'
 
 jobs:
   chunk-workflow:

.github/workflows/chunk-rust.yaml

@@ -12,17 +12,13 @@ on:
       - 'java/compaction/pom.xml'
       - 'java/compaction/compaction-rust/**'
       - 'java/compaction/compaction-core/**'
-      - 'java/ingest/ingest-runner/**'
-      - 'java/common/common-job/**'
-      - 'java/ingest/ingest-status-store/**'
       - 'java/sketches/**'
-      - 'java/statestore/**'
-      - 'java/ingest/ingest-core/**'
       - 'java/parquet/**'
-      - 'java/configuration/**'
       - 'java/common/dynamodb-tools/**'
       - 'java/core/**'
+      - 'java/example-iterators/**'
       - 'java/common/dynamodb-test/**'
+      - 'java/common/localstack-test/**'
 
 jobs:
   chunk-workflow:

.github/workflows/dependency-check.yaml

@@ -31,25 +31,21 @@ jobs:
       - name: Resolve dependencies
         run: mvn de.qaware.maven:go-offline-maven-plugin:resolve-dependencies -Dmaven.repo.local=${{ runner.temp }}/.m2/repository
         working-directory: ./java
-      - name: Explain disabled workflow
-        run: |
-          echo "Temporarily disabled checks as the NVD API is unavailable."
-          echo "See the following issue: https://github.com/gchq/sleeper/issues/3811"
-      # - name: Update CVEs database
-      #   run: mvn --batch-mode dependency-check:update-only -Dmaven.repo.local=${{ runner.temp }}/.m2/repository
-      #   working-directory: ./java
-      # - name: Build with Maven
-      #   run: mvn --batch-mode verify dependency-check:aggregate -Pquick -DskipRust -Dmaven.repo.local=${{ runner.temp }}/.m2/repository
-      #   working-directory: ./java
-      # - name: Cache Maven dependencies & CVEs database
-      #   uses: actions/cache/save@v3
-      #   if: ${{ always() }}
-      #   with:
-      #     path: ${{ runner.temp }}/.m2/repository
-      #     key: ${{ steps.restore-cache.outputs.cache-primary-key }}
-      # - name: Upload dependency check report
-      #   if: ${{ always() }}
-      #   uses: actions/upload-artifact@v4
-      #   with:
-      #     name: Dependency check report
-      #     path: java/target/dependency-check-report.html
+      - name: Update CVEs database
+        run: mvn --batch-mode dependency-check:update-only -Dmaven.repo.local=${{ runner.temp }}/.m2/repository
+        working-directory: ./java
+      - name: Build with Maven
+        run: mvn --batch-mode verify dependency-check:aggregate -Pquick -DskipRust -Dmaven.repo.local=${{ runner.temp }}/.m2/repository
+        working-directory: ./java
+      - name: Cache Maven dependencies & CVEs database
+        uses: actions/cache/save@v3
+        if: ${{ always() }}
+        with:
+          path: ${{ runner.temp }}/.m2/repository
+          key: ${{ steps.restore-cache.outputs.cache-primary-key }}
+      - name: Upload dependency check report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: Dependency check report
+          path: java/target/dependency-check-report.html

.gitignore

@@ -31,3 +31,4 @@ conan-cache/
 scripts/test/deployAll/system-test-instance.properties
 scripts/test/maven/system-test-instance.properties
 input.json
+.approval_tests_temp/