Speedup conformance tests execution by moving the build step to the host machine (#180)

Author: igor-vovk

.dockerignore

@@ -2,5 +2,3 @@
 .github/
 .idea/
 out/
-target/
-**/target/

.github/workflows/scala.yml

@@ -31,29 +31,8 @@ jobs:
       - name: Run tests
         run: sbt test
 
-  conformance:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-
-    strategy:
-      matrix:
-        config:
-          - name: Netty Server
-            profile: netty-server
-          - name: Http4s Server
-            profile: http4s-server
-          - name: Http4s Server (non-stable)
-            profile: http4s-server-nonstable
-          - name: Http4s Client
-            profile: http4s-client
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Run conformance tests for ${{ matrix.config.name }}
-        run: make test-conformance PROFILE=${{ matrix.config.profile }}
+      - name: Run conformance tests
+        run: make test-conformance-stable
 
   dependency-graph:
     name: Update Dependency Graph

Makefile

@@ -1,19 +1,35 @@
-.PHONY: test-conformance
-
 # Define common variables
-DOCKER_BUILD_CMD = docker build -f build/conformance/Dockerfile . --progress=plain
+DOCKER_BUILD_CMD = docker build -f conformance-build/Dockerfile . --progress=plain --output out/
+
+.PHONY: build-conformance
+build-conformance:
+	sbt conformance/stage
+
+.PHONY: test-conformance-http4s-server
+test-conformance-http4s-server: build-conformance
+	@echo "Running conformance tests for http4s server"
+	$(DOCKER_BUILD_CMD) --build-arg launcher=Http4sServerLauncher --build-arg config=suite-http4s.yaml --build-arg parallel_args="--parallel 1"
+	@code=$$(cat out/exit_code | tr -d '\n'); echo "Exiting with code: $$code"; exit $$code
 
-test-conformance:
-	@echo "Running conformance tests with profile: $(PROFILE)"
-ifeq ($(PROFILE),netty-server)
-	$(DOCKER_BUILD_CMD) --build-arg launcher=NettyServerLauncher --build-arg config=suite-netty.yaml
-else ifeq ($(PROFILE),http4s-server)
-	$(DOCKER_BUILD_CMD) --build-arg launcher=Http4sServerLauncher --build-arg config=suite-http4s.yaml
-else ifeq ($(PROFILE),http4s-server-nonstable)
-	$(DOCKER_BUILD_CMD) --build-arg launcher=Http4sServerLauncher --build-arg config=suite-http4s-nonstable.yaml --build-arg stable=false
-else ifeq ($(PROFILE),http4s-client)
+.PHONY: test-conformance-http4s-server-nonstable
+test-conformance-http4s-server-nonstable: build-conformance
+	@echo "Running conformance tests for http4s server (non-stable)"
+	$(DOCKER_BUILD_CMD) --build-arg launcher=Http4sServerLauncher --build-arg config=suite-http4s-nonstable.yaml --build-arg parallel_args="--parallel 1"
+	@code=$$(cat out/exit_code | tr -d '\n'); echo "Exiting with code: $$code"; exit $$code
+
+.PHONY: test-conformance-http4s-client
+test-conformance-http4s-client: build-conformance
+	@echo "Running conformance tests for http4s client"
 	$(DOCKER_BUILD_CMD) --build-arg launcher=Http4sClientLauncher --build-arg config=suite-http4s-client.yaml --build-arg mode=client
-else
-	@echo "Error: Unknown profile '$(PROFILE)'. Supported profiles: netty-server, http4s-server, http4s-server-nonstable, http4s-client."
-	@exit 1
-endif
+	@code=$$(cat out/exit_code | tr -d '\n'); echo "Exiting with code: $$code"; exit $$code
+
+.PHONY: test-conformance-netty-server
+test-conformance-netty-server: build-conformance
+	@echo "Running conformance tests for netty server"
+	$(DOCKER_BUILD_CMD) --build-arg launcher=NettyServerLauncher --build-arg config=suite-netty.yaml --build-arg parallel_args="--parallel 1"
+	@code=$$(cat out/exit_code | tr -d '\n'); echo "Exiting with code: $$code"; exit $$code
+
+.PHONY: test-conformance-stable
+test-conformance-stable: test-conformance-http4s-server \
+						 test-conformance-http4s-client \
+						 test-conformance-netty-server

README.md

@@ -259,16 +259,8 @@ How-tos that go beyond the basic usage:
 
 Run the following command to run Connect-RPC conformance tests:
 
-For the Netty server:
-
-```shell
-make test-conformance PROFILE=netty-server
-```
-
-For the Http4s server:
-
 ```shell
-make test-conformance PROFILE=http4s-server
+make test-conformance-stable
 ```
 
 Execution results are output to STDOUT.

build/conformance/.gitignore

@@ -0,0 +1,31 @@
+FROM eclipse-temurin:24-jre AS base
+
+FROM ghcr.io/igor-vovk/connect-conformance-dockerimage:1.0.4-1 AS conformance
+
+FROM base AS runner
+
+COPY --from=conformance /conformance /conformance
+ADD conformance-build/*.yaml /conformance/
+ADD conformance/target/universal/stage /app
+WORKDIR /conformance
+
+RUN mkdir "/out"
+
+ARG config="suite.yaml"
+ARG mode="server"
+ARG launcher
+ARG parallel_args=""
+
+RUN LOGS_PATH="/out" \
+    ./connectconformance \
+    --conf $config \
+    --mode $mode \
+    $parallel_args \
+    -v -vv --trace \
+    -- \
+    /app/bin/conformance -main org.ivovk.connect_rpc_scala.conformance.$launcher -Dcats.effect.warnOnNonMainThreadDetected=false; \
+    echo $? > /out/exit_code
+
+FROM scratch AS export
+
+COPY --link --from=runner /out /