prestocpp-linux-build-and-unit-test #81
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: prestocpp-linux-build-and-unit-test | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '30 0 * * *' # Runs at 00:30 UTC daily on master(default) branch | |
| pull_request: | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| # Required permissions | |
| permissions: | |
| pull-requests: read | |
| # Set job outputs to values from filter step | |
| outputs: | |
| codechange: ${{ steps.filter.outputs.codechange }} | |
| steps: | |
| # For pull requests it's not necessary to checkout the code | |
| - name: Run changes check for PRs | |
| if: github.event_name != 'schedule' | |
| uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| codechange: | |
| - '!presto-docs/**' | |
| prestocpp-linux-build-for-test: | |
| runs-on: ubuntu-22.04 | |
| needs: changes | |
| container: | |
| image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e | |
| concurrency: | |
| group: ${{ github.workflow }}-prestocpp-linux-build-test-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| env: | |
| CCACHE_DIR: "${{ github.workspace }}/ccache" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| - name: Fix git permissions | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| # Usually actions/checkout does this but as we run in a container | |
| # it doesn't work | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Update velox | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| cd presto-native-execution | |
| make velox-submodule | |
| - name: Install Github CLI for using apache/infrastructure-actions/stash | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm | |
| rpm -iv gh_2.63.2_linux_amd64.rpm | |
| - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| with: | |
| path: '${{ env.CCACHE_DIR }}' | |
| key: ccache-prestocpp-linux-build-for-test | |
| - name: Zero ccache statistics | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: ccache -sz | |
| - name: Build engine | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| source /opt/rh/gcc-toolset-12/enable | |
| cd presto-native-execution | |
| cmake \ | |
| -B _build/release \ | |
| -GNinja \ | |
| -DTREAT_WARNINGS_AS_ERRORS=1 \ | |
| -DENABLE_ALL_WARNINGS=1 \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ | |
| -DPRESTO_ENABLE_JWT=ON \ | |
| -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ | |
| -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ | |
| -DCMAKE_PREFIX_PATH=/usr/local \ | |
| -DThrift_ROOT=/usr/local \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -DMAX_LINK_JOBS=4 | |
| ninja -C _build/release -j 4 | |
| - name: Ccache after | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: ccache -s | |
| - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| with: | |
| path: '${{ env.CCACHE_DIR }}' | |
| key: ccache-prestocpp-linux-build-for-test | |
| - name: Run Unit Tests | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| # Ensure transitive dependency libboost-iostreams is found. | |
| ldconfig /usr/local/lib | |
| cd presto-native-execution/_build/release | |
| ctest -j 4 -VV --output-on-failure --exclude-regex ^velox.* | |
| - name: Upload artifacts | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: presto-native-build | |
| path: | | |
| presto-native-execution/_build/release/presto_cpp/main/presto_server | |
| presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main | |
| prestocpp-linux-presto-e2e-tests: | |
| needs: [changes, prestocpp-linux-build-for-test] | |
| runs-on: ubuntu-22.04 | |
| container: | |
| image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e | |
| env: | |
| MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" | |
| MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| - name: Fix git permissions | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| # Usually actions/checkout does this but as we run in a container | |
| # it doesn't work | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Download artifacts | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: presto-native-build | |
| path: presto-native-execution/_build/release | |
| # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 | |
| - name: Restore execute permissions and library path | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main | |
| # Ensure transitive dependency libboost-iostreams is found. | |
| ldconfig /usr/local/lib | |
| - name: Install OpenJDK8 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17.0.15 | |
| cache: 'maven' | |
| - name: Download nodejs to maven cache | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: .github/bin/download_nodejs | |
| - name: Maven install | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| env: | |
| # Use different Maven options to install. | |
| MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" | |
| run: | | |
| for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) | |
| - name: Run presto-native e2e tests | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" | |
| export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'` | |
| # Convert file paths to comma separated class names | |
| export TESTCLASSES= | |
| for test_file in $TESTFILES | |
| do | |
| tmp=${test_file##*/} | |
| test_class=${tmp%%\.*} | |
| export TESTCLASSES="${TESTCLASSES},$test_class" | |
| done | |
| export TESTCLASSES=${TESTCLASSES#,} | |
| echo "TESTCLASSES = $TESTCLASSES" | |
| # TODO: neeed to enable remote function tests with | |
| # "-Ppresto-native-execution-remote-functions" once | |
| # > https://github.com/facebookincubator/velox/discussions/6163 | |
| # is fixed. | |
| mvn test \ | |
| ${MAVEN_TEST} \ | |
| -pl 'presto-native-execution' \ | |
| -Dtest="${TESTCLASSES}" \ | |
| -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ | |
| -DDATA_DIR=${RUNNER_TEMP} \ | |
| -Duser.timezone=America/Bahia_Banderas \ | |
| -T1C | |
| prestocpp-linux-presto-native-tests: | |
| needs: [changes, prestocpp-linux-build-for-test] | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| storage-format: [ "PARQUET", "DWRF" ] | |
| enable-sidecar: [ "true", "false" ] | |
| container: | |
| image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e | |
| env: | |
| MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" | |
| MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| - name: Fix git permissions | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| # Usually actions/checkout does this but as we run in a container | |
| # it doesn't work | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Download artifacts | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: presto-native-build | |
| path: presto-native-execution/_build/release | |
| # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 | |
| - name: Restore execute permissions and library path | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main | |
| # Ensure transitive dependency libboost-iostreams is found. | |
| ldconfig /usr/local/lib | |
| - name: Install OpenJDK8 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17.0.15 | |
| cache: 'maven' | |
| - name: Download nodejs to maven cache | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: .github/bin/download_nodejs | |
| - name: Maven install | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| env: | |
| # Use different Maven options to install. | |
| MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" | |
| run: | | |
| for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-tests' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) | |
| - name: Run presto-native tests | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" | |
| export TESTFILES=`find ./presto-native-tests/src/test -type f -name 'Test*.java'` | |
| # Convert file paths to comma separated class names | |
| export TESTCLASSES= | |
| for test_file in $TESTFILES | |
| do | |
| tmp=${test_file##*/} | |
| test_class=${tmp%%\.*} | |
| export TESTCLASSES="${TESTCLASSES},$test_class" | |
| done | |
| export TESTCLASSES=${TESTCLASSES#,} | |
| echo "TESTCLASSES = $TESTCLASSES" | |
| mvn test \ | |
| ${MAVEN_TEST} \ | |
| -pl 'presto-native-tests' \ | |
| -DstorageFormat=${{ matrix.storage-format }} \ | |
| -DsidecarEnabled=${{ matrix.enable-sidecar }} \ | |
| -Dtest="${TESTCLASSES}" \ | |
| -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ | |
| -DDATA_DIR=${RUNNER_TEMP} \ | |
| -Duser.timezone=America/Bahia_Banderas \ | |
| -T1C | |
| prestocpp-linux-presto-sidecar-tests: | |
| needs: [changes, prestocpp-linux-build-for-test] | |
| runs-on: ubuntu-22.04 | |
| container: | |
| image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e | |
| env: | |
| MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" | |
| MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| - name: Fix git permissions | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| # Usually actions/checkout does this but as we run in a container | |
| # it doesn't work | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Download artifacts | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: presto-native-build | |
| path: presto-native-execution/_build/release | |
| # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 | |
| - name: Restore execute permissions and library path | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main | |
| # Ensure transitive dependency libboost-iostreams is found. | |
| ldconfig /usr/local/lib | |
| - name: Install OpenJDK8 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17.0.15 | |
| cache: 'maven' | |
| - name: Download nodejs to maven cache | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: .github/bin/download_nodejs | |
| - name: Maven install | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| env: | |
| # Use different Maven options to install. | |
| MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" | |
| run: | | |
| for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) | |
| - name: Run presto-native sidecar tests | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" | |
| export TESTFILES=`find ./presto-native-sidecar-plugin/src/test -type f -name 'Test*.java'` | |
| # Convert file paths to comma separated class names | |
| export TESTCLASSES= | |
| for test_file in $TESTFILES | |
| do | |
| tmp=${test_file##*/} | |
| test_class=${tmp%%\.*} | |
| export TESTCLASSES="${TESTCLASSES},$test_class" | |
| done | |
| export TESTCLASSES=${TESTCLASSES#,} | |
| echo "TESTCLASSES = $TESTCLASSES" | |
| mvn test \ | |
| ${MAVEN_TEST} \ | |
| -pl 'presto-native-sidecar-plugin' \ | |
| -Dtest="${TESTCLASSES}" \ | |
| -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ | |
| -DDATA_DIR=${RUNNER_TEMP} \ | |
| -Duser.timezone=America/Bahia_Banderas \ | |
| -T1C | |
| prestocpp-linux-presto-plan-checker-router-plugin-tests: | |
| needs: [changes, prestocpp-linux-build-for-test] | |
| runs-on: ubuntu-22.04 | |
| container: | |
| image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e | |
| env: | |
| MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" | |
| MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| - name: Fix git permissions | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| # Usually actions/checkout does this but as we run in a container | |
| # it doesn't work | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Download artifacts | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: presto-native-build | |
| path: presto-native-execution/_build/release | |
| # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 | |
| - name: Restore execute permissions and library path | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main | |
| # Ensure transitive dependency libboost-iostreams is found. | |
| ldconfig /usr/local/lib | |
| - name: Install OpenJDK8 | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17.0.15 | |
| cache: 'maven' | |
| - name: Download nodejs to maven cache | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: .github/bin/download_nodejs | |
| - name: Maven install | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| env: | |
| # Use different Maven options to install. | |
| MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" | |
| run: | | |
| for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-plan-checker-router-plugin' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) | |
| - name: Run presto-native plan checker router plugin tests | |
| if: | | |
| github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' | |
| run: | | |
| export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" | |
| mvn test \ | |
| ${MAVEN_TEST} \ | |
| -pl 'presto-plan-checker-router-plugin' \ | |
| -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ | |
| -DDATA_DIR=${RUNNER_TEMP} \ | |
| -Duser.timezone=America/Bahia_Banderas \ | |
| -T1C |