Merge pull request #4298 from ethereum/dev

Release v1.5.0

Author: GabrielAstieres

.github/workflows/docs.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Build docs
         run: make _copy_docs
-      - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: 3.x
       - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3

.github/workflows/generate_vectors.yml

@@ -27,7 +27,7 @@ jobs:
           path: 'consensus-specs'
           ref: ${{ inputs.ref || 'dev' }}
       - name: Setup Python
-        uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: '3.12.4'
           cache: ''

.github/workflows/nightly-tests.yml

@@ -0,0 +1,43 @@
+name: Nightly mainnet tests
+
+defaults:
+  run:
+    shell: zsh {0}
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: The branch, tag or SHA to checkout and build from
+        default: dev
+        type: string
+        required: true
+  schedule:
+    # Every day at 00:00 UTC
+    - cron: '0 0 * * *'
+
+jobs:
+  tests:
+    runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
+    strategy:
+      fail-fast: false
+      matrix:
+        fork:
+          - phase0
+          - altair
+          - bellatrix
+          - capella
+          - deneb
+          - electra
+          - fulu
+          - eip7441
+          - eip7732
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Setup Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.12'
+      - name: test-${{ matrix.fork }}
+        run: make test preset=mainnet fork=${{ matrix.fork }}

.github/workflows/run-tests.yml

@@ -1,46 +1,32 @@
-name: Run spec tests and linter
+name: Run tests
 
 defaults:
   run:
     shell: zsh {0}
 
-env:
-  TEST_PRESET_TYPE: "minimal"
-
 on:
   push:
-    branches:
-      - dev
-      - master
+    branches: [dev, master]
   pull_request:
-  workflow_dispatch:
-    inputs:
-      test_preset_type:
-        default: minimal
-        description: Type of test to run, either mainnet or minimal
-        type: string
-        required: true
-  schedule:
-    - cron: '0 0 * * *'
 
 jobs:
   lint:
-    runs-on: [self-hosted-ghr-custom, size-l-x64, profile-consensusSpecs]
+    runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
     steps:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Setup Python
-        uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
+      - name: Setup python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: '3.12.4'
-          cache: ''
+          python-version: '3.12'
+          cache: 'pip'
       - name: Run linter for pyspec
         run: |
           make lint
           git diff --exit-code
 
   whitespace:
-    runs-on: [self-hosted-ghr-custom, size-l-x64, profile-consensusSpecs]
+    runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
     steps:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -51,56 +37,43 @@ jobs:
             exit 1
           fi
 
-  pyspec-tests:
+  tests:
+    needs: [lint, whitespace]
     runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs]
-    needs: [lint]
     strategy:
       matrix:
-        version: ["phase0", "altair", "bellatrix", "capella", "deneb", "electra", "fulu", "eip7441", "eip7732"]
+        fork:
+          - phase0
+          - altair
+          - bellatrix
+          - capella
+          - deneb
+          - electra
+          - fulu
+          - eip7441
+          - eip7732
     steps:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Setup Rust for dependencies
-        uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0
-      - name: Setup Python
-        uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
+      - name: Setup python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: '3.12.4'
-          cache: ''
-      - name: set preset
-        if: github.event.inputs.test_preset_type != ''
-        run: |
-          echo "spec_test_preset_type=${{ github.event.inputs.test_preset_type || env.TEST_PRESET_TYPE }}" >> $GITHUB_ENV
-      - name: set preset
-        if: ${{ (github.event_name == 'push' && github.ref_name != 'master') || github.event_name == 'pull_request' }}
-        run: |
-          echo "spec_test_preset_type=${{ env.TEST_PRESET_TYPE }}" >> $GITHUB_ENV
-      - name: set preset
-        if: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
-        run: |
-          echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV
-      - name: set preset
-        if: github.event.schedule=='0 0 * * *'
-        run: |
-          echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV
-      - name: test-${{ matrix.version }}
-        run: make test fork=${{ matrix.version }} preset=${{ env.spec_test_preset_type }}
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        if: always()
-        with:
-          name: test-reports-${{ matrix.version }}
-          path: tests/core/pyspec/test-reports
+          python-version: '3.12'
+          cache: 'pip'
+      - name: Run pyspec tests for ${{ matrix.fork }}
+        run: make test preset=minimal fork=${{ matrix.fork }}
 
-  gen-modcheck:
-   runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
-   steps:
+  modcheck:
+    needs: [lint, whitespace]
+    runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
+    steps:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Setup Python
-        uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
+      - name: Setup python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: '3.12.4'
-          cache: ''
+          python-version: '3.12'
+          cache: 'pip'
       - name: Run generators with --modcheck
         run: make gen_all modcheck=true 2>&1 | tee consensustestgen.log
       - name: Check for errors

Makefile

@@ -89,7 +89,6 @@ pyspec: $(VENV) setup.py pyproject.toml
 TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
 
 # Run pyspec tests.
-# Note: for debugging output to show, print to stderr.
 #
 # To run a specific test, append k=<test>, eg:
 #   make test k=test_verify_kzg_proof
@@ -102,13 +101,16 @@ TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
 # To run tests with a specific bls library, append bls=<bls>, eg:
 #   make test bls=arkworks
 test: MAYBE_TEST := $(if $(k),-k=$(k))
+# Disable parallelism which running a specific test.
+# Parallelism makes debugging difficult (print doesn't work).
+test: MAYBE_PARALLEL := $(if $(k),,-n auto)
 test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
 test: PRESET := --preset=$(if $(preset),$(preset),minimal)
 test: BLS := --bls-type=$(if $(bls),$(bls),fastest)
 test: pyspec
 	@mkdir -p $(TEST_REPORT_DIR)
 	@$(PYTHON_VENV) -m pytest \
-		-n auto \
+		$(MAYBE_PARALLEL) \
 		--capture=no \
 		$(MAYBE_TEST) \
 		$(MAYBE_FORK) \
@@ -178,7 +180,8 @@ PYLINT_CONFIG = $(CURDIR)/pylint.ini
 
 PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), $(PYSPEC_DIR)/eth2spec/$S)
 MYPY_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), -p eth2spec.$S)
-MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
+MARKDOWN_FILES = $(CURDIR)/README.md \
+                 $(wildcard $(SPEC_DIR)/*/*.md) \
                  $(wildcard $(SPEC_DIR)/*/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
@@ -202,6 +205,7 @@ SCRIPTS_DIR = $(CURDIR)/scripts
 GENERATOR_ERROR_LOG_FILE = $(TEST_VECTOR_DIR)/testgen_error_log.txt
 GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
 GENERATOR_TARGETS = $(patsubst $(GENERATOR_DIR)/%/, gen_%, $(GENERATORS))
+COMMA:= ,
 
 # List available generators.
 gen_list:
@@ -211,14 +215,34 @@ gen_list:
 
 # Run one generator.
 # This will forcibly rebuild pyspec just in case.
+# To print more details, append verbose=true, eg:
+#   make gen_bls verbose=true
 # To check modules for a generator, append modcheck=true, eg:
 #   make gen_genesis modcheck=true
+# To run the generator for a specific test, append k=<test>, eg:
+#   make gen_operations k=invalid_committee_index
+# To run the generator for a specific fork, append fork=<fork>, eg:
+#   make gen_operations fork=fulu
+# To run the generator for a specific preset, append preset=<preset>, eg:
+#   make gen_operations preset=mainnet
+# To run the generator for a list of tests, forks, and/or presets, append them as comma-separated lists, eg:
+#   make gen_operations k=invalid_committee_index,invalid_too_many_committee_bits
+# Or all at the same time, eg:
+#   make gen_operations preset=mainnet fork=fulu k=invalid_committee_index
+gen_%: MAYBE_VERBOSE := $(if $(filter true,$(verbose)),--verbose)
 gen_%: MAYBE_MODCHECK := $(if $(filter true,$(modcheck)),--modcheck)
+gen_%: MAYBE_TESTS := $(if $(k),--case-list $(subst ${COMMA}, ,$(k)))
+gen_%: MAYBE_FORKS := $(if $(fork),--fork-list $(subst ${COMMA}, ,$(fork)))
+gen_%: MAYBE_PRESETS := $(if $(preset),--preset-list $(subst ${COMMA}, ,$(preset)))
 gen_%: pyspec
 	@mkdir -p $(TEST_VECTOR_DIR)
 	@$(PYTHON_VENV) $(GENERATOR_DIR)/$*/main.py \
 		--output $(TEST_VECTOR_DIR) \
-		$(MAYBE_MODCHECK)
+		$(MAYBE_VERBOSE) \
+		$(MAYBE_MODCHECK) \
+		$(MAYBE_TESTS) \
+		$(MAYBE_FORKS) \
+		$(MAYBE_PRESETS)
 
 # Run all generators then check for errors.
 gen_all: $(GENERATOR_TARGETS)