Fastchess SPRT Test (Linux) #118
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fastchess SPRT Test (Linux) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| base_ref: | |
| description: "Base branch, tag, or commit" | |
| required: false | |
| default: "HandcraftedEngine" | |
| rounds: | |
| description: "Max rounds for SPRT" | |
| required: false | |
| default: "6000" | |
| tc: | |
| description: "Time control (e.g. 60+0.6 or 10+0.1)" | |
| required: false | |
| default: "10+0.1" | |
| elo0: | |
| description: "elo0" | |
| required: false | |
| default: "0.0" | |
| elo1: | |
| description: "elo1" | |
| required: false | |
| default: "2.0" | |
| output_exec: | |
| description: "Executable output file name" | |
| required: false | |
| default: "engine" | |
| permissions: | |
| contents: read | |
| jobs: | |
| build-new: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| path: new | |
| persist-credentials: false | |
| - name: Install deps | |
| run: sudo apt update && sudo apt install -y build-essential cmake | |
| - name: Build new engine | |
| run: | | |
| cd new | |
| mkdir build | |
| cd build | |
| cmake .. -DCMAKE_BUILD_TYPE=Release | |
| make -j$(nproc) | |
| - name: Upload new engine | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: new-engine | |
| path: new/build/engine | |
| build-base: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event.inputs.base_ref }} | |
| path: base | |
| persist-credentials: false | |
| - name: Install deps | |
| run: sudo apt update && sudo apt install -y build-essential cmake | |
| - name: Build base engine | |
| run: | | |
| cd base | |
| mkdir build | |
| cd build | |
| cmake .. -DCMAKE_BUILD_TYPE=Release | |
| make -j$(nproc) | |
| - name: Upload base engine | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: base-engine | |
| path: base/build/${{ github.event.inputs.output_exec }} | |
| prepare-tools: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: | | |
| mkdir tools | |
| cd tools | |
| sudo apt update | |
| sudo apt install -y wget unzip | |
| wget https://github.com/Disservin/fastchess/releases/download/v1.8.0-alpha/fastchess-linux-x86-64.tar | |
| tar -xf fastchess-linux-x86-64.tar | |
| mv fastchess-linux-x86-64/fastchess fastchess | |
| wget https://github.com/official-stockfish/books/raw/refs/heads/master/UHO_Lichess_4852_v1.epd.zip | |
| unzip UHO_Lichess_4852_v1.epd.zip | |
| wget https://github.com/michiguel/Ordo/releases/download/1.0/ordo-linux64 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: tools | |
| path: tools/ | |
| test: | |
| needs: [build-new, build-base, prepare-tools] | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,20,21,22,23,24,25,26,27,28,29] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| - name: Play games | |
| env: | |
| OUTPUT_EXEC: ${{ github.event.inputs.output_exec }} | |
| TC: ${{ github.event.inputs.tc }} | |
| ELO0: ${{ github.event.inputs.elo0 }} | |
| ELO1: ${{ github.event.inputs.elo1 }} | |
| run: | | |
| set -eo pipefail | |
| chmod +x new-engine/engine | |
| chmod +x base-engine/$OUTPUT_EXEC | |
| chmod +x tools/fastchess | |
| chmod +x tools/ordo-linux64 | |
| tools/fastchess \ | |
| -engine cmd=new-engine/engine name=new \ | |
| -engine cmd=base-engine/$OUTPUT_EXEC name=base \ | |
| -openings file=tools/UHO_Lichess_4852_v1.epd format=epd order=random \ | |
| -each tc=$TC -startup-ms 600000 -ucinewgame-ms 600000 -ping-ms 600000\ | |
| -rounds 200 -repeat -log level=err \ | |
| -concurrency $(nproc) -testEnv -sprt elo0=$ELO0 elo1=$ELO1 alpha=0.05 beta=0.05 \ | |
| -pgnout notation=san nodes=true \ | |
| file=games_${{ matrix.shard }}.pgn \ | |
| | tee results_${{ matrix.shard }}.txt | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: shard-${{ matrix.shard }} | |
| path: | | |
| games_${{ matrix.shard }}.pgn | |
| results_${{ matrix.shard }}.txt | |
| evaluate: | |
| needs: test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/download-artifact@v4 | |
| - name: Run Ordo | |
| run: | | |
| chmod +x tools/ordo-linux64 | |
| printf '<%s>\n' shard-*/games_*.pgn | |
| cat shard-*/games_*.pgn > games.pgn | |
| tools/ordo-linux64 -o ratings.txt games.pgn | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| - name: SPRT | |
| env: | |
| ELO0: ${{ github.event.inputs.elo0 }} | |
| ELO1: ${{ github.event.inputs.elo1 }} | |
| run: | | |
| pip install chess scipy | |
| git clone https://github.com/jdart1/stats.git | |
| RESULTS=$(python extract_ptnml.py shard-*/*.pgn | tee results.txt | tail -n1) | |
| python -m stats.sprt \ | |
| --elo0 "$ELO0" \ | |
| --elo1 "$ELO1" \ | |
| --alpha 0.05 \ | |
| --beta 0.05 \ | |
| --results $RESULTS | tee -a results.txt | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: results | |
| path: | | |
| ratings.txt | |
| sprt.txt | |
| results.txt |