games.yml

name: Fastchess SPRT Test (Linux)

on:
  workflow_dispatch:
    inputs:
      base_ref:
        description: "Base branch, tag, or commit"
        required: false
        default: "HandcraftedEngine"
      rounds:
        description: "Max rounds for SPRT"
        required: false
        default: "6000"
      tc:
        description: "Time control (e.g. 60+0.6 or 10+0.1)"
        required: false
        default: "10+0.1"
      elo0:
        description: "elo0"
        required: false
        default: "0.0"
      elo1:
        description: "elo1"
        required: false
        default: "2.0"
      output_exec:
        description: "Executable output file name"
        required: false
        default: "engine"

permissions:
  contents: read

jobs:

  build-new:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v6
        with:
          path: new
          persist-credentials: false

      - name: Install deps
        run: sudo apt update && sudo apt install -y build-essential cmake

      - name: Build new engine
        run: |
          cd new
          mkdir build
          cd build
          cmake .. -DCMAKE_BUILD_TYPE=Release
          make -j$(nproc)

      - name: Upload new engine
        uses: actions/upload-artifact@v4
        with:
          name: new-engine
          path: new/build/engine


  build-base:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ github.event.inputs.base_ref }}
          path: base
          persist-credentials: false

      - name: Install deps
        run: sudo apt update && sudo apt install -y build-essential cmake

      - name: Build base engine
        run: |
          cd base
          mkdir build
          cd build
          cmake .. -DCMAKE_BUILD_TYPE=Release
          make -j$(nproc)

      - name: Upload base engine
        uses: actions/upload-artifact@v4
        with:
          name: base-engine
          path: base/build/${{ github.event.inputs.output_exec }}
  prepare-tools:
    runs-on: ubuntu-latest

    steps:
      - run: |
          mkdir tools
          cd tools

          sudo apt update
          sudo apt install -y wget unzip
          wget https://github.com/Disservin/fastchess/releases/download/v1.8.0-alpha/fastchess-linux-x86-64.tar
          tar -xf fastchess-linux-x86-64.tar
          mv fastchess-linux-x86-64/fastchess fastchess
          wget https://github.com/official-stockfish/books/raw/refs/heads/master/UHO_Lichess_4852_v1.epd.zip
          unzip UHO_Lichess_4852_v1.epd.zip
          wget https://github.com/michiguel/Ordo/releases/download/1.0/ordo-linux64

      - uses: actions/upload-artifact@v4
        with:
          name: tools
          path: tools/
  test:
    needs: [build-new, build-base, prepare-tools]
    strategy:
      fail-fast: true
      matrix:
        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,20,21,22,23,24,25,26,27,28,29]

    runs-on: ubuntu-latest

    steps:
      - uses: actions/download-artifact@v4
      - name: Play games
        env:
          OUTPUT_EXEC: ${{ github.event.inputs.output_exec }}
          TC: ${{ github.event.inputs.tc }}
          ELO0: ${{ github.event.inputs.elo0 }}
          ELO1: ${{ github.event.inputs.elo1 }}
          ROUNDS: ${{ github.event.inputs.rounds }}
        run: |
          set -eo pipefail
          if [[ ! "$ROUNDS" =~ ^[0-9]+$ ]]; then
            echo "rounds must be a decimal integer" >&2
            exit 1
          fi
          rounds_value=$((10#$ROUNDS))
          shard_rounds=$((rounds_value / 30))
            -rounds "$shard_rounds" -repeat -log level=err \
            -concurrency $(nproc) -testEnv -sprt elo0=$ELO0 elo1=$ELO1 alpha=0.05 beta=0.05 \
            -pgnout notation=san nodes=true \
                    file=games_${{ matrix.shard }}.pgn \
            | tee results_${{ matrix.shard }}.txt

      - uses: actions/upload-artifact@v4
        with:
          name: shard-${{ matrix.shard }}
          path: |
            games_${{ matrix.shard }}.pgn
            results_${{ matrix.shard }}.txt

  evaluate:
    needs: test
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - uses: actions/download-artifact@v4

      - name: Run Ordo
        run: |
          chmod +x tools/ordo-linux64
          printf '<%s>\n' shard-*/games_*.pgn
          cat shard-*/games_*.pgn > games.pgn
          tools/ordo-linux64 -o ratings.txt games.pgn

      - uses: actions/setup-python@v6
        with:
          python-version: '3.13'

      - name: SPRT
        env:
          ELO0: ${{ github.event.inputs.elo0 }}
          ELO1: ${{ github.event.inputs.elo1 }}
        run: |
          pip install chess scipy
          git clone https://github.com/jdart1/stats.git
          RESULTS=$(python extract_ptnml.py shard-*/*.pgn | tee results.txt | tail -n1)
          python -m stats.sprt \
              --elo0 "$ELO0" \
              --elo1 "$ELO1" \
              --alpha 0.05 \
              --beta 0.05 \
              --results $RESULTS | tee -a results.txt
              
      - uses: actions/upload-artifact@v4
        with:
          name: results
          path: |
            ratings.txt
            sprt.txt
            results.txt