make ci runs faster, more stable and fix compilation for webgpu only

.github/workflows/cmake-presets.yml

@@ -0,0 +1,105 @@
+name: CMake Presets
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+concurrency:
+  group: cmake-presets-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  msvc-debug:
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: powershell
+
+    strategy:
+      fail-fast: false
+      matrix:
+        preset:
+        - webgpu_app_msvc_debug
+        - unittests_msvc_debug
+
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        submodules: 'true'
+
+    - name: Load CI versions
+      run: |
+        Get-Content .github/ci-versions.env | Add-Content $env:GITHUB_ENV
+
+    - name: Set sccache config
+      run: |
+        "SCCACHE_VERSION=v0.16.0" | Add-Content $env:GITHUB_ENV
+        "ALP_CACHE_CONFIG=${{ matrix.preset }}" | Add-Content $env:GITHUB_ENV
+        "ALP_CACHE_TOOLCHAIN=msvc2022-qt-$env:QT_VERSION-sccache-v0.16.0" | Add-Content $env:GITHUB_ENV
+        "SCCACHE_DIR=${{ github.workspace }}\.sccache" | Add-Content $env:GITHUB_ENV
+        "SCCACHE_CACHE_SIZE=500M" | Add-Content $env:GITHUB_ENV
+
+    - name: Cache sccache
+      uses: actions/cache@v6
+      with:
+        path: ${{ github.workspace }}\.sccache
+        key: sccache-${{ runner.os }}-${{ env.ALP_CACHE_CONFIG }}-${{ env.ALP_CACHE_TOOLCHAIN }}-v1-${{ github.run_id }}-${{ github.run_attempt }}
+        restore-keys: |
+          sccache-${{ runner.os }}-${{ env.ALP_CACHE_CONFIG }}-${{ env.ALP_CACHE_TOOLCHAIN }}-v1-
+
+    - name: Make sure MSVC is found when Ninja generator is in use
+      uses: ilammy/msvc-dev-cmd@v1
+
+    - name: Install tools
+      run: choco install 7zip.install ninja
+
+    - name: Install sccache
+      run: |
+        $Archive = "sccache-$env:SCCACHE_VERSION-x86_64-pc-windows-msvc.zip"
+        $Url = "https://github.com/mozilla/sccache/releases/download/$env:SCCACHE_VERSION/$Archive"
+        Invoke-WebRequest -Uri $Url -OutFile $Archive
+        Expand-Archive -Path $Archive -DestinationPath sccache-download
+        New-Item -ItemType Directory -Force -Path sccache-bin | Out-Null
+        $SccacheExe = Get-ChildItem -Path sccache-download -Filter sccache.exe -Recurse | Select-Object -First 1
+        Copy-Item $SccacheExe.FullName sccache-bin\sccache.exe
+        "${{ github.workspace }}\sccache-bin" | Add-Content $env:GITHUB_PATH
+
+    - name: Install Qt
+      uses: jurplel/install-qt-action@v4
+      with:
+        version: ${{ env.QT_VERSION }}
+        host: windows
+        target: 'desktop'
+        arch: 'win64_msvc2022_64'
+        dir: '${{ github.workspace }}/qt'
+        install-deps: 'true'
+        modules: ${{ env.QT_MODULES }}
+        cache: true
+        aqtsource: 'git+https://github.com/miurahr/aqtinstall.git@bbfb1f7c0590b9eb3fa91356e75bb64fb15d3643'
+
+    - name: Configure
+      env:
+        CMAKE_PREFIX_PATH: ${{ env.QT_ROOT_DIR }}/lib/cmake
+      run: >
+        cmake --preset ${{ matrix.preset }}
+        -DCMAKE_POLICY_DEFAULT_CMP0141=NEW
+        -DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded
+        -DCMAKE_C_COMPILER_LAUNCHER=sccache
+        -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+
+    - name: Reset sccache stats
+      run: |
+        sccache --start-server
+        sccache --zero-stats
+
+    - name: Build
+      run: cmake --build --preset ${{ matrix.preset }}
+
+    - name: Show sccache stats
+      if: always()
+      run: sccache --show-stats

.github/workflows/deploy.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install -y build-essential ninja-build openjdk-17-jdk
+        sudo apt-get install -y build-essential ccache ninja-build openjdk-17-jdk
         echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/" >> $GITHUB_ENV
         # dpkg-query -L  openjdk-17-jdk-headless
 
@@ -94,6 +94,25 @@ jobs:
         echo "ANDROID_BUILD_DIR_ARR=${ANDROID_BUILD_DIR_ARR[*]}" >> $GITHUB_ENV
         echo "DEPLOYMENT_SETTINGS_ARR=${DEPLOYMENT_SETTINGS_ARR[*]}" >> $GITHUB_ENV
 
+    - name: Set ccache config
+      shell: bash
+      run: |
+        echo "ALP_CACHE_CONFIG=android_${{ matrix.qtarch }}_Release" >> "$GITHUB_ENV"
+        echo "ALP_CACHE_TOOLCHAIN=android-${{ matrix.qtarch }}-qt-${QT_VERSION}" >> "$GITHUB_ENV"
+        echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> "$GITHUB_ENV"
+        echo "CCACHE_BASEDIR=${{ github.workspace }}" >> "$GITHUB_ENV"
+        echo "CCACHE_COMPRESS=true" >> "$GITHUB_ENV"
+        echo "CCACHE_COMPILERCHECK=content" >> "$GITHUB_ENV"
+        echo "CCACHE_MAXSIZE=500M" >> "$GITHUB_ENV"
+
+    - name: Cache ccache
+      uses: actions/cache@v6
+      with:
+        path: ${{ github.workspace }}/.ccache
+        key: ccache-${{ runner.os }}-${{ env.ALP_CACHE_CONFIG }}-${{ env.ALP_CACHE_TOOLCHAIN }}-v1-${{ github.run_id }}-${{ github.run_attempt }}
+        restore-keys: |
+          ccache-${{ runner.os }}-${{ env.ALP_CACHE_CONFIG }}-${{ env.ALP_CACHE_TOOLCHAIN }}-v1-
+
     - name: Setup signing keys
       shell: bash
       env:
@@ -153,11 +172,20 @@ jobs:
           -DALP_ENABLE_ASSERTS=ON \
           -DQT_ANDROID_SIGN_APK=ON \
           -DQT_USE_TARGET_ANDROID_BUILD_DIR:BOOL=ON \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
           -S "${{ github.workspace }}"
 
+    - name: Reset ccache stats
+      run: ccache --zero-stats
+
     - name: Build
       run: cmake --build $BUILD_DIR --target apk
 
+    - name: Show ccache stats
+      if: always()
+      run: ccache --show-stats --verbose
+
     - name: Copy android packages
       run: |
         mkdir -p "$INSTALL_DIR"
@@ -202,7 +230,7 @@ jobs:
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install -y build-essential ninja-build
+        sudo apt-get install -y build-essential ccache ninja-build
 
     - uses: actions/checkout@v7
       with:
@@ -215,18 +243,34 @@ jobs:
     - uses: mymindstorm/setup-emsdk@v15
       with:
         version: ${{ env.WEBASSEMBLY_VERSION }}
+        actions-cache-folder: emsdk-cache
+        cache-key: emsdk-${{ runner.os }}-${{ env.WEBASSEMBLY_VERSION }}
 
-    - name: Cache Dawn WebAssembly port package
-      uses: actions/cache@v6
-      with:
-        path: extern/emdawnwebgpu_pkg
-        key: dawn-wasm-port-${{ hashFiles('webgpu/base/CMakeLists.txt', 'misc/scripts/fetch_dawn_port.py') }}
+    - name: Use stable Emscripten path
+      shell: bash
+      run: |
+        STABLE_EMSDK_ROOT="${{ github.workspace }}/emsdk-cache/emsdk-main"
+        if [ ! -x "$STABLE_EMSDK_ROOT/emsdk" ]; then
+          echo "Expected emsdk at $STABLE_EMSDK_ROOT/emsdk"
+          exit 1
+        fi
 
-    - name: Cache Emscripten Dawn port build
-      uses: actions/cache@v6
-      with:
-        path: ${{ env.EMSDK }}/upstream/emscripten/cache
-        key: ${{ matrix.config }}-emscripten-dawn-${{ runner.os }}-emcc-${{ env.WEBASSEMBLY_VERSION }}-${{ hashFiles('webgpu/base/CMakeLists.txt', 'misc/scripts/fetch_dawn_port.py') }}
+        "$STABLE_EMSDK_ROOT/emsdk" activate "$WEBASSEMBLY_VERSION"
+        "$STABLE_EMSDK_ROOT/emsdk" construct_env | while IFS= read -r line; do
+          case "$line" in
+            "PATH += "*)
+              echo "${line#PATH += }" >> "$GITHUB_PATH"
+              ;;
+            *" = "*)
+              name="${line%% = *}"
+              value="${line#* = }"
+              echo "$name=$value" >> "$GITHUB_ENV"
+              ;;
+          esac
+        done
+
+    # Do not cache Dawn here. It is fetched, not built, and measured CI timings
+    # showed no useful speedup from restoring/saving the cache.
 
     - name: Install Qt native version (required for cross building)
       uses: jurplel/install-qt-action@v4
@@ -291,10 +335,28 @@ jobs:
         echo "cmake version:"
         cmake --version
         cd ${{ github.workspace }} && ls -l
-        cat ./qt_wasm_build/build.ninja
 
     - name: Remove StatsWindow, otherwise the webassembly will link against qtwidgets and charts.
       run: rm ./app/StatsWindow.qml
+
+    - name: Set ccache config
+      shell: bash
+      run: |
+        echo "ALP_CACHE_CONFIG=${{ matrix.config }}" >> "$GITHUB_ENV"
+        echo "ALP_CACHE_TOOLCHAIN=emscripten-${WEBASSEMBLY_VERSION}-qt-${QT_VERSION}" >> "$GITHUB_ENV"
+        echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> "$GITHUB_ENV"
+        echo "CCACHE_BASEDIR=${{ github.workspace }}" >> "$GITHUB_ENV"
+        echo "CCACHE_COMPRESS=true" >> "$GITHUB_ENV"
+        echo "CCACHE_COMPILERCHECK=content" >> "$GITHUB_ENV"
+        echo "CCACHE_MAXSIZE=500M" >> "$GITHUB_ENV"
+
+    - name: Cache ccache
+      uses: actions/cache@v6
+      with:
+        path: ${{ github.workspace }}/.ccache
+        key: ccache-${{ runner.os }}-${{ env.ALP_CACHE_CONFIG }}-${{ env.ALP_CACHE_TOOLCHAIN }}-v3-${{ github.run_id }}-${{ github.run_attempt }}
+        restore-keys: |
+          ccache-${{ runner.os }}-${{ env.ALP_CACHE_CONFIG }}-${{ env.ALP_CACHE_TOOLCHAIN }}-v3-
 
     - name: Configure
       env:
@@ -307,11 +369,20 @@ jobs:
         -DALP_WWW_INSTALL_DIR=$INSTALL_DIR
         -DFMT_INSTALL=OFF
         ${{ matrix.cmakeflags }}
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         -S ${{ github.workspace }}
 
+    - name: Reset ccache stats
+      run: ccache --zero-stats
+
     - name: Build
       run: cmake --build $BUILD_DIR --target install
 
+    - name: Show ccache stats
+      if: always()
+      run: ccache --show-stats --verbose
+
     - name: Verify WebGPU app artifacts
       run: |
         test -f "$INSTALL_DIR/webgpu_app.html"
@@ -388,9 +459,9 @@ jobs:
         done
 
     - name: Generate Directory Listings
-      uses: jayanta525/github-pages-directory-listing@v4.0.0
+      uses: gacts/directory-listing@v1
       with:
-        FOLDER: github_page
+        target: github_page
 
     - name: Create Pages artifact
       uses: actions/upload-pages-artifact@v4

.github/workflows/linux.yml

@@ -41,16 +41,13 @@ jobs:
     - name: Load CI versions
       run: cat .github/ci-versions.env >> "$GITHUB_ENV"
 
-    - name: Cache native Dawn
-      uses: actions/cache@v6
-      with:
-        path: extern/dawn
-        key: ${{ runner.os }}-dawn-native-debug-${{ hashFiles('webgpu/base/CMakeLists.txt', 'misc/scripts/fetch_dawn_native.py', 'misc/scripts/install_dawn.py') }}
+    # Do not cache Dawn here. It is fetched, not built, and measured CI timings
+    # showed no useful speedup from restoring/saving the cache.
 
     - name: Install Linux Dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install -y build-essential ninja-build lld libgl1-mesa-dev libxcb-cursor-dev xorg-dev libx11-xcb-dev libxrandr-dev libxcursor-dev libudev-dev libopenal-dev libflac-dev libvorbis-dev libgl1-mesa-dev libegl1-mesa-dev libdrm-dev libgbm-dev xvfb libxcb-cursor0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-render-util0
+        sudo apt-get install -y build-essential ccache ninja-build lld libgl1-mesa-dev libxcb-cursor-dev xorg-dev libx11-xcb-dev libxrandr-dev libxcursor-dev libudev-dev libopenal-dev libflac-dev libvorbis-dev libgl1-mesa-dev libegl1-mesa-dev libdrm-dev libgbm-dev xvfb libxcb-cursor0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-render-util0
 
     - name: Install Clang 17
       if: matrix.compiler == 'clang17'
@@ -75,6 +72,25 @@ jobs:
         install-deps: 'true'
         modules: ${{ env.QT_MODULES }}
         cache: true
+
+    - name: Set ccache config
+      shell: bash
+      run: |
+        echo "ALP_CACHE_CONFIG=linux_${{ matrix.compiler }}_${{ matrix.build_type }}" >> "$GITHUB_ENV"
+        echo "ALP_CACHE_TOOLCHAIN=${{ matrix.compiler }}-qt-${QT_VERSION}" >> "$GITHUB_ENV"
+        echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> "$GITHUB_ENV"
+        echo "CCACHE_BASEDIR=${{ github.workspace }}" >> "$GITHUB_ENV"
+        echo "CCACHE_COMPRESS=true" >> "$GITHUB_ENV"
+        echo "CCACHE_COMPILERCHECK=content" >> "$GITHUB_ENV"
+        echo "CCACHE_MAXSIZE=500M" >> "$GITHUB_ENV"
+
+    - name: Cache ccache
+      uses: actions/cache@v6
+      with:
+        path: ${{ github.workspace }}/.ccache
+        key: ccache-${{ runner.os }}-${{ env.ALP_CACHE_CONFIG }}-${{ env.ALP_CACHE_TOOLCHAIN }}-v1-${{ github.run_id }}-${{ github.run_attempt }}
+        restore-keys: |
+          ccache-${{ runner.os }}-${{ env.ALP_CACHE_CONFIG }}-${{ env.ALP_CACHE_TOOLCHAIN }}-v1-
 
     - name: Debug output
       shell: bash
@@ -98,15 +114,24 @@ jobs:
         -DALP_ENABLE_ADDRESS_SANITIZER=ON
         -DALP_ENABLE_APP_SHUTDOWN_AFTER_60S=ON
         -DALP_USE_LLVM_LINKER=ON
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         -B ./build
 
+    - name: Reset ccache stats
+      run: ccache --zero-stats
+
     - name: Build
       env:
         CC: ${{ matrix.CC }}
         CXX: ${{ matrix.CXX }}
       run: |
         cmake --build ./build
 
+    - name: Show ccache stats
+      if: always()
+      run: ccache --show-stats --verbose
+
     - name: Don't close loaded libraries for better sanitizer output
       env:
         CC: ${{ matrix.CC }}