Enhance CI/CD workflows and documentation for nightly builds

.github/actions/setup-build-env/action.yml

@@ -0,0 +1,58 @@
+name: "Setup Build Environment"
+description: "Python, Rust, uv, and optional protoc for Persisting CI"
+inputs:
+  python-version:
+    description: "Python version"
+    required: false
+    default: "3.12"
+  rust-toolchain:
+    description: "Rust toolchain"
+    required: false
+    default: "stable"
+  install-protoc:
+    description: "Install protobuf compiler (needed by persisting-dlcapt)"
+    required: false
+    default: "true"
+  components:
+    description: "Rust components (comma-separated)"
+    required: false
+    default: "rustfmt, clippy"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ inputs.rust-toolchain }}
+        components: ${{ inputs.components }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+
+    - name: Install protoc
+      if: inputs.install-protoc == 'true' && runner.os == 'Linux'
+      shell: bash
+      run: |
+        sudo apt-get update -qq
+        sudo apt-get install -y -qq --no-install-recommends protobuf-compiler
+        protoc --version
+
+    - name: Install protoc
+      if: inputs.install-protoc == 'true' && runner.os == 'macOS'
+      shell: bash
+      run: |
+        brew install protobuf
+        protoc --version
+
+    - name: Point PyO3 at CI Python
+      shell: bash
+      run: |
+        echo "PYO3_PYTHON=$(python -c 'import sys; print(sys.executable)')" >> "$GITHUB_ENV"

.github/workflows/ci.yml

@@ -0,0 +1,86 @@
+name: CI
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+jobs:
+  lint:
+    name: Lint & Format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Build Environment
+        uses: ./.github/actions/setup-build-env
+        with:
+          install-protoc: "true"
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Rust formatting
+        run: cargo fmt --all -- --check
+
+      - name: Rust clippy
+        # Match local `just clippy` for now; tighten to `-D warnings` once the
+        # workspace is clean (same bar as Pulsing).
+        run: cargo clippy --workspace --all-targets --locked
+
+      - name: Python lint (ruff)
+        run: uvx ruff check persisting/
+
+  rust-test:
+    name: Rust Test (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Build Environment
+        uses: ./.github/actions/setup-build-env
+        with:
+          install-protoc: "true"
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Build
+        run: cargo build --workspace --all-targets --locked
+
+      - name: Test
+        run: cargo test --workspace --locked
+
+  python-test:
+    name: Python Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Build Environment
+        uses: ./.github/actions/setup-build-env
+        with:
+          python-version: "3.12"
+          install-protoc: "true"
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Sync Python deps
+        run: uv sync --extra all --extra dev
+
+      - name: Build extension (maturin develop)
+        run: uv run maturin develop
+
+      - name: Pytest
+        run: uv run pytest tests/ -q

.github/workflows/nightly.yml

@@ -0,0 +1,156 @@
+# Nightly wheels from main — rolling GitHub Release tag `nightly`.
+#
+# Install (after a successful run):
+#   curl -fsSL https://raw.githubusercontent.com/DeepLink-org/Persisting/main/scripts/install-nightly.sh | bash
+# Or browse assets: https://github.com/DeepLink-org/Persisting/releases/tag/nightly
+
+name: Nightly Build
+
+on:
+  schedule:
+    # 03:00 UTC daily
+    - cron: "0 3 * * *"
+  workflow_dispatch:
+  push:
+    branches: [main, master]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: nightly-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  meta:
+    if: github.repository == 'DeepLink-org/Persisting'
+    runs-on: ubuntu-latest
+    outputs:
+      local_version: ${{ steps.ver.outputs.local_version }}
+    steps:
+      - id: ver
+        shell: bash
+        run: |
+          short_sha="${GITHUB_SHA:0:7}"
+          echo "local_version=g${GITHUB_RUN_NUMBER}.${short_sha}" >> "$GITHUB_OUTPUT"
+
+  linux:
+    if: github.repository == 'DeepLink-org/Persisting'
+    needs: meta
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [x86_64]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Build Environment
+        uses: ./.github/actions/setup-build-env
+        with:
+          python-version: "3.12"
+          install-protoc: "true"
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Apply nightly local version (PEP 440)
+        run: python scripts/ci/set_nightly_local_version.py "${{ needs.meta.outputs.local_version }}"
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: >-
+            --release --out dist --zig --compatibility manylinux2014
+            --auditwheel repair
+          sccache: "true"
+          manylinux: auto
+          rust-toolchain: stable
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: nightly-wheels-linux-${{ matrix.target }}
+          path: dist/*.whl
+          retention-days: 14
+
+  macos:
+    if: github.repository == 'DeepLink-org/Persisting'
+    needs: meta
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        target: [x86_64, aarch64]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Build Environment
+        uses: ./.github/actions/setup-build-env
+        with:
+          python-version: "3.12"
+          install-protoc: "true"
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Apply nightly local version (PEP 440)
+        run: python scripts/ci/set_nightly_local_version.py "${{ needs.meta.outputs.local_version }}"
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: >-
+            --release --out dist --find-interpreter
+          sccache: "true"
+          rust-toolchain: stable
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: nightly-wheels-macos-${{ matrix.target }}
+          path: dist/*.whl
+          retention-days: 14
+
+  publish:
+    name: Publish nightly release
+    if: github.repository == 'DeepLink-org/Persisting'
+    needs: [meta, linux, macos]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download wheels
+        uses: actions/download-artifact@v4
+        with:
+          pattern: nightly-wheels-*
+          merge-multiple: true
+          path: dist
+
+      - name: List artifacts
+        run: ls -la dist/
+
+      - name: Publish rolling nightly release
+        uses: ncipollo/release-action@v1
+        with:
+          tag: nightly
+          name: Nightly (main)
+          commit: ${{ github.sha }}
+          allowUpdates: true
+          makeLatest: false
+          prerelease: true
+          removeArtifacts: true
+          artifactErrorsFailBuild: true
+          artifacts: dist/*.whl
+          body: |
+            Rolling nightly wheels built from [`${{ github.sha }}`](${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}) on **${{ github.ref_name }}**.
+
+            **Quick install** (auto-detect platform / Python):
+
+            ```bash
+            curl -fsSL https://raw.githubusercontent.com/${{ github.repository }}/main/scripts/install-nightly.sh | bash
+            ```
+
+            Or pick a `.whl` below and `pip install --force-reinstall <url>`.
+
+            Built with local version `+${{ needs.meta.outputs.local_version }}` (base version from `pyproject.toml`).

crates/persisting-capture/src/proxy/network_policy.rs

@@ -561,6 +561,6 @@ upstream = "http://127.0.0.1:9/v1"
         assert_eq!(cfg.network.allowed_hosts.len(), 3);
         let p = NetworkPolicy::from_config(&cfg).unwrap();
         assert!(assert_egress_allowed(&p, "a.example.com").is_ok());
-        assert!(!assert_egress_allowed(&p, "example.com").is_ok());
+        assert!(assert_egress_allowed(&p, "example.com").is_err());
     }
 }

docs/src/installation.md

@@ -23,6 +23,14 @@ cd Persisting
 pip install -e ".[lance]"
 ```
 
+### Nightly (pre-release wheels)
+
+Daily / `main` pushes refresh the rolling GitHub Release tag [`nightly`](https://github.com/DeepLink-org/Persisting/releases/tag/nightly):
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/DeepLink-org/Persisting/main/scripts/install-nightly.sh | bash
+```
+
 ## Verify
 
 ```python

docs/src/installation.zh.md

@@ -23,6 +23,14 @@ cd Persisting
 pip install -e ".[lance]"
 ```
 
+### Nightly（预发布 wheel）
+
+每日 / `main` 推送会滚动更新 GitHub Release 标签 [`nightly`](https://github.com/DeepLink-org/Persisting/releases/tag/nightly)：
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/DeepLink-org/Persisting/main/scripts/install-nightly.sh | bash
+```
+
 ## 验证
 
 ```python

examples/03_sampler.py

@@ -41,7 +41,9 @@ def sample(
         *args: Any,
         **kwargs: Any,
     ) -> tuple[list[int], list[int]]:
-        taken = ready_indexes[-batch_size:] if len(ready_indexes) >= batch_size else ready_indexes[:]
+        taken = (
+            ready_indexes[-batch_size:] if len(ready_indexes) >= batch_size else ready_indexes[:]
+        )
         taken = list(reversed(taken))
         return taken, taken
 
@@ -64,7 +66,9 @@ async def demo_sequential_sampler() -> None:
 
     print(f"\n  Reading in batches of {batch_size} with SequentialSampler:")
     for step in range(3):
-        records, offset = await queue.get_batch(sampler, batch_size=batch_size, consumed_offset=offset)
+        records, offset = await queue.get_batch(
+            sampler, batch_size=batch_size, consumed_offset=offset
+        )
         if not records:
             break
         print(f"    Step {step + 1}: got {len(records)} records, new offset = {offset}")
@@ -91,7 +95,9 @@ async def demo_custom_reverse_sampler() -> None:
 
     print("\n  Reading in batches of 2 with ReverseSampler (last indices first):")
     for step in range(3):
-        records, offset = await queue.get_batch(sampler, batch_size=batch_size, consumed_offset=offset)
+        records, offset = await queue.get_batch(
+            sampler, batch_size=batch_size, consumed_offset=offset
+        )
         if not records:
             break
         print(f"    Step {step + 1}: got {len(records)} records")