diff --git a/AGENTS.md b/AGENTS.md index 40044b8..2d4fa85 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,9 @@ - `README.md` and `docs/` describe released behavior. Rewrite stale sections in place when behavior changes; do not append historical caveats or parallel inventories. +- Treat user reports and external reviews as evidence to investigate, not as + implemented fact. Verify claims about bounds, clocks, filtering, lifecycle, + and errors against code and tests before documenting or changing them. ## Boundaries @@ -33,8 +36,25 @@ ## Agent workflow - Follow `skills/xprobe-measure-latency/SKILL.md` for measurement tasks. -- Use `doctor`, `discover`, `validate`, then one bounded `measure` call. -- Read evidence pairs and all quality fields before interpreting latency. +- For an unknown workload, progress from readiness and baseline through a short + survey artifact, evidence-based selector narrowing, validation, and one + bounded measurement per stated hypothesis. Do not guess selectors. +- Read evidence pairs, artifact analysis, stream identity, collection quality, + and profiler overhead before interpreting latency. Summed concurrent GPU + duration is not wall time. +- Keep reusable analysis deterministic and bundled with the Skill. Keep model + orchestration in the caller rather than adding CLI commands or a service. + +## Release discipline + +- Start post-release work on a dedicated feature or fix branch. Rebase pull + requests into `master`; do not create merge commits. +- Build Linux release artifacts on Ubuntu 22.04 and enforce a GLIBC_2.34 ceiling + for the CLI and every shipped Agent ELF. +- A release is complete only after downloading the public archive, checking its + digest, testing installation, and inspecting all shipped ELF compatibility. +- Retry CI only when logs identify a transient infrastructure failure. Preserve + real compiler, test, packaging, and compatibility failures for diagnosis. ## Verification @@ -44,5 +64,7 @@ - Run `just test-injection-live` for injection or agent lifecycle changes. - Run `just test-multisource-live` for host/GPU orchestration changes. - Run `just test-agent-contract` for CLI, schema, docs, or Skill changes. +- Test bundled Skill scripts with deterministic fixtures and include them in + `just test-agent-contract`. - Run `just benchmark-gpu` for callback hot-path changes. - Use emoji conventional commits, for example `🐛 fix: restore target registers`. diff --git a/CMakeLists.txt b/CMakeLists.txt index a14a4e9..6e3c190 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.20) -project(xprobe VERSION 0.3.0 LANGUAGES C) +project(xprobe VERSION 0.3.1 LANGUAGES C) include(CTest) diff --git a/Cargo.lock b/Cargo.lock index 0fd5471..d792b4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -452,7 +452,7 @@ dependencies = [ [[package]] name = "xprobe-cli" -version = "0.3.0" +version = "0.3.1" dependencies = [ "clap", "serde_json", @@ -465,7 +465,7 @@ dependencies = [ [[package]] name = "xprobe-collector" -version = "0.3.0" +version = "0.3.1" dependencies = [ "libbpf-rs", "serde_json", @@ -474,7 +474,7 @@ dependencies = [ [[package]] name = "xprobe-core" -version = "0.3.0" +version = "0.3.1" dependencies = [ "nix", "object", @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "xprobe-correlator" -version = "0.3.0" +version = "0.3.1" dependencies = [ "regex", "serde_json", @@ -493,7 +493,7 @@ dependencies = [ [[package]] name = "xprobe-exporter" -version = "0.3.0" +version = "0.3.1" dependencies = [ "serde_json", "xprobe-protocol", @@ -501,7 +501,7 @@ dependencies = [ [[package]] name = "xprobe-protocol" -version = "0.3.0" +version = "0.3.1" dependencies = [ "schemars", "serde", diff --git a/Cargo.toml b/Cargo.toml index 9c39e69..62d80aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.3.0" +version = "0.3.1" edition = "2024" license = "Apache-2.0" repository = "https://github.com/itdevwu/xprobe" diff --git a/README.md b/README.md index ce9dda5..10d11e2 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ contracts, explicit correlation quality, and no daemon or server lifecycle. ```bash curl --proto '=https' --tlsv1.2 -fsSL \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.3.0/install.sh | sh + https://raw.githubusercontent.com/itdevwu/xprobe/v0.3.1/install.sh | sh ``` This installs the released CLI and CUDA 12/13 Agents under `~/.local`. xprobe @@ -32,13 +32,15 @@ Install the version-matched Skill with the open Agent Skills CLI: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.3.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.3.1/skills/xprobe-measure-latency \ --global ``` The installer detects Codex, Claude Code, Cursor, and other compatible agents. Node.js is only needed for this Skill installation, not for xprobe itself. Set `DISABLE_TELEMETRY=1` when anonymous `skills` CLI telemetry is not wanted. +The Skill guides a short survey, derives selectors from trace evidence, and +includes a deterministic multi-stream JSONL analyzer. ## Measure @@ -91,7 +93,7 @@ for safe reactivation. ## Support -| Surface | 0.3.0 support | +| Surface | 0.3.1 support | | --- | --- | | OS/architecture | Linux x86_64, glibc 2.34 or newer | | Host events | ELF function entry/return through PID-scoped uprobes | diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..c57359f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,56 @@ +# Security Policy + +## Supported Versions + +xprobe is a pre-1.0 project. Security fixes are provided for the latest release +only. Reproduce a report against the latest release or the current `master` +branch when possible. + +| Version | Supported | +| --- | --- | +| Latest release | Yes | +| Older releases | No | + +## Reporting a Vulnerability + +Report suspected vulnerabilities through +[GitHub private vulnerability reporting](https://github.com/itdevwu/xprobe/security/advisories/new). +Do not open a public issue for an undisclosed vulnerability. + +Include the affected xprobe version, operating system and kernel, relevant +permission and namespace configuration, reproduction steps, observed impact, +and whether the target process recovered cleanly. Remove application secrets, +captured payloads, and other unrelated sensitive data from the report. + +This project is maintained independently and does not promise a response SLA. +Reports will be assessed as time permits. Please allow a reasonable period for +triage and remediation before public disclosure. + +## Trust Boundary + +xprobe uses eBPF/perf and ptrace capabilities granted by Linux. A live CUDA +measurement may inject a CUPTI Agent into a process that the invoking user is +already permitted to trace. The Agent remains mapped after collection and is +disabled logically. These documented operations intentionally modify target +state and require elevated access in many environments. + +Security boundaries still apply around which process is selected, which Linux +credentials and namespaces authorize access, what data is collected, how target +state is restored, and who can read or control local artifacts and Agent sockets. + +## Scope + +Examples of security issues include: + +- accessing or modifying a process outside the invoking user's Linux authority; +- PID reuse or identity-check failures that attach to a different process; +- failed attach or injection cleanup that does not restore target execution + state; +- Agent sockets, trace artifacts, or installed files with unsafe permissions; +- collection or exposure of sensitive target data beyond the documented event + contract; +- package or installer behavior that violates the documented integrity checks. + +Ordinary profiling overhead, inaccurate measurements without a security impact, +expected mutation disclosed by `validate` and `measure`, and behavior on an +unsupported release are not security vulnerabilities by themselves. diff --git a/docs/agent-integration.md b/docs/agent-integration.md index 1729651..33b3bd2 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -17,15 +17,19 @@ installer's responsibility: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.3.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.3.1/skills/xprobe-measure-latency \ --global ``` For automation, add `--agent codex|claude-code|cursor --copy --yes`. Omit `--global` for repository-scoped installation. The Skill is self-contained; -install its whole directory so its references and example remain available. +install its whole directory so its references, examples, and analysis script +remain available. -The xprobe repository tests installation with `skills` CLI 1.5.20 in isolated +The Skill moves from a short survey artifact to evidence-based selector +narrowing and includes `scripts/analyze_trace.py` for deterministic kernel, +copy, overlap, stream, and gap summaries. The xprobe repository tests +installation with `skills` CLI 1.5.20 in isolated home directories. This pinned test protects released behavior while the documented `skills@1` selector receives compatible path updates. @@ -38,8 +42,9 @@ just test-skill-install The test requires the visible command set to be exactly `doctor`, `discover`, `validate`, and `measure`. It invokes the first three in strict JSON mode, -checks injection requirements, verifies schemas, and checks that the Skill uses -only the four-command bounded workflow and inspects result quality/evidence. +checks injection requirements, verifies schemas, exercises the bundled trace +analyzer, and checks that the Skill uses only the four-command bounded workflow +and inspects result quality/evidence. The installation test uses the real third-party CLI with telemetry disabled and verifies byte-for-byte copies for Codex, Claude Code, and Cursor. diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 089834b..34225f6 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -1,6 +1,6 @@ # CLI contract -xprobe 0.3.0 exposes exactly four public commands: `doctor`, `discover`, +xprobe 0.3.1 exposes exactly four public commands: `doctor`, `discover`, `validate`, and `measure`. ## Common behavior diff --git a/docs/installation.md b/docs/installation.md index e997e5d..a2f9c99 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -11,7 +11,7 @@ The versioned bootstrap installs to `~/.local` without root access: ```bash curl --proto '=https' --tlsv1.2 -fsSL \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.3.0/install.sh | sh + https://raw.githubusercontent.com/itdevwu/xprobe/v0.3.1/install.sh | sh ``` The bootstrap downloads the release archive and its SHA256 file, verifies the @@ -30,7 +30,7 @@ prefix, download the script and pass `--prefix`: ```bash curl --proto '=https' --tlsv1.2 -fsSLO \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.3.0/install.sh + https://raw.githubusercontent.com/itdevwu/xprobe/v0.3.1/install.sh sh install.sh --prefix /opt/xprobe ``` @@ -42,7 +42,7 @@ script with `sudo`. The installer never elevates privileges itself. For a fully explicit archive workflow: ```bash -version=0.3.0 +version=0.3.1 base=https://github.com/itdevwu/xprobe/releases/download/v$version archive=xprobe-$version-linux-x86_64.tar.gz @@ -76,7 +76,7 @@ version-matched Skill: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.3.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.3.1/skills/xprobe-measure-latency \ --global ``` @@ -85,7 +85,7 @@ installation names the target explicitly: ```bash DISABLE_TELEMETRY=1 npx --yes skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.3.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.3.1/skills/xprobe-measure-latency \ --agent codex --global --copy --yes ``` @@ -95,8 +95,8 @@ an agent to execute commands with its granted permissions. Without Node.js, copy the complete `skills/xprobe-measure-latency` directory from the release archive into the Skill directory documented by the target -agent. Do not copy only `SKILL.md`; its local references and example are part of -the contract. +agent. Do not copy only `SKILL.md`; its references, examples, and trace-analysis +script are part of the contract. ## Runtime check diff --git a/install.sh b/install.sh index 8a0effc..45bbde6 100755 --- a/install.sh +++ b/install.sh @@ -2,7 +2,7 @@ set -eu repository=${XPROBE_REPOSITORY:-itdevwu/xprobe} -version=${XPROBE_VERSION:-0.3.0} +version=${XPROBE_VERSION:-0.3.1} if [ -n "${XPROBE_PREFIX:-}" ]; then prefix=$XPROBE_PREFIX elif [ -n "${HOME:-}" ]; then @@ -19,7 +19,7 @@ Install a released xprobe binary and its CUDA Agents. Usage: install.sh [--version VERSION] [--prefix DIR] [--uninstall] Options: - --version VERSION Release to install (default: 0.3.0) + --version VERSION Release to install (default: 0.3.1) --prefix DIR Installation prefix (default: $HOME/.local) --uninstall Remove xprobe from the selected prefix -h, --help Show this help diff --git a/justfile b/justfile index 6ca37a7..0ee885d 100644 --- a/justfile +++ b/justfile @@ -18,9 +18,11 @@ test: build cargo test --workspace ctest --test-dir build --output-on-failure python3 tests/agent-contract/test_contract.py target/debug/xprobe + python3 tests/agent-contract/test_trace_analysis.py test-agent-contract: build python3 tests/agent-contract/test_contract.py target/debug/xprobe + python3 tests/agent-contract/test_trace_analysis.py test-skill-install: tests/agent-contract/test_skill_install.sh diff --git a/skills/xprobe-measure-latency/SKILL.md b/skills/xprobe-measure-latency/SKILL.md index 2ca5529..8a9ace0 100644 --- a/skills/xprobe-measure-latency/SKILL.md +++ b/skills/xprobe-measure-latency/SKILL.md @@ -1,52 +1,72 @@ --- name: xprobe-measure-latency -description: Discover, validate, and measure bounded latency between Linux host functions, CUDA Runtime or Driver APIs, GPU kernels, memory copies, and memory sets with the four-command xprobe CLI. Use for launch delay, request-to-GPU latency, GPU activity duration, and evidence-based profiling of a running PID or completed captures. +description: Investigate unknown Linux CPU/CUDA latency with bounded xprobe captures, derive selectors from trace evidence, analyze multi-stream GPU activity, and measure host functions, CUDA APIs, kernels, memory copies, memory sets, and event-to-event gaps. Use when an agent must profile a running process, narrow a performance regression, inspect an xprobe JSONL artifact, or decide when duration evidence should hand off to Nsight Compute or another microarchitectural profiler. --- -# Measure latency with xprobe +# Investigate latency with xprobe -Use JSON mode and keep collection bounded. Read -[references/cli-contract.md](references/cli-contract.md) for selector and exit -semantics. Read [references/result-quality.md](references/result-quality.md) -before using temporal correlation or normalized clocks. +Use JSON mode and move from a short survey to a narrow measurement. Read +[references/investigation.md](references/investigation.md) before profiling an +unknown workload. Read [references/result-quality.md](references/result-quality.md) +before interpreting correlation, clocks, concurrency, or overhead. The exact +CLI and selector syntax is in [references/cli-contract.md](references/cli-contract.md). ## Workflow -1. Run `xprobe doctor --json --non-interactive --no-color`. Check individual +1. Run `xprobe --version`. Require a version-matched xprobe 0.3.x CLI at 0.3.1 + or later and confirm every JSON response has `schema_version: "2.0"`; do not + assume a pre-0.3 or future protocol is compatible with this Skill. +2. Establish an application-level latency baseline. Wait for process readiness, + CUDA context creation, JIT compilation, and warmup before discovery. Keep a + repeatable request or batch trigger ready for the measurement window. +3. Run `xprobe doctor --json --non-interactive --no-color`. Check individual capabilities; `ok: true` only means diagnosis completed. -2. Run `xprobe discover --pid ROOT_PID --limit 200 --json --non-interactive - --no-color`. It returns only NVML-confirmed CUDA context holders under that - process-tree root. Choose a candidate from workload context; do not make the - CLI guess among workers. -3. Run `xprobe validate --pid WORKER_PID --from SELECTOR --to SELECTOR --match POLICY - --json --non-interactive --no-color`. Stop when `valid` is false. If +4. Run `xprobe discover --pid ROOT_PID --limit 200 --json --non-interactive + --no-color`. It returns NVML-confirmed CUDA context holders under that process + tree. Choose a worker from workload, PID/start-time, command line, and GPU + UUID evidence. Measure workers separately when several ranks are relevant. +5. If selectors are unknown, validate broad activity endpoints, then make one + short duration-bounded survey with `--events-out survey.jsonl`. For a kernel + inventory, use `cuda:kernel_start` to `cuda:kernel_end` with `exact`; keep the + survey short and give `--max-events` headroom. +6. Run `scripts/analyze_trace.py survey.jsonl`. Use its kernel names, selector + hints, duration aggregates, launch variants, stream distribution, busy union, + overlap factor, and adjacent gaps to form one narrow hypothesis. Read + [references/trace-analysis.md](references/trace-analysis.md) when interpreting + the report. +7. Run `xprobe validate --pid WORKER_PID --from SELECTOR --to SELECTOR --match + POLICY --json --non-interactive --no-color`. Stop when `valid` is false. If `agent_activation` is `injection_required`, disclose that `measure` will - ptrace the target and leave the CUPTI shared object mapped. Read - `policy_recommendation`; when it differs from the requested policy, rerun - `validate` explicitly with the recommended or another compatible policy. -4. Run one bounded `xprobe measure --pid PID --from SELECTOR --to SELECTOR - --match POLICY --samples N --timeout-ms MS --json --non-interactive - --no-color`. Use `--events-out PATH [--format jsonl|chrome]` when an artifact - is needed. For a versioned configuration, use `xprobe measure --spec FILE`. -5. Check `status`, matched/unmatched/ambiguous/dropped counts, clock alignment, - `estimated_error_ns`, collection completeness and CUPTI buffer utilization, - correlation method/confidence/score, every warning, and each `evidence` pair - before interpreting latency. + ptrace the target and leave the CUPTI shared object mapped. Use + `policy_recommendation` explicitly; xprobe never changes policy for the caller. +8. Run one bounded `xprobe measure` for that hypothesis. Set samples or duration, + timeout, and max-events; write `--events-out` when the capture may need audit + or offline re-correlation. Use `--spec FILE` for a versioned live configuration. +9. Check `status`, matched/unmatched/ambiguous/dropped counts, collection + completeness, buffer utilization, clock alignment, estimated error, + correlation method/confidence/score, warnings, and every evidence pair. +10. Repeat only with a stated reason: narrow the selector, select another worker + or stream, change an explicitly compatible policy, or test the next boundary. + Recheck application latency after profiling and report observed overhead. For completed captures, replace `--pid` with one or more `--input` arguments. -Use `examples/request-to-first-kernel.json` as a `MeasurementSpec` shape after -replacing the target identity and selectors with values for the selected worker. +Use the [kernel duration](examples/kernel-duration.json), +[same-stream gap](examples/same-stream-kernel-gap.json), +[host span](examples/host-function-span.json), and +[memcpy duration](examples/memcpy-duration.json) specs, plus the +[CUDA synchronization API](examples/cuda-api-duration.json) shape, after +replacing target identity and selectors. Each bounded call answers one +hypothesis; orchestration remains the agent framework's responsibility. -## Guardrails +## Stop conditions -- Do not use unbounded collection. Set samples or duration, timeout, and a - finite event limit. -- Do not continue after target reuse, permission failure, invalid selectors, or - unavailable required collectors. -- Expect a warning on automatic CUPTI injection. Do not suppress or misreport - target mutation; do not manually unload the mapped agent afterward. -- Do not claim exact causality for `first-after` or `nearest`. -- Do not ignore drops, unmatched or ambiguous pairs, unknown clock error, or a - `timed_out` status. -- On a nonzero result after collection, read `details` and `hints` and inspect - the `events-out` artifact before changing selectors, policy, or bounds. +- Stop on target reuse, permission failure, invalid selectors, unavailable + collectors, drops, incomplete capture, unknown clock alignment, or unexamined + ambiguity. Read structured `details`, `hints`, and any failed-capture artifact. +- Do not claim request causality from `first-after` or `nearest`. Do not compare + or sum events across streams as if they were serial. +- Stop using xprobe once evidence isolates time inside one kernel. Kernel + duration cannot explain warp stalls, cache misses, occupancy, instruction mix, + or Tensor Core utilization; hand that question to NCU or PC sampling. +- Avoid continuous or repeated exploratory capture in one production process. + Use short surveys, narrow formal measurements, and a post-profile baseline. diff --git a/skills/xprobe-measure-latency/agents/openai.yaml b/skills/xprobe-measure-latency/agents/openai.yaml index 406d683..cb26558 100644 --- a/skills/xprobe-measure-latency/agents/openai.yaml +++ b/skills/xprobe-measure-latency/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Xprobe Latency Measurement" - short_description: "Measure bounded CPU and GPU event latency" - default_prompt: "Use $xprobe-measure-latency to measure bounded latency between two observable events." + short_description: "Investigate bounded CPU and GPU latency" + default_prompt: "Use $xprobe-measure-latency to survey this workload, derive evidence-based selectors, and measure the suspected latency boundary." diff --git a/skills/xprobe-measure-latency/examples/cuda-api-duration.json b/skills/xprobe-measure-latency/examples/cuda-api-duration.json new file mode 100644 index 0000000..8bb7aea --- /dev/null +++ b/skills/xprobe-measure-latency/examples/cuda-api-duration.json @@ -0,0 +1,15 @@ +{ + "schema_version": "2.0", + "name": "cuda_stream_synchronize_duration", + "target": { + "pid": 1234, + "process_start_time": 987654 + }, + "start_selector": "cuda:runtime_api:cudaStreamSynchronize:entry", + "end_selector": "cuda:runtime_api:cudaStreamSynchronize:exit", + "match_policy": "exact", + "samples": 100, + "duration_ms": null, + "timeout_ms": 30000, + "max_events": 1000 +} diff --git a/skills/xprobe-measure-latency/examples/host-function-span.json b/skills/xprobe-measure-latency/examples/host-function-span.json new file mode 100644 index 0000000..1b586d8 --- /dev/null +++ b/skills/xprobe-measure-latency/examples/host-function-span.json @@ -0,0 +1,15 @@ +{ + "schema_version": "2.0", + "name": "host_function_span", + "target": { + "pid": 1234, + "process_start_time": 987654 + }, + "start_selector": "uprobe:/srv/app/server:handle_request:entry", + "end_selector": "uprobe:/srv/app/server:handle_request:return", + "match_policy": "stack_nested", + "samples": 100, + "duration_ms": null, + "timeout_ms": 30000, + "max_events": 1000 +} diff --git a/skills/xprobe-measure-latency/examples/kernel-duration.json b/skills/xprobe-measure-latency/examples/kernel-duration.json new file mode 100644 index 0000000..6385d94 --- /dev/null +++ b/skills/xprobe-measure-latency/examples/kernel-duration.json @@ -0,0 +1,15 @@ +{ + "schema_version": "2.0", + "name": "kernel_duration", + "target": { + "pid": 1234, + "process_start_time": 987654 + }, + "start_selector": "cuda:kernel_start:name~^vector_add$", + "end_selector": "cuda:kernel_end:name~^vector_add$", + "match_policy": "exact", + "samples": 100, + "duration_ms": null, + "timeout_ms": 30000, + "max_events": 1000 +} diff --git a/skills/xprobe-measure-latency/examples/memcpy-duration.json b/skills/xprobe-measure-latency/examples/memcpy-duration.json new file mode 100644 index 0000000..1864ccb --- /dev/null +++ b/skills/xprobe-measure-latency/examples/memcpy-duration.json @@ -0,0 +1,15 @@ +{ + "schema_version": "2.0", + "name": "host_to_device_memcpy_duration", + "target": { + "pid": 1234, + "process_start_time": 987654 + }, + "start_selector": "cuda:memcpy_start:kind=HtoD", + "end_selector": "cuda:memcpy_end:kind=HtoD", + "match_policy": "exact", + "samples": 100, + "duration_ms": null, + "timeout_ms": 30000, + "max_events": 1000 +} diff --git a/skills/xprobe-measure-latency/examples/same-stream-kernel-gap.json b/skills/xprobe-measure-latency/examples/same-stream-kernel-gap.json new file mode 100644 index 0000000..da047d5 --- /dev/null +++ b/skills/xprobe-measure-latency/examples/same-stream-kernel-gap.json @@ -0,0 +1,15 @@ +{ + "schema_version": "2.0", + "name": "same_stream_kernel_gap", + "target": { + "pid": 1234, + "process_start_time": 987654 + }, + "start_selector": "cuda:kernel_end:name~^producer$", + "end_selector": "cuda:kernel_start:name~^consumer$", + "match_policy": "stream_order", + "samples": 100, + "duration_ms": null, + "timeout_ms": 30000, + "max_events": 2000 +} diff --git a/skills/xprobe-measure-latency/references/cli-contract.md b/skills/xprobe-measure-latency/references/cli-contract.md index 808194f..c73f80e 100644 --- a/skills/xprobe-measure-latency/references/cli-contract.md +++ b/skills/xprobe-measure-latency/references/cli-contract.md @@ -40,6 +40,14 @@ Direct `measure` calls require a positive `--samples` or `--duration-ms` bound. `--timeout-ms` defaults to 30 seconds and `--max-events` to 100,000. Use exactly one source mode: `--pid`, one or more `--input` files, or `--spec`. +Kernel and other GPU activity durations require separate start and end records, +so `max-events` is record capacity rather than sample capacity. Sample completion +is checked after bounded snapshots and does not reserve space in the CUPTI +buffer. `duration-ms` limits correlation from the first selected event and also +sets a live stop from ARM completion; either samples or duration may complete a +call when both are present. Timeout bounds the complete foreground operation and +cleanup. + `--events-out PATH` atomically writes the bounded capture with mode `0600`, not only matched evidence. Collection completeness and CUPTI capacity, observed, retained, dropped, and buffer utilization fields describe capture integrity diff --git a/skills/xprobe-measure-latency/references/investigation.md b/skills/xprobe-measure-latency/references/investigation.md new file mode 100644 index 0000000..f72b7ef --- /dev/null +++ b/skills/xprobe-measure-latency/references/investigation.md @@ -0,0 +1,101 @@ +# Investigation playbook + +## Coordinate the workload + +Wait for the service readiness signal, CUDA context creation, framework warmup, +and JIT compilation. Run `discover` only after the target has submitted CUDA +work. Start a bounded measurement immediately before a controlled request/batch, +or while repeatable traffic is active. A quiet target commonly produces +`NO_MATCHED_SAMPLES`; it is not evidence that the selector is invalid. + +Keep launcher and worker roles separate. `discover --pid ROOT_PID` reports only +CUDA context holders, so a wrapper PID should not be selected unless it appears +as a candidate. Map GPU UUID and command/rank metadata to workload ownership. +Measure multiple relevant workers one at a time. After a restart, rerun discovery +and use PID plus procfs start time; never reuse an old PID-only choice. + +## Survey before selecting + +When kernel names are unknown, first validate and collect all kernel activity for +a short window: + +```bash +xprobe validate --pid "$PID" \ + --from cuda:kernel_start --to cuda:kernel_end --match exact \ + --json --non-interactive --no-color + +xprobe measure --pid "$PID" \ + --from cuda:kernel_start --to cuda:kernel_end --match exact \ + --duration-ms 250 --timeout-ms 5000 --max-events 200000 \ + --events-out survey.jsonl --format jsonl \ + --json --non-interactive --no-color + +skills/xprobe-measure-latency/scripts/analyze_trace.py survey.jsonl \ + > survey-analysis.json +``` + +Treat these numbers as starting bounds, not universal defaults. On +`EVENT_RATE_TOO_HIGH`, inspect the written artifact and error counters. Shorten +the duration or use a simple prefix/suffix/contains selector before raising +capacity. Preserve the artifact as evidence for the change. + +## Derive CUDA selectors + +Use `kernels.by_name` and `selector_hint` from the analysis report. Kernel +selectors accept a regular expression, but the CUPTI hot path can lower only a +short exact, prefix, suffix, or contains literal (under 128 characters). Complex +regex is applied after a broader capture and may fill capacity quickly. Escape +regex metacharacters and validate the final selector. + +For Triton and other JIT kernels, use captured names plus grid/block variants to +identify a launch family, then correlate it with framework cache metadata, +generated source, or application logs. xprobe does not read JIT cache contents +and cannot select by grid/block. Long mangled names should be narrowed to a +short, observed-unique literal instead of copied wholesale. + +## Derive host selectors + +Resolve the mapped object in the target, then inspect its symbols: + +```bash +readlink -f "/proc/$PID/exe" +cat "/proc/$PID/maps" +readelf -Ws /path/to/object +nm -D --defined-only /path/to/object +``` + +Use `uprobe:::entry|return` when a symbol is available. For +stripped or local code, derive a file offset with `readelf`/`objdump` and use +`uprobe::+0xOFFSET:entry|return`. Always pass the exact candidate to +`validate`; do not infer a runtime virtual address from one process and reuse it +as a file offset. + +## Narrow one hypothesis + +Choose one next boundary from evidence: + +- kernel start to end with `exact` for kernel duration; +- CUDA API exit to kernel start with `exact` for launch delay; +- kernel end to next activity start with `stream-order` for one-stream gaps; +- host function entry to return with `stack-nested` for CPU span; +- host marker to GPU activity with `first-after` only as a disclosed heuristic. + +After capture, analyze the artifact and all result quality fields. Re-correlate a +completed artifact with `measure --input` when only selectors or policy change; +do not attach again merely to recompute pairing. + +```bash +xprobe measure --input survey.jsonl \ + --from 'cuda:kernel_start:name~^selected_kernel$' \ + --to 'cuda:kernel_end:name~^selected_kernel$' \ + --match exact --samples 100 --max-events 200000 \ + --json --non-interactive --no-color +``` + +## Escalate at the right boundary + +xprobe can isolate slow kernels, launch gaps, copies, synchronization boundaries, +host spans, and host-to-GPU timing. Once the remaining time is inside a single +kernel, use NCU or PC sampling for stalls, cache behavior, occupancy, instruction +mix, or Tensor Core utilization. Use a CPU sampling profiler when the unresolved +time is inside an uninstrumented host span. diff --git a/skills/xprobe-measure-latency/references/result-quality.md b/skills/xprobe-measure-latency/references/result-quality.md index 919ec3d..b0a2f8f 100644 --- a/skills/xprobe-measure-latency/references/result-quality.md +++ b/skills/xprobe-measure-latency/references/result-quality.md @@ -1,19 +1,61 @@ # Result quality -Prefer `exact` when both CUDA endpoints carry the same CUPTI correlation ID. -Prefer `stack-nested` for entry/return pairs of the same host function and -`stream-order` for activity endpoints on the same device, context, and stream. -Treat `first-after` and `nearest` as temporal heuristics. - -A result is not sufficient evidence when records were dropped, no samples -matched, the selected process identity changed, or clock alignment failed. -Report unmatched and ambiguous counts alongside matched samples. For a broad -selector, state that another eligible event could have changed the pairing. - -Compare latency values only when the result reports a shared or normalized clock -domain. Report `estimated_error_ns`; `null` means no quantified interpolation -error bound. Inspect the full start and end events in every `evidence` pair when -checking selector scope, correlation IDs, streams, and timestamps. - -`completed` means the requested bound was reached. `timed_out` may still contain -useful partial evidence, but it must be identified as partial. +Prefer `exact` when CUDA endpoints carry the same CUPTI correlation ID. Prefer +`stack-nested` for entry/return pairs of the same host function. Use +`stream-order` only for GPU activity endpoints on the same device, context, and +stream. Treat `first-after` and `nearest` as temporal heuristics, never request +causality. + +Inspect every evidence pair for selector scope, process identity, correlation +IDs, device/context/stream, timestamps, and clock domains. A result is not +sufficient when records were dropped, no samples matched, target identity +changed, collection was incomplete, or clock alignment failed. Report unmatched +and ambiguous counts with matched samples. `estimated_error_ns: null` means no +quantified interpolation error bound. + +## Concurrency + +Group GPU evidence by device, context, and stream before interpreting order. +Events on different streams may overlap and timestamp order does not establish a +request relationship. `stream-order` does not cross stream boundaries. Prefer +correlation-ID `exact` when a deterministic relationship exists. + +Summed kernel or activity duration double-counts overlap. Use GPU `busy_union_ns` +for wall-clock busy time and `overlap_factor` to quantify concurrency. Compare +per-stream gaps separately. A top kernel's `summed_kernel_time_share` describes +its share of summed kernel work, not its exclusive share of wall time. + +## Bounds and completion + +`completed` means either requested samples or duration was reached. `timed_out` +is partial and must be reported as such. For one complete kernel duration, start +and end consume at least two CUPTI records; unmatched boundaries and records +admitted by a broad hot-path filter consume additional capacity. Sample +completion is evaluated from snapshots, so a high-rate buffer can reach +`max-events` before the caller observes the requested sample count. + +For narrow start/end activity pairs, begin with: + +```text +minimum_records = samples * (start_records_per_sample + end_records_per_sample) +max_events >= minimum_records + expected_unmatched_records +``` + +Use at least 2x headroom for stable narrow selectors and 4-10x for high-rate or +broad surveys. For a duration survey, size from a pilot artifact's observed +records per second. Increasing max-events without narrowing a noisy selector +only increases profiler work. + +`duration-ms` limits correlation to a window beginning at the first selected +event. In live mode it also sets a collection stop from ARM completion, so finish +readiness and warmup before invoking `measure`. When both samples and duration +are set, either bound completes the call; timeout remains the outer operation and +cleanup limit. + +## Perturbation + +Record an application-level latency distribution before profiling and repeat it +afterward under the same workload. Report the difference and whether automatic +injection occurred. The injected shared object remains mapped but is logically +disabled after collection. Do not interpret a one-off profiled request as an +unperturbed baseline. diff --git a/skills/xprobe-measure-latency/references/trace-analysis.md b/skills/xprobe-measure-latency/references/trace-analysis.md new file mode 100644 index 0000000..54cd252 --- /dev/null +++ b/skills/xprobe-measure-latency/references/trace-analysis.md @@ -0,0 +1,35 @@ +# Trace analysis + +Run the bundled analyzer on an Event JSONL artifact: + +```bash +skills/xprobe-measure-latency/scripts/analyze_trace.py capture.jsonl \ + > capture-analysis.json +``` + +The command rejects non-2.0 events, clock-domain mismatches within an activity, +and negative durations. Unpaired activity boundaries remain visible in +`events` and produce a warning; they are excluded from duration metrics. + +Read the report in this order: + +1. `events`: confirm counts and zero unpaired starts/ends. +2. `gpu.busy_union_ns`: wall-clock union of kernel, memcpy, and memset activity. +3. `gpu.summed_activity_ns` and `overlap_factor`: quantify overlap; do not use + summed activity as wall time when the factor is above 1. +4. `kernels.by_name`: rank total time, then inspect p50/p95 and + `summed_kernel_time_share`. +5. `launch_variants`: separate the same kernel name by grid/block shape before + concluding that its latency distribution is homogeneous. +6. `streams`: compare activity counts and adjacent kernel gaps per + PID/device/context/stream. Cross-stream order is not causal evidence. +7. `memcpy.by_kind`: inspect transfer duration and bytes separately from kernel + work. + +`selector_hint` is exact for names shorter than the CUPTI filter bound. For long +names it may be a prefix or suffix unique only among names observed in that +capture; validate it and repeat a narrow survey before treating it as stable. + +The analyzer reports descriptive activity timing. It does not assign work to a +request, calculate kernel hardware efficiency, or turn temporal proximity into +causality. diff --git a/skills/xprobe-measure-latency/scripts/analyze_trace.py b/skills/xprobe-measure-latency/scripts/analyze_trace.py new file mode 100755 index 0000000..f326eb1 --- /dev/null +++ b/skills/xprobe-measure-latency/scripts/analyze_trace.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 +"""Summarize a bounded xprobe Event JSONL artifact.""" + +import argparse +import collections +import json +import math +import pathlib +import re +import sys + + +ACTIVITY_TYPES = { + "gpu_kernel_start": ("kernel", "start"), + "gpu_kernel_end": ("kernel", "end"), + "gpu_memcpy_start": ("memcpy", "start"), + "gpu_memcpy_end": ("memcpy", "end"), + "gpu_memset_start": ("memset", "start"), + "gpu_memset_end": ("memset", "end"), +} + + +def percentile(values: list[int], percent: int) -> int | None: + if not values: + return None + ordered = sorted(values) + rank = max(1, math.ceil(percent * len(ordered) / 100)) + return ordered[rank - 1] + + +def duration_summary(values: list[int]) -> dict: + if not values: + return { + "count": 0, + "total_ns": 0, + "min_ns": None, + "p50_ns": None, + "p95_ns": None, + "max_ns": None, + } + return { + "count": len(values), + "total_ns": sum(values), + "min_ns": min(values), + "p50_ns": percentile(values, 50), + "p95_ns": percentile(values, 95), + "max_ns": max(values), + } + + +def interval_union_ns(intervals: list[tuple[int, int]]) -> int: + if not intervals: + return 0 + ordered = sorted(intervals) + union = 0 + current_start, current_end = ordered[0] + for start, end in ordered[1:]: + if start <= current_end: + current_end = max(current_end, end) + else: + union += current_end - current_start + current_start, current_end = start, end + return union + current_end - current_start + + +def dimensions(cuda: dict, field: str) -> tuple[int, int, int] | None: + value = cuda.get(field) + if value is None: + return None + return (value["x"], value["y"], value["z"]) + + +def activity_name(kind: str, cuda: dict) -> str: + if kind == "kernel": + return cuda.get("kernel_name") or "" + if kind == "memcpy": + return cuda.get("memcpy_kind") or "unknown" + return "memset" + + +def activity_key(kind: str, event: dict) -> tuple: + cuda = event.get("cuda") or {} + return ( + kind, + event.get("pid"), + cuda.get("device_id"), + cuda.get("context_id"), + cuda.get("stream_id"), + cuda.get("correlation_id"), + activity_name(kind, cuda), + ) + + +def pair_activities(events: list[dict]) -> tuple[list[dict], int, int]: + boundaries = [] + for event in events: + activity = ACTIVITY_TYPES.get(event.get("event_type")) + if activity is not None: + kind, boundary = activity + boundaries.append((event["timestamp_ns"], boundary != "start", kind, boundary, event)) + boundaries.sort(key=lambda item: (item[0], item[1])) + + starts: dict[tuple, collections.deque] = collections.defaultdict(collections.deque) + intervals = [] + unmatched_ends = 0 + for _, _, kind, boundary, event in boundaries: + key = activity_key(kind, event) + if boundary == "start": + starts[key].append(event) + continue + if not starts[key]: + unmatched_ends += 1 + continue + start = starts[key].popleft() + if start["clock_domain"] != event["clock_domain"]: + raise ValueError( + f"activity pair {start.get('event_id')} / {event.get('event_id')} " + "uses different clock domains" + ) + start_ns = start["timestamp_ns"] + end_ns = event["timestamp_ns"] + if end_ns < start_ns: + raise ValueError( + f"activity pair {start.get('event_id')} / {event.get('event_id')} " + "has a negative duration" + ) + cuda = start.get("cuda") or {} + intervals.append( + { + "kind": kind, + "name": activity_name(kind, cuda), + "pid": start.get("pid"), + "device_id": cuda.get("device_id"), + "context_id": cuda.get("context_id"), + "stream_id": cuda.get("stream_id"), + "start_ns": start_ns, + "end_ns": end_ns, + "duration_ns": end_ns - start_ns, + "grid": dimensions(cuda, "grid"), + "block": dimensions(cuda, "block"), + "bytes": cuda.get("bytes"), + } + ) + unmatched_starts = sum(len(queue) for queue in starts.values()) + return intervals, unmatched_starts, unmatched_ends + + +def selector_hint(name: str, names: list[str]) -> dict | None: + if name == "": + return None + escaped = re.escape(name) + if len(name) < 128: + regex = f"^{escaped}$" + scope = "exact_name" + else: + alternatives = [candidate for candidate in names if candidate != name] + candidates = [] + for length in range(min(16, len(name)), min(127, len(name)) + 1): + prefix = name[:length] + if all(not candidate.startswith(prefix) for candidate in alternatives): + candidates.append((length, f"^{re.escape(prefix)}.*", "prefix")) + break + suffix = name[-length:] + if all(not candidate.endswith(suffix) for candidate in alternatives): + candidates.append((length, f".*{re.escape(suffix)}$", "suffix")) + break + if not candidates: + return None + _, regex, filter_kind = min(candidates) + scope = f"{filter_kind}_unique_in_capture" + return { + "name_regex": regex, + "scope": scope, + "start_selector": f"cuda:kernel_start:name~{regex}", + "end_selector": f"cuda:kernel_end:name~{regex}", + } + + +def dimensions_object(value: tuple[int, int, int] | None) -> dict | None: + if value is None: + return None + return {"x": value[0], "y": value[1], "z": value[2]} + + +def kernel_report(intervals: list[dict]) -> dict: + kernels = [interval for interval in intervals if interval["kind"] == "kernel"] + names = sorted({interval["name"] for interval in kernels}) + total = sum(interval["duration_ns"] for interval in kernels) + grouped = collections.defaultdict(list) + for interval in kernels: + grouped[interval["name"]].append(interval) + + reports = [] + for name, group in grouped.items(): + variants = collections.defaultdict(list) + for interval in group: + variants[(interval["grid"], interval["block"])].append(interval["duration_ns"]) + reports.append( + { + "name": name, + "duration": duration_summary([item["duration_ns"] for item in group]), + "summed_kernel_time_share": round( + sum(item["duration_ns"] for item in group) / total, 6 + ) + if total + else None, + "selector_hint": selector_hint(name, names), + "launch_variants": [ + { + "grid": dimensions_object(grid), + "block": dimensions_object(block), + "duration": duration_summary(durations), + } + for (grid, block), durations in sorted( + variants.items(), key=lambda item: repr(item[0]) + ) + ], + } + ) + reports.sort(key=lambda item: (-item["duration"]["total_ns"], item["name"])) + kernel_intervals = [(item["start_ns"], item["end_ns"]) for item in kernels] + busy = interval_union_ns(kernel_intervals) + return { + "duration": duration_summary([item["duration_ns"] for item in kernels]), + "busy_union_ns": busy, + "overlap_factor": round(total / busy, 6) if busy else None, + "by_name": reports, + } + + +def memcpy_report(intervals: list[dict]) -> dict: + copies = [interval for interval in intervals if interval["kind"] == "memcpy"] + grouped = collections.defaultdict(list) + for interval in copies: + grouped[interval["name"]].append(interval) + return { + "duration": duration_summary([item["duration_ns"] for item in copies]), + "total_bytes": sum(item["bytes"] or 0 for item in copies), + "by_kind": [ + { + "kind": kind, + "duration": duration_summary([item["duration_ns"] for item in group]), + "total_bytes": sum(item["bytes"] or 0 for item in group), + } + for kind, group in sorted(grouped.items()) + ], + } + + +def stream_report(intervals: list[dict]) -> list[dict]: + grouped = collections.defaultdict(list) + for interval in intervals: + key = ( + interval["pid"], + interval["device_id"], + interval["context_id"], + interval["stream_id"], + ) + grouped[key].append(interval) + + reports = [] + for (pid, device, context, stream), group in grouped.items(): + kernels = sorted( + (item for item in group if item["kind"] == "kernel"), + key=lambda item: (item["start_ns"], item["end_ns"]), + ) + gaps = [] + overlaps = [] + for previous, current in zip(kernels, kernels[1:]): + delta = current["start_ns"] - previous["end_ns"] + if delta >= 0: + gaps.append(delta) + else: + overlaps.append(-delta) + reports.append( + { + "pid": pid, + "device_id": device, + "context_id": context, + "stream_id": stream, + "event_counts": dict( + sorted(collections.Counter(item["kind"] for item in group).items()) + ), + "summed_activity_ns": sum(item["duration_ns"] for item in group), + "busy_union_ns": interval_union_ns( + [(item["start_ns"], item["end_ns"]) for item in group] + ), + "adjacent_kernel_gaps": duration_summary(gaps), + "adjacent_kernel_overlaps": duration_summary(overlaps), + } + ) + reports.sort( + key=lambda item: ( + item["pid"] if item["pid"] is not None else -1, + item["device_id"] if item["device_id"] is not None else -1, + item["context_id"] if item["context_id"] is not None else -1, + item["stream_id"] if item["stream_id"] is not None else -1, + ) + ) + return reports + + +def load_events(path: pathlib.Path) -> list[dict]: + events = [] + with path.open(encoding="utf-8") as source: + for line_number, line in enumerate(source, 1): + if not line.strip(): + continue + event = json.loads(line) + if event.get("schema_version") != "2.0": + raise ValueError( + f"{path}:{line_number}: expected schema_version 2.0" + ) + events.append(event) + if not events: + raise ValueError(f"{path}: trace contains no events") + return events + + +def analyze(path: pathlib.Path) -> dict: + events = load_events(path) + intervals, unmatched_starts, unmatched_ends = pair_activities(events) + activity_ranges = [(item["start_ns"], item["end_ns"]) for item in intervals] + busy = interval_union_ns(activity_ranges) + summed = sum(item["duration_ns"] for item in intervals) + span = ( + max(item["end_ns"] for item in intervals) + - min(item["start_ns"] for item in intervals) + if intervals + else 0 + ) + warnings = [] + if unmatched_starts or unmatched_ends: + warnings.append( + { + "code": "UNPAIRED_ACTIVITY_BOUNDARIES", + "message": "activity metrics exclude unpaired start or end records", + } + ) + return { + "analysis_version": "1.0", + "input": str(path), + "events": { + "total": len(events), + "by_type": dict( + sorted(collections.Counter(event["event_type"] for event in events).items()) + ), + "paired_activities": len(intervals), + "unmatched_activity_starts": unmatched_starts, + "unmatched_activity_ends": unmatched_ends, + }, + "gpu": { + "span_ns": span, + "busy_union_ns": busy, + "summed_activity_ns": summed, + "overlap_factor": round(summed / busy, 6) if busy else None, + "busy_fraction_of_span": round(busy / span, 6) if span else None, + }, + "kernels": kernel_report(intervals), + "memcpy": memcpy_report(intervals), + "memset": { + "duration": duration_summary( + [ + item["duration_ns"] + for item in intervals + if item["kind"] == "memset" + ] + ) + }, + "streams": stream_report(intervals), + "warnings": warnings, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Summarize kernels, copies, overlap, streams, and gaps in xprobe Event JSONL" + ) + parser.add_argument("trace", type=pathlib.Path) + args = parser.parse_args() + json.dump(analyze(args.trace), sys.stdout, indent=2, sort_keys=True) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/tests/agent-contract/test_contract.py b/tests/agent-contract/test_contract.py index 8cc4aa4..8498e22 100755 --- a/tests/agent-contract/test_contract.py +++ b/tests/agent-contract/test_contract.py @@ -48,15 +48,17 @@ def check_skill(workspace: pathlib.Path) -> None: assert frontmatter is not None assert re.search(r"^name: xprobe-measure-latency$", frontmatter.group(1), re.MULTILINE) assert re.search(r"^description: .+", frontmatter.group(1), re.MULTILINE) + normalized_skill = re.sub(r"\s+", " ", skill) ordered_steps = [ + "xprobe --version", "xprobe doctor", "xprobe discover", "xprobe validate", "xprobe measure", "Check `status`", ] - positions = [skill.index(step) for step in ordered_steps] + positions = [normalized_skill.index(step) for step in ordered_steps] assert positions == sorted(positions) for quality_field in ( "unmatched", @@ -69,16 +71,82 @@ def check_skill(workspace: pathlib.Path) -> None: "confidence", "evidence", ): - assert quality_field in skill - assert "Expect a warning on automatic CUPTI injection" in skill - assert "leave the CUPTI shared object mapped" in skill - assert "Do not use unbounded" in skill + assert quality_field in normalized_skill + for investigation_step in ( + "application-level latency baseline", + "short duration-bounded survey", + "scripts/analyze_trace.py", + "selector hints", + "busy union", + "overlap factor", + "NCU or PC sampling", + ): + assert investigation_step in normalized_skill + assert "leave the CUPTI shared object mapped" in normalized_skill for relative_link in re.findall(r"\]\(([^)]+)\)", skill): target = (skill_root / relative_link).resolve() assert target.is_relative_to(skill_root), relative_link assert target.is_file(), relative_link + analyzer = skill_root / "scripts/analyze_trace.py" + assert analyzer.is_file() + assert os.access(analyzer, os.X_OK) + assert not list(skill_root.rglob("__pycache__")) + assert not list(skill_root.rglob("*.pyc")) + examples = sorted((skill_root / "examples").glob("*.json")) + assert len(examples) >= 5 + policies = set() + for example in examples: + specification = json.loads(example.read_text()) + assert specification["schema_version"] == "2.0", example + assert specification["max_events"] > 0, example + policies.add(specification["match_policy"]) + assert {"exact", "first_after", "stack_nested", "stream_order"} <= policies + + investigation = (skill_root / "references/investigation.md").read_text() + quality = (skill_root / "references/result-quality.md").read_text() + trace_analysis = (skill_root / "references/trace-analysis.md").read_text() + normalized_investigation = re.sub(r"\s+", " ", investigation) + normalized_quality = re.sub(r"\s+", " ", quality) + normalized_trace_analysis = re.sub(r"\s+", " ", trace_analysis) + for required in ( + "Triton", + "procfs start time", + "EVENT_RATE_TOO_HIGH", + "readelf -Ws", + "NO_MATCHED_SAMPLES", + ): + assert required in normalized_investigation + for required in ( + "minimum_records", + "first selected event", + "ARM completion", + "Summed kernel", + "profiler", + ): + assert required in normalized_quality + for required in ( + "busy_union_ns", + "summed_activity_ns", + "launch_variants", + "Cross-stream", + ): + assert required in normalized_trace_analysis + + engineering_rules = re.sub( + r"\s+", " ", (workspace / "AGENTS.md").read_text() + ) + for required in ( + "external reviews as evidence to investigate", + "dedicated feature or fix branch", + "Rebase pull requests", + "GLIBC_2.34 ceiling", + "downloading the public archive", + "transient infrastructure failure", + ): + assert required in engineering_rules + entries = { "codex": workspace / "AGENTS.md", "claude": workspace / "CLAUDE.md", diff --git a/tests/agent-contract/test_trace_analysis.py b/tests/agent-contract/test_trace_analysis.py new file mode 100755 index 0000000..b02e100 --- /dev/null +++ b/tests/agent-contract/test_trace_analysis.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +import json +import pathlib +import subprocess +import tempfile + + +def event( + sequence: int, + event_type: str, + timestamp_ns: int, + stream_id: int, + correlation_id: int, + *, + kernel_name: str | None = None, + grid: tuple[int, int, int] | None = None, + block: tuple[int, int, int] | None = None, + memcpy_kind: str | None = None, + bytes_count: int | None = None, +) -> dict: + def dim(value: tuple[int, int, int] | None) -> dict | None: + if value is None: + return None + return {"x": value[0], "y": value[1], "z": value[2]} + + return { + "schema_version": "2.0", + "event_id": f"event-{sequence}", + "sequence": sequence, + "event_type": event_type, + "pid": 1234, + "timestamp_ns": timestamp_ns, + "clock_domain": "cupti", + "cuda": { + "device_id": 0, + "context_id": 1, + "stream_id": stream_id, + "correlation_id": correlation_id, + "kernel_name": kernel_name, + "grid": dim(grid), + "block": dim(block), + "memcpy_kind": memcpy_kind, + "bytes": bytes_count, + }, + } + + +def pair(sequence: int, kind: str, start: int, end: int, **kwargs: object) -> list[dict]: + return [ + event(sequence, f"gpu_{kind}_start", start, **kwargs), + event(sequence + 1, f"gpu_{kind}_end", end, **kwargs), + ] + + +def main() -> None: + workspace = pathlib.Path(__file__).resolve().parents[2] + script = workspace / "skills/xprobe-measure-latency/scripts/analyze_trace.py" + events = [ + *pair( + 1, + "kernel", + 0, + 100, + stream_id=7, + correlation_id=1, + kernel_name="vector_add", + grid=(1, 1, 1), + block=(32, 1, 1), + ), + *pair( + 3, + "kernel", + 120, + 220, + stream_id=7, + correlation_id=2, + kernel_name="vector_add", + grid=(2, 1, 1), + block=(64, 1, 1), + ), + *pair( + 5, + "kernel", + 50, + 150, + stream_id=9, + correlation_id=3, + kernel_name="reduce", + grid=(1, 1, 1), + block=(128, 1, 1), + ), + *pair( + 7, + "memcpy", + 220, + 270, + stream_id=7, + correlation_id=4, + memcpy_kind="HtoD", + bytes_count=4096, + ), + ] + with tempfile.TemporaryDirectory(prefix="xprobe-trace-analysis-") as directory: + trace = pathlib.Path(directory) / "events.jsonl" + trace.write_text("".join(json.dumps(item) + "\n" for item in events)) + completed = subprocess.run( + [script, trace], check=True, capture_output=True, text=True + ) + unpaired_trace = pathlib.Path(directory) / "unpaired.jsonl" + unpaired_trace.write_text(json.dumps(events[0]) + "\n") + unpaired = json.loads( + subprocess.run( + [script, unpaired_trace], check=True, capture_output=True, text=True + ).stdout + ) + invalid_trace = pathlib.Path(directory) / "invalid.jsonl" + invalid = dict(events[0], schema_version="1.0") + invalid_trace.write_text(json.dumps(invalid) + "\n") + rejected = subprocess.run( + [script, invalid_trace], check=False, capture_output=True, text=True + ) + report = json.loads(completed.stdout) + assert report["events"]["total"] == 8 + assert report["events"]["paired_activities"] == 4 + assert report["gpu"]["busy_union_ns"] == 270 + assert report["gpu"]["summed_activity_ns"] == 350 + assert report["gpu"]["overlap_factor"] == 1.296296 + assert report["kernels"]["busy_union_ns"] == 220 + assert report["kernels"]["overlap_factor"] == 1.363636 + vector_add = report["kernels"]["by_name"][0] + assert vector_add["name"] == "vector_add" + assert vector_add["duration"]["count"] == 2 + assert vector_add["duration"]["total_ns"] == 200 + assert len(vector_add["launch_variants"]) == 2 + assert vector_add["selector_hint"]["scope"] == "exact_name" + assert report["memcpy"]["total_bytes"] == 4096 + assert report["memset"]["duration"]["count"] == 0 + stream = next(item for item in report["streams"] if item["stream_id"] == 7) + assert stream["adjacent_kernel_gaps"]["count"] == 1 + assert stream["adjacent_kernel_gaps"]["p50_ns"] == 20 + assert report["warnings"] == [] + assert unpaired["events"]["unmatched_activity_starts"] == 1 + assert unpaired["warnings"][0]["code"] == "UNPAIRED_ACTIVITY_BOUNDARIES" + assert rejected.returncode != 0 + assert "expected schema_version 2.0" in rejected.stderr + + +if __name__ == "__main__": + main() diff --git a/tests/install/test_install.sh b/tests/install/test_install.sh index 645e86c..2862442 100755 --- a/tests/install/test_install.sh +++ b/tests/install/test_install.sh @@ -24,8 +24,7 @@ elif [ "$#" -eq 0 ]; then "$source_dir/lib/xprobe/cuda13" \ "$source_dir/include/xprobe" \ "$source_dir/docs" \ - "$source_dir/schemas" \ - "$source_dir/skills" + "$source_dir/schemas" cp "$root/install.sh" "$source_dir/install.sh" printf '#!/bin/sh\nprintf "xprobe %s\\n"\n' "$installer_version" >"$source_dir/bin/xprobe" chmod 0755 "$source_dir/bin/xprobe" "$source_dir/install.sh" @@ -35,6 +34,7 @@ elif [ "$#" -eq 0 ]; then printf 'license\n' >"$source_dir/LICENSE" printf 'readme\n' >"$source_dir/README.md" printf 'agents\n' >"$source_dir/AGENTS.md" + cp -R "$root/skills" "$source_dir/skills" else printf 'usage: test_install.sh [release-archive]\n' >&2 exit 2 @@ -70,6 +70,8 @@ test -f "$prefix/share/xprobe/LICENSE" test -d "$prefix/share/xprobe/docs" test -d "$prefix/share/xprobe/schemas" test -d "$prefix/share/xprobe/skills" +test -x \ + "$prefix/share/xprobe/skills/xprobe-measure-latency/scripts/analyze_trace.py" "$prefix/bin/xprobe" --version HOME=$temporary_dir/home FAKE_GLIBC_VERSION=2.34 \