Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .github/workflows/ecosystem-freshness.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: ecosystem-freshness

on:
schedule:
# Nightly at 06:00 UTC
- cron: '0 6 * * *'
workflow_dispatch:
repository_dispatch:
types: [package-published]

jobs:
live-verify:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 24
cache: npm
- run: npm ci
- name: Ecosystem freshness (live)
run: npm run verify:ecosystem:live
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ These three values appear across many files. If any of them change, every refere

## Spec source

The v0.1 specification lives in the canonical spec location: `https://github.com/workspace-json/agents-audit/tree/main/packages/spec` (see `spec.mdx` in that repo). Substantive changes to the spec go through the RFC process; this file is the rendered output of that process, not the place to draft proposals.
The v0.1 specification lives in the canonical spec location: `https://github.com/workspacejson/standard/tree/main/packages/spec` (see `spec.mdx` in that repo). Substantive changes to the spec go through the RFC process; this file is the rendered output of that process, not the place to draft proposals.

## Content files

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ Site runs at `localhost:4321`.

Corrections and improvements to the documentation are welcome via pull request.

Substantive changes to the specification itself should go through the RFC process at [github.com/workspace-json/agents-audit/tree/main/packages/spec](https://github.com/workspace-json/agents-audit/tree/main/packages/spec), not here.
Substantive changes to the specification itself should go through the RFC process at [github.com/workspacejson/standard/tree/main/packages/spec](https://github.com/workspacejson/standard/tree/main/packages/spec), not here.

**Adding your tool to the implementations list:** If your tool reads or writes workspace.json, open a PR editing [`src/content/docs/implementations.mdx`](src/content/docs/implementations.mdx). Add one line with your tool name, a link, and one sentence on what it does. We list any tool that documents workspace.json support in its public docs.

Expand Down
2 changes: 1 addition & 1 deletion astro.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ export default defineConfig({
},
social: [
{ icon: 'rss', label: 'RSS', href: '/blog/rss.xml' },
{ icon: 'github', label: 'GitHub', href: `https://github.com/${contract.tools.audit.repository}` },
{ icon: 'github', label: 'GitHub', href: `https://github.com/${contract.tools.audit.publicationRepository}` },
],
customCss: ['./src/styles/custom.css'],
expressiveCode: {
Expand Down
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"scripts": {
"dev": "astro dev",
"start": "astro dev",
"build": "npm run schema:verify && npm run content:verify && astro build",
"build": "npm run schema:verify && npm run content:verify && npm run verify:ecosystem && astro build",
"build:local": "astro build",
"preview": "astro preview",
"astro": "astro",
Expand All @@ -14,6 +14,8 @@
"schema:verify": "node scripts/materialize-schema.mjs --verify",
"schema:verify-published": "node scripts/verify-published-schema.mjs",
"content:verify": "node scripts/verify-content-contract.mjs",
"verify:ecosystem": "node scripts/verify-ecosystem.mjs",
"verify:ecosystem:live": "node scripts/verify-ecosystem.mjs --live",
"verify:site": "npm run build && playwright test",
"verify:site:contract": "playwright test --project=contract",
"verify:site:update": "npm run build && UPDATE_BASELINES=1 playwright test --update-snapshots",
Expand Down
6 changes: 3 additions & 3 deletions public/schema/v1.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
},
"fileIndex": {
"type": "object",
"description": "Per-file behavioral intelligence keyed by relative path. Read by Buildomator.",
"description": "Per-file behavioral intelligence keyed by repository-root-relative POSIX path (forward slashes, no leading \"./\", no drive letters).",
"additionalProperties": {
"type": "object",
"properties": {
Expand All @@ -74,7 +74,7 @@
"type": "object",
"required": ["files", "rate", "occurrences", "generated"],
"properties": {
"files": { "type": "array", "items": { "type": "string" }, "minItems": 2, "maxItems": 2 },
"files": { "type": "array", "items": { "type": "string" }, "minItems": 2, "maxItems": 2, "description": "Unordered pair (set semantics — position is NOT meaningful; join by membership, not index). Each entry is a repository-root-relative POSIX path." },
"rate": { "type": "number", "minimum": 0, "maximum": 1 },
"occurrences": { "type": "integer", "minimum": 0 },
"generated": { "type": "boolean" }
Expand All @@ -89,7 +89,7 @@
"type": "object",
"required": ["file", "changeCount", "revertCount", "revertRate", "fragilityScore", "excluded"],
"properties": {
"file": { "type": "string" },
"file": { "type": "string", "description": "Repository-root-relative POSIX path (forward slashes, no leading \"./\")." },
"changeCount": { "type": "integer", "minimum": 0 },
"revertCount": { "type": "integer", "minimum": 0 },
"revertRate": { "type": "number", "minimum": 0, "maximum": 1 },
Expand Down
Binary file added public/tally/product/tally_impact_screen.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
10 changes: 10 additions & 0 deletions scripts/verify-content-contract.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,16 @@ if (!tallyPage.includes("from '../../../data/tally-showcase'")) {
fail('tally showcase page must import from the typed contract (../../../data/tally-showcase)');
}

// Verify the page imports Image from astro:assets
if (!tallyPage.includes("from 'astro:assets'")) {
fail('tally showcase page must import Image from astro:assets for the product screenshot');
}

// Verify the page imports the product screenshot from src/assets
if (!tallyPage.includes("from '../../../assets/tally-impact-hero.png'")) {
fail('tally showcase page must import the product screenshot from src/assets/tally-impact-hero.png');
}

// Verify the page uses narrativeActs from the contract
if (!tallyPage.includes('narrativeActs')) {
fail('tally showcase page must reference narrativeActs from the typed contract');
Expand Down
253 changes: 253 additions & 0 deletions scripts/verify-ecosystem.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,253 @@
#!/usr/bin/env node

/**
* Ecosystem freshness gate.
*
* Checks that sources/ecosystem-packages.json is consistent with:
* 1. npm registry dist-tags.latest for each package
* 2. The GitHub tag referenced by releaseRef exists
* 3. npm repository metadata matches declared publicationRepository
* 4. The hosted schema hash equals the pinned tarball schema
* 5. The contract.json tools section agrees with the manifest
*
* Usage:
* node scripts/verify-ecosystem.mjs # check manifest internal consistency + contract agreement
* node scripts/verify-ecosystem.mjs --live # also query npm registry and GitHub API (requires network)
*/

import { readFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const LIVE = process.argv.includes('--live');

const manifest = JSON.parse(
readFileSync(path.join(root, 'sources/ecosystem-packages.json'), 'utf8'),
);
const contract = JSON.parse(
readFileSync(path.join(root, 'src/contract.json'), 'utf8'),
);

const errors = [];
const warnings = [];

function fail(msg) {
errors.push(msg);
}

function warn(msg) {
warnings.push(msg);
}

// 1. Manifest structural validation
for (const pkg of manifest.packages) {
for (const field of ['npmPackage', 'role', 'version', 'publicationRepository', 'releaseRef', 'releaseCommit', 'npmIntegrity', 'freshnessPolicy', 'authorityStatus']) {
if (!pkg[field]) {
fail(`${pkg.npmPackage || 'unknown'}: missing required field "${field}"`);
}
}

// canonicalRepository is required for canonical packages; developmentRepository for transitional
if (pkg.authorityStatus === 'canonical' && !pkg.canonicalRepository) {
fail(`${pkg.npmPackage}: authorityStatus is canonical but missing canonicalRepository`);
}
if (pkg.authorityStatus === 'transitional' && !pkg.developmentRepository) {
fail(`${pkg.npmPackage}: authorityStatus is transitional but missing developmentRepository`);
}

if (!/^\d+\.\d+\.\d+$/.test(pkg.version)) {
fail(`${pkg.npmPackage}: version "${pkg.version}" is not semver`);
}

if (!/^[0-9a-f]{40}$/.test(pkg.releaseCommit)) {
fail(`${pkg.npmPackage}: releaseCommit "${pkg.releaseCommit}" is not a 40-char hex SHA`);
}
}

// 2. Contract agreement: contract.tools.* must agree with manifest
const contractTools = contract.tools;
const manifestByNpm = Object.fromEntries(
manifest.packages.map((p) => [p.npmPackage, p]),
);

function checkContractTool(toolKey, expectedNpmPackage) {
const tool = contractTools[toolKey];
if (!tool) {
warn(`contract.tools.${toolKey} not found (may not be modeled yet)`);
return;
}
const pkg = manifestByNpm[expectedNpmPackage];
if (!pkg) {
fail(`contract.tools.${toolKey} references ${expectedNpmPackage} but manifest has no such package`);
return;
}
if (tool.version && tool.version !== pkg.version) {
fail(`contract.tools.${toolKey}.version (${tool.version}) != manifest ${expectedNpmPackage} version (${pkg.version})`);
}
// For canonical packages, check canonicalRepository; for transitional, check developmentRepository
if (tool.authorityStatus === 'canonical' && tool.canonicalRepository && pkg.canonicalRepository) {
if (tool.canonicalRepository !== pkg.canonicalRepository) {
fail(`contract.tools.${toolKey}.canonicalRepository (${tool.canonicalRepository}) != manifest (${pkg.canonicalRepository})`);
}
}
if (tool.authorityStatus === 'transitional' && tool.developmentRepository && pkg.developmentRepository) {
if (tool.developmentRepository !== pkg.developmentRepository) {
fail(`contract.tools.${toolKey}.developmentRepository (${tool.developmentRepository}) != manifest (${pkg.developmentRepository})`);
}
}
if (tool.publicationRepository && tool.publicationRepository !== pkg.publicationRepository) {
fail(`contract.tools.${toolKey}.publicationRepository (${tool.publicationRepository}) != manifest (${pkg.publicationRepository})`);
}
if (tool.releaseRef && tool.releaseRef !== pkg.releaseRef) {
fail(`contract.tools.${toolKey}.releaseRef (${tool.releaseRef}) != manifest (${pkg.releaseRef})`);
}
if (tool.releaseCommit && tool.releaseCommit !== pkg.releaseCommit) {
fail(`contract.tools.${toolKey}.releaseCommit (${tool.releaseCommit}) != manifest (${pkg.releaseCommit})`);
}
if (tool.authorityStatus && tool.authorityStatus !== pkg.authorityStatus) {
fail(`contract.tools.${toolKey}.authorityStatus (${tool.authorityStatus}) != manifest (${pkg.authorityStatus})`);
}
}

checkContractTool('audit', 'agents-audit');
checkContractTool('cli', '@workspacejson/cli');
checkContractTool('codex', '@workspacejson/codex-mcp');

// 3. Schema hash agreement
const schemaSource = contract.standard.schema.source;
const specPkg = manifestByNpm['@workspacejson/spec'];
if (specPkg && schemaSource.sha256) {
// The schema hash in contract must match the spec package's schema
// (verified via materialize-schema.mjs, which checks the actual bytes)
if (schemaSource.publicationRepository !== specPkg.publicationRepository) {
fail(`contract.schema.source.publicationRepository (${schemaSource.publicationRepository}) != manifest @workspacejson/spec publicationRepository (${specPkg.publicationRepository})`);
}
if (schemaSource.ref && specPkg.releaseRef && schemaSource.ref !== specPkg.releaseRef) {
fail(`contract.schema.source.ref (${schemaSource.ref}) != manifest @workspacejson/spec releaseRef (${specPkg.releaseRef})`);
}
}

// 4. No package in manifest should have a nonexistent canonicalRepository
// (we can only check this with --live)
if (LIVE) {
const GITHUB_API = 'https://api.github.com/repos/';

async function checkGitHubRepo(repo) {
const resp = await fetch(`${GITHUB_API}${repo}`, {
headers: { 'Accept': 'application/vnd.github+json' },
});
if (!resp.ok) {
fail(`GitHub API: ${repo} returned ${resp.status}`);
}
return resp.ok;
}

async function checkNpmLatest(npmPackage) {
const resp = await fetch(`https://registry.npmjs.org/${npmPackage}`);
if (!resp.ok) {
fail(`npm registry: ${npmPackage} returned ${resp.status}`);
return null;
}
const data = await resp.json();
const latest = data['dist-tags']?.latest;
if (!latest) {
fail(`npm registry: ${npmPackage} has no dist-tags.latest`);
return null;
}
return latest;
}

async function checkGitHubTagResolves(repo, ref, expectedCommit) {
const resp = await fetch(`${GITHUB_API}${repo}/git/refs/tags/${ref}`, {
headers: { 'Accept': 'application/vnd.github+json' },
});
if (!resp.ok) {
fail(`GitHub API: tag ${ref} in ${repo} returned ${resp.status}`);
return;
}
const data = await resp.json();
const tagObj = data.object;
if (!tagObj || !tagObj.sha) {
fail(`GitHub API: tag ${ref} in ${repo} has no object.sha`);
return;
}
// If lightweight tag, tagObj.sha is the commit directly
if (tagObj.type === 'commit') {
if (tagObj.sha !== expectedCommit) {
fail(`tag ${ref} in ${repo} resolves to ${tagObj.sha} but manifest says ${expectedCommit}`);
}
return;
}
// If annotated tag, dereference to get the commit
const tagResp = await fetch(`${GITHUB_API}${repo}/git/tags/${tagObj.sha}`, {
headers: { 'Accept': 'application/vnd.github+json' },
});
if (!tagResp.ok) {
fail(`GitHub API: cannot dereference tag ${ref} in ${repo}: ${tagResp.status}`);
return;
}
const tagData = await tagResp.json();
const commitSha = tagData.object?.sha;
if (commitSha !== expectedCommit) {
fail(`tag ${ref} in ${repo} resolves to commit ${commitSha} but manifest says ${expectedCommit}`);
}
}

const seenRepos = new Set();
const seenNpm = new Set();

for (const pkg of manifest.packages) {
// Check canonical or development repo exists
const primaryRepo = pkg.canonicalRepository || pkg.developmentRepository;
if (primaryRepo && !seenRepos.has(primaryRepo)) {
await checkGitHubRepo(primaryRepo);
seenRepos.add(primaryRepo);
}

// Check publication repo exists (if different)
if (pkg.publicationRepository !== primaryRepo && !seenRepos.has(pkg.publicationRepository)) {
await checkGitHubRepo(pkg.publicationRepository);
seenRepos.add(pkg.publicationRepository);
}

// Check GitHub tag exists in publication repo and resolves to expected commit
await checkGitHubTagResolves(pkg.publicationRepository, pkg.releaseRef, pkg.releaseCommit);

// Check npm latest matches manifest version
if (!seenNpm.has(pkg.npmPackage)) {
const latest = await checkNpmLatest(pkg.npmPackage);
if (latest && latest !== pkg.version) {
fail(`npm ${pkg.npmPackage} latest is ${latest} but manifest says ${pkg.version}`);
}
seenNpm.add(pkg.npmPackage);
}
}

// Check npm repository metadata matches declared publicationRepository
for (const pkg of manifest.packages) {
const resp = await fetch(`https://registry.npmjs.org/${pkg.npmPackage}/${pkg.version}`);
if (resp.ok) {
const data = await resp.json();
const repoUrl = data.repository?.url || '';
const expectedRepo = `github.com/${pkg.publicationRepository}`;
if (!repoUrl.includes(expectedRepo)) {
warn(`npm ${pkg.npmPackage}@${pkg.version} repository URL (${repoUrl}) does not contain ${expectedRepo}`);
}
}
}
}

// Report
if (warnings.length > 0) {
console.warn('Warnings:');
for (const w of warnings) console.warn(` WARN: ${w}`);
}

if (errors.length > 0) {
console.error('Ecosystem verification failed:');
for (const e of errors) console.error(` FAIL: ${e}`);
process.exit(1);
}

console.log(`ecosystem verified: ${manifest.packages.length} packages${LIVE ? ' (live)' : ''}`);
Loading
Loading