diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..61e67a0 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "Zeid Data Research", + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "features": { + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/python:1": { "version": "3.12" } + }, + "postCreateCommand": "bash .devcontainer/setup.sh", + "remoteEnv": { + "PATH": "${containerEnv:HOME}/.local/bin:${containerEnv:PATH}" + }, + "customizations": { + "vscode": { + "extensions": ["James-Yu.latex-workshop"] + } + } +} diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100644 index 0000000..a98add9 --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Codespace setup. Makes tectonic and the brand fonts ready so the build and +# bootstrap scripts run with no further steps. Runs once when the Codespace is +# created. Safe to re-run. +mkdir -p "$HOME/.local/bin" +export PATH="$HOME/.local/bin:$PATH" +bash tools/scripts/zd_whitepaper_bootstrap.sh --with-tectonic +echo "" +echo "Codespace ready." +echo "Build a paper: tools/scripts/zd_whitepaper_build.sh research/
/" +echo "Draft a paper: feed tools/whitepaper-agent.md to your command-line agent" diff --git a/docs/whitepaper-pipeline.md b/docs/whitepaper-pipeline.md new file mode 100644 index 0000000..d92a6d1 --- /dev/null +++ b/docs/whitepaper-pipeline.md @@ -0,0 +1,60 @@ +# White paper pipeline + +Automated drafting and placement of white papers into `research/
/`, +with a human pull request gate. No paper reaches `main` without review. + +## One time setup +``` +tools/scripts/zd_whitepaper_bootstrap.sh --with-tectonic +``` +Vendors the OFL brand fonts (DM Serif Display, Inter) into +`templates/whitepaper/fonts/`, writes checksums, and installs `tectonic`. +The fonts are local only and are not committed. Re-run on any new machine. + +## Drafting a paper +Feed the operating prompt at `tools/whitepaper-agent.md` to your command-line +research agent, with either a topic or a path to source material: +``` + # generate mode: research and cite +path/to/notes.md # transform mode: build from existing material +``` +The agent classifies the section, drafts with cited sources, builds the PDF, +and opens a PR into `main`. The agent never merges. + +Once a paper folder exists at `research/
//`, you can build it, +branch, commit, push, and open the PR in one command: +``` +tools/scripts/zd_whitepaper_ship.sh research/
/ +``` +It reads the title and section from `meta.yml`, assembles the PR body from +`sources.json`, refuses to run on a dirty tree, and never merges. + +## Drafting in a Codespace (no local clone) + +If the repo is too large to keep on your machine, draft in the cloud. On the +repo, use the green Code button, open the Codespaces tab, and create a Codespace. +The devcontainer in `.devcontainer/` installs `tectonic` and vendors the brand +fonts on first create, so the build and bootstrap scripts work with no further +setup. From the Codespace terminal: +``` +tools/scripts/zd_whitepaper_build.sh research/
/ +``` +The repo lives in the cloud, nothing is downloaded to your disk. To make startup +instant instead of a one minute setup, enable Codespaces prebuilds for the repo +in Settings, Codespaces. + +## Building a paper by hand +``` +tools/scripts/zd_whitepaper_build.sh research/
/ +``` +Detects the toolchain (tectonic, then latexmk, then xelatex), fails loud if +none is present. PDF lands next to the source. + +## Layout contract +- Papers live at `research/
//` (exactly three levels deep). +- Each paper folder holds `paper.tex`, `paper.pdf`, `sources.json`, `meta.yml`. +- `paper.tex` sets `\def\zdroot{../../../}` and includes the house preamble at + `templates/whitepaper/zd-whitepaper.tex`. +- Every claim maps to a real URL in `sources.json`, or it sits in the mandatory + "Limitations and Unverified Claims" section. Link-check CI enforces resolvable + URLs. diff --git a/research/.gitignore b/research/.gitignore new file mode 100644 index 0000000..6879222 --- /dev/null +++ b/research/.gitignore @@ -0,0 +1,5 @@ +**/*.aux +**/*.log +**/*.out +**/*.fls +**/*.fdb_latexmk diff --git a/templates/whitepaper/fonts/.gitignore b/templates/whitepaper/fonts/.gitignore new file mode 100644 index 0000000..90ef0df --- /dev/null +++ b/templates/whitepaper/fonts/.gitignore @@ -0,0 +1,3 @@ +*.ttf +*.otf +SHA256SUMS diff --git a/templates/whitepaper/fonts/README.md b/templates/whitepaper/fonts/README.md new file mode 100644 index 0000000..0616669 --- /dev/null +++ b/templates/whitepaper/fonts/README.md @@ -0,0 +1,8 @@ +# Vendored fonts (SIL Open Font License 1.1) + +- DMSerifDisplay-Regular.ttf, DM Serif Display, Colophon Foundry, OFL 1.1. +- Inter-Regular.ttf, Inter, Rasmus Andersson, OFL 1.1. Instanced to a static + weight (wght 400, opsz 14) from the upstream variable font. + +Both fonts are redistributable under the OFL. Checksums in SHA256SUMS. +Loaded by path so PDF output is identical on every machine. diff --git a/templates/whitepaper/paper.tex.template b/templates/whitepaper/paper.tex.template new file mode 100644 index 0000000..85821ea --- /dev/null +++ b/templates/whitepaper/paper.tex.template @@ -0,0 +1,22 @@ +\documentclass[11pt]{article} +% Papers live at research/
//, exactly three levels under the repo +% root, so \zdroot is ../../../ . Adjust only if you place a paper elsewhere. +\def\zdroot{../../../} +\input{\zdroot templates/whitepaper/zd-whitepaper.tex} +\begin{document} +\zdtitle{PAPER TITLE}{One paragraph abstract.}{YYYY-MM-DD} + +\section{Background} +Body text with a cited claim.\,[1] + +\section{Analysis} +More body. Every factual claim points to a real source.\,[2] + +\section{Limitations and Unverified Claims} +What the evidence does not yet support. Mandatory section, even if short. + +\section{References} +\begin{enumerate}[label={[\arabic*]}] + \item Source title. \href{https://example.com}{example.com}. Accessed YYYY-MM-DD. +\end{enumerate} +\end{document} diff --git a/templates/whitepaper/zd-whitepaper.tex b/templates/whitepaper/zd-whitepaper.tex new file mode 100644 index 0000000..cca50f7 --- /dev/null +++ b/templates/whitepaper/zd-whitepaper.tex @@ -0,0 +1,54 @@ +% Zeid Data Research white paper preamble. XeLaTeX / tectonic. +% Build from the repo root. Fonts vendored at templates/whitepaper/fonts/. +% Fonts: OFL. DM Serif Display (headings), Inter (body). +\usepackage{geometry} +\geometry{a4paper, margin=1in, headheight=14pt} +\usepackage{xcolor} +\definecolor{zdCarbon}{HTML}{1A1A1A} +\definecolor{zdSlate}{HTML}{4A4A4A} +\definecolor{zdOffWhite}{HTML}{F4F1EA} +\definecolor{zdAmber}{HTML}{E0A800} + +\usepackage{fontspec} +% Vendored fonts. Build runs from the repo root, so Path is repo-root-relative. +% Bold and italic for body text are synthesized from the single regular weight. +% If a font file is missing, the build fails loud. Run the bootstrap to vendor +% fonts before building. Do not ship a paper with a silently broken font. +\setmainfont{Inter-Regular.ttf}[ + Path = \zdroot templates/whitepaper/fonts/ , + AutoFakeBold = 2.0 , + AutoFakeSlant = 0.2 ] +\newfontfamily\zdHead{DMSerifDisplay-Regular.ttf}[ + Path = \zdroot templates/whitepaper/fonts/ ] + +\usepackage{titlesec} +\titleformat{\section}{\zdHead\Large\color{zdCarbon}}{\thesection}{0.6em}{}[{\color{zdAmber}\titlerule}] +\titleformat{\subsection}{\zdHead\large\color{zdCarbon}}{\thesubsection}{0.5em}{} +\titlespacing*{\section}{0pt}{1.4em}{0.5em} + +\usepackage{fancyhdr} +\pagestyle{fancy}\fancyhf{} +\fancyfoot[L]{\footnotesize\color{zdSlate}Zeid Data Research Lab} +\fancyfoot[C]{\footnotesize\color{zdSlate}Evidence first} +\fancyfoot[R]{\footnotesize\color{zdSlate}\thepage} +\renewcommand{\headrulewidth}{0pt} +\renewcommand{\footrulewidth}{0.4pt} +\renewcommand{\footrule}{\hbox to\headwidth{\color{zdAmber}\leaders\hrule height \footrulewidth\hfill}} + +\usepackage{microtype} +\usepackage{booktabs} +\usepackage{enumitem} +\usepackage[hidelinks]{hyperref} +\hypersetup{colorlinks=true, linkcolor=zdCarbon, urlcolor=zdAmber, citecolor=zdAmber} +\usepackage{csquotes} + +\color{zdCarbon} + +% Title block macro. Usage: \zdtitle{Title}{Abstract text}{YYYY-MM-DD} +\newcommand{\zdtitle}[3]{% + \thispagestyle{fancy} + \noindent\colorbox{zdCarbon}{\parbox{\dimexpr\textwidth-2\fboxsep}{% + \vspace{4pt}{\zdHead\LARGE\color{zdOffWhite} #1}\\[4pt] + {\footnotesize\color{zdAmber}Zeid Data Research Lab \quad\textbullet\quad #3}\vspace{4pt}}}% + \vspace{1.2em}\par\noindent{\color{zdAmber}\rule{\textwidth}{1.5pt}}\vspace{0.8em}\par + \noindent\textbf{Abstract.} #2\par\vspace{1.2em}} diff --git a/tools/scripts/zd_whitepaper_bootstrap.sh b/tools/scripts/zd_whitepaper_bootstrap.sh new file mode 100644 index 0000000..7237201 --- /dev/null +++ b/tools/scripts/zd_whitepaper_bootstrap.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# One-time bootstrap for the Zeid Data white paper pipeline. +# Vendors the OFL brand fonts, records checksums, installs tectonic if asked. +# Run from anywhere inside the repo. Idempotent. +# Usage: tools/scripts/zd_whitepaper_bootstrap.sh [--with-tectonic] + +LOG=/tmp/zd_whitepaper_bootstrap.log +: > "$LOG" +red() { if [ -t 2 ]; then printf '\033[31m%s\033[0m\n' "$*" >&2; else printf '%s\n' "$*" >&2; fi; } +say() { printf '%s\n' "$*" | tee -a "$LOG"; } + +ROOT=$(git rev-parse --show-toplevel 2>/dev/null) +if [ -z "$ROOT" ] || [ ! -f "$ROOT/templates/whitepaper/zd-whitepaper.tex" ]; then + d=$(pwd) + while [ "$d" != "/" ]; do + [ -f "$d/templates/whitepaper/zd-whitepaper.tex" ] && { ROOT="$d"; break; } + d=$(dirname "$d") + done +fi +if [ -z "$ROOT" ] || [ ! -f "$ROOT/templates/whitepaper/zd-whitepaper.tex" ]; then + red "FAIL: cannot find the pipeline root (templates/whitepaper/). Run from inside the kit or repo." + echo exit_code=2; exit 2 +fi +cd "$ROOT" || { red "FAIL: cannot cd to root"; echo exit_code=2; exit 2; } +say "root=$ROOT" + +FD=templates/whitepaper/fonts +mkdir -p "$FD" + +validate() { [ -f "$1" ] && [ "$(stat -c%s "$1" 2>/dev/null || echo 0)" -gt 20000 ]; } + +# DM Serif Display Regular (static TTF, OFL) +if validate "$FD/DMSerifDisplay-Regular.ttf"; then + say "OK DMSerifDisplay-Regular.ttf already vendored" +else + if curl -fsSL --retry 2 --max-time 90 \ + "https://github.com/google/fonts/raw/main/ofl/dmserifdisplay/DMSerifDisplay-Regular.ttf" \ + -o "$FD/DMSerifDisplay-Regular.ttf" && validate "$FD/DMSerifDisplay-Regular.ttf"; then + say "OK DMSerifDisplay-Regular.ttf fetched" + else + red "FAIL: could not vendor DM Serif Display"; echo exit_code=5; exit 5 + fi +fi + +# Inter Regular (instanced from the OFL variable font for a clean static weight) +if validate "$FD/Inter-Regular.ttf"; then + say "OK Inter-Regular.ttf already vendored" +else + if ! python3 -c "import fontTools" >/dev/null 2>&1; then + say "installing fonttools" + pip install fonttools --break-system-packages -q >>"$LOG" 2>&1 || pip install --user fonttools -q >>"$LOG" 2>&1 + fi + if curl -fsSL --retry 2 --max-time 120 \ + "https://github.com/google/fonts/raw/main/ofl/inter/Inter%5Bopsz,wght%5D.ttf" \ + -o /tmp/Inter-VF.ttf; then + python3 - "$FD" <<'PY' >>"$LOG" 2>&1 +import sys +from fontTools.ttLib import TTFont +from fontTools.varLib.instancer import instantiateVariableFont +fd=sys.argv[1] +f=TTFont('/tmp/Inter-VF.ttf') +instantiateVariableFont(f, {'wght':400, 'opsz':14}, inplace=True) +f.save(fd+'/Inter-Regular.ttf') +PY + if validate "$FD/Inter-Regular.ttf"; then + say "OK Inter-Regular.ttf instanced (wght=400)" + else + red "FAIL: could not instance Inter"; echo exit_code=6; exit 6 + fi + else + red "FAIL: could not fetch Inter variable font"; echo exit_code=6; exit 6 + fi +fi + +sha256sum "$FD"/*.ttf | tee "$FD/SHA256SUMS" >>"$LOG" +say "checksums written to $FD/SHA256SUMS" + +if [ "$1" = "--with-tectonic" ]; then + if command -v tectonic >/dev/null 2>&1; then + say "OK tectonic already installed: $(tectonic --version)" + else + say "installing tectonic" + curl -fsSL --retry 2 --max-time 180 "https://drop-sh.fullyjustified.net" -o /tmp/install-tectonic.sh >>"$LOG" 2>&1 + DEST="${HOME}/.local/bin"; mkdir -p "$DEST" + ( cd "$DEST" && sh /tmp/install-tectonic.sh ) >>"$LOG" 2>&1 + if [ -x "$DEST/tectonic" ]; then + say "OK tectonic installed to $DEST. Ensure $DEST is on PATH." + else + red "WARN: tectonic install did not land in $DEST. Install it manually." + fi + fi +fi + +say "bootstrap complete" +echo exit_code=0 diff --git a/tools/scripts/zd_whitepaper_build.sh b/tools/scripts/zd_whitepaper_build.sh new file mode 100644 index 0000000..8b1789d --- /dev/null +++ b/tools/scripts/zd_whitepaper_build.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Build a Zeid Data white paper to PDF. Run from anywhere inside the repo. +# Usage: tools/scripts/zd_whitepaper_build.sh research/
/ +# or: tools/scripts/zd_whitepaper_build.sh research/
//paper.tex +# Resolves the repo root itself, detects the toolchain, fails loud if none. + +LOG=/tmp/zd_whitepaper_build.log +: > "$LOG" + +red() { if [ -t 2 ]; then printf '\033[31m%s\033[0m\n' "$*" >&2; else printf '%s\n' "$*" >&2; fi; } +say() { printf '%s\n' "$*" | tee -a "$LOG"; } + +ROOT=$(git rev-parse --show-toplevel 2>/dev/null) +if [ -z "$ROOT" ] || [ ! -f "$ROOT/templates/whitepaper/zd-whitepaper.tex" ]; then + d=$(pwd) + while [ "$d" != "/" ]; do + [ -f "$d/templates/whitepaper/zd-whitepaper.tex" ] && { ROOT="$d"; break; } + d=$(dirname "$d") + done +fi +if [ -z "$ROOT" ] || [ ! -f "$ROOT/templates/whitepaper/zd-whitepaper.tex" ]; then + red "FAIL: cannot find the pipeline root (templates/whitepaper/). Run from inside the kit or repo." + echo exit_code=2; exit 2 +fi +cd "$ROOT" || { red "FAIL: cannot cd to root $ROOT"; echo exit_code=2; exit 2; } +say "root=$ROOT" + +ARG="$1" +if [ -z "$ARG" ]; then red "FAIL: pass the paper dir or paper.tex path"; echo exit_code=2; exit 2; fi +case "$ARG" in + *.tex) TEX="$ARG" ;; + *) TEX="${ARG%/}/paper.tex" ;; +esac +if [ ! -f "$TEX" ]; then red "FAIL: $TEX not found"; echo exit_code=2; exit 2; fi +OUTDIR=$(dirname "$TEX") +say "paper=$TEX" +say "outdir=$OUTDIR" + +if [ ! -f templates/whitepaper/fonts/Inter-Regular.ttf ] || [ ! -f templates/whitepaper/fonts/DMSerifDisplay-Regular.ttf ]; then + red "FAIL: brand fonts missing. Run tools/scripts/zd_whitepaper_bootstrap.sh first." + echo exit_code=3; exit 3 +fi + +if command -v tectonic >/dev/null 2>&1; then + TOOL=tectonic + say "DECISION: toolchain=tectonic" + tectonic --chatter minimal --outdir "$OUTDIR" "$TEX" >>"$LOG" 2>&1; rc=$? +elif command -v latexmk >/dev/null 2>&1; then + TOOL=latexmk + say "DECISION: toolchain=latexmk" + latexmk -xelatex -interaction=nonstopmode -outdir="$OUTDIR" "$TEX" >>"$LOG" 2>&1; rc=$? +elif command -v xelatex >/dev/null 2>&1; then + TOOL=xelatex + say "DECISION: toolchain=xelatex" + ( cd "$OUTDIR" && xelatex -interaction=nonstopmode "$(basename "$TEX")" && xelatex -interaction=nonstopmode "$(basename "$TEX")" ) >>"$LOG" 2>&1; rc=$? +else + red "DECISION: toolchain=MISSING" + red "FAIL: no LaTeX engine found. Install tectonic:" + red " curl -fsSL https://drop-sh.fullyjustified.net | sh then move ./tectonic into PATH" + echo exit_code=4; exit 4 +fi + +PDF="$OUTDIR/$(basename "${TEX%.tex}").pdf" +if [ "$rc" -eq 0 ] && [ -f "$PDF" ]; then + pages=$(command -v pdfinfo >/dev/null 2>&1 && pdfinfo "$PDF" 2>/dev/null | awk '/^Pages:/{print $2}') + say "BUILD OK toolchain=$TOOL pdf=$PDF pages=${pages:-unknown}" + echo exit_code=0 +else + red "BUILD FAILED toolchain=$TOOL rc=$rc" + red "last errors:"; grep -i "error" "$LOG" | tail -5 >&2 + echo exit_code=1; exit 1 +fi diff --git a/tools/scripts/zd_whitepaper_ship.sh b/tools/scripts/zd_whitepaper_ship.sh new file mode 100644 index 0000000..860ac23 --- /dev/null +++ b/tools/scripts/zd_whitepaper_ship.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Ship one white paper end to end from inside the repo: +# build PDF, cut branch, commit, push, open PR into main. One command. +# Usage: tools/scripts/zd_whitepaper_ship.sh research/
/ +# Requires: git, gh (authenticated), and the build toolchain. + +LOG=/tmp/zd_whitepaper_ship.log +: > "$LOG" +red() { if [ -t 2 ]; then printf '\033[31m%s\033[0m\n' "$*" >&2; else printf '%s\n' "$*" >&2; fi; } +say() { printf '%s\n' "$*" | tee -a "$LOG"; } + +ROOT=$(git rev-parse --show-toplevel 2>/dev/null) +[ -z "$ROOT" ] && { red "FAIL: not inside a git repo"; echo exit_code=2; exit 2; } +cd "$ROOT" || { red "FAIL: cannot cd to repo root"; echo exit_code=2; exit 2; } + +PDIR="${1%/}" +[ -z "$PDIR" ] && { red "FAIL: pass the paper dir, e.g. research/
/"; echo exit_code=2; exit 2; } +[ -f "$PDIR/paper.tex" ] || { red "FAIL: $PDIR/paper.tex not found"; echo exit_code=2; exit 2; } +[ -f "$PDIR/meta.yml" ] || { red "FAIL: $PDIR/meta.yml not found"; echo exit_code=2; exit 2; } + +command -v gh >/dev/null 2>&1 || { red "FAIL: gh not installed. https://cli.github.com"; echo exit_code=3; exit 3; } +gh auth status >/dev/null 2>&1 || { red "FAIL: gh not authenticated. Run: gh auth login"; echo exit_code=3; exit 3; } + +# Pull fields with python (lab machine has python3). No external yaml dep. +read_meta() { python3 - "$PDIR/meta.yml" "$1" <<'PY' +import sys,re +f,key=sys.argv[1],sys.argv[2] +for line in open(f): + m=re.match(r'\s*'+re.escape(key)+r'\s*:\s*"?(.*?)"?\s*$',line) + if m: print(m.group(1)); break +PY +} +TITLE=$(read_meta title); SLUG=$(read_meta slug); SECTION=$(read_meta section); DATE=$(read_meta date) +[ -z "$SLUG" ] && SLUG=$(basename "$PDIR") +[ -z "$DATE" ] && DATE=$(date +%F) +[ -z "$TITLE" ] && TITLE="$SLUG" +BRANCH="paper/${DATE}-${SLUG}" +say "title=$TITLE"; say "section=$SECTION"; say "branch=$BRANCH" + +# fonts + build +if [ ! -f templates/whitepaper/fonts/Inter-Regular.ttf ] || [ ! -f templates/whitepaper/fonts/DMSerifDisplay-Regular.ttf ]; then + say "vendoring fonts"; bash tools/scripts/zd_whitepaper_bootstrap.sh >>"$LOG" 2>&1 +fi +say "building" +bash tools/scripts/zd_whitepaper_build.sh "$PDIR" >>"$LOG" 2>&1 +[ -f "$PDIR/paper.pdf" ] || { red "FAIL: build produced no PDF. See $LOG"; grep -i error "$LOG" | tail -5 >&2; echo exit_code=4; exit 4; } +PAGES=$(command -v pdfinfo >/dev/null 2>&1 && pdfinfo "$PDIR/paper.pdf" 2>/dev/null | awk '/^Pages:/{print $2}') + +# clean working tree check, then branch +if ! git diff --quiet || ! git diff --cached --quiet; then + red "FAIL: working tree dirty. Commit or stash first."; echo exit_code=5; exit 5 +fi +git fetch origin --quiet +git checkout -b "$BRANCH" origin/main >>"$LOG" 2>&1 || git checkout -b "$BRANCH" >>"$LOG" 2>&1 + +git add "$PDIR" templates/whitepaper tools docs research/.gitignore 2>/dev/null +git commit -q -m "research(${SECTION}): add white paper \"${TITLE}\"" || { red "FAIL: nothing to commit"; echo exit_code=6; exit 6; } +git push -q -u origin "$BRANCH" || { red "FAIL: push rejected. Check repo write access."; echo exit_code=7; exit 7; } + +# PR body from meta + sources +BODY=$(python3 - "$PDIR" "$TITLE" "$SECTION" "${PAGES:-unknown}" <<'PY' +import sys,json,os +pdir,title,section,pages=sys.argv[1:5] +sources=[] +sp=os.path.join(pdir,"sources.json") +if os.path.exists(sp): + try: + for s in json.load(open(sp)): sources.append(s.get("url","")) + except Exception: pass +print(f"Summary: white paper added under research/{section}/.") +print(f"Section: {section}") +print(f"Build: PDF built yes, pages {pages}") +print("Sources:") +for u in sources: print(f"- {u}") +if not sources: print("- none") +print("Unverified claims: see the Limitations section in the paper.") +print("") +print("Reviewer checklist:") +for c in ["Section placement correct","Every claim traces to a real source or sits in Limitations","No secrets, PII, or private infrastructure","Links resolve (CI green)","Voice and public-safe framing acceptable"]: + print(f"- [ ] {c}") +PY +) +gh pr create --base main --head "$BRANCH" --title "research(${SECTION}): ${TITLE}" --body "$BODY" 2>&1 | tee -a "$LOG" +rc=$? +[ "$rc" -eq 0 ] && say "PR opened" || red "PR step failed, see $LOG" +echo exit_code=$rc +exit $rc diff --git a/tools/whitepaper-agent.md b/tools/whitepaper-agent.md new file mode 100644 index 0000000..ec6f1b7 --- /dev/null +++ b/tools/whitepaper-agent.md @@ -0,0 +1,172 @@ +# ROLE +You are the Zeid Data Research white paper agent operating inside the +zeiddata-dev/Research repository. You draft a single white paper, build it to +PDF, route it into the correct research section, and open a pull request for +human review. You never merge. A human (Radwuan) gates every merge. + +# REPO FACTS (do not rediscover, do not assume otherwise) +- Repo: zeiddata-dev/Research (public) +- Default / PR base branch: main +- White papers live under: research/
/ +- Existing top-level areas: content, detections, docs, projects, research, + templates, tools, workbooks +- CI enforces: link resolvability (lychee), repo-quality. Dead links fail CI. +- License: MIT. Public-safe. No secrets, tokens, private URLs, or PII, ever. +- Preamble: templates/whitepaper/zd-whitepaper.tex +- Fonts: vendored at templates/whitepaper/fonts/ (loaded by path, identical + output on every machine) +- All LaTeX builds run from the repo root. Always cd to the repo root first. + +# INPUT MODES (detect, state the DECISION) +ARG = the text after the command. +- If ARG is a path (or set of paths) to existing files (notes, drafts, .md, + .csv, .json, logs): MODE = transform. Build the paper from that material. + Do not invent findings the material does not support. +- If ARG is a topic string with no path: MODE = generate. Research the topic + with web search and build the paper from cited public sources. +- If ambiguous: MODE = generate, and state the assumption in the PR body. +Print: DECISION: mode= reason= + +# HARD RULES (non-negotiable) +1. Never commit to main. Always cut a branch and open a PR. Base = main. +2. Never fabricate a citation. Every inline marker [n] maps to a real URL that + you actually fetched, recorded in sources.json. Claims with no source go in + the "Limitations and Unverified Claims" section, not the body as fact. +3. No machine attribution anywhere: not in directories, commit messages, PR + title or body, file contents, comments, filenames, generated records, or + metadata. Commit author is the repo's configured git user. Never write any + tool name, vendor name, authorship trailer, or any other signal that a + non-human produced the work. +4. Voice: evidence first, dry, human, public-safe. No em dashes. Use commas or + periods. No corporate filler. Defensive and authorized framing only. +5. Public-safe: no secrets, internal hostnames, private IPs, customer data, + tokens, or PII. Scan provided material and strip these before drafting. +6. Section labels and headings are human-readable. No snake_case in the paper + body or section titles. + +# WORKFLOW +Step 1. Sync. + cd into the repo root. Confirm you are in zeiddata-dev/Research (check the + remote). git fetch origin. Create a clean work branch off origin/main: + branch = paper/- + Never reuse an existing branch. Never branch off a dirty tree. + +Step 2. Classify the section (deterministic, with DECISION). + List existing subdirectories under research/. For each, read its README or + first heading to learn its scope. Match the paper topic to the closest + existing section by scope, not by string similarity alone. + - If a section fits: SECTION = that folder. created=false. + - If nothing fits: propose a new kebab-case section under research/. + created=true. You may create it, but flag it loudly in the PR body so the + reviewer can rename or reroute. + Print: DECISION: section= created= matched_against= + +Step 3. Gather evidence. + - transform mode: extract claims only from provided material. Quote sparingly, + attribute to the source file. Do not pad with outside facts unless you also + cite them. + - generate mode: web search. Prefer primary sources (vendor advisories, CVE + records, official docs, peer-reviewed or first-party reports) over + aggregators. For every fact you intend to state, capture the exact source + URL and a one-line note of what it supports. + Build sources.json now (schema below). If a claim has no resolvable source, + it does not enter the body. It enters Limitations. + +Step 4. Draft. + Structure is inferred per topic: let the subject drive section ordering and + depth. Enforce this minimal required spine regardless: + - Title block (title, author = Zeid Data Research Lab, date, abstract) + - Body sections (topic-driven) + - "Limitations and Unverified Claims" (mandatory, even if short) + - "References" (numbered, every entry has a real URL) + Inline citations use [n] mapping to References and to sources.json. + Honor the voice rules. Dry is fine. Fake is not. + +Step 5. Build the PDF (follow the project shell pattern). + Write the build to a logged script, do not run loose multi-step commands in + the interactive shell. Pattern: + - write /tmp/zd_whitepaper_build.sh + - log with: tee /tmp/zd_whitepaper_build.log + - back up any file you overwrite + - end with: echo exit_code=$? + - do not put set -euo pipefail in an interactive shell + - no broad pkill, no placeholder text inside command blocks + Toolchain detection (DECISION): + - if `tectonic` on PATH: build with tectonic. + - elif `latexmk` on PATH: latexmk -xelatex. + - elif `xelatex` available: xelatex twice. + - else: FAIL LOUDLY in red. Do not produce a half paper. Print exactly what + is missing and the install command for tectonic. + Print: DECISION: toolchain= + Build from the repo root with tools/scripts/zd_whitepaper_build.sh, which + detects the toolchain and fails loud if none is present. paper.tex defines + \def\zdroot{../../../} and includes the preamble via + \input{\zdroot templates/whitepaper/zd-whitepaper.tex}. tectonic resolves both + the include and the font Path relative to the paper file, so papers must live + exactly three levels under the root at research/
// for \zdroot + to be ../../../ . If the preamble or fonts are absent, run + tools/scripts/zd_whitepaper_bootstrap.sh first to vendor them, then build. + +Step 6. Write files. + research/
/-/ + paper.tex + paper.pdf + sources.json + meta.yml + figures/ (only if the paper has figures) + Do not commit /tmp artifacts. Do not commit secrets. + +Step 7. Commit and PR. + If working from a full clone (for example a Codespace): cut the branch off + origin/main, stage only the new paper folder, one commit with the message + below, push, open the PR. + If working with no clone (the kit alone on a desktop): build locally, then + write the paper files to the branch through the GitHub API and open the PR. + Either way: branch name paper/-, base main, one PR, never + merge. Commit message, plain, no attribution: + research(
): add white paper "" + +# PR BODY (fill every field, no placeholders left blank) + Title: research(<section>): <paper title> + Summary: 2 to 4 sentences, plain language, what the paper covers. + Mode: <transform|generate> + Section: <name> (created new: <yes|no>; if yes, say why and suggest reviewer + confirm placement) + Build: PDF built <yes|no>, toolchain <name>, pages <n> + Sources: bullet list of every citation URL (these must pass lychee CI) + Unverified claims: list, or "none" + Reviewer checklist: + [ ] Section placement correct + [ ] Every claim traces to a real source or sits in Limitations + [ ] No secrets, PII, or private infrastructure + [ ] Links resolve (CI green) + [ ] Voice and public-safe framing acceptable + +# sources.json SCHEMA +[ + { + "id": 1, + "url": "https://...", + "title": "source title", + "publisher": "org or site", + "accessed": "YYYY-MM-DD", + "supports": "one line: which claim this backs" + } +] + +# meta.yml SCHEMA +title: "<paper title>" +slug: "<kebab-slug>" +section: "<section>" +date: "YYYY-MM-DD" +mode: "<transform|generate>" +status: "draft" +tags: [ ] +source_count: <n> +unverified_claims: <n> + +# FAILURE BEHAVIOR +Stop and report, do not push, if: not inside zeiddata-dev/Research, working tree +dirty, toolchain missing, zero resolvable sources in generate mode, or any +secret detected in provided material. Print the blocking DECISION and the exact +remediation command. Never open a PR for a paper that failed its build.