diff --git a/loc-tools/README.md b/loc-tools/README.md new file mode 100644 index 0000000..46a1cd1 --- /dev/null +++ b/loc-tools/README.md @@ -0,0 +1,123 @@ +# Localization tooling + +Build-time localization machinery shared by the microbit-apps stack. A consumer +depends on `@microbit-apps/ui-core` and drives this tooling through its API and +two bins. The runtime side (`ui.loc` and the `_loc` namespace) lives in +`loc.ts`; this directory is the Node-side build tooling only. + +## Layout + +- `catalogs.mjs` -- layer discovery and per-language catalog merge. +- `fonts.mjs` -- font glyph-coverage parsing. +- `extract.mjs` -- source-string extraction from `ui.loc(...)` call sites. +- `charsets.mjs` -- generic per-language field data (resolve + validate). +- `emit.mjs` -- `loc.g.ts` generation and default/restore content. +- `gen.mjs` -- the `runLocGen` / `runCoverageMode` orchestrator. +- `index.mjs` -- the public API (imported as `@microbit-apps/ui-core/loc`). +- `bin/loc-strings.mjs`, `bin/loc-gen.mjs` -- the CLI entry points. + +## Bins + +- `loc-strings [srcLang]` -- regenerate the calling project's + `locales/.json` (default `en`) from the display strings passed to + `ui.loc` / `ui.locf` / `ui.locc` in the files listed in `pxt.json`. Runs in + the current working directory. +- `loc-gen [--coverage] [--verbose] [--keep] [lang ...]` -- per-language build + for simple projects. Optional `loc.config.json` (see below). Projects that + need app-specific hooks call `runLocGen` from their own wrapper instead. + +## Layer discovery + +Catalog layers are derived from the dependency graph, never hard-coded. For a +project rooted at `root`: + +1. Walk the project's `pxt.json` dependencies transitively. Every dependency + module resolved under `pxt_modules//` that ships a `locales/` + directory becomes a layer, ordered so a library appears before any library + that depends on it. +2. The project's own `locales/` is the final layer, named `app`. + +Later layers win on merge, so the app overrides its libraries and a library +overrides its own dependencies. The same discovery drives translation catalogs +(`.json`) and charset data (`charsets.json`). + +## Charsets + +Any layer may ship `locales/charsets.json` carrying per-language field data: + +```json +{ + "es": { "alphabetLower": "abc...", "alphabetUpper": "ABC...", "symbols": "+-*/" }, + "fr": { "accentsLower": "àâä...", "accentsUpper": "ÀÂÄ..." } +} +``` + +Each field is an arbitrary string. The tooling is agnostic: it never interprets +field names. Language resolution per layer is the exact language key, else the +base-language prefix (`es` serves `es-ES`). Fields merge across layers per +language, the app layer winning. + +Resolved fields are hard-validated (a failure is an error, never a silent drop): + +- a `Lower` / `Upper` field pair, when both are present, must have equal + code-point length; +- no field may repeat a code point within itself; +- every code point of every field must exist in the build font's coverage. + +Each resolved field is emitted into the generated `loc.g.ts` as a `_loc` member +assignment (` = ` inside `namespace _loc`). The library that +owns a field declares the matching `_loc` member (via namespace reopening in +that library's own source); this tooling only writes the assignment. + +## runLocGen options + +`runLocGen(options, langArgs)` runs the per-language build; `runCoverageMode` +runs the coverage report. Defaults make a bare consumer work with only `root`. + +- `root` (required) -- project directory. +- `srcLang` -- source language (default `"en"`); its build ships no table. +- `langs` -- explicit language list; otherwise `langArgs`, otherwise the source + language plus every discovered language. +- `reservedLangNames` -- non-language basenames in `locales/` to skip during + auto-discovery (the source language and `charsets` are always skipped). +- `font` -- build font for glyph and charset validation (default `"font8"`). +- `textPath` -- font glyph-table source (default the display-shield `text.ts`). +- `hexDir`, `hexName(lang)` -- output directory and per-language file name + (default `/assets/hex` and `..hex`). +- `defaultLocG` -- exact content restored to `loc.g.ts` after a run and written + for the source build (required for a build run). +- `locGPath` -- the generated file (default `/loc.g.ts`). +- `smallFontContext` -- `{ font, strings }`: source strings validated against a + second (small) font instead of the build font, dropped when it cannot render + them. `strings` is an array, a Set, or a function returning either. +- `exclude` -- `{ label, keys(merged) }`: source strings to drop from device + tables (they ship by another route). `label` names the count in the summary. +- `extraInventory` -- extra coverage inventory beyond the layer source + catalogs: entries of `{ path, mapper(catalog) }` or precomputed string arrays. +- `postMerge(lang, merged)` -- called after merge, before drops, for + app-specific side outputs. `merged` is `null` for the source language. +- `coverage`, `verbose`, `keep` -- flags mirrored from the CLI. + +## Thin-wrapper pattern + +Apps with app-specific behavior wrap the API. The wrapper owns everything the +generic tool should not know about, and stays thin: + +```js +import { runLocGen } from "@microbit-apps/ui-core/loc" + +runLocGen( + { + root: ROOT, + defaultLocG: DEFAULT_LOC_G, + hexName: lang => `myapp.${lang}.hex`, + // app-specific hooks: exclude, smallFontContext, extraInventory, postMerge + }, + process.argv.slice(2).filter(a => !a.startsWith("--")), +) +``` + +A project with no such needs skips the wrapper entirely and uses the `loc-gen` +bin, optionally with a `loc.config.json` setting `srcLang`, `font`, `hexName` +(a pattern with a `` placeholder), `reservedLangNames`, `textPath`, or +`hexDir`. diff --git a/loc-tools/bin/loc-gen.mjs b/loc-tools/bin/loc-gen.mjs new file mode 100755 index 0000000..4a8baf3 --- /dev/null +++ b/loc-tools/bin/loc-gen.mjs @@ -0,0 +1,45 @@ +#!/usr/bin/env node +// Thin per-language build CLI for simple consumers. Run from the project root. +// +// Configuration is optional: with no loc.config.json a bare project builds the +// source language plus every locales/.json using stock defaults. A +// loc.config.json (JSON) may set any of: srcLang, font, hexName (a pattern with +// a "" placeholder), reservedLangNames, textPath, hexDir. Apps needing +// app-specific hooks (extra inventory, side outputs, exclusions) call the +// runLocGen API from their own wrapper instead. +// +// Flags: --coverage [--verbose], --keep, and positional language names. + +import { existsSync, readFileSync } from "node:fs" +import { join } from "node:path" +import { runLocGen } from "../gen.mjs" +import { GENERIC_DEFAULT_LOC_G } from "../emit.mjs" + +const root = process.cwd() +const argv = process.argv.slice(2) +const coverage = argv.indexOf("--coverage") >= 0 +const verbose = argv.indexOf("--verbose") >= 0 +const keep = argv.indexOf("--keep") >= 0 +const positional = argv.filter(a => !a.startsWith("--")) + +let config = {} +const configPath = join(root, "loc.config.json") +if (existsSync(configPath)) config = JSON.parse(readFileSync(configPath, "utf8")) + +const options = { + root, + srcLang: config.srcLang, + font: config.font, + textPath: config.textPath ? join(root, config.textPath) : undefined, + hexDir: config.hexDir ? join(root, config.hexDir) : undefined, + hexName: config.hexName ? lang => config.hexName.replace("", lang) : undefined, + reservedLangNames: config.reservedLangNames, + // The CLI always uses the stock default; a custom loc.g.ts default is an + // API-level concern (pass defaultLocG to runLocGen from a wrapper). + defaultLocG: GENERIC_DEFAULT_LOC_G, + coverage, + verbose, + keep, +} + +runLocGen(options, positional) diff --git a/loc-tools/bin/loc-strings.mjs b/loc-tools/bin/loc-strings.mjs new file mode 100755 index 0000000..6d65da1 --- /dev/null +++ b/loc-tools/bin/loc-strings.mjs @@ -0,0 +1,8 @@ +#!/usr/bin/env node +// Regenerate the calling project's source catalog (locales/.json) from +// its listed .ts files. Run from the project root. + +import { runLocStrings } from "../extract.mjs" + +const srcLang = process.argv[2] || "en" +runLocStrings(process.cwd(), srcLang) diff --git a/loc-tools/catalogs.mjs b/loc-tools/catalogs.mjs new file mode 100644 index 0000000..caed439 --- /dev/null +++ b/loc-tools/catalogs.mjs @@ -0,0 +1,63 @@ +// Translation catalog layer discovery and merge. +// +// A build's translations come from ordered layers: each dependency library +// that ships a locales/ directory, in transitive dependency order (a library +// before any library that depends on it), then the consuming project's own +// locales/ last. Later layers win on merge, so the app overrides its +// libraries and a library overrides its own dependencies. + +import { existsSync, readFileSync } from "node:fs" +import { join } from "node:path" + +// Parse a JSON catalog file, or null when it does not exist. +export function readCatalog(path) { + if (!existsSync(path)) return null + return JSON.parse(readFileSync(path, "utf8")) +} + +// Dependency names declared in a pxt.json, or [] when the file is absent. +function readDeps(pxtJsonPath) { + if (!existsSync(pxtJsonPath)) return [] + const pxt = JSON.parse(readFileSync(pxtJsonPath, "utf8")) + return Object.keys(pxt.dependencies || {}) +} + +// The ordered locale layers for a project rooted at `root`: every dependency +// module (resolved under pxt_modules) that carries a locales/ directory, in +// transitive dependency order, followed by the project's own locales/ as the +// final, highest-priority layer named "app". Each layer is { name, localesDir }. +export function discoverLayers(root) { + const modulesDir = join(root, "pxt_modules") + const seen = new Set() + const layers = [] + const visit = name => { + if (seen.has(name)) return + seen.add(name) + const modDir = join(modulesDir, name) + for (const dep of readDeps(join(modDir, "pxt.json"))) visit(dep) + const localesDir = join(modDir, "locales") + if (existsSync(localesDir)) layers.push({ name, localesDir }) + } + for (const dep of readDeps(join(root, "pxt.json"))) visit(dep) + layers.push({ name: "app", localesDir: join(root, "locales") }) + return layers +} + +// Merge the per-language catalog of every layer into one table, later layers +// winning. Returns the merged table and per-layer info (count is null when the +// layer has no file for the language). +export function mergeLayers(layers, lang) { + const merged = {} + const info = [] + for (const layer of layers) { + const path = join(layer.localesDir, lang + ".json") + const cat = readCatalog(path) + if (cat === null) { + info.push({ name: layer.name, path, count: null }) + continue + } + for (const k of Object.keys(cat)) merged[k] = cat[k] + info.push({ name: layer.name, path, count: Object.keys(cat).length }) + } + return { merged, layers: info } +} diff --git a/loc-tools/charsets.mjs b/loc-tools/charsets.mjs new file mode 100644 index 0000000..e322040 --- /dev/null +++ b/loc-tools/charsets.mjs @@ -0,0 +1,85 @@ +// Generic per-language field data. +// +// Any layer (discovered exactly like translation catalogs) may ship +// locales/charsets.json: +// +// { "": { "": "", ... }, ... } +// +// Each field is a string of characters that a consuming library treats as +// per-language configuration -- for example a keyboard alphabet. The tooling +// is agnostic: it never interprets field names, only merges, validates, and +// emits them as `_loc.` assignments (see emit.mjs). The library that +// owns a field declares the matching `_loc` member. +// +// Language resolution per layer: the exact language key, else the base-language +// prefix (so "es" serves "es-ES"). Fields merge across layers per language, +// the app layer winning. + +import { join } from "node:path" +import { readCatalog } from "./catalogs.mjs" +import { toHexCp } from "./fonts.mjs" + +// The base-language prefix of a locale tag ("es" for "es-ES", "fr" for "fr"). +function baseLang(lang) { + const i = lang.indexOf("-") + return i < 0 ? lang : lang.slice(0, i) +} + +// Resolve the merged field set for a language across all layers. Returns an +// object mapping field name to its resolved string (empty when no layer ships +// data for the language). +export function resolveCharsets(layers, lang) { + const merged = {} + for (const layer of layers) { + const data = readCatalog(join(layer.localesDir, "charsets.json")) + if (!data) continue + const entry = data[lang] || data[baseLang(lang)] + if (!entry) continue + for (const field of Object.keys(entry)) merged[field] = entry[field] + } + return merged +} + +// Validate resolved field data. These are hard errors, never silent drops: +// - a `Lower`/`Upper` field pair (both present) must have equal +// code-point length; +// - no field may repeat a code point within itself; +// - every code point of every field must be present in the build font's +// coverage. +export function validateCharsets(fields, fontCoverage, lang) { + for (const [name, str] of Object.entries(fields)) { + const seen = new Set() + for (const ch of str) { + const cp = ch.codePointAt(0) + if (seen.has(cp)) + throw new Error( + `charsets: ${lang} field ${JSON.stringify(name)}: duplicate character ${ch} ${toHexCp(cp)}`, + ) + seen.add(cp) + } + } + + for (const name of Object.keys(fields)) { + if (!name.endsWith("Lower")) continue + const upper = name.slice(0, -"Lower".length) + "Upper" + if (fields[upper] === undefined) continue + const lowerLen = Array.from(fields[name]).length + const upperLen = Array.from(fields[upper]).length + if (lowerLen !== upperLen) + throw new Error( + `charsets: ${lang} pair ${JSON.stringify(name)}/${JSON.stringify(upper)}: ` + + `code-point length ${lowerLen} != ${upperLen}`, + ) + } + + for (const [name, str] of Object.entries(fields)) { + for (const ch of str) { + const cp = ch.codePointAt(0) + if (!fontCoverage.has(cp)) + throw new Error( + `charsets: ${lang} field ${JSON.stringify(name)}: character ${ch} ${toHexCp(cp)} ` + + `not in build font coverage`, + ) + } + } +} diff --git a/loc-tools/emit.mjs b/loc-tools/emit.mjs new file mode 100644 index 0000000..1b3f2ef --- /dev/null +++ b/loc-tools/emit.mjs @@ -0,0 +1,55 @@ +// Generation and restore of the app's loc.g.ts. +// +// loc.g.ts holds one shipped language's translation table (and any per-language +// field data) for a per-language build. It is a transient build intermediate: +// the generator writes a language's content, builds, and restores the default +// state afterward. The default state assigns nothing, so a vanilla build falls +// back to the source strings. + +import { writeFileSync } from "node:fs" + +// A generic assign-nothing loc.g.ts, used when a consumer does not supply its +// own default content. The runtime falls back to the source strings, so a +// vanilla build produces the source-string image with no localization tooling +// involved. A consumer whose build depends on loc.g.ts being listed first in +// pxt.json (for example to fix font-capture ordering) supplies its own text. +export const GENERIC_DEFAULT_LOC_G = `// Generated placeholder for a per-language build. In this default state it +// assigns nothing: the runtime falls back to the source strings. +// +// This file must exist because it is listed in pxt.json "files". A per-language +// build writes a translation table here, builds, and restores this default +// state afterward. Only the default state belongs in a commit. +` + +// Emit loc.g.ts content for one language. `table` is the id-to-string catalog +// (emitted only when non-empty); `charsets` maps field names to their resolved +// strings (each emitted as a `_loc` member assignment). Both live inside one +// reopened `namespace _loc` block. +export function emitLocG(lang, table, charsets) { + const keys = Object.keys(table).sort() + const fields = Object.keys(charsets).sort() + const lines = [] + lines.push("// Generated for a per-language build. Holds the one shipped language's") + lines.push("// translation table and per-language field data.") + lines.push("//") + lines.push("// Transient build intermediate: the generator restores the default state of") + lines.push("// this file after the run. Do not commit this generated content.") + lines.push("// lang: " + lang) + lines.push("namespace _loc {") + if (keys.length > 0) { + const entries = keys.map(k => " " + JSON.stringify(k) + ": " + JSON.stringify(table[k])) + lines.push(" table = {") + lines.push(entries.join(",\n")) + lines.push(" }") + } + for (const field of fields) { + lines.push(" " + field + " = " + JSON.stringify(charsets[field])) + } + lines.push("}") + return lines.join("\n") + "\n" +} + +// Write the default (assign-nothing) state to loc.g.ts. +export function writeDefault(locGPath, defaultLocG) { + writeFileSync(locGPath, defaultLocG) +} diff --git a/scripts/locstrings.mjs b/loc-tools/extract.mjs similarity index 50% rename from scripts/locstrings.mjs rename to loc-tools/extract.mjs index 6c1e271..06d3b04 100644 --- a/scripts/locstrings.mjs +++ b/loc-tools/extract.mjs @@ -1,29 +1,26 @@ -// Regenerates locales/en.json from source. +// Source-string extraction. // -// Scans the .ts files listed in pxt.json "files" (so test files are excluded) -// and extracts the source display strings passed to the localization helpers: +// Scans the .ts files listed in a project's pxt.json "files" (so test files are +// excluded) and extracts the source display strings passed to the localization +// helpers: // // ui.loc("...") / loc("...") // ui.locf("...") / locf("...") (first argument only) // ui.locc("ctx", "...") / locc("ctx", "...") (emits key "ctx#string") // // Arguments must be string literals at the call site (the documented -// convention; see locales/README.md). Any other argument is reported as a -// loud warning naming the file and the argument, so a missed string is never -// dropped silently. A call whose argument is deliberately dynamic (its +// convention; see a repo's locales/README.md). Any other argument is reported +// as a loud warning naming the file and the argument, so a missed string is +// never dropped silently. A call whose argument is deliberately dynamic (its // strings reach the catalog by another route) is suppressed by putting // "locstrings-ignore" in a comment on the call's line. // -// The result is a sorted JSON object mapping each catalog key to its source -// display string. For locc entries the value is the plain string, without the -// context prefix. The file is fully derived: it is regenerated from source, so -// any prior keys not found in the current sources are dropped. +// The result is a sorted object mapping each catalog key to its source display +// string. For locc entries the value is the plain string, without the context +// prefix. The catalog is fully derived from the current sources. import { readFileSync, writeFileSync } from "node:fs" -import { dirname, join } from "node:path" -import { fileURLToPath } from "node:url" - -const root = join(dirname(fileURLToPath(import.meta.url)), "..") +import { join } from "node:path" // A double- or single-quoted string literal, honoring escaped quotes. const STR = `"(?:[^"\\\\]|\\\\.)*"|'(?:[^'\\\\]|\\\\.)*'` @@ -41,13 +38,15 @@ const DEF = `(? !line.includes("locstrings-ignore")) - .join("\n") - const src = stripComments(raw) - if (API_DEF_RE.test(src)) continue - - for (const m of src.matchAll(LOCC_RE)) { - const ctx = resolveArg(m[1], file) - const str = resolveArg(m[2], file) - if (ctx == null || str == null) continue - catalog[ctx + "#" + str] = str - } - for (const m of src.matchAll(LOC_RE)) { - const str = resolveArg(m[1], file) - if (str == null) continue - catalog[str] = str +// Extract the source catalog from a project's listed files. Returns the sorted +// catalog and the count of unresolved (non-literal) helper arguments, each of +// which is reported to the console as it is found. +export function extractCatalog(root) { + const pxt = JSON.parse(readFileSync(join(root, "pxt.json"), "utf8")) + const catalog = {} + let warnings = 0 + + // Resolve a captured helper argument to its string value, or null (with a + // warning) when it is not a string literal. + const resolveArg = (raw, file) => { + const arg = raw.trim() + if (arg == "") return null + if (STR_ONLY.test(arg)) return unquote(arg) + console.warn(`locstrings: WARNING: ${file}: cannot resolve loc argument '${arg}'`) + warnings++ + return null } -} -const sorted = {} -for (const key of Object.keys(catalog).sort()) sorted[key] = catalog[key] + for (const file of pxt.files) { + if (!file.endsWith(".ts")) continue + const raw = readFileSync(join(root, file), "utf8") + .split("\n") + .filter(line => !line.includes("locstrings-ignore")) + .join("\n") + const src = stripComments(raw) + if (API_DEF_RE.test(src)) continue + + for (const m of src.matchAll(LOCC_RE)) { + const ctx = resolveArg(m[1], file) + const str = resolveArg(m[2], file) + if (ctx == null || str == null) continue + catalog[ctx + "#" + str] = str + } + for (const m of src.matchAll(LOC_RE)) { + const str = resolveArg(m[1], file) + if (str == null) continue + catalog[str] = str + } + } -writeFileSync( - join(root, "locales", "en.json"), - JSON.stringify(sorted, null, 4) + "\n", -) + const sorted = {} + for (const key of Object.keys(catalog).sort()) sorted[key] = catalog[key] + return { catalog: sorted, warnings } +} -const count = Object.keys(sorted).length -console.log(`locstrings: wrote ${count} key(s) to locales/en.json`) -if (warnings) console.log(`locstrings: ${warnings} unresolved loc argument(s); see warnings above`) +// Regenerate a project's source catalog file (locales/.json) from its +// listed files. +export function runLocStrings(root, srcLang = "en") { + const { catalog, warnings } = extractCatalog(root) + writeFileSync(join(root, "locales", srcLang + ".json"), JSON.stringify(catalog, null, 4) + "\n") + const count = Object.keys(catalog).length + console.log(`locstrings: wrote ${count} key(s) to locales/${srcLang}.json`) + if (warnings) console.log(`locstrings: ${warnings} unresolved loc argument(s); see warnings above`) + return { count, warnings } +} diff --git a/loc-tools/fonts.mjs b/loc-tools/fonts.mjs new file mode 100644 index 0000000..83a8a30 --- /dev/null +++ b/loc-tools/fonts.mjs @@ -0,0 +1,53 @@ +// Bitmap-font glyph coverage. +// +// A font's renderable code points are read from its hex-literal glyph table in +// the display-shield text source. Each record is 8 bytes: a 2-byte +// little-endian code point followed by 6 data bytes. The set of code points is +// used to decide whether a translated string can render in a given font. + +import { readFileSync } from "node:fs" +import { join } from "node:path" + +// Format a code point as U+XXXX for diagnostics. +export function toHexCp(cp) { + return "U+" + cp.toString(16).toUpperCase().padStart(4, "0") +} + +// Escape regex metacharacters so a value can be matched literally. +function escapeRegExp(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +} + +// Parse the code-point coverage of a named font from a text source file. +export function parseFontCoverage(srcPath, fontName) { + const src = readFileSync(srcPath, "utf8") + const m = src.match( + new RegExp(escapeRegExp(fontName) + "[\\s\\S]*?data:\\s*hex`([\\s\\S]*?)`"), + ) + if (!m) throw new Error(`could not locate ${fontName} hex literal in ${srcPath}`) + const hex = m[1].replace(/\s+/g, "") + const set = new Set() + for (let i = 0; i + 16 <= hex.length; i += 16) { + const lo = parseInt(hex.slice(i, i + 2), 16) + const hi = parseInt(hex.slice(i + 2, i + 4), 16) + set.add(lo | (hi << 8)) + } + return set +} + +// The standard font glyph-table source shipped by the display-shield module. +export function defaultTextPath(root) { + return join(root, "pxt_modules", "display-shield", "text.ts") +} + +// A cached coverage provider over one text source file. `coverage(name)` +// returns the code-point set for a font, parsing it on first use. +export function createFontProvider(textPath) { + const cache = {} + return { + coverage(fontName) { + if (!cache[fontName]) cache[fontName] = parseFontCoverage(textPath, fontName) + return cache[fontName] + }, + } +} diff --git a/loc-tools/gen.mjs b/loc-tools/gen.mjs new file mode 100644 index 0000000..956313b --- /dev/null +++ b/loc-tools/gen.mjs @@ -0,0 +1,376 @@ +// Per-language localization build orchestrator. +// +// For each requested language: merge the discovered catalog layers, drop +// identity and unrenderable entries, resolve and validate per-language field +// data, emit loc.g.ts, build with mkc, and copy the resulting hex. A language +// with no renderable translations and no field data builds no hex. loc.g.ts is +// restored to its default (assign-nothing) state after the run. +// +// Coverage mode reports, per language, how much of the source-string inventory +// the merged catalog translates; it builds nothing and does not write loc.g.ts. + +import { readFileSync, writeFileSync, existsSync, readdirSync, mkdirSync, copyFileSync, statSync } from "node:fs" +import { execSync } from "node:child_process" +import { join } from "node:path" + +import { discoverLayers, mergeLayers, readCatalog } from "./catalogs.mjs" +import { createFontProvider, defaultTextPath, toHexCp } from "./fonts.mjs" +import { emitLocG, writeDefault } from "./emit.mjs" +import { resolveCharsets, validateCharsets } from "./charsets.mjs" + +// Normalize options into a fully-defaulted config. +function resolveConfig(options) { + const root = options.root + if (!root) throw new Error("runLocGen: options.root is required") + const pxt = JSON.parse(readFileSync(join(root, "pxt.json"), "utf8")) + const srcLang = options.srcLang || "en" + const textPath = options.textPath || defaultTextPath(root) + const fonts = createFontProvider(textPath) + + const exclude = options.exclude || null + const excludeLabel = (exclude && exclude.label) || "excluded" + + const small = options.smallFontContext || null + const smallFont = (small && small.font) || "font5" + let smallStrings = null + if (small) { + let s = typeof small.strings === "function" ? small.strings() : small.strings + smallStrings = s instanceof Set ? s : new Set(s || []) + } + + return { + root, + pxt, + srcLang, + fonts, + font: options.font || "font8", + layers: discoverLayers(root), + langs: options.langs || null, + reservedLangNames: options.reservedLangNames || [], + coverage: !!options.coverage, + verbose: !!options.verbose, + keep: !!options.keep, + hexDir: options.hexDir || join(root, "assets", "hex"), + hexName: options.hexName || (lang => `${pxt.name}.${lang}.hex`), + defaultLocG: options.defaultLocG, + locGPath: options.locGPath || join(root, "loc.g.ts"), + exclude, + excludeLabel, + smallFont, + smallStrings, + extraInventory: options.extraInventory || [], + postMerge: options.postMerge || null, + } +} + +// Languages present in the project's own locales/, excluding the source +// language, the reserved charsets file, and any app-declared reserved names. +function discoverLangs(cfg) { + const dir = join(cfg.root, "locales") + if (!existsSync(dir)) return [] + const langs = [] + for (const f of readdirSync(dir)) { + if (!f.endsWith(".json")) continue + const name = f.slice(0, -5) + if (name === cfg.srcLang || name === "charsets") continue + if (cfg.reservedLangNames.indexOf(name) >= 0) continue + langs.push(name) + } + return langs +} + +function buildAndCopy(cfg, lang) { + execSync("npx mkc build", { cwd: cfg.root, stdio: "pipe" }) + if (!existsSync(cfg.hexDir)) mkdirSync(cfg.hexDir, { recursive: true }) + const dst = join(cfg.hexDir, cfg.hexName(lang)) + copyFileSync(join(cfg.root, "built", "binary.hex"), dst) + return { dst, size: statSync(dst).size } +} + +// Validate a set of source strings rendered in the small-font context against +// that font's coverage; drop and report those the font cannot render. +function dropUnrenderable(merged, keys, coverage, lang, tag, only) { + let dropped = 0 + for (const k of Object.keys(merged)) { + if (only && !keys.has(k)) continue + if (!only && keys.has(k)) continue + const val = merged[k] + const missing = [] + for (const ch of val) { + const cp = ch.codePointAt(0) + if (cp < 33) continue + if (!coverage.has(cp)) missing.push(ch + " " + toHexCp(cp)) + } + if (missing.length > 0) { + delete merged[k] + dropped++ + console.log(` [${tag}] drop ${lang} key ${JSON.stringify(k)}: missing ${missing.join(", ")}`) + } + } + return dropped +} + +function processLanguage(cfg, lang) { + const report = { lang } + + if (lang === cfg.srcLang) { + console.log(`\n=== ${lang} ===`) + if (cfg.postMerge) cfg.postMerge(lang, null) + writeDefault(cfg.locGPath, cfg.defaultLocG) + const built = buildAndCopy(cfg, lang) + console.log(" source-string build (no translation table)") + console.log(` hex: ${built.dst} (${built.size} bytes)`) + report.merged = 0 + report.identityDropped = 0 + report.smallDropped = 0 + report.glyphDropped = 0 + report.lengthWarnings = 0 + report.hexPath = built.dst + report.hexSize = built.size + return report + } + + console.log(`\n=== ${lang} ===`) + const { merged, layers } = mergeLayers(cfg.layers, lang) + const layersFound = [] + for (const l of layers) { + if (l.count === null) { + console.log(` layer ${l.name}: missing (${l.path})`) + continue + } + layersFound.push(l.name) + console.log(` layer ${l.name}: ${l.count} entries`) + } + const mergedCount = Object.keys(merged).length + + if (cfg.postMerge) cfg.postMerge(lang, merged) + + // App-declared table key exclusions (source strings that ship elsewhere). + let excluded = 0 + if (cfg.exclude) { + const drop = new Set(cfg.exclude.keys(merged)) + for (const k of Object.keys(merged)) { + if (drop.has(k)) { + delete merged[k] + excluded++ + } + } + } + + // Drop identity entries (value === key): they only waste flash. + let identityDropped = 0 + for (const k of Object.keys(merged)) { + if (merged[k] === k) { + delete merged[k] + identityDropped++ + } + } + + // Small-font-context validation: strings the app renders in the small font + // are validated against its coverage, not the build font. A translation the + // small font cannot render is dropped so the runtime falls back to the + // source string rather than drawing blank. + let smallDropped = 0 + if (cfg.smallStrings && cfg.smallStrings.size > 0) { + const cov = cfg.fonts.coverage(cfg.smallFont) + smallDropped = dropUnrenderable(merged, cfg.smallStrings, cov, lang, cfg.smallFont, true) + } + + // Glyph validation: drop entries whose translation contains code points the + // build font cannot render. Code points below 33 are always acceptable. + // Small-font-context keys are exempt: validated above. + const cov = cfg.fonts.coverage(cfg.font) + const smallKeys = cfg.smallStrings || new Set() + const glyphDropped = dropUnrenderable(merged, smallKeys, cov, lang, cfg.font, false) + + // Length warning (never drops): translated text much longer than source. + let lengthWarnings = 0 + for (const k of Object.keys(merged)) { + const srcLen = Array.from(k).length + const transLen = Array.from(merged[k]).length + if (transLen > srcLen * 1.5 && transLen - srcLen > 3) { + lengthWarnings++ + console.log(` [length] ${lang} key ${JSON.stringify(k)}: src ${srcLen} -> trans ${transLen} chars`) + } + } + + // Per-language field data (hard-validated, never dropped). + const charsets = resolveCharsets(cfg.layers, lang) + if (Object.keys(charsets).length > 0) validateCharsets(charsets, cov, lang) + + const finalCount = Object.keys(merged).length + const hasFields = Object.keys(charsets).length > 0 + + report.merged = mergedCount + report.identityDropped = identityDropped + report.smallDropped = smallDropped + report.glyphDropped = glyphDropped + report.lengthWarnings = lengthWarnings + report.table = finalCount + + // A language none of whose translations survive validation, and with no + // field data, would ship an image identical to the source-string build; + // skip it rather than emit a pointless hex. + if (finalCount === 0 && !hasFields) { + console.log( + ` merged ${mergedCount}, ${cfg.excludeLabel} ${excluded}, identity-dropped ${identityDropped}, ` + + `${cfg.smallFont}-dropped ${smallDropped}, glyph-dropped ${glyphDropped}`, + ) + console.log(" no renderable translations; skipped (no hex built)") + report.skipped = true + return report + } + + writeFileSync(cfg.locGPath, emitLocG(lang, merged, charsets)) + const built = buildAndCopy(cfg, lang) + + console.log( + ` merged ${mergedCount}, ${cfg.excludeLabel} ${excluded}, identity-dropped ${identityDropped}, ` + + `${cfg.smallFont}-dropped ${smallDropped}, glyph-dropped ${glyphDropped}, ` + + `length-warnings ${lengthWarnings}, table ${finalCount}`, + ) + console.log(` hex: ${built.dst} (${built.size} bytes)`) + + report.hexPath = built.dst + report.hexSize = built.size + return report +} + +// The full set of localizable source strings: the union of every layer's +// source catalog (keys) plus any app-declared extra inventory sources. +function sourceInventory(cfg) { + const set = new Set() + for (const layer of cfg.layers) { + const cat = readCatalog(join(layer.localesDir, cfg.srcLang + ".json")) + if (cat) for (const k of Object.keys(cat)) set.add(k) + } + for (const src of cfg.extraInventory) { + if (Array.isArray(src)) { + for (const s of src) set.add(s) + continue + } + const cat = readCatalog(src.path) + if (!cat) continue + for (const s of src.mapper(cat)) set.add(s) + } + return set +} + +function runCoverage(cfg, langArgs) { + const inventory = Array.from(sourceInventory(cfg)).sort() + const total = inventory.length + const langs = langArgs.length > 0 ? langArgs : discoverLangs(cfg) + + console.log("locgen coverage") + console.log(`inventory: ${total} source strings`) + + const rows = [] + const missingByLang = {} + for (const lang of langs) { + const { merged } = mergeLayers(cfg.layers, lang) + let translated = 0 + const missing = [] + for (const s of inventory) { + if (Object.prototype.hasOwnProperty.call(merged, s)) translated++ + else missing.push(s) + } + const pct = total === 0 ? 100 : (translated / total) * 100 + rows.push({ lang, total, translated, missing: missing.length, pct }) + missingByLang[lang] = missing + } + + const head = ["lang", "total", "translated", "missing", "percent"] + const cells = rows.map(r => [ + r.lang, + String(r.total), + String(r.translated), + String(r.missing), + r.pct.toFixed(1) + "%", + ]) + const widths = head.map((h, i) => Math.max(h.length, ...cells.map(c => c[i].length))) + const fmt = c => c.map((v, i) => v.padEnd(widths[i])).join(" ") + console.log("") + console.log(fmt(head)) + for (const c of cells) console.log(fmt(c)) + + if (cfg.verbose) { + for (const lang of langs) { + const missing = missingByLang[lang] + console.log(`\n--- ${lang}: ${missing.length} missing ---`) + for (const s of missing) console.log(" " + s) + } + } +} + +// Run coverage for the given languages (or all discovered when none are named). +export function runCoverageMode(options, langArgs = []) { + const cfg = resolveConfig({ ...options, coverage: true }) + runCoverage(cfg, langArgs) +} + +// Run the per-language build. `options` carry the project root and the hooks +// that specialize it (see loc-tools/README.md). Language selection: options.langs +// if given, else the named `langArgs`, else the source language plus every +// discovered language. +export function runLocGen(options, langArgs = []) { + const cfg = resolveConfig(options) + + if (cfg.coverage) { + runCoverage(cfg, langArgs) + return + } + if (!cfg.defaultLocG) throw new Error("runLocGen: options.defaultLocG is required for a build run") + + let langs = cfg.langs || (langArgs.length > 0 ? langArgs : null) + if (!langs) langs = [cfg.srcLang, ...discoverLangs(cfg)] + + console.log("locgen: " + langs.join(", ")) + + const reports = [] + let leaveModified = false + let failedLang = null + try { + for (const lang of langs) { + try { + reports.push(processLanguage(cfg, lang)) + } catch (err) { + // A build failure leaves loc.g.ts in place for debugging. + leaveModified = true + failedLang = lang + const out = (err.stdout && err.stdout.toString()) || "" + const errOut = (err.stderr && err.stderr.toString()) || "" + console.error(`\nERROR building ${lang}: ${err.message}`) + if (out) console.error(out.split("\n").slice(-25).join("\n")) + if (errOut) console.error(errOut.split("\n").slice(-25).join("\n")) + throw err + } + } + } finally { + if (leaveModified) { + console.error(`\nloc.g.ts left MODIFIED (last: ${failedLang}) for debugging; not restored.`) + } else if (cfg.keep) { + console.log("\nloc.g.ts left in place (--keep); not restored.") + } else { + writeDefault(cfg.locGPath, cfg.defaultLocG) + console.log("\nloc.g.ts restored to default state.") + } + } + + console.log("\n=== summary ===") + const head = ["lang", "merged", "identity", cfg.smallFont, "glyph", "len-warn", "table", "hex bytes"] + console.log(head.join("\t")) + for (const r of reports) { + console.log( + [ + r.lang, + r.merged, + r.identityDropped, + r.smallDropped === undefined ? "-" : r.smallDropped, + r.glyphDropped, + r.lengthWarnings, + r.table === undefined ? "-" : r.table, + r.skipped ? "(skipped)" : r.hexSize, + ].join("\t"), + ) + } +} diff --git a/loc-tools/index.mjs b/loc-tools/index.mjs new file mode 100644 index 0000000..3823087 --- /dev/null +++ b/loc-tools/index.mjs @@ -0,0 +1,13 @@ +// Public localization tooling API. +// +// A consuming app builds a thin wrapper over runLocGen, supplying its root and +// any app-specific hooks (see loc-tools/README.md). Extraction, layer +// discovery, font coverage, and charset resolution are exposed for wrappers and +// tests that need the pieces directly. + +export { runLocGen, runCoverageMode } from "./gen.mjs" +export { extractCatalog, runLocStrings } from "./extract.mjs" +export { discoverLayers, mergeLayers, readCatalog } from "./catalogs.mjs" +export { createFontProvider, parseFontCoverage, defaultTextPath, toHexCp } from "./fonts.mjs" +export { resolveCharsets, validateCharsets } from "./charsets.mjs" +export { emitLocG, writeDefault, GENERIC_DEFAULT_LOC_G } from "./emit.mjs" diff --git a/package.json b/package.json index 4ae4f11..8f24d65 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,16 @@ { + "name": "@microbit-apps/ui-core", + "type": "module", + "exports": { + "./loc": "./loc-tools/index.mjs" + }, + "bin": { + "loc-strings": "./loc-tools/bin/loc-strings.mjs", + "loc-gen": "./loc-tools/bin/loc-gen.mjs" + }, "scripts": { "format": "prettier --write \"**/*.ts\"", - "loc:strings": "node scripts/locstrings.mjs" + "loc:strings": "node loc-tools/bin/loc-strings.mjs" }, "devDependencies": { "prettier": "^3.8.3"