Add localization system

loc.ts

@@ -0,0 +1,90 @@
+/**
+ * Build-time localization runtime.
+ *
+ * The catalog holds the translated strings for exactly one language and is
+ * assigned at startup by a generated per-language file (`loc.g.ts`) in the
+ * consuming app. English strings are used as both the catalog keys and the
+ * fallback: an English build ships no table at all, so `ui.loc` and friends
+ * become the identity function and simply return the source English string.
+ *
+ * One language ships per build image. To localize an app, generate a
+ * `loc.g.ts` that assigns `_loc.table` (the id-to-string catalog) and,
+ * optionally, `_loc.defaultFont` (a language-specific bitmap font). Build one
+ * image per language.
+ *
+ * Catalog keys are the English source strings. Strings that need different
+ * translations depending on where they appear use a `"context#string"` key
+ * (see `ui.locc`) so the same English word can map to different translations.
+ */
+namespace _loc {
+    /**
+     * Translation catalog for the shipped language, assigned by the app's
+     * generated `loc.g.ts`. `undefined` means an English build with no table,
+     * in which case localization is the identity function.
+     */
+    export let table: { [key: string]: string } = undefined
+
+    /**
+     * Per-language default bitmap font, assigned by the app's generated
+     * `loc.g.ts`. `undefined` means the built-in `bitmaps.font8`.
+     */
+    export let defaultFont: ui.TextFont = undefined
+}
+
+namespace ui {
+    /**
+     * Localizes an English source string.
+     *
+     * With no catalog (an English build) the source string is returned
+     * unchanged. Otherwise the catalog is looked up by the source string; a
+     * missing entry falls back to the source string.
+     */
+    export function loc(s: string): string {
+        if (!_loc.table) return s
+        const entry = _loc.table[s]
+        if (entry === undefined) return s
+        return entry
+    }
+
+    /**
+     * Localizes an English source string that needs a context-specific
+     * translation.
+     *
+     * The catalog is looked up by the composite key `context + "#" + s`, which
+     * lets the same English word (for example "on" as a relay state versus
+     * elsewhere) map to different translations. On a miss the lookup falls back
+     * to the plain-string translation via `loc(s)`, and finally to the English
+     * source string. With no catalog the source string is returned unchanged.
+     */
+    export function locc(context: string, s: string): string {
+        if (!_loc.table) return s
+        const entry = _loc.table[context + "#" + s]
+        if (entry === undefined) return loc(s)
+        return entry
+    }
+
+    /**
+     * Localizes an English source string with positional interpolation.
+     *
+     * The source string is localized first, then each `"{i}"` token is
+     * replaced with `args[i]`. Because translation happens before substitution,
+     * a translated string may reorder the placeholders (for example map
+     * "LED {0} {1}" to "{1} {0} DEL").
+     */
+    export function locf(s: string, args: string[]): string {
+        let result = loc(s)
+        for (let i = 0; i < args.length; i++) {
+            result = result.replaceAll("{" + i + "}", args[i])
+        }
+        return result
+    }
+
+    /**
+     * Returns the default bitmap font for the shipped language, or
+     * `bitmaps.font8` when no language-specific font is set.
+     */
+    export function locFont(): TextFont {
+        if (_loc.defaultFont) return _loc.defaultFont
+        return bitmaps.font8
+    }
+}

locales/en.json

@@ -0,0 +1 @@
+{}

package.json

@@ -1,6 +1,7 @@
 {
     "scripts": {
-        "format": "prettier --write \"**/*.ts\""
+        "format": "prettier --write \"**/*.ts\"",
+        "loc:strings": "node scripts/locstrings.mjs"
     },
     "devDependencies": {
         "prettier": "^3.8.3"

pxt.json

@@ -1,8 +1,15 @@
 {
     "name": "ui-core",
     "version": "0.0.4",
+    "dependencies": {
+        "core": "*",
+        "radio": "*",
+        "microphone": "*",
+        "display-shield": "github:microbit-apps/display-shield#v1.1.4"
+    },
     "files": [
         "ns.ts",
+        "loc.ts",
         "geometry.ts",
         "viewport.ts",
         "draw-surface.ts",
@@ -19,23 +26,17 @@
         "modal.ts",
         "icons.ts"
     ],
-    "yotta": {
-        "config": {
-            "DEVICE_BLE": 0
-        }
-    },
-    "supportedTargets": [
-        "microbit"
-    ],
     "testFiles": [
         "test.ts",
         "samples.ts"
     ],
-    "dependencies": {
-        "core": "*",
-        "radio": "*",
-        "microphone": "*",
-        "display-shield": "github:microbit-apps/display-shield#v1.1.4"
-    },
-    "testDependencies": {}
+    "testDependencies": {},
+    "supportedTargets": [
+        "microbit"
+    ],
+    "yotta": {
+        "config": {
+            "DEVICE_BLE": 0
+        }
+    }
 }

scripts/locstrings.mjs

@@ -0,0 +1,158 @@
+// Regenerates locales/en.json from source.
+//
+// Scans the .ts files listed in pxt.json "files" (so test files are excluded)
+// and extracts the source display strings passed to the localization helpers:
+//
+//   ui.loc("...")  / loc("...")
+//   ui.locf("...") / locf("...")   (first argument only)
+//   ui.locc("ctx", "...") / locc("ctx", "...")  (emits key "ctx#string")
+//
+// Arguments must be string literals at the call site (the documented
+// convention; see locales/README.md). Any other argument is reported as a
+// loud warning naming the file and the argument, so a missed string is never
+// dropped silently. A call whose argument is deliberately dynamic (its
+// strings reach the catalog by another route) is suppressed by putting
+// "locstrings-ignore" in a comment on the call's line.
+//
+// The result is a sorted JSON object mapping each catalog key to its source
+// display string. For locc entries the value is the plain string, without the
+// context prefix. The file is fully derived: it is regenerated from source, so
+// any prior keys not found in the current sources are dropped.
+
+import { readFileSync, writeFileSync } from "node:fs"
+import { dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..")
+
+// A double- or single-quoted string literal, honoring escaped quotes.
+const STR = `"(?:[^"\\\\]|\\\\.)*"|'(?:[^'\\\\]|\\\\.)*'`
+// One helper argument: a string literal, or any run up to the next comma or
+// close paren. The string alternative is tried first so commas or parens
+// inside a literal do not truncate the capture.
+const ARG = `(?:${STR}|[^,)]*)`
+
+// Guards for false call-site matches: a helper definition (`function
+// loc(...)`) is not a call, and a preceding identifier character or dot means
+// the name is a property access (obj.loc) or a longer identifier (myloc),
+// not the helper.
+const DEF = `(?<!\\bfunction\\s{1,8})(?<![$\\w.])`
+// loc(...) and locf(...): first argument. The optional "f" plus the required
+// "(" lookahead avoids matching locc and locFont.
+const LOC_RE = new RegExp(`${DEF}(?:ui\\.)?locf?\\s*\\(\\s*(${ARG})`, "g")
+// locc("ctx", "..."): both arguments.
+const LOCC_RE = new RegExp(
+    `${DEF}(?:ui\\.)?locc\\s*\\(\\s*(${ARG})\\s*,\\s*(${ARG})`,
+    "g",
+)
+
+const STR_ONLY = new RegExp(`^(?:${STR})$`)
+
+// Removes line and block comments so commented-out code is neither extracted
+// nor reported as an unresolved argument. String and template literals are
+// copied verbatim so a "//" or "/*" inside a literal is preserved.
+function stripComments(src) {
+    let out = ""
+    let i = 0
+    const n = src.length
+    while (i < n) {
+        const c = src[i]
+        const d = src[i + 1]
+        if (c == "/" && d == "/") {
+            i += 2
+            while (i < n && src[i] != "\n") i++
+            continue
+        }
+        if (c == "/" && d == "*") {
+            i += 2
+            while (i < n && !(src[i] == "*" && src[i + 1] == "/")) i++
+            i += 2
+            continue
+        }
+        if (c == '"' || c == "'" || c == "`") {
+            out += c
+            i++
+            while (i < n) {
+                const ch = src[i]
+                if (ch == "\\") {
+                    out += ch + (i + 1 < n ? src[i + 1] : "")
+                    i += 2
+                    continue
+                }
+                out += ch
+                i++
+                if (ch == c) break
+            }
+            continue
+        }
+        out += c
+        i++
+    }
+    return out
+}
+
+// Turns a source string literal into its runtime string value.
+function unquote(lit) {
+    const body = lit.slice(1, -1)
+    return body.replace(/\\(.)/g, (_, ch) => {
+        if (ch == "n") return "\n"
+        if (ch == "t") return "\t"
+        if (ch == "r") return "\r"
+        return ch
+    })
+}
+
+let warnings = 0
+
+// Resolves a captured helper argument to its string value, or null (with a
+// warning) when it is not a string literal.
+function resolveArg(raw, file) {
+    const arg = raw.trim()
+    if (arg == "") return null
+    if (STR_ONLY.test(arg)) return unquote(arg)
+    console.warn(`locstrings: WARNING: ${file}: cannot resolve loc argument '${arg}'`)
+    warnings++
+    return null
+}
+
+const pxt = JSON.parse(readFileSync(join(root, "pxt.json"), "utf8"))
+const catalog = {}
+
+// The file that implements the localization API passes its own parameters
+// through loc(); it defines the helpers rather than consuming them, so it is
+// not a source of catalog keys.
+const API_DEF_RE = new RegExp(`\\bexport\\s+function\\s+loc\\s*\\(`)
+
+for (const file of pxt.files) {
+    if (!file.endsWith(".ts")) continue
+    const raw = readFileSync(join(root, file), "utf8")
+        .split("\n")
+        .filter(line => !line.includes("locstrings-ignore"))
+        .join("\n")
+    const src = stripComments(raw)
+    if (API_DEF_RE.test(src)) continue
+
+    for (const m of src.matchAll(LOCC_RE)) {
+        const ctx = resolveArg(m[1], file)
+        const str = resolveArg(m[2], file)
+        if (ctx == null || str == null) continue
+        catalog[ctx + "#" + str] = str
+    }
+    for (const m of src.matchAll(LOC_RE)) {
+        const str = resolveArg(m[1], file)
+        if (str == null) continue
+        catalog[str] = str
+    }
+}
+
+const sorted = {}
+for (const key of Object.keys(catalog).sort()) sorted[key] = catalog[key]
+
+writeFileSync(
+    join(root, "locales", "en.json"),
+    JSON.stringify(sorted, null, 4) + "\n",
+)
+
+const count = Object.keys(sorted).length
+console.log(`locstrings: wrote ${count} key(s) to locales/en.json`)
+if (warnings) console.log(`locstrings: ${warnings} unresolved loc argument(s); see warnings above`)

test.ts

@@ -489,11 +489,88 @@ namespace ui.core.tests {
         )
     }
 
+    /**
+     * Smoke harness for the build-time localization runtime.
+     */
+    export function runLocTest(): void {
+        // Identity path: no catalog means source strings pass through.
+        _loc.table = undefined
+        _loc.defaultFont = undefined
+        control.assert(loc("Hello") == "Hello", "loc identity no table")
+        control.assert(
+            locf("Hi {0}", ["Sam"]) == "Hi Sam",
+            "locf identity no table",
+        )
+        control.assert(locc("relay", "on") == "on", "locc identity no table")
+
+        // Table hit and fallback-on-miss.
+        _loc.table = {
+            Hello: "Bonjour",
+            "LED {0} {1}": "{1} {0} DEL",
+        }
+        control.assert(loc("Hello") == "Bonjour", "loc table hit")
+        control.assert(loc("Missing") == "Missing", "loc fallback on miss")
+
+        // locc: composite key, plain-string fallback, English fallback.
+        _loc.table = {
+            "relay#on": "MARCHE",
+            on: "activee",
+        }
+        control.assert(locc("relay", "on") == "MARCHE", "locc composite hit")
+        control.assert(
+            locc("switch", "on") == "activee",
+            "locc plain-string fallback",
+        )
+        control.assert(
+            locc("switch", "off") == "off",
+            "locc source fallback",
+        )
+
+        // locf: interpolation with reordered placeholders in translation.
+        _loc.table = {
+            "LED {0} {1}": "{1} {0} DEL",
+        }
+        control.assert(
+            locf("LED {0} {1}", ["red", "on"]) == "on red DEL",
+            "locf reordered interpolation",
+        )
+        control.assert(
+            locf("Plain {0}", ["x"]) == "Plain x",
+            "locf missing key interpolation",
+        )
+
+        // locFont: default font8 shape when unset, assigned font when set.
+        _loc.table = undefined
+        _loc.defaultFont = undefined
+        const fallbackFont = locFont()
+        control.assert(
+            fallbackFont.charWidth == bitmaps.font8.charWidth,
+            "locFont default char width",
+        )
+        control.assert(
+            fallbackFont.charHeight == bitmaps.font8.charHeight,
+            "locFont default char height",
+        )
+
+        const customFont: TextFont = {
+            charWidth: 3,
+            charHeight: 5,
+            data: hex``,
+        }
+        _loc.defaultFont = customFont
+        control.assert(locFont() == customFont, "locFont returns assigned font")
+
+        // Reset so ordering does not leak state to other tests.
+        _loc.table = undefined
+        _loc.defaultFont = undefined
+    }
+
     runGeometrySmokeTest()
     runViewportSmokeTest(2)
     runAssetResolverSmokeTest()
     runRuntimeSmokeTest()
     runLayoutSmokeTest()
+    runLocTest()
 
     control.__log(1, "All tests passed!")
 }