Add backoff chain, sentence boundary awareness, and repetition penalty

README.md

@@ -149,6 +149,8 @@ archive/             # legacy archived scripts
 - `--fixed-n`: fixed n when mode is `fixed`.
 - `--n-weights`: manual n weights when mode is `manual`.
 - `--n-temperature`: temperature on n-selection distribution.
+- `--repetition-penalty`: divide probability of recently-seen characters by this factor (`1.0` = disabled, recommended range `1.2`–`1.5`).
+- `--repetition-window`: number of recent characters considered for repetition penalty (default `20`).
 
 ## Verbosity modes
 
@@ -167,10 +169,11 @@ archive/             # legacy archived scripts
 Character sampling applies in this order:
 
 1. raw conditional probabilities
-2. `temperature`
-3. `top-k`
-4. `top-p`
-5. renormalize and sample
+2. `repetition-penalty`
+3. `temperature`
+4. `top-k`
+5. `top-p`
+6. renormalize and sample
 
 For n-value selection:
 
@@ -184,6 +187,12 @@ For n-value selection:
 - Fixed global variable dependency in legacy JSONL character search printer.
 - Removed hard-coded local Windows paths from runnable code.
 
+## Improvements
+
+- **Backoff chain**: when the sampled n-gram order yields no candidates, the model now deterministically backs off to n-1, n-2, … down to 1 instead of random retrying. This ensures the longest valid context is always used first.
+- **Sentence boundary awareness**: training splits text on sentence-ending punctuation (`。！？…`) so n-grams never cross sentence boundaries. Punctuation characters are still counted as unigrams.
+- **Repetition penalty**: new `--repetition-penalty` parameter down-weights characters that appeared recently in the generated text, reducing repetitive output. Use `--repetition-window` to control how many recent characters are considered.
+
 ## Contributing
 
 See `CONTRIBUTING.md`.

src/littlelm/build_dictionary.py

@@ -12,15 +12,27 @@
 from .reader import iter_jsonl
 from .utils import ensure_parent_dir, safe_replace
 
-_SPLIT_RE = re.compile(r"[ \u3000\xa0\r\n]+")
+# 空白分隔符（片段内部不跨空白构建 n-gram）
+_SPACE_RE = re.compile(r"[ \u3000\xa0\r\n]+")
+# 句末标点作为边界：n-gram 不跨句末标点，但标点本身保留为独立片段供 1-gram 统计
+_SENT_BOUNDARY_RE = re.compile(r"([。！？…]+)")
 
 
 def split_text(text: str) -> list[str]:
     if text is None:
         return []
     if not isinstance(text, str):
         text = str(text)
-    return [part for part in _SPLIT_RE.split(text) if part]
+    fragments: list[str] = []
+    # 先按句末标点切割，capture group 让标点留在结果里成为独立片段
+    for part in _SENT_BOUNDARY_RE.split(text):
+        if not part:
+            continue
+        # 再按空白切割
+        for sub in _SPACE_RE.split(part):
+            if sub:
+                fragments.append(sub)
+    return fragments
 
 
 def iter_ngrams_from_text(text: str, gram_size: int) -> Iterator[str]:

src/littlelm/constants.py

@@ -12,6 +12,8 @@
 DEFAULT_TOP_K = 0
 DEFAULT_TOP_P = 1.0
 DEFAULT_N_TEMPERATURE = 1.0
+DEFAULT_REPETITION_PENALTY = 1.0
+DEFAULT_REPETITION_WINDOW = 20
 
 RANDOM_SEED_PROMPTS = [
 	"你今天在想什么？",

src/littlelm/sample.py

@@ -15,6 +15,8 @@
     DEFAULT_N_SELECTION_MODE,
     DEFAULT_N_TEMPERATURE,
     DEFAULT_PROMPT_MODE,
+    DEFAULT_REPETITION_PENALTY,
+    DEFAULT_REPETITION_WINDOW,
     DEFAULT_TEMPERATURE,
     DEFAULT_TOP_K,
     DEFAULT_TOP_P,
@@ -46,7 +48,8 @@ class GenerationConfig:
     fixed_n: int | None = None
     n_weights: dict[int, float] | None = None
     n_temperature: float = DEFAULT_N_TEMPERATURE
-    max_try: int = 50
+    repetition_penalty: float = DEFAULT_REPETITION_PENALTY
+    repetition_window: int = DEFAULT_REPETITION_WINDOW
 
 
 def infer_n_from_filename(filename: str | Path) -> int | None:
@@ -188,6 +191,19 @@ def apply_top_p(candidates: list[tuple[str, float]], p: float) -> list[tuple[str
     return picked
 
 
+def apply_repetition_penalty(
+    candidates: list[tuple[str, float]],
+    recent_text: str,
+    penalty: float,
+    window: int = DEFAULT_REPETITION_WINDOW,
+) -> list[tuple[str, float]]:
+    """对最近生成过的字符降权，抑制重复。penalty=1.0 表示不惩罚。"""
+    if penalty <= 1.0:
+        return list(candidates)
+    recent = set(recent_text[-window:])
+    return [(char, prob / penalty if char in recent else prob) for char, prob in candidates]
+
+
 def parse_n_weights(spec: str) -> dict[int, float]:
     weights: dict[int, float] = {}
     text = (spec or "").strip()
@@ -320,7 +336,6 @@ def predict_next_char(
     context: str,
     ngrams_count_folder: str | Path,
     max_n: int = 10,
-    max_try: int = 50,
     debug: bool = False,
 ) -> tuple[str, StepDebugInfo | None]:
     if not isinstance(context, str):
@@ -330,35 +345,30 @@ def predict_next_char(
     if not ngrams_count or not ngrams_count[0]:
         return "", None
 
-    config = GenerationConfig(max_n=max_n, max_try=max_try)
+    config = GenerationConfig(max_n=max_n)
 
     max_available = len(ngrams_count)
     max_k = min(len(context) + 1, config.max_n, max_available)
     if max_k < 1:
         return "", None
 
     available_n = list(range(1, max_k + 1))
-    chosen_n = 1
-    n_distribution: list[tuple[int, float]] = []
-    raw_candidates: list[tuple[str, float]] = []
 
-    for _ in range(config.max_try):
-        if not available_n:
-            break
-        n_distribution = build_n_selection_distribution(available_n, mode="weighted")
-        chosen_n = sample_n_value(n_distribution)
-        raw_candidates = _candidate_char_distribution_for_n(context, chosen_n, ngrams_count)
-        raw_candidates = normalize_distribution(raw_candidates)
+    # 从分布中采样首选 n，然后确定性地逐级回退直到找到候选
+    n_distribution = build_n_selection_distribution(available_n, mode="weighted")
+    chosen_n = sample_n_value(n_distribution)
+
+    raw_candidates: list[tuple[str, float]] = []
+    for try_n in range(chosen_n, 0, -1):
+        raw_candidates = normalize_distribution(
+            _candidate_char_distribution_for_n(context, try_n, ngrams_count)
+        )
         if raw_candidates:
+            chosen_n = try_n
             break
-        available_n = [n for n in available_n if n != chosen_n]
 
     if not raw_candidates:
-        chosen_n = 1
-        n_distribution = [(1, 1.0)]
-        raw_candidates = normalize_distribution(_candidate_char_distribution_for_n(context, 1, ngrams_count))
-        if not raw_candidates:
-            return "", None
+        return "", None
 
     final_candidates = sorted(raw_candidates, key=lambda item: item[1], reverse=True)
     chosen_char = random.choices([c for c, _ in final_candidates], weights=[p for _, p in final_candidates], k=1)[0]
@@ -397,6 +407,8 @@ def generate_text(
     fixed_n: int | None = None,
     n_weights: dict[int, float] | None = None,
     n_temperature: float = DEFAULT_N_TEMPERATURE,
+    repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
+    repetition_window: int = DEFAULT_REPETITION_WINDOW,
     verbosity: str = DEFAULT_VERBOSITY,
     return_debug: bool = False,
 ) -> str | tuple[str, list[StepDebugInfo]]:
@@ -414,6 +426,8 @@ def generate_text(
         fixed_n=fixed_n,
         n_weights=n_weights,
         n_temperature=n_temperature,
+        repetition_penalty=repetition_penalty,
+        repetition_window=repetition_window,
     )
 
     if config.temperature <= 0:
@@ -424,6 +438,8 @@ def generate_text(
         raise ValueError("top-p must be in (0, 1]")
     if config.n_temperature <= 0:
         raise ValueError("n-temperature must be > 0")
+    if config.repetition_penalty < 1.0:
+        raise ValueError("repetition-penalty must be >= 1.0")
     if config.n_selection_mode == "fixed" and config.fixed_n is None:
         raise ValueError("fixed-n is required when n-selection-mode=fixed")
     if config.n_selection_mode == "manual" and not config.n_weights:
@@ -440,36 +456,32 @@ def generate_text(
             break
 
         available_n = list(range(1, max_k + 1))
-        chosen_n = 1
-        n_distribution: list[tuple[int, float]] = []
-        raw_candidates: list[tuple[str, float]] = []
 
-        for _ in range(config.max_try):
-            if not available_n:
-                break
-            n_distribution = build_n_selection_distribution(
-                available_n,
-                mode=config.n_selection_mode,
-                n_temperature=config.n_temperature,
-                fixed_n=config.fixed_n,
-                manual_weights=config.n_weights,
+        # 从分布中采样首选 n，然后确定性地逐级回退直到找到候选
+        n_distribution = build_n_selection_distribution(
+            available_n,
+            mode=config.n_selection_mode,
+            n_temperature=config.n_temperature,
+            fixed_n=config.fixed_n,
+            manual_weights=config.n_weights,
+        )
+        chosen_n = sample_n_value(n_distribution)
+
+        raw_candidates: list[tuple[str, float]] = []
+        for try_n in range(chosen_n, 0, -1):
+            raw_candidates = normalize_distribution(
+                _candidate_char_distribution_for_n(text, try_n, ngrams_count)
             )
-            chosen_n = sample_n_value(n_distribution)
-            raw_candidates = _candidate_char_distribution_for_n(text, chosen_n, ngrams_count)
-            raw_candidates = normalize_distribution(raw_candidates)
             if raw_candidates:
+                chosen_n = try_n
                 break
-            available_n = [n for n in available_n if n != chosen_n]
-
-        if not raw_candidates:
-            chosen_n = 1
-            n_distribution = [(1, 1.0)]
-            raw_candidates = normalize_distribution(_candidate_char_distribution_for_n(text, 1, ngrams_count))
 
         if not raw_candidates:
             break
 
         candidates = normalize_distribution(raw_candidates)
+        candidates = apply_repetition_penalty(candidates, text, config.repetition_penalty, config.repetition_window)
+        candidates = normalize_distribution(candidates)
         candidates = apply_temperature(candidates, config.temperature)
         candidates = normalize_distribution(candidates)
         candidates = apply_top_k(candidates, config.top_k)
@@ -586,6 +598,18 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--fixed-n", type=int, help="Fixed n value when n-selection-mode=fixed.")
     parser.add_argument("--n-weights", help="Manual n weights, for example: 1:0.1,2:0.2,3:0.7")
     parser.add_argument("--n-temperature", type=float, default=DEFAULT_N_TEMPERATURE, help="Temperature for n selection distribution.")
+    parser.add_argument(
+        "--repetition-penalty",
+        type=float,
+        default=DEFAULT_REPETITION_PENALTY,
+        help="Divide probability of recently-seen chars by this factor (1.0 = disabled, >1.0 = penalize).",
+    )
+    parser.add_argument(
+        "--repetition-window",
+        type=int,
+        default=DEFAULT_REPETITION_WINDOW,
+        help="Number of recent characters to consider for repetition penalty.",
+    )
     parser.add_argument("--debug", action="store_true", help="Print debug candidate info.")
     return parser
 
@@ -626,6 +650,8 @@ def main(argv: list[str] | None = None) -> int:
         fixed_n=args.fixed_n,
         n_weights=manual_weights,
         n_temperature=args.n_temperature,
+        repetition_penalty=args.repetition_penalty,
+        repetition_window=args.repetition_window,
         verbosity=verbosity,
         return_debug=True,
     )
@@ -655,6 +681,8 @@ def main(argv: list[str] | None = None) -> int:
     print(f"  n_selection_mode: {args.n_selection_mode}")
     print(f"  fixed_n: {args.fixed_n}")
     print(f"  n_temperature: {args.n_temperature}")
+    print(f"  repetition_penalty: {args.repetition_penalty}")
+    print(f"  repetition_window: {args.repetition_window}")
     print("")
 
     for info in step_infos: