Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 | 56x 4x 4x 4x 15x 15x 2x 15x 2x 15x 6x 9x 9x 9x 2x 7x 2x 7x 7x 1x 6x 1x 5x 21x 21x 32x 32x 4x 4x 4x 6x 6x 6x 2x 2x 2x 4x 4x 4x 1x 1x 2x 2x 2x 4x 4x 4x 3x 3x 3x 5x 5x 5x 5x 3x 3x 2x 2x 1x 1x 19x 2x 17x 1x 16x 1x 15x 3x 12x 2x 10x 32x 32x 31x 31x 31x 46x 46x 46x 45x 45x 44x 44x 56x 56x 56x 55x 55x 55x 55x 3x 3x 3x 31x 58x 58x 58x 754x 754x 58x 4x 4x 30x 30x 301x 299x 30x 53x 52x 52x 51x 53x 51x 50x 50x 50x 50x 7808x 7808x 50x 51x 51x 51x 51x 357x 357x 357x 357x 51x 29x 29x 1x 28x 28x 28x 28x 55x 55x 49x 49x 596x 49x 49x 49x 49x 49x 28x 23x 8x 7x 3x 1x 28x 18x 6x 6x 3x 1x 28x 5x 5x 8x 23x 2x 21x 1x 20x 20x 20x 9x 9x 11x 11x 11x 20x 11x 37x 28x 28x 445x 28x 364x 28x 831x 73x 28x 28x 28x 28x 49x 49x 49x 28x 3x 3x 3x 3x 3x 3x 3x 3x 3x | #!/usr/bin/env node
// SPDX-FileCopyrightText: 2024-2026 Hack23 AB
// SPDX-License-Identifier: Apache-2.0
/**
* @module scripts/discover-untranslated-briefs
* @description Discovery worker for the scheduled `news-translate` workflow.
*
* Scans every `analysis/daily/<date>/<slug>/executive-brief.md` source brief
* (including the legacy `extended/executive-brief.md` location) and computes,
* for each source, which of the 13 non-English translation companions
* (`executive-brief_<lang>.md`) are still missing.
*
* The script emits a prioritised JSON queue of work items that the AI
* translator should pick up, bounded by `--max-briefs` so a single workflow
* run cannot exceed its 60-minute budget. The default cap (2 briefs / run)
* comes from the sizing analysis recorded in the PR description: 2 briefs ×
* 13 languages × ~12 KB ≈ 26 markdown files / ~310 KB of generated output,
* comfortably inside the gh-aw safe-outputs 10 MB patch ceiling and the
* Claude Sonnet 4.6 60-minute wall-clock budget.
*
* Priority rules — `fresh-then-backlog` mode (default):
*
* The queue is built from two pools so the day's newest brief still gets
* timely coverage on at least one of the three daily runs, while the
* long-tail backlog of older briefs (currently ~92 sources / ~1,196 missing
* translations) actually drains rather than being starved by today's wins.
*
* 1. **Fresh slice** (at most 1 entry per run): newest source with any
* missing language. Tie-breakers: more-missing first, slug asc,
* non-extended first.
* 2. **Backlog slice** (remaining `max-briefs - 1` slots): every other
* source with gaps, sorted by `<date>` ASC (oldest first), then
* `missingCount` ASC (finish half-done briefs before starting new
* ones), then `<slug>` ASC, then non-extended first.
*
* Final queue = `freshSlice.concat(backlogSlice)`.
*
* When `--max-briefs 1`, alternate fresh/backlog by run-number parity
* (`--run-number`, default 0, normally driven by `$GITHUB_RUN_NUMBER`) so
* the scheduled cadence still drains backlog while preserving freshness.
*
* Alternative modes (via `--mode`):
*
* - `backlog-only` — drop the fresh slot entirely; oldest-first across the
* entire backlog. Useful for catch-up batches.
* - `newest-first` — legacy behaviour: newest date first, more-missing
* first, slug asc. Retained for one-off operator dispatch where the
* operator explicitly wants today's brief covered first.
*
* Invocation:
*
* node scripts/discover-untranslated-briefs.js \
* [--repo-root <path>] \
* [--max-briefs <n>] # default 2
* [--max-age-days <n>] # default 180; older briefs are skipped
* [--mode <name>] # fresh-then-backlog | backlog-only | newest-first
* [--run-number <n>] # parity selector when --max-briefs 1
* [--max-source-lines <n>] # default 300; briefs above this threshold are
* # flagged largeSource:true and the agent
* # switches to a 2-phase skeleton-then-edit
* # translation strategy (see news-translate.md)
* [--target-brief <id>] # optional operator override: when set, the
* # queue contains ONLY this brief regardless
* # of mode / max-briefs / max-age-days.
* # Accepted forms:
* # YYYY-MM-DD/<slug>
* # YYYY-MM-DD/<slug>/extended
* # analysis/daily/YYYY-MM-DD/<slug>/executive-brief.md
* # analysis/daily/YYYY-MM-DD/<slug>/extended/executive-brief.md
* # Used by operator-dispatched runs that need
* # to (re)translate one specific brief.
* [--output <path>] # default stdout
* [--include-extended] # also scan extended/executive-brief.md
*
* Exit codes:
* 0 — queue successfully written (may be empty if everything is translated)
* 1 — unexpected error (I/O, invalid CLI input)
*
* Output JSON shape:
* {
* "generatedAt": "2026-05-16T08:24:16.909Z",
* "options": { "mode": "fresh-then-backlog", "runNumber": 207, ... },
* "totals": {
* "sourcesScanned": 92,
* "sourcesWithGaps": 92,
* "translationsMissing": 1196,
* "queued": 2,
* "queuedTranslations": 26,
* "freshNewestDate": "2026-05-16",
* "backlogOldestDate": "2025-11-19",
* "topMissingLangs": [
* { "lang": "ja", "count": 92 },
* { "lang": "ko", "count": 92 },
* { "lang": "zh", "count": 92 }
* ]
* },
* "translationRegister": {
* "sv": {
* "name": "Swedish", "family": "Nordic",
* "register": "Formal, impersonal …",
* "fixedTokenNote": "IMF/WEO verbatim — never IVF …",
* "terms": { "European Parliament": "Europaparlamentet", … }
* },
* ...
* },
* "queue": [
* {
* "date": "2026-05-15",
* "slug": "breaking",
* "sourcePath": "analysis/daily/2026-05-15/breaking/executive-brief.md",
* "missingLangs": ["sv","da","no","fi","de","fr","es","nl","ar","he","ja","ko","zh"],
* "missingCount": 13,
* "isExtended": false,
* "sourceH2Count": 8,
* "sourceH2Titles": [
* { "line": 7, "title": "Headline Intelligence" },
* { "line": 96, "title": "IMF Economic Context" },
* { "line": 146, "title": "IMF Economic Context — May 2026 Update" }
* ],
* "sourceFixedTokens": { "IMF": 17, "WEO": 2, "TA-id": 4 },
* "sourceLineCount": 380,
* "largeSource": true
* },
* ...
* ]
* }
*/
import fs from 'node:fs';
import path from 'node:path';
import process from 'node:process';
import { ALL_LANGUAGES } from './constants/language-core.js';
/** Canonical 13 non-English target language codes, derived from ALL_LANGUAGES. */
export const TARGET_LANGS = Object.freeze(ALL_LANGUAGES.filter((lang) => lang !== 'en'));
/** Manual-dispatch upper bound that keeps one 60-minute run inside budget. */
export const MAX_BRIEFS_LIMIT = 4;
/**
* Source-brief line-count threshold above which a brief is flagged as
* `largeSource: true` in the queue. The translator agent uses this signal
* to switch from a one-shot `create` per language to a 2-phase
* skeleton-then-edit strategy that bounds each inference call's output
* size — a one-shot `create` of a ~385-line translated file is what
* triggered the unrecoverable transient-API-error loop in run
* #26181499722 (the queue contained
* `analysis/daily/2026-05-13/election-cycle/executive-brief.md` at 385
* lines; the first Swedish `create` retried 5× then stalled until manual
* cancel). 300 lines is the conservative cutoff: every brief that has
* translated cleanly in recent successful runs has been <250 lines, and
* the largest one-shot output we have empirical evidence of completing
* reliably is ~280 lines.
*/
export const DEFAULT_MAX_SOURCE_LINES = 300;
/** Discovery prioritisation modes. */
export const DISCOVERY_MODES = Object.freeze([
'fresh-then-backlog',
'backlog-only',
'newest-first',
]);
/**
* Parse a `--target-brief` operator override into a `{ date, slug, isExtended }`
* triple. Accepts four equivalent operator-friendly forms so the same input
* works whether the operator copies a path out of the repo, a date/slug pair
* out of the discovery JSON, or types the canonical short form by hand:
*
* 1. `YYYY-MM-DD/<slug>` — short form
* 2. `YYYY-MM-DD/<slug>/extended` — extended legacy path
* 3. `analysis/daily/YYYY-MM-DD/<slug>/executive-brief.md` — full repo path
* 4. `analysis/daily/YYYY-MM-DD/<slug>/extended/executive-brief.md`
*
* Validation is intentionally strict (whitelisted character classes, fixed date
* format, slug character class) — the value flows from a workflow_dispatch
* string input into a filesystem lookup, so a permissive parser would be a
* directory-traversal foothold.
*
* Throws on any malformed spec; never returns null (callers must check for
* empty input BEFORE calling this helper).
*
* @param {string} spec — already-trimmed, non-empty operator input
* @returns {{ date: string, slug: string, isExtended: boolean }}
*/
export function parseTargetBriefSpec(spec) {
// Strip leading "analysis/daily/" prefix and trailing "/executive-brief.md"
// so all four accepted forms collapse to "<date>/<slug>" or
// "<date>/<slug>/extended".
let core = spec;
if (core.startsWith('analysis/daily/')) {
core = core.slice('analysis/daily/'.length);
}
if (core.endsWith('/executive-brief.md')) {
core = core.slice(0, -'/executive-brief.md'.length);
}
// Reject any path-traversal or absolute-path attempts up-front.
if (
core.startsWith('/') ||
core.includes('..') ||
core.includes('\\') ||
core.includes('\0')
) {
throw new Error(
`--target-brief: refusing path-traversal or absolute path in "${spec}"`,
);
}
const parts = core.split('/');
let isExtended = false;
if (parts.length === 3 && parts[2] === 'extended') {
isExtended = true;
} else if (parts.length !== 2) {
throw new Error(
`--target-brief: expected "YYYY-MM-DD/<slug>" or "YYYY-MM-DD/<slug>/extended" (got "${spec}")`,
);
}
const [date, slug] = parts;
if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) {
throw new Error(
`--target-brief: date "${date}" is not in YYYY-MM-DD format (from "${spec}")`,
);
}
// Slug character class matches the existing on-disk convention used by
// src/config/article-horizons.ts (lowercase, digits, dashes).
if (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(slug)) {
throw new Error(
`--target-brief: slug "${slug}" must match [a-z0-9][a-z0-9-]{0,63} (from "${spec}")`,
);
}
return { date, slug, isExtended };
}
/**
* Parse CLI argv into an options object. Exported for unit tests.
* @param {string[]} argv
*/
export function parseArgs(argv) {
const opts = {
repoRoot: process.cwd(),
maxBriefs: 2,
maxAgeDays: 180,
output: null,
includeExtended: false,
mode: 'fresh-then-backlog',
runNumber: 0,
maxSourceLines: DEFAULT_MAX_SOURCE_LINES,
targetBrief: null,
};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
switch (arg) {
case '--repo-root':
opts.repoRoot = argv[i + 1];
i += 1;
break;
case '--max-briefs':
opts.maxBriefs = Number.parseInt(argv[i + 1], 10);
i += 1;
break;
case '--max-age-days':
opts.maxAgeDays = Number.parseInt(argv[i + 1], 10);
i += 1;
break;
case '--output':
opts.output = argv[i + 1];
i += 1;
break;
case '--include-extended':
opts.includeExtended = true;
break;
case '--mode':
opts.mode = argv[i + 1];
i += 1;
break;
case '--run-number':
opts.runNumber = Number(argv[i + 1]);
i += 1;
break;
case '--max-source-lines':
opts.maxSourceLines = Number.parseInt(argv[i + 1], 10);
i += 1;
break;
case '--target-brief': {
const raw = argv[i + 1];
i += 1;
// Normalize and validate. Empty / whitespace-only / the literal
// string "none" is treated as "no override" so the workflow can
// wire `TARGET_BRIEF: ${{ inputs.target_brief }}` without having
// to special-case the empty-default case in bash.
const trimmed = typeof raw === 'string' ? raw.trim() : '';
if (trimmed === '' || trimmed === 'none') {
opts.targetBrief = null;
break;
}
opts.targetBrief = parseTargetBriefSpec(trimmed);
break;
}
case '--help':
case '-h':
printHelp();
process.exit(0);
break;
default:
Eif (arg.startsWith('--')) {
throw new Error(`Unknown flag: ${arg}`);
}
}
}
if (
!Number.isFinite(opts.maxBriefs) ||
opts.maxBriefs < 1 ||
opts.maxBriefs > MAX_BRIEFS_LIMIT
) {
throw new Error(`--max-briefs must be an integer between 1 and ${MAX_BRIEFS_LIMIT}`);
}
if (!Number.isFinite(opts.maxAgeDays) || opts.maxAgeDays < 1) {
throw new Error('--max-age-days must be a positive integer');
}
if (!DISCOVERY_MODES.includes(opts.mode)) {
throw new Error(
`--mode must be one of: ${DISCOVERY_MODES.join(', ')} (got "${opts.mode}")`,
);
}
if (!Number.isInteger(opts.runNumber) || opts.runNumber < 0) {
throw new Error('--run-number must be a non-negative integer');
}
if (!Number.isFinite(opts.maxSourceLines) || opts.maxSourceLines < 1) {
throw new Error('--max-source-lines must be a positive integer');
}
return opts;
}
function printHelp() {
process.stdout.write(
'Usage: discover-untranslated-briefs.js [--repo-root <path>] ' +
'[--max-briefs <n>] [--max-age-days <n>] [--mode <name>] ' +
'[--run-number <n>] [--max-source-lines <n>] [--target-brief <YYYY-MM-DD/slug>] ' +
'[--output <path>] [--include-extended]\n',
);
}
/**
* Return the absolute path of every `executive-brief.md` (and optionally
* `extended/executive-brief.md`) under `analysis/daily/`. Sources are
* filtered to the last `maxAgeDays` so the queue stays focused on recent
* material.
*
* @param {string} repoRoot
* @param {{ includeExtended: boolean, maxAgeDays: number }} options
* @returns {Array<{ absPath: string, relPath: string, date: string, slug: string, isExtended: boolean }>}
*/
export function findExecutiveBriefSources(repoRoot, options) {
const dailyDir = path.join(repoRoot, 'analysis', 'daily');
if (!fs.existsSync(dailyDir)) return [];
const cutoffMs = Date.now() - options.maxAgeDays * 24 * 60 * 60 * 1000;
const sources = [];
for (const dateEntry of fs.readdirSync(dailyDir, { withFileTypes: true })) {
Iif (!dateEntry.isDirectory()) continue;
const date = dateEntry.name;
if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) continue;
const dateMs = Date.parse(`${date}T00:00:00Z`);
if (Number.isFinite(dateMs) && dateMs < cutoffMs) continue;
const dateDir = path.join(dailyDir, date);
for (const slugEntry of fs.readdirSync(dateDir, { withFileTypes: true })) {
Iif (!slugEntry.isDirectory()) continue;
const slug = slugEntry.name;
// Skip translate-run* dirs (legacy artifacts from the previous workflow)
if (slug.startsWith('translate-run')) continue;
const direct = path.join(dateDir, slug, 'executive-brief.md');
Eif (fs.existsSync(direct)) {
sources.push({
absPath: direct,
relPath: path.relative(repoRoot, direct),
date,
slug,
isExtended: false,
});
}
if (options.includeExtended) {
const extended = path.join(dateDir, slug, 'extended', 'executive-brief.md');
Eif (fs.existsSync(extended)) {
sources.push({
absPath: extended,
relPath: path.relative(repoRoot, extended),
date,
slug,
isExtended: true,
});
}
}
}
}
return sources;
}
/**
* For a single source brief, compute which `executive-brief_<lang>.md`
* companions are missing in the same directory.
*
* @param {{ absPath: string }} source
* @returns {string[]} sorted list of missing language codes (subset of TARGET_LANGS)
*/
export function findMissingLangs(source) {
const dir = path.dirname(source.absPath);
const missing = [];
for (const lang of TARGET_LANGS) {
const target = path.join(dir, `executive-brief_${lang}.md`);
if (!fs.existsSync(target)) missing.push(lang);
}
return missing;
}
/**
* Fixed-token classes the translator must preserve verbatim. Aligned with
* `scripts/validate-brief-translations.js` `FIXED_TOKEN_PATTERNS` so the
* discovery report and the validator surface the same shape of evidence.
* Keys are stable identifiers (used by the agent prompt); values are the
* global regex the count is computed against.
*
* @type {ReadonlyArray<{ key: string, pattern: RegExp }>}
*/
const FIXED_TOKEN_CLASSES = Object.freeze([
{ key: 'IMF', pattern: /\bIMF\b/g },
{ key: 'WEO', pattern: /\bWEO\b/g },
{ key: 'World Bank', pattern: /\bWorld Bank\b/g },
{ key: 'Fiscal Monitor', pattern: /\bFiscal Monitor\b/g },
{ key: 'data-vintage', pattern: /data-vintage="WEO-[A-Za-z]+-\d{4}"/g },
{ key: 'TA-id', pattern: /\bTA-\d{1,2}-\d{4}-\d{4}\b/g },
{ key: 'procedure-id', pattern: /\b\d{4}\/\d{4}\([A-Z]{3}\)/g },
]);
/**
* Machine-readable mirror of the per-language register the translator agent
* needs on every run — the style register (§ 4) and canonical EP terminology
* pairs (§ 5) of `analysis/methodologies/executive-brief-translation-guide.md`.
*
* Emitted ONCE into the discovery queue payload (as `translationRegister`,
* filtered to the languages the run actually touches) so the agent reads the
* rows it needs inline from `queue.json` instead of opening the full ~18 KB
* guide in-session on every turn. Bounding what enters the agent's context
* directly bounds the per-session effective-token accumulation that is the
* binding constraint on this workflow (see news-translate.md frontmatter and
* regression run #26641760920).
*
* Each entry:
* - `name` — endonym/English name of the language (operator-facing)
* - `family` — register family (Nordic | EU-core | RTL | CJK)
* - `register` — one-line style-register reminder (guide § 4)
* - `fixedTokenNote` — one-line FIXED-TOKEN preservation trap (guide § 4,
* validator gate #5); the highest-frequency failure mode
* - `terms` — canonical EN→target EP terminology pairs (guide § 5)
*
* This is a deliberate, reviewed duplication of the prose guide: when § 4 / § 5
* change, update this constant in the same PR. The drift-guard test
* `discover-untranslated-briefs.test.js` asserts the shape and a few anchor
* values so the two cannot silently diverge on the fields that matter.
*
* @type {Readonly<Record<string, { name: string, family: string, register: string, fixedTokenNote: string, terms: Record<string, string> }>>}
*/
export const TRANSLATION_REGISTER = Object.freeze({
sv: {
name: 'Swedish',
family: 'Nordic',
register: 'Formal, impersonal (man / Europaparlamentet); avoid du-tilltal.',
fixedTokenNote:
"IMF/WEO verbatim — never IVF or 'Internationella valutafonden'. World Bank/Fiscal Monitor verbatim.",
terms: {
'European Parliament': 'Europaparlamentet',
'Plenary session': 'plenarsammanträde',
Committee: 'utskott',
Rapporteur: 'föredragande',
'Legislative procedure': 'lagstiftningsförfarande',
Resolution: 'resolution',
'Adopted text': 'antagen text',
Vote: 'omröstning',
},
},
da: {
name: 'Danish',
family: 'Nordic',
register: 'Formal, indicative mood; official EP designations.',
fixedTokenNote:
"IMF/WEO verbatim — never IMV or 'Den Internationale Valutafond'. World Bank/Fiscal Monitor verbatim.",
terms: {
'European Parliament': 'Europa-Parlamentet',
'Plenary session': 'plenarmøde',
Committee: 'udvalg',
Rapporteur: 'ordfører',
'Legislative procedure': 'lovgivningsprocedure',
Resolution: 'beslutning',
'Adopted text': 'vedtaget tekst',
Vote: 'afstemning',
},
},
no: {
name: 'Norwegian',
family: 'Nordic',
register: 'Formal, indicative mood; official EP designations.',
fixedTokenNote:
"IMF/WEO verbatim — never IPF/IMV/'Det internasjonale valutafondet'/Pengefondet/'Verdens økonomiske utsikter'. World Bank/Fiscal Monitor verbatim.",
terms: {
'European Parliament': 'Europaparlamentet',
'Plenary session': 'plenumsmøte',
Committee: 'komité',
Rapporteur: 'ordfører',
'Legislative procedure': 'lovgivningsprosedyre',
Resolution: 'resolusjon',
'Adopted text': 'vedtatt tekst',
Vote: 'avstemning',
},
},
fi: {
name: 'Finnish',
family: 'Nordic',
register: 'Formal; partitive + case agreement non-negotiable.',
fixedTokenNote:
"IMF/WEO verbatim — never KVR or 'Kansainvälinen valuuttarahasto'. World Bank/Fiscal Monitor verbatim.",
terms: {
'European Parliament': 'Euroopan parlamentti',
'Plenary session': 'täysistunto',
Committee: 'valiokunta',
Rapporteur: 'esittelijä',
'Legislative procedure': 'lainsäädäntömenettely',
Resolution: 'päätöslauselma',
'Adopted text': 'hyväksytty teksti',
Vote: 'äänestys',
},
},
de: {
name: 'German',
family: 'EU-core',
register:
"Formal (Sie); capitalise nouns; 'Europäisches Parlament' not 'EU-Parlament' in titles.",
fixedTokenNote:
'IMF/WEO/World Bank/Fiscal Monitor verbatim — never IWF.',
terms: {
'European Parliament': 'Europäisches Parlament',
'Plenary session': 'Plenarsitzung',
Committee: 'Ausschuss',
Rapporteur: 'Berichterstatter',
'Legislative procedure': 'Gesetzgebungsverfahren',
Resolution: 'Entschließung',
'Adopted text': 'angenommener Text',
Vote: 'Abstimmung',
},
},
fr: {
name: 'French',
family: 'EU-core',
register:
'Formal (vous); Académie française register; avoid Anglicisms unless source uses one as a fixed token.',
fixedTokenNote:
'IMF/WEO/World Bank/Fiscal Monitor verbatim — never FMI.',
terms: {
'European Parliament': 'Parlement européen',
'Plenary session': 'séance plénière',
Committee: 'commission',
Rapporteur: 'rapporteur',
'Legislative procedure': 'procédure législative',
Resolution: 'résolution',
'Adopted text': 'texte adopté',
Vote: 'vote',
},
},
es: {
name: 'Spanish',
family: 'EU-core',
register: 'Peninsular es-ES; formal; avoid Latin-American vocabulary.',
fixedTokenNote:
'IMF/WEO/World Bank/Fiscal Monitor verbatim — never FMI.',
terms: {
'European Parliament': 'Parlamento Europeo',
'Plenary session': 'sesión plenaria',
Committee: 'comisión',
Rapporteur: 'ponente',
'Legislative procedure': 'procedimiento legislativo',
Resolution: 'resolución',
'Adopted text': 'texto aprobado',
Vote: 'votación',
},
},
nl: {
name: 'Dutch',
family: 'EU-core',
register: 'Formal (u-form); prefer impersonal constructions.',
fixedTokenNote:
"IMF/WEO/World Bank/Fiscal Monitor verbatim — never IMV/'Internationaal Monetair Fonds'/Wereldbank.",
terms: {
'European Parliament': 'Europees Parlement',
'Plenary session': 'plenaire vergadering',
Committee: 'commissie',
Rapporteur: 'rapporteur',
'Legislative procedure': 'wetgevingsprocedure',
Resolution: 'resolutie',
'Adopted text': 'aangenomen tekst',
Vote: 'stemming',
},
},
ar: {
name: 'Arabic',
family: 'RTL',
register:
'Modern Standard Arabic, formal political register; Western Arabic numerals 0–9; no RLM unless disambiguating a Latin token.',
fixedTokenNote:
'Keep IMF/WEO/World Bank/Fiscal Monitor and EP acronyms in Latin script; data-vintage/TA-id/procedure-id verbatim.',
terms: {
'European Parliament': 'البرلمان الأوروبي',
'Plenary session': 'الجلسة العامة',
Committee: 'اللجنة',
Rapporteur: 'المقرر',
'Legislative procedure': 'الإجراء التشريعي',
Resolution: 'قرار',
Vote: 'تصويت',
},
},
he: {
name: 'Hebrew',
family: 'RTL',
register: 'Formal modern Hebrew; no nikud unless the source carries it.',
fixedTokenNote:
'Keep IMF/WEO/World Bank/Fiscal Monitor and EP acronyms in Latin script; data-vintage/TA-id/procedure-id verbatim.',
terms: {
'European Parliament': 'הפרלמנט האירופי',
'Plenary session': 'מליאה',
Committee: 'ועדה',
Rapporteur: 'מדווח',
'Legislative procedure': 'הליך חקיקה',
Resolution: 'החלטה',
Vote: 'הצבעה',
},
},
ja: {
name: 'Japanese',
family: 'CJK',
register: 'です・ます polite register; full-width punctuation 。、「」.',
fixedTokenNote:
'Keep proper nouns (IMF, WEO, World Bank, EP body names) in Latin script; data-vintage/TA-id/procedure-id verbatim.',
terms: {
'European Parliament': '欧州議会',
'Plenary session': '本会議',
Committee: '委員会',
Rapporteur: '報告者',
'Legislative procedure': '立法手続き',
Resolution: '決議',
Vote: '採決',
},
},
ko: {
name: 'Korean',
family: 'CJK',
register: '합쇼체 formal polite; half-width , . ; spaces between eojeol.',
fixedTokenNote:
'Keep proper nouns (IMF, WEO, World Bank, EP body names) in Latin script; data-vintage/TA-id/procedure-id verbatim.',
terms: {
'European Parliament': '유럽의회',
'Plenary session': '본회의',
Committee: '위원회',
Rapporteur: '보고자',
'Legislative procedure': '입법절차',
Resolution: '결의',
Vote: '표결',
},
},
zh: {
name: 'Chinese',
family: 'CJK',
register: 'Simplified zh-CN only; full-width punctuation 。，「」/“”.',
fixedTokenNote:
'Keep proper nouns (IMF, WEO, World Bank, EP body names) in Latin script; data-vintage/TA-id/procedure-id verbatim.',
terms: {
'European Parliament': '欧洲议会',
'Plenary session': '全体会议',
Committee: '委员会',
Rapporteur: '报告员',
'Legislative procedure': '立法程序',
Resolution: '决议',
Vote: '表决',
},
},
});
/**
* Build the per-language register block emitted into the discovery payload.
* Returns only the entries for `langs` (the languages the run actually
* touches), so the agent's in-context copy stays as small as possible.
* Unknown codes are skipped silently — the caller derives `langs` from the
* canonical TARGET_LANGS union, so this is defensive only.
*
* @param {Iterable<string>} langs
* @returns {Record<string, (typeof TRANSLATION_REGISTER)[string]>}
*/
export function buildTranslationRegister(langs) {
const out = {};
for (const lang of langs) {
if (Object.prototype.hasOwnProperty.call(TRANSLATION_REGISTER, lang)) {
out[lang] = TRANSLATION_REGISTER[lang];
}
}
return out;
}
/**
* Count the number of lines in a markdown source file. Returns 0 if the
* file is missing. Used by `buildQueue` to populate `sourceLineCount` and
* the derived `largeSource` flag — the translator agent uses these
* signals to choose between a one-shot `create` per language and a
* 2-phase skeleton-then-edit strategy (Phase A / Phase B / Phase C
* documented in `.github/workflows/news-translate.md` Step 2) when the
* source is too large to translate reliably in a single inference call.
*
* @param {string} absPath
* @returns {number}
*/
export function countLines(absPath) {
if (!fs.existsSync(absPath)) return 0;
const text = fs.readFileSync(absPath, 'utf8');
if (text.length === 0) return 0;
// Match `wc -l` semantics on POSIX: count newlines. A file with content
// but no trailing newline still has 1 logical line.
const newlines = (text.match(/\n/g) || []).length;
return text.endsWith('\n') ? newlines : newlines + 1;
}
/**
* Extract H2 section titles from a markdown source file. Returns the
* 1-based line number and the visible title (with the leading `## `
* stripped). The agent uses this to spot duplicate-titled sections such
* as `## IMF Economic Context` followed by
* `## IMF Economic Context — May 2026 Update`, which were silently
* collapsed across all 13 translations in run #25983007788. Surfacing the
* full title list at discovery time eliminates the ambiguity before any
* translation work begins.
*
* @param {string} absPath
* @returns {Array<{ line: number, title: string }>}
*/
export function extractH2Titles(absPath) {
if (!fs.existsSync(absPath)) return [];
const text = fs.readFileSync(absPath, 'utf8');
const lines = text.split('\n');
const out = [];
for (let i = 0; i < lines.length; i += 1) {
const match = /^##\s+(\S.*)$/.exec(lines[i]);
if (match) out.push({ line: i + 1, title: match[1].trim() });
}
return out;
}
/**
* Count occurrences of each FIXED_TOKEN class in the source brief. Only
* classes with at least one match are emitted, so the queue entry stays
* compact for short briefs.
*
* @param {string} absPath
* @returns {Record<string, number>}
*/
export function countFixedTokens(absPath) {
Iif (!fs.existsSync(absPath)) return {};
const text = fs.readFileSync(absPath, 'utf8');
const counts = {};
for (const { key, pattern } of FIXED_TOKEN_CLASSES) {
const re = new RegExp(pattern.source, pattern.flags);
let n = 0;
while (re.exec(text) !== null) n += 1;
if (n > 0) counts[key] = n;
}
return counts;
}
/**
* Build the prioritised queue. See module docstring for ordering rules.
*
* @param {ReturnType<typeof findExecutiveBriefSources>} sources
* @param {number | {
* maxBriefs: number,
* mode?: string,
* runNumber?: number,
* maxSourceLines?: number,
* targetBrief?: { date: string, slug: string, isExtended: boolean } | null,
* }} options
* Numeric form retained for backward compatibility — equivalent to
* `{ maxBriefs, mode: 'fresh-then-backlog', runNumber: 0, maxSourceLines: DEFAULT_MAX_SOURCE_LINES, targetBrief: null }`.
*/
export function buildQueue(sources, options) {
const opts =
typeof options === 'number'
? {
maxBriefs: options,
mode: 'fresh-then-backlog',
runNumber: 0,
maxSourceLines: DEFAULT_MAX_SOURCE_LINES,
targetBrief: null,
}
: {
maxBriefs: options.maxBriefs,
mode: options.mode || 'fresh-then-backlog',
runNumber: Number.isFinite(options.runNumber) ? options.runNumber : 0,
maxSourceLines: Number.isFinite(options.maxSourceLines)
? options.maxSourceLines
: DEFAULT_MAX_SOURCE_LINES,
targetBrief: options.targetBrief || null,
};
if (!DISCOVERY_MODES.includes(opts.mode)) {
throw new Error(
`buildQueue: invalid mode "${opts.mode}" (expected one of ${DISCOVERY_MODES.join(', ')})`,
);
}
const withGaps = [];
let totalMissing = 0;
const missingByLang = new Map();
for (const source of sources) {
const missing = findMissingLangs(source);
if (missing.length === 0) continue;
totalMissing += missing.length;
for (const lang of missing) {
missingByLang.set(lang, (missingByLang.get(lang) || 0) + 1);
}
// Pre-compute structural targets for the translator agent so it has
// explicit visibility into duplicate-titled H2 sections and the
// verbatim-preserve token budget BEFORE any translation is written.
// Surfacing these here (rather than relying on the agent to discover
// them) prevents the regression observed in run #25983007788, where
// 13 sibling translations of a single brief silently collapsed a
// `## IMF Economic Context — May 2026 Update` section because the
// agent treated it as a duplicate of `## IMF Economic Context`.
const sourceH2Titles = extractH2Titles(source.absPath);
const sourceFixedTokens = countFixedTokens(source.absPath);
const sourceLineCount = countLines(source.absPath);
const largeSource = sourceLineCount > opts.maxSourceLines;
withGaps.push({
date: source.date,
slug: source.slug,
sourcePath: source.relPath,
missingLangs: missing,
missingCount: missing.length,
isExtended: source.isExtended,
sourceH2Titles,
sourceH2Count: sourceH2Titles.length,
sourceFixedTokens,
sourceLineCount,
largeSource,
});
}
// Two canonical orderings drive the three modes. Within the same date,
// small briefs sort BEFORE large ones (`largeSource: false` first) — a
// large brief in the fresh slot triggered the unrecoverable transient-
// API-error loop in run #26181499722. The translator agent can still
// handle large briefs via the 2-phase skeleton-then-edit strategy
// (step 4-large in news-translate.md), but exposing a small candidate
// to the fresh slot first keeps the common case fast.
const newestFirst = (a, b) => {
if (a.date !== b.date) return a.date < b.date ? 1 : -1;
if (a.largeSource !== b.largeSource) return a.largeSource ? 1 : -1;
if (a.missingCount !== b.missingCount) return b.missingCount - a.missingCount;
if (a.slug !== b.slug) return a.slug < b.slug ? -1 : 1;
Eif (a.isExtended !== b.isExtended) return a.isExtended ? 1 : -1;
return 0;
};
const oldestFirstFinishPartial = (a, b) => {
if (a.date !== b.date) return a.date < b.date ? -1 : 1;
Iif (a.largeSource !== b.largeSource) return a.largeSource ? 1 : -1;
// Within the same date, finish briefs that are closer to completion
// first (fewer missing languages → ascending).
if (a.missingCount !== b.missingCount) return a.missingCount - b.missingCount;
if (a.slug !== b.slug) return a.slug < b.slug ? -1 : 1;
Eif (a.isExtended !== b.isExtended) return a.isExtended ? 1 : -1;
return 0;
};
let queue;
if (opts.targetBrief) {
// Operator override: ignore mode / maxBriefs / parity and queue exactly
// the one brief the operator asked for, IF it has any missing languages.
// If the targeted brief is fully translated (no gaps), the queue is
// empty — the workflow's downstream validator handles the empty-queue
// case gracefully (skip with no work to do).
const tb = opts.targetBrief;
queue = withGaps.filter(
(entry) =>
entry.date === tb.date &&
entry.slug === tb.slug &&
entry.isExtended === tb.isExtended,
);
} else if (opts.mode === 'newest-first') {
queue = [...withGaps].sort(newestFirst).slice(0, opts.maxBriefs);
} else if (opts.mode === 'backlog-only') {
queue = [...withGaps].sort(oldestFirstFinishPartial).slice(0, opts.maxBriefs);
} else {
// fresh-then-backlog
const newestSorted = [...withGaps].sort(newestFirst);
const oldestSorted = [...withGaps].sort(oldestFirstFinishPartial);
if (opts.maxBriefs === 1) {
// Alternate fresh/backlog by run-number parity so the scheduled
// cadence still drains backlog while preserving freshness on every
// other slot. Even run-numbers (0, 2, ...) take the fresh slot;
// odd run-numbers take the oldest backlog slot.
const pool = opts.runNumber % 2 === 0 ? newestSorted : oldestSorted;
queue = pool.slice(0, 1);
} else {
const freshSlice = newestSorted.slice(0, 1);
const freshKey = freshSlice[0]
? `${freshSlice[0].date}\u0000${freshSlice[0].slug}\u0000${freshSlice[0].isExtended}`
: null;
const backlogSlice = oldestSorted
.filter(
(entry) => `${entry.date}\u0000${entry.slug}\u0000${entry.isExtended}` !== freshKey,
)
.slice(0, Math.max(0, opts.maxBriefs - 1));
queue = [...freshSlice, ...backlogSlice];
}
}
const queuedTranslations = queue.reduce((sum, item) => sum + item.missingCount, 0);
// Per-language register the run actually needs: the union of every queued
// brief's missing languages, in canonical TARGET_LANGS order. Emitting this
// once (rather than per-entry) lets the translator agent read the style
// register + EP terminology pairs inline from queue.json and skip opening
// the ~18 KB translator guide in-session — bounding the per-session
// effective-token accumulation (see news-translate.md frontmatter budget).
const queuedLangs = new Set();
for (const item of queue) {
for (const lang of item.missingLangs) queuedLangs.add(lang);
}
const translationRegister = buildTranslationRegister(
TARGET_LANGS.filter((lang) => queuedLangs.has(lang)),
);
// Top 3 most-blocked target languages across the entire backlog. Operators
// skim this to spot e.g. "Japanese keeps falling behind" without parsing
// the full queue. Sort by count desc, then language code asc for stable
// tie-breaking.
const topMissingLangs = [...missingByLang.entries()]
.sort((a, b) => (b[1] - a[1]) || (a[0] < b[0] ? -1 : 1))
.slice(0, 3)
.map(([lang, count]) => ({ lang, count }));
// Operator-visibility extents: newest source still carrying gaps (the
// candidate for the fresh slot) and oldest source still carrying gaps
// (the candidate for the backlog slot). Both fall back to null when the
// backlog is empty.
let freshNewestDate = null;
let backlogOldestDate = null;
let largeSourceCount = 0;
for (const entry of withGaps) {
if (freshNewestDate === null || entry.date > freshNewestDate) freshNewestDate = entry.date;
if (backlogOldestDate === null || entry.date < backlogOldestDate) backlogOldestDate = entry.date;
if (entry.largeSource) largeSourceCount += 1;
}
return {
totals: {
sourcesScanned: sources.length,
sourcesWithGaps: withGaps.length,
translationsMissing: totalMissing,
queued: queue.length,
queuedTranslations,
freshNewestDate,
backlogOldestDate,
topMissingLangs,
largeSourceCount,
maxSourceLines: opts.maxSourceLines,
},
translationRegister,
queue,
};
}
/**
* Main entry point.
* @param {string[]} argv
*/
export function main(argv) {
const opts = parseArgs(argv);
const sources = findExecutiveBriefSources(opts.repoRoot, {
includeExtended: opts.includeExtended,
maxAgeDays: opts.maxAgeDays,
});
const { totals, translationRegister, queue } = buildQueue(sources, {
maxBriefs: opts.maxBriefs,
mode: opts.mode,
runNumber: opts.runNumber,
maxSourceLines: opts.maxSourceLines,
targetBrief: opts.targetBrief,
});
const payload = {
generatedAt: new Date().toISOString(),
options: {
maxBriefs: opts.maxBriefs,
maxAgeDays: opts.maxAgeDays,
includeExtended: opts.includeExtended,
mode: opts.mode,
runNumber: opts.runNumber,
maxSourceLines: opts.maxSourceLines,
targetBrief: opts.targetBrief,
},
totals,
translationRegister,
queue,
};
const out = `${JSON.stringify(payload, null, 2)}\n`;
if (opts.output) {
fs.mkdirSync(path.dirname(opts.output), { recursive: true });
fs.writeFileSync(opts.output, out);
} else E{
process.stdout.write(out);
}
return payload;
}
/* c8 ignore start */
if (import.meta.url === `file://${process.argv[1]}`) {
try {
main(process.argv.slice(2));
} catch (err) {
process.stderr.write(`discover-untranslated-briefs: ${err.message}\n`);
process.exit(1);
}
}
/* c8 ignore stop */
|