重构主控编排与运行时预热链路,统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。

同时收敛启动与运维脚本默认行为(含 wiki worker)、更新 Admin 可观测性与相关测试,降低首轮时延并提高运行稳定性。

Made-with: Cursor
This commit is contained in:
oliver 2026-04-26 08:34:33 +08:00
parent 4a23b715a2
commit dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions

View file

@ -0,0 +1,278 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
type behaviorReplacePair struct {
From string `json:"from"`
To string `json:"to"`
}
type behaviorRule struct {
Method string `json:"method"`
MatchAll []string `json:"match_all"`
ResponseFile string `json:"response_file,omitempty"`
ReplacePairs []behaviorReplacePair `json:"replace_pairs,omitempty"`
}
type behaviorFixture struct {
Name string `json:"name"`
Mode string `json:"mode"`
RelPath string `json:"rel_path"`
SourceFile string `json:"source_file"`
ExpectedFile string `json:"expected_file,omitempty"`
ExpectedErrorContains string `json:"expected_error_contains,omitempty"`
ExpectedOutputContains []string `json:"expected_output_contains,omitempty"`
ExpectedOutputNotContains []string `json:"expected_output_not_contains,omitempty"`
Rules []behaviorRule `json:"rules"`
}
type behaviorFixtureTranslator struct {
t *testing.T
dir string
rules []behaviorRule
}
func (tr *behaviorFixtureTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return tr.run("masked", text), nil
}
func (tr *behaviorFixtureTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
return tr.run("raw", text), nil
}
func (tr *behaviorFixtureTranslator) Close() {}
func (tr *behaviorFixtureTranslator) run(method, text string) string {
tr.t.Helper()
for _, rule := range tr.rules {
if rule.Method != method {
continue
}
if !matchesAll(text, rule.MatchAll) {
continue
}
switch {
case rule.ResponseFile != "":
return readFixtureTextInDir(tr.t, tr.dir, rule.ResponseFile)
case len(rule.ReplacePairs) > 0:
out := text
for _, pair := range rule.ReplacePairs {
out = strings.ReplaceAll(out, pair.From, pair.To)
}
return out
default:
return text
}
}
return text
}
func matchesAll(text string, fragments []string) bool {
for _, fragment := range fragments {
if !strings.Contains(text, fragment) {
return false
}
}
return true
}
func TestDocsI18nBehaviorBaselines(t *testing.T) {
t.Parallel()
root := filepath.Join("testdata", "behavior")
entries, err := os.ReadDir(root)
if err != nil {
t.Fatalf("ReadDir(%q): %v", root, err)
}
found := false
for _, entry := range entries {
if !entry.IsDir() {
continue
}
found = true
dir := filepath.Join(root, entry.Name())
fixture := loadBehaviorFixture(t, dir)
name := fixture.Name
if name == "" {
name = entry.Name()
}
t.Run(name, func(t *testing.T) {
t.Parallel()
runBehaviorFixture(t, dir, fixture)
})
}
if !found {
t.Fatalf("no behavior fixtures found under %s", root)
}
}
func loadBehaviorFixture(t *testing.T, dir string) behaviorFixture {
t.Helper()
data, err := os.ReadFile(filepath.Join(dir, "case.json"))
if err != nil {
t.Fatalf("ReadFile(case.json): %v", err)
}
var fixture behaviorFixture
if err := json.Unmarshal(data, &fixture); err != nil {
t.Fatalf("Unmarshal(case.json): %v", err)
}
return fixture
}
func runBehaviorFixture(t *testing.T, dir string, fixture behaviorFixture) {
t.Helper()
source := readFixtureTextInDir(t, dir, fixture.SourceFile)
translator := &behaviorFixtureTranslator{
t: t,
dir: dir,
rules: fixture.Rules,
}
var (
got string
err error
)
switch fixture.Mode {
case "doc_body_chunked":
got, err = translateDocBodyChunked(context.Background(), translator, fixture.RelPath, source, "en", "zh-CN")
case "frontmatter_scalar":
got, err = translateSnippet(
context.Background(),
translator,
&TranslationMemory{entries: map[string]TMEntry{}},
fixture.RelPath+":frontmatter:title",
source,
"en",
"zh-CN",
)
default:
t.Fatalf("unsupported fixture mode %q", fixture.Mode)
}
if fixture.ExpectedErrorContains != "" {
if err == nil {
t.Fatalf("expected error containing %q, got nil", fixture.ExpectedErrorContains)
}
if !strings.Contains(err.Error(), fixture.ExpectedErrorContains) {
t.Fatalf("expected error containing %q, got %v", fixture.ExpectedErrorContains, err)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if fixture.ExpectedFile != "" {
want := readFixtureTextInDir(t, dir, fixture.ExpectedFile)
if normalizeBehaviorText(got) != normalizeBehaviorText(want) {
t.Fatalf("unexpected output\nwant:\n%s\n\ngot:\n%s", want, got)
}
}
for _, fragment := range fixture.ExpectedOutputContains {
if !strings.Contains(got, fragment) {
t.Fatalf("expected output to contain %q\noutput:\n%s", fragment, got)
}
}
for _, fragment := range fixture.ExpectedOutputNotContains {
if strings.Contains(got, fragment) {
t.Fatalf("expected output to exclude %q\noutput:\n%s", fragment, got)
}
}
}
func readFixtureText(t *testing.T, path string) string {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%q): %v", path, err)
}
return string(data)
}
func readFixtureTextInDir(t *testing.T, dir, name string) string {
t.Helper()
resolvedPath, err := resolveFixturePathInDir(dir, name)
if err != nil {
t.Fatal(err)
}
return readFixtureText(t, resolvedPath)
}
func resolveFixturePathInDir(dir, name string) (string, error) {
if filepath.IsAbs(name) {
return "", fmt.Errorf("absolute fixture paths are not allowed: %q", name)
}
clean := filepath.Clean(name)
if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("fixture path escapes dir: %q", name)
}
joined := filepath.Join(dir, clean)
resolvedDir, err := filepath.EvalSymlinks(dir)
if err != nil {
return "", fmt.Errorf("EvalSymlinks(%q): %w", dir, err)
}
resolvedPath, err := filepath.EvalSymlinks(joined)
if err != nil {
return "", fmt.Errorf("EvalSymlinks(%q): %w", joined, err)
}
rel, err := filepath.Rel(resolvedDir, resolvedPath)
if err != nil {
return "", fmt.Errorf("Rel(%q, %q): %w", resolvedDir, resolvedPath, err)
}
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("fixture path resolves outside dir: %q", name)
}
return resolvedPath, nil
}
func normalizeBehaviorText(value string) string {
return strings.TrimSpace(strings.ReplaceAll(value, "\r\n", "\n"))
}
func TestResolveFixturePathInDirRejectsSymlinkEscape(t *testing.T) {
t.Parallel()
root := t.TempDir()
fixtureDir := filepath.Join(root, "fixture")
if err := os.MkdirAll(fixtureDir, 0o755); err != nil {
t.Fatalf("MkdirAll(%q): %v", fixtureDir, err)
}
outsidePath := filepath.Join(root, "outside.txt")
if err := os.WriteFile(outsidePath, []byte("outside\n"), 0o644); err != nil {
t.Fatalf("WriteFile(%q): %v", outsidePath, err)
}
linkPath := filepath.Join(fixtureDir, "outside-link.txt")
if err := os.Symlink(outsidePath, linkPath); err != nil {
if os.IsPermission(err) || runtime.GOOS == "windows" {
t.Skipf("symlink creation unavailable in this test environment: %v", err)
}
t.Fatalf("Symlink(%q, %q): %v", outsidePath, linkPath, err)
}
_, err := resolveFixturePathInDir(fixtureDir, "outside-link.txt")
if err == nil {
t.Fatal("expected symlink escape to fail")
}
if !strings.Contains(err.Error(), "resolves outside dir") {
t.Fatalf("expected outside-dir error, got %v", err)
}
}

View file

@ -0,0 +1,830 @@
package main
import (
"context"
"fmt"
"log"
"os"
"regexp"
"slices"
"strconv"
"strings"
)
const defaultDocChunkMaxBytes = 12000
const defaultDocChunkPromptBudget = 15000
var (
docsFenceRE = regexp.MustCompile(`^\s*(` + "```" + `|~~~)`)
docsComponentTagRE = regexp.MustCompile(`<(/?)([A-Z][A-Za-z0-9]*)\b[^>]*?/?>`)
)
var docsProtocolTokens = []string{
frontmatterTagStart,
frontmatterTagEnd,
bodyTagStart,
bodyTagEnd,
"[[[FM_",
}
type docChunkStructure struct {
fenceCount int
tagCounts map[string]int
}
type docChunkSplitPlan struct {
groups [][]string
reason string
}
func translateDocBodyChunked(ctx context.Context, translator docsTranslator, relPath, body, srcLang, tgtLang string) (string, error) {
if strings.TrimSpace(body) == "" {
return body, nil
}
blocks := splitDocBodyIntoBlocks(body)
groups := groupDocBlocks(blocks, docsI18nDocChunkMaxBytes())
logDocChunkPlan(relPath, blocks, groups)
out := strings.Builder{}
for index, group := range groups {
chunkID := fmt.Sprintf("%s.chunk-%03d", relPath, index+1)
translated, err := translateDocBlockGroup(ctx, translator, chunkID, group, srcLang, tgtLang)
if err != nil {
return "", err
}
out.WriteString(translated)
}
return out.String(), nil
}
func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, srcLang, tgtLang string) (string, error) {
source := strings.Join(blocks, "")
if strings.TrimSpace(source) == "" {
return source, nil
}
if plan, ok := planDocChunkSplit(blocks, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
logDocChunkPlanSplit(chunkID, plan, source)
return translatePlannedDocChunkGroups(ctx, translator, chunkID, plan.groups, srcLang, tgtLang)
}
normalizedSource, commonIndent := stripCommonIndent(source)
log.Printf("docs-i18n: chunk start %s blocks=%d bytes=%d", chunkID, len(blocks), len(source))
translated, err := translator.TranslateRaw(ctx, normalizedSource, srcLang, tgtLang)
if err == nil {
translated = sanitizeDocChunkProtocolWrappers(source, translated)
translated = reapplyCommonIndent(translated, commonIndent)
if validationErr := validateDocChunkTranslation(source, translated); validationErr == nil {
log.Printf("docs-i18n: chunk done %s out_bytes=%d", chunkID, len(translated))
return translated, nil
} else {
err = validationErr
}
}
if len(blocks) <= 1 {
if fallback, fallbackErr := translateDocLeafBlock(ctx, translator, chunkID, source, srcLang, tgtLang); fallbackErr == nil {
return fallback, nil
}
if plan, ok := planSingletonDocChunkRetry(source, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
logDocChunkPlanSplit(chunkID, plan, source)
return translatePlannedDocChunkGroups(ctx, translator, chunkID, plan.groups, srcLang, tgtLang)
}
return "", fmt.Errorf("%s: %w", chunkID, err)
}
if plan, ok := planDocChunkSplit(blocks, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
logDocChunkSplit(chunkID, len(blocks), err)
return translatePlannedDocChunkGroups(ctx, translator, chunkID, plan.groups, srcLang, tgtLang)
}
if plan, ok := splitDocChunkBlocksMidpointSimple(blocks); ok {
logDocChunkSplit(chunkID, len(blocks), err)
return translatePlannedDocChunkGroups(ctx, translator, chunkID, plan.groups, srcLang, tgtLang)
}
return "", fmt.Errorf("%s: %w", chunkID, err)
}
func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source, srcLang, tgtLang string) (string, error) {
sourceStructure := summarizeDocChunkStructure(source)
if sourceStructure.fenceCount != 0 {
return "", fmt.Errorf("%s: raw leaf fallback not applicable", chunkID)
}
normalizedSource, commonIndent := stripCommonIndent(source)
maskedSource, placeholders := maskDocComponentTags(normalizedSource)
translated, err := translator.Translate(ctx, maskedSource, srcLang, tgtLang)
if err != nil {
return "", err
}
translated, err = restoreDocComponentTags(translated, placeholders)
if err != nil {
return "", err
}
translated = sanitizeDocChunkProtocolWrappers(source, translated)
translated = reapplyCommonIndent(translated, commonIndent)
if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil {
return "", validationErr
}
log.Printf("docs-i18n: chunk leaf-fallback done %s out_bytes=%d", chunkID, len(translated))
return translated, nil
}
func splitDocBodyIntoBlocks(body string) []string {
if body == "" {
return nil
}
lines := strings.SplitAfter(body, "\n")
blocks := make([]string, 0, len(lines))
var current strings.Builder
fenceDelimiter := ""
for _, line := range lines {
current.WriteString(line)
fenceDelimiter, _ = updateFenceDelimiter(fenceDelimiter, line)
inFence := fenceDelimiter != ""
if !inFence && strings.TrimSpace(line) == "" {
blocks = append(blocks, current.String())
current.Reset()
}
}
if current.Len() > 0 {
blocks = append(blocks, current.String())
}
if len(blocks) == 0 {
return []string{body}
}
return blocks
}
func groupDocBlocks(blocks []string, maxBytes int) [][]string {
if len(blocks) == 0 {
return nil
}
if maxBytes <= 0 {
maxBytes = defaultDocChunkMaxBytes
}
groups := make([][]string, 0, len(blocks))
current := make([]string, 0, 8)
currentBytes := 0
flush := func() {
if len(current) == 0 {
return
}
groups = append(groups, current)
current = make([]string, 0, 8)
currentBytes = 0
}
for _, block := range blocks {
blockBytes := len(block)
if len(current) > 0 && currentBytes+blockBytes > maxBytes {
flush()
}
if blockBytes > maxBytes {
groups = append(groups, []string{block})
continue
}
current = append(current, block)
currentBytes += blockBytes
}
flush()
return groups
}
func validateDocChunkTranslation(source, translated string) error {
if hasUnexpectedTopLevelProtocolWrapper(source, translated) {
return fmt.Errorf("protocol token leaked: top-level wrapper")
}
sourceLower := strings.ToLower(source)
translatedLower := strings.ToLower(translated)
for _, token := range docsProtocolTokens {
tokenLower := strings.ToLower(token)
if strings.Contains(sourceLower, tokenLower) {
continue
}
if strings.Contains(translatedLower, tokenLower) {
return fmt.Errorf("protocol token leaked: %s", token)
}
}
sourceStructure := summarizeDocChunkStructure(source)
translatedStructure := summarizeDocChunkStructure(translated)
if sourceStructure.fenceCount != translatedStructure.fenceCount {
return fmt.Errorf("code fence mismatch: source=%d translated=%d", sourceStructure.fenceCount, translatedStructure.fenceCount)
}
if !slices.Equal(sortedKeys(sourceStructure.tagCounts), sortedKeys(translatedStructure.tagCounts)) {
return fmt.Errorf("component tag set mismatch")
}
for _, key := range sortedKeys(sourceStructure.tagCounts) {
if sourceStructure.tagCounts[key] != translatedStructure.tagCounts[key] {
return fmt.Errorf("component tag mismatch for %s: source=%d translated=%d", key, sourceStructure.tagCounts[key], translatedStructure.tagCounts[key])
}
}
return nil
}
func sanitizeDocChunkProtocolWrappers(source, translated string) string {
if !containsProtocolWrapperToken(translated) {
return translated
}
trimmedTranslated := strings.TrimSpace(translated)
if !hasUnexpectedTopLevelProtocolWrapper(source, trimmedTranslated) {
return translated
}
if !hasAmbiguousTaggedBodyClose(source, trimmedTranslated) {
_, body, err := parseTaggedDocument(trimmedTranslated)
if err == nil {
if strings.TrimSpace(body) == "" {
return translated
}
return body
}
}
body, ok := stripBodyOnlyWrapper(source, trimmedTranslated)
if !ok || strings.TrimSpace(body) == "" {
return translated
}
return body
}
func stripBodyOnlyWrapper(source, text string) (string, bool) {
sourceLower := strings.ToLower(source)
// When the source itself documents <body> tokens, a bare body-only payload is
// ambiguous: the trailing </body> can be literal translated content instead of
// a real wrapper close. Keep it for validation/retry instead of truncating.
if strings.Contains(sourceLower, strings.ToLower(bodyTagStart)) || strings.Contains(sourceLower, strings.ToLower(bodyTagEnd)) {
return "", false
}
lower := strings.ToLower(text)
bodyStartLower := strings.ToLower(bodyTagStart)
bodyEndLower := strings.ToLower(bodyTagEnd)
if !strings.HasPrefix(lower, bodyStartLower) || !strings.HasSuffix(lower, bodyEndLower) {
return "", false
}
body := text[len(bodyTagStart) : len(text)-len(bodyTagEnd)]
bodyLower := lower[len(bodyTagStart) : len(lower)-len(bodyTagEnd)]
if strings.Contains(bodyLower, bodyStartLower) || strings.Contains(bodyLower, bodyEndLower) {
return "", false
}
return trimTagNewlines(body), true
}
func hasAmbiguousTaggedBodyClose(source, translated string) bool {
sourceLower := strings.ToLower(source)
if !strings.Contains(sourceLower, strings.ToLower(bodyTagStart)) && !strings.Contains(sourceLower, strings.ToLower(bodyTagEnd)) {
return false
}
translatedLower := strings.ToLower(translated)
if !strings.Contains(translatedLower, strings.ToLower(frontmatterTagStart)) {
return false
}
return strings.Count(translatedLower, strings.ToLower(bodyTagEnd)) == 1
}
func maskDocComponentTags(text string) (string, []string) {
placeholders := make([]string, 0, 4)
masked := docsComponentTagRE.ReplaceAllStringFunc(text, func(match string) string {
placeholder := fmt.Sprintf("__OC_DOC_TAG_%03d__", len(placeholders))
placeholders = append(placeholders, match)
return placeholder
})
return masked, placeholders
}
func restoreDocComponentTags(text string, placeholders []string) (string, error) {
restored := text
for index, original := range placeholders {
placeholder := fmt.Sprintf("__OC_DOC_TAG_%03d__", index)
if !strings.Contains(restored, placeholder) {
return "", fmt.Errorf("component tag placeholder missing: %s", placeholder)
}
restored = strings.ReplaceAll(restored, placeholder, original)
}
return restored, nil
}
func logDocChunkSplit(chunkID string, blockCount int, err error) {
if docsI18nVerboseLogs() || blockCount >= 16 {
log.Printf("docs-i18n: chunk split %s blocks=%d err=%v", chunkID, blockCount, err)
}
}
func logDocChunkPlanSplit(chunkID string, plan docChunkSplitPlan, source string) {
if plan.reason == "" {
plan.reason = "unknown"
}
log.Printf("docs-i18n: chunk pre-split %s reason=%s groups=%d bytes=%d", chunkID, plan.reason, len(plan.groups), len(source))
}
func summarizeDocChunkStructure(text string) docChunkStructure {
counts := map[string]int{}
lines := strings.Split(text, "\n")
fenceDelimiter := ""
for _, line := range lines {
var toggled bool
fenceDelimiter, toggled = updateFenceDelimiter(fenceDelimiter, line)
if toggled {
counts["__fence_toggle__"]++
}
for _, match := range docsComponentTagRE.FindAllStringSubmatch(line, -1) {
if len(match) < 3 {
continue
}
fullToken := match[0]
tagName := match[2]
direction := "open"
if match[1] == "/" {
direction = "close"
}
if strings.HasSuffix(fullToken, "/>") {
direction = "self"
}
counts[tagName+":"+direction]++
}
}
return docChunkStructure{
fenceCount: counts["__fence_toggle__"],
tagCounts: countsWithoutFence(counts),
}
}
func countsWithoutFence(counts map[string]int) map[string]int {
filtered := map[string]int{}
for key, value := range counts {
if key == "__fence_toggle__" {
continue
}
filtered[key] = value
}
return filtered
}
func sortedKeys(counts map[string]int) []string {
keys := make([]string, 0, len(counts))
for key := range counts {
keys = append(keys, key)
}
slices.Sort(keys)
return keys
}
func updateFenceDelimiter(current, line string) (string, bool) {
delimiter := leadingFenceDelimiter(line)
if delimiter == "" {
return current, false
}
if current == "" {
return delimiter, true
}
if delimiter[0] == current[0] && len(delimiter) >= len(current) && isClosingFenceLine(line, delimiter) {
return "", true
}
return current, false
}
func leadingFenceDelimiter(line string) string {
trimmed := strings.TrimLeft(line, " \t")
if len(trimmed) < 3 {
return ""
}
switch trimmed[0] {
case '`', '~':
default:
return ""
}
marker := trimmed[0]
index := 0
for index < len(trimmed) && trimmed[index] == marker {
index++
}
if index < 3 {
return ""
}
return trimmed[:index]
}
func isClosingFenceLine(line, delimiter string) bool {
trimmed := strings.TrimLeft(line, " \t")
if !strings.HasPrefix(trimmed, delimiter) {
return false
}
return strings.TrimSpace(trimmed[len(delimiter):]) == ""
}
func hasUnexpectedTopLevelProtocolWrapper(source, translated string) bool {
sourceTrimmed := strings.ToLower(strings.TrimSpace(source))
translatedTrimmed := strings.ToLower(strings.TrimSpace(translated))
checks := []struct {
token string
match func(string) bool
}{
{token: frontmatterTagStart, match: func(text string) bool { return strings.HasPrefix(text, strings.ToLower(frontmatterTagStart)) }},
{token: bodyTagStart, match: func(text string) bool { return strings.HasPrefix(text, strings.ToLower(bodyTagStart)) }},
{token: frontmatterTagEnd, match: func(text string) bool { return strings.HasSuffix(text, strings.ToLower(frontmatterTagEnd)) }},
{token: bodyTagEnd, match: func(text string) bool { return strings.HasSuffix(text, strings.ToLower(bodyTagEnd)) }},
}
for _, check := range checks {
if check.match(translatedTrimmed) && !check.match(sourceTrimmed) {
return true
}
}
return false
}
func containsProtocolWrapperToken(text string) bool {
lower := strings.ToLower(text)
return strings.Contains(lower, strings.ToLower(bodyTagStart)) || strings.Contains(lower, strings.ToLower(frontmatterTagStart))
}
func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID string, groups [][]string, srcLang, tgtLang string) (string, error) {
var out strings.Builder
for index, group := range groups {
translated, err := translateDocBlockGroup(ctx, translator, fmt.Sprintf("%s.%02d", chunkID, index+1), group, srcLang, tgtLang)
if err != nil {
return "", err
}
out.WriteString(translated)
}
return out.String(), nil
}
func planDocChunkSplit(blocks []string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) {
if len(blocks) == 0 {
return docChunkSplitPlan{}, false
}
source := strings.Join(blocks, "")
if strings.TrimSpace(source) == "" {
return docChunkSplitPlan{}, false
}
normalizedSource, _ := stripCommonIndent(source)
estimatedPromptCost := estimateDocPromptCost(normalizedSource)
if len(blocks) > 1 && promptBudget > 0 && estimatedPromptCost > promptBudget {
return splitDocChunkBlocksMidpoint(blocks, estimatedPromptCost, promptBudget)
}
if len(blocks) == 1 {
return planSingletonDocChunk(blocks[0], maxBytes, promptBudget)
}
return docChunkSplitPlan{}, false
}
func splitDocChunkBlocksMidpoint(blocks []string, estimatedPromptCost, promptBudget int) (docChunkSplitPlan, bool) {
if len(blocks) <= 1 {
return docChunkSplitPlan{}, false
}
mid := len(blocks) / 2
if mid <= 0 || mid >= len(blocks) {
return docChunkSplitPlan{}, false
}
return docChunkSplitPlan{
groups: [][]string{blocks[:mid], blocks[mid:]},
reason: fmt.Sprintf("prompt-budget:%d>%d", estimatedPromptCost, promptBudget),
}, true
}
func splitDocChunkBlocksMidpointSimple(blocks []string) (docChunkSplitPlan, bool) {
if len(blocks) <= 1 {
return docChunkSplitPlan{}, false
}
mid := len(blocks) / 2
if mid <= 0 || mid >= len(blocks) {
return docChunkSplitPlan{}, false
}
return docChunkSplitPlan{
groups: [][]string{blocks[:mid], blocks[mid:]},
reason: "retry-midpoint",
}, true
}
func planSingletonDocChunk(block string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) {
normalizedBlock, _ := stripCommonIndent(block)
estimatedPromptCost := estimateDocPromptCost(normalizedBlock)
overBytes := maxBytes > 0 && len(block) > maxBytes
overPrompt := promptBudget > 0 && estimatedPromptCost > promptBudget
if !overBytes && !overPrompt {
return docChunkSplitPlan{}, false
}
return planSingletonDocChunkWithMode(block, maxBytes, promptBudget, false)
}
func planSingletonDocChunkRetry(block string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) {
return planSingletonDocChunkWithMode(block, maxBytes, promptBudget, true)
}
func planSingletonDocChunkWithMode(block string, maxBytes, promptBudget int, force bool) (docChunkSplitPlan, bool) {
if sections := splitDocBlockSections(block); len(sections) > 1 {
if groups := wrapDocChunkSections(sections); len(groups) > 1 {
reason := "singleton-structural"
if force {
reason = "singleton-retry-structural"
}
return docChunkSplitPlan{
groups: groups,
reason: reason,
}, true
}
}
if groups, ok := splitPureFencedDocSectionWithMode(block, maxBytes, promptBudget, force); ok {
reason := "singleton-fence"
if force {
reason = "singleton-retry-fence"
}
return docChunkSplitPlan{
groups: groups,
reason: reason,
}, true
}
if groups, ok := splitPlainDocSectionWithMode(block, maxBytes, promptBudget, force); ok {
reason := "singleton-lines"
if force {
reason = "singleton-retry-lines"
}
return docChunkSplitPlan{
groups: groups,
reason: reason,
}, true
}
return docChunkSplitPlan{}, false
}
func wrapDocChunkSections(sections []string) [][]string {
groups := make([][]string, 0, len(sections))
for _, section := range sections {
if strings.TrimSpace(section) == "" {
continue
}
groups = append(groups, []string{section})
}
return groups
}
func splitDocBlockSections(block string) []string {
lines := strings.SplitAfter(block, "\n")
if len(lines) == 0 {
return nil
}
sections := make([]string, 0, len(lines))
var current strings.Builder
fenceDelimiter := ""
for _, line := range lines {
lineDelimiter := leadingFenceDelimiter(line)
if fenceDelimiter == "" && lineDelimiter != "" {
if current.Len() > 0 {
sections = append(sections, current.String())
current.Reset()
}
current.WriteString(line)
fenceDelimiter = lineDelimiter
continue
}
current.WriteString(line)
if fenceDelimiter != "" {
if lineDelimiter != "" && lineDelimiter[0] == fenceDelimiter[0] && len(lineDelimiter) >= len(fenceDelimiter) && isClosingFenceLine(line, fenceDelimiter) {
sections = append(sections, current.String())
current.Reset()
fenceDelimiter = ""
}
continue
}
if strings.TrimSpace(line) == "" {
sections = append(sections, current.String())
current.Reset()
}
}
if current.Len() > 0 {
sections = append(sections, current.String())
}
if len(sections) <= 1 {
return nil
}
return sections
}
func splitPureFencedDocSection(block string, maxBytes, promptBudget int) ([][]string, bool) {
return splitPureFencedDocSectionWithMode(block, maxBytes, promptBudget, false)
}
func splitPureFencedDocSectionWithMode(block string, maxBytes, promptBudget int, force bool) ([][]string, bool) {
lines := strings.SplitAfter(block, "\n")
if len(lines) < 2 {
return nil, false
}
openingIndex := firstNonEmptyLineIndex(lines)
closingIndex := lastNonEmptyLineIndex(lines)
if openingIndex == -1 || closingIndex <= openingIndex {
return nil, false
}
opening := lines[openingIndex]
delimiter := leadingFenceDelimiter(opening)
if delimiter == "" || !isClosingFenceLine(lines[closingIndex], delimiter) {
return nil, false
}
prefix := strings.Join(lines[:openingIndex], "")
suffix := strings.Join(lines[closingIndex+1:], "")
if strings.TrimSpace(prefix) != "" || strings.TrimSpace(suffix) != "" {
return nil, false
}
closing := lines[closingIndex]
inner := strings.Join(lines[openingIndex+1:closingIndex], "")
groups, ok := splitPlainDocSectionWithMode(inner, maxBytes-len(opening)-len(closing), promptBudget, force)
if !ok {
return nil, false
}
for index, group := range groups {
joined := strings.Join(group, "")
groups[index] = []string{opening + joined + closing}
}
return groups, true
}
func splitPlainDocSection(text string, maxBytes, promptBudget int) ([][]string, bool) {
return splitPlainDocSectionWithMode(text, maxBytes, promptBudget, false)
}
func splitPlainDocSectionWithMode(text string, maxBytes, promptBudget int, force bool) ([][]string, bool) {
if maxBytes <= 0 {
maxBytes = len(text)
}
if promptBudget <= 0 {
promptBudget = defaultDocChunkPromptBudget
}
lines := strings.SplitAfter(text, "\n")
if len(lines) <= 1 {
return nil, false
}
groups := make([][]string, 0, len(lines))
var current strings.Builder
currentBytes := 0
currentPrompt := 0
for _, line := range lines {
linePrompt := estimateDocPromptCost(line)
if len(line) > maxBytes || linePrompt > promptBudget {
return nil, false
}
if currentBytes > 0 && (currentBytes+len(line) > maxBytes || currentPrompt+linePrompt > promptBudget) {
groups = append(groups, []string{current.String()})
current.Reset()
currentBytes = 0
currentPrompt = 0
}
current.WriteString(line)
currentBytes += len(line)
currentPrompt += linePrompt
}
if current.Len() > 0 {
groups = append(groups, []string{current.String()})
}
if len(groups) <= 1 {
if !force {
return nil, false
}
return splitPlainDocSectionMidpoint(lines)
}
return groups, true
}
func splitPlainDocSectionMidpoint(lines []string) ([][]string, bool) {
if len(lines) <= 1 {
return nil, false
}
mid := len(lines) / 2
if mid <= 0 || mid >= len(lines) {
return nil, false
}
left := strings.Join(lines[:mid], "")
right := strings.Join(lines[mid:], "")
if strings.TrimSpace(left) == "" || strings.TrimSpace(right) == "" {
return nil, false
}
return [][]string{{left}, {right}}, true
}
func firstNonEmptyLineIndex(lines []string) int {
for index, line := range lines {
if strings.TrimSpace(line) != "" {
return index
}
}
return -1
}
func lastNonEmptyLineIndex(lines []string) int {
for index := len(lines) - 1; index >= 0; index-- {
if strings.TrimSpace(lines[index]) != "" {
return index
}
}
return -1
}
func docsI18nDocChunkMaxBytes() int {
value := strings.TrimSpace(os.Getenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES"))
if value == "" {
return defaultDocChunkMaxBytes
}
parsed, err := strconv.Atoi(value)
if err != nil || parsed <= 0 {
return defaultDocChunkMaxBytes
}
return parsed
}
func docsI18nDocChunkPromptBudget() int {
value := strings.TrimSpace(os.Getenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_PROMPT_BUDGET"))
if value == "" {
return defaultDocChunkPromptBudget
}
parsed, err := strconv.Atoi(value)
if err != nil || parsed <= 0 {
return defaultDocChunkPromptBudget
}
return parsed
}
func estimateDocPromptCost(text string) int {
cost := len(text)
cost += strings.Count(text, "`") * 6
cost += strings.Count(text, "|") * 4
cost += strings.Count(text, "{") * 4
cost += strings.Count(text, "}") * 4
cost += strings.Count(text, "[") * 4
cost += strings.Count(text, "]") * 4
cost += strings.Count(text, ":") * 2
cost += strings.Count(text, "<") * 4
cost += strings.Count(text, ">") * 4
return cost
}
func stripCommonIndent(text string) (string, string) {
lines := strings.SplitAfter(text, "\n")
common := ""
for _, line := range lines {
trimmed := strings.TrimRight(line, "\r\n")
if strings.TrimSpace(trimmed) == "" {
continue
}
indent := leadingIndent(trimmed)
if common == "" {
common = indent
continue
}
common = commonIndentPrefix(common, indent)
if common == "" {
return text, ""
}
}
if common == "" {
return text, ""
}
var out strings.Builder
for _, line := range lines {
trimmed := strings.TrimRight(line, "\r\n")
if strings.TrimSpace(trimmed) == "" {
out.WriteString(line)
continue
}
if strings.HasPrefix(line, common) {
out.WriteString(strings.TrimPrefix(line, common))
continue
}
out.WriteString(line)
}
return out.String(), common
}
func reapplyCommonIndent(text, indent string) string {
if indent == "" || text == "" {
return text
}
lines := strings.SplitAfter(text, "\n")
var out strings.Builder
for _, line := range lines {
trimmed := strings.TrimRight(line, "\r\n")
if strings.TrimSpace(trimmed) == "" {
out.WriteString(line)
continue
}
out.WriteString(indent)
out.WriteString(line)
}
return out.String()
}
func leadingIndent(line string) string {
index := 0
for index < len(line) {
if line[index] != ' ' && line[index] != '\t' {
break
}
index++
}
return line[:index]
}
func commonIndentPrefix(a, b string) string {
limit := len(a)
if len(b) < limit {
limit = len(b)
}
index := 0
for index < limit && a[index] == b[index] {
index++
}
return a[:index]
}

View file

@ -0,0 +1,200 @@
package main
import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
)
const (
frontmatterTagStart = "<frontmatter>"
frontmatterTagEnd = "</frontmatter>"
bodyTagStart = "<body>"
bodyTagEnd = "</body>"
)
func processFileDoc(ctx context.Context, translator docsTranslator, docsRoot, filePath, srcLang, tgtLang string, overwrite bool) (bool, string, error) {
absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
if err != nil {
return false, "", err
}
content, err := os.ReadFile(absPath)
if err != nil {
return false, "", err
}
currentHash := hashBytes(content)
outputPath := filepath.Join(docsRoot, tgtLang, relPath)
if !overwrite {
skip, err := shouldSkipDoc(outputPath, currentHash)
if err != nil {
return false, "", err
}
if skip {
return true, "", nil
}
}
sourceFront, sourceBody := splitFrontMatter(string(content))
frontData := map[string]any{}
if strings.TrimSpace(sourceFront) != "" {
if err := yaml.Unmarshal([]byte(sourceFront), &frontData); err != nil {
return false, "", fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
}
}
docTM := &TranslationMemory{entries: map[string]TMEntry{}}
if err := translateFrontMatter(ctx, translator, docTM, frontData, relPath, srcLang, tgtLang); err != nil {
return false, "", fmt.Errorf("frontmatter translation failed for %s: %w", relPath, err)
}
updatedFront, err := encodeFrontMatter(frontData, relPath, content)
if err != nil {
return false, "", err
}
translatedBody, err := translateDocBodyChunked(ctx, translator, relPath, sourceBody, srcLang, tgtLang)
if err != nil {
return false, "", fmt.Errorf("body translate failed for %s: %w", relPath, err)
}
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
return false, "", err
}
output := updatedFront + translatedBody
return false, outputPath, os.WriteFile(outputPath, []byte(output), 0o644)
}
func formatTaggedDocument(frontMatter, body string) string {
return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", frontmatterTagStart, frontMatter, frontmatterTagEnd, bodyTagStart, body, bodyTagEnd)
}
func parseTaggedDocument(text string) (string, string, error) {
frontStart := strings.Index(text, frontmatterTagStart)
if frontStart == -1 {
return "", "", fmt.Errorf("missing %s", frontmatterTagStart)
}
frontStart += len(frontmatterTagStart)
frontEnd := strings.Index(text[frontStart:], frontmatterTagEnd)
if frontEnd == -1 {
return "", "", fmt.Errorf("missing %s", frontmatterTagEnd)
}
frontEnd += frontStart
bodyStart := strings.Index(text[frontEnd:], bodyTagStart)
if bodyStart == -1 {
return "", "", fmt.Errorf("missing %s", bodyTagStart)
}
bodyStart += frontEnd + len(bodyTagStart)
bodyEnd := findTaggedBodyEnd(text, bodyStart)
if bodyEnd == -1 {
return "", "", fmt.Errorf("missing %s", bodyTagEnd)
}
body := trimTagNewlines(text[bodyStart:bodyEnd])
suffix := strings.TrimSpace(text[bodyEnd+len(bodyTagEnd):])
prefix := strings.TrimSpace(text[:frontStart-len(frontmatterTagStart)])
if prefix != "" || suffix != "" {
return "", "", fmt.Errorf("unexpected text outside tagged sections")
}
frontMatter := trimTagNewlines(text[frontStart:frontEnd])
return frontMatter, body, nil
}
func findTaggedBodyEnd(text string, bodyStart int) int {
if bodyStart < 0 || bodyStart > len(text) {
return -1
}
search := text[bodyStart:]
candidate := -1
offset := 0
for {
index := strings.Index(search[offset:], bodyTagEnd)
if index == -1 {
return candidate
}
index += offset
absolute := bodyStart + index
suffix := strings.TrimSpace(text[absolute+len(bodyTagEnd):])
if suffix == "" {
candidate = absolute
}
offset = index + len(bodyTagEnd)
if offset >= len(search) {
return candidate
}
}
}
func trimTagNewlines(value string) string {
value = strings.TrimPrefix(value, "\n")
value = strings.TrimSuffix(value, "\n")
return value
}
func shouldSkipDoc(outputPath string, sourceHash string) (bool, error) {
data, err := os.ReadFile(outputPath)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
frontMatter, _ := splitFrontMatter(string(data))
if frontMatter == "" {
return false, nil
}
frontData := map[string]any{}
if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
return false, nil
}
storedHash := extractSourceHash(frontData)
if storedHash == "" {
return false, nil
}
return strings.EqualFold(storedHash, sourceHash), nil
}
func extractSourceHash(frontData map[string]any) string {
xi, ok := frontData["x-i18n"].(map[string]any)
if !ok {
return ""
}
value, ok := xi["source_hash"].(string)
if !ok {
return ""
}
return strings.TrimSpace(value)
}
func logDocChunkPlan(relPath string, blocks []string, groups [][]string) {
totalBytes := 0
for _, block := range blocks {
totalBytes += len(block)
}
log.Printf("docs-i18n: body-chunks %s blocks=%d groups=%d bytes=%d", relPath, len(blocks), len(groups), totalBytes)
}
func resolveDocsPath(docsRoot, filePath string) (string, string, error) {
absPath, err := filepath.Abs(filePath)
if err != nil {
return "", "", err
}
relPath, err := filepath.Rel(docsRoot, absPath)
if err != nil {
return "", "", err
}
if relPath == "." || relPath == "" {
return "", "", fmt.Errorf("file %s resolves to docs root %s", absPath, docsRoot)
}
if filepath.IsAbs(relPath) || relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) {
return "", "", fmt.Errorf("file %s not under docs root %s", absPath, docsRoot)
}
return absPath, relPath, nil
}

View file

@ -0,0 +1,904 @@
package main
import (
"context"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
)
type docChunkTranslator struct{}
func (docChunkTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (docChunkTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
switch {
case strings.Contains(text, "Alpha block") && strings.Contains(text, "Beta block"):
return strings.ReplaceAll(text, "</Accordion>", ""), nil
default:
replacer := strings.NewReplacer(
"Alpha block", "阿尔法段",
"Beta block", "贝塔段",
"Code sample", "代码示例",
)
return replacer.Replace(text), nil
}
}
func (docChunkTranslator) Close() {}
type docLeafFallbackTranslator struct{}
func (docLeafFallbackTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
replacer := strings.NewReplacer(
"Gateway refuses to start unless `local`.", "Gateway 只有在 `local` 时才会启动。",
"`gateway.auth.mode: \"trusted-proxy\"`", "`gateway.auth.mode: \"trusted-proxy\"`",
)
return replacer.Replace(text), nil
}
func (docLeafFallbackTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
if strings.Contains(text, "Gateway refuses to start unless `local`.") {
return strings.Replace(text, "Gateway refuses to start unless `local`.", "<Tip>Gateway only starts in local mode.</Tip>", 1), nil
}
return text, nil
}
func (docLeafFallbackTranslator) Close() {}
type docFrontmatterTranslator struct{}
func (docFrontmatterTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
replacer := strings.NewReplacer(
"Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS", "在 Fly.io 上逐步部署 OpenClaw,包含持久化存储和 HTTPS",
"Deploying OpenClaw on Fly.io", "在 Fly.io 上部署 OpenClaw",
"Setting up Fly volumes, secrets, and first-run config", "设置 Fly volume、密钥和首次运行配置",
)
return replacer.Replace(text), nil
}
func (docFrontmatterTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
return "extra text outside tagged sections", nil
}
func (docFrontmatterTranslator) Close() {}
type docFrontmatterFallbackTranslator struct{}
func (docFrontmatterFallbackTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
switch text {
case "Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS":
return strings.Join([]string{
"<frontmatter>",
"title: Fly.io",
"summary: \"在 Fly.io 上部署 OpenClaw 的逐步指南,包含持久化存储和 HTTPS 设置\"",
"read_when:",
" - 在 Fly.io 上部署 OpenClaw",
" - 设置 Fly 卷、机密和初始运行配置",
"</frontmatter>",
"",
"<body>",
"# Fly.io 部署",
"</body>",
}, "\n"), nil
case "Deploying OpenClaw on Fly.io":
return "在 Fly.io 上部署 OpenClaw", nil
case "Setting up Fly volumes, secrets, and first-run config":
return "设置 Fly 卷、机密和初始运行配置", nil
default:
return text, nil
}
}
func (docFrontmatterFallbackTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (docFrontmatterFallbackTranslator) Close() {}
type docProtocolLeakTranslator struct{}
func (docProtocolLeakTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (docProtocolLeakTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
switch {
case strings.Contains(text, "First chunk") && strings.Contains(text, "Second chunk"):
return strings.Join([]string{
"<frontmatter>",
"title: leaked",
"</frontmatter>",
"",
"<body>",
"First translated",
"",
"Second translated",
"</body>",
}, "\n"), nil
default:
replacer := strings.NewReplacer(
"First chunk", "First translated",
"Second chunk", "Second translated",
)
return replacer.Replace(text), nil
}
}
func (docProtocolLeakTranslator) Close() {}
type docWrappedLeafTranslator struct{}
func (docWrappedLeafTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (docWrappedLeafTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
return strings.Join([]string{
"<frontmatter>",
"title: leaked",
"</frontmatter>",
"",
"<body>",
"# Fly.io 部署",
"</body>",
}, "\n"), nil
}
func (docWrappedLeafTranslator) Close() {}
type docComponentLeafFallbackTranslator struct{}
func (docComponentLeafFallbackTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return strings.ReplaceAll(text, "Yes.", "是的。"), nil
}
func (docComponentLeafFallbackTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
if strings.Contains(text, "Can I use Claude Max subscription without an API key?") {
return strings.ReplaceAll(text, "Yes.\n", "Yes.\n</Accordion>\n"), nil
}
return text, nil
}
func (docComponentLeafFallbackTranslator) Close() {}
type docPromptBudgetTranslator struct {
rawInputs []string
}
func (t *docPromptBudgetTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (t *docPromptBudgetTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
t.rawInputs = append(t.rawInputs, text)
replacer := strings.NewReplacer(
"First chunk with `json5` and { braces }", "第一块,含 `json5` 和 { braces }",
"Second chunk with | table | pipes |", "第二块,含 | table | pipes |",
)
return replacer.Replace(text), nil
}
func (t *docPromptBudgetTranslator) Close() {}
type uppercaseWrapperTranslator struct{}
func (uppercaseWrapperTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (uppercaseWrapperTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
return "<BODY>\n" + strings.ReplaceAll(text, "Regular paragraph.", "Translated paragraph.") + "\n</BODY>\n", nil
}
func (uppercaseWrapperTranslator) Close() {}
type oversizedBlockTranslator struct {
rawInputs []string
}
func (t *oversizedBlockTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (t *oversizedBlockTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
t.rawInputs = append(t.rawInputs, text)
return strings.ReplaceAll(text, "Line ", "Translated line "), nil
}
func (t *oversizedBlockTranslator) Close() {}
type singletonFenceRetryTranslator struct {
rawInputs []string
}
func (t *singletonFenceRetryTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (t *singletonFenceRetryTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
t.rawInputs = append(t.rawInputs, text)
if strings.Contains(text, "Line 01") && strings.Contains(text, "Line 04") {
return strings.Replace(text, "\n```\n", "\n", 1), nil
}
return strings.ReplaceAll(text, "Line ", "Translated line "), nil
}
func (t *singletonFenceRetryTranslator) Close() {}
func TestParseTaggedDocumentRejectsMissingBodyCloseAtEOF(t *testing.T) {
t.Parallel()
input := "<frontmatter>\ntitle: Test\n</frontmatter>\n<body>\nTranslated body\n"
_, _, err := parseTaggedDocument(input)
if err == nil {
t.Fatal("expected error for missing </body>")
}
}
func TestParseTaggedDocumentRejectsTrailingTextOutsideTags(t *testing.T) {
t.Parallel()
input := "<frontmatter>\ntitle: Test\n</frontmatter>\n<body>\nTranslated body\n</body>\nextra"
_, _, err := parseTaggedDocument(input)
if err == nil {
t.Fatal("expected error for trailing text")
}
}
func TestFindTaggedBodyEndSearchesFromBodyStart(t *testing.T) {
t.Parallel()
text := strings.Join([]string{
"<frontmatter>",
"summary: literal </body> token in frontmatter",
"</frontmatter>",
"<body>",
"Translated body",
"</body>",
}, "\n")
bodyStart := strings.Index(text, bodyTagStart)
if bodyStart == -1 {
t.Fatal("expected body tag in test input")
}
bodyStart += len(bodyTagStart)
bodyEnd := findTaggedBodyEnd(text, bodyStart)
if bodyEnd == -1 {
t.Fatal("expected closing body tag to be found")
}
body := trimTagNewlines(text[bodyStart:bodyEnd])
if body != "Translated body" {
t.Fatalf("expected body slice to ignore pre-body literal token, got %q", body)
}
}
func TestSplitDocBodyIntoBlocksKeepsFenceTogether(t *testing.T) {
t.Parallel()
body := strings.Join([]string{
"<Accordion title=\"Alpha block\">",
"",
"Code sample:",
"```ts",
"console.log('hello')",
"```",
"",
"Beta block",
"",
"</Accordion>",
"",
}, "\n")
blocks := splitDocBodyIntoBlocks(body)
if len(blocks) != 4 {
t.Fatalf("expected 4 blocks, got %d", len(blocks))
}
if !strings.Contains(blocks[1], "```ts") || !strings.Contains(blocks[1], "```") {
t.Fatalf("expected code fence to stay in a single block:\n%s", blocks[1])
}
if !strings.Contains(blocks[2], "Beta block") {
t.Fatalf("expected Beta paragraph in its own block:\n%s", blocks[2])
}
}
func TestSplitDocBodyIntoBlocksKeepsNestedTripleBackticksInsideFourBacktickFence(t *testing.T) {
t.Parallel()
body := strings.Join([]string{
"````md",
"```ts",
"console.log('nested example')",
"```",
"````",
"",
"Outside paragraph",
"",
}, "\n")
blocks := splitDocBodyIntoBlocks(body)
if len(blocks) != 2 {
t.Fatalf("expected 2 blocks, got %d", len(blocks))
}
if !strings.Contains(blocks[0], "console.log('nested example')") || !strings.Contains(blocks[0], "````") {
t.Fatalf("expected the full fenced example to stay in one block:\n%s", blocks[0])
}
if !strings.Contains(blocks[1], "Outside paragraph") {
t.Fatalf("expected trailing paragraph in second block:\n%s", blocks[1])
}
}
func TestSanitizeDocChunkProtocolWrappersStripsOuterWrapperAroundBodyExamples(t *testing.T) {
t.Parallel()
source := strings.Join([]string{
"Paragraph mentioning literal tokens `<body>` and `</body>`.",
"",
"<html>",
" <body>",
" literal example",
" </body>",
"</html>",
}, "\n")
translated := strings.Join([]string{
"<frontmatter>",
"title: leaked",
"</frontmatter>",
"",
"<body>",
"提到字面量 `<body>` 和 `</body>` 的段落。",
"",
"<html>",
" <body>",
" literal example",
" </body>",
"</html>",
"</body>",
}, "\n")
sanitized := sanitizeDocChunkProtocolWrappers(source, translated)
if strings.Contains(sanitized, frontmatterTagStart) || strings.HasPrefix(strings.TrimSpace(sanitized), bodyTagStart) {
t.Fatalf("expected outer wrapper stripped, got:\n%s", sanitized)
}
if !strings.Contains(sanitized, "<html>") || !strings.Contains(sanitized, "<body>") || !strings.Contains(sanitized, "</body>") {
t.Fatalf("expected inner HTML example preserved, got:\n%s", sanitized)
}
}
func TestTranslateDocBodyChunkedFallsBackToSmallerChunks(t *testing.T) {
body := strings.Join([]string{
"<Accordion title=\"Alpha block\">",
"Alpha block",
"</Accordion>",
"",
"Beta block",
"",
}, "\n")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
translated, err := translateDocBodyChunked(context.Background(), docChunkTranslator{}, "help/faq.md", body, "en", "zh-CN")
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if !strings.Contains(translated, "阿尔法段") || !strings.Contains(translated, "贝塔段") {
t.Fatalf("expected translated text after chunk split, got:\n%s", translated)
}
if strings.Count(translated, "</Accordion>") != 1 {
t.Fatalf("expected closing Accordion tag to be preserved after fallback split:\n%s", translated)
}
}
func TestStripAndReapplyCommonIndent(t *testing.T) {
t.Parallel()
source := strings.Join([]string{
" <Step title=\"Example\">",
" - item one",
" - item two",
" </Step>",
"",
}, "\n")
normalized, indent := stripCommonIndent(source)
if indent != " " {
t.Fatalf("expected common indent of four spaces, got %q", indent)
}
if strings.HasPrefix(normalized, " ") {
t.Fatalf("expected normalized text without common indent:\n%s", normalized)
}
roundTrip := reapplyCommonIndent(normalized, indent)
if roundTrip != source {
t.Fatalf("expected indent round-trip to preserve source\nwant:\n%s\ngot:\n%s", source, roundTrip)
}
}
func TestTranslateDocBodyChunkedFallsBackToMaskedTranslateForLeafValidationFailure(t *testing.T) {
body := strings.Join([]string{
"- `mode`: `local` or `remote`. Gateway refuses to start unless `local`.",
"- `gateway.auth.mode: \"trusted-proxy\"`: delegate auth to a reverse proxy.",
"",
}, "\n")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
translated, err := translateDocBodyChunked(
context.Background(),
docLeafFallbackTranslator{},
"gateway/configuration-reference.md",
body,
"en",
"zh-CN",
)
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if strings.Contains(translated, "<Tip>") {
t.Fatalf("expected masked fallback to remove hallucinated component tags:\n%s", translated)
}
if !strings.Contains(translated, "Gateway 只有在 `local` 时才会启动。") {
t.Fatalf("expected fallback translation to be applied:\n%s", translated)
}
}
func TestValidateDocChunkTranslationRejectsProtocolTokenLeakage(t *testing.T) {
t.Parallel()
source := "Regular paragraph.\n\n"
translated := "<frontmatter>\ntitle: leaked\n</frontmatter>\n<body>\nRegular paragraph.\n</body>\n"
err := validateDocChunkTranslation(source, translated)
if err == nil {
t.Fatal("expected protocol token leakage to be rejected")
}
if !strings.Contains(err.Error(), "protocol token leaked") {
t.Fatalf("expected protocol token leakage error, got %v", err)
}
}
func TestValidateDocChunkTranslationRejectsTopLevelBodyWrapperLeakEvenWhenSourceMentionsBodyTag(t *testing.T) {
t.Parallel()
source := "Use `<body>` in examples, but keep prose outside wrappers.\n"
translated := "<body>\nTranslated paragraph.\n"
err := validateDocChunkTranslation(source, translated)
if err == nil {
t.Fatal("expected top-level wrapper leakage to be rejected")
}
if !strings.Contains(err.Error(), "protocol token leaked") {
t.Fatalf("expected protocol token leakage error, got %v", err)
}
}
func TestTranslateDocBodyChunkedSplitsOnProtocolTokenLeakage(t *testing.T) {
body := strings.Join([]string{
"First chunk",
"",
"Second chunk",
"",
}, "\n")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
translated, err := translateDocBodyChunked(context.Background(), docProtocolLeakTranslator{}, "gateway/configuration-reference.md", body, "en", "zh-CN")
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if strings.Contains(translated, "<frontmatter>") || strings.Contains(translated, "<body>") || strings.Contains(translated, "[[[FM_") {
t.Fatalf("expected protocol wrapper leakage to be removed after split:\n%s", translated)
}
if !strings.Contains(translated, "First translated") || !strings.Contains(translated, "Second translated") {
t.Fatalf("expected split chunks to translate successfully:\n%s", translated)
}
}
func TestTranslateDocBodyChunkedStripsUppercaseBodyWrapper(t *testing.T) {
body := "Regular paragraph.\n"
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
translated, err := translateDocBodyChunked(context.Background(), uppercaseWrapperTranslator{}, "gateway/configuration-reference.md", body, "en", "zh-CN")
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if strings.Contains(strings.ToLower(translated), "<body>") {
t.Fatalf("expected uppercase wrapper to be stripped:\n%s", translated)
}
if !strings.Contains(translated, "Translated paragraph.") {
t.Fatalf("expected translated body content to survive unwrap:\n%s", translated)
}
}
func TestSanitizeDocChunkProtocolWrappersKeepsBodyOnlyWrapperWhenSourceMentionsBodyTag(t *testing.T) {
t.Parallel()
source := "Use `<body>` and `</body>` in examples, but keep the paragraph text plain.\n"
translated := "<body>\nTranslated paragraph.\n</body>\n"
got := sanitizeDocChunkProtocolWrappers(source, translated)
if got != translated {
t.Fatalf("expected ambiguous body-only wrapper to remain unchanged for retry\nwant:\n%s\ngot:\n%s", translated, got)
}
}
func TestSanitizeDocChunkProtocolWrappersKeepsLegitimateTopLevelBodyBlock(t *testing.T) {
t.Parallel()
source := "<body>\nLiteral HTML block.\n</body>\n"
translated := "<body>\nLiteral HTML block.\n</body>\n"
got := sanitizeDocChunkProtocolWrappers(source, translated)
if got != translated {
t.Fatalf("expected legitimate top-level body block to remain unchanged\nwant:\n%s\ngot:\n%s", translated, got)
}
}
func TestSanitizeDocChunkProtocolWrappersStripsBodyOnlyWrapperWhenSourceHasNoBodyTokens(t *testing.T) {
t.Parallel()
source := "Regular paragraph.\n"
translated := "<body>\nTranslated paragraph.\n</body>\n"
got := sanitizeDocChunkProtocolWrappers(source, translated)
if strings.Contains(got, "<body>") || strings.Contains(got, "</body>") {
t.Fatalf("expected body-only wrapper to be stripped, got %q", got)
}
if strings.TrimSpace(got) != "Translated paragraph." {
t.Fatalf("unexpected sanitized body %q", got)
}
}
func TestSanitizeDocChunkProtocolWrappersKeepsAmbiguousTaggedWrapperForRetry(t *testing.T) {
t.Parallel()
source := strings.Join([]string{
"Paragraph mentioning literal tokens `<body>` and `</body>`.",
"",
"Closing example:",
"</body>",
}, "\n")
translated := strings.Join([]string{
"<frontmatter>",
"title: leaked",
"</frontmatter>",
"",
"<body>",
"提到字面量 `<body>` 和 `</body>` 的段落。",
}, "\n")
got := sanitizeDocChunkProtocolWrappers(source, translated)
if got != translated {
t.Fatalf("expected ambiguous tagged wrapper to remain unchanged for retry\nwant:\n%s\ngot:\n%s", translated, got)
}
}
func TestSplitDocBodyIntoBlocksKeepsInfoStringExampleInsideFence(t *testing.T) {
t.Parallel()
body := strings.Join([]string{
"```md",
"```ts",
"console.log('inside example')",
"```",
"",
"Outside paragraph",
"",
}, "\n")
blocks := splitDocBodyIntoBlocks(body)
if len(blocks) != 2 {
t.Fatalf("expected 2 blocks, got %d", len(blocks))
}
if !strings.Contains(blocks[0], "console.log('inside example')") || !strings.Contains(blocks[0], "```ts") {
t.Fatalf("expected fenced example to stay together:\n%s", blocks[0])
}
if !strings.Contains(blocks[1], "Outside paragraph") {
t.Fatalf("expected trailing paragraph in second block:\n%s", blocks[1])
}
}
func TestTranslateDocBodyChunkedPreSplitsOversizedPromptBudget(t *testing.T) {
body := strings.Join([]string{
"First chunk with `json5` and { braces }",
"",
"Second chunk with | table | pipes |",
"",
}, "\n")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_PROMPT_BUDGET", "60")
translator := &docPromptBudgetTranslator{}
translated, err := translateDocBodyChunked(
context.Background(),
translator,
"gateway/configuration-reference.md",
body,
"en",
"zh-CN",
)
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
for _, input := range translator.rawInputs {
if strings.Contains(input, "First chunk with `json5` and { braces }") && strings.Contains(input, "Second chunk with | table | pipes |") {
t.Fatalf("expected prompt budget guard to split before raw translation, saw combined input:\n%s", input)
}
}
if !strings.Contains(translated, "第一块") || !strings.Contains(translated, "第二块") {
t.Fatalf("expected split chunks to translate successfully:\n%s", translated)
}
}
func TestTranslateDocBodyChunkedSplitsOversizedSingletonBlock(t *testing.T) {
body := strings.Join([]string{
"Line 01",
"Line 02",
"Line 03",
"Line 04",
"Line 05",
"Line 06",
"",
}, "\n")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "24")
translator := &oversizedBlockTranslator{}
translated, err := translateDocBodyChunked(context.Background(), translator, "gateway/configuration-reference.md", body, "en", "zh-CN")
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if len(translator.rawInputs) < 2 {
t.Fatalf("expected oversized singleton block to be split before translation, saw %d input(s)", len(translator.rawInputs))
}
for _, input := range translator.rawInputs {
if len(input) > 24 {
t.Fatalf("expected split chunk under byte budget, got %d bytes:\n%s", len(input), input)
}
}
if !strings.Contains(translated, "Translated line 01") || !strings.Contains(translated, "Translated line 06") {
t.Fatalf("expected translated singleton parts to be reassembled:\n%s", translated)
}
}
func TestTranslateDocBodyChunkedSplitsSingletonBlockWhenPromptBudgetExceeded(t *testing.T) {
lineA := "Alpha chunk with { braces }\n"
lineB := "Beta chunk with | pipes |\n"
body := lineA + lineB + "\n"
budget := max(estimateDocPromptCost(lineA), estimateDocPromptCost(lineB)) + 1
if estimateDocPromptCost(body) <= budget {
t.Fatalf("test setup expected combined singleton prompt cost to exceed budget; cost=%d budget=%d", estimateDocPromptCost(body), budget)
}
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_PROMPT_BUDGET", strconv.Itoa(budget))
translator := &oversizedBlockTranslator{}
translated, err := translateDocBodyChunked(context.Background(), translator, "gateway/configuration-reference.md", body, "en", "zh-CN")
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if len(translator.rawInputs) < 2 {
t.Fatalf("expected prompt-budget singleton split before translation, saw %d input(s)", len(translator.rawInputs))
}
for _, input := range translator.rawInputs {
if estimateDocPromptCost(input) > budget {
t.Fatalf("expected split chunk under prompt budget, got cost=%d budget=%d:\n%s", estimateDocPromptCost(input), budget, input)
}
}
if !strings.Contains(translated, "Alpha chunk") || !strings.Contains(translated, "Beta chunk") {
t.Fatalf("expected translated singleton parts to be reassembled:\n%s", translated)
}
}
func TestTranslateDocBodyChunkedSplitsOversizedFenceBeforeTrailingProse(t *testing.T) {
body := strings.Join([]string{
"```md",
"Line 01",
"Line 02",
"Line 03",
"Line 04",
"```",
"Trailing paragraph after the fence.",
"",
}, "\n")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "24")
translator := &oversizedBlockTranslator{}
translated, err := translateDocBodyChunked(context.Background(), translator, "gateway/configuration-reference.md", body, "en", "zh-CN")
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if len(translator.rawInputs) < 3 {
t.Fatalf("expected oversized fenced block with trailing prose to split, saw %d input(s)", len(translator.rawInputs))
}
for _, input := range translator.rawInputs {
if strings.Contains(input, "Line 01") || strings.Contains(input, "Line 02") || strings.Contains(input, "Line 03") || strings.Contains(input, "Line 04") {
if !strings.Contains(input, "```md") || !strings.Contains(input, "```") {
t.Fatalf("expected fenced split input to keep matched fence wrappers:\n%s", input)
}
}
}
if !strings.Contains(translated, "Translated line 01") || !strings.Contains(translated, "Trailing paragraph after the fence.") {
t.Fatalf("expected fence content and trailing prose to survive split:\n%s", translated)
}
}
func TestTranslateDocBodyChunkedRetriesSingletonFenceAfterValidationFailure(t *testing.T) {
body := strings.Join([]string{
"```md",
"Line 01",
"Line 02",
"Line 03",
"Line 04",
"```",
"",
}, "\n")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_PROMPT_BUDGET", "4096")
translator := &singletonFenceRetryTranslator{}
translated, err := translateDocBodyChunked(context.Background(), translator, "gateway/configuration-reference.md", body, "en", "zh-CN")
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if len(translator.rawInputs) < 3 {
t.Fatalf("expected singleton fence retry to split after validation failure, saw %d input(s)", len(translator.rawInputs))
}
if !strings.Contains(translator.rawInputs[0], "Line 01") || !strings.Contains(translator.rawInputs[0], "Line 04") {
t.Fatalf("expected first raw attempt to include the original fenced block:\n%s", translator.rawInputs[0])
}
for _, input := range translator.rawInputs[1:] {
if strings.Contains(input, "Line 01") || strings.Contains(input, "Line 02") || strings.Contains(input, "Line 03") || strings.Contains(input, "Line 04") {
if !strings.Contains(input, "```md") || !strings.Contains(input, "```") {
t.Fatalf("expected split retry inputs to preserve fence wrappers:\n%s", input)
}
}
}
if !strings.Contains(translated, "Translated line 01") || !strings.Contains(translated, "Translated line 04") {
t.Fatalf("expected singleton fence retry to reassemble translated output:\n%s", translated)
}
}
func TestTranslateDocBodyChunkedUnwrapsTaggedLeafProtocolLeakage(t *testing.T) {
body := "# Fly.io Deployment\n\n"
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
translated, err := translateDocBodyChunked(
context.Background(),
docWrappedLeafTranslator{},
"install/fly.md",
body,
"en",
"zh-CN",
)
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if strings.Contains(translated, "<frontmatter>") || strings.Contains(translated, "<body>") {
t.Fatalf("expected wrapped leaf translation to unwrap protocol tags:\n%s", translated)
}
if !strings.Contains(translated, "# Fly.io 部署") {
t.Fatalf("expected unwrapped body translation:\n%s", translated)
}
}
func TestTranslateDocBodyChunkedFallsBackForComponentLeafValidationFailure(t *testing.T) {
body := " <Accordion title=\"Can I use Claude Max subscription without an API key?\">\n Yes.\n\n"
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
translated, err := translateDocBodyChunked(
context.Background(),
docComponentLeafFallbackTranslator{},
"help/faq.md",
body,
"en",
"zh-CN",
)
if err != nil {
t.Fatalf("translateDocBodyChunked returned error: %v", err)
}
if strings.Contains(translated, "</Accordion>") {
t.Fatalf("expected component leaf fallback to avoid hallucinated closing tag:\n%s", translated)
}
if !strings.Contains(translated, "是的。") {
t.Fatalf("expected body text to be translated after component leaf fallback:\n%s", translated)
}
if !strings.Contains(translated, "<Accordion title=\"Can I use Claude Max subscription without an API key?\">") {
t.Fatalf("expected Accordion opening tag to be preserved:\n%s", translated)
}
}
func TestProcessFileDocUsesFieldLevelFrontmatterTranslation(t *testing.T) {
t.Parallel()
docsRoot := t.TempDir()
sourcePath := filepath.Join(docsRoot, "install")
if err := os.MkdirAll(sourcePath, 0o755); err != nil {
t.Fatalf("mkdir failed: %v", err)
}
sourceFile := filepath.Join(sourcePath, "fly.md")
source := strings.Join([]string{
"---",
"title: Fly.io",
"summary: \"Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS\"",
"read_when:",
" - Deploying OpenClaw on Fly.io",
" - Setting up Fly volumes, secrets, and first-run config",
"---",
"",
}, "\n")
if err := os.WriteFile(sourceFile, []byte(source), 0o644); err != nil {
t.Fatalf("write failed: %v", err)
}
skipped, outputPath, err := processFileDoc(context.Background(), docFrontmatterTranslator{}, docsRoot, sourceFile, "en", "zh-CN", true)
if err != nil {
t.Fatalf("processFileDoc returned error: %v", err)
}
if skipped {
t.Fatal("expected file to be processed")
}
if outputPath == "" {
t.Fatal("expected output path")
}
output, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("read output failed: %v", err)
}
text := string(output)
if !strings.Contains(text, "在 Fly.io 上逐步部署 OpenClaw,包含持久化存储和 HTTPS") {
t.Fatalf("expected translated summary in output:\n%s", text)
}
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
t.Fatalf("expected translated read_when entry in output:\n%s", text)
}
}
func TestProcessFileDocRejectsSuspiciousFrontmatterScalarExpansion(t *testing.T) {
t.Parallel()
docsRoot := t.TempDir()
sourcePath := filepath.Join(docsRoot, "install")
if err := os.MkdirAll(sourcePath, 0o755); err != nil {
t.Fatalf("mkdir failed: %v", err)
}
sourceFile := filepath.Join(sourcePath, "fly.md")
source := strings.Join([]string{
"---",
"title: Fly.io",
"summary: \"Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS\"",
"read_when:",
" - Deploying OpenClaw on Fly.io",
" - Setting up Fly volumes, secrets, and first-run config",
"---",
"",
}, "\n")
if err := os.WriteFile(sourceFile, []byte(source), 0o644); err != nil {
t.Fatalf("write failed: %v", err)
}
skipped, outputPath, err := processFileDoc(context.Background(), docFrontmatterFallbackTranslator{}, docsRoot, sourceFile, "en", "zh-CN", true)
if err != nil {
t.Fatalf("processFileDoc returned error: %v", err)
}
if skipped {
t.Fatal("expected file to be processed")
}
output, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("read output failed: %v", err)
}
text := string(output)
if strings.Contains(text, "<frontmatter>") || strings.Contains(text, "<body>") {
t.Fatalf("expected suspicious frontmatter expansion to be rejected:\n%s", text)
}
if !strings.Contains(text, "summary: Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS") {
t.Fatalf("expected original summary to be preserved after fallback:\n%s", text)
}
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
t.Fatalf("expected read_when translation to survive fallback:\n%s", text)
}
}

View file

@ -0,0 +1,29 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"os"
)
type GlossaryEntry struct {
Source string `json:"source"`
Target string `json:"target"`
}
func LoadGlossary(path string) ([]GlossaryEntry, error) {
data, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, err
}
var entries []GlossaryEntry
if err := json.Unmarshal(data, &entries); err != nil {
return nil, fmt.Errorf("glossary parse failed: %w", err)
}
return entries, nil
}

View file

@ -0,0 +1,10 @@
module github.com/openclaw/openclaw/scripts/docs-i18n
go 1.24.0
require (
github.com/joshp123/pi-golang v0.0.4
github.com/yuin/goldmark v1.7.8
golang.org/x/net v0.50.0
gopkg.in/yaml.v3 v3.0.1
)

View file

@ -0,0 +1,10 @@
github.com/joshp123/pi-golang v0.0.4 h1:82HISyKNN8bIl2lvAd65462LVCQIsjhaUFQxyQgg5Xk=
github.com/joshp123/pi-golang v0.0.4/go.mod h1:9mHEQkeJELYzubXU3b86/T8yedI/iAOKx0Tz0c41qes=
github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic=
github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -0,0 +1,160 @@
package main
import (
"context"
"io"
"strings"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/text"
"golang.org/x/net/html"
"sort"
)
type htmlReplacement struct {
Start int
Stop int
Value string
}
func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, srcLang, tgtLang string) (string, error) {
source := []byte(body)
r := text.NewReader(source)
md := goldmark.New(
goldmark.WithExtensions(extension.GFM),
)
doc := md.Parser().Parse(r)
replacements := make([]htmlReplacement, 0, 8)
_ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
if !entering {
return ast.WalkContinue, nil
}
block, ok := n.(*ast.HTMLBlock)
if !ok {
return ast.WalkContinue, nil
}
start, stop, ok := htmlBlockSpan(block, source)
if !ok {
return ast.WalkSkipChildren, nil
}
htmlText := string(source[start:stop])
translated, err := translateHTMLBlock(ctx, translator, htmlText, srcLang, tgtLang)
if err != nil {
return ast.WalkStop, err
}
replacements = append(replacements, htmlReplacement{Start: start, Stop: stop, Value: translated})
return ast.WalkSkipChildren, nil
})
if len(replacements) == 0 {
return body, nil
}
return applyHTMLReplacements(body, replacements), nil
}
func htmlBlockSpan(block *ast.HTMLBlock, source []byte) (int, int, bool) {
lines := block.Lines()
if lines.Len() == 0 {
return 0, 0, false
}
start := lines.At(0).Start
stop := lines.At(lines.Len() - 1).Stop
if start >= stop {
return 0, 0, false
}
return start, stop, true
}
func applyHTMLReplacements(body string, replacements []htmlReplacement) string {
if len(replacements) == 0 {
return body
}
sortHTMLReplacements(replacements)
var out strings.Builder
last := 0
for _, rep := range replacements {
if rep.Start < last {
continue
}
out.WriteString(body[last:rep.Start])
out.WriteString(rep.Value)
last = rep.Stop
}
out.WriteString(body[last:])
return out.String()
}
func sortHTMLReplacements(replacements []htmlReplacement) {
sort.Slice(replacements, func(i, j int) bool {
return replacements[i].Start < replacements[j].Start
})
}
func translateHTMLBlock(ctx context.Context, translator docsTranslator, htmlText, srcLang, tgtLang string) (string, error) {
tokenizer := html.NewTokenizer(strings.NewReader(htmlText))
var out strings.Builder
skipDepth := 0
for {
tt := tokenizer.Next()
if tt == html.ErrorToken {
if err := tokenizer.Err(); err != nil && err != io.EOF {
return "", err
}
break
}
raw := string(tokenizer.Raw())
tok := tokenizer.Token()
switch tt {
case html.StartTagToken:
out.WriteString(raw)
if isSkipTag(strings.ToLower(tok.Data)) {
skipDepth++
}
case html.EndTagToken:
out.WriteString(raw)
if isSkipTag(strings.ToLower(tok.Data)) && skipDepth > 0 {
skipDepth--
}
case html.SelfClosingTagToken:
out.WriteString(raw)
case html.TextToken:
if shouldTranslateHTMLText(skipDepth, raw) {
translated, err := translator.Translate(ctx, raw, srcLang, tgtLang)
if err != nil {
return "", err
}
out.WriteString(translated)
} else {
out.WriteString(raw)
}
default:
out.WriteString(raw)
}
}
return out.String(), nil
}
func shouldTranslateHTMLText(skipDepth int, text string) bool {
if strings.TrimSpace(text) == "" {
return false
}
return skipDepth == 0
}
func isSkipTag(tag string) bool {
switch tag {
case "code", "pre", "script", "style":
return true
default:
return false
}
}

View file

@ -0,0 +1,398 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"regexp"
"strings"
"gopkg.in/yaml.v3"
)
type routeIndex struct {
targetLang string
redirects map[string]string
sourceRoutes map[string]struct{}
localizedRoutes map[string]struct{}
localePrefixes map[string]struct{}
}
type docsConfig struct {
Redirects []docsRedirect `json:"redirects"`
}
type docsRedirect struct {
Source string `json:"source"`
Destination string `json:"destination"`
}
var (
localeDirRe = regexp.MustCompile(`^[a-z]{2,3}(?:-[A-Za-z0-9]{2,8})?$`)
fencedBacktickCodeBlock = regexp.MustCompile("(?ms)(^|\\n)[ \\t]*```[^\\n]*\\n.*?\\n[ \\t]*```[ \\t]*(?:\\n|$)")
fencedTildeCodeBlock = regexp.MustCompile("(?ms)(^|\\n)[ \\t]*~~~[^\\n]*\\n.*?\\n[ \\t]*~~~[ \\t]*(?:\\n|$)")
markdownLinkTargetRe = regexp.MustCompile(`!?\[[^\]]*\]\(([^)]+)\)`)
hrefDoubleQuotedValueRe = regexp.MustCompile(`\bhref\s*=\s*"([^"]*)"`)
hrefSingleQuotedValueRe = regexp.MustCompile(`\bhref\s*=\s*'([^']*)'`)
)
func loadRouteIndex(docsRoot, targetLang string) (*routeIndex, error) {
index := &routeIndex{
targetLang: strings.TrimSpace(targetLang),
redirects: map[string]string{},
sourceRoutes: map[string]struct{}{},
localizedRoutes: map[string]struct{}{},
localePrefixes: map[string]struct{}{},
}
if err := index.loadRedirects(filepath.Join(docsRoot, "docs.json")); err != nil {
return nil, err
}
if err := index.loadRoutes(docsRoot); err != nil {
return nil, err
}
return index, nil
}
func (ri *routeIndex) loadRedirects(configPath string) error {
data, err := os.ReadFile(configPath)
if err != nil {
return err
}
var config docsConfig
if err := json.Unmarshal(data, &config); err != nil {
return err
}
for _, item := range config.Redirects {
source := normalizeRoute(item.Source)
destination := normalizeRoute(item.Destination)
if source == "" || destination == "" {
continue
}
ri.redirects[source] = destination
}
return nil
}
func (ri *routeIndex) loadRoutes(docsRoot string) error {
localePrefixes, err := discoverLocalePrefixes(docsRoot)
if err != nil {
return err
}
if ri.targetLang != "" {
localePrefixes[ri.targetLang] = struct{}{}
}
ri.localePrefixes = localePrefixes
return filepath.WalkDir(docsRoot, func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
return nil
}
if !isMarkdownFile(path) {
return nil
}
relPath, err := filepath.Rel(docsRoot, path)
if err != nil {
return err
}
relPath = normalizeSlashes(relPath)
firstSegment := firstPathSegment(relPath)
content, err := os.ReadFile(path)
if err != nil {
return err
}
permalinks := extractPermalinks(content)
switch {
case firstSegment == ri.targetLang:
trimmedRel := strings.TrimPrefix(relPath, firstSegment+"/")
addRouteCandidates(ri.localizedRoutes, trimmedRel, permalinks)
case ri.isLocalePrefix(firstSegment):
return nil
default:
addRouteCandidates(ri.sourceRoutes, relPath, permalinks)
}
return nil
})
}
func discoverLocalePrefixes(docsRoot string) (map[string]struct{}, error) {
entries, err := os.ReadDir(docsRoot)
if err != nil {
return nil, err
}
locales := map[string]struct{}{}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
name := entry.Name()
if !localeDirRe.MatchString(name) {
continue
}
locales[name] = struct{}{}
}
return locales, nil
}
func isMarkdownFile(path string) bool {
return strings.HasSuffix(path, ".md") || strings.HasSuffix(path, ".mdx")
}
func normalizeSlashes(path string) string {
return strings.ReplaceAll(path, "\\", "/")
}
func firstPathSegment(relPath string) string {
if relPath == "" {
return ""
}
parts := strings.SplitN(relPath, "/", 2)
return parts[0]
}
func addRouteCandidates(routes map[string]struct{}, relPath string, permalinks []string) {
base := strings.TrimSuffix(strings.TrimSuffix(relPath, ".md"), ".mdx")
if base != relPath {
addRoute(routes, normalizeRoute(base))
switch {
case base == "index":
addRoute(routes, "/")
case strings.HasSuffix(base, "/index"):
addRoute(routes, normalizeRoute(strings.TrimSuffix(base, "/index")))
}
}
for _, permalink := range permalinks {
addRoute(routes, normalizeRoute(permalink))
}
}
func addRoute(routes map[string]struct{}, route string) {
if route == "" {
return
}
routes[route] = struct{}{}
}
func extractPermalinks(content []byte) []string {
frontMatter, _ := splitFrontMatter(string(content))
if strings.TrimSpace(frontMatter) == "" {
return nil
}
data := map[string]any{}
if err := yaml.Unmarshal([]byte(frontMatter), &data); err != nil {
return nil
}
raw, ok := data["permalink"].(string)
if !ok {
return nil
}
permalink := strings.TrimSpace(raw)
if permalink == "" {
return nil
}
return []string{permalink}
}
func normalizeRoute(path string) string {
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return ""
}
stripped := strings.Trim(trimmed, "/")
if stripped == "" {
return "/"
}
return "/" + stripped
}
func (ri *routeIndex) localizeBodyLinks(body string) string {
if ri == nil || ri.targetLang == "" || strings.EqualFold(ri.targetLang, "en") {
return body
}
state := NewPlaceholderState(body)
placeholders := make([]string, 0, 8)
mapping := map[string]string{}
masked := maskMatches(body, fencedBacktickCodeBlock, state.Next, &placeholders, mapping)
masked = maskMatches(masked, fencedTildeCodeBlock, state.Next, &placeholders, mapping)
masked = maskMatches(masked, inlineCodeRe, state.Next, &placeholders, mapping)
masked = rewriteMarkdownLinkTargets(masked, ri)
masked = rewriteHrefTargets(masked, ri)
return unmaskMarkdown(masked, placeholders, mapping)
}
func rewriteMarkdownLinkTargets(text string, ri *routeIndex) string {
matches := markdownLinkTargetRe.FindAllStringSubmatchIndex(text, -1)
if len(matches) == 0 {
return text
}
var out strings.Builder
pos := 0
for _, span := range matches {
fullStart, targetStart, targetEnd := span[0], span[2], span[3]
if fullStart < pos {
continue
}
out.WriteString(text[pos:targetStart])
target := text[targetStart:targetEnd]
if text[fullStart] == '!' {
out.WriteString(target)
} else {
out.WriteString(ri.localizeURL(target))
}
pos = targetEnd
}
out.WriteString(text[pos:])
return out.String()
}
func rewriteHrefTargets(text string, ri *routeIndex) string {
text = rewriteCapturedTargets(text, hrefDoubleQuotedValueRe, 2, ri)
text = rewriteCapturedTargets(text, hrefSingleQuotedValueRe, 2, ri)
return text
}
func rewriteCapturedTargets(text string, re *regexp.Regexp, groupIndex int, ri *routeIndex) string {
matches := re.FindAllStringSubmatchIndex(text, -1)
if len(matches) == 0 {
return text
}
var out strings.Builder
pos := 0
for _, span := range matches {
start, end := span[groupIndex], span[groupIndex+1]
if start < pos || start < 0 || end < 0 {
continue
}
out.WriteString(text[pos:start])
out.WriteString(ri.localizeURL(text[start:end]))
pos = end
}
out.WriteString(text[pos:])
return out.String()
}
func (ri *routeIndex) localizeURL(raw string) string {
trimmed := strings.TrimSpace(raw)
if trimmed == "" {
return raw
}
if strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "//") {
return raw
}
if hasURLScheme(trimmed) {
return raw
}
pathPart, suffix := splitURLSuffix(trimmed)
if !strings.HasPrefix(pathPart, "/") {
return raw
}
normalized := normalizeRoute(pathPart)
if ri.routeHasLocalePrefix(normalized) {
return raw
}
canonical, ok := ri.resolveRoute(normalized)
if !ok {
return raw
}
if _, ok := ri.localizedRoutes[canonical]; !ok {
return raw
}
return prefixLocaleRoute(ri.targetLang, canonical) + suffix
}
func hasURLScheme(raw string) bool {
switch {
case strings.HasPrefix(raw, "http://"), strings.HasPrefix(raw, "https://"):
return true
case strings.HasPrefix(raw, "mailto:"), strings.HasPrefix(raw, "tel:"):
return true
case strings.HasPrefix(raw, "data:"), strings.HasPrefix(raw, "javascript:"):
return true
default:
return false
}
}
func splitURLSuffix(raw string) (string, string) {
index := strings.IndexAny(raw, "?#")
if index == -1 {
return raw, ""
}
return raw[:index], raw[index:]
}
func prefixLocaleRoute(lang, route string) string {
if route == "/" {
return "/" + lang
}
return "/" + lang + route
}
func (ri *routeIndex) routeHasLocalePrefix(route string) bool {
if route == "/" {
return false
}
firstSegment := strings.TrimPrefix(route, "/")
firstSegment = strings.SplitN(firstSegment, "/", 2)[0]
return ri.isLocalePrefix(firstSegment)
}
func (ri *routeIndex) isLocalePrefix(segment string) bool {
if segment == "" {
return false
}
_, ok := ri.localePrefixes[segment]
return ok
}
func (ri *routeIndex) resolveRoute(route string) (string, bool) {
current := normalizeRoute(route)
if current == "" {
return "", false
}
seen := map[string]struct{}{current: {}}
for {
next, ok := ri.redirects[current]
if !ok {
break
}
current = next
if _, ok := seen[current]; ok {
return "", false
}
seen[current] = struct{}{}
}
if current == "/" {
_, ok := ri.localizedRoutes[current]
return current, ok
}
if _, ok := ri.sourceRoutes[current]; ok {
return current, true
}
if _, ok := ri.localizedRoutes[current]; ok {
return current, true
}
return "", false
}

View file

@ -0,0 +1,172 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestLocalizeBodyLinks(t *testing.T) {
docsRoot := setupDocsTree(t)
routes, err := loadRouteIndex(docsRoot, "zh-CN")
if err != nil {
t.Fatalf("loadRouteIndex failed: %v", err)
}
tests := []struct {
name string
input string
want string
}{
{
name: "markdown link",
input: `See [Config](/gateway/configuration).`,
want: `See [Config](/zh-CN/gateway/configuration).`,
},
{
name: "href attribute",
input: `<Card href="/gateway/configuration" title="Config" />`,
want: `<Card href="/zh-CN/gateway/configuration" title="Config" />`,
},
{
name: "redirect source resolves to canonical localized page",
input: `See [Sandbox](/sandboxing).`,
want: `See [Sandbox](/zh-CN/gateway/sandboxing).`,
},
{
name: "fragment is preserved",
input: `See [Hooks](/gateway/configuration#hooks).`,
want: `See [Hooks](/zh-CN/gateway/configuration#hooks).`,
},
{
name: "images stay unchanged",
input: `![Diagram](/images/diagram.svg)`,
want: `![Diagram](/images/diagram.svg)`,
},
{
name: "already localized stays unchanged",
input: `See [Config](/zh-CN/gateway/configuration).`,
want: `See [Config](/zh-CN/gateway/configuration).`,
},
{
name: "missing localized page stays unchanged",
input: `See [FAQ](/help/faq).`,
want: `See [FAQ](/help/faq).`,
},
{
name: "permalink route localizes",
input: `See [Formal verification](/security/formal-verification).`,
want: `See [Formal verification](/zh-CN/security/formal-verification).`,
},
{
name: "inline code stays unchanged",
input: "Use `[Config](/gateway/configuration)` in examples.\n\n" +
"See [Config](/gateway/configuration).",
want: "Use `[Config](/gateway/configuration)` in examples.\n\n" +
"See [Config](/zh-CN/gateway/configuration).",
},
{
name: "fenced code block stays unchanged",
input: "```md\n[Config](/gateway/configuration)\n```\n\n" +
"See [Config](/gateway/configuration).",
want: "```md\n[Config](/gateway/configuration)\n```\n\n" +
"See [Config](/zh-CN/gateway/configuration).",
},
{
name: "inline code does not swallow later paragraphs",
input: strings.Join([]string{
"Use `channels.matrix.accounts` and `name`.",
"",
"See [Config](/gateway/configuration).",
"",
"Then review [Troubleshooting](/channels/troubleshooting).",
}, "\n"),
want: strings.Join([]string{
"Use `channels.matrix.accounts` and `name`.",
"",
"See [Config](/zh-CN/gateway/configuration).",
"",
"Then review [Troubleshooting](/zh-CN/channels/troubleshooting).",
}, "\n"),
},
{
name: "indented fenced code block does not swallow later paragraphs",
input: strings.Join([]string{
"1. Setup:",
"",
" ```bash",
" echo hi",
" ```",
"",
"Use `channels.matrix.accounts` and `name`.",
"",
"For triage: [/channels/troubleshooting](/channels/troubleshooting).",
"See [Config](/gateway/configuration).",
}, "\n"),
want: strings.Join([]string{
"1. Setup:",
"",
" ```bash",
" echo hi",
" ```",
"",
"Use `channels.matrix.accounts` and `name`.",
"",
"For triage: [/channels/troubleshooting](/zh-CN/channels/troubleshooting).",
"See [Config](/zh-CN/gateway/configuration).",
}, "\n"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := routes.localizeBodyLinks(tt.input)
if got != tt.want {
t.Fatalf("unexpected rewrite\nwant: %q\ngot: %q", tt.want, got)
}
})
}
}
func setupDocsTree(t *testing.T) string {
t.Helper()
root := t.TempDir()
writeFile(t, filepath.Join(root, "docs.json"), `{
"redirects": [
{ "source": "/sandboxing", "destination": "/gateway/sandboxing" }
]
}`)
files := map[string]string{
"index.md": "# Home\n",
"channels/troubleshooting.md": "# Troubleshooting\n",
"gateway/configuration.md": "# Config\n",
"gateway/sandboxing.md": "# Sandboxing\n",
"security/formal-verification.md": "---\npermalink: /security/formal-verification/\n---\n\n# Formal verification\n",
"help/faq.md": "# FAQ\n",
"zh-CN/index.md": "# Home\n",
"zh-CN/channels/troubleshooting.md": "# Troubleshooting\n",
"zh-CN/gateway/configuration.md": "# Config\n",
"zh-CN/gateway/sandboxing.md": "# Sandboxing\n",
"zh-CN/security/formal-verification.md": "---\npermalink: /security/formal-verification/\n---\n\n# Formal verification\n",
"ja-JP/index.md": "# Home\n",
}
for relPath, content := range files {
writeFile(t, filepath.Join(root, relPath), content)
}
return root
}
func writeFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir failed for %s: %v", path, err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("write failed for %s: %v", path, err)
}
}

View file

@ -0,0 +1,339 @@
package main
import (
"context"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"sync"
"time"
)
type docJob struct {
index int
path string
rel string
}
type docResult struct {
index int
rel string
output string
duration time.Duration
skipped bool
err error
}
type runConfig struct {
targetLang string
sourceLang string
docsRoot string
tmPath string
mode string
thinking string
overwrite bool
maxFiles int
parallel int
}
func main() {
var (
targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)")
sourceLang = flag.String("src", "en", "source language")
docsRoot = flag.String("docs", "docs", "docs root")
tmPath = flag.String("tm", "", "translation memory path")
mode = flag.String("mode", "segment", "translation mode (segment|doc)")
thinking = flag.String("thinking", "high", "thinking level (low|high)")
overwrite = flag.Bool("overwrite", false, "overwrite existing translations")
maxFiles = flag.Int("max", 0, "max files to process (0 = all)")
parallel = flag.Int("parallel", 1, "parallel workers for doc mode")
)
flag.Parse()
files := flag.Args()
if len(files) == 0 {
fatal(fmt.Errorf("no doc files provided"))
}
if err := runDocsI18N(context.Background(), runConfig{
targetLang: *targetLang,
sourceLang: *sourceLang,
docsRoot: *docsRoot,
tmPath: *tmPath,
mode: *mode,
thinking: *thinking,
overwrite: *overwrite,
maxFiles: *maxFiles,
parallel: *parallel,
}, files, func(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (docsTranslator, error) {
return NewPiTranslator(srcLang, tgtLang, glossary, thinking)
}); err != nil {
fatal(err)
}
}
func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslator docsTranslatorFactory) error {
if len(files) == 0 {
return fmt.Errorf("no doc files provided")
}
resolvedDocsRoot, err := filepath.Abs(cfg.docsRoot)
if err != nil {
return err
}
tmPath := cfg.tmPath
if tmPath == "" {
tmPath = filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("%s.tm.jsonl", cfg.targetLang))
}
glossaryPath := filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("glossary.%s.json", cfg.targetLang))
glossary, err := LoadGlossary(glossaryPath)
if err != nil {
return err
}
tm, err := LoadTranslationMemory(tmPath)
if err != nil {
return err
}
ordered, err := orderFiles(resolvedDocsRoot, files)
if err != nil {
return err
}
totalFiles := len(ordered)
preSkipped := 0
if cfg.mode == "doc" && !cfg.overwrite {
filtered, skipped, err := filterDocQueue(resolvedDocsRoot, cfg.targetLang, ordered)
if err != nil {
return err
}
ordered = filtered
preSkipped = skipped
}
if cfg.maxFiles > 0 && cfg.maxFiles < len(ordered) {
ordered = ordered[:cfg.maxFiles]
}
parallel := cfg.parallel
if parallel < 1 {
parallel = 1
}
log.SetFlags(log.LstdFlags)
start := time.Now()
processed := 0
skipped := 0
localizedFiles := []string{}
var runErr error
log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", cfg.mode, totalFiles, len(ordered), preSkipped, cfg.overwrite, cfg.thinking, parallel)
switch cfg.mode {
case "doc":
if parallel > 1 {
proc, skip, outputs, err := runDocParallel(ctx, ordered, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, parallel, glossary, cfg.thinking, newTranslator)
processed += proc
skipped += skip
localizedFiles = append(localizedFiles, outputs...)
if err != nil {
runErr = err
}
} else {
translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking)
if err != nil {
return err
}
defer translator.Close()
proc, skip, outputs, err := runDocSequential(ctx, ordered, translator, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite)
processed += proc
skipped += skip
localizedFiles = append(localizedFiles, outputs...)
if err != nil {
runErr = err
}
}
case "segment":
if parallel > 1 {
return fmt.Errorf("parallel processing is only supported in doc mode")
}
translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking)
if err != nil {
return err
}
defer translator.Close()
proc, outputs, err := runSegmentSequential(ctx, ordered, translator, tm, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang)
processed += proc
localizedFiles = append(localizedFiles, outputs...)
if err != nil {
runErr = err
}
default:
return fmt.Errorf("unknown mode: %s", cfg.mode)
}
if err := tm.Save(); err != nil && runErr == nil {
runErr = err
}
if err := postprocessLocalizedDocs(resolvedDocsRoot, cfg.targetLang, localizedFiles); err != nil && runErr == nil {
runErr = err
}
elapsed := time.Since(start).Round(time.Millisecond)
log.Printf("docs-i18n: completed processed=%d skipped=%d elapsed=%s", processed, skipped, elapsed)
return runErr
}
func runDocSequential(ctx context.Context, ordered []string, translator docsTranslator, docsRoot, srcLang, tgtLang string, overwrite bool) (int, int, []string, error) {
processed := 0
skipped := 0
outputs := []string{}
for index, file := range ordered {
relPath := resolveRelPath(docsRoot, file)
log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
start := time.Now()
skip, outputPath, err := processFileDoc(ctx, translator, docsRoot, file, srcLang, tgtLang, overwrite)
if err != nil {
return processed, skipped, outputs, err
}
if skip {
skipped++
log.Printf("docs-i18n: [%d/%d] skipped %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
} else {
processed++
outputs = append(outputs, outputPath)
log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
}
}
return processed, skipped, outputs, nil
}
func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tgtLang string, overwrite bool, parallel int, glossary []GlossaryEntry, thinking string, newTranslator docsTranslatorFactory) (int, int, []string, error) {
jobs := make(chan docJob)
results := make(chan docResult, len(ordered))
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var wg sync.WaitGroup
for worker := 0; worker < parallel; worker++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
translator, err := newTranslator(srcLang, tgtLang, glossary, thinking)
if err != nil {
results <- docResult{err: err}
return
}
defer translator.Close()
for job := range jobs {
if ctx.Err() != nil {
return
}
log.Printf("docs-i18n: [w%d %d/%d] start %s", workerID, job.index, len(ordered), job.rel)
start := time.Now()
skip, outputPath, err := processFileDoc(ctx, translator, docsRoot, job.path, srcLang, tgtLang, overwrite)
results <- docResult{
index: job.index,
rel: job.rel,
output: outputPath,
duration: time.Since(start),
skipped: skip,
err: err,
}
if err != nil {
cancel()
return
}
}
}(worker + 1)
}
go func() {
defer close(jobs)
for index, file := range ordered {
job := docJob{index: index + 1, path: file, rel: resolveRelPath(docsRoot, file)}
select {
case <-ctx.Done():
return
case jobs <- job:
}
}
}()
go func() {
wg.Wait()
close(results)
}()
processed := 0
skipped := 0
outputs := []string{}
var firstErr error
for result := range results {
if result.err != nil && firstErr == nil {
firstErr = result.err
}
if result.skipped {
skipped++
log.Printf("docs-i18n: [w* %d/%d] skipped %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
} else if result.err == nil {
processed++
outputs = append(outputs, result.output)
log.Printf("docs-i18n: [w* %d/%d] done %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
}
}
return processed, skipped, outputs, firstErr
}
func runSegmentSequential(ctx context.Context, ordered []string, translator docsTranslator, tm *TranslationMemory, docsRoot, srcLang, tgtLang string) (int, []string, error) {
processed := 0
outputs := []string{}
for index, file := range ordered {
relPath := resolveRelPath(docsRoot, file)
log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
start := time.Now()
_, outputPath, err := processFile(ctx, translator, tm, docsRoot, file, srcLang, tgtLang)
if err != nil {
return processed, outputs, err
}
processed++
outputs = append(outputs, outputPath)
log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
}
return processed, outputs, nil
}
func resolveRelPath(docsRoot, file string) string {
relPath := file
if _, rel, err := resolveDocsPath(docsRoot, file); err == nil {
relPath = rel
}
return relPath
}
func filterDocQueue(docsRoot, targetLang string, ordered []string) ([]string, int, error) {
pending := make([]string, 0, len(ordered))
skipped := 0
for _, file := range ordered {
absPath, relPath, err := resolveDocsPath(docsRoot, file)
if err != nil {
return nil, skipped, err
}
content, err := os.ReadFile(absPath)
if err != nil {
return nil, skipped, err
}
sourceHash := hashBytes(content)
outputPath := filepath.Join(docsRoot, targetLang, relPath)
skip, err := shouldSkipDoc(outputPath, sourceHash)
if err != nil {
return nil, skipped, err
}
if skip {
skipped++
continue
}
pending = append(pending, file)
}
return pending, skipped, nil
}

View file

@ -0,0 +1,108 @@
package main
import (
"context"
"path/filepath"
"strings"
"testing"
)
type fakeDocsTranslator struct{}
func (fakeDocsTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (fakeDocsTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
// Keep the fake translator deterministic so this test exercises the
// docs-i18n pipeline wiring and final link relocalization, not model output.
replaced := strings.NewReplacer(
"Gateway", "网关",
"See ", "参见 ",
).Replace(text)
return replaced, nil
}
func (fakeDocsTranslator) Close() {}
type invalidFrontmatterTranslator struct{}
func (invalidFrontmatterTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
return "<body>\n" + text + "\n</body>\n", nil
}
func (invalidFrontmatterTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
return text, nil
}
func (invalidFrontmatterTranslator) Close() {}
func TestRunDocsI18NRewritesFinalLocalizedPageLinks(t *testing.T) {
t.Parallel()
docsRoot := t.TempDir()
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
writeFile(t, filepath.Join(docsRoot, "gateway", "index.md"), stringsJoin(
"---",
"title: Gateway",
"---",
"",
"See [Troubleshooting](/gateway/troubleshooting).",
"",
"See [Example provider](/providers/example-provider).",
))
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
writeFile(t, filepath.Join(docsRoot, "providers", "example-provider.md"), "# Example provider\n")
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), "# 故障排除\n")
writeFile(t, filepath.Join(docsRoot, "zh-CN", "providers", "example-provider.md"), "# 示例 provider\n")
// This is the higher-level regression for the bug fixed in this PR:
// if the pipeline stops wiring postprocess through the main flow, the final
// localized output page will keep stale English-root links and this test fails.
err := runDocsI18N(context.Background(), runConfig{
targetLang: "zh-CN",
sourceLang: "en",
docsRoot: docsRoot,
mode: "doc",
thinking: "high",
overwrite: true,
parallel: 1,
}, []string{filepath.Join(docsRoot, "gateway", "index.md")}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
return fakeDocsTranslator{}, nil
})
if err != nil {
t.Fatalf("runDocsI18N failed: %v", err)
}
got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"))
expected := []string{
"参见 [Troubleshooting](/zh-CN/gateway/troubleshooting).",
"参见 [Example provider](/zh-CN/providers/example-provider).",
}
for _, want := range expected {
if !containsLine(got, want) {
t.Fatalf("expected final localized page link %q in output:\n%s", want, got)
}
}
}
func TestTranslateSnippetDoesNotCacheFallbackToSource(t *testing.T) {
t.Parallel()
tm := &TranslationMemory{entries: map[string]TMEntry{}}
source := "Gateway"
translated, err := translateSnippet(context.Background(), invalidFrontmatterTranslator{}, tm, "gateway/index.md:frontmatter:title", source, "en", "zh-CN")
if err != nil {
t.Fatalf("translateSnippet returned error: %v", err)
}
if translated != source {
t.Fatalf("expected fallback to source text, got %q", translated)
}
cacheKey := cacheKey(cacheNamespace(), "en", "zh-CN", "gateway/index.md:frontmatter:title", hashText(source))
if _, ok := tm.Get(cacheKey); ok {
t.Fatalf("expected fallback translation not to be cached")
}
}

View file

@ -0,0 +1,131 @@
package main
import (
"sort"
"strings"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/text"
)
func extractSegments(body, relPath string) ([]Segment, error) {
source := []byte(body)
r := text.NewReader(source)
md := goldmark.New(
goldmark.WithExtensions(extension.GFM),
)
doc := md.Parser().Parse(r)
segments := make([]Segment, 0, 128)
skipDepth := 0
var lastBlock ast.Node
err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
switch n.(type) {
case *ast.CodeBlock, *ast.FencedCodeBlock, *ast.CodeSpan, *ast.HTMLBlock, *ast.RawHTML:
if entering {
skipDepth++
} else {
skipDepth--
}
return ast.WalkContinue, nil
}
if !entering || skipDepth > 0 {
return ast.WalkContinue, nil
}
textNode, ok := n.(*ast.Text)
if !ok {
return ast.WalkContinue, nil
}
block := blockParent(textNode)
if block == nil {
return ast.WalkContinue, nil
}
textValue := string(textNode.Segment.Value(source))
if strings.TrimSpace(textValue) == "" {
return ast.WalkContinue, nil
}
start := textNode.Segment.Start
stop := textNode.Segment.Stop
if len(segments) > 0 && lastBlock == block {
last := &segments[len(segments)-1]
gap := string(source[last.Stop:start])
if strings.TrimSpace(gap) == "" {
last.Stop = stop
return ast.WalkContinue, nil
}
}
segments = append(segments, Segment{Start: start, Stop: stop})
lastBlock = block
return ast.WalkContinue, nil
})
if err != nil {
return nil, err
}
filtered := make([]Segment, 0, len(segments))
for _, seg := range segments {
textValue := string(source[seg.Start:seg.Stop])
trimmed := strings.TrimSpace(textValue)
if trimmed == "" {
continue
}
textHash := hashText(textValue)
segmentID := segmentID(relPath, textHash)
filtered = append(filtered, Segment{
Start: seg.Start,
Stop: seg.Stop,
Text: textValue,
TextHash: textHash,
SegmentID: segmentID,
})
}
sort.Slice(filtered, func(i, j int) bool {
return filtered[i].Start < filtered[j].Start
})
return filtered, nil
}
func blockParent(n ast.Node) ast.Node {
for node := n.Parent(); node != nil; node = node.Parent() {
if isTranslatableBlock(node) {
return node
}
}
return nil
}
func isTranslatableBlock(n ast.Node) bool {
switch n.(type) {
case *ast.Paragraph, *ast.Heading, *ast.ListItem:
return true
default:
return false
}
}
func applyTranslations(body string, segments []Segment) string {
if len(segments) == 0 {
return body
}
var out strings.Builder
last := 0
for _, seg := range segments {
if seg.Start < last {
continue
}
out.WriteString(body[last:seg.Start])
out.WriteString(seg.Translated)
last = seg.Stop
}
out.WriteString(body[last:])
return out.String()
}

View file

@ -0,0 +1,89 @@
package main
import (
"fmt"
"regexp"
"strings"
)
var (
inlineCodeRe = regexp.MustCompile("`[^`]+`")
angleLinkRe = regexp.MustCompile(`<https?://[^>]+>`)
linkURLRe = regexp.MustCompile(`\[[^\]]*\]\(([^)]+)\)`)
placeholderRe = regexp.MustCompile(`__OC_I18N_\d+__`)
)
func maskMarkdown(text string, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
masked := maskMatches(text, inlineCodeRe, nextPlaceholder, placeholders, mapping)
masked = maskMatches(masked, angleLinkRe, nextPlaceholder, placeholders, mapping)
masked = maskLinkURLs(masked, nextPlaceholder, placeholders, mapping)
return masked
}
func maskMatches(text string, re *regexp.Regexp, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
matches := re.FindAllStringIndex(text, -1)
if len(matches) == 0 {
return text
}
var out strings.Builder
pos := 0
for _, span := range matches {
start, end := span[0], span[1]
if start < pos {
continue
}
out.WriteString(text[pos:start])
placeholder := nextPlaceholder()
mapping[placeholder] = text[start:end]
*placeholders = append(*placeholders, placeholder)
out.WriteString(placeholder)
pos = end
}
out.WriteString(text[pos:])
return out.String()
}
func maskLinkURLs(text string, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
matches := linkURLRe.FindAllStringSubmatchIndex(text, -1)
if len(matches) == 0 {
return text
}
var out strings.Builder
pos := 0
for _, span := range matches {
fullStart := span[0]
urlStart, urlEnd := span[2], span[3]
if urlStart < 0 || urlEnd < 0 {
continue
}
if fullStart < pos {
continue
}
out.WriteString(text[pos:urlStart])
placeholder := nextPlaceholder()
mapping[placeholder] = text[urlStart:urlEnd]
*placeholders = append(*placeholders, placeholder)
out.WriteString(placeholder)
pos = urlEnd
}
out.WriteString(text[pos:])
return out.String()
}
func unmaskMarkdown(text string, placeholders []string, mapping map[string]string) string {
out := text
for _, placeholder := range placeholders {
original := mapping[placeholder]
out = strings.ReplaceAll(out, placeholder, original)
}
return out
}
func validatePlaceholders(text string, placeholders []string) error {
for _, placeholder := range placeholders {
if !strings.Contains(text, placeholder) {
return fmt.Errorf("placeholder missing: %s", placeholder)
}
}
return nil
}

View file

@ -0,0 +1,37 @@
package main
import (
"path/filepath"
"sort"
)
type orderedFile struct {
path string
rel string
}
func orderFiles(docsRoot string, files []string) ([]string, error) {
entries := make([]orderedFile, 0, len(files))
for _, file := range files {
abs, err := filepath.Abs(file)
if err != nil {
return nil, err
}
rel, err := filepath.Rel(docsRoot, abs)
if err != nil {
rel = abs
}
entries = append(entries, orderedFile{path: file, rel: rel})
}
if len(entries) == 0 {
return nil, nil
}
sort.Slice(entries, func(i, j int) bool {
return entries[i].rel < entries[j].rel
})
ordered := make([]string, 0, len(entries))
for _, entry := range entries {
ordered = append(ordered, entry.path)
}
return ordered, nil
}

View file

@ -0,0 +1,130 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
)
const (
envDocsPiExecutable = "OPENCLAW_DOCS_I18N_PI_EXECUTABLE"
envDocsPiArgs = "OPENCLAW_DOCS_I18N_PI_ARGS"
envDocsPiPackageVersion = "OPENCLAW_DOCS_I18N_PI_PACKAGE_VERSION"
envDocsPiOmitProvider = "OPENCLAW_DOCS_I18N_PI_OMIT_PROVIDER"
defaultPiPackageVersion = "0.58.3"
)
type docsPiCommand struct {
Executable string
Args []string
}
var (
materializedPiRuntimeMu sync.Mutex
materializedPiRuntimeCommand docsPiCommand
materializedPiRuntimeErr error
)
func resolveDocsPiCommand(ctx context.Context) (docsPiCommand, error) {
if executable := strings.TrimSpace(os.Getenv(envDocsPiExecutable)); executable != "" {
return docsPiCommand{
Executable: executable,
Args: strings.Fields(os.Getenv(envDocsPiArgs)),
}, nil
}
piPath, err := exec.LookPath("pi")
if err == nil && !shouldMaterializePiRuntime(piPath) {
return docsPiCommand{Executable: piPath}, nil
}
return ensureMaterializedPiRuntime(ctx)
}
func shouldMaterializePiRuntime(piPath string) bool {
realPath, err := filepath.EvalSymlinks(piPath)
if err != nil {
realPath = piPath
}
return strings.Contains(filepath.ToSlash(realPath), "/Projects/pi-mono/")
}
func ensureMaterializedPiRuntime(ctx context.Context) (docsPiCommand, error) {
materializedPiRuntimeMu.Lock()
defer materializedPiRuntimeMu.Unlock()
if materializedPiRuntimeErr == nil && materializedPiRuntimeCommand.Executable != "" {
return materializedPiRuntimeCommand, nil
}
runtimeDir, err := getMaterializedPiRuntimeDir()
if err != nil {
materializedPiRuntimeErr = err
return docsPiCommand{}, err
}
cliPath := filepath.Join(runtimeDir, "node_modules", "@mariozechner", "pi-coding-agent", "dist", "cli.js")
if _, err := os.Stat(cliPath); errors.Is(err, os.ErrNotExist) {
installCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
if err := os.MkdirAll(runtimeDir, 0o755); err != nil {
materializedPiRuntimeErr = err
return docsPiCommand{}, err
}
packageVersion := getMaterializedPiPackageVersion()
install := exec.CommandContext(
installCtx,
"npm",
"install",
"--silent",
"--no-audit",
"--no-fund",
fmt.Sprintf("@mariozechner/pi-coding-agent@%s", packageVersion),
)
install.Dir = runtimeDir
install.Env = os.Environ()
output, err := install.CombinedOutput()
if err != nil {
materializedPiRuntimeErr = fmt.Errorf("materialize pi runtime: %w (%s)", err, strings.TrimSpace(string(output)))
return docsPiCommand{}, materializedPiRuntimeErr
}
}
materializedPiRuntimeCommand = docsPiCommand{
Executable: "node",
Args: []string{cliPath},
}
materializedPiRuntimeErr = nil
return materializedPiRuntimeCommand, nil
}
func getMaterializedPiRuntimeDir() (string, error) {
cacheDir, err := os.UserCacheDir()
if err != nil {
cacheDir = os.TempDir()
}
return filepath.Join(cacheDir, "openclaw", "docs-i18n", "pi-runtime", getMaterializedPiPackageVersion()), nil
}
func getMaterializedPiPackageVersion() string {
if version := strings.TrimSpace(os.Getenv(envDocsPiPackageVersion)); version != "" {
return version
}
return defaultPiPackageVersion
}
func docsPiOmitProvider() bool {
switch strings.ToLower(strings.TrimSpace(os.Getenv(envDocsPiOmitProvider))) {
case "1", "true", "yes", "on":
return true
default:
return false
}
}

View file

@ -0,0 +1,351 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
)
type docsPiClientOptions struct {
SystemPrompt string
Thinking string
}
type docsPiClient struct {
process *exec.Cmd
stdin io.WriteCloser
stderr bytes.Buffer
events chan piEvent
promptLock sync.Mutex
closeOnce sync.Once
closed chan struct{}
requestID uint64
}
type piEvent struct {
Type string
Raw json.RawMessage
}
type agentEndPayload struct {
Type string `json:"type,omitempty"`
Messages []agentMessage `json:"messages"`
}
type rpcResponse struct {
ID string `json:"id,omitempty"`
Type string `json:"type"`
Command string `json:"command,omitempty"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
}
type agentMessage struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
StopReason string `json:"stopReason,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
}
type contentBlock struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
}
func startDocsPiClient(ctx context.Context, options docsPiClientOptions) (*docsPiClient, error) {
command, err := resolveDocsPiCommand(ctx)
if err != nil {
return nil, err
}
args := append([]string{}, command.Args...)
args = append(args, "--mode", "rpc")
if provider := docsPiProviderArg(); provider != "" && !docsPiOmitProvider() {
args = append(args, "--provider", provider)
}
args = append(args,
"--model", docsPiModelRef(),
"--thinking", options.Thinking,
"--no-session",
)
if strings.TrimSpace(options.SystemPrompt) != "" {
args = append(args, "--system-prompt", options.SystemPrompt)
}
process := exec.Command(command.Executable, args...)
agentDir, err := resolveDocsPiAgentDir()
if err != nil {
return nil, err
}
process.Env = append(os.Environ(), fmt.Sprintf("PI_CODING_AGENT_DIR=%s", agentDir))
stdin, err := process.StdinPipe()
if err != nil {
return nil, err
}
stdout, err := process.StdoutPipe()
if err != nil {
return nil, err
}
stderr, err := process.StderrPipe()
if err != nil {
return nil, err
}
client := &docsPiClient{
process: process,
stdin: stdin,
events: make(chan piEvent, 256),
closed: make(chan struct{}),
}
if err := process.Start(); err != nil {
return nil, err
}
go client.captureStderr(stderr)
go client.readStdout(stdout)
return client, nil
}
func (client *docsPiClient) Prompt(ctx context.Context, message string) (string, error) {
client.promptLock.Lock()
defer client.promptLock.Unlock()
command := map[string]string{
"type": "prompt",
"id": fmt.Sprintf("req-%d", atomic.AddUint64(&client.requestID, 1)),
"message": message,
}
payload, err := json.Marshal(command)
if err != nil {
return "", err
}
if _, err := client.stdin.Write(append(payload, '\n')); err != nil {
return "", err
}
for {
select {
case <-ctx.Done():
return "", ctx.Err()
case <-client.closed:
return "", errors.New("pi process closed")
case event, ok := <-client.events:
if !ok {
return "", errors.New("pi event stream closed")
}
if event.Type == "response" {
response, err := decodeRpcResponse(event.Raw)
if err != nil {
return "", err
}
if !response.Success {
if strings.TrimSpace(response.Error) == "" {
return "", errors.New("pi prompt failed")
}
return "", errors.New(strings.TrimSpace(response.Error))
}
continue
}
if event.Type == "agent_end" {
return extractTranslationResult(event.Raw)
}
}
}
}
func (client *docsPiClient) Stderr() string {
return client.stderr.String()
}
func (client *docsPiClient) Close() error {
client.closeOnce.Do(func() {
close(client.closed)
if client.stdin != nil {
_ = client.stdin.Close()
}
if client.process != nil && client.process.Process != nil {
_ = client.process.Process.Signal(syscall.SIGTERM)
}
done := make(chan struct{})
go func() {
if client.process != nil {
_ = client.process.Wait()
}
close(done)
}()
select {
case <-done:
case <-time.After(2 * time.Second):
if client.process != nil && client.process.Process != nil {
_ = client.process.Process.Kill()
}
}
})
return nil
}
func (client *docsPiClient) captureStderr(stderr io.Reader) {
_, _ = io.Copy(&client.stderr, stderr)
}
func (client *docsPiClient) readStdout(stdout io.Reader) {
defer close(client.events)
reader := bufio.NewReader(stdout)
for {
line, err := reader.ReadBytes('\n')
line = bytes.TrimSpace(line)
if len(line) > 0 {
var envelope struct {
Type string `json:"type"`
}
if json.Unmarshal(line, &envelope) == nil && envelope.Type != "" {
select {
case client.events <- piEvent{Type: envelope.Type, Raw: append([]byte{}, line...)}:
case <-client.closed:
return
}
}
}
if err != nil {
return
}
}
}
func extractTranslationResult(raw json.RawMessage) (string, error) {
var payload agentEndPayload
if err := json.Unmarshal(raw, &payload); err != nil {
return "", err
}
for index := len(payload.Messages) - 1; index >= 0; index-- {
message := payload.Messages[index]
if message.Role != "assistant" {
continue
}
if message.ErrorMessage != "" || isTerminalPiStopReason(message.StopReason) {
text, _ := extractContentText(message.Content)
return "", formatPiAgentError(message, text)
}
text, err := extractContentText(message.Content)
if err != nil {
return "", err
}
return text, nil
}
return "", errors.New("assistant message not found")
}
func isTerminalPiStopReason(stopReason string) bool {
switch strings.ToLower(strings.TrimSpace(stopReason)) {
case "error", "terminated", "cancelled", "canceled", "aborted":
return true
default:
return false
}
}
func formatPiAgentError(message agentMessage, assistantText string) error {
parts := []string{}
if msg := strings.TrimSpace(message.ErrorMessage); msg != "" {
parts = append(parts, msg)
}
if stop := strings.TrimSpace(message.StopReason); stop != "" {
parts = append(parts, "stopReason="+stop)
}
if preview := previewPiAssistantText(assistantText); preview != "" {
parts = append(parts, "assistant="+preview)
}
if len(parts) == 0 {
parts = append(parts, "unknown error")
}
return fmt.Errorf("pi error: %s", strings.Join(parts, "; "))
}
func previewPiAssistantText(text string) string {
trimmed := strings.TrimSpace(text)
if trimmed == "" {
return ""
}
trimmed = strings.ReplaceAll(trimmed, "\n", " ")
trimmed = strings.Join(strings.Fields(trimmed), " ")
const limit = 160
if len(trimmed) <= limit {
return trimmed
}
return trimmed[:limit] + "..."
}
func extractContentText(content json.RawMessage) (string, error) {
trimmed := strings.TrimSpace(string(content))
if trimmed == "" {
return "", nil
}
if strings.HasPrefix(trimmed, "\"") {
var text string
if err := json.Unmarshal(content, &text); err != nil {
return "", err
}
return text, nil
}
var blocks []contentBlock
if err := json.Unmarshal(content, &blocks); err != nil {
return "", err
}
var parts []string
for _, block := range blocks {
if block.Type == "text" && block.Text != "" {
parts = append(parts, block.Text)
}
}
return strings.Join(parts, ""), nil
}
func decodeRpcResponse(raw json.RawMessage) (rpcResponse, error) {
var response rpcResponse
if err := json.Unmarshal(raw, &response); err != nil {
return rpcResponse{}, err
}
return response, nil
}
func getDocsPiAgentDir() (string, error) {
cacheDir, err := os.UserCacheDir()
if err != nil {
cacheDir = os.TempDir()
}
dir := filepath.Join(cacheDir, "openclaw", "docs-i18n", "agent")
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
return dir, nil
}
func resolveDocsPiAgentDir() (string, error) {
if override := strings.TrimSpace(os.Getenv("PI_CODING_AGENT_DIR")); override != "" {
if err := os.MkdirAll(override, 0o700); err != nil {
return "", err
}
return override, nil
}
return getDocsPiAgentDir()
}

View file

@ -0,0 +1,84 @@
package main
import (
"strings"
"testing"
)
func TestExtractTranslationResultIncludesStopReasonAndPreview(t *testing.T) {
t.Parallel()
raw := []byte(`{
"type":"agent_end",
"messages":[
{
"role":"assistant",
"stopReason":"terminated",
"content":[
{"type":"text","text":"provider disconnected while streaming the translation chunk"}
]
}
]
}`)
_, err := extractTranslationResult(raw)
if err == nil {
t.Fatal("expected error")
}
message := err.Error()
for _, want := range []string{
"pi error:",
"stopReason=terminated",
"assistant=provider disconnected while streaming the translation chunk",
} {
if !strings.Contains(message, want) {
t.Fatalf("expected %q in error, got %q", want, message)
}
}
}
func TestPreviewPiAssistantTextTruncatesAndFlattensWhitespace(t *testing.T) {
t.Parallel()
input := "line one\n\nline two\tline three " + strings.Repeat("x", 200)
preview := previewPiAssistantText(input)
if strings.Contains(preview, "\n") {
t.Fatalf("expected flattened whitespace, got %q", preview)
}
if !strings.HasPrefix(preview, "line one line two line three ") {
t.Fatalf("unexpected preview prefix: %q", preview)
}
if !strings.HasSuffix(preview, "...") {
t.Fatalf("expected truncation suffix, got %q", preview)
}
}
func TestExtractTranslationResultReturnsPiErrorBeforeDecodingStructuredErrorContent(t *testing.T) {
t.Parallel()
raw := []byte(`{
"type":"agent_end",
"messages":[
{
"role":"assistant",
"stopReason":"terminated",
"content":{"type":"error","message":"provider disconnected"}
}
]
}`)
_, err := extractTranslationResult(raw)
if err == nil {
t.Fatal("expected error")
}
message := err.Error()
if !strings.Contains(message, "pi error:") {
t.Fatalf("expected normalized pi error, got %q", message)
}
if !strings.Contains(message, "stopReason=terminated") {
t.Fatalf("expected stopReason in error, got %q", message)
}
if strings.Contains(message, "cannot unmarshal") {
t.Fatalf("expected terminal pi error before decode failure, got %q", message)
}
}

View file

@ -0,0 +1,30 @@
package main
import (
"fmt"
)
type PlaceholderState struct {
counter int
used map[string]struct{}
}
func NewPlaceholderState(text string) *PlaceholderState {
used := map[string]struct{}{}
for _, hit := range placeholderRe.FindAllString(text, -1) {
used[hit] = struct{}{}
}
return &PlaceholderState{counter: 900000, used: used}
}
func (s *PlaceholderState) Next() string {
for {
candidate := fmt.Sprintf("__OC_I18N_%d__", s.counter)
s.counter++
if _, ok := s.used[candidate]; ok {
continue
}
s.used[candidate] = struct{}{}
return candidate
}
}

View file

@ -0,0 +1,279 @@
package main
import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"time"
"gopkg.in/yaml.v3"
)
func processFile(ctx context.Context, translator docsTranslator, tm *TranslationMemory, docsRoot, filePath, srcLang, tgtLang string) (bool, string, error) {
absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
if err != nil {
return false, "", err
}
content, err := os.ReadFile(absPath)
if err != nil {
return false, "", err
}
frontMatter, body := splitFrontMatter(string(content))
frontData := map[string]any{}
if frontMatter != "" {
if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
return false, "", fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
}
}
if err := translateFrontMatter(ctx, translator, tm, frontData, relPath, srcLang, tgtLang); err != nil {
return false, "", err
}
body, err = translateHTMLBlocks(ctx, translator, body, srcLang, tgtLang)
if err != nil {
return false, "", err
}
segments, err := extractSegments(body, relPath)
if err != nil {
return false, "", err
}
namespace := cacheNamespace()
for i := range segments {
seg := &segments[i]
seg.CacheKey = cacheKey(namespace, srcLang, tgtLang, seg.SegmentID, seg.TextHash)
if entry, ok := tm.Get(seg.CacheKey); ok {
seg.Translated = entry.Translated
continue
}
translated, err := translator.Translate(ctx, seg.Text, srcLang, tgtLang)
if err != nil {
return false, "", fmt.Errorf("translate failed (%s): %w", relPath, err)
}
seg.Translated = translated
entry := TMEntry{
CacheKey: seg.CacheKey,
SegmentID: seg.SegmentID,
SourcePath: relPath,
TextHash: seg.TextHash,
Text: seg.Text,
Translated: translated,
Provider: docsPiProvider(),
Model: docsPiModel(),
SrcLang: srcLang,
TgtLang: tgtLang,
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
tm.Put(entry)
}
translatedBody := applyTranslations(body, segments)
updatedFront, err := encodeFrontMatter(frontData, relPath, content)
if err != nil {
return false, "", err
}
outputPath := filepath.Join(docsRoot, tgtLang, relPath)
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
return false, "", err
}
output := updatedFront + translatedBody
return false, outputPath, os.WriteFile(outputPath, []byte(output), 0o644)
}
func splitFrontMatter(content string) (string, string) {
if !strings.HasPrefix(content, "---") {
return "", content
}
lines := strings.Split(content, "\n")
if len(lines) < 2 {
return "", content
}
endIndex := -1
for i := 1; i < len(lines); i++ {
if strings.TrimSpace(lines[i]) == "---" {
endIndex = i
break
}
}
if endIndex == -1 {
return "", content
}
front := strings.Join(lines[1:endIndex], "\n")
body := strings.Join(lines[endIndex+1:], "\n")
if strings.HasPrefix(body, "\n") {
body = body[1:]
}
return front, body
}
func encodeFrontMatter(frontData map[string]any, relPath string, source []byte) (string, error) {
if frontData == nil {
frontData = map[string]any{}
}
frontData["x-i18n"] = map[string]any{
"source_path": relPath,
"source_hash": hashBytes(source),
"provider": docsPiProvider(),
"model": docsPiModel(),
"workflow": workflowVersion,
"generated_at": time.Now().UTC().Format(time.RFC3339),
}
encoded, err := yaml.Marshal(frontData)
if err != nil {
return "", err
}
return fmt.Sprintf("---\n%s---\n\n", string(encoded)), nil
}
func translateFrontMatter(ctx context.Context, translator docsTranslator, tm *TranslationMemory, data map[string]any, relPath, srcLang, tgtLang string) error {
if len(data) == 0 {
return nil
}
if summary, ok := data["summary"].(string); ok {
if docsI18nVerboseLogs() {
log.Printf("docs-i18n: frontmatter start %s field=summary bytes=%d", relPath, len(summary))
}
translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:summary", summary, srcLang, tgtLang)
if err != nil {
return err
}
if docsI18nVerboseLogs() {
log.Printf("docs-i18n: frontmatter done %s field=summary out_bytes=%d", relPath, len(translated))
}
data["summary"] = translated
}
if title, ok := data["title"].(string); ok {
if docsI18nVerboseLogs() {
log.Printf("docs-i18n: frontmatter start %s field=title bytes=%d", relPath, len(title))
}
translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:title", title, srcLang, tgtLang)
if err != nil {
return err
}
if docsI18nVerboseLogs() {
log.Printf("docs-i18n: frontmatter done %s field=title out_bytes=%d", relPath, len(translated))
}
data["title"] = translated
}
if readWhen, ok := data["read_when"].([]any); ok {
translated := make([]any, 0, len(readWhen))
for idx, item := range readWhen {
textValue, ok := item.(string)
if !ok {
translated = append(translated, item)
continue
}
if docsI18nVerboseLogs() {
log.Printf("docs-i18n: frontmatter start %s field=read_when[%d] bytes=%d", relPath, idx, len(textValue))
}
value, err := translateSnippet(ctx, translator, tm, fmt.Sprintf("%s:frontmatter:read_when:%d", relPath, idx), textValue, srcLang, tgtLang)
if err != nil {
return err
}
if docsI18nVerboseLogs() {
log.Printf("docs-i18n: frontmatter done %s field=read_when[%d] out_bytes=%d", relPath, idx, len(value))
}
translated = append(translated, value)
}
data["read_when"] = translated
}
return nil
}
func docsI18nVerboseLogs() bool {
value := strings.TrimSpace(os.Getenv("OPENCLAW_DOCS_I18N_VERBOSE_LOGS"))
if value == "" {
return false
}
switch strings.ToLower(value) {
case "1", "true", "yes", "on", "debug", "verbose":
return true
default:
return false
}
}
func translateSnippet(ctx context.Context, translator docsTranslator, tm *TranslationMemory, segmentID, textValue, srcLang, tgtLang string) (string, error) {
if strings.TrimSpace(textValue) == "" {
return textValue, nil
}
namespace := cacheNamespace()
textHash := hashText(textValue)
ck := cacheKey(namespace, srcLang, tgtLang, segmentID, textHash)
if entry, ok := tm.Get(ck); ok {
return entry.Translated, nil
}
translated, err := translator.Translate(ctx, textValue, srcLang, tgtLang)
if err != nil {
return "", err
}
shouldCache := true
if validationErr := validateFrontmatterScalarTranslation(textValue, translated); validationErr != nil {
log.Printf("docs-i18n: frontmatter fallback %s reason=%v", segmentID, validationErr)
translated = textValue
shouldCache = false
}
entry := TMEntry{
CacheKey: ck,
SegmentID: segmentID,
SourcePath: segmentID,
TextHash: textHash,
Text: textValue,
Translated: translated,
Provider: docsPiProvider(),
Model: docsPiModel(),
SrcLang: srcLang,
TgtLang: tgtLang,
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
if shouldCache {
tm.Put(entry)
}
return translated, nil
}
func validateFrontmatterScalarTranslation(source, translated string) error {
trimmed := strings.TrimSpace(translated)
if trimmed == "" {
return fmt.Errorf("empty translation")
}
lower := strings.ToLower(trimmed)
if strings.Contains(lower, "<frontmatter>") || strings.Contains(lower, "</frontmatter>") || strings.Contains(lower, "<body>") || strings.Contains(lower, "</body>") {
return fmt.Errorf("tagged document wrapper detected")
}
if strings.Contains(trimmed, "[[[FM_") {
return fmt.Errorf("frontmatter marker leaked into scalar translation")
}
if strings.Contains(trimmed, "\n---\n") || strings.HasPrefix(trimmed, "---\n") {
return fmt.Errorf("yaml document boundary detected")
}
if !strings.Contains(source, "\n") && strings.Count(trimmed, "\n") >= 3 {
return fmt.Errorf("unexpected multiline expansion")
}
sourceLen := len(strings.TrimSpace(source))
translatedLen := len(trimmed)
if sourceLen > 0 {
limit := sourceLen*8 + 256
if limit < 512 {
limit = 512
}
if translatedLen > limit {
return fmt.Errorf("unexpected size expansion source=%d translated=%d", sourceLen, translatedLen)
}
}
for _, key := range []string{"title:", "summary:", "read_when:"} {
if strings.Contains(lower, "\n"+key) || strings.HasPrefix(lower, key) {
return fmt.Errorf("frontmatter key leaked into scalar translation")
}
}
return nil
}

View file

@ -0,0 +1,163 @@
package main
import (
"fmt"
"strings"
)
func prettyLanguageLabel(lang string) string {
trimmed := strings.TrimSpace(lang)
if trimmed == "" {
return lang
}
switch {
case strings.EqualFold(trimmed, "en"):
return "English"
case strings.EqualFold(trimmed, "zh-CN"):
return "Simplified Chinese"
case strings.EqualFold(trimmed, "ja-JP"):
return "Japanese"
case strings.EqualFold(trimmed, "uk"):
return "Ukrainian"
default:
return trimmed
}
}
func translationPrompt(srcLang, tgtLang string, glossary []GlossaryEntry) string {
srcLabel := prettyLanguageLabel(srcLang)
tgtLabel := prettyLanguageLabel(tgtLang)
glossaryBlock := buildGlossaryPrompt(glossary)
switch {
case strings.EqualFold(tgtLang, "zh-CN"):
// Keep this prompt as stable as possible; it has lots of tuning baked into the wording.
return strings.TrimSpace(fmt.Sprintf(zhCNPromptTemplate, srcLabel, tgtLabel, glossaryBlock))
case strings.EqualFold(tgtLang, "ja-JP"):
return strings.TrimSpace(fmt.Sprintf(jaJPPromptTemplate, srcLabel, tgtLabel, glossaryBlock))
default:
return strings.TrimSpace(fmt.Sprintf(genericPromptTemplate, srcLabel, tgtLabel, glossaryBlock))
}
}
const zhCNPromptTemplate = `You are a translation function, not a chat assistant.
Translate from %s to %s.
Rules:
- Output ONLY the translated text. No preamble, no questions, no commentary.
- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
- All prose must be Chinese. If any English sentence remains outside code/URLs/product names, it is wrong.
- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
- Translate only the contents inside those tags.
- Preserve YAML structure inside <frontmatter>; translate only values.
- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
- Translate headings/labels like "Exit codes" and "Optional scripts".
- Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
- Preserve HTML tags and attributes exactly.
- Do not translate code spans/blocks, config keys, CLI flags, or env vars.
- Do not alter URLs or anchors.
- Preserve placeholders exactly: __OC_I18N_####__.
- Do not remove, reorder, or summarize content.
- Use fluent, idiomatic technical Chinese; avoid slang or jokes.
- Use neutral documentation tone; prefer “你/你的”, avoid “您/您的”.
- Glossary terms are mandatory. When a source term matches a glossary entry, use
the glossary target exactly, including headings, link labels, and short
UI-style labels.
- If a glossary target is identical to the source text, preserve that term in
English exactly as written.
- Insert a space between Latin characters and CJK text (W3C CLREQ), e.g., “Gateway 网关”, “Skills 配置”.
- Use Chinese quotation marks “ and ” for Chinese prose; keep ASCII quotes inside code spans/blocks or literal CLI/keys.
- Keep product names in English: OpenClaw, Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
- For the OpenClaw Gateway, use “Gateway 网关”.
- Keep these terms in English: Skills, local loopback, Tailscale.
- Never output an empty response; if unsure, return the source text unchanged.
%s
If the input is empty, output empty.
If the input contains only placeholders, output it unchanged.`
const jaJPPromptTemplate = `You are a translation function, not a chat assistant.
Translate from %s to %s.
Rules:
- Output ONLY the translated text. No preamble, no questions, no commentary.
- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
- All prose must be Japanese. If any English sentence remains outside code/URLs/product names, it is wrong.
- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
- Translate only the contents inside those tags.
- Preserve YAML structure inside <frontmatter>; translate only values.
- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
- Translate headings/labels like "Exit codes" and "Optional scripts".
- Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
- Preserve HTML tags and attributes exactly.
- Do not translate code spans/blocks, config keys, CLI flags, or env vars.
- Do not alter URLs or anchors.
- Preserve placeholders exactly: __OC_I18N_####__.
- Do not remove, reorder, or summarize content.
- Use fluent, idiomatic technical Japanese; avoid slang or jokes.
- Use neutral documentation tone; avoid overly formal honorifics (e.g., avoid “〜でございます”).
- Glossary terms are mandatory. When a source term matches a glossary entry, use
the glossary target exactly, including headings, link labels, and short
UI-style labels.
- If a glossary target is identical to the source text, preserve that term in
English exactly as written.
- Use Japanese quotation marks 「 and 」 for Japanese prose; keep ASCII quotes inside code spans/blocks or literal CLI/keys.
- Do not add or remove spacing around Latin text just because it borders Japanese; keep spacing stable unless required by Japanese grammar.
- Keep product names in English: OpenClaw, Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
- Keep these terms in English: Skills, local loopback, Tailscale.
- Never output an empty response; if unsure, return the source text unchanged.
%s
If the input is empty, output empty.
If the input contains only placeholders, output it unchanged.`
const genericPromptTemplate = `You are a translation function, not a chat assistant.
Translate from %s to %s.
Rules:
- Output ONLY the translated text. No preamble, no questions, no commentary.
- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
- If any English sentence remains outside code/URLs/product names, it is likely wrong.
- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
- Translate only the contents inside those tags.
- Preserve YAML structure inside <frontmatter>; translate only values.
- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
- Translate headings/labels like "Exit codes" and "Optional scripts".
- Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
- Preserve HTML tags and attributes exactly.
- Do not translate code spans/blocks, config keys, CLI flags, or env vars.
- Do not alter URLs or anchors.
- Preserve placeholders exactly: __OC_I18N_####__.
- Do not remove, reorder, or summarize content.
- Use fluent, idiomatic technical language in the target language; avoid slang or jokes.
- Use neutral documentation tone.
- Glossary terms are mandatory. When a source term matches a glossary entry, use
the glossary target exactly, including headings, link labels, and short
UI-style labels.
- If a glossary target is identical to the source text, preserve that term in
English exactly as written.
- Keep product names in English: OpenClaw, Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
- Keep these terms in English: Skills, local loopback, Tailscale.
- Never output an empty response; if unsure, return the source text unchanged.
%s
If the input is empty, output empty.
If the input contains only placeholders, output it unchanged.`
func buildGlossaryPrompt(glossary []GlossaryEntry) string {
if len(glossary) == 0 {
return ""
}
var lines []string
lines = append(lines, "Required terminology (use exactly when the source term matches):")
for _, entry := range glossary {
if entry.Source == "" || entry.Target == "" {
continue
}
lines = append(lines, fmt.Sprintf("- %s -> %s", entry.Source, entry.Target))
}
return strings.Join(lines, "\n")
}

View file

@ -0,0 +1,40 @@
package main
import (
"os"
)
func postprocessLocalizedDocs(docsRoot, targetLang string, localizedFiles []string) error {
if targetLang == "" || targetLang == "en" || len(localizedFiles) == 0 {
return nil
}
routes, err := loadRouteIndex(docsRoot, targetLang)
if err != nil {
return err
}
for _, path := range localizedFiles {
content, err := os.ReadFile(path)
if err != nil {
return err
}
frontMatter, body := splitFrontMatter(string(content))
rewrittenBody := routes.localizeBodyLinks(body)
if rewrittenBody == body {
continue
}
output := rewrittenBody
if frontMatter != "" {
output = "---\n" + frontMatter + "\n---\n\n" + rewrittenBody
}
if err := os.WriteFile(path, []byte(output), 0o644); err != nil {
return err
}
}
return nil
}

View file

@ -0,0 +1,220 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestPostprocessLocalizedDocsFixesStaleLinksAfterLaterPagesExist(t *testing.T) {
t.Parallel()
docsRoot := t.TempDir()
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
writeFile(t, filepath.Join(docsRoot, "gateway", "index.md"), "# Gateway\n")
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"), stringsJoin(
"---",
"title: 网关",
"x-i18n:",
" source_hash: test",
"---",
"",
"See [Troubleshooting](/gateway/troubleshooting).",
))
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), stringsJoin(
"---",
"title: 故障排除",
"x-i18n:",
" source_hash: test",
"---",
"",
"# 故障排除",
))
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{
filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"),
filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"),
}); err != nil {
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
}
got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"))
if !strings.Contains(got, "---\ntitle: 网关\nx-i18n:\n source_hash: test\n---\n\n") {
t.Fatalf("front matter corrupted after rewrite:\n%s", got)
}
want := "See [Troubleshooting](/zh-CN/gateway/troubleshooting)."
if !containsLine(got, want) {
t.Fatalf("expected rewritten localized link %q in output:\n%s", want, got)
}
}
func TestPostprocessLocalizedDocsRewritesPublishedPageLinksForEachLocale(t *testing.T) {
t.Parallel()
tests := []struct {
name string
lang string
title string
wantPrefix string
}{
{name: "zh-CN", lang: "zh-CN", title: "网关", wantPrefix: "/zh-CN"},
{name: "ja-JP", lang: "ja-JP", title: "ゲートウェイ", wantPrefix: "/ja-JP"},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
docsRoot := t.TempDir()
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
writeFile(t, filepath.Join(docsRoot, "gateway", "index.md"), "# Gateway\n")
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
writeFile(t, filepath.Join(docsRoot, "providers", "example-provider.md"), "# Example provider\n")
writeFile(t, filepath.Join(docsRoot, tt.lang, "gateway", "troubleshooting.md"), "# Localized troubleshooting\n")
writeFile(t, filepath.Join(docsRoot, tt.lang, "providers", "example-provider.md"), "# Localized example provider\n")
pagePath := filepath.Join(docsRoot, tt.lang, "gateway", "index.md")
writeFile(t, pagePath, stringsJoin(
"---",
"title: "+tt.title,
"x-i18n:",
" source_hash: test",
"---",
"",
"See [Troubleshooting](/gateway/troubleshooting).",
"",
"See [Example provider](/providers/example-provider).",
"",
`<Card href="/gateway/troubleshooting" title="Troubleshooting" />`,
`<Card href="`+tt.wantPrefix+`/providers/example-provider" title="Example provider" />`,
))
if err := postprocessLocalizedDocs(docsRoot, tt.lang, []string{pagePath}); err != nil {
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
}
got := mustReadFile(t, pagePath)
expectedLinks := []string{
"See [Troubleshooting](" + tt.wantPrefix + "/gateway/troubleshooting).",
"See [Example provider](" + tt.wantPrefix + "/providers/example-provider).",
`<Card href="` + tt.wantPrefix + `/gateway/troubleshooting" title="Troubleshooting" />`,
`<Card href="` + tt.wantPrefix + `/providers/example-provider" title="Example provider" />`,
}
for _, want := range expectedLinks {
if !containsLine(got, want) {
t.Fatalf("expected rewritten link %q in output:\n%s", want, got)
}
}
})
}
}
func TestPostprocessLocalizedDocsOnlyTouchesScopedFiles(t *testing.T) {
t.Parallel()
docsRoot := t.TempDir()
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), "# 故障排除\n")
scopedPath := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
unscopedPath := filepath.Join(docsRoot, "zh-CN", "help", "index.md")
writeFile(t, scopedPath, stringsJoin(
"---",
"title: 网关",
"---",
"",
"See [Troubleshooting](/gateway/troubleshooting).",
))
writeFile(t, unscopedPath, stringsJoin(
"---",
"title: 帮助",
"---",
"",
"See [Troubleshooting](/gateway/troubleshooting).",
))
beforeUnscoped := mustReadFile(t, unscopedPath)
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{scopedPath}); err != nil {
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
}
gotScoped := mustReadFile(t, scopedPath)
if !containsLine(gotScoped, "See [Troubleshooting](/zh-CN/gateway/troubleshooting).") {
t.Fatalf("expected scoped file rewrite, got:\n%s", gotScoped)
}
afterUnscoped := mustReadFile(t, unscopedPath)
if afterUnscoped != beforeUnscoped {
t.Fatalf("expected unscoped file to remain unchanged\nbefore:\n%s\nafter:\n%s", beforeUnscoped, afterUnscoped)
}
}
func TestPostprocessLocalizedDocsContinuesAfterUnchangedFile(t *testing.T) {
t.Parallel()
docsRoot := t.TempDir()
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), "# 故障排除\n")
unchangedPath := filepath.Join(docsRoot, "zh-CN", "gateway", "already-localized.md")
needsRewritePath := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
writeFile(t, unchangedPath, stringsJoin(
"---",
"title: 已本地化",
"---",
"",
"See [Troubleshooting](/zh-CN/gateway/troubleshooting).",
))
writeFile(t, needsRewritePath, stringsJoin(
"---",
"title: 网关",
"---",
"",
"See [Troubleshooting](/gateway/troubleshooting).",
))
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{unchangedPath, needsRewritePath}); err != nil {
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
}
got := mustReadFile(t, needsRewritePath)
if !containsLine(got, "See [Troubleshooting](/zh-CN/gateway/troubleshooting).") {
t.Fatalf("expected later file rewrite after unchanged file, got:\n%s", got)
}
}
func stringsJoin(lines ...string) string {
result := ""
for i, line := range lines {
if i > 0 {
result += "\n"
}
result += line
}
return result
}
func mustReadFile(t *testing.T, path string) string {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read failed for %s: %v", path, err)
}
return string(data)
}
func containsLine(text, want string) bool {
for _, line := range strings.Split(text, "\n") {
if line == want {
return true
}
}
return false
}

View file

@ -0,0 +1,11 @@
package main
type Segment struct {
Start int
Stop int
Text string
TextHash string
SegmentID string
Translated string
CacheKey string
}

View file

@ -0,0 +1,24 @@
{
"name": "fenced singleton retries after malformed raw output",
"mode": "doc_body_chunked",
"rel_path": "gateway/configuration-reference.md",
"source_file": "source.txt",
"expected_output_contains": [
"Translated line 01",
"Translated line 02",
"Translated line 03",
"Translated line 04"
],
"expected_output_not_contains": ["Line 01", "Line 02", "Line 03", "Line 04"],
"rules": [
{
"method": "raw",
"match_all": ["Line 01", "Line 04"],
"response_file": "raw-malformed.txt"
},
{
"method": "raw",
"replace_pairs": [{ "from": "Line ", "to": "Translated line " }]
}
]
}

View file

@ -0,0 +1,6 @@
```md
Line 01
Line 02
Line 03
Line 04

View file

@ -0,0 +1,7 @@
```md
Line 01
Line 02
Line 03
Line 04
```

View file

@ -0,0 +1,14 @@
{
"name": "frontmatter scalar falls back instead of keeping tagged wrapper",
"mode": "frontmatter_scalar",
"rel_path": "install/fly.md",
"source_file": "source.txt",
"expected_file": "expected.txt",
"rules": [
{
"method": "masked",
"match_all": ["Deploying OpenClaw on Fly.io"],
"response_file": "masked-tagged-wrapper.txt"
}
]
}

View file

@ -0,0 +1 @@
Deploying OpenClaw on Fly.io

View file

@ -0,0 +1,7 @@
<frontmatter>
title: Fly.io
</frontmatter>
<body>
# Fly.io 部署
</body>

View file

@ -0,0 +1 @@
Deploying OpenClaw on Fly.io

View file

@ -0,0 +1,25 @@
{
"name": "protocol leak retries on smaller chunks",
"mode": "doc_body_chunked",
"rel_path": "gateway/configuration-reference.md",
"source_file": "source.txt",
"expected_file": "expected.txt",
"expected_output_not_contains": ["<frontmatter>", "<body>", "[[[FM_"],
"rules": [
{
"method": "raw",
"match_all": ["First chunk", "Second chunk"],
"response_file": "raw-leaked.txt"
},
{
"method": "raw",
"match_all": ["First chunk"],
"response_file": "raw-first.txt"
},
{
"method": "raw",
"match_all": ["Second chunk"],
"response_file": "raw-second.txt"
}
]
}

View file

@ -0,0 +1,4 @@
First translated
Second translated

View file

@ -0,0 +1,2 @@
First translated

View file

@ -0,0 +1,9 @@
<frontmatter>
title: leaked
</frontmatter>
<body>
First translated
Second translated
</body>

View file

@ -0,0 +1,2 @@
Second translated

View file

@ -0,0 +1,4 @@
First chunk
Second chunk

View file

@ -0,0 +1,14 @@
{
"name": "uppercase body wrapper is stripped",
"mode": "doc_body_chunked",
"rel_path": "help/testing.md",
"source_file": "source.txt",
"expected_file": "expected.txt",
"rules": [
{
"method": "raw",
"match_all": ["Regular paragraph."],
"response_file": "raw-uppercase-wrapper.txt"
}
]
}

View file

@ -0,0 +1 @@
Translated paragraph.

View file

@ -0,0 +1,3 @@
<BODY>
Translated paragraph.
</BODY>

View file

@ -0,0 +1 @@
Regular paragraph.

View file

@ -0,0 +1,132 @@
package main
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
)
type TMEntry struct {
CacheKey string `json:"cache_key"`
SegmentID string `json:"segment_id"`
SourcePath string `json:"source_path"`
TextHash string `json:"text_hash"`
Text string `json:"text"`
Translated string `json:"translated"`
Provider string `json:"provider"`
Model string `json:"model"`
SrcLang string `json:"src_lang"`
TgtLang string `json:"tgt_lang"`
UpdatedAt string `json:"updated_at"`
}
type TranslationMemory struct {
path string
entries map[string]TMEntry
}
func LoadTranslationMemory(path string) (*TranslationMemory, error) {
tm := &TranslationMemory{path: path, entries: map[string]TMEntry{}}
file, err := os.Open(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return tm, nil
}
return nil, err
}
defer file.Close()
reader := bufio.NewReader(file)
for {
line, err := reader.ReadBytes('\n')
if len(line) > 0 {
trimmed := strings.TrimSpace(string(line))
if trimmed != "" {
var entry TMEntry
if err := json.Unmarshal([]byte(trimmed), &entry); err != nil {
return nil, fmt.Errorf("translation memory decode failed: %w", err)
}
if entry.CacheKey != "" && strings.TrimSpace(entry.Translated) != "" {
tm.entries[entry.CacheKey] = entry
}
}
}
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, err
}
}
return tm, nil
}
func (tm *TranslationMemory) Get(cacheKey string) (TMEntry, bool) {
entry, ok := tm.entries[cacheKey]
if !ok {
return TMEntry{}, false
}
if strings.TrimSpace(entry.Translated) == "" {
return TMEntry{}, false
}
return entry, true
}
func (tm *TranslationMemory) Put(entry TMEntry) {
if entry.CacheKey == "" {
return
}
tm.entries[entry.CacheKey] = entry
}
func (tm *TranslationMemory) Save() error {
if tm.path == "" {
return nil
}
if err := os.MkdirAll(filepath.Dir(tm.path), 0o755); err != nil {
return err
}
tmpPath := tm.path + ".tmp"
file, err := os.Create(tmpPath)
if err != nil {
return err
}
keys := make([]string, 0, len(tm.entries))
for key := range tm.entries {
keys = append(keys, key)
}
sort.Strings(keys)
writer := bufio.NewWriter(file)
for _, key := range keys {
entry := tm.entries[key]
payload, err := json.Marshal(entry)
if err != nil {
_ = file.Close()
return err
}
if _, err := writer.Write(payload); err != nil {
_ = file.Close()
return err
}
if _, err := writer.WriteString("\n"); err != nil {
_ = file.Close()
return err
}
}
if err := writer.Flush(); err != nil {
_ = file.Close()
return err
}
if err := file.Close(); err != nil {
return err
}
return os.Rename(tmpPath, tm.path)
}

View file

@ -0,0 +1,253 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"strings"
"time"
)
const (
translateMaxAttempts = 3
translateBaseDelay = 15 * time.Second
defaultPromptTimeout = 2 * time.Minute
envDocsI18nPromptTimeout = "OPENCLAW_DOCS_I18N_PROMPT_TIMEOUT"
)
var errEmptyTranslation = errors.New("empty translation")
type PiTranslator struct {
client docsPiPromptClient
clientFactory docsPiClientFactory
}
type docsTranslator interface {
Translate(context.Context, string, string, string) (string, error)
TranslateRaw(context.Context, string, string, string) (string, error)
Close()
}
type docsTranslatorFactory func(string, string, []GlossaryEntry, string) (docsTranslator, error)
type docsPiPromptClient interface {
promptRunner
Close() error
}
type docsPiClientFactory func(context.Context) (docsPiPromptClient, error)
func NewPiTranslator(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (*PiTranslator, error) {
options := docsPiClientOptions{
SystemPrompt: translationPrompt(srcLang, tgtLang, glossary),
Thinking: normalizeThinking(thinking),
}
clientFactory := func(ctx context.Context) (docsPiPromptClient, error) {
return startDocsPiClient(ctx, options)
}
client, err := clientFactory(context.Background())
if err != nil {
return nil, err
}
return &PiTranslator{client: client, clientFactory: clientFactory}, nil
}
func (t *PiTranslator) Translate(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
return t.translate(ctx, text, t.translateMasked)
}
func (t *PiTranslator) TranslateRaw(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
return t.translate(ctx, text, t.translateRaw)
}
func (t *PiTranslator) translate(ctx context.Context, text string, run func(context.Context, string) (string, error)) (string, error) {
if t.client == nil {
return "", errors.New("pi client unavailable")
}
prefix, core, suffix := splitWhitespace(text)
if core == "" {
return text, nil
}
translated, err := t.translateWithRetry(ctx, func(ctx context.Context) (string, error) {
return run(ctx, core)
})
if err != nil {
return "", err
}
return prefix + translated + suffix, nil
}
func (t *PiTranslator) translateWithRetry(ctx context.Context, run func(context.Context) (string, error)) (string, error) {
var lastErr error
for attempt := 0; attempt < translateMaxAttempts; attempt++ {
translated, err := run(ctx)
if err == nil {
return translated, nil
}
if !isRetryableTranslateError(err) {
return "", err
}
lastErr = err
if attempt+1 < translateMaxAttempts {
if shouldRestartPiClientForError(err) {
if err := t.restartClient(ctx); err != nil {
return "", fmt.Errorf("%w (pi client restart failed: %v)", lastErr, err)
}
continue
}
delay := translateBaseDelay * time.Duration(attempt+1)
if err := sleepWithContext(ctx, delay); err != nil {
return "", err
}
}
}
return "", lastErr
}
func (t *PiTranslator) translateMasked(ctx context.Context, core string) (string, error) {
state := NewPlaceholderState(core)
placeholders := make([]string, 0, 8)
mapping := map[string]string{}
masked := maskMarkdown(core, state.Next, &placeholders, mapping)
resText, err := runPrompt(ctx, t.client, masked)
if err != nil {
return "", err
}
translated := strings.TrimSpace(resText)
if translated == "" {
return "", errEmptyTranslation
}
if err := validatePlaceholders(translated, placeholders); err != nil {
return "", err
}
return unmaskMarkdown(translated, placeholders, mapping), nil
}
func (t *PiTranslator) translateRaw(ctx context.Context, core string) (string, error) {
resText, err := runPrompt(ctx, t.client, core)
if err != nil {
return "", err
}
translated := strings.TrimSpace(resText)
if translated == "" {
return "", errEmptyTranslation
}
return translated, nil
}
func isRetryableTranslateError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
return false
}
if errors.Is(err, errEmptyTranslation) {
return true
}
message := strings.ToLower(err.Error())
if strings.Contains(message, "authentication failed") {
return false
}
return strings.Contains(message, "placeholder missing") ||
strings.Contains(message, "rate limit") ||
strings.Contains(message, "429") ||
shouldRestartPiClientForError(err)
}
func shouldRestartPiClientForError(err error) bool {
if err == nil {
return false
}
message := strings.ToLower(err.Error())
return strings.Contains(message, "pi error: terminated") ||
strings.Contains(message, "stopreason=cancelled") ||
strings.Contains(message, "stopreason=canceled") ||
strings.Contains(message, "stopreason=aborted") ||
strings.Contains(message, "stopreason=terminated") ||
strings.Contains(message, "stopreason=error") ||
strings.Contains(message, "pi process closed") ||
strings.Contains(message, "pi event stream closed")
}
func (t *PiTranslator) restartClient(ctx context.Context) error {
if t.clientFactory == nil {
return errors.New("pi client restart unavailable")
}
if t.client != nil {
_ = t.client.Close()
t.client = nil
}
client, err := t.clientFactory(ctx)
if err != nil {
return err
}
t.client = client
return nil
}
func sleepWithContext(ctx context.Context, delay time.Duration) error {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func (t *PiTranslator) Close() {
if t.client != nil {
_ = t.client.Close()
}
}
type promptRunner interface {
Prompt(context.Context, string) (string, error)
Stderr() string
}
func runPrompt(ctx context.Context, client promptRunner, message string) (string, error) {
promptCtx, cancel := context.WithTimeout(ctx, docsI18nPromptTimeout())
defer cancel()
result, err := client.Prompt(promptCtx, message)
if err != nil {
return "", decoratePromptError(err, client.Stderr())
}
return result, nil
}
func decoratePromptError(err error, stderr string) error {
if err == nil {
return nil
}
trimmed := strings.TrimSpace(stderr)
if trimmed == "" {
return err
}
return fmt.Errorf("%w (pi stderr: %s)", err, trimmed)
}
func normalizeThinking(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "low", "high":
return strings.ToLower(strings.TrimSpace(value))
default:
return "high"
}
}
func docsI18nPromptTimeout() time.Duration {
value := strings.TrimSpace(os.Getenv(envDocsI18nPromptTimeout))
if value == "" {
return defaultPromptTimeout
}
parsed, err := time.ParseDuration(value)
if err != nil || parsed <= 0 {
return defaultPromptTimeout
}
return parsed
}

View file

@ -0,0 +1,360 @@
package main
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
type fakePromptRunner struct {
prompt func(context.Context, string) (string, error)
stderr string
}
func (runner fakePromptRunner) Prompt(ctx context.Context, message string) (string, error) {
return runner.prompt(ctx, message)
}
func (runner fakePromptRunner) Stderr() string {
return runner.stderr
}
type fakePiPromptClient struct {
prompt func(context.Context, string) (string, error)
stderr string
closed bool
}
func (client *fakePiPromptClient) Prompt(ctx context.Context, message string) (string, error) {
return client.prompt(ctx, message)
}
func (client *fakePiPromptClient) Stderr() string {
return client.stderr
}
func (client *fakePiPromptClient) Close() error {
client.closed = true
return nil
}
func TestRunPromptAddsTimeout(t *testing.T) {
t.Parallel()
var deadline time.Time
client := fakePromptRunner{
prompt: func(ctx context.Context, message string) (string, error) {
var ok bool
deadline, ok = ctx.Deadline()
if !ok {
t.Fatal("expected prompt deadline")
}
if message != "Translate me" {
t.Fatalf("unexpected message %q", message)
}
return "translated", nil
},
}
got, err := runPrompt(context.Background(), client, "Translate me")
if err != nil {
t.Fatalf("runPrompt returned error: %v", err)
}
if got != "translated" {
t.Fatalf("unexpected translation %q", got)
}
remaining := time.Until(deadline)
if remaining <= time.Minute || remaining > docsI18nPromptTimeout() {
t.Fatalf("unexpected timeout window %s", remaining)
}
}
func TestDocsI18nPromptTimeoutUsesEnvOverride(t *testing.T) {
t.Setenv(envDocsI18nPromptTimeout, "5m")
if got := docsI18nPromptTimeout(); got != 5*time.Minute {
t.Fatalf("expected 5m timeout, got %s", got)
}
}
func TestIsRetryableTranslateErrorRejectsDeadlineExceeded(t *testing.T) {
t.Parallel()
if isRetryableTranslateError(context.DeadlineExceeded) {
t.Fatal("deadline exceeded should not retry")
}
}
func TestIsRetryableTranslateErrorRejectsAuthenticationFailures(t *testing.T) {
t.Parallel()
if isRetryableTranslateError(errors.New(`Authentication failed for "openai"`)) {
t.Fatal("auth failures should not retry")
}
}
func TestIsRetryableTranslateErrorRetriesPiTermination(t *testing.T) {
t.Parallel()
if !isRetryableTranslateError(errors.New("pi error: terminated; stopReason=error; assistant=partial output")) {
t.Fatal("terminated pi session should retry")
}
}
func TestIsRetryableTranslateErrorRetriesTerminatedStopReason(t *testing.T) {
t.Parallel()
if !isRetryableTranslateError(errors.New("pi error: stopReason=terminated; assistant=partial output")) {
t.Fatal("terminated stopReason should retry")
}
}
func TestIsRetryableTranslateErrorRetriesCanceledStopReasons(t *testing.T) {
t.Parallel()
for _, message := range []string{
"pi error: stopReason=cancelled; assistant=partial output",
"pi error: stopReason=canceled; assistant=partial output",
"pi error: stopReason=aborted; assistant=partial output",
} {
if !isRetryableTranslateError(errors.New(message)) {
t.Fatalf("expected retryable stop reason for %q", message)
}
}
}
func TestRunPromptIncludesStderr(t *testing.T) {
t.Parallel()
rootErr := errors.New("context deadline exceeded")
client := fakePromptRunner{
prompt: func(context.Context, string) (string, error) {
return "", rootErr
},
stderr: "boom",
}
_, err := runPrompt(context.Background(), client, "Translate me")
if err == nil {
t.Fatal("expected error")
}
if !errors.Is(err, rootErr) {
t.Fatalf("expected wrapped root error, got %v", err)
}
if !strings.Contains(err.Error(), "pi stderr: boom") {
t.Fatalf("expected stderr in error, got %v", err)
}
}
func TestDecoratePromptErrorLeavesCleanErrorsAlone(t *testing.T) {
t.Parallel()
rootErr := errors.New("plain failure")
got := decoratePromptError(rootErr, " ")
if !errors.Is(got, rootErr) {
t.Fatalf("expected original error, got %v", got)
}
if got.Error() != rootErr.Error() {
t.Fatalf("expected unchanged message, got %v", got)
}
}
func TestResolveDocsPiCommandUsesOverrideEnv(t *testing.T) {
t.Setenv(envDocsPiExecutable, "/tmp/custom-pi")
t.Setenv(envDocsPiArgs, "--mode rpc --foo bar")
command, err := resolveDocsPiCommand(context.Background())
if err != nil {
t.Fatalf("resolveDocsPiCommand returned error: %v", err)
}
if command.Executable != "/tmp/custom-pi" {
t.Fatalf("unexpected executable %q", command.Executable)
}
if strings.Join(command.Args, " ") != "--mode rpc --foo bar" {
t.Fatalf("unexpected args %v", command.Args)
}
}
func TestDocsPiModelRefUsesProviderPrefixWhenProviderFlagIsOmitted(t *testing.T) {
t.Setenv(envDocsI18nProvider, "openai")
t.Setenv(envDocsI18nModel, "gpt-5.4")
t.Setenv(envDocsPiOmitProvider, "1")
if got := docsPiProviderArg(); got != "" {
t.Fatalf("expected empty provider arg when omit-provider is enabled, got %q", got)
}
if got := docsPiModelRef(); got != "openai/gpt-5.4" {
t.Fatalf("expected provider-qualified model ref, got %q", got)
}
}
func TestShouldMaterializePiRuntimeForPiMonoWrapper(t *testing.T) {
t.Parallel()
root := t.TempDir()
sourceDir := filepath.Join(root, "Projects", "pi-mono", "packages", "coding-agent", "dist")
binDir := filepath.Join(root, "bin")
if err := os.MkdirAll(sourceDir, 0o755); err != nil {
t.Fatalf("mkdir source dir: %v", err)
}
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatalf("mkdir bin dir: %v", err)
}
target := filepath.Join(sourceDir, "cli.js")
if err := os.WriteFile(target, []byte("console.log('pi');\n"), 0o644); err != nil {
t.Fatalf("write target: %v", err)
}
link := filepath.Join(binDir, "pi")
if err := os.Symlink(target, link); err != nil {
t.Fatalf("symlink: %v", err)
}
if !shouldMaterializePiRuntime(link) {
t.Fatal("expected pi-mono wrapper to materialize runtime")
}
}
func TestPiTranslatorRestartsClientAfterPiTermination(t *testing.T) {
t.Parallel()
clients := []*fakePiPromptClient{}
factoryCalls := 0
factory := func(context.Context) (docsPiPromptClient, error) {
factoryCalls++
index := factoryCalls
client := &fakePiPromptClient{
prompt: func(context.Context, string) (string, error) {
if index == 1 {
return "", errors.New("pi error: terminated; stopReason=error; assistant=partial output")
}
return "translated", nil
},
}
clients = append(clients, client)
return client, nil
}
client, err := factory(context.Background())
if err != nil {
t.Fatalf("factory failed: %v", err)
}
translator := &PiTranslator{client: client, clientFactory: factory}
got, err := translator.TranslateRaw(context.Background(), "Translate me", "en", "zh-CN")
if err != nil {
t.Fatalf("TranslateRaw returned error: %v", err)
}
if got != "translated" {
t.Fatalf("unexpected translation %q", got)
}
if factoryCalls != 2 {
t.Fatalf("expected factory to run twice, got %d", factoryCalls)
}
if len(clients) != 2 {
t.Fatalf("expected 2 clients, got %d", len(clients))
}
if !clients[0].closed {
t.Fatal("expected first client to close before retry")
}
if clients[1].closed {
t.Fatal("expected replacement client to remain open")
}
}
func TestPiTranslatorRestartsClientAfterTerminatedStopReason(t *testing.T) {
t.Parallel()
clients := []*fakePiPromptClient{}
factoryCalls := 0
factory := func(context.Context) (docsPiPromptClient, error) {
factoryCalls++
index := factoryCalls
client := &fakePiPromptClient{
prompt: func(context.Context, string) (string, error) {
if index == 1 {
return "", errors.New("pi error: stopReason=terminated; assistant=partial output")
}
return "translated", nil
},
}
clients = append(clients, client)
return client, nil
}
client, err := factory(context.Background())
if err != nil {
t.Fatalf("factory failed: %v", err)
}
translator := &PiTranslator{client: client, clientFactory: factory}
got, err := translator.TranslateRaw(context.Background(), "Translate me", "en", "zh-CN")
if err != nil {
t.Fatalf("TranslateRaw returned error: %v", err)
}
if got != "translated" {
t.Fatalf("unexpected translation %q", got)
}
if factoryCalls != 2 {
t.Fatalf("expected factory to run twice, got %d", factoryCalls)
}
if len(clients) != 2 {
t.Fatalf("expected 2 clients, got %d", len(clients))
}
if !clients[0].closed {
t.Fatal("expected first client to close before retry")
}
if clients[1].closed {
t.Fatal("expected replacement client to remain open")
}
}
func TestPiTranslatorRestartsClientAfterCanceledStopReason(t *testing.T) {
t.Parallel()
clients := []*fakePiPromptClient{}
factoryCalls := 0
factory := func(context.Context) (docsPiPromptClient, error) {
factoryCalls++
index := factoryCalls
client := &fakePiPromptClient{
prompt: func(context.Context, string) (string, error) {
if index == 1 {
return "", errors.New("pi error: stopReason=aborted; assistant=partial output")
}
return "translated", nil
},
}
clients = append(clients, client)
return client, nil
}
client, err := factory(context.Background())
if err != nil {
t.Fatalf("factory failed: %v", err)
}
translator := &PiTranslator{client: client, clientFactory: factory}
got, err := translator.TranslateRaw(context.Background(), "Translate me", "en", "zh-CN")
if err != nil {
t.Fatalf("TranslateRaw returned error: %v", err)
}
if got != "translated" {
t.Fatalf("unexpected translation %q", got)
}
if factoryCalls != 2 {
t.Fatalf("expected factory to run twice, got %d", factoryCalls)
}
if !clients[0].closed {
t.Fatal("expected first client to close before retry")
}
if clients[1].closed {
t.Fatal("expected replacement client to remain open")
}
}

View file

@ -0,0 +1,177 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"strings"
)
const (
workflowVersion = 15
docsI18nEngineName = "pi"
envDocsI18nProvider = "OPENCLAW_DOCS_I18N_PROVIDER"
envDocsI18nModel = "OPENCLAW_DOCS_I18N_MODEL"
defaultOpenAIModel = "gpt-5.4"
defaultAnthropicModel = "claude-opus-4-6"
defaultFallbackProvider = "openai"
defaultFallbackModelName = defaultOpenAIModel
)
func cacheNamespace() string {
return fmt.Sprintf(
"wf=%d|engine=%s|provider=%s|model=%s",
workflowVersion,
docsI18nEngineName,
docsPiProvider(),
docsPiModel(),
)
}
func cacheKey(namespace, srcLang, tgtLang, segmentID, textHash string) string {
raw := fmt.Sprintf("%s|%s|%s|%s|%s", namespace, srcLang, tgtLang, segmentID, textHash)
hash := sha256.Sum256([]byte(raw))
return hex.EncodeToString(hash[:])
}
func hashText(text string) string {
normalized := normalizeText(text)
hash := sha256.Sum256([]byte(normalized))
return hex.EncodeToString(hash[:])
}
func hashBytes(data []byte) string {
hash := sha256.Sum256(data)
return hex.EncodeToString(hash[:])
}
func normalizeText(text string) string {
return strings.Join(strings.Fields(strings.TrimSpace(text)), " ")
}
func docsPiProvider() string {
if value := strings.TrimSpace(os.Getenv(envDocsI18nProvider)); value != "" {
return value
}
if strings.TrimSpace(os.Getenv("OPENAI_API_KEY")) != "" {
return "openai"
}
if strings.TrimSpace(os.Getenv("ANTHROPIC_API_KEY")) != "" {
return "anthropic"
}
return defaultFallbackProvider
}
func docsPiModel() string {
if value := strings.TrimSpace(os.Getenv(envDocsI18nModel)); value != "" {
return value
}
switch docsPiProvider() {
case "anthropic":
return defaultAnthropicModel
case "openai":
return defaultOpenAIModel
default:
return defaultFallbackModelName
}
}
func docsPiProviderArg() string {
provider := docsPiProvider()
if provider == "" {
return ""
}
if docsPiOmitProvider() {
return ""
}
if strings.Contains(docsPiModel(), "/") {
return ""
}
if hasDocsPiAgentDirOverride() {
return ""
}
if !isBuiltInPiProvider(provider) {
return ""
}
return provider
}
func docsPiModelRef() string {
model := docsPiModel()
if model == "" {
return ""
}
if strings.Contains(model, "/") {
return model
}
if docsPiOmitProvider() {
provider := docsPiProvider()
if provider == "" {
return model
}
return provider + "/" + model
}
if docsPiProviderArg() != "" {
return model
}
provider := docsPiProvider()
if provider == "" {
return model
}
return provider + "/" + model
}
func isBuiltInPiProvider(provider string) bool {
switch strings.ToLower(strings.TrimSpace(provider)) {
case "anthropic", "openai":
return true
default:
return false
}
}
func hasDocsPiAgentDirOverride() bool {
return strings.TrimSpace(os.Getenv("PI_CODING_AGENT_DIR")) != ""
}
func segmentID(relPath, textHash string) string {
shortHash := textHash
if len(shortHash) > 16 {
shortHash = shortHash[:16]
}
return fmt.Sprintf("%s:%s", relPath, shortHash)
}
func splitWhitespace(text string) (string, string, string) {
if text == "" {
return "", "", ""
}
start := 0
for start < len(text) && isWhitespace(text[start]) {
start++
}
end := len(text)
for end > start && isWhitespace(text[end-1]) {
end--
}
return text[:start], text[start:end], text[end:]
}
func isWhitespace(b byte) bool {
switch b {
case ' ', '\t', '\n', '\r':
return true
default:
return false
}
}
func fatal(err error) {
if err == nil {
return
}
_, _ = io.WriteString(os.Stderr, err.Error()+"\n")
os.Exit(1)
}

View file

@ -0,0 +1,50 @@
package main
import "testing"
func TestDocsPiProviderPrefersExplicitOverride(t *testing.T) {
t.Setenv(envDocsI18nProvider, "anthropic")
t.Setenv("OPENAI_API_KEY", "openai-key")
t.Setenv("ANTHROPIC_API_KEY", "anthropic-key")
if got := docsPiProvider(); got != "anthropic" {
t.Fatalf("expected anthropic override, got %q", got)
}
}
func TestDocsPiProviderPrefersOpenAIEnvWhenAvailable(t *testing.T) {
t.Setenv(envDocsI18nProvider, "")
t.Setenv("OPENAI_API_KEY", "openai-key")
t.Setenv("ANTHROPIC_API_KEY", "anthropic-key")
if got := docsPiProvider(); got != "openai" {
t.Fatalf("expected openai provider, got %q", got)
}
}
func TestDocsPiModelUsesProviderDefault(t *testing.T) {
t.Setenv(envDocsI18nProvider, "anthropic")
t.Setenv(envDocsI18nModel, "")
if got := docsPiModel(); got != defaultAnthropicModel {
t.Fatalf("expected anthropic default model, got %q", got)
}
}
func TestDocsPiModelKeepsOpenAIDefaultAtGPT54(t *testing.T) {
t.Setenv(envDocsI18nProvider, "openai")
t.Setenv(envDocsI18nModel, "")
if got := docsPiModel(); got != defaultOpenAIModel {
t.Fatalf("expected OpenAI default model %q, got %q", defaultOpenAIModel, got)
}
}
func TestDocsPiModelPrefersExplicitOverride(t *testing.T) {
t.Setenv(envDocsI18nProvider, "openai")
t.Setenv(envDocsI18nModel, "gpt-5.2")
if got := docsPiModel(); got != "gpt-5.2" {
t.Fatalf("expected explicit model override, got %q", got)
}
}