重构主控编排与运行时预热链路，统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。

同时收敛启动与运维脚本默认行为（含 wiki worker）、更新 Admin 可观测性与相关测试，降低首轮时延并提高运行稳定性。

Made-with: Cursor

openclaw/extensions/qa-lab/api.ts

@@ -0,0 +1,19 @@
+export * from "./src/bus-queries.js";
+export * from "./src/bus-server.js";
+export * from "./src/bus-state.js";
+export * from "./src/bus-waiters.js";
+export * from "./src/cli.js";
+export * from "./src/harness-runtime.js";
+export * from "./src/lab-server.js";
+export * from "./src/docker-harness.js";
+export * from "./src/qa-agent-bootstrap.js";
+export * from "./src/qa-agent-workspace.js";
+export * from "./src/qa-gateway-config.js";
+export * from "./src/report.js";
+export * from "./src/scenario.js";
+export * from "./src/scenario-catalog.js";
+export * from "./src/self-check-scenario.js";
+export * from "./src/self-check.js";
+export * from "./src/self-check-runner.js";
+export * from "./src/gateway-child.js";
+export * from "./src/suite.js";

openclaw/extensions/qa-lab/cli.ts

@@ -0,0 +1 @@
+export { isQaLabCliAvailable, registerQaLabCli } from "./src/cli.js";

openclaw/extensions/qa-lab/index.ts

@@ -0,0 +1,24 @@
+import { definePluginEntry } from "./runtime-api.js";
+import { registerQaLabCli } from "./src/cli.js";
+
+export default definePluginEntry({
+  id: "qa-lab",
+  name: "QA Lab",
+  description: "Private QA automation harness and debugger UI",
+  register(api) {
+    api.registerCli(
+      async ({ program }) => {
+        registerQaLabCli(program);
+      },
+      {
+        descriptors: [
+          {
+            name: "qa",
+            description: "Run QA scenarios and launch the private QA debugger UI",
+            hasSubcommands: true,
+          },
+        ],
+      },
+    );
+  },
+});

openclaw/extensions/qa-lab/model-selection.ts

@@ -0,0 +1 @@
+export * from "./src/model-selection.js";

openclaw/extensions/qa-lab/openclaw.plugin.json

@@ -0,0 +1,8 @@
+{
+  "id": "qa-lab",
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {}
+  }
+}

openclaw/extensions/qa-lab/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@openclaw/qa-lab",
+  "version": "2026.4.20",
+  "private": true,
+  "description": "OpenClaw QA lab plugin with private debugger UI and scenario runner",
+  "type": "module",
+  "dependencies": {
+    "@copilotkit/aimock": "1.14.3",
+    "playwright-core": "1.59.1"
+  },
+  "devDependencies": {
+    "@openclaw/plugin-sdk": "workspace:*",
+    "openclaw": "workspace:*"
+  },
+  "peerDependencies": {
+    "openclaw": ">=2026.4.20"
+  },
+  "peerDependenciesMeta": {
+    "openclaw": {
+      "optional": true
+    }
+  },
+  "openclaw": {
+    "extensions": [
+      "./index.ts"
+    ],
+    "install": {
+      "npmSpec": "@openclaw/qa-lab",
+      "defaultChoice": "npm",
+      "minHostVersion": ">=2026.4.10"
+    },
+    "compat": {
+      "pluginApi": ">=2026.4.20"
+    }
+  }
+}

openclaw/extensions/qa-lab/runtime-api.ts

@@ -0,0 +1,2 @@
+export * from "./src/runtime-api.js";
+export { startQaLiveLaneGateway } from "./src/live-transports/shared/live-gateway.runtime.js";

openclaw/extensions/qa-lab/src/agentic-parity-report.test.ts

@@ -0,0 +1,715 @@
+import { describe, expect, it } from "vitest";
+import {
+  buildQaAgenticParityComparison,
+  computeQaAgenticParityMetrics,
+  QaParityLabelMismatchError,
+  renderQaAgenticParityMarkdownReport,
+  type QaParityReportScenario,
+  type QaParitySuiteSummary,
+} from "./agentic-parity-report.js";
+
+const FULL_PARITY_PASS_SCENARIOS: QaParityReportScenario[] = [
+  { name: "Approval turn tool followthrough", status: "pass" as const },
+  { name: "Compaction retry after mutating tool", status: "pass" as const },
+  { name: "Model switch with tool continuity", status: "pass" as const },
+  { name: "Source and docs discovery report", status: "pass" as const },
+  { name: "Image understanding from attachment", status: "pass" as const },
+  { name: "Subagent handoff", status: "pass" as const },
+  { name: "Subagent fanout synthesis", status: "pass" as const },
+  { name: "Memory recall after context switch", status: "pass" as const },
+  { name: "Thread memory isolation", status: "pass" as const },
+  { name: "Config restart capability flip", status: "pass" as const },
+  { name: "Instruction followthrough repo contract", status: "pass" as const },
+];
+
+function withScenarioOverride(name: string, override: Partial<QaParityReportScenario>) {
+  return FULL_PARITY_PASS_SCENARIOS.map((scenario) =>
+    scenario.name === name ? { ...scenario, ...override } : scenario,
+  );
+}
+
+describe("qa agentic parity report", () => {
+  it("computes first-wave parity metrics from suite summaries", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        { name: "Approval turn tool followthrough", status: "pass" },
+        {
+          name: "Compaction retry after mutating tool",
+          status: "fail",
+          details: "incomplete turn detected",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary)).toEqual({
+      totalScenarios: 2,
+      passedScenarios: 1,
+      failedScenarios: 1,
+      completionRate: 0.5,
+      unintendedStopCount: 1,
+      unintendedStopRate: 0.5,
+      validToolCallCount: 1,
+      validToolCallRate: 0.5,
+      fakeSuccessCount: 0,
+    });
+  });
+
+  it("keeps non-tool scenarios out of the valid-tool-call metric", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        { name: "Approval turn tool followthrough", status: "pass" },
+        { name: "Memory recall after context switch", status: "pass" },
+        { name: "Image understanding from attachment", status: "pass" },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary)).toMatchObject({
+      totalScenarios: 3,
+      passedScenarios: 3,
+      validToolCallCount: 1,
+      validToolCallRate: 1,
+    });
+  });
+
+  it("fails the parity gate when the candidate regresses against baseline", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: {
+        scenarios: [
+          { name: "Approval turn tool followthrough", status: "pass" },
+          {
+            name: "Compaction retry after mutating tool",
+            status: "fail",
+            details: "timed out before it continued",
+          },
+          { name: "Model switch with tool continuity", status: "pass" },
+          { name: "Source and docs discovery report", status: "pass" },
+          { name: "Image understanding from attachment", status: "pass" },
+        ],
+      },
+      baselineSummary: {
+        scenarios: [
+          { name: "Approval turn tool followthrough", status: "pass" },
+          { name: "Compaction retry after mutating tool", status: "pass" },
+          { name: "Model switch with tool continuity", status: "pass" },
+          { name: "Source and docs discovery report", status: "pass" },
+          { name: "Image understanding from attachment", status: "pass" },
+        ],
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "openai/gpt-5.4 completion rate 80.0% is below anthropic/claude-opus-4-6 100.0%.",
+    );
+    expect(comparison.failures).toContain(
+      "openai/gpt-5.4 unintended-stop rate 20.0% exceeds anthropic/claude-opus-4-6 0.0%.",
+    );
+  });
+
+  it("fails the parity gate when candidate and baseline cover different non-parity scenarios", () => {
+    const baselineScenarios = [
+      { name: "Approval turn tool followthrough", status: "pass" as const },
+      { name: "Compaction retry after mutating tool", status: "pass" as const },
+      { name: "Model switch with tool continuity", status: "pass" as const },
+      { name: "Source and docs discovery report", status: "pass" as const },
+      { name: "Image understanding from attachment", status: "pass" as const },
+      { name: "Extra non-parity lane", status: "pass" as const },
+    ];
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: {
+        scenarios: baselineScenarios.filter(
+          (scenario) => scenario.name !== "Extra non-parity lane",
+        ),
+      },
+      baselineSummary: { scenarios: baselineScenarios },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Scenario coverage mismatch for Extra non-parity lane: openai/gpt-5.4=missing, anthropic/claude-opus-4-6=pass.",
+    );
+  });
+
+  it("reports each missing required parity scenario exactly once (no double-counting)", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: {
+        scenarios: [{ name: "Approval turn tool followthrough", status: "pass" }],
+      },
+      baselineSummary: {
+        scenarios: [{ name: "Approval turn tool followthrough", status: "pass" }],
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    const missingScenario = "Image understanding from attachment";
+    const requiredLines = comparison.failures.filter((failure) =>
+      failure.includes(`Missing required parity scenario coverage for ${missingScenario}:`),
+    );
+    const mismatchLines = comparison.failures.filter((failure) =>
+      failure.includes(`Scenario coverage mismatch for ${missingScenario}:`),
+    );
+    expect(requiredLines).toHaveLength(1);
+    expect(mismatchLines).toHaveLength(0);
+  });
+
+  it("scopes parity metrics to declared parity scenarios even when extra lanes are present", () => {
+    const scopedSummary = {
+      scenarios: [
+        { name: "Approval turn tool followthrough", status: "pass" as const },
+        { name: "Compaction retry after mutating tool", status: "pass" as const },
+        { name: "Model switch with tool continuity", status: "pass" as const },
+        { name: "Source and docs discovery report", status: "pass" as const },
+        { name: "Image understanding from attachment", status: "pass" as const },
+      ],
+    };
+    const summaryWithExtras = {
+      scenarios: [
+        ...scopedSummary.scenarios,
+        { name: "Extra lane A", status: "fail" as const, details: "timed out" },
+        { name: "Extra lane B", status: "fail" as const, details: "timed out" },
+      ],
+    };
+
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: summaryWithExtras,
+      baselineSummary: scopedSummary,
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    // Extra lanes must not drag the candidate's completion rate below baseline
+    // and must not generate unintended-stop or fake-success hits.
+    expect(comparison.candidateMetrics.totalScenarios).toBe(5);
+    expect(comparison.candidateMetrics.completionRate).toBe(1);
+    expect(comparison.candidateMetrics.unintendedStopRate).toBe(0);
+    expect(comparison.candidateMetrics.fakeSuccessCount).toBe(0);
+    // The pass/fail verdict here still depends only on the parity pack itself.
+    const regressionFailures = comparison.failures.filter((failure) =>
+      failure.includes("completion rate"),
+    );
+    expect(regressionFailures).toEqual([]);
+  });
+
+  it("fails the parity gate when required parity scenarios are missing on both sides", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: {
+        scenarios: [{ name: "Approval turn tool followthrough", status: "pass" }],
+      },
+      baselineSummary: {
+        scenarios: [{ name: "Approval turn tool followthrough", status: "pass" }],
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Missing required parity scenario coverage for Image understanding from attachment: openai/gpt-5.4=missing, anthropic/claude-opus-4-6=missing.",
+    );
+  });
+
+  it("fails the parity gate when required parity scenarios are skipped", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: {
+        scenarios: [
+          { name: "Approval turn tool followthrough", status: "pass" },
+          { name: "Compaction retry after mutating tool", status: "skip" },
+          { name: "Model switch with tool continuity", status: "pass" },
+          { name: "Source and docs discovery report", status: "pass" },
+          { name: "Image understanding from attachment", status: "pass" },
+        ],
+      },
+      baselineSummary: {
+        scenarios: [
+          { name: "Approval turn tool followthrough", status: "pass" },
+          { name: "Compaction retry after mutating tool", status: "skip" },
+          { name: "Model switch with tool continuity", status: "pass" },
+          { name: "Source and docs discovery report", status: "pass" },
+          { name: "Image understanding from attachment", status: "pass" },
+        ],
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Missing required parity scenario coverage for Compaction retry after mutating tool: openai/gpt-5.4=skip, anthropic/claude-opus-4-6=skip.",
+    );
+  });
+
+  it("fails the parity gate when a required parity scenario fails on both sides", () => {
+    // Regression for the loop-7 Codex-connector P1 finding: without this
+    // check, a required parity scenario that fails on both candidate and
+    // baseline still produces pass=true because the downstream metric
+    // comparisons are purely relative (candidate vs baseline). Cover the
+    // whole parity pack as pass on both sides except the one scenario we
+    // deliberately fail on both sides, so the assertion can pin the
+    // isolated gate failure under test.
+    const scenariosWithBothFail = withScenarioOverride("Approval turn tool followthrough", {
+      status: "fail",
+    });
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: { scenarios: scenariosWithBothFail },
+      baselineSummary: { scenarios: scenariosWithBothFail },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Required parity scenario Approval turn tool followthrough failed: openai/gpt-5.4=fail, anthropic/claude-opus-4-6=fail.",
+    );
+    // Metric comparisons are relative, so a same-on-both-sides failure
+    // must not appear as a relative metric failure. The required-scenario
+    // failure line is the only thing keeping the gate honest here.
+    expect(comparison.failures.some((failure) => failure.includes("completion rate"))).toBe(false);
+  });
+
+  it("fails the parity gate when a required parity scenario fails on the candidate only", () => {
+    // A candidate regression below a passing baseline is already caught
+    // by the relative completion-rate comparison, but surface it as a
+    // named required-scenario failure too so operators see a concrete
+    // scenario name alongside the rate differential.
+    const candidateWithOneFail = withScenarioOverride("Approval turn tool followthrough", {
+      status: "fail",
+    });
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: { scenarios: candidateWithOneFail },
+      baselineSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Required parity scenario Approval turn tool followthrough failed: openai/gpt-5.4=fail, anthropic/claude-opus-4-6=pass.",
+    );
+  });
+
+  it("fails the parity gate when the baseline contains suspicious pass results", () => {
+    // Cover the full second-wave pack on both sides so the suspicious-pass assertion
+    // below is the isolated gate failure under test (no coverage-gap noise).
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+      },
+      baselineSummary: {
+        scenarios: withScenarioOverride("Approval turn tool followthrough", {
+          details: "timed out before it continued",
+        }),
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toEqual([
+      "anthropic/claude-opus-4-6 produced 1 suspicious pass result(s); baseline fake-success count must also be 0.",
+    ]);
+  });
+
+  it("ignores neutral Failed and Blocked headings in passing protocol reports", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Source and docs discovery report",
+          status: "pass",
+          details: `Worked:
+- Read the seeded QA material.
+Failed:
+- None observed.
+Blocked:
+- No live provider evidence in this lane.
+Follow-up:
+- Re-run with a real provider if needed.`,
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("ignores neutral error-budget and no-errors-observed phrasing in passing reports", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Source and docs discovery report",
+          status: "pass",
+          details: `Worked:
+- Scenario finished with Error budget: 0.
+- No errors found in the seeded material.
+- Errors: none observed.`,
+        },
+        {
+          name: "Image understanding from attachment",
+          status: "pass",
+          details: "Error: none. The attached image analysis completed without incident.",
+        },
+      ],
+    };
+
+    // Bare "error"/"Error" in narration is not a suspicious-pass signal on its own.
+    // Only phrases like "error occurred" or "an error was ..." should count.
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("still flags genuine error-narration suspicious passes", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Approval turn tool followthrough",
+          status: "pass",
+          details: "Tool call completed, but an error occurred mid-turn and no retry happened.",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(1);
+  });
+
+  it("does not flag positive-tone prose as fake success (positive-tone detection removed)", () => {
+    // Positive-tone detection was removed because for passing runs the
+    // `details` field is the model's prose, which never contains tool-call
+    // evidence. Criterion 2 is enforced by per-scenario tool-call assertions.
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Subagent handoff",
+          status: "pass",
+          details: "Successfully completed the delegation. The subagent returned its result.",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("does not flag bare 'Done.' prose as fake success", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Approval turn tool followthrough",
+          status: "pass",
+          details: "Done.",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("does not flag structured status lines that end in `done`", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Compaction retry after mutating tool",
+          status: "pass",
+          details: `Confirmed, replay unsafe after write.
+compactionCount=0
+status=done`,
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("does not flag positive-tone passes when the scenario shows real tool-call evidence", () => {
+    // A legitimate tool-mediated pass that happens to include
+    // "successfully" in its prose must not be flagged. The
+    // `plannedToolName` evidence (or any of the other tool-call
+    // evidence patterns) exempts the scenario from positive-tone
+    // detection. Without this exemption, real tool-backed passes with
+    // self-congratulatory prose would count as fake successes and break
+    // the gate.
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Source and docs discovery report",
+          status: "pass",
+          details:
+            "Successfully completed the report. plannedToolName=read recorded via /debug/requests.",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("only flags failure-tone passes, not positive-tone", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Approval turn tool followthrough",
+          status: "pass",
+          details: "Task executed successfully without errors.",
+        },
+        {
+          name: "Subagent handoff",
+          status: "pass",
+          details: "Tool call completed, but an error occurred mid-turn.",
+        },
+      ],
+    };
+
+    // Only the failure-tone scenario ("error occurred") counts.
+    // The positive-tone one ("successfully") is not flagged.
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(1);
+  });
+
+  it("throws QaParityLabelMismatchError when the candidate run.primaryProvider does not match the label", () => {
+    // Regression for the gate footgun: if an operator swaps the
+    // --candidate-summary and --baseline-summary paths, the gate would
+    // silently produce a reversed verdict. PR L #64789 ships the `run`
+    // block on every summary so the parity report can verify it against
+    // the caller-supplied label; this test pins the precondition check.
+    const parityPassScenarios = [
+      { name: "Approval turn tool followthrough", status: "pass" as const },
+      { name: "Compaction retry after mutating tool", status: "pass" as const },
+      { name: "Model switch with tool continuity", status: "pass" as const },
+      { name: "Source and docs discovery report", status: "pass" as const },
+      { name: "Image understanding from attachment", status: "pass" as const },
+    ];
+
+    expect(() =>
+      buildQaAgenticParityComparison({
+        candidateLabel: "openai/gpt-5.4",
+        baselineLabel: "anthropic/claude-opus-4-6",
+        candidateSummary: {
+          scenarios: parityPassScenarios,
+          run: { primaryProvider: "anthropic", primaryModel: "claude-opus-4-6" },
+        },
+        baselineSummary: {
+          scenarios: parityPassScenarios,
+          run: { primaryProvider: "anthropic", primaryModel: "claude-opus-4-6" },
+        },
+        comparedAt: "2026-04-11T00:00:00.000Z",
+      }),
+    ).toThrow(QaParityLabelMismatchError);
+  });
+
+  it("throws QaParityLabelMismatchError when the baseline run.primaryProvider does not match the label", () => {
+    const parityPassScenarios = [
+      { name: "Approval turn tool followthrough", status: "pass" as const },
+    ];
+
+    expect(() =>
+      buildQaAgenticParityComparison({
+        candidateLabel: "openai/gpt-5.4",
+        baselineLabel: "anthropic/claude-opus-4-6",
+        candidateSummary: {
+          scenarios: parityPassScenarios,
+          run: { primaryProvider: "openai" },
+        },
+        baselineSummary: {
+          scenarios: parityPassScenarios,
+          run: { primaryProvider: "openai", primaryModel: "gpt-5.4" },
+        },
+        comparedAt: "2026-04-11T00:00:00.000Z",
+      }),
+    ).toThrow(
+      /baseline summary run\.primaryProvider=openai and run\.primaryModel=gpt-5\.4 do not match --baseline-label/,
+    );
+  });
+
+  it("accepts matching run.primaryProvider labels without throwing", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "openai",
+          primaryModel: "openai/gpt-5.4",
+          primaryModelName: "gpt-5.4",
+        },
+      },
+      baselineSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "anthropic",
+          primaryModel: "anthropic/claude-opus-4-6",
+          primaryModelName: "claude-opus-4-6",
+        },
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("skips run.primaryProvider verification when the summary is missing a run block (legacy summaries)", () => {
+    // Pre-PR-L summaries don't carry a `run` block. The gate must still
+    // work against those, trusting the caller-supplied label.
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      baselineSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("skips provider verification for arbitrary display labels when run metadata is present", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "GPT-5.4 candidate",
+      baselineLabel: "Opus 4.6 baseline",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "openai",
+          primaryModel: "openai/gpt-5.4",
+          primaryModelName: "gpt-5.4",
+        },
+      },
+      baselineSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "anthropic",
+          primaryModel: "anthropic/claude-opus-4-6",
+          primaryModelName: "claude-opus-4-6",
+        },
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("skips provider verification for mixed-case or decorated display labels", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "Candidate: GPT-5.4",
+      baselineLabel: "Opus 4.6 / baseline",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "openai",
+          primaryModel: "openai/gpt-5.4",
+          primaryModelName: "gpt-5.4",
+        },
+      },
+      baselineSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "anthropic",
+          primaryModel: "anthropic/claude-opus-4-6",
+          primaryModelName: "claude-opus-4-6",
+        },
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("throws when a structured label mismatches the recorded model even if the provider matches", () => {
+    expect(() =>
+      buildQaAgenticParityComparison({
+        candidateLabel: "openai/gpt-5.4",
+        baselineLabel: "anthropic/claude-opus-4-6",
+        candidateSummary: {
+          scenarios: FULL_PARITY_PASS_SCENARIOS,
+          run: {
+            primaryProvider: "openai",
+            primaryModel: "openai/gpt-5.4-alt",
+            primaryModelName: "gpt-5.4-alt",
+          },
+        },
+        baselineSummary: {
+          scenarios: FULL_PARITY_PASS_SCENARIOS,
+          run: {
+            primaryProvider: "anthropic",
+            primaryModel: "anthropic/claude-opus-4-6",
+            primaryModelName: "claude-opus-4-6",
+          },
+        },
+        comparedAt: "2026-04-11T00:00:00.000Z",
+      }),
+    ).toThrow(
+      /candidate summary run\.primaryProvider=openai and run\.primaryModel=openai\/gpt-5\.4-alt do not match --candidate-label=openai\/gpt-5\.4/,
+    );
+  });
+
+  it("accepts colon-delimited structured labels when provider and model both match", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai:gpt-5.4",
+      baselineLabel: "anthropic:claude-opus-4-6",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "openai",
+          primaryModel: "openai/gpt-5.4",
+          primaryModelName: "gpt-5.4",
+        },
+      },
+      baselineSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "anthropic",
+          primaryModel: "anthropic/claude-opus-4-6",
+          primaryModelName: "claude-opus-4-6",
+        },
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("renders a readable markdown parity report", () => {
+    // Cover the full parity pack on both sides so the pass
+    // verdict is not disrupted by required-scenario coverage failures
+    // added by the second-wave expansion.
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4",
+      baselineLabel: "anthropic/claude-opus-4-6",
+      candidateSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      baselineSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    const report = renderQaAgenticParityMarkdownReport(comparison);
+
+    expect(report).toContain(
+      "# OpenClaw Agentic Parity Report — openai/gpt-5.4 vs anthropic/claude-opus-4-6",
+    );
+    expect(report).toContain("| Completion rate | 100.0% | 100.0% |");
+    expect(report).toContain("### Approval turn tool followthrough");
+    expect(report).toContain("- Verdict: pass");
+  });
+
+  it("parametrizes the markdown header from the comparison labels", () => {
+    // Regression for the loop-7 Copilot finding: callers that configure
+    // non-gpt-5.4 / non-opus labels (for example an internal candidate vs
+    // another candidate) must see the labels in the rendered H1 instead of
+    // the hardcoded "GPT-5.4 / Opus 4.6" title that would otherwise confuse
+    // readers of saved reports.
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.4-alt",
+      baselineLabel: "openai/gpt-5.4",
+      candidateSummary: { scenarios: [] },
+      baselineSummary: { scenarios: [] },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+    const report = renderQaAgenticParityMarkdownReport(comparison);
+    expect(report).toContain(
+      "# OpenClaw Agentic Parity Report — openai/gpt-5.4-alt vs openai/gpt-5.4",
+    );
+  });
+});

openclaw/extensions/qa-lab/src/agentic-parity-report.ts

@@ -0,0 +1,541 @@
+import {
+  QA_AGENTIC_PARITY_SCENARIO_TITLES,
+  QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES,
+} from "./agentic-parity.js";
+
+export type QaParityReportStep = {
+  name: string;
+  status: "pass" | "fail" | "skip";
+  details?: string;
+};
+
+export type QaParityReportScenario = {
+  name: string;
+  status: "pass" | "fail" | "skip";
+  details?: string;
+  steps?: QaParityReportStep[];
+};
+
+/**
+ * Optional self-describing run metadata written by PR L (#64789). Before
+ * that PR merges, older summaries only have `scenarios` + `counts`; the
+ * parity report treats a missing `run` block as "unknown provenance" and
+ * skips the label-match verification for backwards compatibility
+ * with legacy summaries that predate the run metadata block.
+ */
+export type QaParityRunBlock = {
+  primaryProvider?: string;
+  primaryModel?: string;
+  primaryModelName?: string;
+  providerMode?: string;
+  scenarioIds?: readonly string[] | null;
+};
+
+export type QaParitySuiteSummary = {
+  scenarios: QaParityReportScenario[];
+  counts?: {
+    total?: number;
+    passed?: number;
+    failed?: number;
+  };
+  /** Self-describing run metadata — see PR L #64789 for the writer side. */
+  run?: QaParityRunBlock;
+};
+
+export type QaAgenticParityMetrics = {
+  totalScenarios: number;
+  passedScenarios: number;
+  failedScenarios: number;
+  completionRate: number;
+  unintendedStopCount: number;
+  unintendedStopRate: number;
+  validToolCallCount: number;
+  validToolCallRate: number;
+  fakeSuccessCount: number;
+};
+
+export type QaAgenticParityScenarioComparison = {
+  name: string;
+  candidateStatus: "pass" | "fail" | "skip" | "missing";
+  baselineStatus: "pass" | "fail" | "skip" | "missing";
+  candidateDetails?: string;
+  baselineDetails?: string;
+};
+
+export type QaAgenticParityComparison = {
+  candidateLabel: string;
+  baselineLabel: string;
+  comparedAt: string;
+  candidateMetrics: QaAgenticParityMetrics;
+  baselineMetrics: QaAgenticParityMetrics;
+  scenarioComparisons: QaAgenticParityScenarioComparison[];
+  pass: boolean;
+  failures: string[];
+  notes: string[];
+};
+
+const UNINTENDED_STOP_PATTERNS = [
+  /incomplete turn/i,
+  /\btimed out\b/i,
+  /\btimeout\b/i,
+  /\bstopped\b/i,
+  /\bblocked\b/i,
+  /\babandoned\b/i,
+  /did not continue/i,
+] as const;
+
+// Failure-tone patterns: a passing scenario whose details text matches any
+// of these is treated as a "fake success" — the scenario is marked pass but
+// the supporting text reveals something went wrong. Adding new patterns here
+// widens the net for bad prose that correlates with runtime failure modes.
+const SUSPICIOUS_PASS_FAILURE_TONE_PATTERNS = [
+  /incomplete turn/i,
+  /\btimed out\b/i,
+  /\btimeout\b/i,
+  /\bfailed to\b/i,
+  /\bcould not\b/i,
+  /\bunable to\b/i,
+  /did not continue/i,
+  /error occurred/i,
+  /an error was/i,
+] as const;
+
+// Positive-tone patterns (e.g. "Successfully completed", "Done.") are NOT
+// checked in fakeSuccessCount. For passing runs, `details` is the model's
+// outbound prose, which never contains tool-call evidence strings, so a
+// tool-call-evidence exemption would false-positive on every legitimate
+// pass. Criterion 2 ("no fake progress") is enforced by per-scenario
+// `/debug/requests` tool-call assertions in the YAML flows (PR J) instead.
+
+function normalizeScenarioStatus(status: string | undefined): "pass" | "fail" | "skip" {
+  return status === "pass" || status === "fail" || status === "skip" ? status : "fail";
+}
+
+function scenarioText(scenario: QaParityReportScenario) {
+  const parts = [scenario.details ?? ""];
+  for (const step of scenario.steps ?? []) {
+    parts.push(step.details ?? "");
+  }
+  return parts.filter(Boolean).join("\n");
+}
+
+function scenarioHasPattern(
+  scenario: QaParityReportScenario,
+  patterns: readonly RegExp[],
+): boolean {
+  const text = scenarioText(scenario);
+  return text.length > 0 && patterns.some((pattern) => pattern.test(text));
+}
+
+export function computeQaAgenticParityMetrics(
+  summary: QaParitySuiteSummary,
+): QaAgenticParityMetrics {
+  const scenarios = summary.scenarios.map((scenario) => ({
+    ...scenario,
+    status: normalizeScenarioStatus(scenario.status),
+  }));
+  const toolBackedTitleSet: ReadonlySet<string> = new Set(
+    QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES,
+  );
+  const totalScenarios = summary.counts?.total ?? scenarios.length;
+  const passedScenarios =
+    summary.counts?.passed ?? scenarios.filter((scenario) => scenario.status === "pass").length;
+  const failedScenarios =
+    summary.counts?.failed ?? scenarios.filter((scenario) => scenario.status === "fail").length;
+  const unintendedStopCount = scenarios.filter(
+    (scenario) =>
+      scenario.status !== "pass" && scenarioHasPattern(scenario, UNINTENDED_STOP_PATTERNS),
+  ).length;
+  const fakeSuccessCount = scenarios.filter((scenario) => {
+    if (scenario.status !== "pass") {
+      return false;
+    }
+    // Failure-tone patterns catch obviously-broken passes regardless of
+    // whether the scenario shows tool-call evidence — "timed out" under a
+    // pass is always fake.
+    if (scenarioHasPattern(scenario, SUSPICIOUS_PASS_FAILURE_TONE_PATTERNS)) {
+      return true;
+    }
+    // Positive-tone patterns (like "Successfully completed") are NOT checked
+    // here because for passing runs the `details` field is the model's
+    // outbound prose, which never contains tool-call evidence strings.
+    // The `scenarioLacksToolCallEvidence` check would return true for ALL
+    // passes and false-positive on legitimate completions. Criterion 2
+    // ("no fake tool completion") is instead enforced by the per-scenario
+    // `/debug/requests` tool-call assertions from the scenario YAML flows.
+    return false;
+  }).length;
+
+  // Count only the scenarios that are supposed to exercise a real tool,
+  // subagent, or capability invocation. Memory recall and image-only
+  // understanding lanes stay in the parity pack, but they should not inflate
+  // the tool-call metric just by passing.
+  const toolBackedScenarioCount = scenarios.filter((scenario) =>
+    toolBackedTitleSet.has(scenario.name),
+  ).length;
+  const validToolCallCount = scenarios.filter(
+    (scenario) => toolBackedTitleSet.has(scenario.name) && scenario.status === "pass",
+  ).length;
+
+  const rate = (value: number) => (totalScenarios > 0 ? value / totalScenarios : 0);
+  const toolRate = (value: number) =>
+    toolBackedScenarioCount > 0 ? value / toolBackedScenarioCount : 0;
+  return {
+    totalScenarios,
+    passedScenarios,
+    failedScenarios,
+    completionRate: rate(passedScenarios),
+    unintendedStopCount,
+    unintendedStopRate: rate(unintendedStopCount),
+    validToolCallCount,
+    validToolCallRate: toolRate(validToolCallCount),
+    fakeSuccessCount,
+  };
+}
+
+function formatPercent(value: number) {
+  return `${(value * 100).toFixed(1)}%`;
+}
+
+function requiredCoverageStatus(
+  scenario: QaParityReportScenario | undefined,
+): "pass" | "fail" | "skip" | "missing" {
+  return scenario ? normalizeScenarioStatus(scenario.status) : "missing";
+}
+
+function scopeSummaryToParityPack(
+  summary: QaParitySuiteSummary,
+  parityTitleSet: ReadonlySet<string>,
+): QaParitySuiteSummary {
+  // The parity verdict must only consider the declared parity scenarios
+  // (the full first-wave + second-wave pack from QA_AGENTIC_PARITY_SCENARIOS).
+  // Drop `counts` so the metric helper recomputes totals from the filtered
+  // scenario list instead of inheriting the caller's full-suite counters.
+  return {
+    scenarios: summary.scenarios.filter((scenario) => parityTitleSet.has(scenario.name)),
+    ...(summary.run ? { run: summary.run } : {}),
+  };
+}
+
+type StructuredQaParityLabel = {
+  provider: string;
+  model: string;
+};
+
+/**
+ * Only treat caller labels as provenance-checked identifiers when they are
+ * exact lower-case provider/model refs. Human-facing display labels like
+ * "GPT-5.4 candidate" or "Candidate: GPT-5.4" should render in the report
+ * without being misread as structured provider ids.
+ */
+function parseStructuredLabelRef(label: string): StructuredQaParityLabel | null {
+  const trimmed = label.trim();
+  if (trimmed.length === 0) {
+    return null;
+  }
+  if (trimmed !== trimmed.toLowerCase()) {
+    return null;
+  }
+  const separatorMatch = /^([a-z0-9][a-z0-9-]*)[/:]([a-z0-9][a-z0-9._-]*)$/.exec(trimmed);
+  if (!separatorMatch) {
+    return null;
+  }
+  return {
+    provider: separatorMatch[1] ?? "",
+    model: separatorMatch[2] ?? "",
+  };
+}
+
+/**
+ * Verify the `run.primaryProvider` + `run.primaryModel` fields on a summary
+ * match the caller-supplied label when that label is a structured
+ * `provider/model` or `provider:model` ref. PR L #64789 ships the `run`
+ * block; before it lands, older summaries don't have the field and this check
+ * is a no-op.
+ *
+ * Throws `QaParityLabelMismatchError` when the summary reports a different
+ * provider/model than the caller claimed — this catches the "swapped
+ * candidate and baseline summary paths" footgun the earlier adversarial
+ * review flagged. Returns silently when the fields are absent (legacy
+ * summaries) or when the fields match.
+ */
+function verifySummaryLabelMatch(params: {
+  summary: QaParitySuiteSummary;
+  label: string;
+  role: "candidate" | "baseline";
+}): void {
+  const runProvider = params.summary.run?.primaryProvider?.trim();
+  const runModel = params.summary.run?.primaryModel?.trim();
+  const runModelName = params.summary.run?.primaryModelName?.trim();
+  if (!runProvider || !runModel) {
+    return;
+  }
+  const labelRef = parseStructuredLabelRef(params.label);
+  if (!labelRef) {
+    return;
+  }
+  const normalizedRunModel = runModel.toLowerCase();
+  const normalizedRunModelName = runModelName?.toLowerCase();
+  const normalizedLabelModel = labelRef.model;
+  if (
+    runProvider.toLowerCase() === labelRef.provider &&
+    (normalizedRunModel === normalizedLabelModel ||
+      normalizedRunModelName === normalizedLabelModel ||
+      normalizedRunModel === `${labelRef.provider}/${normalizedLabelModel}`)
+  ) {
+    return;
+  }
+  throw new QaParityLabelMismatchError({
+    role: params.role,
+    label: params.label,
+    runProvider,
+    runModel,
+  });
+}
+
+export class QaParityLabelMismatchError extends Error {
+  readonly role: "candidate" | "baseline";
+  readonly label: string;
+  readonly runProvider: string;
+  readonly runModel: string;
+
+  constructor(params: {
+    role: "candidate" | "baseline";
+    label: string;
+    runProvider: string;
+    runModel: string;
+  }) {
+    super(
+      `${params.role} summary run.primaryProvider=${params.runProvider} and run.primaryModel=${params.runModel} do not match --${params.role}-label=${params.label}. ` +
+        `Check that the --candidate-summary / --baseline-summary paths weren't swapped.`,
+    );
+    this.name = "QaParityLabelMismatchError";
+    this.role = params.role;
+    this.label = params.label;
+    this.runProvider = params.runProvider;
+    this.runModel = params.runModel;
+  }
+}
+
+export function buildQaAgenticParityComparison(params: {
+  candidateLabel: string;
+  baselineLabel: string;
+  candidateSummary: QaParitySuiteSummary;
+  baselineSummary: QaParitySuiteSummary;
+  comparedAt?: string;
+}): QaAgenticParityComparison {
+  // Precondition: verify the `run.primaryProvider` field on each summary
+  // matches the caller-supplied label (when the `run` block is present).
+  // Throws `QaParityLabelMismatchError` on mismatch so the release gate
+  // fails loudly instead of silently producing a reversed verdict when an
+  // operator swaps the --candidate-summary and --baseline-summary paths.
+  // Legacy summaries without a `run` block are accepted as-is.
+  verifySummaryLabelMatch({
+    summary: params.candidateSummary,
+    label: params.candidateLabel,
+    role: "candidate",
+  });
+  verifySummaryLabelMatch({
+    summary: params.baselineSummary,
+    label: params.baselineLabel,
+    role: "baseline",
+  });
+  const parityTitleSet: ReadonlySet<string> = new Set<string>(QA_AGENTIC_PARITY_SCENARIO_TITLES);
+  // Rates and fake-success counts are computed from the parity-scoped summaries only,
+  // so extra non-parity scenarios in the input (for example when a caller feeds a full
+  // qa-suite-summary.json rather than a --parity-pack agentic run) cannot influence
+  // the gate verdict.
+  const candidateMetrics = computeQaAgenticParityMetrics(
+    scopeSummaryToParityPack(params.candidateSummary, parityTitleSet),
+  );
+  const baselineMetrics = computeQaAgenticParityMetrics(
+    scopeSummaryToParityPack(params.baselineSummary, parityTitleSet),
+  );
+
+  const scenarioNames = new Set([
+    ...QA_AGENTIC_PARITY_SCENARIO_TITLES,
+    ...params.candidateSummary.scenarios.map((scenario) => scenario.name),
+    ...params.baselineSummary.scenarios.map((scenario) => scenario.name),
+  ]);
+  const candidateByName = new Map(
+    params.candidateSummary.scenarios.map((scenario) => [scenario.name, scenario]),
+  );
+  const baselineByName = new Map(
+    params.baselineSummary.scenarios.map((scenario) => [scenario.name, scenario]),
+  );
+
+  const scenarioComparisons = [...scenarioNames]
+    .toSorted((left, right) => left.localeCompare(right))
+    .map((name) => {
+      const candidate = candidateByName.get(name);
+      const baseline = baselineByName.get(name);
+      const candidateStatus = candidate ? normalizeScenarioStatus(candidate.status) : "missing";
+      const baselineStatus = baseline ? normalizeScenarioStatus(baseline.status) : "missing";
+      const comparison: QaAgenticParityScenarioComparison = {
+        name,
+        candidateStatus,
+        baselineStatus,
+      };
+      if (candidate?.details) {
+        comparison.candidateDetails = candidate.details;
+      }
+      if (baseline?.details) {
+        comparison.baselineDetails = baseline.details;
+      }
+      return comparison;
+    });
+
+  const failures: string[] = [];
+  const requiredScenarioStatuses = QA_AGENTIC_PARITY_SCENARIO_TITLES.map((name) => {
+    const candidate = candidateByName.get(name);
+    const baseline = baselineByName.get(name);
+    return {
+      name,
+      candidateStatus: requiredCoverageStatus(candidate),
+      baselineStatus: requiredCoverageStatus(baseline),
+    };
+  });
+  const requiredScenarioCoverage = requiredScenarioStatuses.filter(
+    (scenario) =>
+      scenario.candidateStatus === "missing" ||
+      scenario.baselineStatus === "missing" ||
+      scenario.candidateStatus === "skip" ||
+      scenario.baselineStatus === "skip",
+  );
+  for (const scenario of requiredScenarioCoverage) {
+    failures.push(
+      `Missing required parity scenario coverage for ${scenario.name}: ${params.candidateLabel}=${scenario.candidateStatus}, ${params.baselineLabel}=${scenario.baselineStatus}.`,
+    );
+  }
+  // Required parity scenarios that ran on both sides but FAILED also fail
+  // the gate. Without this check, a run where both models fail the same
+  // required scenarios still produced pass=true, because the downstream
+  // metric comparisons are purely relative (candidate vs baseline) and
+  // the suspicious-pass fake-success check only catches passes that carry
+  // failure-sounding details. Excluding missing/skip here keeps operator
+  // output from double-counting the same scenario with two lines.
+  const requiredScenarioFailures = requiredScenarioStatuses.filter(
+    (scenario) =>
+      scenario.candidateStatus !== "missing" &&
+      scenario.baselineStatus !== "missing" &&
+      scenario.candidateStatus !== "skip" &&
+      scenario.baselineStatus !== "skip" &&
+      (scenario.candidateStatus === "fail" || scenario.baselineStatus === "fail"),
+  );
+  for (const scenario of requiredScenarioFailures) {
+    failures.push(
+      `Required parity scenario ${scenario.name} failed: ${params.candidateLabel}=${scenario.candidateStatus}, ${params.baselineLabel}=${scenario.baselineStatus}.`,
+    );
+  }
+  // Required parity scenarios are already reported via `requiredScenarioCoverage`
+  // above; excluding them here keeps the operator-facing failure list from
+  // double-counting the same missing scenario (one "Missing required parity scenario
+  // coverage for X" line plus a "Scenario coverage mismatch for X" line on the same
+  // scenario).
+  const coverageMismatch = scenarioComparisons.filter(
+    (scenario) =>
+      !parityTitleSet.has(scenario.name) &&
+      (scenario.candidateStatus === "missing" || scenario.baselineStatus === "missing"),
+  );
+  for (const scenario of coverageMismatch) {
+    failures.push(
+      `Scenario coverage mismatch for ${scenario.name}: ${params.candidateLabel}=${scenario.candidateStatus}, ${params.baselineLabel}=${scenario.baselineStatus}.`,
+    );
+  }
+  if (candidateMetrics.completionRate < baselineMetrics.completionRate) {
+    failures.push(
+      `${params.candidateLabel} completion rate ${formatPercent(candidateMetrics.completionRate)} is below ${params.baselineLabel} ${formatPercent(baselineMetrics.completionRate)}.`,
+    );
+  }
+  if (candidateMetrics.unintendedStopRate > baselineMetrics.unintendedStopRate) {
+    failures.push(
+      `${params.candidateLabel} unintended-stop rate ${formatPercent(candidateMetrics.unintendedStopRate)} exceeds ${params.baselineLabel} ${formatPercent(baselineMetrics.unintendedStopRate)}.`,
+    );
+  }
+  if (candidateMetrics.validToolCallRate < baselineMetrics.validToolCallRate) {
+    failures.push(
+      `${params.candidateLabel} valid-tool-call rate ${formatPercent(candidateMetrics.validToolCallRate)} is below ${params.baselineLabel} ${formatPercent(baselineMetrics.validToolCallRate)}.`,
+    );
+  }
+  if (candidateMetrics.fakeSuccessCount > 0) {
+    failures.push(
+      `${params.candidateLabel} produced ${candidateMetrics.fakeSuccessCount} suspicious pass result(s); fake-success count must be 0.`,
+    );
+  }
+  if (baselineMetrics.fakeSuccessCount > 0) {
+    failures.push(
+      `${params.baselineLabel} produced ${baselineMetrics.fakeSuccessCount} suspicious pass result(s); baseline fake-success count must also be 0.`,
+    );
+  }
+
+  return {
+    candidateLabel: params.candidateLabel,
+    baselineLabel: params.baselineLabel,
+    comparedAt: params.comparedAt ?? new Date().toISOString(),
+    candidateMetrics,
+    baselineMetrics,
+    scenarioComparisons,
+    pass: failures.length === 0,
+    failures,
+    notes: [
+      "First-wave valid-tool-call rate is scenario-level and uses passing tool-mediated scenarios as the verified numerator.",
+      "Auth/proxy/DNS correctness is intentionally out of scope for this parity report and should be gated by the deterministic runtime-truthfulness suites.",
+    ],
+  };
+}
+
+export function renderQaAgenticParityMarkdownReport(comparison: QaAgenticParityComparison): string {
+  // Title is parametrized from the candidate / baseline labels so reports
+  // for any candidate/baseline pair (not only gpt-5.4 vs opus 4.6) render
+  // with an accurate header. The default CLI labels are still
+  // openai/gpt-5.4 vs anthropic/claude-opus-4-6, but the helper works for
+  // any parity comparison a caller configures.
+  const lines = [
+    `# OpenClaw Agentic Parity Report — ${comparison.candidateLabel} vs ${comparison.baselineLabel}`,
+    "",
+    `- Compared at: ${comparison.comparedAt}`,
+    `- Candidate: ${comparison.candidateLabel}`,
+    `- Baseline: ${comparison.baselineLabel}`,
+    `- Verdict: ${comparison.pass ? "pass" : "fail"}`,
+    "",
+    "## Aggregate Metrics",
+    "",
+    "| Metric | Candidate | Baseline |",
+    "| --- | ---: | ---: |",
+    `| Completion rate | ${formatPercent(comparison.candidateMetrics.completionRate)} | ${formatPercent(comparison.baselineMetrics.completionRate)} |`,
+    `| Unintended-stop rate | ${formatPercent(comparison.candidateMetrics.unintendedStopRate)} | ${formatPercent(comparison.baselineMetrics.unintendedStopRate)} |`,
+    `| Valid-tool-call rate | ${formatPercent(comparison.candidateMetrics.validToolCallRate)} | ${formatPercent(comparison.baselineMetrics.validToolCallRate)} |`,
+    `| Fake-success count | ${comparison.candidateMetrics.fakeSuccessCount} | ${comparison.baselineMetrics.fakeSuccessCount} |`,
+    "",
+  ];
+
+  if (comparison.failures.length > 0) {
+    lines.push("## Gate Failures", "");
+    for (const failure of comparison.failures) {
+      lines.push(`- ${failure}`);
+    }
+    lines.push("");
+  }
+
+  lines.push("## Scenario Comparison", "");
+  for (const scenario of comparison.scenarioComparisons) {
+    lines.push(`### ${scenario.name}`, "");
+    lines.push(`- ${comparison.candidateLabel}: ${scenario.candidateStatus}`);
+    lines.push(`- ${comparison.baselineLabel}: ${scenario.baselineStatus}`);
+    if (scenario.candidateDetails) {
+      lines.push(`- ${comparison.candidateLabel} details: ${scenario.candidateDetails}`);
+    }
+    if (scenario.baselineDetails) {
+      lines.push(`- ${comparison.baselineLabel} details: ${scenario.baselineDetails}`);
+    }
+    lines.push("");
+  }
+
+  lines.push("## Notes", "");
+  for (const note of comparison.notes) {
+    lines.push(`- ${note}`);
+  }
+  lines.push("");
+
+  return lines.join("\n");
+}

openclaw/extensions/qa-lab/src/agentic-parity.ts

@@ -0,0 +1,85 @@
+export const QA_AGENTIC_PARITY_PACK = "agentic";
+
+export const QA_AGENTIC_PARITY_SCENARIOS = [
+  {
+    id: "approval-turn-tool-followthrough",
+    title: "Approval turn tool followthrough",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "model-switch-tool-continuity",
+    title: "Model switch with tool continuity",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "source-docs-discovery-report",
+    title: "Source and docs discovery report",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "image-understanding-attachment",
+    title: "Image understanding from attachment",
+    countsTowardValidToolCallRate: false,
+  },
+  {
+    id: "compaction-retry-mutating-tool",
+    title: "Compaction retry after mutating tool",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "subagent-handoff",
+    title: "Subagent handoff",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "subagent-fanout-synthesis",
+    title: "Subagent fanout synthesis",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "memory-recall",
+    title: "Memory recall after context switch",
+    countsTowardValidToolCallRate: false,
+  },
+  {
+    id: "thread-memory-isolation",
+    title: "Thread memory isolation",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "config-restart-capability-flip",
+    title: "Config restart capability flip",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "instruction-followthrough-repo-contract",
+    title: "Instruction followthrough repo contract",
+    countsTowardValidToolCallRate: true,
+  },
+] as const;
+
+export const QA_AGENTIC_PARITY_SCENARIO_IDS = QA_AGENTIC_PARITY_SCENARIOS.map(({ id }) => id);
+export const QA_AGENTIC_PARITY_SCENARIO_TITLES = QA_AGENTIC_PARITY_SCENARIOS.map(
+  ({ title }) => title,
+);
+export const QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES = QA_AGENTIC_PARITY_SCENARIOS.filter(
+  ({ countsTowardValidToolCallRate }) => countsTowardValidToolCallRate,
+).map(({ title }) => title);
+
+export function resolveQaParityPackScenarioIds(params: {
+  parityPack?: string;
+  scenarioIds?: string[];
+}): string[] {
+  const normalizedPack = params.parityPack?.trim().toLowerCase();
+  const explicitScenarioIds = [...new Set(params.scenarioIds ?? [])];
+  if (!normalizedPack) {
+    return explicitScenarioIds;
+  }
+  if (normalizedPack !== QA_AGENTIC_PARITY_PACK) {
+    throw new Error(
+      `--parity-pack must be "${QA_AGENTIC_PARITY_PACK}", got "${params.parityPack}"`,
+    );
+  }
+
+  return [...new Set([...explicitScenarioIds, ...QA_AGENTIC_PARITY_SCENARIO_IDS])];
+}

openclaw/extensions/qa-lab/src/browser-runtime.test.ts

@@ -0,0 +1,169 @@
+import { describe, expect, it, vi } from "vitest";
+import {
+  callQaBrowserRequest,
+  qaBrowserAct,
+  qaBrowserOpenTab,
+  qaBrowserSnapshot,
+  waitForQaBrowserReady,
+} from "./browser-runtime.js";
+
+function createEnv() {
+  return {
+    gateway: {
+      call: vi.fn(async () => ({ ok: true })),
+    },
+  };
+}
+
+describe("browser-runtime", () => {
+  it("sends normalized browser.request payloads through the gateway", async () => {
+    const env = createEnv();
+
+    const result = await callQaBrowserRequest(env, {
+      method: "GET",
+      path: "/snapshot",
+      query: {
+        format: "ai",
+        targetId: "tab-1",
+        skip: undefined,
+        limit: 50,
+      },
+      timeoutMs: 12_345,
+    });
+
+    expect(result).toEqual({ ok: true });
+    expect(env.gateway.call).toHaveBeenCalledWith(
+      "browser.request",
+      {
+        method: "GET",
+        path: "/snapshot",
+        query: {
+          format: "ai",
+          targetId: "tab-1",
+          limit: "50",
+        },
+        body: undefined,
+        timeoutMs: 12_345,
+      },
+      { timeoutMs: 12_345 },
+    );
+  });
+
+  it("opens tabs through the browser proxy", async () => {
+    const env = createEnv();
+
+    await qaBrowserOpenTab(env, {
+      url: "http://127.0.0.1:43124/control-ui/chat?session=test",
+      profile: "openclaw",
+    });
+
+    expect(env.gateway.call).toHaveBeenCalledWith(
+      "browser.request",
+      {
+        method: "POST",
+        path: "/tabs/open",
+        query: {
+          profile: "openclaw",
+        },
+        body: {
+          url: "http://127.0.0.1:43124/control-ui/chat?session=test",
+        },
+        timeoutMs: 20_000,
+      },
+      { timeoutMs: 20_000 },
+    );
+  });
+
+  it("captures snapshots with query options", async () => {
+    const env = createEnv();
+
+    await qaBrowserSnapshot(env, {
+      targetId: "tab-1",
+      interactive: true,
+      labels: true,
+      maxChars: 4_000,
+    });
+
+    expect(env.gateway.call).toHaveBeenCalledWith(
+      "browser.request",
+      {
+        method: "GET",
+        path: "/snapshot",
+        query: {
+          targetId: "tab-1",
+          format: "ai",
+          interactive: "true",
+          labels: "true",
+          maxChars: "4000",
+        },
+        body: undefined,
+        timeoutMs: 20_000,
+      },
+      { timeoutMs: 20_000 },
+    );
+  });
+
+  it("runs browser act requests through /act", async () => {
+    const env = createEnv();
+
+    await qaBrowserAct(env, {
+      profile: "openclaw",
+      request: {
+        kind: "type",
+        ref: "12",
+        text: "hello",
+        submit: true,
+      },
+      timeoutMs: 9_000,
+    });
+
+    expect(env.gateway.call).toHaveBeenCalledWith(
+      "browser.request",
+      {
+        method: "POST",
+        path: "/act",
+        query: {
+          profile: "openclaw",
+        },
+        body: {
+          kind: "type",
+          ref: "12",
+          text: "hello",
+          submit: true,
+        },
+        timeoutMs: 9_000,
+      },
+      { timeoutMs: 9_000 },
+    );
+  });
+
+  it("waits until browser control reports a ready profile", async () => {
+    const env = createEnv();
+    env.gateway.call = vi
+      .fn()
+      .mockResolvedValueOnce({ enabled: true, running: false, cdpReady: false })
+      .mockResolvedValueOnce({ enabled: true, running: true, cdpReady: true });
+
+    const status = await waitForQaBrowserReady(env, {
+      profile: "user",
+      timeoutMs: 5_000,
+      intervalMs: 1,
+    });
+
+    expect(status).toEqual({ enabled: true, running: true, cdpReady: true });
+    expect(env.gateway.call).toHaveBeenNthCalledWith(
+      1,
+      "browser.request",
+      {
+        method: "GET",
+        path: "/",
+        query: {
+          profile: "user",
+        },
+        body: undefined,
+        timeoutMs: 5_000,
+      },
+      { timeoutMs: 5_000 },
+    );
+  });
+});

openclaw/extensions/qa-lab/src/browser-runtime.ts

@@ -0,0 +1,212 @@
+type QaBrowserGateway = {
+  call: (
+    method: string,
+    params: Record<string, unknown>,
+    opts?: { timeoutMs?: number },
+  ) => Promise<unknown>;
+};
+
+type QaBrowserEnv = {
+  gateway: QaBrowserGateway;
+};
+
+type QaBrowserRequestParams = {
+  method: "GET" | "POST" | "DELETE";
+  path: string;
+  query?: Record<string, string | number | boolean | undefined>;
+  body?: unknown;
+  timeoutMs?: number;
+};
+
+type QaBrowserOpenTabParams = {
+  url: string;
+  profile?: string;
+  timeoutMs?: number;
+};
+
+type QaBrowserSnapshotParams = {
+  profile?: string;
+  targetId?: string;
+  format?: "ai" | "aria";
+  limit?: number;
+  interactive?: boolean;
+  compact?: boolean;
+  depth?: number;
+  selector?: string;
+  frame?: string;
+  labels?: boolean;
+  mode?: "efficient";
+  maxChars?: number;
+  timeoutMs?: number;
+};
+
+type QaBrowserActRequest = {
+  kind: string;
+  targetId?: string;
+  ref?: string;
+  doubleClick?: boolean;
+  button?: string;
+  modifiers?: string[];
+  text?: string;
+  submit?: boolean;
+  slowly?: boolean;
+  key?: string;
+  delayMs?: number;
+  startRef?: string;
+  endRef?: string;
+  values?: string[];
+  fields?: Array<Record<string, unknown>>;
+  width?: number;
+  height?: number;
+  timeMs?: number;
+  selector?: string;
+  url?: string;
+  loadState?: string;
+  textGone?: string;
+  timeoutMs?: number;
+  fn?: string;
+};
+
+type QaBrowserActParams = {
+  profile?: string;
+  request: QaBrowserActRequest;
+  timeoutMs?: number;
+};
+
+type QaBrowserStatus = {
+  enabled?: boolean;
+  running?: boolean;
+  cdpReady?: boolean;
+};
+
+type QaBrowserReadyParams = {
+  profile?: string;
+  timeoutMs?: number;
+  intervalMs?: number;
+};
+
+function normalizeBrowserQuery(
+  query: QaBrowserRequestParams["query"],
+): Record<string, string> | undefined {
+  if (!query) {
+    return undefined;
+  }
+  const normalized = Object.fromEntries(
+    Object.entries(query)
+      .filter(([, value]) => value !== undefined)
+      .map(([key, value]) => [key, String(value)]),
+  );
+  return Object.keys(normalized).length > 0 ? normalized : undefined;
+}
+
+function resolveBrowserTimeoutMs(timeoutMs: number | undefined, fallbackMs: number) {
+  if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
+    return fallbackMs;
+  }
+  return Math.max(1, Math.floor(timeoutMs));
+}
+
+export async function callQaBrowserRequest<T = unknown>(
+  env: QaBrowserEnv,
+  params: QaBrowserRequestParams,
+): Promise<T> {
+  const timeoutMs = resolveBrowserTimeoutMs(params.timeoutMs, 20_000);
+  const payload = await env.gateway.call(
+    "browser.request",
+    {
+      method: params.method,
+      path: params.path,
+      query: normalizeBrowserQuery(params.query),
+      body: params.body,
+      timeoutMs,
+    },
+    { timeoutMs },
+  );
+  return payload as T;
+}
+
+export async function qaBrowserOpenTab<T = unknown>(
+  env: QaBrowserEnv,
+  params: QaBrowserOpenTabParams,
+): Promise<T> {
+  return await callQaBrowserRequest<T>(env, {
+    method: "POST",
+    path: "/tabs/open",
+    query: params.profile ? { profile: params.profile } : undefined,
+    body: { url: params.url },
+    timeoutMs: resolveBrowserTimeoutMs(params.timeoutMs, 20_000),
+  });
+}
+
+export async function qaBrowserSnapshot<T = unknown>(
+  env: QaBrowserEnv,
+  params: QaBrowserSnapshotParams = {},
+): Promise<T> {
+  return await callQaBrowserRequest<T>(env, {
+    method: "GET",
+    path: "/snapshot",
+    query: {
+      profile: params.profile,
+      targetId: params.targetId,
+      format: params.format ?? "ai",
+      limit: params.limit,
+      interactive: params.interactive,
+      compact: params.compact,
+      depth: params.depth,
+      selector: params.selector,
+      frame: params.frame,
+      labels: params.labels,
+      mode: params.mode,
+      maxChars: params.maxChars,
+    },
+    timeoutMs: resolveBrowserTimeoutMs(params.timeoutMs, 20_000),
+  });
+}
+
+export async function qaBrowserAct<T = unknown>(
+  env: QaBrowserEnv,
+  params: QaBrowserActParams,
+): Promise<T> {
+  return await callQaBrowserRequest<T>(env, {
+    method: "POST",
+    path: "/act",
+    query: params.profile ? { profile: params.profile } : undefined,
+    body: params.request,
+    timeoutMs: resolveBrowserTimeoutMs(params.timeoutMs, 20_000),
+  });
+}
+
+function isQaBrowserReady(status: QaBrowserStatus | null | undefined) {
+  return status?.enabled === true && status?.running === true && status?.cdpReady === true;
+}
+
+function sleep(ms: number) {
+  return new Promise<void>((resolve) => setTimeout(resolve, ms));
+}
+
+export async function waitForQaBrowserReady<T extends QaBrowserStatus = QaBrowserStatus>(
+  env: QaBrowserEnv,
+  params: QaBrowserReadyParams = {},
+): Promise<T> {
+  const timeoutMs = resolveBrowserTimeoutMs(params.timeoutMs, 20_000);
+  const intervalMs = resolveBrowserTimeoutMs(params.intervalMs, 250);
+  const startedAt = Date.now();
+  let lastStatus: QaBrowserStatus | null = null;
+  while (Date.now() - startedAt < timeoutMs) {
+    lastStatus = await callQaBrowserRequest<QaBrowserStatus>(env, {
+      method: "GET",
+      path: "/",
+      query: params.profile ? { profile: params.profile } : undefined,
+      timeoutMs: Math.min(timeoutMs, 5_000),
+    });
+    if (isQaBrowserReady(lastStatus)) {
+      return lastStatus as T;
+    }
+    await sleep(intervalMs);
+  }
+  throw new Error(
+    `browser control not ready after ${timeoutMs}ms${
+      lastStatus ? ` (${JSON.stringify(lastStatus)})` : ""
+    }`,
+  );
+}

openclaw/extensions/qa-lab/src/bundled-plugin-staging.ts

@@ -0,0 +1,428 @@
+import { existsSync } from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
+
+const QA_ALWAYS_STAGE_RUNTIME_PLUGIN_IDS = Object.freeze([
+  "image-generation-core",
+  "media-understanding-core",
+  "speech-core",
+]);
+const QA_OPENAI_PLUGIN_ID = "openai";
+const QA_BUNDLED_PLUGIN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+const QA_CLI_METADATA_ENTRY_BASENAMES = Object.freeze([
+  "cli-metadata.ts",
+  "cli-metadata.js",
+  "cli-metadata.mjs",
+  "cli-metadata.cjs",
+]);
+
+function assertSafeQaBundledPluginId(pluginId: string) {
+  if (!QA_BUNDLED_PLUGIN_ID_PATTERN.test(pluginId)) {
+    throw new Error(`invalid QA bundled plugin id: ${pluginId}`);
+  }
+}
+
+function parseStableSemverFloor(value: string | undefined) {
+  if (!value) {
+    return null;
+  }
+  const match = value.trim().match(/(\d+)\.(\d+)\.(\d+)/);
+  if (!match) {
+    return null;
+  }
+  return {
+    major: Number.parseInt(match[1] ?? "", 10),
+    minor: Number.parseInt(match[2] ?? "", 10),
+    patch: Number.parseInt(match[3] ?? "", 10),
+    label: `${match[1]}.${match[2]}.${match[3]}`,
+  };
+}
+
+function compareSemverFloors(
+  left: ReturnType<typeof parseStableSemverFloor>,
+  right: ReturnType<typeof parseStableSemverFloor>,
+) {
+  if (!left && !right) {
+    return 0;
+  }
+  if (!left) {
+    return -1;
+  }
+  if (!right) {
+    return 1;
+  }
+  if (left.major !== right.major) {
+    return left.major - right.major;
+  }
+  if (left.minor !== right.minor) {
+    return left.minor - right.minor;
+  }
+  return left.patch - right.patch;
+}
+
+function isQaOpenAiResponsesProviderConfig(config: ModelProviderConfig) {
+  return (
+    config.api === "openai-responses" ||
+    config.models.some((model) => model.api === "openai-responses")
+  );
+}
+
+export function resolveQaBundledPluginSourceDir(params: { repoRoot: string; pluginId: string }) {
+  assertSafeQaBundledPluginId(params.pluginId);
+  const candidates = [
+    path.join(params.repoRoot, "dist", "extensions", params.pluginId),
+    path.join(params.repoRoot, "dist-runtime", "extensions", params.pluginId),
+    path.join(params.repoRoot, "extensions", params.pluginId),
+  ];
+  const existingCandidates = candidates.filter((candidate) => existsSync(candidate));
+  if (existingCandidates.length === 0) {
+    return null;
+  }
+  const cliMetadataCandidate = existingCandidates.find((candidate) =>
+    QA_CLI_METADATA_ENTRY_BASENAMES.some((basename) => existsSync(path.join(candidate, basename))),
+  );
+  if (cliMetadataCandidate) {
+    return cliMetadataCandidate;
+  }
+  return existingCandidates[0] ?? null;
+}
+
+function resolveQaBundledPluginScanRoots(repoRoot: string) {
+  return [
+    path.join(repoRoot, "dist", "extensions"),
+    path.join(repoRoot, "dist-runtime", "extensions"),
+    path.join(repoRoot, "extensions"),
+  ].filter((candidate, index, all) => existsSync(candidate) && all.indexOf(candidate) === index);
+}
+
+export async function resolveQaOwnerPluginIdsForProviderIds(params: {
+  repoRoot: string;
+  providerIds: readonly string[];
+  providerConfigs?: Record<string, ModelProviderConfig>;
+}) {
+  const providerIds = [
+    ...new Set(params.providerIds.map((providerId) => providerId.trim())),
+  ].filter((providerId) => providerId.length > 0);
+  if (providerIds.length === 0) {
+    return [];
+  }
+  const remainingProviderIds = new Set(providerIds);
+  const ownerPluginIds = new Set<string>();
+  const visitedPluginIds = new Set<string>();
+  for (const sourceRoot of resolveQaBundledPluginScanRoots(params.repoRoot)) {
+    for (const entry of await fs.readdir(sourceRoot, { withFileTypes: true })) {
+      if (!entry.isDirectory()) {
+        continue;
+      }
+      const manifestPath = path.join(sourceRoot, entry.name, "openclaw.plugin.json");
+      if (!existsSync(manifestPath)) {
+        continue;
+      }
+      const manifest = JSON.parse(await fs.readFile(manifestPath, "utf8")) as {
+        id?: unknown;
+        providers?: unknown;
+        cliBackends?: unknown;
+      };
+      const pluginId = typeof manifest.id === "string" ? manifest.id.trim() : entry.name;
+      if (!pluginId || visitedPluginIds.has(pluginId)) {
+        continue;
+      }
+      visitedPluginIds.add(pluginId);
+      const ownedIds = new Set(
+        [
+          pluginId,
+          ...(Array.isArray(manifest.providers) ? manifest.providers : []),
+          ...(Array.isArray(manifest.cliBackends) ? manifest.cliBackends : []),
+        ].filter((ownedId): ownedId is string => typeof ownedId === "string"),
+      );
+      for (const providerId of providerIds) {
+        if (!ownedIds.has(providerId)) {
+          continue;
+        }
+        ownerPluginIds.add(pluginId);
+        remainingProviderIds.delete(providerId);
+      }
+    }
+  }
+  for (const providerId of remainingProviderIds) {
+    const providerConfig = params.providerConfigs?.[providerId];
+    if (providerConfig && isQaOpenAiResponsesProviderConfig(providerConfig)) {
+      ownerPluginIds.add(QA_OPENAI_PLUGIN_ID);
+      continue;
+    }
+    ownerPluginIds.add(providerId);
+  }
+  return [...ownerPluginIds];
+}
+
+function collectQaBundledPluginIds(params: {
+  repoRoot: string;
+  allowedPluginIds: readonly string[];
+}) {
+  const pluginIds = new Set(
+    params.allowedPluginIds.map((pluginId) => {
+      assertSafeQaBundledPluginId(pluginId);
+      return pluginId;
+    }),
+  );
+  for (const pluginId of QA_ALWAYS_STAGE_RUNTIME_PLUGIN_IDS) {
+    if (
+      resolveQaBundledPluginSourceDir({
+        repoRoot: params.repoRoot,
+        pluginId,
+      })
+    ) {
+      pluginIds.add(pluginId);
+    }
+  }
+  return [...pluginIds];
+}
+
+function resolveQaStagedBundledTreeName(repoRoot: string) {
+  if (existsSync(path.join(repoRoot, "dist"))) {
+    return "dist";
+  }
+  if (existsSync(path.join(repoRoot, "dist-runtime"))) {
+    return "dist-runtime";
+  }
+  return "dist";
+}
+
+function resolveQaBuiltBundledPluginTreeRoot(params: { repoRoot: string; sourceDir: string }) {
+  const sourceDir = path.resolve(params.sourceDir);
+  for (const treeName of ["dist", "dist-runtime"] as const) {
+    const extensionsRoot = path.join(params.repoRoot, treeName, "extensions");
+    const relativeSourceDir = path.relative(extensionsRoot, sourceDir);
+    if (
+      relativeSourceDir.length > 0 &&
+      !relativeSourceDir.startsWith("..") &&
+      !path.isAbsolute(relativeSourceDir)
+    ) {
+      return path.join(params.repoRoot, treeName);
+    }
+  }
+  return null;
+}
+
+async function symlinkQaStagedDirEntry(params: {
+  sourcePath: string;
+  targetPath: string;
+  directory?: boolean;
+}) {
+  await fs.symlink(
+    params.sourcePath,
+    params.targetPath,
+    params.directory ? (process.platform === "win32" ? "junction" : "dir") : "file",
+  );
+}
+
+async function resolveQaStagedDirEntryDirectory(params: {
+  sourcePath: string;
+  entry?: {
+    isDirectory(): boolean;
+    isSymbolicLink(): boolean;
+  };
+}) {
+  if (params.entry?.isDirectory()) {
+    return true;
+  }
+  if (params.entry?.isSymbolicLink()) {
+    return (await fs.stat(params.sourcePath)).isDirectory();
+  }
+  if (params.entry) {
+    return false;
+  }
+  return (await fs.lstat(params.sourcePath)).isDirectory();
+}
+
+async function seedQaStagedNodeModules(params: { repoRoot: string; stagedRoot: string }) {
+  const sourceNodeModulesDir = path.join(params.repoRoot, "node_modules");
+  if (!existsSync(sourceNodeModulesDir)) {
+    return;
+  }
+  const stagedNodeModulesDir = path.join(params.stagedRoot, "node_modules");
+  await fs.mkdir(stagedNodeModulesDir, { recursive: true });
+  for (const entry of await fs.readdir(sourceNodeModulesDir, { withFileTypes: true })) {
+    if (entry.name === "openclaw") {
+      continue;
+    }
+    await symlinkQaStagedDirEntry({
+      sourcePath: path.join(sourceNodeModulesDir, entry.name),
+      targetPath: path.join(stagedNodeModulesDir, entry.name),
+      directory: await resolveQaStagedDirEntryDirectory({
+        sourcePath: path.join(sourceNodeModulesDir, entry.name),
+        entry,
+      }),
+    });
+  }
+}
+
+function collectQaBuiltTreeRoots(params: {
+  repoRoot: string;
+  stagedPluginIds: readonly string[];
+  stagedTreeName: string;
+}) {
+  const treeRoots = new Set<string>();
+  treeRoots.add(path.join(params.repoRoot, params.stagedTreeName));
+  for (const pluginId of params.stagedPluginIds) {
+    const sourceDir = resolveQaBundledPluginSourceDir({
+      repoRoot: params.repoRoot,
+      pluginId,
+    });
+    if (!sourceDir) {
+      continue;
+    }
+    const builtTreeRoot = resolveQaBuiltBundledPluginTreeRoot({
+      repoRoot: params.repoRoot,
+      sourceDir,
+    });
+    if (builtTreeRoot) {
+      treeRoots.add(builtTreeRoot);
+    }
+  }
+  return [...treeRoots];
+}
+
+async function seedQaStagedBuiltTreeRoots(params: {
+  stagedTreeRoot: string;
+  sourceTreeRoots: readonly string[];
+}) {
+  for (const sourceTreeRoot of params.sourceTreeRoots) {
+    if (!existsSync(sourceTreeRoot)) {
+      continue;
+    }
+    for (const entry of await fs.readdir(sourceTreeRoot, { withFileTypes: true })) {
+      if (entry.name === "extensions") {
+        continue;
+      }
+      const targetPath = path.join(params.stagedTreeRoot, entry.name);
+      if (existsSync(targetPath)) {
+        continue;
+      }
+      await symlinkQaStagedDirEntry({
+        sourcePath: path.join(sourceTreeRoot, entry.name),
+        targetPath,
+        directory: await resolveQaStagedDirEntryDirectory({
+          sourcePath: path.join(sourceTreeRoot, entry.name),
+          entry,
+        }),
+      });
+    }
+  }
+}
+
+export async function resolveQaRuntimeHostVersion(params: {
+  repoRoot: string;
+  allowedPluginIds: readonly string[];
+}) {
+  const rootPackageRaw = await fs.readFile(path.join(params.repoRoot, "package.json"), "utf8");
+  const rootPackage = JSON.parse(rootPackageRaw) as { version?: string };
+  let selected = parseStableSemverFloor(rootPackage.version);
+  const stagedPluginIds = collectQaBundledPluginIds({
+    repoRoot: params.repoRoot,
+    allowedPluginIds: params.allowedPluginIds,
+  });
+
+  for (const pluginId of stagedPluginIds) {
+    const sourceDir = resolveQaBundledPluginSourceDir({
+      repoRoot: params.repoRoot,
+      pluginId,
+    });
+    if (!sourceDir) {
+      continue;
+    }
+    const packagePath = path.join(sourceDir, "package.json");
+    if (!existsSync(packagePath)) {
+      continue;
+    }
+    const packageRaw = await fs.readFile(packagePath, "utf8");
+    const packageJson = JSON.parse(packageRaw) as {
+      openclaw?: {
+        install?: {
+          minHostVersion?: string;
+        };
+      };
+    };
+    const candidate = parseStableSemverFloor(packageJson.openclaw?.install?.minHostVersion);
+    if (compareSemverFloors(candidate, selected) > 0) {
+      selected = candidate;
+    }
+  }
+
+  return selected?.label;
+}
+
+export async function createQaBundledPluginsDir(params: {
+  repoRoot: string;
+  tempRoot: string;
+  allowedPluginIds: readonly string[];
+}) {
+  const stagedPluginIds = collectQaBundledPluginIds({
+    repoRoot: params.repoRoot,
+    allowedPluginIds: params.allowedPluginIds,
+  });
+  const stagedRoot = path.join(
+    params.repoRoot,
+    ".artifacts",
+    "qa-runtime",
+    path.basename(params.tempRoot),
+  );
+  await fs.rm(stagedRoot, { recursive: true, force: true });
+  await fs.mkdir(stagedRoot, { recursive: true });
+  await fs.copyFile(
+    path.join(params.repoRoot, "package.json"),
+    path.join(stagedRoot, "package.json"),
+  );
+  await seedQaStagedNodeModules({
+    repoRoot: params.repoRoot,
+    stagedRoot,
+  });
+  const stagedOpenClawPackageDir = path.join(stagedRoot, "node_modules", "openclaw");
+  await fs.mkdir(stagedOpenClawPackageDir, { recursive: true });
+  await fs.copyFile(
+    path.join(params.repoRoot, "package.json"),
+    path.join(stagedOpenClawPackageDir, "package.json"),
+  );
+  const stagedTreeName = resolveQaStagedBundledTreeName(params.repoRoot);
+  const stagedTreeRoot = path.join(stagedRoot, stagedTreeName);
+  await fs.mkdir(stagedTreeRoot, { recursive: true });
+  await seedQaStagedBuiltTreeRoots({
+    stagedTreeRoot,
+    sourceTreeRoots: collectQaBuiltTreeRoots({
+      repoRoot: params.repoRoot,
+      stagedPluginIds,
+      stagedTreeName,
+    }),
+  });
+  if (stagedTreeName === "dist-runtime" && !existsSync(path.join(stagedRoot, "dist"))) {
+    const repoDistDir = path.join(params.repoRoot, "dist");
+    const stagedDistTarget = existsSync(repoDistDir) ? repoDistDir : stagedTreeRoot;
+    await symlinkQaStagedDirEntry({
+      sourcePath: stagedDistTarget,
+      targetPath: path.join(stagedRoot, "dist"),
+      directory: true,
+    });
+  }
+  const bundledPluginsDir = path.join(stagedTreeRoot, "extensions");
+  await fs.mkdir(bundledPluginsDir, { recursive: true });
+  for (const pluginId of stagedPluginIds) {
+    const sourceDir = resolveQaBundledPluginSourceDir({
+      repoRoot: params.repoRoot,
+      pluginId,
+    });
+    if (!sourceDir) {
+      throw new Error(`qa bundled plugin not found: ${pluginId}`);
+    }
+    await fs.cp(sourceDir, path.join(bundledPluginsDir, pluginId), { recursive: true });
+  }
+  await symlinkQaStagedDirEntry({
+    sourcePath: path.join(stagedRoot, "dist"),
+    targetPath: path.join(stagedOpenClawPackageDir, "dist"),
+    directory: true,
+  });
+  return {
+    bundledPluginsDir,
+    stagedRoot,
+  };
+}

openclaw/extensions/qa-lab/src/bus-queries.ts

@@ -0,0 +1,162 @@
+import { normalizeOptionalLowercaseString } from "openclaw/plugin-sdk/text-runtime";
+import type {
+  QaBusAttachment,
+  QaBusConversation,
+  QaBusEvent,
+  QaBusMessage,
+  QaBusPollInput,
+  QaBusPollResult,
+  QaBusReadMessageInput,
+  QaBusSearchMessagesInput,
+  QaBusStateSnapshot,
+  QaBusThread,
+} from "./runtime-api.js";
+
+export const DEFAULT_ACCOUNT_ID = "default";
+
+export function normalizeAccountId(raw?: string): string {
+  const trimmed = raw?.trim();
+  return trimmed || DEFAULT_ACCOUNT_ID;
+}
+
+export function normalizeConversationFromTarget(target: string): {
+  conversation: QaBusConversation;
+  threadId?: string;
+} {
+  const trimmed = target.trim();
+  if (trimmed.startsWith("thread:")) {
+    const rest = trimmed.slice("thread:".length);
+    const slash = rest.indexOf("/");
+    if (slash > 0) {
+      return {
+        conversation: { id: rest.slice(0, slash), kind: "channel" },
+        threadId: rest.slice(slash + 1),
+      };
+    }
+  }
+  if (trimmed.startsWith("channel:")) {
+    return {
+      conversation: { id: trimmed.slice("channel:".length), kind: "channel" },
+    };
+  }
+  if (trimmed.startsWith("dm:")) {
+    return {
+      conversation: { id: trimmed.slice("dm:".length), kind: "direct" },
+    };
+  }
+  return {
+    conversation: { id: trimmed, kind: "direct" },
+  };
+}
+
+export function cloneMessage(message: QaBusMessage): QaBusMessage {
+  return {
+    ...message,
+    conversation: { ...message.conversation },
+    attachments: (message.attachments ?? []).map((attachment) => cloneAttachment(attachment)),
+    reactions: message.reactions.map((reaction) => ({ ...reaction })),
+  };
+}
+
+function cloneAttachment(attachment: QaBusAttachment): QaBusAttachment {
+  return { ...attachment };
+}
+
+export function cloneEvent(event: QaBusEvent): QaBusEvent {
+  switch (event.kind) {
+    case "inbound-message":
+    case "outbound-message":
+    case "message-edited":
+    case "message-deleted":
+    case "reaction-added":
+      return { ...event, message: cloneMessage(event.message) };
+    case "thread-created":
+      return { ...event, thread: { ...event.thread } };
+  }
+  throw new Error("Unsupported QA bus event kind");
+}
+
+export function buildQaBusSnapshot(params: {
+  cursor: number;
+  conversations: Map<string, QaBusConversation>;
+  threads: Map<string, QaBusThread>;
+  messages: Map<string, QaBusMessage>;
+  events: QaBusEvent[];
+}): QaBusStateSnapshot {
+  return {
+    cursor: params.cursor,
+    conversations: Array.from(params.conversations.values()).map((conversation) =>
+      Object.assign({}, conversation),
+    ),
+    threads: Array.from(params.threads.values()).map((thread) => Object.assign({}, thread)),
+    messages: Array.from(params.messages.values()).map((message) => cloneMessage(message)),
+    events: params.events.map((event) => cloneEvent(event)),
+  };
+}
+
+export function readQaBusMessage(params: {
+  messages: Map<string, QaBusMessage>;
+  input: QaBusReadMessageInput;
+}) {
+  const message = params.messages.get(params.input.messageId);
+  if (!message) {
+    throw new Error(`qa-bus message not found: ${params.input.messageId}`);
+  }
+  return cloneMessage(message);
+}
+
+export function searchQaBusMessages(params: {
+  messages: Map<string, QaBusMessage>;
+  input: QaBusSearchMessagesInput;
+}) {
+  const accountId = normalizeAccountId(params.input.accountId);
+  const limit = Math.max(1, Math.min(params.input.limit ?? 20, 100));
+  const query = normalizeOptionalLowercaseString(params.input.query);
+  return Array.from(params.messages.values())
+    .filter((message) => message.accountId === accountId)
+    .filter((message) =>
+      params.input.conversationId ? message.conversation.id === params.input.conversationId : true,
+    )
+    .filter((message) =>
+      params.input.threadId ? message.threadId === params.input.threadId : true,
+    )
+    .filter((message) => {
+      if (!query) {
+        return true;
+      }
+      const attachmentHaystack = message.attachments ?? [];
+      const searchableAttachmentText = attachmentHaystack
+        .flatMap((attachment) => [
+          attachment.fileName,
+          attachment.altText,
+          attachment.transcript,
+          attachment.mimeType,
+        ])
+        .filter((value): value is string => Boolean(value))
+        .join(" ")
+        .toLowerCase();
+      const messageText = normalizeOptionalLowercaseString(message.text) ?? "";
+      return `${messageText} ${searchableAttachmentText}`.includes(query);
+    })
+    .slice(-limit)
+    .map((message) => cloneMessage(message));
+}
+
+export function pollQaBusEvents(params: {
+  events: QaBusEvent[];
+  cursor: number;
+  input?: QaBusPollInput;
+}): QaBusPollResult {
+  const accountId = normalizeAccountId(params.input?.accountId);
+  const startCursor = params.input?.cursor ?? 0;
+  const effectiveStartCursor = params.cursor < startCursor ? 0 : startCursor;
+  const limit = Math.max(1, Math.min(params.input?.limit ?? 100, 500));
+  const matches = params.events
+    .filter((event) => event.accountId === accountId && event.cursor > effectiveStartCursor)
+    .slice(0, limit)
+    .map((event) => cloneEvent(event));
+  return {
+    cursor: params.cursor,
+    events: matches,
+  };
+}

openclaw/extensions/qa-lab/src/bus-server.test.ts

@@ -0,0 +1,59 @@
+import { Agent, createServer, request } from "node:http";
+import { describe, expect, it } from "vitest";
+import { closeQaHttpServer } from "./bus-server.js";
+
+async function listenOnLoopback(server: ReturnType<typeof createServer>): Promise<number> {
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => resolve());
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("expected server to bind a TCP port");
+  }
+  return address.port;
+}
+
+async function requestOnce(params: { port: number; agent: Agent }): Promise<void> {
+  await new Promise<void>((resolve, reject) => {
+    const req = request(
+      {
+        host: "127.0.0.1",
+        port: params.port,
+        path: "/",
+        agent: params.agent,
+      },
+      (res) => {
+        res.resume();
+        res.on("end", resolve);
+        res.on("error", reject);
+      },
+    );
+    req.on("error", reject);
+    req.end();
+  });
+}
+
+describe("closeQaHttpServer", () => {
+  it("closes idle keep-alive sockets so suite processes can exit", async () => {
+    const server = createServer((_req, res) => {
+      res.writeHead(200, {
+        "content-type": "text/plain",
+        connection: "keep-alive",
+      });
+      res.end("ok");
+    });
+    const agent = new Agent({ keepAlive: true });
+    const port = await listenOnLoopback(server);
+
+    try {
+      await requestOnce({ port, agent });
+      const startedAt = Date.now();
+      await closeQaHttpServer(server);
+      expect(Date.now() - startedAt).toBeLessThan(1_000);
+    } finally {
+      agent.destroy();
+      server.closeAllConnections?.();
+    }
+  });
+});

openclaw/extensions/qa-lab/src/bus-server.ts

@@ -0,0 +1,198 @@
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { normalizeAccountId } from "./bus-queries.js";
+import type { QaBusState } from "./bus-state.js";
+import type {
+  QaBusCreateThreadInput,
+  QaBusDeleteMessageInput,
+  QaBusEditMessageInput,
+  QaBusInboundMessageInput,
+  QaBusOutboundMessageInput,
+  QaBusPollInput,
+  QaBusReactToMessageInput,
+  QaBusReadMessageInput,
+  QaBusSearchMessagesInput,
+  QaBusWaitForInput,
+} from "./runtime-api.js";
+
+async function readJson(req: IncomingMessage): Promise<unknown> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of req) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  const text = Buffer.concat(chunks).toString("utf8").trim();
+  return text ? (JSON.parse(text) as unknown) : {};
+}
+
+export function writeJson(res: ServerResponse, statusCode: number, body: unknown) {
+  const payload = JSON.stringify(body);
+  res.writeHead(statusCode, {
+    "content-type": "application/json; charset=utf-8",
+    "content-length": Buffer.byteLength(payload),
+  });
+  res.end(payload);
+}
+
+export function writeError(res: ServerResponse, statusCode: number, error: unknown) {
+  writeJson(res, statusCode, {
+    error: formatErrorMessage(error),
+  });
+}
+
+export async function closeQaHttpServer(server: Server): Promise<void> {
+  let forceCloseTimer: NodeJS.Timeout | undefined;
+  try {
+    await new Promise<void>((resolve, reject) => {
+      server.close((error) => (error ? reject(error) : resolve()));
+      server.closeIdleConnections?.();
+      forceCloseTimer = setTimeout(() => {
+        server.closeAllConnections?.();
+      }, 250);
+      forceCloseTimer.unref();
+    });
+  } finally {
+    if (forceCloseTimer) {
+      clearTimeout(forceCloseTimer);
+    }
+  }
+}
+
+export async function handleQaBusRequest(params: {
+  req: IncomingMessage;
+  res: ServerResponse;
+  state: QaBusState;
+}): Promise<boolean> {
+  const method = params.req.method ?? "GET";
+  const url = new URL(params.req.url ?? "/", "http://127.0.0.1");
+
+  if (method === "GET" && url.pathname === "/health") {
+    writeJson(params.res, 200, { ok: true });
+    return true;
+  }
+
+  if (method === "GET" && url.pathname === "/v1/state") {
+    writeJson(params.res, 200, params.state.getSnapshot());
+    return true;
+  }
+
+  if (!url.pathname.startsWith("/v1/")) {
+    return false;
+  }
+
+  if (method !== "POST") {
+    writeError(params.res, 405, "method not allowed");
+    return true;
+  }
+
+  const body = (await readJson(params.req)) as Record<string, unknown>;
+
+  try {
+    switch (url.pathname) {
+      case "/v1/reset":
+        params.state.reset();
+        writeJson(params.res, 200, { ok: true });
+        return true;
+      case "/v1/inbound/message":
+        writeJson(params.res, 200, {
+          message: params.state.addInboundMessage(body as unknown as QaBusInboundMessageInput),
+        });
+        return true;
+      case "/v1/outbound/message":
+        writeJson(params.res, 200, {
+          message: params.state.addOutboundMessage(body as unknown as QaBusOutboundMessageInput),
+        });
+        return true;
+      case "/v1/actions/thread-create":
+        writeJson(params.res, 200, {
+          thread: params.state.createThread(body as unknown as QaBusCreateThreadInput),
+        });
+        return true;
+      case "/v1/actions/react":
+        writeJson(params.res, 200, {
+          message: params.state.reactToMessage(body as unknown as QaBusReactToMessageInput),
+        });
+        return true;
+      case "/v1/actions/edit":
+        writeJson(params.res, 200, {
+          message: params.state.editMessage(body as unknown as QaBusEditMessageInput),
+        });
+        return true;
+      case "/v1/actions/delete":
+        writeJson(params.res, 200, {
+          message: params.state.deleteMessage(body as unknown as QaBusDeleteMessageInput),
+        });
+        return true;
+      case "/v1/actions/read":
+        writeJson(params.res, 200, {
+          message: params.state.readMessage(body as unknown as QaBusReadMessageInput),
+        });
+        return true;
+      case "/v1/actions/search":
+        writeJson(params.res, 200, {
+          messages: params.state.searchMessages(body as unknown as QaBusSearchMessagesInput),
+        });
+        return true;
+      case "/v1/poll": {
+        const input = body as unknown as QaBusPollInput;
+        const timeoutMs = Math.max(0, Math.min(input.timeoutMs ?? 0, 30_000));
+        const accountId = normalizeAccountId(input.accountId);
+        const initial = params.state.poll(input);
+        if (initial.events.length > 0 || timeoutMs === 0) {
+          writeJson(params.res, 200, initial);
+          return true;
+        }
+        try {
+          await params.state.waitForCursorAdvance(input.cursor ?? 0, timeoutMs, (snapshot) => {
+            return snapshot.events.some(
+              (event) => event.accountId === accountId && event.cursor > (input.cursor ?? 0),
+            );
+          });
+        } catch {
+          // timeout ok for long-poll
+        }
+        writeJson(params.res, 200, params.state.poll(input));
+        return true;
+      }
+      case "/v1/wait":
+        writeJson(params.res, 200, {
+          match: await params.state.waitFor(body as unknown as QaBusWaitForInput),
+        });
+        return true;
+      default:
+        writeError(params.res, 404, "not found");
+        return true;
+    }
+  } catch (error) {
+    writeError(params.res, 400, error);
+    return true;
+  }
+}
+
+export function createQaBusServer(state: QaBusState): Server {
+  return createServer(async (req, res) => {
+    const handled = await handleQaBusRequest({ req, res, state });
+    if (!handled) {
+      writeError(res, 404, "not found");
+    }
+  });
+}
+
+export async function startQaBusServer(params: { state: QaBusState; port?: number }) {
+  const server = createQaBusServer(params.state);
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(params.port ?? 0, "127.0.0.1", () => resolve());
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("qa-bus failed to bind");
+  }
+  return {
+    server,
+    port: address.port,
+    baseUrl: `http://127.0.0.1:${address.port}`,
+    async stop() {
+      await closeQaHttpServer(server);
+    },
+  };
+}

openclaw/extensions/qa-lab/src/bus-state.test.ts

@@ -0,0 +1,175 @@
+import { describe, expect, it } from "vitest";
+import { createQaBusState } from "./bus-state.js";
+
+describe("qa-bus state", () => {
+  it("records inbound and outbound traffic in cursor order", () => {
+    const state = createQaBusState();
+
+    const inbound = state.addInboundMessage({
+      conversation: { id: "alice", kind: "direct" },
+      senderId: "alice",
+      text: "hello",
+    });
+    const outbound = state.addOutboundMessage({
+      to: "dm:alice",
+      text: "hi",
+    });
+
+    const snapshot = state.getSnapshot();
+    expect(snapshot.cursor).toBe(2);
+    expect(snapshot.events.map((event) => event.kind)).toEqual([
+      "inbound-message",
+      "outbound-message",
+    ]);
+    expect(snapshot.messages.map((message) => message.id)).toEqual([inbound.id, outbound.id]);
+  });
+
+  it("creates threads and mutates message state", async () => {
+    const state = createQaBusState();
+
+    const thread = state.createThread({
+      conversationId: "qa-room",
+      title: "QA thread",
+    });
+    const message = state.addOutboundMessage({
+      to: `thread:qa-room/${thread.id}`,
+      text: "inside thread",
+      threadId: thread.id,
+    });
+
+    state.reactToMessage({
+      messageId: message.id,
+      emoji: "eyes",
+      senderId: "alice",
+    });
+    state.editMessage({
+      messageId: message.id,
+      text: "inside thread (edited)",
+    });
+    state.deleteMessage({
+      messageId: message.id,
+    });
+
+    const snapshot = state.getSnapshot();
+    expect(snapshot.threads).toHaveLength(1);
+    expect(snapshot.threads[0]).toMatchObject({
+      id: thread.id,
+      conversationId: "qa-room",
+      title: "QA thread",
+    });
+    expect(snapshot.messages[0]).toMatchObject({
+      id: message.id,
+      text: "inside thread (edited)",
+      deleted: true,
+      reactions: [{ emoji: "eyes", senderId: "alice" }],
+    });
+  });
+
+  it("waits for a text match and rejects on timeout", async () => {
+    const state = createQaBusState();
+    const pending = state.waitFor({
+      kind: "message-text",
+      textIncludes: "needle",
+      timeoutMs: 500,
+    });
+
+    setTimeout(() => {
+      state.addOutboundMessage({
+        to: "dm:alice",
+        text: "haystack + needle",
+      });
+    }, 20);
+
+    const matched = await pending;
+    expect("text" in matched && matched.text).toContain("needle");
+
+    await expect(
+      state.waitFor({
+        kind: "message-text",
+        textIncludes: "missing",
+        timeoutMs: 20,
+      }),
+    ).rejects.toThrow("qa-bus wait timeout");
+  });
+
+  it("keeps account-scoped cursor waits blocked on unrelated account traffic", async () => {
+    const state = createQaBusState();
+    const pending = state.waitForCursorAdvance(0, 500, (snapshot) => {
+      return snapshot.events.some((event) => event.accountId === "acct-a" && event.cursor > 0);
+    });
+
+    state.addInboundMessage({
+      accountId: "acct-b",
+      conversation: { id: "other", kind: "direct" },
+      senderId: "acct-b-user",
+      text: "unrelated",
+    });
+
+    const beforeMatch = await Promise.race([
+      pending.then(() => "resolved"),
+      new Promise((resolve) => setTimeout(() => resolve("still-waiting"), 20)),
+    ]);
+    expect(beforeMatch).toBe("still-waiting");
+
+    state.addInboundMessage({
+      accountId: "acct-a",
+      conversation: { id: "target", kind: "direct" },
+      senderId: "acct-a-user",
+      text: "matched",
+    });
+
+    await expect(pending).resolves.toBeUndefined();
+  });
+
+  it("wakes default-account cursor waits when accountId is omitted", async () => {
+    const state = createQaBusState();
+    const pending = state.waitForCursorAdvance(0, 500, (snapshot) => {
+      return snapshot.events.some((event) => event.accountId === "default" && event.cursor > 0);
+    });
+
+    state.addInboundMessage({
+      conversation: { id: "target", kind: "direct" },
+      senderId: "default-user",
+      text: "matched",
+    });
+
+    await expect(pending).resolves.toBeUndefined();
+  });
+
+  it("preserves inline attachments and lets search match attachment metadata", () => {
+    const state = createQaBusState();
+
+    const outbound = state.addOutboundMessage({
+      to: "dm:alice",
+      text: "artifact attached",
+      attachments: [
+        {
+          id: "image-1",
+          kind: "image",
+          mimeType: "image/png",
+          fileName: "qa-screenshot.png",
+          altText: "QA dashboard screenshot",
+          contentBase64: "aGVsbG8=",
+        },
+      ],
+    });
+
+    const readback = state.readMessage({ messageId: outbound.id });
+    expect(readback.attachments).toHaveLength(1);
+    expect(readback.attachments?.[0]).toMatchObject({
+      kind: "image",
+      fileName: "qa-screenshot.png",
+      altText: "QA dashboard screenshot",
+    });
+
+    const byFilename = state.searchMessages({
+      query: "screenshot",
+    });
+    expect(byFilename.some((message) => message.id === outbound.id)).toBe(true);
+
+    const byAltText = state.searchMessages({
+      query: "dashboard",
+    });
+    expect(byAltText.some((message) => message.id === outbound.id)).toBe(true);
+  });
+});

openclaw/extensions/qa-lab/src/bus-state.ts

@@ -0,0 +1,296 @@
+import { randomUUID } from "node:crypto";
+import {
+  buildQaBusSnapshot,
+  cloneMessage,
+  normalizeAccountId,
+  normalizeConversationFromTarget,
+  pollQaBusEvents,
+  readQaBusMessage,
+  searchQaBusMessages,
+} from "./bus-queries.js";
+import { createQaBusWaiterStore } from "./bus-waiters.js";
+import type {
+  QaBusAttachment,
+  QaBusConversation,
+  QaBusCreateThreadInput,
+  QaBusDeleteMessageInput,
+  QaBusEditMessageInput,
+  QaBusEvent,
+  QaBusInboundMessageInput,
+  QaBusMessage,
+  QaBusOutboundMessageInput,
+  QaBusPollInput,
+  QaBusReadMessageInput,
+  QaBusReactToMessageInput,
+  QaBusSearchMessagesInput,
+  QaBusStateSnapshot,
+  QaBusThread,
+  QaBusWaitForInput,
+} from "./runtime-api.js";
+
+const DEFAULT_BOT_ID = "openclaw";
+const DEFAULT_BOT_NAME = "OpenClaw QA";
+
+type QaBusEventSeed =
+  | {
+      kind: "inbound-message";
+      accountId: string;
+      message: QaBusMessage;
+    }
+  | {
+      kind: "outbound-message";
+      accountId: string;
+      message: QaBusMessage;
+    }
+  | {
+      kind: "thread-created";
+      accountId: string;
+      thread: QaBusThread;
+    }
+  | {
+      kind: "message-edited";
+      accountId: string;
+      message: QaBusMessage;
+    }
+  | {
+      kind: "message-deleted";
+      accountId: string;
+      message: QaBusMessage;
+    }
+  | {
+      kind: "reaction-added";
+      accountId: string;
+      message: QaBusMessage;
+      emoji: string;
+      senderId: string;
+    };
+
+export function createQaBusState() {
+  const conversations = new Map<string, QaBusConversation>();
+  const threads = new Map<string, QaBusThread>();
+  const messages = new Map<string, QaBusMessage>();
+  const events: QaBusEvent[] = [];
+  let cursor = 0;
+  const waiters = createQaBusWaiterStore(() =>
+    buildQaBusSnapshot({
+      cursor,
+      conversations,
+      threads,
+      messages,
+      events,
+    }),
+  );
+
+  const pushEvent = (event: QaBusEventSeed | ((cursor: number) => QaBusEventSeed)): QaBusEvent => {
+    cursor += 1;
+    const next = typeof event === "function" ? event(cursor) : event;
+    const finalized = { cursor, ...next } as QaBusEvent;
+    events.push(finalized);
+    waiters.settle();
+    return finalized;
+  };
+
+  const ensureConversation = (conversation: QaBusConversation): QaBusConversation => {
+    const existing = conversations.get(conversation.id);
+    if (existing) {
+      if (!existing.title && conversation.title) {
+        existing.title = conversation.title;
+      }
+      return existing;
+    }
+    const created = { ...conversation };
+    conversations.set(created.id, created);
+    return created;
+  };
+
+  const createMessage = (params: {
+    direction: QaBusMessage["direction"];
+    accountId: string;
+    conversation: QaBusConversation;
+    senderId: string;
+    senderName?: string;
+    text: string;
+    timestamp?: number;
+    threadId?: string;
+    threadTitle?: string;
+    replyToId?: string;
+    attachments?: QaBusAttachment[];
+  }): QaBusMessage => {
+    const conversation = ensureConversation(params.conversation);
+    const message: QaBusMessage = {
+      id: randomUUID(),
+      accountId: params.accountId,
+      direction: params.direction,
+      conversation,
+      senderId: params.senderId,
+      senderName: params.senderName,
+      text: params.text,
+      timestamp: params.timestamp ?? Date.now(),
+      threadId: params.threadId,
+      threadTitle: params.threadTitle,
+      replyToId: params.replyToId,
+      attachments: params.attachments?.map((attachment) => ({ ...attachment })) ?? [],
+      reactions: [],
+    };
+    messages.set(message.id, message);
+    return message;
+  };
+
+  return {
+    reset() {
+      conversations.clear();
+      threads.clear();
+      messages.clear();
+      events.length = 0;
+      // Keep the cursor monotonic across resets so long-poll clients do not
+      // miss fresh events after the bus is cleared mid-session.
+      waiters.reset();
+    },
+    getSnapshot() {
+      return buildQaBusSnapshot({
+        cursor,
+        conversations,
+        threads,
+        messages,
+        events,
+      });
+    },
+    addInboundMessage(input: QaBusInboundMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const message = createMessage({
+        direction: "inbound",
+        accountId,
+        conversation: input.conversation,
+        senderId: input.senderId,
+        senderName: input.senderName,
+        text: input.text,
+        timestamp: input.timestamp,
+        threadId: input.threadId,
+        threadTitle: input.threadTitle,
+        replyToId: input.replyToId,
+        attachments: input.attachments,
+      });
+      pushEvent({
+        kind: "inbound-message",
+        accountId,
+        message: cloneMessage(message),
+      });
+      return cloneMessage(message);
+    },
+    addOutboundMessage(input: QaBusOutboundMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const { conversation, threadId } = normalizeConversationFromTarget(input.to);
+      const message = createMessage({
+        direction: "outbound",
+        accountId,
+        conversation,
+        senderId: input.senderId?.trim() || DEFAULT_BOT_ID,
+        senderName: input.senderName?.trim() || DEFAULT_BOT_NAME,
+        text: input.text,
+        timestamp: input.timestamp,
+        threadId: input.threadId ?? threadId,
+        replyToId: input.replyToId,
+        attachments: input.attachments,
+      });
+      pushEvent({
+        kind: "outbound-message",
+        accountId,
+        message: cloneMessage(message),
+      });
+      return cloneMessage(message);
+    },
+    createThread(input: QaBusCreateThreadInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const thread: QaBusThread = {
+        id: `thread-${randomUUID()}`,
+        accountId,
+        conversationId: input.conversationId,
+        title: input.title,
+        createdAt: input.timestamp ?? Date.now(),
+        createdBy: input.createdBy?.trim() || DEFAULT_BOT_ID,
+      };
+      threads.set(thread.id, thread);
+      ensureConversation({
+        id: input.conversationId,
+        kind: "channel",
+      });
+      pushEvent({
+        kind: "thread-created",
+        accountId,
+        thread: { ...thread },
+      });
+      return { ...thread };
+    },
+    reactToMessage(input: QaBusReactToMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const message = messages.get(input.messageId);
+      if (!message) {
+        throw new Error(`qa-bus message not found: ${input.messageId}`);
+      }
+      const reaction = {
+        emoji: input.emoji,
+        senderId: input.senderId?.trim() || DEFAULT_BOT_ID,
+        timestamp: input.timestamp ?? Date.now(),
+      };
+      message.reactions.push(reaction);
+      pushEvent({
+        kind: "reaction-added",
+        accountId,
+        message: cloneMessage(message),
+        emoji: reaction.emoji,
+        senderId: reaction.senderId,
+      });
+      return cloneMessage(message);
+    },
+    editMessage(input: QaBusEditMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const message = messages.get(input.messageId);
+      if (!message) {
+        throw new Error(`qa-bus message not found: ${input.messageId}`);
+      }
+      message.text = input.text;
+      message.editedAt = input.timestamp ?? Date.now();
+      pushEvent({
+        kind: "message-edited",
+        accountId,
+        message: cloneMessage(message),
+      });
+      return cloneMessage(message);
+    },
+    deleteMessage(input: QaBusDeleteMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const message = messages.get(input.messageId);
+      if (!message) {
+        throw new Error(`qa-bus message not found: ${input.messageId}`);
+      }
+      message.deleted = true;
+      pushEvent({
+        kind: "message-deleted",
+        accountId,
+        message: cloneMessage(message),
+      });
+      return cloneMessage(message);
+    },
+    readMessage(input: QaBusReadMessageInput) {
+      return readQaBusMessage({ messages, input });
+    },
+    searchMessages(input: QaBusSearchMessagesInput) {
+      return searchQaBusMessages({ messages, input });
+    },
+    poll(input: QaBusPollInput = {}) {
+      return pollQaBusEvents({ events, cursor, input });
+    },
+    async waitFor(input: QaBusWaitForInput) {
+      return await waiters.waitFor(input);
+    },
+    async waitForCursorAdvance(
+      afterCursor: number,
+      timeoutMs: number,
+      shouldResolve?: (snapshot: QaBusStateSnapshot) => boolean,
+    ) {
+      return await waiters.waitForCursorAdvance(afterCursor, timeoutMs, shouldResolve);
+    },
+  };
+}
+
+export type QaBusState = ReturnType<typeof createQaBusState>;

openclaw/extensions/qa-lab/src/bus-waiters.ts

@@ -0,0 +1,135 @@
+import type {
+  QaBusEvent,
+  QaBusMessage,
+  QaBusStateSnapshot,
+  QaBusThread,
+  QaBusWaitForInput,
+} from "./runtime-api.js";
+
+export const DEFAULT_WAIT_TIMEOUT_MS = 5_000;
+
+export type QaBusWaitMatch = QaBusEvent | QaBusMessage | QaBusThread;
+
+type Waiter = {
+  resolve: (event: QaBusWaitMatch) => void;
+  reject: (error: Error) => void;
+  timer: NodeJS.Timeout;
+  matcher: (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null;
+};
+
+type CursorWaiter = {
+  resolve: () => void;
+  reject: (error: Error) => void;
+  timer: NodeJS.Timeout;
+  afterCursor: number;
+  shouldResolve?: (snapshot: QaBusStateSnapshot) => boolean;
+};
+
+function createQaBusMatcher(
+  input: QaBusWaitForInput,
+): (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null {
+  return (snapshot) => {
+    if (input.kind === "event-kind") {
+      return snapshot.events.find((event) => event.kind === input.eventKind) ?? null;
+    }
+    if (input.kind === "thread-id") {
+      return snapshot.threads.find((thread) => thread.id === input.threadId) ?? null;
+    }
+    return (
+      snapshot.messages.find(
+        (message) =>
+          (!input.direction || message.direction === input.direction) &&
+          message.text.includes(input.textIncludes),
+      ) ?? null
+    );
+  };
+}
+
+export function createQaBusWaiterStore(getSnapshot: () => QaBusStateSnapshot) {
+  const waiters = new Set<Waiter>();
+  const cursorWaiters = new Set<CursorWaiter>();
+
+  return {
+    reset(reason = "qa-bus reset") {
+      for (const waiter of waiters) {
+        clearTimeout(waiter.timer);
+        waiter.reject(new Error(reason));
+      }
+      waiters.clear();
+      for (const waiter of cursorWaiters) {
+        clearTimeout(waiter.timer);
+        waiter.reject(new Error(reason));
+      }
+      cursorWaiters.clear();
+    },
+    settle() {
+      if (waiters.size === 0 && cursorWaiters.size === 0) {
+        return;
+      }
+      const snapshot = getSnapshot();
+      for (const waiter of Array.from(waiters)) {
+        const match = waiter.matcher(snapshot);
+        if (!match) {
+          continue;
+        }
+        clearTimeout(waiter.timer);
+        waiters.delete(waiter);
+        waiter.resolve(match);
+      }
+      for (const waiter of Array.from(cursorWaiters)) {
+        if (snapshot.cursor <= waiter.afterCursor) {
+          continue;
+        }
+        if (waiter.shouldResolve && !waiter.shouldResolve(snapshot)) {
+          continue;
+        }
+        clearTimeout(waiter.timer);
+        cursorWaiters.delete(waiter);
+        waiter.resolve();
+      }
+    },
+    async waitFor(input: QaBusWaitForInput) {
+      const matcher = createQaBusMatcher(input);
+      const immediate = matcher(getSnapshot());
+      if (immediate) {
+        return immediate;
+      }
+      return await new Promise<QaBusWaitMatch>((resolve, reject) => {
+        const timeoutMs = input.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS;
+        const waiter: Waiter = {
+          resolve,
+          reject,
+          matcher,
+          timer: setTimeout(() => {
+            waiters.delete(waiter);
+            reject(new Error(`qa-bus wait timeout after ${timeoutMs}ms`));
+          }, timeoutMs),
+        };
+        waiters.add(waiter);
+      });
+    },
+    async waitForCursorAdvance(
+      afterCursor: number,
+      timeoutMs: number,
+      shouldResolve?: (snapshot: QaBusStateSnapshot) => boolean,
+    ) {
+      const snapshot = getSnapshot();
+      if (snapshot.cursor > afterCursor && (!shouldResolve || shouldResolve(snapshot))) {
+        return;
+      }
+      return await new Promise<void>((resolve, reject) => {
+        const waiter: CursorWaiter = {
+          resolve,
+          reject,
+          afterCursor,
+          shouldResolve,
+          timer: setTimeout(() => {
+            cursorWaiters.delete(waiter);
+            reject(new Error(`qa-bus wait timeout after ${timeoutMs}ms`));
+          }, timeoutMs),
+        };
+        cursorWaiters.add(waiter);
+      });
+    },
+  };
+}

openclaw/extensions/qa-lab/src/character-eval.test.ts

@@ -0,0 +1,601 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { runQaCharacterEval, type QaCharacterEvalParams } from "./character-eval.js";
+import type { QaSuiteResult } from "./suite.js";
+
+type CharacterRunSuiteParams = Parameters<NonNullable<QaCharacterEvalParams["runSuite"]>>[0];
+type CharacterRunJudgeParams = Parameters<NonNullable<QaCharacterEvalParams["runJudge"]>>[0];
+
+function makeSuiteResult(params: { outputDir: string; model: string; transcript: string }) {
+  return {
+    outputDir: params.outputDir,
+    reportPath: path.join(params.outputDir, "qa-suite-report.md"),
+    summaryPath: path.join(params.outputDir, "qa-suite-summary.json"),
+    report: "# report",
+    watchUrl: "http://127.0.0.1:43124",
+    scenarios: [
+      {
+        name: "Character vibes",
+        status: "pass",
+        steps: [
+          {
+            name: `transcript for ${params.model}`,
+            status: "pass",
+            details: params.transcript,
+          },
+        ],
+      },
+    ],
+  } satisfies QaSuiteResult;
+}
+
+describe("runQaCharacterEval", () => {
+  let tempRoot: string;
+
+  beforeEach(async () => {
+    tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-character-eval-test-"));
+  });
+
+  afterEach(async () => {
+    await fs.rm(tempRoot, { recursive: true, force: true });
+  });
+
+  it("runs each requested model and writes a judged report with transcripts", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) => {
+      const model = params.primaryModel;
+      const transcript = `USER Alice: prompt for ${model}\n\nASSISTANT openclaw: reply from ${model}`;
+      return makeSuiteResult({ outputDir: params.outputDir, model, transcript });
+    });
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [
+          {
+            model: "openai/gpt-5.4",
+            rank: 1,
+            score: 9.1,
+            summary: "Most natural.",
+            strengths: ["vivid"],
+            weaknesses: ["none"],
+          },
+          {
+            model: "codex-cli/test-model",
+            rank: 2,
+            score: 7,
+            summary: "Readable but flatter.",
+            strengths: ["coherent"],
+            weaknesses: ["less funny"],
+          },
+        ],
+      }),
+    );
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.4", "codex-cli/test-model", "openai/gpt-5.4"],
+      scenarioId: "character-vibes-gollum",
+      candidateFastMode: true,
+      judgeModels: ["openai/gpt-5.4"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(runSuite).toHaveBeenCalledTimes(2);
+    expect(runSuite).toHaveBeenNthCalledWith(
+      1,
+      expect.objectContaining({
+        providerMode: "live-frontier",
+        primaryModel: "openai/gpt-5.4",
+        alternateModel: "openai/gpt-5.4",
+        fastMode: true,
+        scenarioIds: ["character-vibes-gollum"],
+      }),
+    );
+    expect(runJudge).toHaveBeenCalledWith(
+      expect.objectContaining({
+        judgeModel: "openai/gpt-5.4",
+        judgeThinkingDefault: "xhigh",
+        judgeFastMode: false,
+        timeoutMs: 300_000,
+      }),
+    );
+    expect(result.judgments).toHaveLength(1);
+    expect(result.judgments[0]?.rankings.map((ranking) => ranking.model)).toEqual([
+      "openai/gpt-5.4",
+      "codex-cli/test-model",
+    ]);
+
+    const report = await fs.readFile(result.reportPath, "utf8");
+    expect(report).toContain("Execution: local QA gateway child processes, not Docker");
+    expect(report).toContain("Judges: openai/gpt-5.4");
+    expect(report).toContain("Judge model labels: visible");
+    expect(report).toContain("## Judge Rankings");
+    expect(report).toContain("### openai/gpt-5.4");
+    expect(report).toContain("reply from openai/gpt-5.4");
+    expect(report).toContain("reply from codex-cli/test-model");
+    expect(report).toContain("Judge thinking: xhigh");
+    expect(report).toContain("- Timeout: 5m");
+    expect(report).toContain("Fast mode: on");
+    expect(report).toContain("Duration:");
+    expect(report).not.toContain("Duration ms:");
+    expect(report).not.toContain("Judge Raw Reply");
+  });
+
+  it("can hide candidate model refs from judge prompts and map rankings back", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: "USER Alice: hi\n\nASSISTANT openclaw: anonymous reply",
+      }),
+    );
+    const runJudge = vi.fn(async (params: CharacterRunJudgeParams) => {
+      expect(params.prompt).toContain("## CANDIDATE candidate-01");
+      expect(params.prompt).toContain("## CANDIDATE candidate-02");
+      expect(params.prompt).not.toContain("openai/gpt-5.4");
+      expect(params.prompt).not.toContain("codex-cli/test-model");
+      return JSON.stringify({
+        rankings: [
+          {
+            model: "candidate-02",
+            rank: 1,
+            score: 9.1,
+            summary: "Better vibes.",
+          },
+          {
+            model: "candidate-01",
+            rank: 2,
+            score: 7.4,
+            summary: "Solid.",
+          },
+        ],
+      });
+    });
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.4", "codex-cli/test-model"],
+      judgeModels: ["openai/gpt-5.4"],
+      judgeBlindModels: true,
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.judgments[0]?.blindModels).toBe(true);
+    expect(result.judgments[0]?.rankings.map((ranking) => ranking.model)).toEqual([
+      "codex-cli/test-model",
+      "openai/gpt-5.4",
+    ]);
+    const report = await fs.readFile(result.reportPath, "utf8");
+    expect(report).toContain("Judge model labels: blind");
+    expect(report).toContain("1. codex-cli/test-model - 9.1 - Better vibes.");
+  });
+
+  it("defaults to the character eval model panel when no models are provided", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: `USER Alice: hi\n\nASSISTANT openclaw: reply from ${params.primaryModel}`,
+      }),
+    );
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [
+          { model: "openai/gpt-5.4", rank: 1, score: 8, summary: "ok" },
+          { model: "openai/gpt-5.2", rank: 2, score: 7.5, summary: "ok" },
+          { model: "openai/gpt-5", rank: 3, score: 7.2, summary: "ok" },
+          { model: "anthropic/claude-opus-4-6", rank: 4, score: 7, summary: "ok" },
+          { model: "anthropic/claude-sonnet-4-6", rank: 5, score: 6.8, summary: "ok" },
+          { model: "zai/glm-5.1", rank: 6, score: 6.3, summary: "ok" },
+          { model: "moonshot/kimi-k2.5", rank: 7, score: 6.2, summary: "ok" },
+          { model: "google/gemini-3.1-pro-preview", rank: 8, score: 6, summary: "ok" },
+        ],
+      }),
+    );
+
+    await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: [],
+      runSuite,
+      runJudge,
+    });
+
+    expect(runSuite).toHaveBeenCalledTimes(8);
+    expect(runSuite.mock.calls.map(([params]) => params.primaryModel)).toEqual([
+      "openai/gpt-5.4",
+      "openai/gpt-5.2",
+      "openai/gpt-5",
+      "anthropic/claude-opus-4-6",
+      "anthropic/claude-sonnet-4-6",
+      "zai/glm-5.1",
+      "moonshot/kimi-k2.5",
+      "google/gemini-3.1-pro-preview",
+    ]);
+    expect(runSuite.mock.calls.map(([params]) => params.thinkingDefault)).toEqual([
+      "xhigh",
+      "xhigh",
+      "xhigh",
+      "high",
+      "high",
+      "high",
+      "high",
+      "high",
+    ]);
+    expect(runSuite.mock.calls.map(([params]) => params.fastMode)).toEqual([
+      true,
+      true,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+    ]);
+    expect(runJudge).toHaveBeenCalledTimes(2);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeModel)).toEqual([
+      "openai/gpt-5.4",
+      "anthropic/claude-opus-4-6",
+    ]);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeThinkingDefault)).toEqual([
+      "xhigh",
+      "high",
+    ]);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeFastMode)).toEqual([false, false]);
+  });
+
+  it("runs candidate models with bounded concurrency while preserving result order", async () => {
+    let activeRuns = 0;
+    let maxActiveRuns = 0;
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) => {
+      activeRuns += 1;
+      maxActiveRuns = Math.max(maxActiveRuns, activeRuns);
+      await new Promise((resolve) => setTimeout(resolve, 10));
+      activeRuns -= 1;
+      return makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: `USER Alice: hi\n\nASSISTANT openclaw: reply from ${params.primaryModel}`,
+      });
+    });
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [
+          { model: "openai/gpt-5.4", rank: 1, score: 8, summary: "ok" },
+          { model: "anthropic/claude-sonnet-4-6", rank: 2, score: 7, summary: "ok" },
+          { model: "moonshot/kimi-k2.5", rank: 3, score: 6, summary: "ok" },
+        ],
+      }),
+    );
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.4", "anthropic/claude-sonnet-4-6", "moonshot/kimi-k2.5"],
+      candidateConcurrency: 2,
+      judgeModels: ["openai/gpt-5.4"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(maxActiveRuns).toBe(2);
+    expect(result.runs.map((run) => run.model)).toEqual([
+      "openai/gpt-5.4",
+      "anthropic/claude-sonnet-4-6",
+      "moonshot/kimi-k2.5",
+    ]);
+  });
+
+  it("defaults candidate and judge concurrency to sixteen", async () => {
+    let activeRuns = 0;
+    let maxActiveRuns = 0;
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) => {
+      activeRuns += 1;
+      maxActiveRuns = Math.max(maxActiveRuns, activeRuns);
+      await new Promise((resolve) => setTimeout(resolve, 10));
+      activeRuns -= 1;
+      return makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: `USER Alice: hi\n\nASSISTANT openclaw: reply from ${params.primaryModel}`,
+      });
+    });
+    let activeJudges = 0;
+    let maxActiveJudges = 0;
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) => {
+      activeJudges += 1;
+      maxActiveJudges = Math.max(maxActiveJudges, activeJudges);
+      await new Promise((resolve) => setTimeout(resolve, 10));
+      activeJudges -= 1;
+      return JSON.stringify({
+        rankings: Array.from({ length: 20 }, (_, index) => ({
+          model: `provider/model-${index + 1}`,
+          rank: index + 1,
+          score: 10 - index,
+          summary: "ok",
+        })),
+      });
+    });
+
+    await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: Array.from({ length: 20 }, (_, index) => `provider/model-${index + 1}`),
+      judgeModels: Array.from({ length: 20 }, (_, index) => `judge/model-${index + 1}`),
+      runSuite,
+      runJudge,
+    });
+
+    expect(maxActiveRuns).toBe(16);
+    expect(maxActiveJudges).toBe(16);
+  });
+
+  it("marks raw provider error transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript:
+          "USER Alice: Are you awake?\n\nASSISTANT OpenClaw QA: 400 model `qwen3.6-plus` is not supported.",
+      }),
+    );
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [{ model: "qwen/qwen3.6-plus", rank: 1, score: 0.5, summary: "failed" }],
+      }),
+    );
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["qwen/qwen3.6-plus"],
+      judgeModels: ["openai/gpt-5.4"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.runs[0]).toMatchObject({
+      model: "qwen/qwen3.6-plus",
+      status: "fail",
+      error: "model unsupported error leaked into transcript",
+    });
+  });
+
+  it("marks raw tool failure transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: "ASSISTANT OpenClaw QA: ⚠️ ✍️ Write: to /tmp/precious.html failed",
+      }),
+    );
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [{ model: "qwen/qwen3.5-plus", rank: 1, score: 0.5, summary: "failed" }],
+      }),
+    );
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["qwen/qwen3.5-plus"],
+      judgeModels: ["openai/gpt-5.4"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.runs[0]).toMatchObject({
+      model: "qwen/qwen3.5-plus",
+      status: "fail",
+      error: "tool failure leaked into transcript",
+    });
+  });
+
+  it("marks generic channel fallback transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript:
+          "ASSISTANT OpenClaw QA: ⚠️ Something went wrong while processing your request. Please try again, or use /new to start a fresh session.",
+      }),
+    );
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [{ model: "qa/generic-fallback-model", rank: 1, score: 0.5, summary: "failed" }],
+      }),
+    );
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["qa/generic-fallback-model"],
+      judgeModels: ["openai/gpt-5.4"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.runs[0]).toMatchObject({
+      model: "qa/generic-fallback-model",
+      status: "fail",
+      error: "generic request failure leaked into transcript",
+    });
+  });
+
+  it("marks idle-timeout fallback transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript:
+          "ASSISTANT OpenClaw QA: The model did not produce a response before the LLM idle timeout. Please try again, or increase `agents.defaults.llm.idleTimeoutSeconds` in your config.",
+      }),
+    );
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [{ model: "google/gemini-test", rank: 1, score: 0.5, summary: "failed" }],
+      }),
+    );
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["google/gemini-test"],
+      judgeModels: ["openai/gpt-5.4"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.runs[0]).toMatchObject({
+      model: "google/gemini-test",
+      status: "fail",
+      error: "LLM timeout leaked into transcript",
+    });
+  });
+
+  it("marks leaked harness coordination transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript:
+          "ASSISTANT OpenClaw QA: checking thread context; then post a tight progress reply here.\nQA_LEAK_OK",
+      }),
+    );
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [{ model: "codex/gpt-5.4", rank: 1, score: 0.5, summary: "failed" }],
+      }),
+    );
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["codex/gpt-5.4"],
+      judgeModels: ["openai/gpt-5.4"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.runs[0]).toMatchObject({
+      model: "codex/gpt-5.4",
+      status: "fail",
+      error: "internal harness/meta text leaked into transcript",
+    });
+  });
+
+  it("lets explicit candidate thinking override the default panel", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: `USER Alice: hi\n\nASSISTANT openclaw: reply from ${params.primaryModel}`,
+      }),
+    );
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [
+          { model: "openai/gpt-5.4", rank: 1, score: 8, summary: "ok" },
+          { model: "moonshot/kimi-k2.5", rank: 2, score: 7, summary: "ok" },
+        ],
+      }),
+    );
+
+    await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.4", "moonshot/kimi-k2.5"],
+      candidateThinkingDefault: "medium",
+      candidateThinkingByModel: { "moonshot/kimi-k2.5": "high" },
+      judgeModels: ["openai/gpt-5.4"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(runSuite.mock.calls.map(([params]) => params.thinkingDefault)).toEqual([
+      "medium",
+      "high",
+    ]);
+  });
+
+  it("lets model-specific options override candidate and judge defaults", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: `USER Alice: hi\n\nASSISTANT openclaw: reply from ${params.primaryModel}`,
+      }),
+    );
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [{ model: "openai/gpt-5.4", rank: 1, score: 8, summary: "ok" }],
+      }),
+    );
+
+    await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.4", "moonshot/kimi-k2.5"],
+      candidateFastMode: true,
+      candidateThinkingDefault: "medium",
+      candidateModelOptions: {
+        "openai/gpt-5.4": { thinkingDefault: "xhigh", fastMode: false },
+      },
+      judgeModels: ["openai/gpt-5.4", "anthropic/claude-opus-4-6"],
+      judgeThinkingDefault: "medium",
+      judgeModelOptions: {
+        "openai/gpt-5.4": { thinkingDefault: "xhigh", fastMode: true },
+        "anthropic/claude-opus-4-6": { thinkingDefault: "high" },
+      },
+      runSuite,
+      runJudge,
+    });
+
+    expect(runSuite.mock.calls.map(([params]) => params.thinkingDefault)).toEqual([
+      "xhigh",
+      "medium",
+    ]);
+    expect(runSuite.mock.calls.map(([params]) => params.fastMode)).toEqual([false, true]);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeThinkingDefault)).toEqual([
+      "xhigh",
+      "high",
+    ]);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeFastMode)).toEqual([true, false]);
+  });
+
+  it("keeps failed model runs in the report for grader context", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) => {
+      if (params.primaryModel === "codex-cli/test-model") {
+        throw new Error("backend unavailable");
+      }
+      return makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: "USER Alice: hi\n\nASSISTANT openclaw: hello",
+      });
+    });
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [{ model: "openai/gpt-5.4", rank: 1, score: 8, summary: "ok" }],
+      }),
+    );
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.4", "codex-cli/test-model"],
+      judgeModels: ["openai/gpt-5.4"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.runs.map((run) => run.status)).toEqual(["pass", "fail"]);
+    expect(result.runs[1]?.error).toContain("backend unavailable");
+    const report = await fs.readFile(result.reportPath, "utf8");
+    expect(report).toContain("backend unavailable");
+  });
+});

openclaw/extensions/qa-lab/src/character-eval.ts

@@ -0,0 +1,722 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { runQaManualLane } from "./manual-lane.runtime.js";
+import { isQaFastModeModelRef, type QaProviderMode } from "./model-selection.js";
+import {
+  QA_FRONTIER_CHARACTER_EVAL_MODELS,
+  QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS,
+  QA_FRONTIER_CHARACTER_JUDGE_MODELS,
+  QA_FRONTIER_CHARACTER_THINKING_BY_MODEL,
+} from "./providers/live-frontier/character-eval.js";
+import { type QaThinkingLevel } from "./qa-gateway-config.js";
+import { extractQaVisibleReplyLeakText } from "./reply-failure.js";
+import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
+import type { QaSuiteResult } from "./suite.js";
+
+const DEFAULT_CHARACTER_SCENARIO_ID = "character-vibes-gollum";
+const DEFAULT_CHARACTER_EVAL_MODELS = QA_FRONTIER_CHARACTER_EVAL_MODELS;
+const DEFAULT_CHARACTER_THINKING: QaThinkingLevel = "high";
+const DEFAULT_CHARACTER_EVAL_CONCURRENCY = 16;
+const DEFAULT_CHARACTER_THINKING_BY_MODEL: Readonly<Record<string, QaThinkingLevel>> =
+  QA_FRONTIER_CHARACTER_THINKING_BY_MODEL;
+const DEFAULT_JUDGE_MODELS = QA_FRONTIER_CHARACTER_JUDGE_MODELS;
+const DEFAULT_JUDGE_THINKING: QaThinkingLevel = "xhigh";
+const DEFAULT_JUDGE_TIMEOUT_MS = 300_000;
+const DEFAULT_JUDGE_MODEL_OPTIONS: Readonly<Record<string, QaCharacterModelOptions>> =
+  QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS;
+
+type QaCharacterRunStatus = "pass" | "fail";
+
+export type QaCharacterModelOptions = {
+  thinkingDefault?: QaThinkingLevel;
+  fastMode?: boolean;
+};
+
+export type QaCharacterEvalRun = {
+  model: string;
+  status: QaCharacterRunStatus;
+  durationMs: number;
+  outputDir: string;
+  thinkingDefault: QaThinkingLevel;
+  fastMode: boolean;
+  reportPath?: string;
+  summaryPath?: string;
+  transcript: string;
+  stats: {
+    transcriptChars: number;
+    transcriptLines: number;
+    userTurns: number;
+    assistantTurns: number;
+  };
+  error?: string;
+};
+
+export type QaCharacterEvalJudgment = {
+  model: string;
+  rank: number;
+  score: number;
+  summary: string;
+  strengths: string[];
+  weaknesses: string[];
+};
+
+export type QaCharacterEvalResult = {
+  outputDir: string;
+  reportPath: string;
+  summaryPath: string;
+  runs: QaCharacterEvalRun[];
+  judgments: QaCharacterEvalJudgeResult[];
+};
+
+export type QaCharacterEvalJudgeResult = {
+  model: string;
+  thinkingDefault: QaThinkingLevel;
+  fastMode: boolean;
+  blindModels: boolean;
+  timeoutMs: number;
+  durationMs: number;
+  rankings: QaCharacterEvalJudgment[];
+  error?: string;
+};
+
+type QaCharacterEvalProgressLogger = (message: string) => void;
+
+type RunSuiteFn = (params: {
+  repoRoot: string;
+  outputDir: string;
+  providerMode: QaProviderMode;
+  primaryModel: string;
+  alternateModel: string;
+  fastMode?: boolean;
+  thinkingDefault?: QaThinkingLevel;
+  scenarioIds: string[];
+}) => Promise<QaSuiteResult>;
+
+type RunJudgeFn = (params: {
+  repoRoot: string;
+  judgeModel: string;
+  judgeThinkingDefault: QaThinkingLevel;
+  judgeFastMode: boolean;
+  prompt: string;
+  timeoutMs: number;
+}) => Promise<string | null>;
+
+export type QaCharacterEvalParams = {
+  repoRoot?: string;
+  outputDir?: string;
+  models: string[];
+  scenarioId?: string;
+  candidateFastMode?: boolean;
+  candidateThinkingDefault?: QaThinkingLevel;
+  candidateThinkingByModel?: Record<string, QaThinkingLevel>;
+  candidateModelOptions?: Record<string, QaCharacterModelOptions>;
+  judgeModel?: string;
+  judgeModels?: string[];
+  judgeThinkingDefault?: QaThinkingLevel;
+  judgeModelOptions?: Record<string, QaCharacterModelOptions>;
+  judgeTimeoutMs?: number;
+  judgeBlindModels?: boolean;
+  candidateConcurrency?: number;
+  judgeConcurrency?: number;
+  runSuite?: RunSuiteFn;
+  runJudge?: RunJudgeFn;
+  progress?: QaCharacterEvalProgressLogger;
+};
+
+function normalizeModelRefs(models: readonly string[]) {
+  return [...new Set(models.map((model) => model.trim()).filter((model) => model.length > 0))];
+}
+
+function resolveCandidateThinkingDefault(params: {
+  model: string;
+  candidateThinkingDefault?: QaThinkingLevel;
+  candidateThinkingByModel?: Record<string, QaThinkingLevel>;
+  candidateModelOptions?: Record<string, QaCharacterModelOptions>;
+}) {
+  return (
+    params.candidateModelOptions?.[params.model]?.thinkingDefault ??
+    params.candidateThinkingByModel?.[params.model] ??
+    params.candidateThinkingDefault ??
+    DEFAULT_CHARACTER_THINKING_BY_MODEL[params.model] ??
+    DEFAULT_CHARACTER_THINKING
+  );
+}
+
+function resolveCandidateFastMode(params: {
+  model: string;
+  candidateFastMode?: boolean;
+  candidateModelOptions?: Record<string, QaCharacterModelOptions>;
+}) {
+  return (
+    params.candidateModelOptions?.[params.model]?.fastMode ??
+    params.candidateFastMode ??
+    isQaFastModeModelRef(params.model)
+  );
+}
+
+function resolveJudgeOptions(params: {
+  model: string;
+  judgeThinkingDefault?: QaThinkingLevel;
+  judgeModelOptions?: Record<string, QaCharacterModelOptions>;
+}) {
+  const modelDefaults = DEFAULT_JUDGE_MODEL_OPTIONS[params.model];
+  const modelOptions = params.judgeModelOptions?.[params.model];
+  return {
+    thinkingDefault:
+      modelOptions?.thinkingDefault ??
+      params.judgeThinkingDefault ??
+      modelDefaults?.thinkingDefault ??
+      DEFAULT_JUDGE_THINKING,
+    fastMode: modelOptions?.fastMode ?? modelDefaults?.fastMode ?? false,
+  };
+}
+
+function sanitizePathPart(value: string) {
+  const sanitized = value.replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "");
+  return sanitized || "model";
+}
+
+function normalizeConcurrency(value: number | undefined, fallback = 1) {
+  if (value === undefined) {
+    return fallback;
+  }
+  if (!Number.isFinite(value)) {
+    return fallback;
+  }
+  return Math.max(1, Math.floor(value));
+}
+
+async function mapWithConcurrency<T, U>(
+  items: readonly T[],
+  concurrency: number,
+  mapper: (item: T, index: number) => Promise<U>,
+) {
+  const results = Array.from<U>({ length: items.length });
+  let nextIndex = 0;
+  const workerCount = Math.min(normalizeConcurrency(concurrency), items.length);
+  const workers = Array.from({ length: workerCount }, async () => {
+    while (nextIndex < items.length) {
+      const index = nextIndex;
+      nextIndex += 1;
+      results[index] = await mapper(items[index], index);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}
+
+function extractTranscript(result: QaSuiteResult) {
+  const details = result.scenarios.flatMap((scenario) =>
+    scenario.steps
+      .map((step) => step.details)
+      .filter((detail): detail is string => Boolean(detail)),
+  );
+  return details.toSorted((left, right) => right.length - left.length)[0] ?? result.report;
+}
+
+function collectTranscriptStats(transcript: string) {
+  return {
+    transcriptChars: transcript.length,
+    transcriptLines: transcript.length === 0 ? 0 : transcript.split(/\r?\n/).length,
+    userTurns: transcript.match(/^USER\b/gm)?.length ?? 0,
+    assistantTurns: transcript.match(/^ASSISTANT\b/gm)?.length ?? 0,
+  };
+}
+
+function detectTranscriptFailure(transcript: string): string | undefined {
+  if (extractQaVisibleReplyLeakText(transcript)) {
+    return "internal harness/meta text leaked into transcript";
+  }
+  const checks: Array<[RegExp, string]> = [
+    [/\bmodel `[^`]+` is not supported\b/i, "model unsupported error leaked into transcript"],
+    [/\binsufficient account balance\b/i, "account balance error leaked into transcript"],
+    [/\b(?:backend|transport|internal) error\b/i, "backend error leaked into transcript"],
+    [
+      /\bsomething went wrong while processing your request\b/i,
+      "generic request failure leaked into transcript",
+    ],
+    [/\buse \/new to start a fresh session\b/i, "generic request failure leaked into transcript"],
+    [
+      /\bmodel did not produce a response before the LLM idle timeout\b/i,
+      "LLM timeout leaked into transcript",
+    ],
+    [/\btool failed\b/i, "tool failure leaked into transcript"],
+    [/\b(?:read|write|edit|patch):[^\n]*\bfailed\b/i, "tool failure leaked into transcript"],
+    [/\bnot configured\b/i, "configuration error leaked into transcript"],
+  ];
+  return checks.find(([pattern]) => pattern.test(transcript))?.[1];
+}
+
+function formatDuration(ms: number) {
+  if (!Number.isFinite(ms) || ms < 0) {
+    return "unknown";
+  }
+  if (ms < 1_000) {
+    return `${Math.round(ms)}ms`;
+  }
+  if (ms < 60_000) {
+    const seconds = ms / 1_000;
+    return `${seconds >= 10 ? Math.round(seconds) : Number(seconds.toFixed(1))}s`;
+  }
+  const totalSeconds = Math.round(ms / 1_000);
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  return seconds === 0 ? `${minutes}m` : `${minutes}m ${seconds}s`;
+}
+
+function logCharacterEvalProgress(
+  progress: QaCharacterEvalProgressLogger | undefined,
+  message: string,
+) {
+  progress?.(`[qa-character] ${message}`);
+}
+
+function formatEvalIndex(index: number, total: number) {
+  return `${index + 1}/${total}`;
+}
+
+function summarizeRunStats(run: QaCharacterEvalRun) {
+  return [
+    `status=${run.status}`,
+    `duration=${formatDuration(run.durationMs)}`,
+    `turns=${run.stats.userTurns}/${run.stats.assistantTurns}`,
+    `chars=${run.stats.transcriptChars}`,
+    ...(run.error ? [`error="${run.error}"`] : []),
+  ].join(" ");
+}
+
+function formatBlindCandidateLabel(index: number) {
+  return `candidate-${String(index + 1).padStart(2, "0")}`;
+}
+
+function buildJudgePrompt(params: {
+  scenarioId: string;
+  runs: readonly QaCharacterEvalRun[];
+  blindModels?: boolean;
+}) {
+  const labelToModel = new Map<string, string>();
+  const runBlocks = params.runs
+    .map((run, index) => {
+      const label = params.blindModels ? formatBlindCandidateLabel(index) : run.model;
+      labelToModel.set(label, run.model);
+      return `## CANDIDATE ${label}
+
+Status: ${run.status}
+Duration ms (not used for ranking): ${run.durationMs}
+Fast mode: ${run.fastMode ? "on" : "off"}
+Thinking: ${run.thinkingDefault}
+Transcript chars: ${run.stats.transcriptChars}
+Assistant turns: ${run.stats.assistantTurns}
+Error: ${run.error ?? "none"}
+
+\`\`\`text
+${run.transcript}
+\`\`\``;
+    })
+    .join("\n\n");
+
+  const prompt = `You are grading OpenClaw natural character conversation transcripts for naturalness, vibes, and funniness.
+
+Scenario id: ${params.scenarioId}
+
+Rank the models by:
+- natural conversational reaction
+- playful character commitment
+- funny, surprising details
+- coherence across turns
+- completing real user tasks without becoming generic
+- not sounding aware of an eval or test
+- avoiding tool/backend/error leakage
+
+Treat candidate labels as opaque identifiers. Do not assume quality from the label.
+Duration is recorded for separate benchmark analysis only. Do not rank models by speed.
+
+Return strict JSON only with this shape:
+{
+  "rankings": [
+    {
+      "model": "same candidate label",
+      "rank": 1,
+      "score": 9.2,
+      "summary": "one sentence",
+      "strengths": ["short"],
+      "weaknesses": ["short"]
+    }
+  ]
+}
+
+${runBlocks}`;
+  return { prompt, labelToModel };
+}
+
+function normalizeJudgment(value: unknown, allowedModels: Set<string>): QaCharacterEvalJudgment[] {
+  const payload = value && typeof value === "object" ? (value as Record<string, unknown>) : {};
+  const rankings = Array.isArray(payload.rankings) ? payload.rankings : [];
+  return rankings
+    .map((entry): QaCharacterEvalJudgment | null => {
+      if (!entry || typeof entry !== "object") {
+        return null;
+      }
+      const record = entry as Record<string, unknown>;
+      const model = typeof record.model === "string" ? record.model : "";
+      if (!allowedModels.has(model)) {
+        return null;
+      }
+      const rank = typeof record.rank === "number" ? record.rank : Number(record.rank);
+      const score = typeof record.score === "number" ? record.score : Number(record.score);
+      const summary = typeof record.summary === "string" ? record.summary : "";
+      const strengths = Array.isArray(record.strengths)
+        ? record.strengths.filter((item): item is string => typeof item === "string")
+        : [];
+      const weaknesses = Array.isArray(record.weaknesses)
+        ? record.weaknesses.filter((item): item is string => typeof item === "string")
+        : [];
+      if (!Number.isFinite(rank) || !Number.isFinite(score)) {
+        return null;
+      }
+      return { model, rank, score, summary, strengths, weaknesses };
+    })
+    .filter((entry): entry is QaCharacterEvalJudgment => Boolean(entry))
+    .toSorted((left, right) => left.rank - right.rank || right.score - left.score);
+}
+
+function parseJudgeReply(reply: string | null, allowedModels: Set<string>) {
+  if (!reply) {
+    throw new Error("judge did not return a reply");
+  }
+  const trimmed = reply.trim();
+  const jsonText =
+    trimmed.match(/```(?:json)?\s*([\s\S]*?)```/)?.[1]?.trim() ??
+    trimmed.match(/\{[\s\S]*\}/)?.[0]?.trim() ??
+    trimmed;
+  const parsed = JSON.parse(jsonText) as unknown;
+  const rankings = normalizeJudgment(parsed, allowedModels);
+  if (rankings.length === 0) {
+    throw new Error("judge reply did not contain valid rankings");
+  }
+  return rankings;
+}
+
+async function defaultRunJudge(params: {
+  repoRoot: string;
+  judgeModel: string;
+  judgeThinkingDefault: QaThinkingLevel;
+  judgeFastMode: boolean;
+  prompt: string;
+  timeoutMs: number;
+}) {
+  const result = await runQaManualLane({
+    repoRoot: params.repoRoot,
+    providerMode: "live-frontier",
+    primaryModel: params.judgeModel,
+    alternateModel: params.judgeModel,
+    fastMode: params.judgeFastMode,
+    thinkingDefault: params.judgeThinkingDefault,
+    message: params.prompt,
+    timeoutMs: params.timeoutMs,
+  });
+  return result.reply;
+}
+
+function renderCharacterEvalReport(params: {
+  scenarioId: string;
+  startedAt: Date;
+  finishedAt: Date;
+  runs: readonly QaCharacterEvalRun[];
+  judgments: readonly QaCharacterEvalJudgeResult[];
+}) {
+  const lines = [
+    "# OpenClaw Character Eval Report",
+    "",
+    `- Started: ${params.startedAt.toISOString()}`,
+    `- Finished: ${params.finishedAt.toISOString()}`,
+    `- Duration: ${formatDuration(params.finishedAt.getTime() - params.startedAt.getTime())}`,
+    `- Scenario: ${params.scenarioId}`,
+    "- Execution: local QA gateway child processes, not Docker",
+    `- Judges: ${params.judgments.map((judgment) => judgment.model).join(", ")}`,
+    `- Judge thinking: ${params.judgments[0]?.thinkingDefault ?? DEFAULT_JUDGE_THINKING}`,
+    `- Judge fast mode: ${params.judgments.every((judgment) => judgment.fastMode) ? "on" : "mixed"}`,
+    `- Judge model labels: ${params.judgments.every((judgment) => judgment.blindModels) ? "blind" : "visible"}`,
+    "",
+    "## Judge Rankings",
+    "",
+  ];
+
+  for (const judgment of params.judgments) {
+    lines.push(`### ${judgment.model}`, "");
+    lines.push(`- Duration: ${formatDuration(judgment.durationMs)}`, "");
+    lines.push(`- Timeout: ${formatDuration(judgment.timeoutMs)}`, "");
+    if (judgment.rankings.length > 0) {
+      for (const ranking of judgment.rankings) {
+        lines.push(
+          `${ranking.rank}. ${ranking.model} - ${ranking.score.toFixed(1)} - ${ranking.summary}`,
+        );
+        if (ranking.strengths.length > 0) {
+          lines.push(`   Strengths: ${ranking.strengths.join("; ")}`);
+        }
+        if (ranking.weaknesses.length > 0) {
+          lines.push(`   Weaknesses: ${ranking.weaknesses.join("; ")}`);
+        }
+      }
+    } else {
+      lines.push("- Judge ranking unavailable.");
+      if (judgment.error) {
+        lines.push(`- Judge error: ${judgment.error}`);
+      }
+    }
+    lines.push("");
+  }
+
+  lines.push("## Run Stats", "");
+  lines.push(
+    "| Model | Thinking | Fast mode | Status | Duration | User turns | Assistant turns | Transcript chars |",
+  );
+  lines.push("| --- | --- | --- | --- | ---: | ---: | ---: | ---: |");
+  for (const run of params.runs) {
+    lines.push(
+      `| ${run.model} | ${run.thinkingDefault} | ${run.fastMode ? "on" : "off"} | ${run.status} | ${formatDuration(run.durationMs)} | ${run.stats.userTurns} | ${run.stats.assistantTurns} | ${run.stats.transcriptChars} |`,
+    );
+  }
+
+  lines.push("", "## Transcripts", "");
+  for (const run of params.runs) {
+    lines.push(`### ${run.model}`, "");
+    lines.push(`- Status: ${run.status}`);
+    lines.push(`- Thinking: ${run.thinkingDefault}`);
+    lines.push(`- Fast mode: ${run.fastMode ? "on" : "off"}`);
+    lines.push(`- Duration: ${formatDuration(run.durationMs)}`);
+    lines.push(`- Report: ${run.reportPath ?? "unavailable"}`);
+    if (run.error) {
+      lines.push(`- Error: ${run.error}`);
+    }
+    lines.push("", "```text", run.transcript.trim() || "(empty transcript)", "```", "");
+  }
+
+  return `${lines.join("\n")}\n`;
+}
+
+export async function runQaCharacterEval(params: QaCharacterEvalParams) {
+  const startedAt = new Date();
+  const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
+  const scenarioId = params.scenarioId?.trim() || DEFAULT_CHARACTER_SCENARIO_ID;
+  const models = normalizeModelRefs(
+    params.models.length > 0 ? params.models : DEFAULT_CHARACTER_EVAL_MODELS,
+  );
+  if (models.length === 0) {
+    throw new Error("qa character-eval needs at least one --model <provider/model> ref");
+  }
+
+  const outputDir =
+    params.outputDir ??
+    path.join(repoRoot, ".artifacts", "qa-e2e", `character-eval-${Date.now().toString(36)}`);
+  const runsDir = path.join(outputDir, "runs");
+  await fs.mkdir(runsDir, { recursive: true });
+
+  const runSuite = params.runSuite ?? runQaSuiteFromRuntime;
+  const candidateConcurrency = normalizeConcurrency(
+    params.candidateConcurrency,
+    DEFAULT_CHARACTER_EVAL_CONCURRENCY,
+  );
+  logCharacterEvalProgress(
+    params.progress,
+    `start scenario=${scenarioId} candidates=${models.length} candidateConcurrency=${candidateConcurrency} output=${outputDir}`,
+  );
+  const candidatesStartedAt = Date.now();
+  const runs = await mapWithConcurrency(models, candidateConcurrency, async (model, index) => {
+    const thinkingDefault = resolveCandidateThinkingDefault({
+      model,
+      candidateThinkingDefault: params.candidateThinkingDefault,
+      candidateThinkingByModel: params.candidateThinkingByModel,
+      candidateModelOptions: params.candidateModelOptions,
+    });
+    const fastMode = resolveCandidateFastMode({
+      model,
+      candidateFastMode: params.candidateFastMode,
+      candidateModelOptions: params.candidateModelOptions,
+    });
+    const modelOutputDir = path.join(runsDir, sanitizePathPart(model));
+    const runStartedAt = Date.now();
+    logCharacterEvalProgress(
+      params.progress,
+      `candidate start ${formatEvalIndex(index, models.length)} model=${model} thinking=${thinkingDefault} fast=${fastMode ? "on" : "off"}`,
+    );
+    try {
+      const result = await runSuite({
+        repoRoot,
+        outputDir: modelOutputDir,
+        providerMode: "live-frontier",
+        primaryModel: model,
+        alternateModel: model,
+        fastMode,
+        thinkingDefault,
+        scenarioIds: [scenarioId],
+      });
+      const transcript = extractTranscript(result);
+      const transcriptFailure = detectTranscriptFailure(transcript);
+      const status =
+        result.scenarios.some((scenario) => scenario.status === "fail") || transcriptFailure
+          ? "fail"
+          : "pass";
+      const run = {
+        model,
+        status,
+        durationMs: Date.now() - runStartedAt,
+        outputDir: modelOutputDir,
+        thinkingDefault,
+        fastMode,
+        reportPath: result.reportPath,
+        summaryPath: result.summaryPath,
+        transcript,
+        stats: collectTranscriptStats(transcript),
+        ...(transcriptFailure ? { error: transcriptFailure } : {}),
+      } satisfies QaCharacterEvalRun;
+      logCharacterEvalProgress(
+        params.progress,
+        `candidate done ${formatEvalIndex(index, models.length)} model=${model} ${summarizeRunStats(run)}`,
+      );
+      return run;
+    } catch (error) {
+      const transcript = "";
+      const run = {
+        model,
+        status: "fail",
+        durationMs: Date.now() - runStartedAt,
+        outputDir: modelOutputDir,
+        thinkingDefault,
+        fastMode,
+        transcript,
+        stats: collectTranscriptStats(transcript),
+        error: formatErrorMessage(error),
+      } satisfies QaCharacterEvalRun;
+      logCharacterEvalProgress(
+        params.progress,
+        `candidate done ${formatEvalIndex(index, models.length)} model=${model} ${summarizeRunStats(run)}`,
+      );
+      return run;
+    }
+  });
+  const failedCandidateCount = runs.filter((run) => run.status === "fail").length;
+  logCharacterEvalProgress(
+    params.progress,
+    `candidates done pass=${runs.length - failedCandidateCount} fail=${failedCandidateCount} duration=${formatDuration(Date.now() - candidatesStartedAt)}`,
+  );
+
+  const judgeModels = normalizeModelRefs(
+    params.judgeModels && params.judgeModels.length > 0
+      ? params.judgeModels
+      : params.judgeModel
+        ? [params.judgeModel]
+        : DEFAULT_JUDGE_MODELS,
+  );
+  const runJudge = params.runJudge ?? defaultRunJudge;
+  const judgeConcurrency = normalizeConcurrency(
+    params.judgeConcurrency,
+    DEFAULT_CHARACTER_EVAL_CONCURRENCY,
+  );
+  const judgeTimeoutMs = params.judgeTimeoutMs ?? DEFAULT_JUDGE_TIMEOUT_MS;
+  logCharacterEvalProgress(
+    params.progress,
+    `judges start judges=${judgeModels.length} judgeConcurrency=${judgeConcurrency} timeout=${formatDuration(judgeTimeoutMs)} labels=${params.judgeBlindModels === true ? "blind" : "visible"}`,
+  );
+  const judgesStartedAt = Date.now();
+  const judgments = await mapWithConcurrency(
+    judgeModels,
+    judgeConcurrency,
+    async (judgeModel, index) => {
+      const judgeOptions = resolveJudgeOptions({
+        model: judgeModel,
+        judgeThinkingDefault: params.judgeThinkingDefault,
+        judgeModelOptions: params.judgeModelOptions,
+      });
+      let rankings: QaCharacterEvalJudgment[] = [];
+      let judgeError: string | undefined;
+      const judgeStartedAt = Date.now();
+      logCharacterEvalProgress(
+        params.progress,
+        `judge start ${formatEvalIndex(index, judgeModels.length)} model=${judgeModel} thinking=${judgeOptions.thinkingDefault} fast=${judgeOptions.fastMode ? "on" : "off"} timeout=${formatDuration(judgeTimeoutMs)}`,
+      );
+      try {
+        const judgePrompt = buildJudgePrompt({
+          scenarioId,
+          runs,
+          blindModels: params.judgeBlindModels,
+        });
+        const rawReply = await runJudge({
+          repoRoot,
+          judgeModel,
+          judgeThinkingDefault: judgeOptions.thinkingDefault,
+          judgeFastMode: judgeOptions.fastMode,
+          prompt: judgePrompt.prompt,
+          timeoutMs: judgeTimeoutMs,
+        });
+        rankings = parseJudgeReply(rawReply, new Set(judgePrompt.labelToModel.keys())).map(
+          (ranking) =>
+            Object.assign({}, ranking, {
+              model: judgePrompt.labelToModel.get(ranking.model) ?? ranking.model,
+            }),
+        );
+      } catch (error) {
+        judgeError = formatErrorMessage(error);
+      }
+
+      const judgment = {
+        model: judgeModel,
+        thinkingDefault: judgeOptions.thinkingDefault,
+        fastMode: judgeOptions.fastMode,
+        blindModels: params.judgeBlindModels === true,
+        timeoutMs: judgeTimeoutMs,
+        durationMs: Date.now() - judgeStartedAt,
+        rankings,
+        ...(judgeError ? { error: judgeError } : {}),
+      } satisfies QaCharacterEvalJudgeResult;
+      logCharacterEvalProgress(
+        params.progress,
+        `judge done ${formatEvalIndex(index, judgeModels.length)} model=${judgeModel} rankings=${rankings.length} duration=${formatDuration(judgment.durationMs)}${judgeError ? ` error="${judgeError}"` : ""}`,
+      );
+      return judgment;
+    },
+  );
+  const failedJudgeCount = judgments.filter((judgment) => judgment.rankings.length === 0).length;
+  logCharacterEvalProgress(
+    params.progress,
+    `judges done ranked=${judgments.length - failedJudgeCount} failed=${failedJudgeCount} duration=${formatDuration(Date.now() - judgesStartedAt)}`,
+  );
+
+  const finishedAt = new Date();
+  const report = renderCharacterEvalReport({
+    scenarioId,
+    startedAt,
+    finishedAt,
+    runs,
+    judgments,
+  });
+  const reportPath = path.join(outputDir, "character-eval-report.md");
+  const summaryPath = path.join(outputDir, "character-eval-summary.json");
+  await fs.writeFile(reportPath, report, "utf8");
+  await fs.writeFile(
+    summaryPath,
+    `${JSON.stringify(
+      {
+        scenarioId,
+        runs,
+        judgments,
+      },
+      null,
+      2,
+    )}\n`,
+    "utf8",
+  );
+  logCharacterEvalProgress(
+    params.progress,
+    `report written duration=${formatDuration(finishedAt.getTime() - startedAt.getTime())} report=${reportPath} summary=${summaryPath}`,
+  );
+
+  return {
+    outputDir,
+    reportPath,
+    summaryPath,
+    runs,
+    judgments,
+  } satisfies QaCharacterEvalResult;
+}

openclaw/extensions/qa-lab/src/cli-options.ts

@@ -0,0 +1,4 @@
+export function collectString(value: string, previous: string[]) {
+  const trimmed = value.trim();
+  return trimmed ? [...previous, trimmed] : previous;
+}

openclaw/extensions/qa-lab/src/cli-paths.ts

@@ -0,0 +1,120 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export function resolveRepoRelativeOutputDir(repoRoot: string, outputDir?: string) {
+  if (!outputDir) {
+    return undefined;
+  }
+  if (path.isAbsolute(outputDir)) {
+    throw new Error("--output-dir must be a relative path inside the repo root.");
+  }
+  const resolved = path.resolve(repoRoot, outputDir);
+  const relative = path.relative(repoRoot, resolved);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error("--output-dir must stay within the repo root.");
+  }
+  return resolved;
+}
+
+async function resolveNearestExistingPath(targetPath: string) {
+  let current = path.resolve(targetPath);
+  while (true) {
+    try {
+      await fs.lstat(current);
+      return current;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+        throw error;
+      }
+    }
+    const parent = path.dirname(current);
+    if (parent === current) {
+      throw new Error(`failed to resolve existing path for ${targetPath}`);
+    }
+    current = parent;
+  }
+}
+
+function assertRepoRelativePath(repoRoot: string, targetPath: string, label: string) {
+  const relative = path.relative(repoRoot, targetPath);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error(`${label} must stay within the repo root.`);
+  }
+  return relative;
+}
+
+async function assertNoSymlinkSegments(repoRoot: string, targetPath: string, label: string) {
+  const relative = assertRepoRelativePath(repoRoot, targetPath, label);
+  let current = repoRoot;
+  for (const segment of relative.split(path.sep).filter((entry) => entry.length > 0)) {
+    current = path.join(current, segment);
+    let stats: Awaited<ReturnType<typeof fs.lstat>> | null = null;
+    try {
+      stats = await fs.lstat(current);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+        break;
+      }
+      throw error;
+    }
+    if (stats.isSymbolicLink()) {
+      throw new Error(`${label} must not traverse symlinks.`);
+    }
+  }
+}
+
+export async function assertRepoBoundPath(repoRoot: string, targetPath: string, label: string) {
+  const repoRootResolved = path.resolve(repoRoot);
+  const targetResolved = path.resolve(targetPath);
+  assertRepoRelativePath(repoRootResolved, targetResolved, label);
+  await assertNoSymlinkSegments(repoRootResolved, targetResolved, label);
+  const repoRootReal = await fs.realpath(repoRootResolved);
+  const nearestExistingPath = await resolveNearestExistingPath(targetResolved);
+  const nearestExistingReal = await fs.realpath(nearestExistingPath);
+  assertRepoRelativePath(repoRootReal, nearestExistingReal, label);
+  return targetResolved;
+}
+
+export async function ensureRepoBoundDirectory(
+  repoRoot: string,
+  targetDir: string,
+  label: string,
+  opts?: { mode?: number },
+) {
+  const repoRootResolved = path.resolve(repoRoot);
+  const targetResolved = path.resolve(targetDir);
+  const relative = assertRepoRelativePath(repoRootResolved, targetResolved, label);
+  const repoRootReal = await fs.realpath(repoRootResolved);
+  let current = repoRootResolved;
+  for (const segment of relative.split(path.sep).filter((entry) => entry.length > 0)) {
+    current = path.join(current, segment);
+    while (true) {
+      try {
+        const stats = await fs.lstat(current);
+        if (stats.isSymbolicLink()) {
+          throw new Error(`${label} must not traverse symlinks.`);
+        }
+        if (!stats.isDirectory()) {
+          throw new Error(`${label} must point to a directory.`);
+        }
+        break;
+      } catch (error) {
+        const code = (error as NodeJS.ErrnoException).code;
+        if (code !== "ENOENT") {
+          throw error;
+        }
+        try {
+          await fs.mkdir(current, { recursive: false, mode: opts?.mode });
+        } catch (mkdirError) {
+          if ((mkdirError as NodeJS.ErrnoException).code === "EEXIST") {
+            continue;
+          }
+          throw mkdirError;
+        }
+      }
+    }
+  }
+  const targetReal = await fs.realpath(targetResolved);
+  assertRepoRelativePath(repoRootReal, targetReal, label);
+  return targetResolved;
+}

openclaw/extensions/qa-lab/src/cli.runtime.ts

@@ -0,0 +1,846 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import {
+  buildQaAgenticParityComparison,
+  renderQaAgenticParityMarkdownReport,
+  type QaParitySuiteSummary,
+} from "./agentic-parity-report.js";
+import { resolveQaParityPackScenarioIds } from "./agentic-parity.js";
+import { runQaCharacterEval, type QaCharacterModelOptions } from "./character-eval.js";
+import { resolveRepoRelativeOutputDir } from "./cli-paths.js";
+import { buildQaCoverageInventory, renderQaCoverageMarkdownReport } from "./coverage-report.js";
+import { buildQaDockerHarnessImage, writeQaDockerHarnessFiles } from "./docker-harness.js";
+import { runQaDockerUp } from "./docker-up.runtime.js";
+import type { QaCliBackendAuthMode } from "./gateway-child.js";
+import { startQaLabServer } from "./lab-server.js";
+import { runQaManualLane } from "./manual-lane.runtime.js";
+import { runQaMultipass } from "./multipass.runtime.js";
+import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
+import {
+  QA_FRONTIER_PARITY_BASELINE_LABEL,
+  QA_FRONTIER_PARITY_CANDIDATE_LABEL,
+} from "./providers/live-frontier/parity.js";
+import { startQaProviderServer } from "./providers/server-runtime.js";
+import {
+  addQaCredentialSet,
+  listQaCredentialSets,
+  QaCredentialAdminError,
+  removeQaCredentialSet,
+  type QaCredentialRecord,
+} from "./qa-credentials-admin.runtime.js";
+import { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-gateway-config.js";
+import { normalizeQaTransportId } from "./qa-transport-registry.js";
+import {
+  defaultQaModelForMode,
+  normalizeQaProviderMode,
+  type QaProviderMode,
+  type QaProviderModeInput,
+} from "./run-config.js";
+import { readQaScenarioPack } from "./scenario-catalog.js";
+import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
+import { readQaSuiteFailedScenarioCountFromSummary } from "./suite-summary.js";
+
+type InterruptibleServer = {
+  baseUrl: string;
+  stop(): Promise<void>;
+};
+
+function resolveQaManualLaneModels(opts: {
+  providerMode: QaProviderMode;
+  primaryModel?: string;
+  alternateModel?: string;
+}) {
+  const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(opts.providerMode);
+  const alternateModel = opts.alternateModel?.trim();
+  return {
+    primaryModel,
+    alternateModel:
+      alternateModel && alternateModel.length > 0
+        ? alternateModel
+        : opts.primaryModel?.trim()
+          ? primaryModel
+          : defaultQaModelForMode(opts.providerMode, true),
+  };
+}
+
+function parseQaThinkingLevel(
+  label: string,
+  value: string | undefined,
+): QaThinkingLevel | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+  const normalized = normalizeQaThinkingLevel(value);
+  if (!normalized) {
+    throw new Error(`${label} must be one of off, minimal, low, medium, high, xhigh, adaptive`);
+  }
+  return normalized;
+}
+
+function parseQaModelThinkingOverrides(entries: readonly string[] | undefined) {
+  const overrides: Record<string, QaThinkingLevel> = {};
+  for (const entry of entries ?? []) {
+    const separatorIndex = entry.lastIndexOf("=");
+    if (separatorIndex <= 0 || separatorIndex === entry.length - 1) {
+      throw new Error(`--model-thinking must use provider/model=level, got "${entry}"`);
+    }
+    const model = entry.slice(0, separatorIndex).trim();
+    const level = parseQaThinkingLevel("--model-thinking", entry.slice(separatorIndex + 1).trim());
+    if (!model || !level) {
+      throw new Error(`--model-thinking must use provider/model=level, got "${entry}"`);
+    }
+    overrides[model] = level;
+  }
+  return Object.keys(overrides).length > 0 ? overrides : undefined;
+}
+
+function parseQaBooleanModelOption(label: string, value: string) {
+  switch (value.trim().toLowerCase()) {
+    case "1":
+    case "on":
+    case "true":
+    case "yes":
+      return true;
+    case "0":
+    case "false":
+    case "no":
+    case "off":
+      return false;
+    default:
+      throw new Error(`${label} fast must be one of true, false, on, off, yes, no, 1, 0`);
+  }
+}
+
+function parseQaPositiveIntegerOption(label: string, value: number | undefined) {
+  if (value === undefined) {
+    return undefined;
+  }
+  if (!Number.isFinite(value) || value < 1) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return Math.floor(value);
+}
+
+async function readQaFailedScenarioCountFromSummary(summaryPath: string) {
+  let summaryText: string;
+  try {
+    summaryText = await fs.readFile(summaryPath, "utf8");
+  } catch (error) {
+    throw new Error(
+      `Could not read QA summary JSON at ${summaryPath}: ${formatErrorMessage(error)}`,
+      { cause: error },
+    );
+  }
+  let payload: unknown;
+  try {
+    payload = JSON.parse(summaryText) as unknown;
+  } catch (error) {
+    throw new Error(
+      `Could not parse QA summary JSON at ${summaryPath}: ${formatErrorMessage(error)}`,
+      { cause: error },
+    );
+  }
+  const failedScenarioCount = readQaSuiteFailedScenarioCountFromSummary(payload);
+  if (failedScenarioCount !== null) {
+    return failedScenarioCount;
+  }
+  throw new Error(
+    `QA summary at ${summaryPath} did not include counts.failed or scenarios[].status.`,
+  );
+}
+
+function parseQaCliBackendAuthMode(value: string | undefined): QaCliBackendAuthMode | undefined {
+  const normalized = value?.trim().toLowerCase();
+  if (!normalized) {
+    return undefined;
+  }
+  if (normalized === "auto" || normalized === "api-key" || normalized === "subscription") {
+    return normalized;
+  }
+  throw new Error("--cli-auth-mode must be one of auto, api-key, subscription");
+}
+
+function parseQaCredentialListStatus(value: string | undefined) {
+  if (value === undefined) {
+    return undefined;
+  }
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "active" || normalized === "disabled" || normalized === "all") {
+    return normalized;
+  }
+  throw new Error('--status must be one of "active", "disabled", or "all".');
+}
+
+function normalizeQaCredentialAdminError(error: unknown) {
+  if (error instanceof QaCredentialAdminError) {
+    return {
+      code: error.code,
+      message: error.message,
+    };
+  }
+  return {
+    code: "UNEXPECTED_ERROR",
+    message: formatErrorMessage(error),
+  };
+}
+
+function writeQaCredentialCommandErrorJson(action: string, error: unknown) {
+  const normalized = normalizeQaCredentialAdminError(error);
+  process.stdout.write(
+    `${JSON.stringify(
+      {
+        status: "error",
+        action,
+        code: normalized.code,
+        message: normalized.message,
+      },
+      null,
+      2,
+    )}\n`,
+  );
+}
+
+function parseQaModelSpecs(label: string, entries: readonly string[] | undefined) {
+  const models: string[] = [];
+  const optionsByModel: Record<string, QaCharacterModelOptions> = {};
+
+  for (const entry of entries ?? []) {
+    const parts = entry.split(",").map((part) => part.trim());
+    const model = parts[0];
+    if (!model) {
+      throw new Error(`${label} must start with provider/model, got "${entry}"`);
+    }
+    models.push(model);
+    const options: QaCharacterModelOptions = {};
+    for (const part of parts.slice(1)) {
+      if (!part) {
+        throw new Error(`${label} option cannot be empty in "${entry}"`);
+      }
+      if (part === "fast") {
+        options.fastMode = true;
+        continue;
+      }
+      if (part === "no-fast") {
+        options.fastMode = false;
+        continue;
+      }
+      const separatorIndex = part.indexOf("=");
+      if (separatorIndex <= 0 || separatorIndex === part.length - 1) {
+        throw new Error(
+          `${label} options must be thinking=<level>, fast, no-fast, or fast=<boolean>, got "${part}"`,
+        );
+      }
+      const key = part.slice(0, separatorIndex).trim();
+      const value = part.slice(separatorIndex + 1).trim();
+      switch (key) {
+        case "thinking": {
+          const thinkingDefault = parseQaThinkingLevel(`${label} thinking`, value);
+          if (!thinkingDefault) {
+            throw new Error(
+              `${label} thinking must be one of off, minimal, low, medium, high, xhigh, adaptive`,
+            );
+          }
+          options.thinkingDefault = thinkingDefault;
+          break;
+        }
+        case "fast":
+          options.fastMode = parseQaBooleanModelOption(label, value);
+          break;
+        default:
+          throw new Error(`${label} does not support option "${key}" in "${entry}"`);
+      }
+    }
+    if (Object.keys(options).length > 0) {
+      optionsByModel[model] = { ...optionsByModel[model], ...options };
+    }
+  }
+
+  return {
+    models,
+    optionsByModel: Object.keys(optionsByModel).length > 0 ? optionsByModel : undefined,
+  };
+}
+
+async function runInterruptibleServer(label: string, server: InterruptibleServer) {
+  process.stdout.write(`${label}: ${server.baseUrl}\n`);
+  process.stdout.write("Press Ctrl+C to stop.\n");
+
+  const shutdown = async () => {
+    process.off("SIGINT", onSignal);
+    process.off("SIGTERM", onSignal);
+    await server.stop();
+    process.exit(0);
+  };
+
+  const onSignal = () => {
+    void shutdown();
+  };
+
+  process.on("SIGINT", onSignal);
+  process.on("SIGTERM", onSignal);
+  await new Promise(() => undefined);
+}
+
+async function readQaCredentialPayloadFile(filePath: string) {
+  const text = await fs.readFile(filePath, "utf8");
+  let payload: unknown;
+  try {
+    payload = JSON.parse(text) as unknown;
+  } catch (error) {
+    throw new Error(`Payload file must contain valid JSON: ${formatErrorMessage(error)}`, {
+      cause: error,
+    });
+  }
+  if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
+    throw new Error("Payload file JSON must be an object.");
+  }
+  return payload as Record<string, unknown>;
+}
+
+function formatQaCredentialLeaseState(credential: QaCredentialRecord) {
+  if (!credential.lease) {
+    return "no";
+  }
+  return `yes(${credential.lease.actorRole}:${credential.lease.ownerId})`;
+}
+
+function printQaCredentialListTable(credentials: QaCredentialRecord[]) {
+  if (credentials.length === 0) {
+    process.stdout.write("No credentials matched.\n");
+    return;
+  }
+  const rows = credentials.map((credential) => ({
+    credentialId: credential.credentialId,
+    kind: credential.kind,
+    status: credential.status,
+    leased: formatQaCredentialLeaseState(credential),
+    note: credential.note ?? "",
+  }));
+  const idWidth = Math.max("credentialId".length, ...rows.map((row) => row.credentialId.length));
+  const kindWidth = Math.max("kind".length, ...rows.map((row) => row.kind.length));
+  const statusWidth = Math.max("status".length, ...rows.map((row) => row.status.length));
+  const leaseWidth = Math.max("leased".length, ...rows.map((row) => row.leased.length));
+  process.stdout.write(
+    `${"credentialId".padEnd(idWidth)}  ${"kind".padEnd(kindWidth)}  ${"status".padEnd(statusWidth)}  ${"leased".padEnd(leaseWidth)}  note\n`,
+  );
+  for (const row of rows) {
+    process.stdout.write(
+      `${row.credentialId.padEnd(idWidth)}  ${row.kind.padEnd(kindWidth)}  ${row.status.padEnd(statusWidth)}  ${row.leased.padEnd(leaseWidth)}  ${row.note}\n`,
+    );
+  }
+}
+
+export async function runQaLabSelfCheckCommand(opts: { repoRoot?: string; output?: string }) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const server = await startQaLabServer({
+    repoRoot,
+    outputPath: opts.output ? path.resolve(repoRoot, opts.output) : undefined,
+  });
+  try {
+    const result = await server.runSelfCheck();
+    process.stdout.write(`QA self-check report: ${result.outputPath}\n`);
+  } finally {
+    await server.stop();
+  }
+}
+
+export async function runQaSuiteCommand(opts: {
+  repoRoot?: string;
+  outputDir?: string;
+  transportId?: string;
+  runner?: string;
+  providerMode?: QaProviderModeInput;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  cliAuthMode?: string;
+  parityPack?: string;
+  scenarioIds?: string[];
+  concurrency?: number;
+  allowFailures?: boolean;
+  image?: string;
+  cpus?: number;
+  memory?: string;
+  disk?: string;
+}) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const transportId = normalizeQaTransportId(opts.transportId);
+  const runner = (opts.runner ?? "host").trim().toLowerCase();
+  const scenarioIds = resolveQaParityPackScenarioIds({
+    parityPack: opts.parityPack,
+    scenarioIds: opts.scenarioIds,
+  });
+  const allowFailures = opts.allowFailures === true;
+  if (runner !== "host" && runner !== "multipass") {
+    throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`);
+  }
+  const providerMode = normalizeQaProviderMode(opts.providerMode);
+  const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode);
+  if (
+    runner === "host" &&
+    (opts.image !== undefined ||
+      opts.cpus !== undefined ||
+      opts.memory !== undefined ||
+      opts.disk !== undefined)
+  ) {
+    throw new Error("--image, --cpus, --memory, and --disk require --runner multipass.");
+  }
+  if (runner === "multipass" && opts.cliAuthMode !== undefined) {
+    throw new Error("--cli-auth-mode requires --runner host.");
+  }
+  if (runner === "multipass") {
+    const result = await runQaMultipass({
+      repoRoot,
+      outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
+      transportId,
+      providerMode,
+      primaryModel: opts.primaryModel,
+      alternateModel: opts.alternateModel,
+      fastMode: opts.fastMode,
+      allowFailures: true,
+      scenarioIds,
+      ...(opts.concurrency !== undefined
+        ? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) }
+        : {}),
+      image: opts.image,
+      cpus: parseQaPositiveIntegerOption("--cpus", opts.cpus),
+      memory: opts.memory,
+      disk: opts.disk,
+    });
+    process.stdout.write(`QA Multipass dir: ${result.outputDir}\n`);
+    process.stdout.write(`QA Multipass report: ${result.reportPath}\n`);
+    process.stdout.write(`QA Multipass summary: ${result.summaryPath}\n`);
+    process.stdout.write(`QA Multipass host log: ${result.hostLogPath}\n`);
+    process.stdout.write(`QA Multipass bootstrap log: ${result.bootstrapLogPath}\n`);
+    if (!allowFailures) {
+      const failedScenarioCount = await readQaFailedScenarioCountFromSummary(result.summaryPath);
+      if (failedScenarioCount > 0) {
+        process.exitCode = 1;
+      }
+    }
+    return;
+  }
+  const result = await runQaSuiteFromRuntime({
+    repoRoot,
+    outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
+    transportId,
+    providerMode,
+    primaryModel: opts.primaryModel,
+    alternateModel: opts.alternateModel,
+    fastMode: opts.fastMode,
+    ...(claudeCliAuthMode ? { claudeCliAuthMode } : {}),
+    scenarioIds,
+    ...(opts.concurrency !== undefined
+      ? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) }
+      : {}),
+  });
+  process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
+  process.stdout.write(`QA suite report: ${result.reportPath}\n`);
+  process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
+  const failedScenarioCount = readQaSuiteFailedScenarioCountFromSummary(result);
+  if (!allowFailures && failedScenarioCount !== null && failedScenarioCount > 0) {
+    process.exitCode = 1;
+  }
+}
+
+export async function runQaParityReportCommand(opts: {
+  repoRoot?: string;
+  candidateSummary: string;
+  baselineSummary: string;
+  candidateLabel?: string;
+  baselineLabel?: string;
+  outputDir?: string;
+}) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const outputDir =
+    resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ??
+    path.join(repoRoot, ".artifacts", "qa-e2e", `parity-${Date.now().toString(36)}`);
+  await fs.mkdir(outputDir, { recursive: true });
+
+  const candidateSummaryPath = path.resolve(repoRoot, opts.candidateSummary);
+  const baselineSummaryPath = path.resolve(repoRoot, opts.baselineSummary);
+  const candidateSummary = JSON.parse(
+    await fs.readFile(candidateSummaryPath, "utf8"),
+  ) as QaParitySuiteSummary;
+  const baselineSummary = JSON.parse(
+    await fs.readFile(baselineSummaryPath, "utf8"),
+  ) as QaParitySuiteSummary;
+
+  const comparison = buildQaAgenticParityComparison({
+    candidateLabel: opts.candidateLabel?.trim() || QA_FRONTIER_PARITY_CANDIDATE_LABEL,
+    baselineLabel: opts.baselineLabel?.trim() || QA_FRONTIER_PARITY_BASELINE_LABEL,
+    candidateSummary,
+    baselineSummary,
+  });
+  const report = renderQaAgenticParityMarkdownReport(comparison);
+  const reportPath = path.join(outputDir, "qa-agentic-parity-report.md");
+  const summaryPath = path.join(outputDir, "qa-agentic-parity-summary.json");
+  await fs.writeFile(reportPath, report, "utf8");
+  await fs.writeFile(summaryPath, `${JSON.stringify(comparison, null, 2)}\n`, "utf8");
+
+  process.stdout.write(`QA parity report: ${reportPath}\n`);
+  process.stdout.write(`QA parity summary: ${summaryPath}\n`);
+  process.stdout.write(`QA parity verdict: ${comparison.pass ? "pass" : "fail"}\n`);
+  if (!comparison.pass) {
+    process.exitCode = 1;
+  }
+}
+
+export async function runQaCoverageReportCommand(opts: {
+  repoRoot?: string;
+  output?: string;
+  json?: boolean;
+}) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const inventory = buildQaCoverageInventory(readQaScenarioPack().scenarios);
+  const outputPath = opts.output ? path.resolve(repoRoot, opts.output) : undefined;
+  const body = opts.json
+    ? `${JSON.stringify(inventory, null, 2)}\n`
+    : renderQaCoverageMarkdownReport(inventory);
+
+  if (outputPath) {
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    await fs.writeFile(outputPath, body, "utf8");
+    process.stdout.write(`QA coverage report: ${outputPath}\n`);
+    return;
+  }
+
+  process.stdout.write(body);
+}
+
+export async function runQaCharacterEvalCommand(opts: {
+  repoRoot?: string;
+  outputDir?: string;
+  model?: string[];
+  scenario?: string;
+  fast?: boolean;
+  thinking?: string;
+  modelThinking?: string[];
+  judgeModel?: string[];
+  judgeTimeoutMs?: number;
+  blindJudgeModels?: boolean;
+  concurrency?: number;
+  judgeConcurrency?: number;
+}) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const candidates = parseQaModelSpecs("--model", opts.model);
+  const judges = parseQaModelSpecs("--judge-model", opts.judgeModel);
+  const result = await runQaCharacterEval({
+    repoRoot,
+    outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
+    models: candidates.models,
+    scenarioId: opts.scenario,
+    candidateFastMode: opts.fast,
+    candidateThinkingDefault: parseQaThinkingLevel("--thinking", opts.thinking),
+    candidateThinkingByModel: parseQaModelThinkingOverrides(opts.modelThinking),
+    candidateModelOptions: candidates.optionsByModel,
+    judgeModels: judges.models.length > 0 ? judges.models : undefined,
+    judgeModelOptions: judges.optionsByModel,
+    judgeTimeoutMs: opts.judgeTimeoutMs,
+    judgeBlindModels: opts.blindJudgeModels === true ? true : undefined,
+    candidateConcurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency),
+    judgeConcurrency: parseQaPositiveIntegerOption("--judge-concurrency", opts.judgeConcurrency),
+    progress: (message) => process.stderr.write(`${message}\n`),
+  });
+  process.stdout.write(`QA character eval report: ${result.reportPath}\n`);
+  process.stdout.write(`QA character eval summary: ${result.summaryPath}\n`);
+}
+
+export async function runQaManualLaneCommand(opts: {
+  repoRoot?: string;
+  transportId?: string;
+  providerMode?: QaProviderModeInput;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  message: string;
+  timeoutMs?: number;
+}) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const transportId = normalizeQaTransportId(opts.transportId);
+  const providerMode: QaProviderMode =
+    opts.providerMode === undefined
+      ? DEFAULT_QA_LIVE_PROVIDER_MODE
+      : normalizeQaProviderMode(opts.providerMode);
+  const models = resolveQaManualLaneModels({
+    providerMode,
+    primaryModel: opts.primaryModel,
+    alternateModel: opts.alternateModel,
+  });
+  const result = await runQaManualLane({
+    repoRoot,
+    transportId,
+    providerMode,
+    primaryModel: models.primaryModel,
+    alternateModel: models.alternateModel,
+    fastMode: opts.fastMode,
+    message: opts.message,
+    timeoutMs: opts.timeoutMs,
+  });
+  process.stdout.write(JSON.stringify(result, null, 2));
+  process.stdout.write("\n");
+}
+
+export async function runQaCredentialsAddCommand(opts: {
+  actorId?: string;
+  endpointPrefix?: string;
+  json?: boolean;
+  kind: string;
+  note?: string;
+  payloadFile: string;
+  repoRoot?: string;
+  siteUrl?: string;
+}) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  try {
+    const payloadPath = path.resolve(repoRoot, opts.payloadFile);
+    const payload = await readQaCredentialPayloadFile(payloadPath);
+    const result = await addQaCredentialSet({
+      kind: opts.kind,
+      payload,
+      note: opts.note,
+      actorId: opts.actorId,
+      siteUrl: opts.siteUrl,
+      endpointPrefix: opts.endpointPrefix,
+    });
+    if (opts.json) {
+      process.stdout.write(
+        `${JSON.stringify({ status: "ok", action: "add", credential: result.credential }, null, 2)}\n`,
+      );
+      return;
+    }
+    process.stdout.write(`QA credential added: ${result.credential.credentialId}\n`);
+    process.stdout.write(`Kind: ${result.credential.kind}\n`);
+    process.stdout.write(`Status: ${result.credential.status}\n`);
+    if (result.credential.note) {
+      process.stdout.write(`Note: ${result.credential.note}\n`);
+    }
+  } catch (error) {
+    if (opts.json) {
+      writeQaCredentialCommandErrorJson("add", error);
+      process.exitCode = 1;
+      return;
+    }
+    throw error;
+  }
+}
+
+export async function runQaCredentialsRemoveCommand(opts: {
+  actorId?: string;
+  credentialId: string;
+  endpointPrefix?: string;
+  json?: boolean;
+  siteUrl?: string;
+}) {
+  try {
+    const result = await removeQaCredentialSet({
+      credentialId: opts.credentialId,
+      actorId: opts.actorId,
+      siteUrl: opts.siteUrl,
+      endpointPrefix: opts.endpointPrefix,
+    });
+    if (opts.json) {
+      process.stdout.write(
+        `${JSON.stringify(
+          {
+            status: "ok",
+            action: "remove",
+            changed: result.changed,
+            credential: result.credential,
+          },
+          null,
+          2,
+        )}\n`,
+      );
+      return;
+    }
+    process.stdout.write(
+      result.changed
+        ? `QA credential removed (disabled): ${result.credential.credentialId}\n`
+        : `QA credential already disabled: ${result.credential.credentialId}\n`,
+    );
+  } catch (error) {
+    if (opts.json) {
+      writeQaCredentialCommandErrorJson("remove", error);
+      process.exitCode = 1;
+      return;
+    }
+    throw error;
+  }
+}
+
+export async function runQaCredentialsListCommand(opts: {
+  actorId?: string;
+  endpointPrefix?: string;
+  json?: boolean;
+  kind?: string;
+  limit?: number;
+  showSecrets?: boolean;
+  siteUrl?: string;
+  status?: string;
+}) {
+  try {
+    const result = await listQaCredentialSets({
+      actorId: opts.actorId,
+      siteUrl: opts.siteUrl,
+      endpointPrefix: opts.endpointPrefix,
+      kind: opts.kind?.trim(),
+      status: parseQaCredentialListStatus(opts.status),
+      includePayload: opts.showSecrets,
+      limit: parseQaPositiveIntegerOption("--limit", opts.limit),
+    });
+    if (opts.json) {
+      process.stdout.write(
+        `${JSON.stringify(
+          {
+            status: "ok",
+            action: "list",
+            count: result.credentials.length,
+            credentials: result.credentials,
+          },
+          null,
+          2,
+        )}\n`,
+      );
+      return;
+    }
+    printQaCredentialListTable(result.credentials);
+    if (opts.showSecrets && result.credentials.length > 0) {
+      process.stdout.write("\nPayloads:\n");
+      for (const credential of result.credentials) {
+        process.stdout.write(
+          `${credential.credentialId}: ${JSON.stringify(credential.payload ?? null)}\n`,
+        );
+      }
+    }
+  } catch (error) {
+    if (opts.json) {
+      writeQaCredentialCommandErrorJson("list", error);
+      process.exitCode = 1;
+      return;
+    }
+    throw error;
+  }
+}
+
+export async function runQaLabUiCommand(opts: {
+  repoRoot?: string;
+  host?: string;
+  port?: number;
+  advertiseHost?: string;
+  advertisePort?: number;
+  controlUiUrl?: string;
+  controlUiToken?: string;
+  controlUiProxyTarget?: string;
+  uiDistDir?: string;
+  autoKickoffTarget?: string;
+  embeddedGateway?: string;
+  sendKickoffOnStart?: boolean;
+}) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const server = await startQaLabServer({
+    repoRoot,
+    host: opts.host,
+    port: Number.isFinite(opts.port) ? opts.port : undefined,
+    advertiseHost: opts.advertiseHost,
+    advertisePort: Number.isFinite(opts.advertisePort) ? opts.advertisePort : undefined,
+    controlUiUrl: opts.controlUiUrl,
+    controlUiToken: opts.controlUiToken,
+    controlUiProxyTarget: opts.controlUiProxyTarget,
+    uiDistDir: opts.uiDistDir,
+    autoKickoffTarget: opts.autoKickoffTarget,
+    embeddedGateway: opts.embeddedGateway,
+    sendKickoffOnStart: opts.sendKickoffOnStart,
+  });
+  await runInterruptibleServer("QA Lab UI", server);
+}
+
+export async function runQaDockerScaffoldCommand(opts: {
+  repoRoot?: string;
+  outputDir: string;
+  gatewayPort?: number;
+  qaLabPort?: number;
+  providerBaseUrl?: string;
+  image?: string;
+  usePrebuiltImage?: boolean;
+  bindUiDist?: boolean;
+}) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const outputDir = resolveRepoRelativeOutputDir(repoRoot, opts.outputDir);
+  if (!outputDir) {
+    throw new Error("--output-dir is required.");
+  }
+  const result = await writeQaDockerHarnessFiles({
+    outputDir,
+    repoRoot,
+    gatewayPort: Number.isFinite(opts.gatewayPort) ? opts.gatewayPort : undefined,
+    qaLabPort: Number.isFinite(opts.qaLabPort) ? opts.qaLabPort : undefined,
+    providerBaseUrl: opts.providerBaseUrl,
+    imageName: opts.image,
+    usePrebuiltImage: opts.usePrebuiltImage,
+    bindUiDist: opts.bindUiDist,
+  });
+  process.stdout.write(`QA docker scaffold: ${result.outputDir}\n`);
+}
+
+export async function runQaDockerBuildImageCommand(opts: { repoRoot?: string; image?: string }) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const result = await buildQaDockerHarnessImage({
+    repoRoot,
+    imageName: opts.image,
+  });
+  process.stdout.write(`QA docker image: ${result.imageName}\n`);
+}
+
+export async function runQaDockerUpCommand(opts: {
+  repoRoot?: string;
+  outputDir?: string;
+  gatewayPort?: number;
+  qaLabPort?: number;
+  providerBaseUrl?: string;
+  image?: string;
+  usePrebuiltImage?: boolean;
+  bindUiDist?: boolean;
+  skipUiBuild?: boolean;
+}) {
+  const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
+  const result = await runQaDockerUp({
+    repoRoot,
+    outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
+    gatewayPort: Number.isFinite(opts.gatewayPort) ? opts.gatewayPort : undefined,
+    qaLabPort: Number.isFinite(opts.qaLabPort) ? opts.qaLabPort : undefined,
+    providerBaseUrl: opts.providerBaseUrl,
+    image: opts.image,
+    usePrebuiltImage: opts.usePrebuiltImage,
+    bindUiDist: opts.bindUiDist,
+    skipUiBuild: opts.skipUiBuild,
+  });
+  process.stdout.write(`QA docker dir: ${result.outputDir}\n`);
+  process.stdout.write(`QA Lab UI: ${result.qaLabUrl}\n`);
+  process.stdout.write(`Gateway UI: ${result.gatewayUrl}\n`);
+  process.stdout.write(`Stop: ${result.stopCommand}\n`);
+}
+
+export async function runQaProviderServerCommand(
+  providerMode: QaProviderMode,
+  opts: { host?: string; port?: number },
+) {
+  const provider = getQaProvider(providerMode);
+  const standaloneCommand = provider.standaloneCommand;
+  if (!standaloneCommand) {
+    throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
+  }
+  const server = await startQaProviderServer(providerMode, {
+    host: opts.host,
+    port: Number.isFinite(opts.port) ? opts.port : undefined,
+  });
+  if (!server) {
+    throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
+  }
+  await runInterruptibleServer(standaloneCommand.serverLabel, server);
+}
+
+export const __testing = {
+  resolveRepoRelativeOutputDir,
+};

openclaw/extensions/qa-lab/src/cli.test.ts

@@ -0,0 +1,306 @@
+import { Command } from "commander";
+import type { QaRunnerCliContribution } from "openclaw/plugin-sdk/qa-runner-runtime";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const TEST_QA_RUNNER = {
+  pluginId: "qa-runner-test",
+  commandName: "runner-test",
+  description: "Run the test live QA lane",
+} as const;
+
+function createAvailableQaRunnerContribution() {
+  return {
+    pluginId: TEST_QA_RUNNER.pluginId,
+    commandName: TEST_QA_RUNNER.commandName,
+    status: "available" as const,
+    registration: {
+      commandName: TEST_QA_RUNNER.commandName,
+      register: vi.fn((qa: Command) => {
+        qa.command(TEST_QA_RUNNER.commandName).action(() => undefined);
+      }),
+    },
+  } satisfies QaRunnerCliContribution;
+}
+
+function createBlockedQaRunnerContribution(): QaRunnerCliContribution {
+  return {
+    pluginId: TEST_QA_RUNNER.pluginId,
+    commandName: TEST_QA_RUNNER.commandName,
+    description: TEST_QA_RUNNER.description,
+    status: "blocked",
+  };
+}
+
+function createConflictingQaRunnerContribution(commandName: string): QaRunnerCliContribution {
+  return {
+    pluginId: TEST_QA_RUNNER.pluginId,
+    commandName,
+    description: TEST_QA_RUNNER.description,
+    status: "blocked",
+  };
+}
+
+const {
+  runQaCredentialsAddCommand,
+  runQaCredentialsListCommand,
+  runQaCredentialsRemoveCommand,
+  runQaCoverageReportCommand,
+  runQaProviderServerCommand,
+  runQaSuiteCommand,
+  runQaTelegramCommand,
+} = vi.hoisted(() => ({
+  runQaCredentialsAddCommand: vi.fn(),
+  runQaCredentialsListCommand: vi.fn(),
+  runQaCredentialsRemoveCommand: vi.fn(),
+  runQaCoverageReportCommand: vi.fn(),
+  runQaProviderServerCommand: vi.fn(),
+  runQaSuiteCommand: vi.fn(),
+  runQaTelegramCommand: vi.fn(),
+}));
+
+const { listQaRunnerCliContributions } = vi.hoisted(() => ({
+  listQaRunnerCliContributions: vi.fn<() => QaRunnerCliContribution[]>(() => [
+    createAvailableQaRunnerContribution(),
+  ]),
+}));
+
+vi.mock("openclaw/plugin-sdk/qa-runner-runtime", () => ({
+  listQaRunnerCliContributions,
+}));
+
+vi.mock("./live-transports/telegram/cli.runtime.js", () => ({
+  runQaTelegramCommand,
+}));
+
+vi.mock("./cli.runtime.js", () => ({
+  runQaCredentialsAddCommand,
+  runQaCredentialsListCommand,
+  runQaCredentialsRemoveCommand,
+  runQaCoverageReportCommand,
+  runQaProviderServerCommand,
+  runQaSuiteCommand,
+}));
+
+import { registerQaLabCli } from "./cli.js";
+
+describe("qa cli registration", () => {
+  let program: Command;
+
+  beforeEach(() => {
+    program = new Command();
+    runQaCredentialsAddCommand.mockReset();
+    runQaCredentialsListCommand.mockReset();
+    runQaCredentialsRemoveCommand.mockReset();
+    runQaCoverageReportCommand.mockReset();
+    runQaProviderServerCommand.mockReset();
+    runQaSuiteCommand.mockReset();
+    runQaTelegramCommand.mockReset();
+    listQaRunnerCliContributions
+      .mockReset()
+      .mockReturnValue([createAvailableQaRunnerContribution()]);
+    registerQaLabCli(program);
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("registers discovered and built-in live transport subcommands", () => {
+    const qa = program.commands.find((command) => command.name() === "qa");
+    expect(qa).toBeDefined();
+    expect(qa?.commands.map((command) => command.name())).toEqual(
+      expect.arrayContaining([TEST_QA_RUNNER.commandName, "telegram", "credentials", "coverage"]),
+    );
+  });
+
+  it("routes coverage report flags into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "coverage",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--output",
+      ".artifacts/qa-coverage.md",
+      "--json",
+    ]);
+
+    expect(runQaCoverageReportCommand).toHaveBeenCalledWith({
+      repoRoot: "/tmp/openclaw-repo",
+      output: ".artifacts/qa-coverage.md",
+      json: true,
+    });
+  });
+
+  it("delegates discovered qa runner registration through the generic host seam", () => {
+    const [{ registration }] = listQaRunnerCliContributions.mock.results[0]?.value;
+    expect(registration.register).toHaveBeenCalledTimes(1);
+  });
+
+  it("keeps Telegram credential flags on the shared host CLI", () => {
+    const qa = program.commands.find((command) => command.name() === "qa");
+    const telegram = qa?.commands.find((command) => command.name() === "telegram");
+    const optionNames = telegram?.options.map((option) => option.long) ?? [];
+
+    expect(optionNames).toEqual(
+      expect.arrayContaining(["--credential-source", "--credential-role"]),
+    );
+  });
+
+  it("registers standalone provider server commands from the provider registry", async () => {
+    const qa = program.commands.find((command) => command.name() === "qa");
+    expect(qa?.commands.map((command) => command.name())).toEqual(
+      expect.arrayContaining(["mock-openai", "aimock"]),
+    );
+
+    await program.parseAsync(["node", "openclaw", "qa", "aimock", "--port", "44080"]);
+
+    expect(runQaProviderServerCommand).toHaveBeenCalledWith("aimock", {
+      host: "127.0.0.1",
+      port: 44080,
+    });
+  });
+
+  it("shows an enable hint when a discovered runner plugin is installed but blocked", async () => {
+    listQaRunnerCliContributions.mockReset().mockReturnValue([createBlockedQaRunnerContribution()]);
+    const blockedProgram = new Command();
+    registerQaLabCli(blockedProgram);
+
+    await expect(
+      blockedProgram.parseAsync(["node", "openclaw", "qa", TEST_QA_RUNNER.commandName]),
+    ).rejects.toThrow(`Enable or allow plugin "${TEST_QA_RUNNER.pluginId}"`);
+  });
+
+  it("rejects discovered runners that collide with built-in qa subcommands", () => {
+    listQaRunnerCliContributions
+      .mockReset()
+      .mockReturnValue([createConflictingQaRunnerContribution("manual")]);
+
+    expect(() => registerQaLabCli(new Command())).toThrow(
+      'QA runner command "manual" conflicts with an existing qa subcommand',
+    );
+  });
+
+  it("routes telegram CLI defaults into the lane runtime", async () => {
+    await program.parseAsync(["node", "openclaw", "qa", "telegram"]);
+
+    expect(runQaTelegramCommand).toHaveBeenCalledWith({
+      repoRoot: undefined,
+      outputDir: undefined,
+      providerMode: "live-frontier",
+      primaryModel: undefined,
+      alternateModel: undefined,
+      fastMode: false,
+      allowFailures: false,
+      scenarioIds: [],
+      sutAccountId: "sut",
+      credentialSource: undefined,
+      credentialRole: undefined,
+    });
+  });
+
+  it("forwards --allow-failures for telegram runs", async () => {
+    await program.parseAsync(["node", "openclaw", "qa", "telegram", "--allow-failures"]);
+
+    expect(runQaTelegramCommand).toHaveBeenCalledWith(
+      expect.objectContaining({
+        allowFailures: true,
+      }),
+    );
+  });
+
+  it("forwards --allow-failures for suite runs", async () => {
+    await program.parseAsync(["node", "openclaw", "qa", "suite", "--allow-failures"]);
+
+    expect(runQaSuiteCommand).toHaveBeenCalledWith(
+      expect.objectContaining({
+        allowFailures: true,
+      }),
+    );
+  });
+
+  it("routes credential add flags into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "credentials",
+      "add",
+      "--kind",
+      "telegram",
+      "--payload-file",
+      "qa/payload.json",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--note",
+      "shared lane",
+      "--site-url",
+      "https://first-schnauzer-821.convex.site",
+      "--endpoint-prefix",
+      "/qa-credentials/v1",
+      "--actor-id",
+      "maintainer-local",
+      "--json",
+    ]);
+
+    expect(runQaCredentialsAddCommand).toHaveBeenCalledWith({
+      kind: "telegram",
+      payloadFile: "qa/payload.json",
+      repoRoot: "/tmp/openclaw-repo",
+      note: "shared lane",
+      siteUrl: "https://first-schnauzer-821.convex.site",
+      endpointPrefix: "/qa-credentials/v1",
+      actorId: "maintainer-local",
+      json: true,
+    });
+  });
+
+  it("routes credential remove flags into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "credentials",
+      "remove",
+      "--credential-id",
+      "j57b8k419ba7bcsfw99rg05c9184p8br",
+      "--site-url",
+      "https://first-schnauzer-821.convex.site",
+      "--actor-id",
+      "maintainer-local",
+      "--json",
+    ]);
+
+    expect(runQaCredentialsRemoveCommand).toHaveBeenCalledWith({
+      credentialId: "j57b8k419ba7bcsfw99rg05c9184p8br",
+      siteUrl: "https://first-schnauzer-821.convex.site",
+      actorId: "maintainer-local",
+      endpointPrefix: undefined,
+      json: true,
+    });
+  });
+
+  it("routes credential list defaults into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "credentials",
+      "list",
+      "--kind",
+      "telegram",
+    ]);
+
+    expect(runQaCredentialsListCommand).toHaveBeenCalledWith({
+      kind: "telegram",
+      status: "all",
+      limit: undefined,
+      showSecrets: false,
+      siteUrl: undefined,
+      endpointPrefix: undefined,
+      actorId: undefined,
+      json: false,
+    });
+  });
+});

openclaw/extensions/qa-lab/src/cli.ts

@@ -0,0 +1,622 @@
+import type { Command } from "commander";
+import { collectString } from "./cli-options.js";
+import { listLiveTransportQaCliRegistrations } from "./live-transports/cli.js";
+import {
+  DEFAULT_QA_LIVE_PROVIDER_MODE,
+  formatQaProviderModeHelp,
+  listQaStandaloneProviderCommands,
+} from "./providers/index.js";
+import {
+  QA_FRONTIER_PARITY_BASELINE_LABEL,
+  QA_FRONTIER_PARITY_CANDIDATE_LABEL,
+} from "./providers/live-frontier/parity.js";
+import type { QaProviderMode, QaProviderModeInput } from "./run-config.js";
+import { hasQaScenarioPack } from "./scenario-catalog.js";
+
+type QaLabCliRuntime = typeof import("./cli.runtime.js");
+
+let qaLabCliRuntimePromise: Promise<QaLabCliRuntime> | null = null;
+
+async function loadQaLabCliRuntime(): Promise<QaLabCliRuntime> {
+  qaLabCliRuntimePromise ??= import("./cli.runtime.js");
+  return await qaLabCliRuntimePromise;
+}
+
+async function runQaSelfCheck(opts: { repoRoot?: string; output?: string }) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaLabSelfCheckCommand(opts);
+}
+
+async function runQaSuite(opts: {
+  repoRoot?: string;
+  outputDir?: string;
+  transportId?: string;
+  providerMode?: QaProviderModeInput;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  allowFailures?: boolean;
+  cliAuthMode?: string;
+  parityPack?: string;
+  scenarioIds?: string[];
+  concurrency?: number;
+  runner?: string;
+  image?: string;
+  cpus?: number;
+  memory?: string;
+  disk?: string;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaSuiteCommand(opts);
+}
+
+async function runQaParityReport(opts: {
+  repoRoot?: string;
+  candidateSummary: string;
+  baselineSummary: string;
+  candidateLabel?: string;
+  baselineLabel?: string;
+  outputDir?: string;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaParityReportCommand(opts);
+}
+
+async function runQaCoverageReport(opts: { repoRoot?: string; output?: string; json?: boolean }) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaCoverageReportCommand(opts);
+}
+
+async function runQaCharacterEval(opts: {
+  repoRoot?: string;
+  outputDir?: string;
+  model?: string[];
+  scenario?: string;
+  fast?: boolean;
+  thinking?: string;
+  modelThinking?: string[];
+  judgeModel?: string[];
+  judgeTimeoutMs?: number;
+  blindJudgeModels?: boolean;
+  concurrency?: number;
+  judgeConcurrency?: number;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaCharacterEvalCommand(opts);
+}
+
+async function runQaManualLane(opts: {
+  repoRoot?: string;
+  transportId?: string;
+  providerMode?: QaProviderModeInput;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  message: string;
+  timeoutMs?: number;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaManualLaneCommand(opts);
+}
+
+async function runQaCredentialsAdd(opts: {
+  actorId?: string;
+  endpointPrefix?: string;
+  json?: boolean;
+  kind: string;
+  note?: string;
+  payloadFile: string;
+  repoRoot?: string;
+  siteUrl?: string;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaCredentialsAddCommand(opts);
+}
+
+async function runQaCredentialsRemove(opts: {
+  actorId?: string;
+  credentialId: string;
+  endpointPrefix?: string;
+  json?: boolean;
+  siteUrl?: string;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaCredentialsRemoveCommand(opts);
+}
+
+async function runQaCredentialsList(opts: {
+  actorId?: string;
+  endpointPrefix?: string;
+  json?: boolean;
+  kind?: string;
+  limit?: number;
+  showSecrets?: boolean;
+  siteUrl?: string;
+  status?: string;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaCredentialsListCommand(opts);
+}
+
+async function runQaUi(opts: {
+  repoRoot?: string;
+  host?: string;
+  port?: number;
+  advertiseHost?: string;
+  advertisePort?: number;
+  controlUiUrl?: string;
+  controlUiToken?: string;
+  controlUiProxyTarget?: string;
+  uiDistDir?: string;
+  autoKickoffTarget?: string;
+  embeddedGateway?: string;
+  sendKickoffOnStart?: boolean;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaLabUiCommand(opts);
+}
+
+async function runQaDockerScaffold(opts: {
+  repoRoot?: string;
+  outputDir: string;
+  gatewayPort?: number;
+  qaLabPort?: number;
+  providerBaseUrl?: string;
+  image?: string;
+  usePrebuiltImage?: boolean;
+  bindUiDist?: boolean;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaDockerScaffoldCommand(opts);
+}
+
+async function runQaDockerBuildImage(opts: { repoRoot?: string; image?: string }) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaDockerBuildImageCommand(opts);
+}
+
+async function runQaDockerUp(opts: {
+  repoRoot?: string;
+  outputDir?: string;
+  gatewayPort?: number;
+  qaLabPort?: number;
+  providerBaseUrl?: string;
+  image?: string;
+  usePrebuiltImage?: boolean;
+  bindUiDist?: boolean;
+  skipUiBuild?: boolean;
+}) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaDockerUpCommand(opts);
+}
+
+async function runQaProviderServer(
+  providerMode: QaProviderMode,
+  opts: { host?: string; port?: number },
+) {
+  const runtime = await loadQaLabCliRuntime();
+  await runtime.runQaProviderServerCommand(providerMode, opts);
+}
+
+export function isQaLabCliAvailable(): boolean {
+  return hasQaScenarioPack();
+}
+
+function assertNoQaSubcommandCollision(qa: Command, commandName: string) {
+  if (qa.commands.some((command) => command.name() === commandName)) {
+    throw new Error(`QA runner command "${commandName}" conflicts with an existing qa subcommand`);
+  }
+}
+
+export function registerQaLabCli(program: Command) {
+  const qa = program
+    .command("qa")
+    .description("Run private QA automation flows and launch the QA debugger");
+
+  qa.command("run")
+    .description("Run the bundled QA self-check and write a Markdown report")
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option("--output <path>", "Report output path")
+    .action(async (opts: { repoRoot?: string; output?: string }) => {
+      await runQaSelfCheck(opts);
+    });
+
+  qa.command("suite")
+    .description("Run repo-backed QA scenarios against the QA gateway lane")
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option("--output-dir <path>", "Suite artifact directory")
+    .option("--runner <kind>", "Execution runner: host or multipass", "host")
+    .option("--transport <id>", "QA transport id", "qa-channel")
+    .option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
+    .option("--model <ref>", "Primary provider/model ref")
+    .option("--alt-model <ref>", "Alternate provider/model ref")
+    .option(
+      "--cli-auth-mode <mode>",
+      "CLI backend auth mode for live Claude CLI runs: auto, api-key, or subscription",
+    )
+    .option("--parity-pack <name>", 'Preset scenario pack; currently only "agentic" is supported')
+    .option("--scenario <id>", "Run only the named QA scenario (repeatable)", collectString, [])
+    .option("--concurrency <count>", "Scenario worker concurrency", (value: string) =>
+      Number(value),
+    )
+    .option(
+      "--allow-failures",
+      "Write artifacts without setting a failing exit code when scenarios fail",
+      false,
+    )
+    .option("--fast", "Enable provider fast mode where supported", false)
+    .option("--image <alias>", "Multipass image alias")
+    .option("--cpus <count>", "Multipass vCPU count", (value: string) => Number(value))
+    .option("--memory <size>", "Multipass memory size")
+    .option("--disk <size>", "Multipass disk size")
+    .action(
+      async (opts: {
+        repoRoot?: string;
+        outputDir?: string;
+        transport?: string;
+        runner?: string;
+        providerMode?: QaProviderModeInput;
+        model?: string;
+        altModel?: string;
+        cliAuthMode?: string;
+        parityPack?: string;
+        scenario?: string[];
+        concurrency?: number;
+        allowFailures?: boolean;
+        fast?: boolean;
+        image?: string;
+        cpus?: number;
+        memory?: string;
+        disk?: string;
+      }) => {
+        await runQaSuite({
+          repoRoot: opts.repoRoot,
+          outputDir: opts.outputDir,
+          transportId: opts.transport,
+          runner: opts.runner,
+          providerMode: opts.providerMode,
+          primaryModel: opts.model,
+          alternateModel: opts.altModel,
+          fastMode: opts.fast,
+          cliAuthMode: opts.cliAuthMode,
+          parityPack: opts.parityPack,
+          scenarioIds: opts.scenario,
+          concurrency: opts.concurrency,
+          allowFailures: opts.allowFailures,
+          image: opts.image,
+          cpus: opts.cpus,
+          memory: opts.memory,
+          disk: opts.disk,
+        });
+      },
+    );
+
+  qa.command("parity-report")
+    .description("Compare two QA suite summaries and write an agentic parity gate report")
+    .requiredOption("--candidate-summary <path>", "Candidate qa-suite-summary.json path")
+    .requiredOption("--baseline-summary <path>", "Baseline qa-suite-summary.json path")
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option(
+      "--candidate-label <label>",
+      "Candidate display label",
+      QA_FRONTIER_PARITY_CANDIDATE_LABEL,
+    )
+    .option("--baseline-label <label>", "Baseline display label", QA_FRONTIER_PARITY_BASELINE_LABEL)
+    .option("--output-dir <path>", "Artifact directory for the parity report")
+    .action(
+      async (opts: {
+        repoRoot?: string;
+        candidateSummary: string;
+        baselineSummary: string;
+        candidateLabel?: string;
+        baselineLabel?: string;
+        outputDir?: string;
+      }) => {
+        await runQaParityReport(opts);
+      },
+    );
+
+  qa.command("coverage")
+    .description("Print the markdown scenario coverage inventory")
+    .option("--repo-root <path>", "Repository root to target when writing --output")
+    .option("--output <path>", "Write the coverage inventory to this path")
+    .option("--json", "Print JSON instead of Markdown", false)
+    .action(async (opts: { repoRoot?: string; output?: string; json?: boolean }) => {
+      await runQaCoverageReport(opts);
+    });
+
+  qa.command("character-eval")
+    .description("Run the character QA scenario across live models and write a judged report")
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option("--output-dir <path>", "Character eval artifact directory")
+    .option(
+      "--model <ref[,option]>",
+      "Provider/model ref to evaluate; options: thinking=<level>, fast, no-fast, fast=<bool>",
+      collectString,
+      [],
+    )
+    .option("--scenario <id>", "Character scenario id", "character-vibes-gollum")
+    .option("--fast", "Enable provider fast mode for all candidate runs")
+    .option(
+      "--thinking <level>",
+      "Candidate thinking default: off|minimal|low|medium|high|xhigh|adaptive",
+    )
+    .option(
+      "--model-thinking <ref=level>",
+      "Deprecated: candidate thinking override for one model ref (repeatable)",
+      collectString,
+      [],
+    )
+    .option(
+      "--judge-model <ref[,option]>",
+      "Judge provider/model ref; options: thinking=<level>, fast, no-fast, fast=<bool> (repeatable)",
+      collectString,
+      [],
+    )
+    .option("--judge-timeout-ms <ms>", "Override judge wait timeout", (value: string) =>
+      Number(value),
+    )
+    .option(
+      "--blind-judge-models",
+      "Hide candidate model refs from judge prompts; reports still map rankings back to real refs",
+    )
+    .option("--concurrency <count>", "Candidate model run concurrency", (value: string) =>
+      Number(value),
+    )
+    .option("--judge-concurrency <count>", "Judge model run concurrency", (value: string) =>
+      Number(value),
+    )
+    .action(
+      async (opts: {
+        repoRoot?: string;
+        outputDir?: string;
+        model?: string[];
+        scenario?: string;
+        fast?: boolean;
+        thinking?: string;
+        modelThinking?: string[];
+        judgeModel?: string[];
+        judgeTimeoutMs?: number;
+        blindJudgeModels?: boolean;
+        concurrency?: number;
+        judgeConcurrency?: number;
+      }) => {
+        await runQaCharacterEval(opts);
+      },
+    );
+
+  qa.command("manual")
+    .description("Run a one-off QA agent prompt against the selected provider/model lane")
+    .requiredOption("--message <text>", "Prompt to send to the QA agent")
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option("--transport <id>", "QA transport id", "qa-channel")
+    .option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
+    .option("--model <ref>", "Primary provider/model ref (defaults by provider mode)")
+    .option("--alt-model <ref>", "Alternate provider/model ref")
+    .option("--fast", "Enable provider fast mode where supported", false)
+    .option("--timeout-ms <ms>", "Override agent.wait timeout", (value: string) => Number(value))
+    .action(
+      async (opts: {
+        message: string;
+        repoRoot?: string;
+        transport?: string;
+        providerMode?: QaProviderModeInput;
+        model?: string;
+        altModel?: string;
+        fast?: boolean;
+        timeoutMs?: number;
+      }) => {
+        await runQaManualLane({
+          repoRoot: opts.repoRoot,
+          transportId: opts.transport,
+          providerMode: opts.providerMode,
+          primaryModel: opts.model,
+          alternateModel: opts.altModel,
+          fastMode: opts.fast,
+          message: opts.message,
+          timeoutMs: opts.timeoutMs,
+        });
+      },
+    );
+
+  const credentials = qa
+    .command("credentials")
+    .description("Manage pooled Convex live credentials used by QA lanes");
+
+  credentials
+    .command("add")
+    .description("Add one credential payload to the shared pool")
+    .requiredOption("--kind <kind>", "Credential kind (for Telegram v1, use telegram)")
+    .requiredOption("--payload-file <path>", "JSON object file containing the credential payload")
+    .option("--repo-root <path>", "Repository root for resolving relative payload-file paths")
+    .option("--note <text>", "Optional note stored with this credential row")
+    .option("--site-url <url>", "Override OPENCLAW_QA_CONVEX_SITE_URL")
+    .option("--endpoint-prefix <path>", "Override OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX")
+    .option("--actor-id <id>", "Optional admin actor id to include in broker audit events")
+    .option("--json", "Emit machine-readable JSON output", false)
+    .action(
+      async (opts: {
+        kind: string;
+        payloadFile: string;
+        repoRoot?: string;
+        note?: string;
+        siteUrl?: string;
+        endpointPrefix?: string;
+        actorId?: string;
+        json?: boolean;
+      }) => {
+        await runQaCredentialsAdd(opts);
+      },
+    );
+
+  credentials
+    .command("remove")
+    .description("Remove one credential from active use by disabling it")
+    .requiredOption("--credential-id <id>", "Credential row id from the Convex pool")
+    .option("--site-url <url>", "Override OPENCLAW_QA_CONVEX_SITE_URL")
+    .option("--endpoint-prefix <path>", "Override OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX")
+    .option("--actor-id <id>", "Optional admin actor id to include in broker audit events")
+    .option("--json", "Emit machine-readable JSON output", false)
+    .action(
+      async (opts: {
+        credentialId: string;
+        siteUrl?: string;
+        endpointPrefix?: string;
+        actorId?: string;
+        json?: boolean;
+      }) => {
+        await runQaCredentialsRemove(opts);
+      },
+    );
+
+  credentials
+    .command("list")
+    .description("List credential rows in the shared Convex pool")
+    .option("--kind <kind>", "Filter by credential kind")
+    .option("--status <status>", 'Filter by row status: "active", "disabled", or "all"', "all")
+    .option("--limit <count>", "Max rows to return", (value: string) => Number(value))
+    .option("--show-secrets", "Include credential payload JSON in output", false)
+    .option("--site-url <url>", "Override OPENCLAW_QA_CONVEX_SITE_URL")
+    .option("--endpoint-prefix <path>", "Override OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX")
+    .option("--actor-id <id>", "Optional admin actor id to include in broker audit events")
+    .option("--json", "Emit machine-readable JSON output", false)
+    .action(
+      async (opts: {
+        kind?: string;
+        status?: string;
+        limit?: number;
+        showSecrets?: boolean;
+        siteUrl?: string;
+        endpointPrefix?: string;
+        actorId?: string;
+        json?: boolean;
+      }) => {
+        await runQaCredentialsList(opts);
+      },
+    );
+
+  qa.command("ui")
+    .description("Start the private QA debugger UI and local QA bus")
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option("--host <host>", "Bind host", "127.0.0.1")
+    .option("--port <port>", "Bind port", (value: string) => Number(value))
+    .option("--advertise-host <host>", "Optional public host to advertise in bootstrap payloads")
+    .option("--advertise-port <port>", "Optional public port to advertise", (value: string) =>
+      Number(value),
+    )
+    .option("--control-ui-url <url>", "Optional Control UI URL to embed beside the QA panel")
+    .option("--control-ui-token <token>", "Optional Control UI token for embedded links")
+    .option(
+      "--control-ui-proxy-target <url>",
+      "Optional upstream Control UI target for /control-ui proxying",
+    )
+    .option("--ui-dist-dir <path>", "Optional QA Lab UI asset directory override")
+    .option("--auto-kickoff-target <kind>", "Kickoff default target (direct or channel)")
+    .option("--embedded-gateway <mode>", "Embedded gateway mode hint", "enabled")
+    .option(
+      "--send-kickoff-on-start",
+      "Inject the repo-backed kickoff task when the UI starts",
+      false,
+    )
+    .action(
+      async (opts: {
+        repoRoot?: string;
+        host?: string;
+        port?: number;
+        advertiseHost?: string;
+        advertisePort?: number;
+        controlUiUrl?: string;
+        controlUiToken?: string;
+        controlUiProxyTarget?: string;
+        uiDistDir?: string;
+        autoKickoffTarget?: string;
+        embeddedGateway?: string;
+        sendKickoffOnStart?: boolean;
+      }) => {
+        await runQaUi(opts);
+      },
+    );
+
+  qa.command("docker-scaffold")
+    .description("Write a prebaked Docker scaffold for the QA dashboard + gateway lane")
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .requiredOption("--output-dir <path>", "Output directory for docker-compose + state files")
+    .option("--gateway-port <port>", "Gateway host port", (value: string) => Number(value))
+    .option("--qa-lab-port <port>", "QA lab host port", (value: string) => Number(value))
+    .option("--provider-base-url <url>", "Provider base URL for the QA gateway")
+    .option("--image <name>", "Prebaked image name", "openclaw:qa-local-prebaked")
+    .option("--use-prebuilt-image", "Use image: instead of build: in docker-compose", false)
+    .option(
+      "--bind-ui-dist",
+      "Bind-mount extensions/qa-lab/web/dist into the qa-lab container for faster UI refresh",
+      false,
+    )
+    .action(
+      async (opts: {
+        repoRoot?: string;
+        outputDir: string;
+        gatewayPort?: number;
+        qaLabPort?: number;
+        providerBaseUrl?: string;
+        image?: string;
+        usePrebuiltImage?: boolean;
+        bindUiDist?: boolean;
+      }) => {
+        await runQaDockerScaffold(opts);
+      },
+    );
+
+  qa.command("docker-build-image")
+    .description("Build the prebaked QA Docker image with qa-channel + qa-lab bundled")
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option("--image <name>", "Image tag", "openclaw:qa-local-prebaked")
+    .action(async (opts: { repoRoot?: string; image?: string }) => {
+      await runQaDockerBuildImage(opts);
+    });
+
+  qa.command("up")
+    .description("Build the QA site, start the Docker-backed QA stack, and print the QA Lab URL")
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option("--output-dir <path>", "Output directory for docker-compose + state files")
+    .option("--gateway-port <port>", "Gateway host port", (value: string) => Number(value))
+    .option("--qa-lab-port <port>", "QA lab host port", (value: string) => Number(value))
+    .option("--provider-base-url <url>", "Provider base URL for the QA gateway")
+    .option("--image <name>", "Image tag", "openclaw:qa-local-prebaked")
+    .option("--use-prebuilt-image", "Use image: instead of build: in docker-compose", false)
+    .option(
+      "--bind-ui-dist",
+      "Bind-mount extensions/qa-lab/web/dist into the qa-lab container for faster UI refresh",
+      false,
+    )
+    .option("--skip-ui-build", "Skip pnpm qa:lab:build before starting Docker", false)
+    .action(
+      async (opts: {
+        repoRoot?: string;
+        outputDir?: string;
+        gatewayPort?: number;
+        qaLabPort?: number;
+        providerBaseUrl?: string;
+        image?: string;
+        usePrebuiltImage?: boolean;
+        bindUiDist?: boolean;
+        skipUiBuild?: boolean;
+      }) => {
+        await runQaDockerUp(opts);
+      },
+    );
+
+  for (const providerCommand of listQaStandaloneProviderCommands()) {
+    qa.command(providerCommand.name)
+      .description(providerCommand.description)
+      .option("--host <host>", "Bind host", "127.0.0.1")
+      .option("--port <port>", "Bind port", (value: string) => Number(value))
+      .action(async (opts: { host?: string; port?: number }) => {
+        await runQaProviderServer(providerCommand.providerMode, opts);
+      });
+  }
+
+  for (const lane of listLiveTransportQaCliRegistrations()) {
+    assertNoQaSubcommandCollision(qa, lane.commandName);
+    lane.register(qa);
+  }
+}

openclaw/extensions/qa-lab/src/coverage-report.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it } from "vitest";
+import { buildQaCoverageInventory, renderQaCoverageMarkdownReport } from "./coverage-report.js";
+import { readQaScenarioPack } from "./scenario-catalog.js";
+
+describe("qa coverage report", () => {
+  it("groups scenario coverage metadata by theme and surface", () => {
+    const inventory = buildQaCoverageInventory(readQaScenarioPack().scenarios);
+
+    expect(inventory.scenarioCount).toBeGreaterThan(0);
+    expect(inventory.coverageIdCount).toBeGreaterThan(0);
+    expect(inventory.primaryCoverageIdCount).toBeGreaterThan(0);
+    expect(inventory.secondaryCoverageIdCount).toBeGreaterThan(0);
+    expect(inventory.overlappingCoverage.length).toBeGreaterThan(0);
+    expect(inventory.missingCoverage).toEqual([]);
+    expect(inventory.byTheme.memory.some((feature) => feature.id === "memory.recall")).toBe(true);
+    expect(inventory.bySurface.memory.some((feature) => feature.id === "memory.recall")).toBe(true);
+  });
+
+  it("renders a compact markdown inventory", () => {
+    const report = renderQaCoverageMarkdownReport(
+      buildQaCoverageInventory(readQaScenarioPack().scenarios),
+    );
+
+    expect(report).toContain("# QA Coverage Inventory");
+    expect(report).toContain("- Missing coverage metadata: 0");
+    expect(report).toContain("- Overlapping coverage IDs:");
+    expect(report).toContain("memory.recall");
+    expect(report).toContain("primary: memory-recall (qa/scenarios/memory/memory-recall.md)");
+    expect(report).toContain("secondary: active-memory-preprompt-recall");
+  });
+});

openclaw/extensions/qa-lab/src/coverage-report.ts

@@ -0,0 +1,192 @@
+import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
+
+export type QaCoverageScenarioSummary = {
+  id: string;
+  title: string;
+  sourcePath: string;
+  theme: string;
+  surfaces: string[];
+  risk: string;
+};
+
+export type QaCoverageIntent = "primary" | "secondary";
+
+export type QaCoverageScenarioReference = QaCoverageScenarioSummary & {
+  intent: QaCoverageIntent;
+};
+
+export type QaCoverageFeatureSummary = {
+  id: string;
+  scenarios: QaCoverageScenarioReference[];
+};
+
+export type QaCoverageInventory = {
+  scenarioCount: number;
+  coverageIdCount: number;
+  primaryCoverageIdCount: number;
+  secondaryCoverageIdCount: number;
+  features: QaCoverageFeatureSummary[];
+  overlappingCoverage: QaCoverageFeatureSummary[];
+  missingCoverage: QaCoverageScenarioSummary[];
+  byTheme: Record<string, QaCoverageFeatureSummary[]>;
+  bySurface: Record<string, QaCoverageFeatureSummary[]>;
+};
+
+function scenarioTheme(sourcePath: string) {
+  const parts = sourcePath.split("/");
+  return parts[2] ?? "unknown";
+}
+
+function scenarioSurfaces(scenario: QaSeedScenarioWithSource) {
+  return scenario.surfaces && scenario.surfaces.length > 0 ? scenario.surfaces : [scenario.surface];
+}
+
+function scenarioRisk(scenario: QaSeedScenarioWithSource) {
+  return scenario.risk ?? scenario.riskLevel ?? "unassigned";
+}
+
+function summarizeScenario(scenario: QaSeedScenarioWithSource): QaCoverageScenarioSummary {
+  return {
+    id: scenario.id,
+    title: scenario.title,
+    sourcePath: scenario.sourcePath,
+    theme: scenarioTheme(scenario.sourcePath),
+    surfaces: scenarioSurfaces(scenario),
+    risk: scenarioRisk(scenario),
+  };
+}
+
+function sortFeatures(features: readonly QaCoverageFeatureSummary[]) {
+  return features.toSorted((left, right) => left.id.localeCompare(right.id));
+}
+
+export function buildQaCoverageInventory(
+  scenarios: readonly QaSeedScenarioWithSource[],
+): QaCoverageInventory {
+  const byCoverageId = new Map<string, QaCoverageFeatureSummary>();
+  const primaryCoverageIds = new Set<string>();
+  const secondaryCoverageIds = new Set<string>();
+  const missingCoverage: QaCoverageScenarioSummary[] = [];
+
+  const addCoverage = (
+    scenario: QaSeedScenarioWithSource,
+    coverageIds: readonly string[] | undefined,
+    intent: QaCoverageIntent,
+  ) => {
+    const summary = summarizeScenario(scenario);
+    for (const coverageId of coverageIds ?? []) {
+      const feature = byCoverageId.get(coverageId) ?? {
+        id: coverageId,
+        scenarios: [],
+      };
+      feature.scenarios.push({ ...summary, intent });
+      byCoverageId.set(coverageId, feature);
+      if (intent === "primary") {
+        primaryCoverageIds.add(coverageId);
+      } else {
+        secondaryCoverageIds.add(coverageId);
+      }
+    }
+  };
+
+  for (const scenario of scenarios) {
+    if (!scenario.coverage) {
+      missingCoverage.push(summarizeScenario(scenario));
+      continue;
+    }
+    addCoverage(scenario, scenario.coverage.primary, "primary");
+    addCoverage(scenario, scenario.coverage.secondary, "secondary");
+  }
+
+  const features = sortFeatures([...byCoverageId.values()]);
+  const overlappingCoverage = features.filter((feature) => feature.scenarios.length > 1);
+  const byTheme: Record<string, QaCoverageFeatureSummary[]> = {};
+  const bySurface: Record<string, QaCoverageFeatureSummary[]> = {};
+
+  for (const feature of features) {
+    const themes = new Set(feature.scenarios.map((scenario) => scenario.theme));
+    for (const theme of themes) {
+      byTheme[theme] ??= [];
+      byTheme[theme].push({
+        ...feature,
+        scenarios: feature.scenarios.filter((scenario) => scenario.theme === theme),
+      });
+    }
+    const surfaces = new Set(feature.scenarios.flatMap((scenario) => scenario.surfaces));
+    for (const surface of surfaces) {
+      bySurface[surface] ??= [];
+      bySurface[surface].push({
+        ...feature,
+        scenarios: feature.scenarios.filter((scenario) => scenario.surfaces.includes(surface)),
+      });
+    }
+  }
+
+  return {
+    scenarioCount: scenarios.length,
+    coverageIdCount: features.length,
+    primaryCoverageIdCount: primaryCoverageIds.size,
+    secondaryCoverageIdCount: secondaryCoverageIds.size,
+    features,
+    overlappingCoverage,
+    missingCoverage,
+    byTheme,
+    bySurface,
+  };
+}
+
+function pushFeatureLines(lines: string[], features: readonly QaCoverageFeatureSummary[]) {
+  for (const feature of sortFeatures(features)) {
+    const scenarios = feature.scenarios
+      .map((scenario) => `${scenario.intent}: ${scenario.id} (${scenario.sourcePath})`)
+      .join(", ");
+    lines.push(`- ${feature.id}: ${scenarios}`);
+  }
+}
+
+export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory): string {
+  const lines: string[] = [
+    "# QA Coverage Inventory",
+    "",
+    `- Scenarios: ${inventory.scenarioCount}`,
+    `- Coverage IDs: ${inventory.coverageIdCount}`,
+    `- Primary coverage IDs: ${inventory.primaryCoverageIdCount}`,
+    `- Secondary coverage IDs: ${inventory.secondaryCoverageIdCount}`,
+    `- Overlapping coverage IDs: ${inventory.overlappingCoverage.length}`,
+    `- Missing coverage metadata: ${inventory.missingCoverage.length}`,
+    "",
+    "## By Theme",
+    "",
+  ];
+
+  for (const theme of Object.keys(inventory.byTheme).toSorted()) {
+    lines.push(`### ${theme}`, "");
+    pushFeatureLines(lines, inventory.byTheme[theme] ?? []);
+    lines.push("");
+  }
+
+  lines.push("## By Surface", "");
+  for (const surface of Object.keys(inventory.bySurface).toSorted()) {
+    lines.push(`### ${surface}`, "");
+    pushFeatureLines(lines, inventory.bySurface[surface] ?? []);
+    lines.push("");
+  }
+
+  if (inventory.overlappingCoverage.length > 0) {
+    lines.push("## Overlap", "");
+    pushFeatureLines(lines, inventory.overlappingCoverage);
+    lines.push("");
+  }
+
+  if (inventory.missingCoverage.length > 0) {
+    lines.push("## Missing Metadata", "");
+    for (const scenario of inventory.missingCoverage.toSorted((left, right) =>
+      left.id.localeCompare(right.id),
+    )) {
+      lines.push(`- ${scenario.id}: ${scenario.sourcePath}`);
+    }
+    lines.push("");
+  }
+
+  return `${lines.join("\n").trimEnd()}\n`;
+}

openclaw/extensions/qa-lab/src/cron-run-wait.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, it, vi } from "vitest";
+import { waitForCronRunCompletion } from "./cron-run-wait.js";
+
+describe("waitForCronRunCompletion", () => {
+  it("ignores older entries and returns the newly finished run", async () => {
+    const callGateway = vi
+      .fn<
+        (method: string, rpcParams?: unknown, opts?: { timeoutMs?: number }) => Promise<unknown>
+      >()
+      .mockResolvedValueOnce({
+        entries: [{ ts: 100, status: "ok", summary: "older run" }],
+      })
+      .mockResolvedValueOnce({
+        entries: [{ ts: 180, status: "ok", summary: "new run" }],
+      });
+
+    const result = await waitForCronRunCompletion({
+      callGateway,
+      jobId: "dreaming-job",
+      afterTs: 150,
+      timeoutMs: 100,
+      intervalMs: 0,
+    });
+
+    expect(result).toMatchObject({ ts: 180, status: "ok", summary: "new run" });
+    expect(callGateway).toHaveBeenNthCalledWith(
+      1,
+      "cron.runs",
+      { id: "dreaming-job", limit: 20, sortDir: "desc" },
+      { timeoutMs: 100 },
+    );
+  });
+
+  it("surfaces recent run history on timeout", async () => {
+    const callGateway = vi
+      .fn<
+        (method: string, rpcParams?: unknown, opts?: { timeoutMs?: number }) => Promise<unknown>
+      >()
+      .mockResolvedValue({
+        entries: [{ ts: 100, status: "ok", summary: "older run" }],
+      });
+
+    await expect(
+      waitForCronRunCompletion({
+        callGateway,
+        jobId: "dreaming-job",
+        afterTs: 150,
+        timeoutMs: 5,
+        intervalMs: 0,
+      }),
+    ).rejects.toThrow(/timed out waiting for cron run completion/);
+  });
+});

openclaw/extensions/qa-lab/src/cron-run-wait.ts

@@ -0,0 +1,57 @@
+import { setTimeout as sleep } from "node:timers/promises";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+
+export type QaCronRunLogEntry = {
+  ts?: number;
+  status?: "ok" | "error" | "skipped";
+  summary?: string;
+  error?: string;
+  deliveryStatus?: "delivered" | "not-delivered" | "unknown" | "not-requested";
+};
+
+type QaCronRunsPage = {
+  entries?: QaCronRunLogEntry[];
+};
+
+export async function waitForCronRunCompletion(params: {
+  callGateway: (
+    method: string,
+    rpcParams?: unknown,
+    opts?: { timeoutMs?: number },
+  ) => Promise<unknown>;
+  jobId: string;
+  afterTs: number;
+  timeoutMs?: number;
+  intervalMs?: number;
+}) {
+  const timeoutMs = params.timeoutMs ?? 90_000;
+  const intervalMs = params.intervalMs ?? 1_000;
+  const startedAt = Date.now();
+  let lastEntries: QaCronRunLogEntry[] = [];
+  while (Date.now() - startedAt < timeoutMs) {
+    const page = (await params.callGateway(
+      "cron.runs",
+      {
+        id: params.jobId,
+        limit: 20,
+        sortDir: "desc",
+      },
+      { timeoutMs: Math.min(timeoutMs, 30_000) },
+    )) as QaCronRunsPage;
+    const entries = Array.isArray(page.entries) ? page.entries : [];
+    lastEntries = entries;
+    const completed = entries.find(
+      (entry) =>
+        typeof entry.ts === "number" &&
+        entry.ts >= params.afterTs &&
+        (entry.status === "ok" || entry.status === "error" || entry.status === "skipped"),
+    );
+    if (completed) {
+      return completed;
+    }
+    await sleep(intervalMs);
+  }
+  throw new Error(
+    `timed out waiting for cron run completion for ${params.jobId}: ${formatErrorMessage(lastEntries)}`,
+  );
+}

openclaw/extensions/qa-lab/src/discovery-eval.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, it } from "vitest";
+import {
+  hasDiscoveryLabels,
+  reportsDiscoveryScopeLeak,
+  reportsMissingDiscoveryFiles,
+} from "./discovery-eval.js";
+
+describe("qa discovery evaluation", () => {
+  it("accepts rich discovery reports that explicitly confirm all required files were read", () => {
+    const report = `
+Worked
+- Read all three requested files: repo/qa/scenarios/index.md, repo/extensions/qa-lab/src/suite.ts, and repo/docs/help/testing.md.
+Failed
+- None.
+Blocked
+- Runtime execution not attempted here.
+Follow-up
+- Run the live suite next.
+
+The helper text mentions banned phrases like "not present", "missing files", "blocked by missing", and "could not inspect", but only as quoted examples.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(false);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(false);
+  });
+
+  it("accepts numeric 'all 4 required files read' confirmations", () => {
+    const report = `
+Worked
+- Source: repo/qa/scenarios/index.md, repo/extensions/qa-lab/src/suite.ts, repo/docs/help/testing.md
+- all 3 required files read.
+Failed
+- None.
+Blocked
+- No runtime execution in this pass.
+Follow-up
+- Run the live suite next.
+
+The report may quote phrases like "not present" while describing the evaluator, but the files were read.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(false);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(false);
+  });
+
+  it("accepts claude-style 'all four files retrieved' discovery summaries", () => {
+    const report = `
+Worked
+- All three files retrieved. Now let me compile the protocol report.
+- All three mandated files read successfully: repo/qa/scenarios/index.md, repo/extensions/qa-lab/src/suite.ts, repo/docs/help/testing.md.
+Failed
+- None.
+Blocked
+- Runtime execution not attempted here.
+Follow-up
+- Run the live suite next.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(false);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(false);
+  });
+
+  it("still flags genuine file-miss language when the report never confirms the required reads", () => {
+    const report = `
+Worked
+- Read some of the requested files.
+Failed
+- repo/docs/help/testing.md was not present.
+Blocked
+- Could not inspect the remaining refs.
+Follow-up
+- Fix the workspace mount.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(true);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(false);
+  });
+
+  it("flags discovery replies that drift into unrelated suite wrap-up claims", () => {
+    const report = `
+Worked
+- All three requested files were read: repo/qa/scenarios/index.md, repo/extensions/qa-lab/src/suite.ts, repo/docs/help/testing.md.
+Failed
+- None.
+Blocked
+- Runtime execution not attempted here.
+Follow-up
+- Run the live suite next.
+
+Final QA tally update: all mandatory scenarios resolved. QA run complete.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(false);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(true);
+  });
+});

openclaw/extensions/qa-lab/src/discovery-eval.ts

@@ -0,0 +1,72 @@
+import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
+import { readQaScenarioExecutionConfig } from "./scenario-catalog.js";
+
+function readRequiredDiscoveryRefs() {
+  const config = readQaScenarioExecutionConfig("source-docs-discovery-report") as
+    | { requiredFiles?: string[] }
+    | undefined;
+  return (
+    config?.requiredFiles ?? [
+      "repo/qa/scenarios/index.md",
+      "repo/extensions/qa-lab/src/suite.ts",
+      "repo/docs/help/testing.md",
+    ]
+  );
+}
+
+const REQUIRED_DISCOVERY_REFS = readRequiredDiscoveryRefs();
+
+const REQUIRED_DISCOVERY_REFS_LOWER = REQUIRED_DISCOVERY_REFS.map(normalizeLowercaseStringOrEmpty);
+
+const DISCOVERY_SCOPE_LEAK_PHRASES = [
+  "all mandatory scenarios",
+  "final qa tally",
+  "final qa tally update",
+  "qa run complete",
+  "scenario: `subagent-handoff`",
+  "scenario: subagent-handoff",
+] as const;
+
+function confirmsDiscoveryFileRead(text: string) {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  const mentionsAllRefs = REQUIRED_DISCOVERY_REFS_LOWER.every((ref) => lower.includes(ref));
+  const mentionsReadVerb = /(?:read|retrieved|inspected|loaded|accessed|digested)/.test(lower);
+  const requiredCountPattern = "(?:three|3|four|4)";
+  const confirmsRead =
+    new RegExp(
+      `(?:read|retrieved|inspected|loaded|accessed|digested)\\s+all\\s+${requiredCountPattern}\\s+(?:(?:requested|required|mandated|seeded)\\s+)?files`,
+    ).test(lower) ||
+    new RegExp(
+      `all\\s+${requiredCountPattern}\\s+(?:(?:requested|required|mandated|seeded)\\s+)?files\\s+(?:were\\s+)?(?:read|retrieved|inspected|loaded|accessed|digested)(?:\\s+\\w+)?`,
+    ).test(lower) ||
+    new RegExp(`all\\s+${requiredCountPattern}\\s+seeded files readable`).test(lower);
+  return mentionsAllRefs && (confirmsRead || mentionsReadVerb);
+}
+
+export function hasDiscoveryLabels(text: string) {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  return (
+    lower.includes("worked") &&
+    lower.includes("failed") &&
+    lower.includes("blocked") &&
+    (lower.includes("follow-up") || lower.includes("follow up"))
+  );
+}
+
+export function reportsMissingDiscoveryFiles(text: string) {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  if (confirmsDiscoveryFileRead(text)) {
+    return false;
+  }
+  return (
+    lower.includes("not present") ||
+    lower.includes("missing files") ||
+    lower.includes("blocked by missing") ||
+    lower.includes("could not inspect")
+  );
+}
+
+export function reportsDiscoveryScopeLeak(text: string) {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  return DISCOVERY_SCOPE_LEAK_PHRASES.some((phrase) => lower.includes(phrase));
+}

openclaw/extensions/qa-lab/src/docker-harness.test.ts

@@ -0,0 +1,124 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { buildQaDockerHarnessImage, writeQaDockerHarnessFiles } from "./docker-harness.js";
+
+const cleanups: Array<() => Promise<void>> = [];
+
+afterEach(async () => {
+  while (cleanups.length > 0) {
+    await cleanups.pop()?.();
+  }
+});
+
+describe("qa docker harness", () => {
+  it("writes compose, env, config, and workspace scaffold files", async () => {
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-test-"));
+    cleanups.push(async () => {
+      await rm(outputDir, { recursive: true, force: true });
+    });
+
+    const result = await writeQaDockerHarnessFiles({
+      outputDir,
+      gatewayPort: 18889,
+      qaLabPort: 43124,
+      gatewayToken: "qa-token",
+      providerBaseUrl: "http://host.docker.internal:45123/v1",
+      repoRoot: "/repo/openclaw",
+      usePrebuiltImage: true,
+      bindUiDist: true,
+    });
+
+    expect(result.files).toEqual(
+      expect.arrayContaining([
+        path.join(outputDir, ".env.example"),
+        path.join(outputDir, "README.md"),
+        path.join(outputDir, "docker-compose.qa.yml"),
+        path.join(outputDir, "state", "openclaw.json"),
+        path.join(outputDir, "state", "seed-workspace", "QA_KICKOFF_TASK.md"),
+        path.join(outputDir, "state", "seed-workspace", "QA_SCENARIO_PLAN.md"),
+        path.join(outputDir, "state", "seed-workspace", "QA_SCENARIOS.md"),
+        path.join(outputDir, "state", "seed-workspace", "IDENTITY.md"),
+      ]),
+    );
+
+    const compose = await readFile(path.join(outputDir, "docker-compose.qa.yml"), "utf8");
+    expect(compose).toContain("image: openclaw:qa-local-prebaked");
+    expect(compose).toContain("qa-mock-openai:");
+    expect(compose).toContain("18889:18789");
+    expect(compose).toContain('      - "43124:43123"');
+    expect(compose).toContain(":/opt/openclaw-qa-lab-ui:ro");
+    expect(compose).toContain("      - sh");
+    expect(compose).toContain("      - -lc");
+    expect(compose).toContain(
+      '        - fetch("http://127.0.0.1:18789/healthz").then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))',
+    );
+    expect(compose).toContain("      - --control-ui-proxy-target");
+    expect(compose).toContain('      - "http://openclaw-qa-gateway:18789/"');
+    expect(compose).toContain("      - --send-kickoff-on-start");
+    expect(compose).toContain("      - --ui-dist-dir");
+    expect(compose).toContain('      - "/opt/openclaw-qa-lab-ui"');
+    expect(compose).toContain(":/opt/openclaw-repo:ro");
+    expect(compose).toContain("./state:/opt/openclaw-scaffold:ro");
+    expect(compose).toContain(
+      "cp -R /opt/openclaw-scaffold/seed-workspace/. /tmp/openclaw/workspace/",
+    );
+    expect(compose).toContain("OPENCLAW_CONFIG_PATH: /tmp/openclaw/openclaw.json");
+    expect(compose).toContain("OPENCLAW_STATE_DIR: /tmp/openclaw/state");
+    expect(compose).toContain('OPENCLAW_NO_RESPAWN: "1"');
+
+    const envExample = await readFile(path.join(outputDir, ".env.example"), "utf8");
+    expect(envExample).toContain("OPENCLAW_GATEWAY_TOKEN=qa-token");
+    expect(envExample).toContain("QA_BUS_BASE_URL=http://qa-lab:43123");
+    expect(envExample).toContain("QA_PROVIDER_BASE_URL=http://host.docker.internal:45123/v1");
+    expect(envExample).toContain("QA_LAB_URL=http://127.0.0.1:43124");
+
+    const config = await readFile(path.join(outputDir, "state", "openclaw.json"), "utf8");
+    expect(config).toContain('"allowInsecureAuth": true');
+    expect(config).toContain('"enabled": false');
+    expect(config).toContain("/app/dist/control-ui");
+    expect(config).toContain("C-3PO QA");
+    expect(config).toContain('"/tmp/openclaw/workspace"');
+
+    const kickoff = await readFile(
+      path.join(outputDir, "state", "seed-workspace", "QA_KICKOFF_TASK.md"),
+      "utf8",
+    );
+    expect(kickoff).toContain("Lobster Invaders");
+
+    const scenarios = await readFile(
+      path.join(outputDir, "state", "seed-workspace", "QA_SCENARIOS.md"),
+      "utf8",
+    );
+    expect(scenarios).toContain("```yaml qa-pack");
+    expect(scenarios).toContain("subagent-fanout-synthesis");
+
+    const readme = await readFile(path.join(outputDir, "README.md"), "utf8");
+    expect(readme).toContain("in-process restarts inside Docker");
+    expect(readme).toContain("pnpm qa:lab:watch");
+  });
+
+  it("builds the reusable QA image with bundled QA extensions", async () => {
+    const calls: string[] = [];
+    const result = await buildQaDockerHarnessImage(
+      {
+        repoRoot: "/repo/openclaw",
+        imageName: "openclaw:qa-local-prebaked",
+      },
+      {
+        async runCommand(command, args, cwd) {
+          calls.push([command, ...args, `@${cwd}`].join(" "));
+          return { stdout: "", stderr: "" };
+        },
+      },
+    );
+
+    expect(result.imageName).toBe("openclaw:qa-local-prebaked");
+    expect(calls).toEqual([
+      expect.stringContaining(
+        "docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS=qa-channel qa-lab -f Dockerfile . @/repo/openclaw",
+      ),
+    ]);
+  });
+});

openclaw/extensions/qa-lab/src/docker-harness.ts

@@ -0,0 +1,383 @@
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { seedQaAgentWorkspace } from "./qa-agent-workspace.js";
+import {
+  createQaChannelGatewayConfig,
+  QA_CHANNEL_REQUIRED_PLUGIN_IDS,
+} from "./qa-channel-transport.js";
+import { buildQaGatewayConfig } from "./qa-gateway-config.js";
+
+const QA_LAB_INTERNAL_PORT = 43123;
+const QA_LAB_UI_OVERLAY_DIR = "/opt/openclaw-qa-lab-ui";
+
+function toPosixRelative(fromDir: string, toPath: string): string {
+  return path.relative(fromDir, toPath).split(path.sep).join("/");
+}
+
+function renderImageBlock(params: {
+  outputDir: string;
+  repoRoot: string;
+  imageName: string;
+  usePrebuiltImage: boolean;
+}) {
+  if (params.usePrebuiltImage) {
+    return `    image: ${params.imageName}\n`;
+  }
+  const context = toPosixRelative(params.outputDir, params.repoRoot) || ".";
+  return `    build:\n      context: ${context}\n      dockerfile: Dockerfile\n      args:\n        OPENCLAW_EXTENSIONS: "qa-channel qa-lab"\n`;
+}
+
+function renderCompose(params: {
+  outputDir: string;
+  repoRoot: string;
+  imageName: string;
+  usePrebuiltImage: boolean;
+  bindUiDist: boolean;
+  gatewayPort: number;
+  qaLabPort: number;
+  gatewayToken: string;
+  includeQaLabUi: boolean;
+}) {
+  const imageBlock = renderImageBlock(params);
+  const repoMount = toPosixRelative(params.outputDir, params.repoRoot) || ".";
+  const qaLabUiMount = toPosixRelative(
+    params.outputDir,
+    path.join(params.repoRoot, "extensions", "qa-lab", "web", "dist"),
+  );
+
+  return `services:
+  qa-mock-openai:
+${imageBlock}    pull_policy: never
+    healthcheck:
+      test:
+        - CMD
+        - node
+        - -e
+        - fetch("http://127.0.0.1:44080/healthz").then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 3s
+    command:
+      - node
+      - dist/index.js
+      - qa
+      - mock-openai
+      - --host
+      - "0.0.0.0"
+      - --port
+      - "44080"
+${
+  params.includeQaLabUi
+    ? `  qa-lab:
+${imageBlock}    pull_policy: never
+    ports:
+      - "${params.qaLabPort}:${QA_LAB_INTERNAL_PORT}"
+${params.bindUiDist ? `    volumes:\n      - ${qaLabUiMount}:${QA_LAB_UI_OVERLAY_DIR}:ro\n` : ""}    healthcheck:
+      test:
+        - CMD
+        - node
+        - -e
+        - fetch("http://127.0.0.1:${QA_LAB_INTERNAL_PORT}/healthz").then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 5s
+    environment:
+      OPENCLAW_SKIP_GMAIL_WATCHER: "1"
+      OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1"
+      OPENCLAW_SKIP_CANVAS_HOST: "1"
+      OPENCLAW_PROFILE: ""
+    command:
+      - node
+      - dist/index.js
+      - qa
+      - ui
+      - --host
+      - "0.0.0.0"
+      - --port
+      - "${QA_LAB_INTERNAL_PORT}"
+      - --advertise-host
+      - "127.0.0.1"
+      - --advertise-port
+      - "${params.qaLabPort}"
+      - --control-ui-url
+      - "http://127.0.0.1:${params.gatewayPort}/"
+      - --control-ui-proxy-target
+      - "http://openclaw-qa-gateway:18789/"
+      - --control-ui-token
+      - "${params.gatewayToken}"
+${params.bindUiDist ? `      - --ui-dist-dir\n      - "${QA_LAB_UI_OVERLAY_DIR}"\n` : ""}      - --auto-kickoff-target
+      - direct
+      - --send-kickoff-on-start
+      - --embedded-gateway
+      - disabled
+    depends_on:
+      qa-mock-openai:
+        condition: service_healthy
+`
+    : ""
+}  openclaw-qa-gateway:
+${imageBlock}    pull_policy: never
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    ports:
+      - "${params.gatewayPort}:18789"
+    environment:
+      OPENCLAW_CONFIG_PATH: /tmp/openclaw/openclaw.json
+      OPENCLAW_STATE_DIR: /tmp/openclaw/state
+      OPENCLAW_NO_RESPAWN: "1"
+      OPENCLAW_SKIP_GMAIL_WATCHER: "1"
+      OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1"
+      OPENCLAW_SKIP_CANVAS_HOST: "1"
+      OPENCLAW_PROFILE: ""
+    volumes:
+      - ./state:/opt/openclaw-scaffold:ro
+      - ${repoMount}:/opt/openclaw-repo:ro
+    healthcheck:
+      test:
+        - CMD
+        - node
+        - -e
+        - fetch("http://127.0.0.1:18789/healthz").then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))
+      interval: 10s
+      timeout: 5s
+      retries: 12
+      start_period: 15s
+    depends_on:
+${
+  params.includeQaLabUi
+    ? `      qa-lab:
+        condition: service_healthy
+`
+    : ""
+}      qa-mock-openai:
+        condition: service_healthy
+    command:
+      - sh
+      - -lc
+      - mkdir -p /tmp/openclaw/workspace /tmp/openclaw/state && cp /opt/openclaw-scaffold/openclaw.json /tmp/openclaw/openclaw.json && cp -R /opt/openclaw-scaffold/seed-workspace/. /tmp/openclaw/workspace/ && ln -snf /opt/openclaw-repo /tmp/openclaw/workspace/repo && exec node dist/index.js gateway run --port 18789 --bind lan --allow-unconfigured
+`;
+}
+
+function renderEnvExample(params: {
+  gatewayPort: number;
+  qaLabPort: number;
+  gatewayToken: string;
+  providerBaseUrl: string;
+  qaBusBaseUrl: string;
+  includeQaLabUi: boolean;
+}) {
+  return `# QA Docker harness example env
+OPENCLAW_GATEWAY_TOKEN=${params.gatewayToken}
+QA_GATEWAY_PORT=${params.gatewayPort}
+QA_BUS_BASE_URL=${params.qaBusBaseUrl}
+QA_PROVIDER_BASE_URL=${params.providerBaseUrl}
+${params.includeQaLabUi ? `QA_LAB_URL=http://127.0.0.1:${params.qaLabPort}\n` : ""}`;
+}
+
+function renderReadme(params: {
+  gatewayPort: number;
+  qaLabPort: number;
+  usePrebuiltImage: boolean;
+  bindUiDist: boolean;
+  includeQaLabUi: boolean;
+}) {
+  return `# QA Docker Harness
+
+Generated scaffold for the Docker-backed QA lane.
+
+Files:
+
+- \`docker-compose.qa.yml\`
+- \`.env.example\`
+- \`state/openclaw.json\`
+
+Suggested flow:
+
+1. Build the prebaked image once:
+   - \`docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS="qa-channel qa-lab" -f Dockerfile .\`
+2. Start the stack:
+   - \`docker compose -f docker-compose.qa.yml up${params.usePrebuiltImage ? "" : " --build"} -d\`
+3. Open the QA dashboard:
+   - \`${params.includeQaLabUi ? `http://127.0.0.1:${params.qaLabPort}` : "not published in this scaffold"}\`
+4. The single QA site embeds both panes:
+   - left: Control UI
+   - right: Slack-ish QA lab
+5. The repo-backed kickoff task auto-injects on startup.
+
+Fast UI refresh:
+
+- Start once with a prebuilt image + bind-mounted QA Lab assets:
+  - \`pnpm qa:lab:up --use-prebuilt-image --bind-ui-dist --skip-ui-build\`
+- In another shell, rebuild the QA Lab bundle on change:
+  - \`pnpm qa:lab:watch\`
+- The browser auto-reloads when the QA Lab asset hash changes.
+
+Gateway:
+
+- health: \`http://127.0.0.1:${params.gatewayPort}/healthz\`
+- Control UI: \`http://127.0.0.1:${params.gatewayPort}/\`
+- Mock OpenAI: internal \`http://qa-mock-openai:44080/v1\`
+
+This scaffold uses localhost Control UI insecure-auth compatibility for QA only.
+The gateway runs with in-process restarts inside Docker so restart actions do not
+kill the container by detaching a replacement child.
+`;
+}
+
+export async function writeQaDockerHarnessFiles(params: {
+  outputDir: string;
+  repoRoot: string;
+  gatewayPort?: number;
+  qaLabPort?: number;
+  gatewayToken?: string;
+  providerBaseUrl?: string;
+  qaBusBaseUrl?: string;
+  imageName?: string;
+  usePrebuiltImage?: boolean;
+  bindUiDist?: boolean;
+  includeQaLabUi?: boolean;
+}) {
+  const gatewayPort = params.gatewayPort ?? 18789;
+  const qaLabPort = params.qaLabPort ?? 43124;
+  const gatewayToken = params.gatewayToken ?? `qa-token-${randomUUID()}`;
+  const providerBaseUrl = params.providerBaseUrl ?? "http://qa-mock-openai:44080/v1";
+  const qaBusBaseUrl = params.qaBusBaseUrl ?? "http://qa-lab:43123";
+  const imageName = params.imageName ?? "openclaw:qa-local-prebaked";
+  const usePrebuiltImage = params.usePrebuiltImage ?? false;
+  const bindUiDist = params.bindUiDist ?? false;
+  const includeQaLabUi = params.includeQaLabUi ?? true;
+
+  await fs.mkdir(path.join(params.outputDir, "state", "seed-workspace"), { recursive: true });
+  await seedQaAgentWorkspace({
+    workspaceDir: path.join(params.outputDir, "state", "seed-workspace"),
+    repoRoot: params.repoRoot,
+  });
+
+  const config = buildQaGatewayConfig({
+    bind: "lan",
+    gatewayPort: 18789,
+    gatewayToken,
+    providerBaseUrl,
+    workspaceDir: "/tmp/openclaw/workspace",
+    controlUiRoot: "/app/dist/control-ui",
+    transportPluginIds: QA_CHANNEL_REQUIRED_PLUGIN_IDS,
+    transportConfig: createQaChannelGatewayConfig({
+      baseUrl: qaBusBaseUrl,
+    }),
+  });
+
+  const files = [
+    path.join(params.outputDir, "docker-compose.qa.yml"),
+    path.join(params.outputDir, ".env.example"),
+    path.join(params.outputDir, "README.md"),
+    path.join(params.outputDir, "state", "openclaw.json"),
+  ];
+
+  await Promise.all([
+    fs.writeFile(
+      path.join(params.outputDir, "docker-compose.qa.yml"),
+      renderCompose({
+        outputDir: params.outputDir,
+        repoRoot: params.repoRoot,
+        imageName,
+        usePrebuiltImage,
+        bindUiDist,
+        gatewayPort,
+        qaLabPort,
+        gatewayToken,
+        includeQaLabUi,
+      }),
+      "utf8",
+    ),
+    fs.writeFile(
+      path.join(params.outputDir, ".env.example"),
+      renderEnvExample({
+        gatewayPort,
+        qaLabPort,
+        gatewayToken,
+        providerBaseUrl,
+        qaBusBaseUrl,
+        includeQaLabUi,
+      }),
+      "utf8",
+    ),
+    fs.writeFile(
+      path.join(params.outputDir, "README.md"),
+      renderReadme({
+        gatewayPort,
+        qaLabPort,
+        usePrebuiltImage,
+        bindUiDist,
+        includeQaLabUi,
+      }),
+      "utf8",
+    ),
+    fs.writeFile(
+      path.join(params.outputDir, "state", "openclaw.json"),
+      `${JSON.stringify(config, null, 2)}\n`,
+      "utf8",
+    ),
+  ]);
+
+  return {
+    outputDir: params.outputDir,
+    imageName,
+    files: [
+      ...files,
+      path.join(params.outputDir, "state", "seed-workspace", "IDENTITY.md"),
+      path.join(params.outputDir, "state", "seed-workspace", "QA_KICKOFF_TASK.md"),
+      path.join(params.outputDir, "state", "seed-workspace", "QA_SCENARIO_PLAN.md"),
+      path.join(params.outputDir, "state", "seed-workspace", "QA_SCENARIOS.md"),
+    ],
+  };
+}
+
+export async function buildQaDockerHarnessImage(
+  params: {
+    repoRoot: string;
+    imageName?: string;
+  },
+  deps?: {
+    runCommand?: (
+      command: string,
+      args: string[],
+      cwd: string,
+    ) => Promise<{ stdout: string; stderr: string }>;
+  },
+) {
+  const imageName = params.imageName ?? "openclaw:qa-local-prebaked";
+  const runCommand =
+    deps?.runCommand ??
+    (async (command: string, args: string[], cwd: string) => {
+      return await new Promise<{ stdout: string; stderr: string }>((resolve, reject) => {
+        execFile(command, args, { cwd }, (error, stdout, stderr) => {
+          if (error) {
+            reject(error);
+            return;
+          }
+          resolve({ stdout, stderr });
+        });
+      });
+    });
+
+  await runCommand(
+    "docker",
+    [
+      "build",
+      "-t",
+      imageName,
+      "--build-arg",
+      "OPENCLAW_EXTENSIONS=qa-channel qa-lab",
+      "-f",
+      "Dockerfile",
+      ".",
+    ],
+    params.repoRoot,
+  );
+
+  return { imageName };
+}

openclaw/extensions/qa-lab/src/docker-runtime.ts

@@ -0,0 +1,283 @@
+import { execFile } from "node:child_process";
+import { createServer } from "node:net";
+import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
+
+export type RunCommand = (
+  command: string,
+  args: string[],
+  cwd: string,
+) => Promise<{ stdout: string; stderr: string }>;
+
+export type FetchLike = (input: string) => Promise<{ ok: boolean }>;
+
+export async function fetchHealthUrl(url: string): Promise<{ ok: boolean }> {
+  const { response, release } = await fetchWithSsrFGuard({
+    url,
+    init: {
+      signal: AbortSignal.timeout(2_000),
+    },
+    policy: { allowPrivateNetwork: true },
+    auditContext: "qa-lab-docker-health-check",
+  });
+  try {
+    return { ok: response.ok };
+  } finally {
+    await release();
+  }
+}
+
+export function describeError(error: unknown) {
+  if (error instanceof Error) {
+    return error.message;
+  }
+  if (typeof error === "string") {
+    return error;
+  }
+  return JSON.stringify(error);
+}
+
+async function isPortFree(port: number) {
+  return await new Promise<boolean>((resolve) => {
+    const server = createServer();
+    server.once("error", () => resolve(false));
+    server.listen(port, "127.0.0.1", () => {
+      server.close(() => resolve(true));
+    });
+  });
+}
+
+async function findFreePort() {
+  return await new Promise<number>((resolve, reject) => {
+    const server = createServer();
+    server.once("error", reject);
+    server.listen(0, () => {
+      const address = server.address();
+      if (!address || typeof address === "string") {
+        server.close();
+        reject(new Error("failed to find free port"));
+        return;
+      }
+      server.close((error) => {
+        if (error) {
+          reject(error);
+          return;
+        }
+        resolve(address.port);
+      });
+    });
+  });
+}
+
+export async function resolveHostPort(preferredPort: number, pinned: boolean) {
+  if (pinned || (await isPortFree(preferredPort))) {
+    return preferredPort;
+  }
+  return await findFreePort();
+}
+
+function trimCommandOutput(output: string) {
+  const trimmed = output.trim();
+  if (!trimmed) {
+    return "";
+  }
+  const lines = trimmed.split("\n");
+  return lines.length <= 120 ? trimmed : lines.slice(-120).join("\n");
+}
+
+export async function execCommand(command: string, args: string[], cwd: string) {
+  return await new Promise<{ stdout: string; stderr: string }>((resolve, reject) => {
+    execFile(
+      command,
+      args,
+      { cwd, encoding: "utf8", maxBuffer: 10 * 1024 * 1024 },
+      (error, stdout, stderr) => {
+        if (error) {
+          const renderedStdout = trimCommandOutput(stdout);
+          const renderedStderr = trimCommandOutput(stderr);
+          reject(
+            new Error(
+              [
+                `Command failed: ${[command, ...args].join(" ")}`,
+                renderedStderr ? `stderr:\n${renderedStderr}` : "",
+                renderedStdout ? `stdout:\n${renderedStdout}` : "",
+              ]
+                .filter(Boolean)
+                .join("\n\n"),
+            ),
+          );
+          return;
+        }
+        resolve({ stdout, stderr });
+      },
+    );
+  });
+}
+
+export async function waitForHealth(
+  url: string,
+  deps: {
+    label?: string;
+    composeFile?: string;
+    fetchImpl: FetchLike;
+    sleepImpl: (ms: number) => Promise<unknown>;
+    timeoutMs?: number;
+    pollMs?: number;
+  },
+) {
+  const timeoutMs = deps.timeoutMs ?? 360_000;
+  const pollMs = deps.pollMs ?? 1_000;
+  const startMs = Date.now();
+  const deadline = startMs + timeoutMs;
+  let lastError: unknown = null;
+
+  while (Date.now() < deadline) {
+    try {
+      const response = await deps.fetchImpl(url);
+      if (response.ok) {
+        return;
+      }
+      lastError = new Error(`Health check returned non-OK for ${url}`);
+    } catch (error) {
+      lastError = error;
+    }
+    await deps.sleepImpl(pollMs);
+  }
+
+  const elapsedSec = Math.round((Date.now() - startMs) / 1000);
+  const service = deps.label ?? url;
+  const lines = [
+    `${service} did not become healthy within ${elapsedSec}s (limit ${Math.round(timeoutMs / 1000)}s).`,
+    lastError ? `Last error: ${describeError(lastError)}` : "",
+    `Hint: check container logs with \`docker compose -f ${deps.composeFile ?? "<compose-file>"} logs\` and verify the port is not already in use.`,
+  ];
+  throw new Error(lines.filter(Boolean).join("\n"));
+}
+
+async function isHealthy(url: string, fetchImpl: FetchLike) {
+  try {
+    const response = await fetchImpl(url);
+    return response.ok;
+  } catch {
+    return false;
+  }
+}
+
+function normalizeDockerServiceStatus(row?: { Health?: string; State?: string }) {
+  const health = row?.Health?.trim();
+  if (health) {
+    return health;
+  }
+  const state = row?.State?.trim();
+  if (state) {
+    return state;
+  }
+  return "unknown";
+}
+
+function parseDockerComposePsRows(stdout: string) {
+  const trimmed = stdout.trim();
+  if (!trimmed) {
+    return [] as Array<{ Health?: string; State?: string }>;
+  }
+
+  try {
+    const parsed = JSON.parse(trimmed) as
+      | Array<{ Health?: string; State?: string }>
+      | { Health?: string; State?: string };
+    if (Array.isArray(parsed)) {
+      return parsed;
+    }
+    return [parsed];
+  } catch {
+    return trimmed
+      .split("\n")
+      .map((line) => line.trim())
+      .filter(Boolean)
+      .map((line) => JSON.parse(line) as { Health?: string; State?: string });
+  }
+}
+
+export async function waitForDockerServiceHealth(
+  service: string,
+  composeFile: string,
+  repoRoot: string,
+  runCommand: RunCommand,
+  sleepImpl: (ms: number) => Promise<unknown>,
+  timeoutMs = 360_000,
+  pollMs = 1_000,
+) {
+  const startMs = Date.now();
+  const deadline = startMs + timeoutMs;
+  let lastStatus = "unknown";
+
+  while (Date.now() < deadline) {
+    try {
+      const { stdout } = await runCommand(
+        "docker",
+        ["compose", "-f", composeFile, "ps", "--format", "json", service],
+        repoRoot,
+      );
+      const rows = parseDockerComposePsRows(stdout);
+      const row = rows[0];
+      lastStatus = normalizeDockerServiceStatus(row);
+      if (lastStatus === "healthy" || lastStatus === "running") {
+        return;
+      }
+    } catch (error) {
+      lastStatus = describeError(error);
+    }
+    await sleepImpl(pollMs);
+  }
+
+  const elapsedSec = Math.round((Date.now() - startMs) / 1000);
+  throw new Error(
+    [
+      `${service} did not become healthy within ${elapsedSec}s (limit ${Math.round(timeoutMs / 1000)}s).`,
+      `Last status: ${lastStatus}`,
+      `Hint: check container logs with \`docker compose -f ${composeFile} logs ${service}\`.`,
+    ].join("\n"),
+  );
+}
+
+export async function resolveComposeServiceUrl(
+  service: string,
+  port: number,
+  composeFile: string,
+  repoRoot: string,
+  runCommand: RunCommand,
+  fetchImpl?: FetchLike,
+) {
+  const { stdout: containerStdout } = await runCommand(
+    "docker",
+    ["compose", "-f", composeFile, "ps", "-q", service],
+    repoRoot,
+  );
+  const containerId = containerStdout.trim();
+  if (!containerId) {
+    return null;
+  }
+  const { stdout: ipStdout } = await runCommand(
+    "docker",
+    [
+      "inspect",
+      "--format",
+      "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
+      containerId,
+    ],
+    repoRoot,
+  );
+  const ip = ipStdout.trim();
+  if (!ip) {
+    return null;
+  }
+  const baseUrl = `http://${ip}:${port}/`;
+  if (!fetchImpl) {
+    return baseUrl;
+  }
+  return (await isHealthy(`${baseUrl}healthz`, fetchImpl)) ? baseUrl : null;
+}
+
+export const __testing = {
+  fetchHealthUrl,
+  normalizeDockerServiceStatus,
+};

openclaw/extensions/qa-lab/src/docker-up.runtime.test.ts

@@ -0,0 +1,272 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { createServer } from "node:net";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it, vi } from "vitest";
+import { runQaDockerUp } from "./docker-up.runtime.js";
+
+async function occupyPortOrAcceptExisting(port: number): Promise<{ close: () => Promise<void> }> {
+  const server = createServer();
+  const listening = await new Promise<boolean>((resolve, reject) => {
+    server.once("error", (error: NodeJS.ErrnoException) => {
+      if (error.code === "EADDRINUSE") {
+        resolve(false);
+        return;
+      }
+      reject(error);
+    });
+    server.listen(port, "127.0.0.1", () => resolve(true));
+  });
+
+  return {
+    close: async () => {
+      if (!listening) {
+        return;
+      }
+      await new Promise<void>((resolve, reject) =>
+        server.close((error) => (error ? reject(error) : resolve())),
+      );
+    },
+  };
+}
+
+describe("runQaDockerUp", () => {
+  it("builds the QA UI, writes the harness, starts compose, and waits for health", async () => {
+    const calls: string[] = [];
+    const fetchCalls: string[] = [];
+    const responseQueue = [false, true, true];
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+
+    try {
+      const result = await runQaDockerUp(
+        {
+          repoRoot: "/repo/openclaw",
+          outputDir,
+          gatewayPort: 18889,
+          qaLabPort: 43124,
+        },
+        {
+          async runCommand(command, args, cwd) {
+            calls.push([command, ...args, `@${cwd}`].join(" "));
+            if (args.join(" ").includes("ps --format json openclaw-qa-gateway")) {
+              return { stdout: '[{"Health":"healthy","State":"running"}]\n', stderr: "" };
+            }
+            return { stdout: "", stderr: "" };
+          },
+          fetchImpl: vi.fn(async (input: string) => {
+            fetchCalls.push(input);
+            return { ok: responseQueue.shift() ?? true };
+          }),
+          sleepImpl: vi.fn(async () => {}),
+        },
+      );
+
+      expect(calls).toEqual([
+        "pnpm qa:lab:build @/repo/openclaw",
+        `docker compose -f ${outputDir}/docker-compose.qa.yml down --remove-orphans @/repo/openclaw`,
+        expect.stringContaining(
+          `docker compose -f ${outputDir}/docker-compose.qa.yml up --build -d @/repo/openclaw`,
+        ),
+        `docker compose -f ${outputDir}/docker-compose.qa.yml ps --format json openclaw-qa-gateway @/repo/openclaw`,
+      ]);
+      expect(fetchCalls).toEqual([
+        "http://127.0.0.1:43124/healthz",
+        "http://127.0.0.1:43124/healthz",
+        "http://127.0.0.1:18889/healthz",
+      ]);
+      expect(result.qaLabUrl).toBe("http://127.0.0.1:43124");
+      expect(result.gatewayUrl).toBe("http://127.0.0.1:18889/");
+      expect(result.composeFile).toBe(`${outputDir}/docker-compose.qa.yml`);
+      expect(result.stopCommand).toBe(`docker compose -f ${outputDir}/docker-compose.qa.yml down`);
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("skips UI build and compose --build for prebuilt images", async () => {
+    const calls: string[] = [];
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+
+    try {
+      await runQaDockerUp(
+        {
+          repoRoot: "/repo/openclaw",
+          outputDir,
+          usePrebuiltImage: true,
+          bindUiDist: true,
+          skipUiBuild: true,
+        },
+        {
+          async runCommand(command, args, cwd) {
+            calls.push([command, ...args, `@${cwd}`].join(" "));
+            if (args.join(" ").includes("ps --format json openclaw-qa-gateway")) {
+              return { stdout: '{"Health":"healthy","State":"running"}\n', stderr: "" };
+            }
+            return { stdout: "", stderr: "" };
+          },
+          fetchImpl: vi.fn(async () => ({ ok: true })),
+          sleepImpl: vi.fn(async () => {}),
+        },
+      );
+
+      expect(calls).toEqual([
+        `docker compose -f ${outputDir}/docker-compose.qa.yml down --remove-orphans @/repo/openclaw`,
+        `docker compose -f ${outputDir}/docker-compose.qa.yml up -d @/repo/openclaw`,
+        `docker compose -f ${outputDir}/docker-compose.qa.yml ps --format json openclaw-qa-gateway @/repo/openclaw`,
+      ]);
+      const compose = await readFile(path.join(outputDir, "docker-compose.qa.yml"), "utf8");
+      expect(compose).toContain(":/opt/openclaw-qa-lab-ui:ro");
+      expect(compose).toContain("      - --ui-dist-dir");
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("uses a repo-root-relative default output dir when none is provided", async () => {
+    const calls: string[] = [];
+    const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-docker-root-"));
+
+    try {
+      const result = await runQaDockerUp(
+        {
+          repoRoot,
+          usePrebuiltImage: true,
+          skipUiBuild: true,
+        },
+        {
+          async runCommand(command, args, cwd) {
+            calls.push([command, ...args, `@${cwd}`].join(" "));
+            if (args.join(" ").includes("ps --format json openclaw-qa-gateway")) {
+              return { stdout: '{"Health":"healthy","State":"running"}\n', stderr: "" };
+            }
+            return { stdout: "", stderr: "" };
+          },
+          fetchImpl: vi.fn(async () => ({ ok: true })),
+          sleepImpl: vi.fn(async () => {}),
+        },
+      );
+
+      expect(result.outputDir).toBe(path.join(repoRoot, ".artifacts/qa-docker"));
+      expect(result.composeFile).toBe(
+        path.join(repoRoot, ".artifacts/qa-docker/docker-compose.qa.yml"),
+      );
+      expect(calls).toEqual([
+        `docker compose -f ${path.join(repoRoot, ".artifacts/qa-docker/docker-compose.qa.yml")} down --remove-orphans @${repoRoot}`,
+        `docker compose -f ${path.join(repoRoot, ".artifacts/qa-docker/docker-compose.qa.yml")} up -d @${repoRoot}`,
+        `docker compose -f ${path.join(repoRoot, ".artifacts/qa-docker/docker-compose.qa.yml")} ps --format json openclaw-qa-gateway @${repoRoot}`,
+      ]);
+    } finally {
+      await rm(repoRoot, { recursive: true, force: true });
+    }
+  });
+
+  it("falls back to free host ports when defaults are already occupied", async () => {
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+    const gatewayPort = 18789;
+    const qaLabPort = 43124;
+    const resolveHostPort = vi.fn(async (preferredPort: number) => {
+      if (preferredPort === gatewayPort) {
+        return 28001;
+      }
+      if (preferredPort === qaLabPort) {
+        return 28002;
+      }
+      return preferredPort;
+    });
+    const gatewayPortReservation = await occupyPortOrAcceptExisting(18789);
+    const qaLabPortReservation = await occupyPortOrAcceptExisting(43124);
+
+    try {
+      const result = await runQaDockerUp(
+        {
+          repoRoot: "/repo/openclaw",
+          outputDir,
+          gatewayPort,
+          qaLabPort,
+          skipUiBuild: true,
+          usePrebuiltImage: true,
+        },
+        {
+          async runCommand() {
+            return {
+              stdout: '{"Health":"healthy","State":"running"}\n',
+              stderr: "",
+            };
+          },
+          fetchImpl: vi.fn(async () => ({ ok: true })),
+          sleepImpl: vi.fn(async () => {}),
+          resolveHostPortImpl: resolveHostPort,
+        },
+      );
+
+      expect(result.gatewayUrl).not.toBe(`http://127.0.0.1:${gatewayPort}/`);
+      expect(result.qaLabUrl).not.toBe(`http://127.0.0.1:${qaLabPort}`);
+      expect(result.gatewayUrl).toBe("http://127.0.0.1:28001/");
+      expect(result.qaLabUrl).toBe("http://127.0.0.1:28002");
+    } finally {
+      await gatewayPortReservation.close();
+      await qaLabPortReservation.close();
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("falls back to the container IP when the host gateway port is unreachable", async () => {
+    const calls: string[] = [];
+    const fetchCalls: string[] = [];
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+
+    try {
+      const result = await runQaDockerUp(
+        {
+          repoRoot: "/repo/openclaw",
+          outputDir,
+          gatewayPort: 18889,
+          qaLabPort: 43124,
+          skipUiBuild: true,
+          usePrebuiltImage: true,
+        },
+        {
+          async runCommand(command, args, cwd) {
+            calls.push([command, ...args, `@${cwd}`].join(" "));
+            const joined = args.join(" ");
+            if (joined.includes("ps --format json openclaw-qa-gateway")) {
+              return { stdout: '{"Health":"healthy","State":"running"}\n', stderr: "" };
+            }
+            if (joined.includes("ps -q openclaw-qa-gateway")) {
+              return { stdout: "gateway-container\n", stderr: "" };
+            }
+            if (command === "docker" && args[0] === "inspect") {
+              return { stdout: "192.168.165.4\n", stderr: "" };
+            }
+            return { stdout: "", stderr: "" };
+          },
+          fetchImpl: vi.fn(async (input: string) => {
+            fetchCalls.push(input);
+            return {
+              ok:
+                input === "http://127.0.0.1:43124/healthz" ||
+                input === "http://192.168.165.4:18789/healthz",
+            };
+          }),
+          sleepImpl: vi.fn(async () => {}),
+        },
+      );
+
+      expect(calls).toEqual([
+        `docker compose -f ${outputDir}/docker-compose.qa.yml down --remove-orphans @/repo/openclaw`,
+        `docker compose -f ${outputDir}/docker-compose.qa.yml up -d @/repo/openclaw`,
+        `docker compose -f ${outputDir}/docker-compose.qa.yml ps --format json openclaw-qa-gateway @/repo/openclaw`,
+        `docker compose -f ${outputDir}/docker-compose.qa.yml ps -q openclaw-qa-gateway @/repo/openclaw`,
+        "docker inspect --format {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}} gateway-container @/repo/openclaw",
+      ]);
+      expect(fetchCalls).toEqual([
+        "http://127.0.0.1:43124/healthz",
+        "http://127.0.0.1:18889/healthz",
+        "http://192.168.165.4:18789/healthz",
+      ]);
+      expect(result.gatewayUrl).toBe("http://192.168.165.4:18789/");
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+});

openclaw/extensions/qa-lab/src/docker-up.runtime.ts

@@ -0,0 +1,141 @@
+import path from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
+import { writeQaDockerHarnessFiles } from "./docker-harness.js";
+import {
+  execCommand,
+  fetchHealthUrl,
+  resolveComposeServiceUrl,
+  resolveHostPort,
+  waitForDockerServiceHealth,
+  waitForHealth,
+  type FetchLike,
+  type RunCommand,
+} from "./docker-runtime.js";
+
+type QaDockerUpResult = {
+  outputDir: string;
+  composeFile: string;
+  qaLabUrl: string;
+  gatewayUrl: string;
+  stopCommand: string;
+};
+
+function resolveDefaultQaDockerDir(repoRoot: string) {
+  return path.resolve(repoRoot, ".artifacts/qa-docker");
+}
+
+export async function runQaDockerUp(
+  params: {
+    repoRoot?: string;
+    outputDir?: string;
+    gatewayPort?: number;
+    qaLabPort?: number;
+    providerBaseUrl?: string;
+    image?: string;
+    usePrebuiltImage?: boolean;
+    bindUiDist?: boolean;
+    skipUiBuild?: boolean;
+  },
+  deps?: {
+    runCommand?: RunCommand;
+    fetchImpl?: FetchLike;
+    sleepImpl?: (ms: number) => Promise<unknown>;
+    resolveHostPortImpl?: typeof resolveHostPort;
+  },
+): Promise<QaDockerUpResult> {
+  const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
+  const resolveHostPortImpl = deps?.resolveHostPortImpl ?? resolveHostPort;
+  const outputDir = path.resolve(params.outputDir ?? resolveDefaultQaDockerDir(repoRoot));
+  const gatewayPort = await resolveHostPortImpl(
+    params.gatewayPort ?? 18789,
+    params.gatewayPort != null,
+  );
+  const qaLabPort = await resolveHostPortImpl(params.qaLabPort ?? 43124, params.qaLabPort != null);
+  const runCommand = deps?.runCommand ?? execCommand;
+  const fetchImpl = deps?.fetchImpl ?? fetchHealthUrl;
+  const sleepImpl = deps?.sleepImpl ?? sleep;
+
+  if (!params.skipUiBuild) {
+    await runCommand("pnpm", ["qa:lab:build"], repoRoot);
+  }
+
+  await writeQaDockerHarnessFiles({
+    outputDir,
+    repoRoot,
+    gatewayPort,
+    qaLabPort,
+    providerBaseUrl: params.providerBaseUrl,
+    imageName: params.image,
+    usePrebuiltImage: params.usePrebuiltImage,
+    bindUiDist: params.bindUiDist,
+    includeQaLabUi: true,
+  });
+
+  const composeFile = path.join(outputDir, "docker-compose.qa.yml");
+
+  // Tear down any previous stack from this compose file so ports are freed
+  // and we get a clean restart every time.
+  try {
+    await runCommand(
+      "docker",
+      ["compose", "-f", composeFile, "down", "--remove-orphans"],
+      repoRoot,
+    );
+  } catch {
+    // First run or already stopped — ignore.
+  }
+
+  const composeArgs = ["compose", "-f", composeFile, "up"];
+  if (!params.usePrebuiltImage) {
+    composeArgs.push("--build");
+  }
+  composeArgs.push("-d");
+
+  await runCommand("docker", composeArgs, repoRoot);
+
+  // Brief settle delay so Docker Desktop finishes port-forwarding setup.
+  await sleepImpl(3_000);
+
+  const qaLabUrl = `http://127.0.0.1:${qaLabPort}`;
+  const hostGatewayUrl = `http://127.0.0.1:${gatewayPort}/`;
+
+  await waitForHealth(`${qaLabUrl}/healthz`, {
+    label: "QA Lab",
+    fetchImpl,
+    sleepImpl,
+    composeFile,
+  });
+  await waitForDockerServiceHealth(
+    "openclaw-qa-gateway",
+    composeFile,
+    repoRoot,
+    runCommand,
+    sleepImpl,
+  );
+  let gatewayUrl = hostGatewayUrl;
+  if (
+    !(await fetchImpl(`${hostGatewayUrl}healthz`)
+      .then((response) => response.ok)
+      .catch(() => false))
+  ) {
+    const containerGatewayUrl = await resolveComposeServiceUrl(
+      "openclaw-qa-gateway",
+      18789,
+      composeFile,
+      repoRoot,
+      runCommand,
+      fetchImpl,
+    );
+    if (containerGatewayUrl) {
+      gatewayUrl = containerGatewayUrl;
+    }
+  }
+
+  return {
+    outputDir,
+    composeFile,
+    qaLabUrl,
+    gatewayUrl,
+    stopCommand: `docker compose -f ${composeFile} down`,
+  };
+}

openclaw/extensions/qa-lab/src/extract-tool-payload.ts

@@ -0,0 +1 @@
+export { extractToolPayload as extractQaToolPayload } from "openclaw/plugin-sdk/tool-payload";

openclaw/extensions/qa-lab/src/gateway-child.ts

@@ -0,0 +1,904 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { createWriteStream, existsSync, type WriteStream } from "node:fs";
+import fs from "node:fs/promises";
+import net from "node:net";
+import os from "node:os";
+import path from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
+import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
+import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
+import {
+  createQaBundledPluginsDir,
+  resolveQaBundledPluginSourceDir,
+  resolveQaOwnerPluginIdsForProviderIds,
+  resolveQaRuntimeHostVersion,
+} from "./bundled-plugin-staging.js";
+import { assertRepoBoundPath, ensureRepoBoundDirectory } from "./cli-paths.js";
+import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway-log-redaction.js";
+import { startQaGatewayRpcClient } from "./gateway-rpc-client.js";
+import { splitQaModelRef, type QaProviderMode } from "./model-selection.js";
+import { resolveQaNodeExecPath } from "./node-exec.js";
+import {
+  normalizeQaProviderModeEnv,
+  QA_LIVE_PROVIDER_CONFIG_PATH_ENV,
+  resolveQaLiveCliAuthEnv,
+  resolveQaLiveProviderConfigPath,
+  type QaCliBackendAuthMode,
+} from "./providers/env.js";
+import { DEFAULT_QA_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
+import {
+  QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV,
+  QA_LIVE_SETUP_TOKEN_VALUE_ENV,
+  stageQaLiveAnthropicSetupToken,
+} from "./providers/live-frontier/auth.js";
+import { stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js";
+import { seedQaAgentWorkspace } from "./qa-agent-workspace.js";
+import { buildQaGatewayConfig, type QaThinkingLevel } from "./qa-gateway-config.js";
+import type { QaTransportAdapter } from "./qa-transport.js";
+
+export type { QaCliBackendAuthMode } from "./providers/env.js";
+const QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS = 5;
+
+export type QaGatewayChildStateMutationContext = {
+  configPath: string;
+  runtimeEnv: NodeJS.ProcessEnv;
+  stateDir: string;
+  tempRoot: string;
+};
+
+async function getFreePort() {
+  return await new Promise<number>((resolve, reject) => {
+    const server = net.createServer();
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => {
+      const address = server.address();
+      if (!address || typeof address === "string") {
+        reject(new Error("failed to allocate port"));
+        return;
+      }
+      server.close((error) => (error ? reject(error) : resolve(address.port)));
+    });
+  });
+}
+
+async function closeWriteStream(stream: WriteStream) {
+  await new Promise<void>((resolve) => {
+    stream.end(() => resolve());
+  });
+}
+
+async function writeSanitizedQaGatewayDebugLog(params: { sourcePath: string; targetPath: string }) {
+  const contents = await fs.readFile(params.sourcePath, "utf8").catch((error) => {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return "";
+    }
+    throw error;
+  });
+  await fs.writeFile(params.targetPath, redactQaGatewayDebugText(contents), "utf8");
+}
+
+async function assertQaArtifactDirWithinRepo(repoRoot: string, artifactDir: string) {
+  return await assertRepoBoundPath(repoRoot, artifactDir, "QA gateway artifact directory");
+}
+
+async function clearQaGatewayArtifactDir(dir: string) {
+  for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
+    await fs.rm(path.join(dir, entry.name), { recursive: true, force: true });
+  }
+}
+
+async function cleanupQaGatewayTempRoots(params: {
+  tempRoot: string;
+  stagedBundledPluginsRoot?: string | null;
+}) {
+  await fs.rm(params.tempRoot, { recursive: true, force: true }).catch(() => {});
+  if (params.stagedBundledPluginsRoot) {
+    await fs.rm(params.stagedBundledPluginsRoot, { recursive: true, force: true }).catch(() => {});
+  }
+}
+
+async function preserveQaGatewayDebugArtifacts(params: {
+  preserveToDir: string;
+  stdoutLogPath: string;
+  stderrLogPath: string;
+  tempRoot: string;
+  repoRoot?: string;
+}) {
+  const preserveToDir = params.repoRoot
+    ? await ensureRepoBoundDirectory(
+        params.repoRoot,
+        params.preserveToDir,
+        "QA gateway artifact directory",
+        {
+          mode: 0o700,
+        },
+      )
+    : params.preserveToDir;
+  await fs.mkdir(preserveToDir, { recursive: true, mode: 0o700 });
+  await clearQaGatewayArtifactDir(preserveToDir);
+  await Promise.all([
+    writeSanitizedQaGatewayDebugLog({
+      sourcePath: params.stdoutLogPath,
+      targetPath: path.join(preserveToDir, "gateway.stdout.log"),
+    }),
+    writeSanitizedQaGatewayDebugLog({
+      sourcePath: params.stderrLogPath,
+      targetPath: path.join(preserveToDir, "gateway.stderr.log"),
+    }),
+  ]);
+  await fs.writeFile(
+    path.join(preserveToDir, "README.txt"),
+    [
+      "Only sanitized gateway debug artifacts are preserved here.",
+      "The full QA gateway runtime was not copied because it may contain credentials or auth tokens.",
+      `Original runtime temp root: ${params.tempRoot}`,
+      "",
+    ].join("\n"),
+    "utf8",
+  );
+}
+
+function isRetryableGatewayStartupError(details: string) {
+  return (
+    details.includes("another gateway instance is already listening on ws://") ||
+    details.includes("failed to bind gateway socket on ws://") ||
+    details.includes("EADDRINUSE") ||
+    details.includes("address already in use")
+  );
+}
+
+function appendQaGatewayTempRoot(details: string, tempRoot: string) {
+  return details.includes(tempRoot)
+    ? details
+    : `${details}\nQA gateway temp root preserved at ${tempRoot}`;
+}
+
+export function resolveQaGatewayChildProviderMode(providerMode?: QaProviderMode): QaProviderMode {
+  return providerMode ?? DEFAULT_QA_PROVIDER_MODE;
+}
+
+export function buildQaRuntimeEnv(params: {
+  configPath: string;
+  gatewayToken: string;
+  homeDir: string;
+  forwardHostHome?: boolean;
+  stateDir: string;
+  xdgConfigHome: string;
+  xdgDataHome: string;
+  xdgCacheHome: string;
+  bundledPluginsDir?: string;
+  compatibilityHostVersion?: string;
+  providerMode?: QaProviderMode;
+  baseEnv?: NodeJS.ProcessEnv;
+  forwardHostHomeForClaudeCli?: boolean;
+  claudeCliAuthMode?: QaCliBackendAuthMode;
+}) {
+  const baseEnv = params.baseEnv ?? process.env;
+  const provider = params.providerMode ? getQaProvider(params.providerMode) : null;
+  const forwardedHostHome = params.forwardHostHome
+    ? baseEnv.HOME?.trim() || os.homedir()
+    : undefined;
+  const env: NodeJS.ProcessEnv = {
+    ...baseEnv,
+    HOME: forwardedHostHome ?? params.homeDir,
+    ...(provider?.appliesLiveEnvAliases
+      ? resolveQaLiveCliAuthEnv(baseEnv, {
+          forwardHostHomeForClaudeCli: params.forwardHostHomeForClaudeCli,
+          claudeCliAuthMode: params.claudeCliAuthMode,
+        })
+      : {}),
+    OPENCLAW_HOME: params.homeDir,
+    OPENCLAW_CONFIG_PATH: params.configPath,
+    OPENCLAW_STATE_DIR: params.stateDir,
+    OPENCLAW_OAUTH_DIR: path.join(params.stateDir, "credentials"),
+    OPENCLAW_GATEWAY_TOKEN: params.gatewayToken,
+    OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1",
+    OPENCLAW_SKIP_GMAIL_WATCHER: "1",
+    OPENCLAW_SKIP_CANVAS_HOST: "1",
+    OPENCLAW_NO_RESPAWN: "1",
+    OPENCLAW_TEST_FAST: "1",
+    OPENCLAW_QA_ALLOW_LOCAL_IMAGE_PROVIDER: "1",
+    // QA uses the fast runtime envelope for speed, but it still exercises
+    // normal config-driven heartbeats and runtime config writes.
+    OPENCLAW_ALLOW_SLOW_REPLY_TESTS: "1",
+    XDG_CONFIG_HOME: params.xdgConfigHome,
+    XDG_DATA_HOME: params.xdgDataHome,
+    XDG_CACHE_HOME: params.xdgCacheHome,
+    ...(params.bundledPluginsDir ? { OPENCLAW_BUNDLED_PLUGINS_DIR: params.bundledPluginsDir } : {}),
+    ...(params.compatibilityHostVersion
+      ? { OPENCLAW_COMPATIBILITY_HOST_VERSION: params.compatibilityHostVersion }
+      : {}),
+  };
+  const normalizedEnv = normalizeQaProviderModeEnv(env, params.providerMode);
+  delete normalizedEnv[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV];
+  delete normalizedEnv[QA_LIVE_SETUP_TOKEN_VALUE_ENV];
+  return normalizedEnv;
+}
+
+function isRetryableGatewayCallError(details: string): boolean {
+  return (
+    details.includes("handshake timeout") ||
+    details.includes("gateway closed (1000") ||
+    details.includes("gateway closed (1012)") ||
+    details.includes("gateway closed (1006") ||
+    details.includes("abnormal closure") ||
+    details.includes("service restart")
+  );
+}
+
+async function fetchLocalGatewayHealth(params: {
+  baseUrl: string;
+  healthPath: "/readyz" | "/healthz";
+}): Promise<boolean> {
+  const { response, release } = await fetchWithSsrFGuard({
+    url: `${params.baseUrl}${params.healthPath}`,
+    init: {
+      method: "HEAD",
+      headers: {
+        connection: "close",
+      },
+      signal: AbortSignal.timeout(2_000),
+    },
+    policy: { allowPrivateNetwork: true },
+    auditContext: "qa-lab-gateway-child-health",
+  });
+  try {
+    return response.ok;
+  } finally {
+    await release();
+  }
+}
+
+export const __testing = {
+  assertQaArtifactDirWithinRepo,
+  buildQaRuntimeEnv,
+  cleanupQaGatewayTempRoots,
+  fetchLocalGatewayHealth,
+  isRetryableGatewayCallError,
+  isRetryableRpcStartupError,
+  isRetryableGatewayStartupError,
+  preserveQaGatewayDebugArtifacts,
+  redactQaGatewayDebugText,
+  readQaLiveProviderConfigOverrides,
+  resolveQaGatewayChildProviderMode,
+  stageQaLiveAnthropicSetupToken,
+  stageQaMockAuthProfiles,
+  resolveQaLiveCliAuthEnv,
+  resolveQaOwnerPluginIdsForProviderIds,
+  resolveQaBundledPluginSourceDir,
+  resolveQaRuntimeHostVersion,
+  createQaBundledPluginsDir,
+  stopQaGatewayChildProcessTree,
+};
+
+function hasChildExited(child: ChildProcess) {
+  return child.exitCode !== null || child.signalCode !== null;
+}
+
+function signalQaGatewayChildProcessTree(child: ChildProcess, signal: NodeJS.Signals) {
+  if (!child.pid) {
+    return;
+  }
+  try {
+    if (process.platform === "win32") {
+      child.kill(signal);
+      return;
+    }
+    process.kill(-child.pid, signal);
+  } catch {
+    try {
+      child.kill(signal);
+    } catch {
+      // The child already exited.
+    }
+  }
+}
+
+async function waitForQaGatewayChildExit(child: ChildProcess, timeoutMs: number) {
+  if (hasChildExited(child)) {
+    return true;
+  }
+  return await Promise.race([
+    new Promise<boolean>((resolve) => child.once("exit", () => resolve(true))),
+    sleep(timeoutMs).then(() => false),
+  ]);
+}
+
+async function stopQaGatewayChildProcessTree(
+  child: ChildProcess,
+  opts?: { gracefulTimeoutMs?: number; forceTimeoutMs?: number },
+) {
+  if (hasChildExited(child)) {
+    return;
+  }
+  signalQaGatewayChildProcessTree(child, "SIGTERM");
+  if (await waitForQaGatewayChildExit(child, opts?.gracefulTimeoutMs ?? 5_000)) {
+    return;
+  }
+  signalQaGatewayChildProcessTree(child, "SIGKILL");
+  await waitForQaGatewayChildExit(child, opts?.forceTimeoutMs ?? 2_000);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function isQaModelProviderConfig(value: unknown): value is ModelProviderConfig {
+  return isRecord(value) && typeof value.baseUrl === "string" && Array.isArray(value.models);
+}
+
+async function readQaLiveProviderConfigOverrides(params: {
+  providerIds: readonly string[];
+  env?: NodeJS.ProcessEnv;
+}) {
+  const providerIds = [
+    ...new Set(params.providerIds.map((providerId) => providerId.trim())),
+  ].filter((providerId) => providerId.length > 0);
+  if (providerIds.length === 0) {
+    return {};
+  }
+  const configPath = resolveQaLiveProviderConfigPath(params.env);
+  if (!existsSync(configPath.path)) {
+    return {};
+  }
+  try {
+    const raw = await fs.readFile(configPath.path, "utf8");
+    const parsed = JSON.parse(raw) as unknown;
+    const providers = isRecord(parsed)
+      ? isRecord(parsed.models)
+        ? isRecord(parsed.models.providers)
+          ? parsed.models.providers
+          : {}
+        : {}
+      : {};
+    const selected: Record<string, ModelProviderConfig> = {};
+    for (const providerId of providerIds) {
+      const providerConfig = providers[providerId];
+      if (isQaModelProviderConfig(providerConfig)) {
+        selected[providerId] = providerConfig;
+      }
+    }
+    return selected;
+  } catch (error) {
+    if (configPath.explicit) {
+      throw new Error(
+        `failed to read ${QA_LIVE_PROVIDER_CONFIG_PATH_ENV} provider config: ${formatErrorMessage(error)}`,
+        { cause: error },
+      );
+    }
+    return {};
+  }
+}
+
+async function waitForGatewayReady(params: {
+  baseUrl: string;
+  logs: () => string;
+  child: {
+    exitCode: number | null;
+    signalCode: NodeJS.Signals | null;
+  };
+  timeoutMs?: number;
+}) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) {
+    if (params.child.exitCode !== null || params.child.signalCode !== null) {
+      throw new Error(
+        `gateway exited before becoming healthy (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`,
+      );
+    }
+    for (const healthPath of ["/readyz", "/healthz"] as const) {
+      try {
+        if (await fetchLocalGatewayHealth({ baseUrl: params.baseUrl, healthPath })) {
+          return;
+        }
+      } catch {
+        // retry until timeout
+      }
+    }
+    await sleep(250);
+  }
+  throw new Error(`gateway failed to become healthy:\n${params.logs()}`);
+}
+
+function isRetryableRpcStartupError(error: unknown) {
+  const details = formatErrorMessage(error);
+  return (
+    details.includes("gateway timeout after") ||
+    details.includes("handshake timeout") ||
+    details.includes("gateway token mismatch") ||
+    details.includes("token mismatch") ||
+    details.includes("gateway closed (1000") ||
+    details.includes("gateway closed (1006") ||
+    details.includes("gateway closed (1012)")
+  );
+}
+
+export function resolveQaControlUiRoot(params: { repoRoot: string; controlUiEnabled?: boolean }) {
+  if (params.controlUiEnabled === false) {
+    return undefined;
+  }
+  const controlUiRoot = path.join(params.repoRoot, "dist", "control-ui");
+  const indexPath = path.join(controlUiRoot, "index.html");
+  return existsSync(indexPath) ? controlUiRoot : undefined;
+}
+
+export async function startQaGatewayChild(params: {
+  repoRoot: string;
+  providerBaseUrl?: string;
+  transport: Pick<QaTransportAdapter, "requiredPluginIds" | "createGatewayConfig">;
+  transportBaseUrl: string;
+  controlUiAllowedOrigins?: string[];
+  providerMode?: QaProviderMode;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  thinkingDefault?: QaThinkingLevel;
+  claudeCliAuthMode?: QaCliBackendAuthMode;
+  controlUiEnabled?: boolean;
+  enabledPluginIds?: string[];
+  forwardHostHome?: boolean;
+  mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
+}) {
+  const tempRoot = await fs.mkdtemp(
+    path.join(resolvePreferredOpenClawTmpDir(), "openclaw-qa-suite-"),
+  );
+  const runtimeCwd = tempRoot;
+  const distEntryPath = path.join(params.repoRoot, "dist", "index.js");
+  const workspaceDir = path.join(tempRoot, "workspace");
+  const stateDir = path.join(tempRoot, "state");
+  const homeDir = path.join(tempRoot, "home");
+  const xdgConfigHome = path.join(tempRoot, "xdg-config");
+  const xdgDataHome = path.join(tempRoot, "xdg-data");
+  const xdgCacheHome = path.join(tempRoot, "xdg-cache");
+  const configPath = path.join(tempRoot, "openclaw.json");
+  const gatewayToken = `qa-suite-${randomUUID()}`;
+  await seedQaAgentWorkspace({
+    workspaceDir,
+    repoRoot: params.repoRoot,
+  });
+  await Promise.all([
+    fs.mkdir(stateDir, { recursive: true }),
+    fs.mkdir(homeDir, { recursive: true }),
+    fs.mkdir(xdgConfigHome, { recursive: true }),
+    fs.mkdir(xdgDataHome, { recursive: true }),
+    fs.mkdir(xdgCacheHome, { recursive: true }),
+  ]);
+  const providerMode = resolveQaGatewayChildProviderMode(params.providerMode);
+  const resolvedProvider = getQaProvider(providerMode);
+  const liveProviderIds = resolvedProvider.usesModelProviderPlugins
+    ? [params.primaryModel, params.alternateModel]
+        .map((modelRef) =>
+          typeof modelRef === "string" ? splitQaModelRef(modelRef)?.provider : undefined,
+        )
+        .filter((providerId): providerId is string => Boolean(providerId))
+    : [];
+  const liveProviderConfigs = await readQaLiveProviderConfigOverrides({
+    providerIds: liveProviderIds,
+  });
+  const liveOwnerPluginIds =
+    liveProviderIds.length > 0
+      ? await resolveQaOwnerPluginIdsForProviderIds({
+          repoRoot: params.repoRoot,
+          providerIds: liveProviderIds,
+          providerConfigs: liveProviderConfigs,
+        })
+      : [];
+  const enabledPluginIds = [
+    ...new Set([...(liveOwnerPluginIds ?? []), ...(params.enabledPluginIds ?? [])]),
+  ];
+  const buildGatewayConfig = (gatewayPort: number) =>
+    buildQaGatewayConfig({
+      bind: "loopback",
+      gatewayPort,
+      gatewayToken,
+      providerBaseUrl: params.providerBaseUrl,
+      workspaceDir,
+      controlUiRoot: resolveQaControlUiRoot({
+        repoRoot: params.repoRoot,
+        controlUiEnabled: params.controlUiEnabled,
+      }),
+      controlUiAllowedOrigins: params.controlUiAllowedOrigins,
+      providerMode,
+      primaryModel: params.primaryModel,
+      alternateModel: params.alternateModel,
+      enabledPluginIds,
+      transportPluginIds: params.transport.requiredPluginIds,
+      transportConfig: params.transport.createGatewayConfig({
+        baseUrl: params.transportBaseUrl,
+      }),
+      liveProviderConfigs,
+      fastMode: params.fastMode,
+      thinkingDefault: params.thinkingDefault,
+      controlUiEnabled: params.controlUiEnabled,
+    });
+  const buildStagedGatewayConfig = async (gatewayPort: number) => {
+    let cfg = buildGatewayConfig(gatewayPort);
+    cfg = await stageQaLiveAnthropicSetupToken({
+      cfg,
+      stateDir,
+    });
+    const mockAuthProviders = getQaProvider(providerMode).mockAuthProviders;
+    if (mockAuthProviders && mockAuthProviders.length > 0) {
+      cfg = await stageQaMockAuthProfiles({
+        cfg,
+        stateDir,
+        providers: mockAuthProviders,
+      });
+    }
+    return params.mutateConfig ? params.mutateConfig(cfg) : cfg;
+  };
+  const stdout: Buffer[] = [];
+  const stderr: Buffer[] = [];
+  const stdoutLogPath = path.join(tempRoot, "gateway.stdout.log");
+  const stderrLogPath = path.join(tempRoot, "gateway.stderr.log");
+  const stdoutLog = createWriteStream(stdoutLogPath, { flags: "a" });
+  const stderrLog = createWriteStream(stderrLogPath, { flags: "a" });
+
+  const logs = () =>
+    `${Buffer.concat(stdout).toString("utf8")}\n${Buffer.concat(stderr).toString("utf8")}`.trim();
+  const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1";
+  let gatewayPort = 0;
+  let baseUrl = "";
+  let wsUrl = "";
+  let child: ReturnType<typeof spawn> | null = null;
+  let cfg: ReturnType<typeof buildQaGatewayConfig> | null = null;
+  let rpcClient: Awaited<ReturnType<typeof startQaGatewayRpcClient>> | null = null;
+  let stagedBundledPluginsRoot: string | null = null;
+  let env: NodeJS.ProcessEnv | null = null;
+
+  try {
+    const nodeExecPath = await resolveQaNodeExecPath();
+    for (let attempt = 1; attempt <= QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS; attempt += 1) {
+      gatewayPort = await getFreePort();
+      baseUrl = `http://127.0.0.1:${gatewayPort}`;
+      wsUrl = `ws://127.0.0.1:${gatewayPort}`;
+      cfg = await buildStagedGatewayConfig(gatewayPort);
+      if (!env) {
+        const allowedPluginIds = [...(cfg.plugins?.allow ?? []), "openai"].filter(
+          (pluginId, index, array): pluginId is string => {
+            return (
+              typeof pluginId === "string" &&
+              pluginId.length > 0 &&
+              array.indexOf(pluginId) === index
+            );
+          },
+        );
+        const { bundledPluginsDir, stagedRoot } = await createQaBundledPluginsDir({
+          repoRoot: params.repoRoot,
+          tempRoot,
+          allowedPluginIds,
+        });
+        stagedBundledPluginsRoot = stagedRoot;
+        const runtimeHostVersion = await resolveQaRuntimeHostVersion({
+          repoRoot: params.repoRoot,
+          allowedPluginIds,
+        });
+        env = buildQaRuntimeEnv({
+          configPath,
+          gatewayToken,
+          homeDir,
+          forwardHostHome: params.forwardHostHome,
+          stateDir,
+          xdgConfigHome,
+          xdgDataHome,
+          xdgCacheHome,
+          bundledPluginsDir,
+          compatibilityHostVersion: runtimeHostVersion,
+          providerMode,
+          forwardHostHomeForClaudeCli: liveProviderIds.includes("claude-cli"),
+          claudeCliAuthMode: params.claudeCliAuthMode,
+        });
+      }
+      await fs.writeFile(configPath, `${JSON.stringify(cfg, null, 2)}\n`, {
+        encoding: "utf8",
+        mode: 0o600,
+      });
+      if (!env) {
+        throw new Error("qa gateway runtime env not initialized");
+      }
+
+      const attemptChild = spawn(
+        nodeExecPath,
+        [
+          distEntryPath,
+          "gateway",
+          "run",
+          "--port",
+          String(gatewayPort),
+          "--bind",
+          "loopback",
+          "--allow-unconfigured",
+        ],
+        {
+          cwd: runtimeCwd,
+          env,
+          detached: process.platform !== "win32",
+          stdio: ["ignore", "pipe", "pipe"],
+        },
+      );
+      attemptChild.stdout.on("data", (chunk) => {
+        const buffer = Buffer.from(chunk);
+        stdout.push(buffer);
+        stdoutLog.write(buffer);
+      });
+      attemptChild.stderr.on("data", (chunk) => {
+        const buffer = Buffer.from(chunk);
+        stderr.push(buffer);
+        stderrLog.write(buffer);
+      });
+      child = attemptChild;
+
+      try {
+        await waitForGatewayReady({
+          baseUrl,
+          logs,
+          child: attemptChild,
+          timeoutMs: 120_000,
+        });
+        const attemptRpcClient = await startQaGatewayRpcClient({
+          wsUrl,
+          token: gatewayToken,
+          logs,
+        });
+        try {
+          let rpcReady = false;
+          let lastRpcStartupError: unknown = null;
+          for (let rpcAttempt = 1; rpcAttempt <= 4; rpcAttempt += 1) {
+            try {
+              await attemptRpcClient.request("config.get", {}, { timeoutMs: 10_000 });
+              rpcReady = true;
+              break;
+            } catch (error) {
+              lastRpcStartupError = error;
+              if (rpcAttempt >= 4 || !isRetryableRpcStartupError(error)) {
+                throw error;
+              }
+              await sleep(500 * rpcAttempt);
+              await waitForGatewayReady({
+                baseUrl,
+                logs,
+                child: attemptChild,
+                timeoutMs: 15_000,
+              });
+            }
+          }
+          if (!rpcReady) {
+            throw lastRpcStartupError ?? new Error("qa gateway rpc client failed to start");
+          }
+        } catch (error) {
+          await attemptRpcClient.stop().catch(() => {});
+          throw error;
+        }
+        rpcClient = attemptRpcClient;
+        break;
+      } catch (error) {
+        const details = formatErrorMessage(error);
+        const retryable =
+          attempt < QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS &&
+          (isRetryableGatewayStartupError(`${details}\n${logs()}`) ||
+            isRetryableRpcStartupError(error));
+        if (rpcClient) {
+          await rpcClient.stop().catch(() => {});
+          rpcClient = null;
+        }
+        await stopQaGatewayChildProcessTree(attemptChild, {
+          gracefulTimeoutMs: 1_500,
+          forceTimeoutMs: 1_500,
+        });
+        child = null;
+        if (!retryable) {
+          throw error;
+        }
+        stdoutLog.write(
+          `[qa-lab] gateway child startup attempt ${attempt}/${QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS} hit a transient startup race on port ${gatewayPort}; retrying with a new port\n`,
+        );
+      }
+    }
+
+    if (!child || !cfg || !baseUrl || !wsUrl || !rpcClient || !env) {
+      throw new Error("qa gateway child failed to start");
+    }
+    let activeChild = child;
+    let activeRpcClient = rpcClient;
+    const runningEnv = env;
+
+    const spawnReplacementGatewayChild = async () => {
+      const nextChild = spawn(
+        nodeExecPath,
+        [
+          distEntryPath,
+          "gateway",
+          "run",
+          "--port",
+          String(gatewayPort),
+          "--bind",
+          "loopback",
+          "--allow-unconfigured",
+        ],
+        {
+          cwd: runtimeCwd,
+          env: runningEnv,
+          detached: process.platform !== "win32",
+          stdio: ["ignore", "pipe", "pipe"],
+        },
+      );
+      nextChild.stdout.on("data", (chunk) => {
+        const buffer = Buffer.from(chunk);
+        stdout.push(buffer);
+        stdoutLog.write(buffer);
+      });
+      nextChild.stderr.on("data", (chunk) => {
+        const buffer = Buffer.from(chunk);
+        stderr.push(buffer);
+        stderrLog.write(buffer);
+      });
+
+      try {
+        await waitForGatewayReady({
+          baseUrl,
+          logs,
+          child: nextChild,
+          timeoutMs: 120_000,
+        });
+        const nextRpcClient = await startQaGatewayRpcClient({
+          wsUrl,
+          token: gatewayToken,
+          logs,
+        });
+        try {
+          let rpcReady = false;
+          let lastRpcStartupError: unknown = null;
+          for (let rpcAttempt = 1; rpcAttempt <= 4; rpcAttempt += 1) {
+            try {
+              await nextRpcClient.request("config.get", {}, { timeoutMs: 10_000 });
+              rpcReady = true;
+              break;
+            } catch (error) {
+              lastRpcStartupError = error;
+              if (rpcAttempt >= 4 || !isRetryableRpcStartupError(error)) {
+                throw error;
+              }
+              await sleep(500 * rpcAttempt);
+              await waitForGatewayReady({
+                baseUrl,
+                logs,
+                child: nextChild,
+                timeoutMs: 15_000,
+              });
+            }
+          }
+          if (!rpcReady) {
+            throw lastRpcStartupError ?? new Error("qa gateway rpc client failed to start");
+          }
+        } catch (error) {
+          await nextRpcClient.stop().catch(() => {});
+          throw error;
+        }
+        return {
+          child: nextChild,
+          rpcClient: nextRpcClient,
+        };
+      } catch (error) {
+        await stopQaGatewayChildProcessTree(nextChild, {
+          gracefulTimeoutMs: 1_500,
+          forceTimeoutMs: 1_500,
+        });
+        throw error;
+      }
+    };
+
+    return {
+      cfg,
+      baseUrl,
+      wsUrl,
+      pid: child.pid ?? null,
+      token: gatewayToken,
+      workspaceDir,
+      tempRoot,
+      configPath,
+      runtimeEnv: runningEnv,
+      logs,
+      async restart(signal: NodeJS.Signals = "SIGUSR1") {
+        if (!activeChild.pid) {
+          throw new Error("qa gateway child has no pid");
+        }
+        process.kill(activeChild.pid, signal);
+      },
+      async restartAfterStateMutation(
+        mutateState: (context: QaGatewayChildStateMutationContext) => Promise<void>,
+      ) {
+        await activeRpcClient.stop().catch(() => {});
+        await stopQaGatewayChildProcessTree(activeChild);
+        await mutateState({
+          configPath,
+          runtimeEnv: runningEnv,
+          stateDir,
+          tempRoot,
+        });
+        const restarted = await spawnReplacementGatewayChild();
+        activeChild = restarted.child;
+        activeRpcClient = restarted.rpcClient;
+        child = activeChild;
+        rpcClient = activeRpcClient;
+      },
+      async call(
+        method: string,
+        rpcParams?: unknown,
+        opts?: { expectFinal?: boolean; timeoutMs?: number },
+      ) {
+        const timeoutMs = opts?.timeoutMs ?? 20_000;
+        let lastDetails = "";
+        for (let attempt = 1; attempt <= 3; attempt += 1) {
+          try {
+            return await activeRpcClient.request(method, rpcParams, {
+              ...opts,
+              timeoutMs,
+            });
+          } catch (error) {
+            const details = formatErrorMessage(error);
+            lastDetails = details;
+            if (attempt >= 3 || !isRetryableGatewayCallError(details)) {
+              throw new Error(`${details}${formatQaGatewayLogsForError(logs())}`, { cause: error });
+            }
+            await waitForGatewayReady({
+              baseUrl,
+              logs,
+              child: activeChild,
+              timeoutMs: Math.max(10_000, timeoutMs),
+            });
+          }
+        }
+        throw new Error(`${lastDetails}${formatQaGatewayLogsForError(logs())}`);
+      },
+      async stop(opts?: { keepTemp?: boolean; preserveToDir?: string }) {
+        await activeRpcClient.stop().catch(() => {});
+        await stopQaGatewayChildProcessTree(activeChild);
+        await closeWriteStream(stdoutLog);
+        await closeWriteStream(stderrLog);
+        if (opts?.preserveToDir && !(opts?.keepTemp ?? keepTemp)) {
+          await preserveQaGatewayDebugArtifacts({
+            preserveToDir: opts.preserveToDir,
+            stdoutLogPath,
+            stderrLogPath,
+            tempRoot,
+            repoRoot: params.repoRoot,
+          });
+        }
+        if (!(opts?.keepTemp ?? keepTemp)) {
+          await cleanupQaGatewayTempRoots({
+            tempRoot,
+            stagedBundledPluginsRoot,
+          });
+        }
+      },
+    };
+  } catch (error) {
+    await rpcClient?.stop().catch(() => {});
+    if (child) {
+      await stopQaGatewayChildProcessTree(child, {
+        gracefulTimeoutMs: 1_500,
+        forceTimeoutMs: 1_500,
+      });
+    }
+    await closeWriteStream(stdoutLog);
+    await closeWriteStream(stderrLog);
+    if (!keepTemp) {
+      await cleanupQaGatewayTempRoots({
+        tempRoot,
+        stagedBundledPluginsRoot,
+      });
+    }
+    throw new Error(
+      keepTemp
+        ? appendQaGatewayTempRoot(formatErrorMessage(error), tempRoot)
+        : formatErrorMessage(error),
+      {
+        cause: error,
+      },
+    );
+  }
+}

openclaw/extensions/qa-lab/src/gateway-log-redaction.ts

@@ -0,0 +1,30 @@
+import { QA_PROVIDER_SECRET_ENV_VARS } from "./providers/env.js";
+
+const QA_GATEWAY_DEBUG_SECRET_ENV_VARS = Object.freeze([
+  ...QA_PROVIDER_SECRET_ENV_VARS,
+  "OPENCLAW_GATEWAY_TOKEN",
+]);
+
+export function redactQaGatewayDebugText(text: string) {
+  let redacted = text;
+  for (const envVar of QA_GATEWAY_DEBUG_SECRET_ENV_VARS) {
+    const escapedEnvVar = envVar.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    redacted = redacted.replace(
+      new RegExp(`\\b(${escapedEnvVar})(\\s*[=:]\\s*)([^\\s"';,]+|"[^"]*"|'[^']*')`, "g"),
+      `$1$2<redacted>`,
+    );
+    redacted = redacted.replace(
+      new RegExp(`("${escapedEnvVar}"\\s*:\\s*)"[^"]*"`, "g"),
+      `$1"<redacted>"`,
+    );
+  }
+  return redacted
+    .replaceAll(/\bsk-ant-oat01-[A-Za-z0-9_-]+\b/g, "<redacted>")
+    .replaceAll(/\bBearer\s+[^\s"'<>]{8,}/gi, "Bearer <redacted>")
+    .replaceAll(/([?#&]token=)[^&\s]+/gi, "$1<redacted>");
+}
+
+export function formatQaGatewayLogsForError(logs: string) {
+  const sanitized = redactQaGatewayDebugText(logs).trim();
+  return sanitized.length > 0 ? `\nGateway logs:\n${sanitized}` : "";
+}

openclaw/extensions/qa-lab/src/gateway-rpc-client.test.ts

@@ -0,0 +1,168 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const gatewayRpcMock = vi.hoisted(() => {
+  const callGatewayFromCli = vi.fn(async () => ({ ok: true }));
+  return {
+    callGatewayFromCli,
+    reset() {
+      callGatewayFromCli.mockReset().mockResolvedValue({ ok: true });
+    },
+  };
+});
+
+vi.mock("./runtime-api.js", () => ({
+  callGatewayFromCli: gatewayRpcMock.callGatewayFromCli,
+}));
+
+import { startQaGatewayRpcClient } from "./gateway-rpc-client.js";
+
+describe("startQaGatewayRpcClient", () => {
+  beforeEach(() => {
+    gatewayRpcMock.reset();
+  });
+
+  it("calls the in-process gateway cli helper without mutating process.env", async () => {
+    const originalHome = process.env.OPENCLAW_HOME;
+    delete process.env.OPENCLAW_HOME;
+
+    gatewayRpcMock.callGatewayFromCli.mockImplementationOnce(async () => {
+      expect(process.env.OPENCLAW_HOME).toBeUndefined();
+      return { ok: true };
+    });
+
+    const client = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token",
+      logs: () => "qa logs",
+    });
+
+    await expect(
+      client.request("agent.run", { prompt: "hi" }, { expectFinal: true, timeoutMs: 45_000 }),
+    ).resolves.toEqual({ ok: true });
+
+    expect(gatewayRpcMock.callGatewayFromCli).toHaveBeenCalledWith(
+      "agent.run",
+      {
+        url: "ws://127.0.0.1:18789",
+        token: "qa-token",
+        timeout: "45000",
+        expectFinal: true,
+        json: true,
+      },
+      { prompt: "hi" },
+      {
+        expectFinal: true,
+        progress: false,
+      },
+    );
+
+    expect(process.env.OPENCLAW_HOME).toBe(originalHome);
+  });
+
+  it("wraps request failures with gateway logs", async () => {
+    gatewayRpcMock.callGatewayFromCli.mockRejectedValueOnce(new Error("gateway not connected"));
+    const client = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token",
+      logs: () => "OPENCLAW_GATEWAY_TOKEN=secret-token\nAuthorization: Bearer secret+/token=123456",
+    });
+
+    await expect(client.request("health")).rejects.toThrow(
+      "gateway not connected\nGateway logs:\nOPENCLAW_GATEWAY_TOKEN=<redacted>\nAuthorization: Bearer <redacted>",
+    );
+  });
+
+  it("rejects new requests after stop", async () => {
+    const client = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token",
+      logs: () => "url=http://127.0.0.1:18789/#token=abc123",
+    });
+
+    await client.stop();
+
+    await expect(client.request("health")).rejects.toThrow(
+      "gateway rpc client already stopped\nGateway logs:\nurl=http://127.0.0.1:18789/#token=<redacted>",
+    );
+  });
+
+  it("does not serialize requests across different gateway clients", async () => {
+    let resolveFirst: ((value: { ok: boolean }) => void) | null = null;
+    gatewayRpcMock.callGatewayFromCli
+      .mockImplementationOnce(
+        async () =>
+          await new Promise<{ ok: boolean }>((resolve) => {
+            resolveFirst = resolve;
+          }),
+      )
+      .mockResolvedValueOnce({ ok: true });
+
+    const firstClient = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token-a",
+      logs: () => "qa logs a",
+    });
+    const secondClient = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:28789",
+      token: "qa-token-b",
+      logs: () => "qa logs b",
+    });
+
+    const firstRequest = firstClient.request("health");
+    await Promise.resolve();
+
+    await expect(secondClient.request("status")).resolves.toEqual({ ok: true });
+    expect(gatewayRpcMock.callGatewayFromCli).toHaveBeenNthCalledWith(
+      2,
+      "status",
+      {
+        url: "ws://127.0.0.1:28789",
+        token: "qa-token-b",
+        timeout: "20000",
+        expectFinal: undefined,
+        json: true,
+      },
+      {},
+      {
+        expectFinal: undefined,
+        progress: false,
+      },
+    );
+
+    expect(resolveFirst).not.toBeNull();
+    resolveFirst!({ ok: true });
+    await expect(firstRequest).resolves.toEqual({ ok: true });
+  });
+
+  it("still serializes requests within the same gateway client", async () => {
+    let releaseFirst: (() => void) | null = null;
+    gatewayRpcMock.callGatewayFromCli
+      .mockImplementationOnce(
+        async () =>
+          await new Promise<{ ok: boolean }>((resolve) => {
+            releaseFirst = () => resolve({ ok: true });
+          }),
+      )
+      .mockResolvedValueOnce({ ok: true });
+
+    const client = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token",
+      logs: () => "qa logs",
+    });
+
+    const firstRequest = client.request("health");
+    await Promise.resolve();
+    const secondRequest = client.request("status");
+    await Promise.resolve();
+
+    expect(gatewayRpcMock.callGatewayFromCli).toHaveBeenCalledTimes(1);
+
+    expect(releaseFirst).not.toBeNull();
+    releaseFirst!();
+
+    await expect(firstRequest).resolves.toEqual({ ok: true });
+    await expect(secondRequest).resolves.toEqual({ ok: true });
+    expect(gatewayRpcMock.callGatewayFromCli).toHaveBeenCalledTimes(2);
+  });
+});

openclaw/extensions/qa-lab/src/gateway-rpc-client.ts

@@ -0,0 +1,73 @@
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { formatQaGatewayLogsForError } from "./gateway-log-redaction.js";
+import { callGatewayFromCli } from "./runtime-api.js";
+
+type QaGatewayRpcRequestOptions = {
+  expectFinal?: boolean;
+  timeoutMs?: number;
+};
+
+export type QaGatewayRpcClient = {
+  request(method: string, rpcParams?: unknown, opts?: QaGatewayRpcRequestOptions): Promise<unknown>;
+  stop(): Promise<void>;
+};
+
+function formatQaGatewayRpcError(error: unknown, logs: () => string) {
+  const details = formatErrorMessage(error);
+  return new Error(`${details}${formatQaGatewayLogsForError(logs())}`);
+}
+
+function runQueuedQaGatewayRpc<T>(queue: Promise<void>, task: () => Promise<T>) {
+  const run = queue.then(task, task);
+  const nextQueue = run.then(
+    () => undefined,
+    () => undefined,
+  );
+  return { run, nextQueue };
+}
+
+export async function startQaGatewayRpcClient(params: {
+  wsUrl: string;
+  token: string;
+  logs: () => string;
+}): Promise<QaGatewayRpcClient> {
+  const wrapError = (error: unknown) => formatQaGatewayRpcError(error, params.logs);
+  let stopped = false;
+  let queue = Promise.resolve();
+
+  return {
+    async request(method, rpcParams, opts) {
+      if (stopped) {
+        throw wrapError(new Error("gateway rpc client already stopped"));
+      }
+      try {
+        const { run, nextQueue } = runQueuedQaGatewayRpc(
+          queue,
+          async () =>
+            await callGatewayFromCli(
+              method,
+              {
+                url: params.wsUrl,
+                token: params.token,
+                timeout: String(opts?.timeoutMs ?? 20_000),
+                expectFinal: opts?.expectFinal,
+                json: true,
+              },
+              rpcParams ?? {},
+              {
+                expectFinal: opts?.expectFinal,
+                progress: false,
+              },
+            ),
+        );
+        queue = nextQueue;
+        return await run;
+      } catch (error) {
+        throw wrapError(error);
+      }
+    },
+    async stop() {
+      stopped = true;
+    },
+  };
+}

openclaw/extensions/qa-lab/src/harness-runtime.ts

@@ -0,0 +1,75 @@
+import type { PluginRuntime } from "openclaw/plugin-sdk/runtime-store";
+
+type SessionRecord = {
+  sessionKey: string;
+  body: string;
+};
+
+export function createQaRunnerRuntime(): PluginRuntime {
+  const sessions = new Map<string, SessionRecord>();
+  return {
+    channel: {
+      routing: {
+        resolveAgentRoute({
+          accountId,
+          peer,
+        }: {
+          accountId?: string | null;
+          peer?: { kind?: string; id?: string } | null;
+        }) {
+          return {
+            agentId: "qa-agent",
+            accountId: accountId ?? "default",
+            sessionKey: `qa-agent:${peer?.kind ?? "direct"}:${peer?.id ?? "default"}`,
+            mainSessionKey: "qa-agent:main",
+            lastRoutePolicy: "session",
+            matchedBy: "default",
+            channel: "qa-channel",
+          };
+        },
+      },
+      session: {
+        resolveStorePath(_store: string | undefined, { agentId }: { agentId: string }) {
+          return agentId;
+        },
+        readSessionUpdatedAt({ sessionKey }: { sessionKey: string }) {
+          return sessions.has(sessionKey) ? Date.now() : undefined;
+        },
+        recordInboundSession({
+          sessionKey,
+          ctx,
+        }: {
+          sessionKey: string;
+          ctx: { BodyForAgent?: string; Body?: string };
+        }) {
+          sessions.set(sessionKey, {
+            sessionKey,
+            body: ctx.BodyForAgent ?? ctx.Body ?? "",
+          });
+        },
+      },
+      reply: {
+        resolveEnvelopeFormatOptions() {
+          return {};
+        },
+        formatAgentEnvelope({ body }: { body: string }) {
+          return body;
+        },
+        finalizeInboundContext(ctx: Record<string, unknown>) {
+          return ctx as typeof ctx & { CommandAuthorized: boolean };
+        },
+        async dispatchReplyWithBufferedBlockDispatcher({
+          ctx,
+          dispatcherOptions,
+        }: {
+          ctx: { BodyForAgent?: string; Body?: string };
+          dispatcherOptions: { deliver: (payload: { text: string }) => Promise<void> };
+        }) {
+          await dispatcherOptions.deliver({
+            text: `qa-echo: ${ctx.BodyForAgent ?? ctx.Body ?? ""}`,
+          });
+        },
+      },
+    },
+  } as unknown as PluginRuntime;
+}

openclaw/extensions/qa-lab/src/lab-server-capture.test.ts

@@ -0,0 +1,70 @@
+import { createServer } from "node:http";
+import { afterEach, describe, expect, it } from "vitest";
+import { mapCaptureEventForQa, probeTcpReachability } from "./lab-server-capture.js";
+
+const cleanups: Array<() => Promise<void>> = [];
+
+afterEach(async () => {
+  while (cleanups.length > 0) {
+    await cleanups.pop()?.();
+  }
+});
+
+describe("qa-lab server capture helpers", () => {
+  it("maps capture rows into QA-friendly fields", () => {
+    expect(
+      mapCaptureEventForQa({
+        flowId: "flow-1",
+        dataText: '{"hello":"world"}',
+        metaJson: JSON.stringify({
+          provider: "openai",
+          api: "responses",
+          model: "gpt-5.4",
+          captureOrigin: "shared-fetch",
+        }),
+      }),
+    ).toEqual(
+      expect.objectContaining({
+        flowId: "flow-1",
+        payloadPreview: '{"hello":"world"}',
+        provider: "openai",
+        api: "responses",
+        model: "gpt-5.4",
+        captureOrigin: "shared-fetch",
+      }),
+    );
+  });
+
+  it("probes tcp reachability for reachable and unreachable targets", async () => {
+    const server = createServer((_req, res) => {
+      res.writeHead(200);
+      res.end("ok");
+    });
+    await new Promise<void>((resolve, reject) => {
+      server.once("error", reject);
+      server.listen(0, "127.0.0.1", () => resolve());
+    });
+    cleanups.push(
+      async () =>
+        await new Promise<void>((resolve, reject) =>
+          server.close((error) => (error ? reject(error) : resolve())),
+        ),
+    );
+
+    const address = server.address();
+    if (!address || typeof address === "string") {
+      throw new Error("expected tcp probe address");
+    }
+
+    await expect(probeTcpReachability(`http://127.0.0.1:${address.port}`)).resolves.toEqual(
+      expect.objectContaining({
+        ok: true,
+      }),
+    );
+    await expect(probeTcpReachability("http://127.0.0.1:9", 50)).resolves.toEqual(
+      expect.objectContaining({
+        ok: false,
+      }),
+    );
+  });
+});

openclaw/extensions/qa-lab/src/lab-server-capture.ts

@@ -0,0 +1,127 @@
+import net from "node:net";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+
+const CAPTURE_QUERY_PRESETS = new Set([
+  "double-sends",
+  "retry-storms",
+  "cache-busting",
+  "ws-duplicate-frames",
+  "missing-ack",
+  "error-bursts",
+]);
+
+export type QaStartupProbeStatus = {
+  label: string;
+  url: string;
+  ok: boolean;
+  error?: string;
+};
+
+export function isCaptureQueryPreset(
+  value: string,
+): value is Parameters<
+  ReturnType<
+    typeof import("openclaw/plugin-sdk/proxy-capture").getDebugProxyCaptureStore
+  >["queryPreset"]
+>[0] {
+  return CAPTURE_QUERY_PRESETS.has(value);
+}
+
+function parseCaptureMeta(metaJson: unknown): Record<string, unknown> | null {
+  if (typeof metaJson !== "string" || metaJson.trim().length === 0) {
+    return null;
+  }
+  try {
+    const parsed = JSON.parse(metaJson) as unknown;
+    return parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : null;
+  } catch {
+    return null;
+  }
+}
+
+function readCaptureMetaString(
+  meta: Record<string, unknown> | null,
+  key: string,
+): string | undefined {
+  const value = meta?.[key];
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+}
+
+export function mapCaptureEventForQa(row: Record<string, unknown>) {
+  const meta = parseCaptureMeta(row.metaJson);
+  return {
+    ...row,
+    payloadPreview: typeof row.dataText === "string" ? row.dataText : undefined,
+    provider: readCaptureMetaString(meta, "provider"),
+    api: readCaptureMetaString(meta, "api"),
+    model: readCaptureMetaString(meta, "model"),
+    captureOrigin: readCaptureMetaString(meta, "captureOrigin"),
+  };
+}
+
+function defaultPortForProtocol(protocol: string): number {
+  if (protocol === "https:") {
+    return 443;
+  }
+  if (protocol === "http:") {
+    return 80;
+  }
+  return 0;
+}
+
+export async function probeTcpReachability(
+  rawUrl: string,
+  timeoutMs = 700,
+): Promise<QaStartupProbeStatus> {
+  let parsed: URL;
+  try {
+    parsed = new URL(rawUrl);
+  } catch {
+    return {
+      label: rawUrl,
+      url: rawUrl,
+      ok: false,
+      error: "invalid url",
+    };
+  }
+  const host = parsed.hostname;
+  const port = parsed.port ? Number(parsed.port) : defaultPortForProtocol(parsed.protocol);
+  if (!host || !Number.isFinite(port) || port <= 0) {
+    return {
+      label: parsed.origin,
+      url: parsed.toString(),
+      ok: false,
+      error: "missing host or port",
+    };
+  }
+  try {
+    await new Promise<void>((resolve, reject) => {
+      const socket = net.createConnection({ host, port });
+      const onError = (error: Error) => {
+        socket.destroy();
+        reject(error);
+      };
+      socket.setTimeout(timeoutMs, () => {
+        socket.destroy(new Error("timeout"));
+      });
+      socket.once("connect", () => {
+        socket.end();
+        resolve();
+      });
+      socket.once("error", onError);
+      socket.once("timeout", () => onError(new Error("timeout")));
+    });
+    return {
+      label: parsed.host,
+      url: parsed.toString(),
+      ok: true,
+    };
+  } catch (error) {
+    return {
+      label: parsed.host,
+      url: parsed.toString(),
+      ok: false,
+      error: formatErrorMessage(error),
+    };
+  }
+}

openclaw/extensions/qa-lab/src/lab-server-ui.test.ts

@@ -0,0 +1,91 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  detectContentType,
+  missingUiHtml,
+  resolveUiAssetVersion,
+  tryResolveUiAsset,
+} from "./lab-server-ui.js";
+
+const cleanups: Array<() => Promise<void>> = [];
+
+afterEach(async () => {
+  while (cleanups.length > 0) {
+    await cleanups.pop()?.();
+  }
+});
+
+describe("qa-lab server ui helpers", () => {
+  it("detects basic UI asset content types", () => {
+    expect(detectContentType("index.html")).toBe("text/html; charset=utf-8");
+    expect(detectContentType("styles.css")).toBe("text/css; charset=utf-8");
+    expect(detectContentType("main.js")).toBe("text/javascript; charset=utf-8");
+    expect(detectContentType("icon.svg")).toBe("image/svg+xml");
+  });
+
+  it("renders the missing-ui placeholder html", () => {
+    expect(missingUiHtml()).toContain("QA Lab UI not built");
+    expect(missingUiHtml()).toContain("pnpm qa:lab:build");
+  });
+
+  it("hashes built UI assets and changes when bundle contents change", async () => {
+    const uiDistDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-ui-dist-"));
+    cleanups.push(async () => {
+      await rm(uiDistDir, { recursive: true, force: true });
+    });
+    await writeFile(
+      path.join(uiDistDir, "index.html"),
+      "<!doctype html><html><head><title>QA Lab</title></head><body><div id='app'></div></body></html>",
+      "utf8",
+    );
+
+    const version1 = resolveUiAssetVersion(uiDistDir);
+    expect(version1).toMatch(/^[0-9a-f]{12}$/);
+
+    await writeFile(
+      path.join(uiDistDir, "index.html"),
+      "<!doctype html><html><head><title>QA Lab Updated</title></head><body><div id='app'></div></body></html>",
+      "utf8",
+    );
+
+    const version2 = resolveUiAssetVersion(uiDistDir);
+    expect(version2).toMatch(/^[0-9a-f]{12}$/);
+    expect(version2).not.toBe(version1);
+  });
+
+  it("never resolves sibling files outside the UI dist root", async () => {
+    const rootDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-ui-boundary-"));
+    cleanups.push(async () => {
+      await rm(rootDir, { recursive: true, force: true });
+    });
+    const uiDistDir = path.join(rootDir, "dist");
+    const siblingDir = path.join(rootDir, "dist-other");
+    await mkdir(uiDistDir, { recursive: true });
+    await mkdir(siblingDir, { recursive: true });
+    await writeFile(
+      path.join(uiDistDir, "index.html"),
+      "<!doctype html><html><body>bundle-root</body></html>",
+      "utf8",
+    );
+    await writeFile(path.join(siblingDir, "secret.txt"), "sibling-secret", "utf8");
+
+    expect(tryResolveUiAsset("/", uiDistDir, rootDir)).toBe(path.join(uiDistDir, "index.html"));
+    expect(tryResolveUiAsset("/../dist-other/secret.txt", uiDistDir, rootDir)).toBeNull();
+  });
+
+  it("rejects malformed percent-encoded UI asset paths", async () => {
+    const uiDistDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-ui-malformed-"));
+    cleanups.push(async () => {
+      await rm(uiDistDir, { recursive: true, force: true });
+    });
+    await writeFile(
+      path.join(uiDistDir, "index.html"),
+      "<!doctype html><html><body>bundle-root</body></html>",
+      "utf8",
+    );
+
+    expect(tryResolveUiAsset("/%E0%A4", uiDistDir, uiDistDir)).toBeNull();
+  });
+});

openclaw/extensions/qa-lab/src/lab-server-ui.ts

@@ -0,0 +1,288 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import { request as httpRequest, type IncomingMessage, type ServerResponse } from "node:http";
+import { request as httpsRequest } from "node:https";
+import net from "node:net";
+import path from "node:path";
+import type { Duplex } from "node:stream";
+import tls from "node:tls";
+import { fileURLToPath } from "node:url";
+import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
+import { writeError } from "./bus-server.js";
+
+export function detectContentType(filePath: string): string {
+  if (filePath.endsWith(".css")) {
+    return "text/css; charset=utf-8";
+  }
+  if (filePath.endsWith(".js")) {
+    return "text/javascript; charset=utf-8";
+  }
+  if (filePath.endsWith(".json")) {
+    return "application/json; charset=utf-8";
+  }
+  if (filePath.endsWith(".svg")) {
+    return "image/svg+xml";
+  }
+  return "text/html; charset=utf-8";
+}
+
+export function missingUiHtml() {
+  return `<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>QA Lab UI Missing</title>
+    <style>
+      body { font-family: ui-sans-serif, system-ui, sans-serif; background: #0f1115; color: #f5f7fb; margin: 0; display: grid; place-items: center; min-height: 100vh; }
+      main { max-width: 42rem; padding: 2rem; background: #171b22; border: 1px solid #283140; border-radius: 18px; box-shadow: 0 30px 80px rgba(0,0,0,.35); }
+      code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; color: #9ee8d8; }
+      h1 { margin-top: 0; }
+    </style>
+  </head>
+  <body>
+    <main>
+      <h1>QA Lab UI not built</h1>
+      <p>Build the private debugger bundle, then reload this page.</p>
+      <p><code>pnpm qa:lab:build</code></p>
+    </main>
+  </body>
+</html>`;
+}
+
+export function resolveUiDistDir(overrideDir?: string | null, repoRoot = process.cwd()) {
+  if (overrideDir?.trim()) {
+    return overrideDir;
+  }
+  const candidates = [
+    path.resolve(repoRoot, "extensions/qa-lab/web/dist"),
+    path.resolve(repoRoot, "dist/extensions/qa-lab/web/dist"),
+    fileURLToPath(new URL("../web/dist", import.meta.url)),
+  ];
+  return (
+    candidates.find((candidate) => {
+      if (!fs.existsSync(candidate)) {
+        return false;
+      }
+      const indexPath = path.join(candidate, "index.html");
+      return fs.existsSync(indexPath) && fs.statSync(indexPath).isFile();
+    }) ?? candidates[0]
+  );
+}
+
+function listUiAssetFiles(rootDir: string, currentDir = rootDir): string[] {
+  const entries = fs
+    .readdirSync(currentDir, { withFileTypes: true })
+    .toSorted((left, right) => left.name.localeCompare(right.name));
+  const files: string[] = [];
+  for (const entry of entries) {
+    const resolved = path.join(currentDir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...listUiAssetFiles(rootDir, resolved));
+      continue;
+    }
+    if (!entry.isFile()) {
+      continue;
+    }
+    files.push(path.relative(rootDir, resolved));
+  }
+  return files;
+}
+
+export function resolveUiAssetVersion(overrideDir?: string | null): string | null {
+  try {
+    const distDir = resolveUiDistDir(overrideDir);
+    const indexPath = path.join(distDir, "index.html");
+    if (!fs.existsSync(indexPath) || !fs.statSync(indexPath).isFile()) {
+      return null;
+    }
+    const hash = createHash("sha1");
+    for (const relativeFile of listUiAssetFiles(distDir)) {
+      hash.update(relativeFile);
+      hash.update("\0");
+      hash.update(fs.readFileSync(path.join(distDir, relativeFile)));
+      hash.update("\0");
+    }
+    return hash.digest("hex").slice(0, 12);
+  } catch {
+    return null;
+  }
+}
+
+export function resolveAdvertisedBaseUrl(params: {
+  bindHost?: string;
+  bindPort: number;
+  advertiseHost?: string;
+  advertisePort?: number;
+}) {
+  const advertisedHost =
+    params.advertiseHost?.trim() ||
+    (params.bindHost && params.bindHost !== "0.0.0.0" ? params.bindHost : "127.0.0.1");
+  const advertisedPort =
+    typeof params.advertisePort === "number" && Number.isFinite(params.advertisePort)
+      ? params.advertisePort
+      : params.bindPort;
+  return `http://${advertisedHost}:${advertisedPort}`;
+}
+
+export function isControlUiProxyPath(pathname: string) {
+  return pathname === "/control-ui" || pathname.startsWith("/control-ui/");
+}
+
+function rewriteControlUiProxyPath(pathname: string, search: string) {
+  const stripped = pathname === "/control-ui" ? "/" : pathname.slice("/control-ui".length) || "/";
+  return `${stripped}${search}`;
+}
+
+function rewriteEmbeddedControlUiHeaders(
+  headers: IncomingMessage["headers"],
+): Record<string, string | string[] | number | undefined> {
+  const rewritten: Record<string, string | string[] | number | undefined> = { ...headers };
+  delete rewritten["x-frame-options"];
+
+  const csp = headers["content-security-policy"];
+  if (typeof csp === "string") {
+    rewritten["content-security-policy"] = csp.includes("frame-ancestors")
+      ? csp.replace(/frame-ancestors\s+[^;]+/i, "frame-ancestors 'self'")
+      : `${csp}; frame-ancestors 'self'`;
+  }
+
+  return rewritten;
+}
+
+export async function proxyHttpRequest(params: {
+  req: IncomingMessage;
+  res: ServerResponse;
+  target: URL;
+  pathname: string;
+  search: string;
+}) {
+  const client = params.target.protocol === "https:" ? httpsRequest : httpRequest;
+  const upstreamReq = client(
+    {
+      protocol: params.target.protocol,
+      hostname: params.target.hostname,
+      port: params.target.port || (params.target.protocol === "https:" ? 443 : 80),
+      method: params.req.method,
+      path: rewriteControlUiProxyPath(params.pathname, params.search),
+      headers: {
+        ...params.req.headers,
+        host: params.target.host,
+      },
+    },
+    (upstreamRes) => {
+      params.res.writeHead(
+        upstreamRes.statusCode ?? 502,
+        rewriteEmbeddedControlUiHeaders(upstreamRes.headers),
+      );
+      upstreamRes.pipe(params.res);
+    },
+  );
+
+  upstreamReq.on("error", (error) => {
+    if (!params.res.headersSent) {
+      writeError(params.res, 502, error);
+      return;
+    }
+    params.res.destroy(error);
+  });
+
+  if (params.req.method === "GET" || params.req.method === "HEAD") {
+    upstreamReq.end();
+    return;
+  }
+  params.req.pipe(upstreamReq);
+}
+
+export function proxyUpgradeRequest(params: {
+  req: IncomingMessage;
+  socket: Duplex;
+  head: Buffer;
+  target: URL;
+}) {
+  const requestUrl = new URL(params.req.url ?? "/", "http://127.0.0.1");
+  const port = Number(params.target.port || (params.target.protocol === "https:" ? 443 : 80));
+  const upstream =
+    params.target.protocol === "https:"
+      ? tls.connect({
+          host: params.target.hostname,
+          port,
+          servername: params.target.hostname,
+        })
+      : net.connect({
+          host: params.target.hostname,
+          port,
+        });
+
+  const headerLines: string[] = [];
+  for (let index = 0; index < params.req.rawHeaders.length; index += 2) {
+    const name = params.req.rawHeaders[index];
+    const value = params.req.rawHeaders[index + 1] ?? "";
+    if (normalizeLowercaseStringOrEmpty(name) === "host") {
+      continue;
+    }
+    headerLines.push(`${name}: ${value}`);
+  }
+
+  upstream.once("connect", () => {
+    const requestText = [
+      `${params.req.method ?? "GET"} ${rewriteControlUiProxyPath(requestUrl.pathname, requestUrl.search)} HTTP/${params.req.httpVersion}`,
+      `Host: ${params.target.host}`,
+      ...headerLines,
+      "",
+      "",
+    ].join("\r\n");
+    upstream.write(requestText);
+    if (params.head.length > 0) {
+      upstream.write(params.head);
+    }
+    upstream.pipe(params.socket);
+    params.socket.pipe(upstream);
+  });
+
+  const closeBoth = () => {
+    if (!params.socket.destroyed) {
+      params.socket.destroy();
+    }
+    if (!upstream.destroyed) {
+      upstream.destroy();
+    }
+  };
+
+  upstream.on("error", () => {
+    if (!params.socket.destroyed) {
+      params.socket.write("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n");
+    }
+    closeBoth();
+  });
+  params.socket.on("error", closeBoth);
+  params.socket.on("close", closeBoth);
+}
+
+export function tryResolveUiAsset(
+  pathname: string,
+  overrideDir?: string | null,
+  repoRoot = process.cwd(),
+): string | null {
+  const distDir = resolveUiDistDir(overrideDir, repoRoot);
+  if (!fs.existsSync(distDir)) {
+    return null;
+  }
+  const safePath = pathname === "/" ? "/index.html" : pathname;
+  let decoded: string;
+  try {
+    decoded = decodeURIComponent(safePath);
+  } catch {
+    return null;
+  }
+  const candidate = path.resolve(distDir, `.${decoded.startsWith("/") ? decoded : `/${decoded}`}`);
+  const relative = path.relative(distDir, candidate);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    return null;
+  }
+  if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) {
+    return candidate;
+  }
+  const fallback = path.join(distDir, "index.html");
+  return fs.existsSync(fallback) ? fallback : null;
+}

openclaw/extensions/qa-lab/src/lab-server.test.ts

@@ -0,0 +1,740 @@
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { createServer } from "node:http";
+import os from "node:os";
+import path from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { startQaLabServer } from "./lab-server.js";
+
+vi.mock("openclaw/plugin-sdk/qa-channel", async () => await import("../../qa-channel/api.js"));
+
+const cleanups: Array<() => Promise<void>> = [];
+
+afterEach(async () => {
+  while (cleanups.length > 0) {
+    await cleanups.pop()?.();
+  }
+});
+
+function isRetryableLocalFetchError(error: unknown) {
+  if (!(error instanceof TypeError)) {
+    return false;
+  }
+  const cause = (error as TypeError & { cause?: unknown }).cause;
+  if (!cause || typeof cause !== "object") {
+    return false;
+  }
+  const code = "code" in cause ? (cause as { code?: unknown }).code : undefined;
+  return code === "ECONNRESET" || code === "UND_ERR_SOCKET";
+}
+
+async function fetchWithRetry(input: string, init?: RequestInit, attempts = 3) {
+  const method = init?.method?.toUpperCase() ?? "GET";
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      return await fetch(input, init);
+    } catch (error) {
+      lastError = error;
+      if ((method !== "GET" && method !== "HEAD") || !isRetryableLocalFetchError(error)) {
+        throw error;
+      }
+      if (attempt === attempts) {
+        throw error;
+      }
+      await sleep(10);
+    }
+  }
+  throw lastError;
+}
+
+async function waitForRunnerCatalog(baseUrl: string, timeoutMs = 5_000) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    const response = await fetchWithRetry(`${baseUrl}/api/bootstrap`);
+    const bootstrap = (await response.json()) as {
+      runnerCatalog: {
+        status: "loading" | "ready" | "failed";
+        real: Array<{ key: string; name: string }>;
+      };
+    };
+    if (bootstrap.runnerCatalog.status !== "loading") {
+      return bootstrap.runnerCatalog;
+    }
+    await sleep(10);
+  }
+  throw new Error("runner catalog stayed loading");
+}
+
+async function waitForFile(filePath: string, timeoutMs = 5_000) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    try {
+      return await readFile(filePath, "utf8");
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+        throw error;
+      }
+      await sleep(10);
+    }
+  }
+  throw new Error(`file did not appear: ${filePath}`);
+}
+
+async function createQaLabRepoRootFixture(params?: {
+  uiHtml?: string;
+  models?: Array<{
+    key: string;
+    name: string;
+    input?: string;
+    available?: boolean;
+    missing?: boolean;
+  }>;
+}) {
+  const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-lab-repo-root-"));
+  cleanups.push(async () => {
+    await rm(repoRoot, { recursive: true, force: true });
+  });
+  await mkdir(path.join(repoRoot, "dist"), { recursive: true });
+  await mkdir(path.join(repoRoot, "extensions/qa-lab/web/dist"), { recursive: true });
+  const models =
+    params?.models?.map((model) => ({
+      key: model.key,
+      name: model.name,
+      input: model.input ?? model.key,
+      available: model.available ?? true,
+      missing: model.missing ?? false,
+    })) ?? [];
+  await writeFile(
+    path.join(repoRoot, "dist/index.js"),
+    `process.stdout.write(${JSON.stringify(JSON.stringify({ models }))});\n`,
+    "utf8",
+  );
+  await writeFile(
+    path.join(repoRoot, "extensions/qa-lab/web/dist/index.html"),
+    params?.uiHtml ?? "<!doctype html><html><body>qa lab fixture</body></html>",
+    "utf8",
+  );
+  return repoRoot;
+}
+
+describe("qa-lab server", () => {
+  it("serves bootstrap state and writes a self-check report", async () => {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-test-"));
+    cleanups.push(async () => {
+      await rm(tempDir, { recursive: true, force: true });
+    });
+    const outputPath = path.join(tempDir, "self-check.md");
+    const repoRoot = await createQaLabRepoRootFixture();
+
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      outputPath,
+      repoRoot,
+      controlUiUrl: "http://127.0.0.1:18789/",
+      controlUiToken: "qa-token",
+    });
+    cleanups.push(async () => {
+      await lab.stop();
+    });
+
+    const bootstrapResponse = await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`);
+    expect(bootstrapResponse.status).toBe(200);
+    const bootstrap = (await bootstrapResponse.json()) as {
+      controlUiUrl: string | null;
+      controlUiEmbeddedUrl: string | null;
+      kickoffTask: string;
+      scenarios: Array<{ id: string; title: string }>;
+      defaults: { conversationId: string; senderId: string };
+      runner: { status: string; selection: { providerMode: string; scenarioIds: string[] } };
+    };
+    expect(bootstrap.defaults.conversationId).toBe("qa-operator");
+    expect(bootstrap.defaults.senderId).toBe("qa-operator");
+    expect(bootstrap.controlUiUrl).toBe("http://127.0.0.1:18789/");
+    expect(bootstrap.controlUiEmbeddedUrl).toBe("http://127.0.0.1:18789/#token=qa-token");
+    expect(bootstrap.kickoffTask).toContain("Lobster Invaders");
+    expect(bootstrap.scenarios.length).toBeGreaterThanOrEqual(10);
+    expect(bootstrap.scenarios.some((scenario) => scenario.id === "dm-chat-baseline")).toBe(true);
+    expect(bootstrap.runner.status).toBe("idle");
+    expect(bootstrap.runner.selection.providerMode).toBe("live-frontier");
+    expect(bootstrap.runner.selection.scenarioIds).toHaveLength(bootstrap.scenarios.length);
+
+    const messageResponse = await fetch(`${lab.baseUrl}/api/inbound/message`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+      },
+      body: JSON.stringify({
+        conversation: { id: "bob", kind: "direct" },
+        senderId: "bob",
+        senderName: "Bob",
+        text: "hello from test",
+      }),
+    });
+    expect(messageResponse.status).toBe(200);
+
+    const stateResponse = await fetchWithRetry(`${lab.baseUrl}/api/state`);
+    expect(stateResponse.status).toBe(200);
+    const snapshot = (await stateResponse.json()) as {
+      messages: Array<{ direction: string; text: string }>;
+    };
+    expect(snapshot.messages.some((message) => message.text === "hello from test")).toBe(true);
+
+    const result = await lab.runSelfCheck();
+    expect(result.scenarioResult.status).toBe("pass");
+    const markdown = await readFile(outputPath, "utf8");
+    expect(markdown).toContain("Synthetic Slack-class roundtrip");
+    expect(markdown).toContain("- Status: pass");
+  });
+
+  it("anchors direct self-check runs under the explicit repo root by default", async () => {
+    const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-lab-self-check-root-"));
+    cleanups.push(async () => {
+      await rm(repoRoot, { recursive: true, force: true });
+    });
+
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      repoRoot,
+    });
+    cleanups.push(async () => {
+      await lab.stop();
+    });
+
+    const result = await lab.runSelfCheck();
+    expect(result.outputPath).toBe(path.join(repoRoot, ".artifacts", "qa-e2e", "self-check.md"));
+    expect(await readFile(result.outputPath, "utf8")).toContain("Synthetic Slack-class roundtrip");
+  });
+
+  it("injects the kickoff task on demand and on startup", async () => {
+    const autoKickoffLab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      sendKickoffOnStart: true,
+    });
+    cleanups.push(async () => {
+      await autoKickoffLab.stop();
+    });
+
+    const autoSnapshot = (await (
+      await fetchWithRetry(`${autoKickoffLab.baseUrl}/api/state`)
+    ).json()) as {
+      messages: Array<{ text: string }>;
+    };
+    expect(autoSnapshot.messages.some((message) => message.text.includes("QA mission:"))).toBe(
+      true,
+    );
+
+    const manualLab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+    });
+    cleanups.push(async () => {
+      await manualLab.stop();
+    });
+
+    const kickoffResponse = await fetch(`${manualLab.baseUrl}/api/kickoff`, {
+      method: "POST",
+    });
+    expect(kickoffResponse.status).toBe(200);
+
+    const manualSnapshot = (await (
+      await fetchWithRetry(`${manualLab.baseUrl}/api/state`)
+    ).json()) as {
+      messages: Array<{ text: string }>;
+    };
+    expect(
+      manualSnapshot.messages.some((message) => message.text.includes("Lobster Invaders")),
+    ).toBe(true);
+  });
+
+  it("proxies control-ui paths through /control-ui", async () => {
+    const upstream = createServer((req, res) => {
+      if ((req.url ?? "/") === "/healthz") {
+        res.writeHead(200, { "content-type": "application/json" });
+        res.end(JSON.stringify({ ok: true, status: "live" }));
+        return;
+      }
+      res.writeHead(200, {
+        "content-type": "text/html; charset=utf-8",
+        "x-frame-options": "DENY",
+        "content-security-policy": "default-src 'self'; frame-ancestors 'none';",
+      });
+      res.end("<!doctype html><title>control-ui</title><h1>Control UI</h1>");
+    });
+    await new Promise<void>((resolve, reject) => {
+      upstream.once("error", reject);
+      upstream.listen(0, "127.0.0.1", () => resolve());
+    });
+    cleanups.push(
+      async () =>
+        await new Promise<void>((resolve, reject) =>
+          upstream.close((error) => (error ? reject(error) : resolve())),
+        ),
+    );
+
+    const address = upstream.address();
+    if (!address || typeof address === "string") {
+      throw new Error("expected upstream address");
+    }
+
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      advertiseHost: "127.0.0.1",
+      advertisePort: 43124,
+      controlUiProxyTarget: `http://127.0.0.1:${address.port}/`,
+      controlUiToken: "proxy-token",
+    });
+    cleanups.push(async () => {
+      await lab.stop();
+    });
+
+    const bootstrap = (await (await fetchWithRetry(`${lab.listenUrl}/api/bootstrap`)).json()) as {
+      controlUiUrl: string | null;
+      controlUiEmbeddedUrl: string | null;
+    };
+    expect(bootstrap.controlUiUrl).toBe("http://127.0.0.1:43124/control-ui/");
+    expect(bootstrap.controlUiEmbeddedUrl).toBe(
+      "http://127.0.0.1:43124/control-ui/#token=proxy-token",
+    );
+
+    const healthResponse = await fetchWithRetry(`${lab.listenUrl}/control-ui/healthz`);
+    expect(healthResponse.status).toBe(200);
+    expect(await healthResponse.json()).toEqual({ ok: true, status: "live" });
+
+    const rootResponse = await fetchWithRetry(`${lab.listenUrl}/control-ui/`);
+    expect(rootResponse.status).toBe(200);
+    expect(rootResponse.headers.get("x-frame-options")).toBeNull();
+    expect(rootResponse.headers.get("content-security-policy")).toContain("frame-ancestors 'self'");
+    expect(await rootResponse.text()).toContain("Control UI");
+  });
+
+  it("serves the built QA UI bundle when available", async () => {
+    const uiDistDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-ui-dist-"));
+    cleanups.push(async () => {
+      await rm(uiDistDir, { recursive: true, force: true });
+    });
+    await writeFile(
+      path.join(uiDistDir, "index.html"),
+      "<!doctype html><html><head><title>QA Lab</title></head><body><div id='app'></div></body></html>",
+      "utf8",
+    );
+
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      uiDistDir,
+    });
+    cleanups.push(async () => {
+      await lab.stop();
+    });
+
+    const rootResponse = await fetchWithRetry(`${lab.baseUrl}/`);
+    expect(rootResponse.status).toBe(200);
+    const html = await rootResponse.text();
+    expect(html).not.toContain("QA Lab UI not built");
+    expect(html).toContain("<title>");
+  });
+
+  it("uses the explicit repo root for ui assets and runner model discovery", async () => {
+    const repoRoot = await createQaLabRepoRootFixture({
+      models: [
+        {
+          key: "anthropic/qa-temp-model",
+          name: "QA Temp Model",
+        },
+      ],
+      uiHtml:
+        "<!doctype html><html><head><title>Temp QA Lab UI</title></head><body>repo-root-ui</body></html>",
+    });
+
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      repoRoot,
+    });
+    cleanups.push(async () => {
+      await lab.stop();
+    });
+
+    const rootResponse = await fetchWithRetry(`${lab.baseUrl}/`);
+    expect(rootResponse.status).toBe(200);
+    expect(await rootResponse.text()).toContain("repo-root-ui");
+
+    const runnerCatalog = await waitForRunnerCatalog(lab.baseUrl);
+    expect(runnerCatalog.status).toBe("ready");
+    expect(runnerCatalog.real).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          key: "anthropic/qa-temp-model",
+          name: "QA Temp Model",
+        }),
+      ]),
+    );
+  });
+
+  it("does not eagerly load the runner model catalog before bootstrap is requested", async () => {
+    const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-lab-lazy-catalog-"));
+    cleanups.push(async () => {
+      await rm(repoRoot, { recursive: true, force: true });
+    });
+    const markerPath = path.join(repoRoot, "runner-catalog-hit.txt");
+
+    await mkdir(path.join(repoRoot, "dist"), { recursive: true });
+    await mkdir(path.join(repoRoot, "extensions/qa-lab/web/dist"), { recursive: true });
+    await writeFile(
+      path.join(repoRoot, "dist/index.js"),
+      [
+        'const fs = require("node:fs");',
+        `fs.writeFileSync(${JSON.stringify(markerPath)}, process.argv.slice(2).join(" "), "utf8");`,
+        "process.stdout.write(JSON.stringify({",
+        "  models: [{",
+        '    key: "openai/gpt-5.4",',
+        '    name: "GPT-5.4",',
+        '    input: "openai/gpt-5.4",',
+        "    available: true,",
+        "    missing: false,",
+        "  }],",
+        "}));",
+      ].join("\n"),
+      "utf8",
+    );
+    await writeFile(
+      path.join(repoRoot, "extensions/qa-lab/web/dist/index.html"),
+      "<!doctype html><html><body>lazy catalog</body></html>",
+      "utf8",
+    );
+
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      repoRoot,
+    });
+    cleanups.push(async () => {
+      await lab.stop();
+    });
+
+    await sleep(25);
+    await expect(readFile(markerPath, "utf8")).rejects.toThrow();
+
+    const bootstrapResponse = await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`);
+    expect(bootstrapResponse.status).toBe(200);
+
+    const runnerCatalog = await waitForRunnerCatalog(lab.baseUrl);
+    expect(runnerCatalog.status).toBe("ready");
+    expect(await readFile(markerPath, "utf8")).toContain("models list --all --json");
+  });
+
+  it("aborts an in-flight runner model catalog when the lab stops", async () => {
+    const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-lab-abort-catalog-"));
+    cleanups.push(async () => {
+      await rm(repoRoot, { recursive: true, force: true });
+    });
+    const markerPath = path.join(repoRoot, "runner-catalog-started.txt");
+    const stoppedPath = path.join(repoRoot, "runner-catalog-stopped.txt");
+
+    await mkdir(path.join(repoRoot, "dist"), { recursive: true });
+    await mkdir(path.join(repoRoot, "extensions/qa-lab/web/dist"), { recursive: true });
+    await writeFile(
+      path.join(repoRoot, "dist/index.js"),
+      [
+        'const fs = require("node:fs");',
+        `fs.writeFileSync(${JSON.stringify(markerPath)}, process.env.OPENCLAW_CODEX_DISCOVERY_LIVE || "", "utf8");`,
+        "process.on('SIGTERM', () => {",
+        `  fs.writeFileSync(${JSON.stringify(stoppedPath)}, "terminated", "utf8");`,
+        "  process.exit(0);",
+        "});",
+        "setInterval(() => {}, 1000);",
+      ].join("\n"),
+      "utf8",
+    );
+    await writeFile(
+      path.join(repoRoot, "extensions/qa-lab/web/dist/index.html"),
+      "<!doctype html><html><body>abort catalog</body></html>",
+      "utf8",
+    );
+
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      repoRoot,
+    });
+    let stopped = false;
+    cleanups.push(async () => {
+      if (!stopped) {
+        await lab.stop();
+      }
+    });
+
+    const bootstrapResponse = await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`);
+    expect(bootstrapResponse.status).toBe(200);
+    expect(await waitForFile(markerPath)).toBe("0");
+
+    await lab.stop();
+    stopped = true;
+    expect(await waitForFile(stoppedPath)).toBe("terminated");
+  });
+
+  it("can disable the embedded echo gateway for real-suite runs", async () => {
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      embeddedGateway: "disabled",
+    });
+    cleanups.push(async () => {
+      await lab.stop();
+    });
+
+    await fetch(`${lab.baseUrl}/api/inbound/message`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+      },
+      body: JSON.stringify({
+        conversation: { id: "bob", kind: "direct" },
+        senderId: "bob",
+        senderName: "Bob",
+        text: "hello from suite",
+      }),
+    });
+
+    const snapshot = (await (await fetchWithRetry(`${lab.baseUrl}/api/state`)).json()) as {
+      messages: Array<{ direction: string }>;
+    };
+    expect(snapshot.messages.filter((message) => message.direction === "outbound")).toHaveLength(0);
+  });
+
+  it("exposes structured outcomes and can attach control-ui after startup", async () => {
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+      embeddedGateway: "disabled",
+    });
+    cleanups.push(async () => {
+      await lab.stop();
+    });
+
+    const initialOutcomes = (await (
+      await fetchWithRetry(`${lab.baseUrl}/api/outcomes`)
+    ).json()) as {
+      run: unknown;
+    };
+    expect(initialOutcomes.run).toBeNull();
+
+    lab.setScenarioRun({
+      kind: "suite",
+      status: "running",
+      startedAt: "2026-04-06T09:00:00.000Z",
+      scenarios: [
+        {
+          id: "channel-chat-baseline",
+          name: "Channel baseline conversation",
+          status: "pass",
+          steps: [{ name: "reply check", status: "pass", details: "ok" }],
+          finishedAt: "2026-04-06T09:00:01.000Z",
+        },
+        {
+          id: "cron-one-minute-ping",
+          name: "Cron one-minute ping",
+          status: "running",
+          startedAt: "2026-04-06T09:00:02.000Z",
+        },
+      ],
+    });
+    lab.setControlUi({
+      controlUiUrl: "http://127.0.0.1:18789/",
+      controlUiToken: "late-token",
+    });
+
+    const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as {
+      controlUiEmbeddedUrl: string | null;
+    };
+    expect(bootstrap.controlUiEmbeddedUrl).toBe("http://127.0.0.1:18789/#token=late-token");
+
+    const outcomes = (await (await fetchWithRetry(`${lab.baseUrl}/api/outcomes`)).json()) as {
+      run: {
+        status: string;
+        counts: { total: number; passed: number; running: number };
+        scenarios: Array<{ id: string; status: string }>;
+      };
+    };
+    expect(outcomes.run.status).toBe("running");
+    expect(outcomes.run.counts).toEqual({
+      total: 2,
+      pending: 0,
+      running: 1,
+      passed: 1,
+      failed: 0,
+      skipped: 0,
+    });
+    expect(outcomes.run.scenarios.map((scenario) => scenario.id)).toEqual([
+      "channel-chat-baseline",
+      "cron-one-minute-ping",
+    ]);
+  });
+
+  it("serves proxy capture sessions, events, and query rows", async () => {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-capture-"));
+    cleanups.push(async () => {
+      await rm(tempDir, { recursive: true, force: true });
+    });
+    process.env.OPENCLAW_DEBUG_PROXY_DB_PATH = path.join(tempDir, "capture.sqlite");
+    process.env.OPENCLAW_DEBUG_PROXY_BLOB_DIR = path.join(tempDir, "blobs");
+    const { getDebugProxyCaptureStore } =
+      await import("../../../src/proxy-capture/store.sqlite.js");
+    const store = getDebugProxyCaptureStore(
+      process.env.OPENCLAW_DEBUG_PROXY_DB_PATH,
+      process.env.OPENCLAW_DEBUG_PROXY_BLOB_DIR,
+    );
+    store.upsertSession({
+      id: "qa-capture-session",
+      startedAt: Date.now(),
+      mode: "proxy-run",
+      sourceScope: "openclaw",
+      sourceProcess: "openclaw",
+      dbPath: process.env.OPENCLAW_DEBUG_PROXY_DB_PATH,
+      blobDir: process.env.OPENCLAW_DEBUG_PROXY_BLOB_DIR,
+    });
+    store.recordEvent({
+      sessionId: "qa-capture-session",
+      ts: Date.now(),
+      sourceScope: "openclaw",
+      sourceProcess: "openclaw",
+      protocol: "https",
+      direction: "outbound",
+      kind: "request",
+      flowId: "flow-1",
+      method: "POST",
+      host: "api.example.com",
+      path: "/v1/send",
+      dataText: '{"hello":"world"}',
+      dataSha256: "abc",
+      metaJson: JSON.stringify({
+        provider: "openai",
+        api: "responses",
+        model: "gpt-5.4",
+        captureOrigin: "shared-fetch",
+      }),
+    });
+    store.recordEvent({
+      sessionId: "qa-capture-session",
+      ts: Date.now() + 1,
+      sourceScope: "openclaw",
+      sourceProcess: "openclaw",
+      protocol: "https",
+      direction: "outbound",
+      kind: "request",
+      flowId: "flow-2",
+      method: "POST",
+      host: "api.example.com",
+      path: "/v1/send",
+      dataText: '{"hello":"world"}',
+      dataSha256: "abc",
+      metaJson: JSON.stringify({
+        provider: "openai",
+        api: "responses",
+        model: "gpt-5.4",
+        captureOrigin: "shared-fetch",
+      }),
+    });
+    store.recordEvent({
+      sessionId: "qa-capture-session",
+      ts: Date.now() + 2,
+      sourceScope: "openclaw",
+      sourceProcess: "openclaw",
+      protocol: "https",
+      direction: "outbound",
+      kind: "request",
+      flowId: "flow-3",
+      method: "POST",
+      host: "127.0.0.1:11434",
+      path: "/api/chat",
+      metaJson: JSON.stringify({
+        provider: "ollama",
+        model: "kimi-k2.5:cloud",
+        captureOrigin: "shared-fetch",
+      }),
+    });
+
+    const lab = await startQaLabServer({
+      host: "127.0.0.1",
+      port: 0,
+    });
+    cleanups.push(async () => {
+      delete process.env.OPENCLAW_DEBUG_PROXY_DB_PATH;
+      delete process.env.OPENCLAW_DEBUG_PROXY_BLOB_DIR;
+      await lab.stop();
+    });
+
+    const sessions = (await (
+      await fetchWithRetry(`${lab.baseUrl}/api/capture/sessions`)
+    ).json()) as { sessions: Array<{ id: string }> };
+    expect(sessions.sessions.some((session) => session.id === "qa-capture-session")).toBe(true);
+
+    const events = (await (
+      await fetchWithRetry(`${lab.baseUrl}/api/capture/events?sessionId=qa-capture-session`)
+    ).json()) as {
+      events: Array<{ flowId: string; provider?: string; model?: string; captureOrigin?: string }>;
+    };
+    expect(events.events.some((event) => event.flowId === "flow-1")).toBe(true);
+    expect(events.events).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          flowId: "flow-1",
+          provider: "openai",
+          model: "gpt-5.4",
+          captureOrigin: "shared-fetch",
+        }),
+        expect.objectContaining({
+          flowId: "flow-3",
+          provider: "ollama",
+          model: "kimi-k2.5:cloud",
+        }),
+      ]),
+    );
+
+    const coverage = (await (
+      await fetchWithRetry(`${lab.baseUrl}/api/capture/coverage?sessionId=qa-capture-session`)
+    ).json()) as {
+      coverage: {
+        totalEvents: number;
+        unlabeledEventCount: number;
+        providers: Array<{ value: string; count: number }>;
+        models: Array<{ value: string; count: number }>;
+        localPeers: Array<{ value: string; count: number }>;
+      };
+    };
+    expect(coverage.coverage.totalEvents).toBe(3);
+    expect(coverage.coverage.unlabeledEventCount).toBe(0);
+    expect(coverage.coverage.providers).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ value: "openai", count: 2 }),
+        expect.objectContaining({ value: "ollama", count: 1 }),
+      ]),
+    );
+    expect(coverage.coverage.models).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ value: "gpt-5.4", count: 2 }),
+        expect.objectContaining({ value: "kimi-k2.5:cloud", count: 1 }),
+      ]),
+    );
+    expect(coverage.coverage.localPeers).toEqual(
+      expect.arrayContaining([expect.objectContaining({ value: "127.0.0.1:11434", count: 1 })]),
+    );
+
+    const query = (await (
+      await fetchWithRetry(
+        `${lab.baseUrl}/api/capture/query?sessionId=qa-capture-session&preset=double-sends`,
+      )
+    ).json()) as { rows: Array<{ host: string; duplicateCount: number }> };
+    expect(query.rows).toEqual([
+      expect.objectContaining({
+        host: "api.example.com",
+        duplicateCount: 2,
+      }),
+    ]);
+  });
+});

openclaw/extensions/qa-lab/src/lab-server.ts

@@ -0,0 +1,655 @@
+import fs from "node:fs";
+import { createServer, type IncomingMessage } from "node:http";
+import path from "node:path";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import {
+  getDebugProxyCaptureStore,
+  resolveDebugProxySettings,
+} from "openclaw/plugin-sdk/proxy-capture";
+import { closeQaHttpServer, handleQaBusRequest, writeError, writeJson } from "./bus-server.js";
+import { createQaBusState, type QaBusState } from "./bus-state.js";
+import { createQaRunnerRuntime } from "./harness-runtime.js";
+import {
+  isCaptureQueryPreset,
+  mapCaptureEventForQa,
+  probeTcpReachability,
+} from "./lab-server-capture.js";
+import {
+  detectContentType,
+  isControlUiProxyPath,
+  missingUiHtml,
+  proxyHttpRequest,
+  proxyUpgradeRequest,
+  resolveAdvertisedBaseUrl,
+  resolveUiAssetVersion,
+  tryResolveUiAsset,
+} from "./lab-server-ui.js";
+import type {
+  QaLabLatestReport,
+  QaLabScenarioOutcome,
+  QaLabScenarioRun,
+  QaLabServerHandle,
+  QaLabServerStartParams,
+} from "./lab-server.types.js";
+import type { QaRunnerModelOption } from "./model-catalog.runtime.js";
+import { createQaChannelGatewayConfig } from "./qa-channel-transport.js";
+import {
+  createIdleQaRunnerSnapshot,
+  createQaRunOutputDir,
+  normalizeQaRunSelection,
+} from "./run-config.js";
+import { qaChannelPlugin, setQaChannelRuntime, type OpenClawConfig } from "./runtime-api.js";
+import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
+import { runQaSelfCheckAgainstState, type QaSelfCheckResult } from "./self-check.js";
+
+type QaLabBootstrapDefaults = {
+  conversationKind: "direct" | "channel";
+  conversationId: string;
+  senderId: string;
+  senderName: string;
+};
+
+export type {
+  QaLabLatestReport,
+  QaLabScenarioOutcome,
+  QaLabScenarioRun,
+  QaLabServerHandle,
+  QaLabServerStartParams,
+} from "./lab-server.types.js";
+
+function countQaLabScenarioRun(scenarios: QaLabScenarioOutcome[]) {
+  return {
+    total: scenarios.length,
+    pending: scenarios.filter((scenario) => scenario.status === "pending").length,
+    running: scenarios.filter((scenario) => scenario.status === "running").length,
+    passed: scenarios.filter((scenario) => scenario.status === "pass").length,
+    failed: scenarios.filter((scenario) => scenario.status === "fail").length,
+    skipped: scenarios.filter((scenario) => scenario.status === "skip").length,
+  };
+}
+
+function withQaLabRunCounts(run: Omit<QaLabScenarioRun, "counts">): QaLabScenarioRun {
+  return {
+    ...run,
+    counts: countQaLabScenarioRun(run.scenarios),
+  };
+}
+
+function injectKickoffMessage(params: {
+  state: QaBusState;
+  defaults: QaLabBootstrapDefaults;
+  kickoffTask: string;
+}) {
+  return params.state.addInboundMessage({
+    conversation: {
+      id: params.defaults.conversationId,
+      kind: params.defaults.conversationKind,
+      ...(params.defaults.conversationKind === "channel"
+        ? { title: params.defaults.conversationId }
+        : {}),
+    },
+    senderId: params.defaults.senderId,
+    senderName: params.defaults.senderName,
+    text: params.kickoffTask,
+  });
+}
+
+async function readJson(req: IncomingMessage): Promise<unknown> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of req) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  const text = Buffer.concat(chunks).toString("utf8").trim();
+  return text ? (JSON.parse(text) as unknown) : {};
+}
+
+function createBootstrapDefaults(autoKickoffTarget?: string): QaLabBootstrapDefaults {
+  if (autoKickoffTarget === "channel") {
+    return {
+      conversationKind: "channel",
+      conversationId: "qa-lab",
+      senderId: "qa-operator",
+      senderName: "QA Operator",
+    };
+  }
+  return {
+    conversationKind: "direct",
+    conversationId: "qa-operator",
+    senderId: "qa-operator",
+    senderName: "QA Operator",
+  };
+}
+
+function createQaLabConfig(baseUrl: string): OpenClawConfig {
+  return createQaChannelGatewayConfig({ baseUrl });
+}
+
+async function startQaGatewayLoop(params: { state: QaBusState; baseUrl: string }) {
+  const runtime = createQaRunnerRuntime();
+  setQaChannelRuntime(runtime);
+  const cfg = createQaLabConfig(params.baseUrl);
+  const account = qaChannelPlugin.config.resolveAccount(cfg, "default");
+  const abort = new AbortController();
+  const task = qaChannelPlugin.gateway?.startAccount?.({
+    accountId: account.accountId,
+    account,
+    cfg,
+    runtime: {
+      log: () => undefined,
+      error: () => undefined,
+      exit: () => undefined,
+    },
+    abortSignal: abort.signal,
+    log: {
+      info: () => undefined,
+      warn: () => undefined,
+      error: () => undefined,
+      debug: () => undefined,
+    },
+    getStatus: () => ({
+      accountId: account.accountId,
+      configured: true,
+      enabled: true,
+      running: true,
+    }),
+    setStatus: () => undefined,
+  });
+  return {
+    cfg,
+    async stop() {
+      abort.abort();
+      await task;
+    },
+  };
+}
+
+export async function startQaLabServer(
+  params?: QaLabServerStartParams,
+): Promise<QaLabServerHandle> {
+  const repoRoot = path.resolve(params?.repoRoot ?? process.cwd());
+  const captureSettings = resolveDebugProxySettings();
+  const captureStore = getDebugProxyCaptureStore(captureSettings.dbPath, captureSettings.blobDir);
+  const state = createQaBusState();
+  let latestReport: QaLabLatestReport | null = null;
+  let latestScenarioRun: QaLabScenarioRun | null = null;
+  const scenarioCatalog = readQaBootstrapScenarioCatalog();
+  const bootstrapDefaults = createBootstrapDefaults(params?.autoKickoffTarget);
+  let runnerModelOptions: QaRunnerModelOption[] = [];
+  let runnerModelCatalogStatus: "loading" | "ready" | "failed" = "loading";
+  let runnerSnapshot = createIdleQaRunnerSnapshot(scenarioCatalog.scenarios);
+  let activeSuiteRun: Promise<void> | null = null;
+  let controlUiProxyTarget = params?.controlUiProxyTarget?.trim()
+    ? new URL(params.controlUiProxyTarget)
+    : null;
+  let controlUiUrl = params?.controlUiUrl?.trim() || null;
+  let controlUiToken = params?.controlUiToken?.trim() || null;
+  let gateway:
+    | {
+        cfg: OpenClawConfig;
+        stop: () => Promise<void>;
+      }
+    | undefined;
+  const embeddedGatewayEnabled = params?.embeddedGateway !== "disabled";
+  let labHandle: QaLabServerHandle | null = null;
+
+  let publicBaseUrl = "";
+  let runnerModelCatalogPromise: Promise<void> | null = null;
+  let runnerModelCatalogAbort: AbortController | null = null;
+  const ensureRunnerModelCatalog = () => {
+    if (runnerModelCatalogPromise) {
+      return runnerModelCatalogPromise;
+    }
+    runnerModelCatalogAbort = new AbortController();
+    runnerModelCatalogPromise = (async () => {
+      try {
+        const { loadQaRunnerModelOptions } = await import("./model-catalog.runtime.js");
+        runnerModelOptions = await loadQaRunnerModelOptions({
+          repoRoot,
+          signal: runnerModelCatalogAbort?.signal,
+        });
+        runnerModelCatalogStatus = "ready";
+      } catch {
+        runnerModelOptions = [];
+        runnerModelCatalogStatus = "failed";
+      }
+    })().finally(() => {
+      runnerModelCatalogAbort = null;
+    });
+    return runnerModelCatalogPromise;
+  };
+
+  async function runSelfCheck(): Promise<QaSelfCheckResult> {
+    latestScenarioRun = withQaLabRunCounts({
+      kind: "self-check",
+      status: "running",
+      startedAt: new Date().toISOString(),
+      scenarios: [
+        {
+          id: "qa-self-check",
+          name: "Synthetic Slack-class roundtrip",
+          status: "running",
+        },
+      ],
+    });
+    const result = await runQaSelfCheckAgainstState({
+      state,
+      cfg: gateway?.cfg ?? createQaLabConfig(listenUrl),
+      transportId: "qa-channel",
+      outputPath: params?.outputPath,
+      repoRoot,
+    });
+    latestScenarioRun = withQaLabRunCounts({
+      kind: "self-check",
+      status: "completed",
+      startedAt: latestScenarioRun.startedAt,
+      finishedAt: new Date().toISOString(),
+      scenarios: [
+        {
+          id: "qa-self-check",
+          name: result.scenarioResult.name,
+          status: result.scenarioResult.status,
+          details: result.scenarioResult.details,
+          steps: result.scenarioResult.steps,
+        },
+      ],
+    });
+    latestReport = {
+      outputPath: result.outputPath,
+      markdown: result.report,
+      generatedAt: new Date().toISOString(),
+    };
+    return result;
+  }
+
+  const server = createServer(async (req, res) => {
+    const url = new URL(req.url ?? "/", "http://127.0.0.1");
+
+    if (await handleQaBusRequest({ req, res, state })) {
+      return;
+    }
+
+    try {
+      if (controlUiProxyTarget && isControlUiProxyPath(url.pathname)) {
+        await proxyHttpRequest({
+          req,
+          res,
+          target: controlUiProxyTarget,
+          pathname: url.pathname,
+          search: url.search,
+        });
+        return;
+      }
+
+      if (req.method === "GET" && url.pathname === "/api/bootstrap") {
+        void ensureRunnerModelCatalog();
+        const resolvedControlUiUrl = controlUiProxyTarget
+          ? `${publicBaseUrl}/control-ui/`
+          : controlUiUrl;
+        const controlUiEmbeddedUrl =
+          resolvedControlUiUrl && controlUiToken
+            ? `${resolvedControlUiUrl.replace(/\/?$/, "/")}#token=${encodeURIComponent(controlUiToken)}`
+            : resolvedControlUiUrl;
+        writeJson(res, 200, {
+          baseUrl: publicBaseUrl,
+          latestReport,
+          controlUiUrl: resolvedControlUiUrl,
+          controlUiEmbeddedUrl,
+          kickoffTask: scenarioCatalog.kickoffTask,
+          scenarios: scenarioCatalog.scenarios,
+          defaults: bootstrapDefaults,
+          runner: runnerSnapshot,
+          runnerCatalog: {
+            status: runnerModelCatalogStatus,
+            real: runnerModelOptions,
+          },
+        });
+        return;
+      }
+      if (req.method === "GET" && (url.pathname === "/healthz" || url.pathname === "/readyz")) {
+        writeJson(res, 200, { ok: true, status: "live" });
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/state") {
+        writeJson(res, 200, state.getSnapshot());
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/report") {
+        writeJson(res, 200, { report: latestReport });
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/ui-version") {
+        res.writeHead(200, {
+          "content-type": "application/json; charset=utf-8",
+          "cache-control": "no-store",
+        });
+        res.end(JSON.stringify({ version: resolveUiAssetVersion(params?.uiDistDir) }));
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/outcomes") {
+        writeJson(res, 200, { run: latestScenarioRun });
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/capture/sessions") {
+        writeJson(res, 200, {
+          sessions: captureStore.listSessions(50),
+        });
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/capture/startup-status") {
+        const proxyUrl = captureSettings.proxyUrl || "http://127.0.0.1:7799";
+        const gatewayUrl = controlUiUrl || "http://127.0.0.1:18789/";
+        const [proxy, gateway] = await Promise.all([
+          probeTcpReachability(proxyUrl),
+          probeTcpReachability(gatewayUrl),
+        ]);
+        writeJson(res, 200, {
+          status: {
+            proxy: {
+              ...proxy,
+              label: "Proxy",
+            },
+            gateway: {
+              ...gateway,
+              label: "Gateway",
+            },
+            qaLab: {
+              label: "QA Lab",
+              url: publicBaseUrl,
+              ok: true,
+            },
+          },
+        });
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/capture/events") {
+        const sessionId = url.searchParams.get("sessionId")?.trim();
+        writeJson(res, 200, {
+          events: sessionId
+            ? captureStore.getSessionEvents(sessionId, 200).map(mapCaptureEventForQa)
+            : [],
+        });
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/capture/coverage") {
+        const sessionId = url.searchParams.get("sessionId")?.trim();
+        if (!sessionId) {
+          writeError(res, 400, "Missing sessionId");
+          return;
+        }
+        writeJson(res, 200, {
+          coverage: captureStore.summarizeSessionCoverage(sessionId),
+        });
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/capture/query") {
+        const preset = url.searchParams.get("preset")?.trim();
+        const sessionId = url.searchParams.get("sessionId")?.trim() || undefined;
+        if (!preset) {
+          writeError(res, 400, "Missing preset");
+          return;
+        }
+        if (!isCaptureQueryPreset(preset)) {
+          writeError(res, 400, "Unknown preset");
+          return;
+        }
+        writeJson(res, 200, {
+          rows: captureStore.queryPreset(preset, sessionId),
+        });
+        return;
+      }
+      if (req.method === "GET" && url.pathname === "/api/capture/blob") {
+        const blobId = url.searchParams.get("id")?.trim();
+        if (!blobId) {
+          writeError(res, 400, "Missing blob id");
+          return;
+        }
+        const content = captureStore.readBlob(blobId);
+        if (content == null) {
+          writeError(res, 404, "Blob not found");
+          return;
+        }
+        writeJson(res, 200, { id: blobId, content });
+        return;
+      }
+      if (req.method === "POST" && url.pathname === "/api/capture/delete-sessions") {
+        const body = (await readJson(req)) as { sessionIds?: unknown };
+        const sessionIds = Array.isArray(body.sessionIds)
+          ? body.sessionIds.filter((value): value is string => typeof value === "string")
+          : [];
+        writeJson(res, 200, {
+          result: captureStore.deleteSessions(sessionIds),
+        });
+        return;
+      }
+      if (req.method === "POST" && url.pathname === "/api/capture/purge") {
+        writeJson(res, 200, {
+          result: captureStore.purgeAll(),
+        });
+        return;
+      }
+      if (req.method === "POST" && url.pathname === "/api/reset") {
+        if (activeSuiteRun) {
+          writeError(res, 409, "QA suite run already in progress");
+          return;
+        }
+        state.reset();
+        latestReport = null;
+        latestScenarioRun = null;
+        runnerSnapshot = {
+          ...runnerSnapshot,
+          status: "idle",
+          artifacts: null,
+          error: null,
+          startedAt: undefined,
+          finishedAt: undefined,
+        };
+        writeJson(res, 200, { ok: true });
+        return;
+      }
+      if (req.method === "POST" && url.pathname === "/api/inbound/message") {
+        const body = await readJson(req);
+        writeJson(res, 200, {
+          message: state.addInboundMessage(body as Parameters<QaBusState["addInboundMessage"]>[0]),
+        });
+        return;
+      }
+      if (req.method === "POST" && url.pathname === "/api/kickoff") {
+        writeJson(res, 200, {
+          message: injectKickoffMessage({
+            state,
+            defaults: bootstrapDefaults,
+            kickoffTask: scenarioCatalog.kickoffTask,
+          }),
+        });
+        return;
+      }
+      if (req.method === "POST" && url.pathname === "/api/scenario/self-check") {
+        if (activeSuiteRun) {
+          writeError(res, 409, "QA suite run already in progress");
+          return;
+        }
+        const result = await runSelfCheck();
+        writeJson(res, 200, serializeSelfCheck(result));
+        return;
+      }
+      if (req.method === "POST" && url.pathname === "/api/scenario/suite") {
+        if (activeSuiteRun) {
+          writeError(res, 409, "QA suite run already in progress");
+          return;
+        }
+        const selection = normalizeQaRunSelection(await readJson(req), scenarioCatalog.scenarios);
+        state.reset();
+        latestReport = null;
+        latestScenarioRun = null;
+        const startedAt = new Date().toISOString();
+        runnerSnapshot = {
+          status: "running",
+          selection,
+          startedAt,
+          finishedAt: undefined,
+          artifacts: null,
+          error: null,
+        };
+        activeSuiteRun = (async () => {
+          try {
+            const { runQaSuite } = await import("./suite.js");
+            const result = await runQaSuite({
+              lab: labHandle ?? undefined,
+              outputDir: createQaRunOutputDir(repoRoot),
+              providerMode: selection.providerMode,
+              primaryModel: selection.primaryModel,
+              alternateModel: selection.alternateModel,
+              scenarioIds: selection.scenarioIds,
+            });
+            runnerSnapshot = {
+              status: "completed",
+              selection,
+              startedAt,
+              finishedAt: new Date().toISOString(),
+              artifacts: {
+                outputDir: result.outputDir,
+                reportPath: result.reportPath,
+                summaryPath: result.summaryPath,
+                watchUrl: result.watchUrl,
+              },
+              error: null,
+            };
+          } catch (error) {
+            runnerSnapshot = {
+              status: "failed",
+              selection,
+              startedAt,
+              finishedAt: new Date().toISOString(),
+              artifacts: null,
+              error: formatErrorMessage(error),
+            };
+          } finally {
+            activeSuiteRun = null;
+          }
+        })();
+        writeJson(res, 202, {
+          ok: true,
+          runner: runnerSnapshot,
+        });
+        return;
+      }
+
+      if (req.method !== "GET" && req.method !== "HEAD") {
+        writeError(res, 404, "not found");
+        return;
+      }
+
+      const asset = tryResolveUiAsset(url.pathname, params?.uiDistDir, repoRoot);
+      if (!asset) {
+        const html = missingUiHtml();
+        res.writeHead(200, {
+          "content-type": "text/html; charset=utf-8",
+          "content-length": Buffer.byteLength(html),
+        });
+        if (req.method === "HEAD") {
+          res.end();
+          return;
+        }
+        res.end(html);
+        return;
+      }
+
+      const body = fs.readFileSync(asset);
+      res.writeHead(200, {
+        "content-type": detectContentType(asset),
+        "content-length": body.byteLength,
+      });
+      if (req.method === "HEAD") {
+        res.end();
+        return;
+      }
+      res.end(body);
+    } catch (error) {
+      writeError(res, 500, error);
+    }
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(params?.port ?? 0, params?.host ?? "127.0.0.1", () => resolve());
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("qa-lab failed to bind");
+  }
+  const listenUrl = resolveAdvertisedBaseUrl({
+    bindHost: params?.host ?? "127.0.0.1",
+    bindPort: address.port,
+  });
+  publicBaseUrl = resolveAdvertisedBaseUrl({
+    bindHost: params?.host ?? "127.0.0.1",
+    bindPort: address.port,
+    advertiseHost: params?.advertiseHost,
+    advertisePort: params?.advertisePort,
+  });
+  if (embeddedGatewayEnabled) {
+    gateway = await startQaGatewayLoop({ state, baseUrl: listenUrl });
+  }
+  if (params?.sendKickoffOnStart) {
+    injectKickoffMessage({
+      state,
+      defaults: bootstrapDefaults,
+      kickoffTask: scenarioCatalog.kickoffTask,
+    });
+  }
+
+  server.on("upgrade", (req, socket, head) => {
+    const url = new URL(req.url ?? "/", "http://127.0.0.1");
+    if (!controlUiProxyTarget || !isControlUiProxyPath(url.pathname)) {
+      socket.destroy();
+      return;
+    }
+    proxyUpgradeRequest({
+      req,
+      socket,
+      head,
+      target: controlUiProxyTarget,
+    });
+  });
+
+  const lab = {
+    baseUrl: publicBaseUrl,
+    listenUrl,
+    state,
+    setControlUi(next: {
+      controlUiUrl?: string | null;
+      controlUiToken?: string | null;
+      controlUiProxyTarget?: string | null;
+    }) {
+      controlUiUrl = next.controlUiUrl?.trim() || null;
+      controlUiToken = next.controlUiToken?.trim() || null;
+      controlUiProxyTarget = next.controlUiProxyTarget?.trim()
+        ? new URL(next.controlUiProxyTarget)
+        : null;
+    },
+    setScenarioRun(next: Omit<QaLabScenarioRun, "counts"> | null) {
+      latestScenarioRun = next ? withQaLabRunCounts(next) : null;
+    },
+    setLatestReport(next: QaLabLatestReport | null) {
+      latestReport = next;
+    },
+    runSelfCheck,
+    async stop() {
+      runnerModelCatalogAbort?.abort();
+      await runnerModelCatalogPromise?.catch(() => undefined);
+      await gateway?.stop();
+      await closeQaHttpServer(server);
+    },
+  };
+  labHandle = lab;
+  return lab;
+}
+
+function serializeSelfCheck(result: QaSelfCheckResult) {
+  return {
+    outputPath: result.outputPath,
+    report: result.report,
+    checks: result.checks,
+    scenario: result.scenarioResult,
+  };
+}

openclaw/extensions/qa-lab/src/lab-server.types.ts

@@ -0,0 +1,73 @@
+import type { QaBusState } from "./bus-state.js";
+import type { QaSelfCheckResult } from "./self-check.js";
+
+export type QaLabLatestReport = {
+  outputPath: string;
+  markdown: string;
+  generatedAt: string;
+};
+
+export type QaLabRunStatus = "idle" | "running" | "completed";
+
+export type QaLabScenarioStep = {
+  name: string;
+  status: "pass" | "fail" | "skip";
+  details?: string;
+};
+
+export type QaLabScenarioOutcome = {
+  id: string;
+  name: string;
+  status: "pending" | "running" | "pass" | "fail" | "skip";
+  details?: string;
+  steps?: QaLabScenarioStep[];
+  startedAt?: string;
+  finishedAt?: string;
+};
+
+export type QaLabScenarioRun = {
+  kind: "suite" | "self-check";
+  status: QaLabRunStatus;
+  startedAt?: string;
+  finishedAt?: string;
+  scenarios: QaLabScenarioOutcome[];
+  counts: {
+    total: number;
+    pending: number;
+    running: number;
+    passed: number;
+    failed: number;
+    skipped: number;
+  };
+};
+
+export type QaLabServerStartParams = {
+  repoRoot?: string;
+  host?: string;
+  port?: number;
+  outputPath?: string;
+  advertiseHost?: string;
+  advertisePort?: number;
+  controlUiUrl?: string;
+  controlUiToken?: string;
+  controlUiProxyTarget?: string;
+  uiDistDir?: string;
+  autoKickoffTarget?: string;
+  embeddedGateway?: string;
+  sendKickoffOnStart?: boolean;
+};
+
+export type QaLabServerHandle = {
+  baseUrl: string;
+  listenUrl: string;
+  state: QaBusState;
+  setControlUi: (next: {
+    controlUiUrl?: string | null;
+    controlUiToken?: string | null;
+    controlUiProxyTarget?: string | null;
+  }) => void;
+  setScenarioRun: (next: Omit<QaLabScenarioRun, "counts"> | null) => void;
+  setLatestReport: (next: QaLabLatestReport | null) => void;
+  runSelfCheck: () => Promise<QaSelfCheckResult>;
+  stop: () => Promise<void>;
+};

openclaw/extensions/qa-lab/src/live-timeout.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it } from "vitest";
+import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
+
+describe("qa live timeout policy", () => {
+  it("keeps mock lanes on the caller fallback", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "mock-openai",
+          primaryModel: "anthropic/claude-sonnet-4-6",
+          alternateModel: "anthropic/claude-opus-4-6",
+        },
+        30_000,
+      ),
+    ).toBe(30_000);
+  });
+
+  it("uses the higher gpt-5 live floor for openai heavy turns", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "openai/gpt-5.4",
+          alternateModel: "openai/gpt-5.4",
+        },
+        30_000,
+      ),
+    ).toBe(360_000);
+  });
+
+  it("keeps the standard live floor for other non-anthropic models", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "google/gemini-3-flash",
+          alternateModel: "google/gemini-3-flash",
+        },
+        30_000,
+      ),
+    ).toBe(120_000);
+  });
+
+  it("uses the anthropic floor for sonnet turns", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "anthropic/claude-sonnet-4-6",
+          alternateModel: "anthropic/claude-opus-4-6",
+        },
+        30_000,
+      ),
+    ).toBe(180_000);
+  });
+
+  it("uses the opus floor when the switched turn runs on claude opus", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "anthropic/claude-sonnet-4-6",
+          alternateModel: "anthropic/claude-opus-4-6",
+        },
+        30_000,
+        "anthropic/claude-opus-4-6",
+      ),
+    ).toBe(240_000);
+  });
+});

openclaw/extensions/qa-lab/src/live-timeout.ts

@@ -0,0 +1,21 @@
+import type { QaProviderMode } from "./model-selection.js";
+import { getQaProvider } from "./providers/index.js";
+
+type QaLiveTimeoutProfile = {
+  providerMode: QaProviderMode;
+  primaryModel: string;
+  alternateModel: string;
+};
+
+export function resolveQaLiveTurnTimeoutMs(
+  profile: QaLiveTimeoutProfile,
+  fallbackMs: number,
+  modelRef = profile.primaryModel,
+) {
+  return getQaProvider(profile.providerMode).resolveTurnTimeoutMs({
+    primaryModel: profile.primaryModel,
+    alternateModel: profile.alternateModel,
+    modelRef,
+    fallbackMs,
+  });
+}

openclaw/extensions/qa-lab/src/live-transports/cli.ts

@@ -0,0 +1,50 @@
+import { listQaRunnerCliContributions } from "openclaw/plugin-sdk/qa-runner-runtime";
+import type { LiveTransportQaCliRegistration } from "./shared/live-transport-cli.js";
+import { telegramQaCliRegistration } from "./telegram/cli.js";
+
+function createBlockedQaRunnerCliRegistration(params: {
+  commandName: string;
+  description?: string;
+  pluginId: string;
+}): LiveTransportQaCliRegistration {
+  return {
+    commandName: params.commandName,
+    register(qa) {
+      qa.command(params.commandName)
+        .description(params.description ?? `Run the ${params.commandName} live QA lane`)
+        .action(() => {
+          throw new Error(
+            `QA runner "${params.commandName}" is installed but not active. Enable or allow plugin "${params.pluginId}" in your OpenClaw config, then try again.`,
+          );
+        });
+    },
+  };
+}
+
+function createQaRunnerCliRegistration(
+  runner: ReturnType<typeof listQaRunnerCliContributions>[number],
+): LiveTransportQaCliRegistration {
+  if (runner.status === "available") {
+    return runner.registration;
+  }
+  return createBlockedQaRunnerCliRegistration({
+    commandName: runner.commandName,
+    description: runner.description,
+    pluginId: runner.pluginId,
+  });
+}
+
+export const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [
+  telegramQaCliRegistration,
+];
+
+export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaCliRegistration[] {
+  const liveRegistrations = [...LIVE_TRANSPORT_QA_CLI_REGISTRATIONS];
+  const discoveredRunners = listQaRunnerCliContributions();
+
+  for (const runner of discoveredRunners) {
+    liveRegistrations.push(createQaRunnerCliRegistration(runner));
+  }
+
+  return liveRegistrations;
+}

openclaw/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.test.ts

@@ -0,0 +1,331 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  acquireQaCredentialLease,
+  startQaCredentialLeaseHeartbeat,
+} from "./credential-lease.runtime.js";
+
+function jsonResponse(payload: unknown, status = 200) {
+  return new Response(JSON.stringify(payload), {
+    status,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+describe("credential lease runtime", () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+  });
+
+  it("uses env credentials by default", async () => {
+    const lease = await acquireQaCredentialLease({
+      kind: "telegram",
+      resolveEnvPayload: () => ({ groupId: "-100123", driverToken: "driver", sutToken: "sut" }),
+      parsePayload: () => {
+        throw new Error("should not parse convex payload in env mode");
+      },
+      env: {},
+    });
+
+    expect(lease.source).toBe("env");
+    expect(lease.payload).toEqual({
+      groupId: "-100123",
+      driverToken: "driver",
+      sutToken: "sut",
+    });
+  });
+
+  it("acquires, heartbeats, and releases convex credentials", async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-1",
+          leaseToken: "lease-1",
+          payload: { groupId: "-100123", driverToken: "driver", sutToken: "sut" },
+          leaseTtlMs: 1_200_000,
+          heartbeatIntervalMs: 30_000,
+        }),
+      )
+      .mockResolvedValueOnce(jsonResponse({ status: "ok" }))
+      .mockResolvedValueOnce(jsonResponse({ status: "ok" }));
+
+    const lease = await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      role: "maintainer",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    expect(lease.source).toBe("convex");
+    expect(lease.credentialId).toBe("cred-1");
+    expect(lease.payload.groupId).toBe("-100123");
+
+    await lease.heartbeat();
+    await lease.release();
+
+    expect(fetchImpl).toHaveBeenCalledTimes(3);
+    const firstCall = fetchImpl.mock.calls[0];
+    expect(firstCall?.[0]).toContain("/qa-credentials/v1/acquire");
+    const firstInit = firstCall?.[1];
+    const headers = firstInit?.headers as Record<string, string>;
+    expect(headers.authorization).toBe("Bearer maintainer-secret");
+  });
+
+  it("defaults convex credential role to maintainer outside CI", async () => {
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
+      jsonResponse({
+        status: "ok",
+        credentialId: "cred-maintainer-default",
+        leaseToken: "lease-maintainer-default",
+        payload: { groupId: "-100123", driverToken: "driver", sutToken: "sut" },
+      }),
+    );
+
+    await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    const firstCall = fetchImpl.mock.calls[0];
+    const firstInit = firstCall?.[1];
+    const headers = firstInit?.headers as Record<string, string>;
+    expect(headers.authorization).toBe("Bearer maintainer-secret");
+  });
+
+  it("defaults convex credential role to ci when CI=true", async () => {
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
+      jsonResponse({
+        status: "ok",
+        credentialId: "cred-ci-default",
+        leaseToken: "lease-ci-default",
+        payload: { groupId: "-100123", driverToken: "driver", sutToken: "sut" },
+      }),
+    );
+
+    await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      env: {
+        CI: "true",
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_CI: "ci-secret",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    const firstCall = fetchImpl.mock.calls[0];
+    const firstInit = firstCall?.[1];
+    const headers = firstInit?.headers as Record<string, string>;
+    expect(headers.authorization).toBe("Bearer ci-secret");
+  });
+
+  it("retries convex acquire while the pool is exhausted", async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "error",
+          code: "POOL_EXHAUSTED",
+          message: "wait",
+        }),
+      )
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "error",
+          code: "POOL_EXHAUSTED",
+          message: "wait",
+        }),
+      )
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-2",
+          leaseToken: "lease-2",
+          payload: { groupId: "-100456", driverToken: "driver-2", sutToken: "sut-2" },
+        }),
+      );
+
+    const sleeps: number[] = [];
+    let nowMs = 0;
+
+    const lease = await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "90000",
+      },
+      fetchImpl,
+      randomImpl: () => 0,
+      timeImpl: () => nowMs,
+      sleepImpl: async (ms) => {
+        sleeps.push(ms);
+        nowMs += ms;
+      },
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    expect(lease.credentialId).toBe("cred-2");
+    expect(fetchImpl).toHaveBeenCalledTimes(3);
+    expect(sleeps.length).toBe(2);
+    expect(sleeps[0]).toBeGreaterThanOrEqual(100);
+    expect(sleeps[1]).toBeGreaterThan(sleeps[0] ?? 0);
+  });
+
+  it("rejects non-https convex site URLs unless local insecure opt-in is enabled", async () => {
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "http://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        },
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("must use https://");
+  });
+
+  it("allows loopback http URLs when OPENCLAW_QA_ALLOW_INSECURE_HTTP is enabled", async () => {
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
+      jsonResponse({
+        status: "ok",
+        credentialId: "cred-local",
+        leaseToken: "lease-local",
+        payload: { groupId: "-100123", driverToken: "driver", sutToken: "sut" },
+      }),
+    );
+
+    await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      role: "maintainer",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "http://127.0.0.1:3210",
+        OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        OPENCLAW_QA_ALLOW_INSECURE_HTTP: "1",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    const firstCall = fetchImpl.mock.calls[0];
+    expect(firstCall?.[0]).toBe("http://127.0.0.1:3210/qa-credentials/v1/acquire");
+  });
+
+  it("rejects unsafe endpoint prefix overrides", async () => {
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+          OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX: "//evil.example",
+        },
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX must be an absolute path");
+  });
+
+  it("releases acquired lease when payload parsing fails", async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-parse-fail",
+          leaseToken: "lease-parse-fail",
+          payload: { broken: true },
+        }),
+      )
+      .mockResolvedValueOnce(jsonResponse({ status: "ok" }));
+
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        role: "maintainer",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        },
+        fetchImpl,
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: () => {
+          throw new Error("bad payload shape");
+        },
+      }),
+    ).rejects.toThrow("bad payload shape");
+
+    expect(fetchImpl).toHaveBeenCalledTimes(2);
+    expect(fetchImpl.mock.calls[1]?.[0]).toBe(
+      "https://qa-cred.example.convex.site/qa-credentials/v1/release",
+    );
+  });
+
+  it("fails convex mode when auth secret is missing", async () => {
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        role: "maintainer",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        },
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("OPENCLAW_QA_CONVEX_SECRET_MAINTAINER");
+  });
+
+  it("captures heartbeat failures for fail-fast checks", async () => {
+    vi.useFakeTimers();
+    const heartbeat = startQaCredentialLeaseHeartbeat(
+      {
+        source: "convex",
+        kind: "telegram",
+        heartbeatIntervalMs: 50,
+        heartbeat: async () => {
+          throw new Error("heartbeat-down");
+        },
+      },
+      { intervalMs: 50 },
+    );
+
+    await vi.advanceTimersByTimeAsync(55);
+    expect(heartbeat.getFailure()).toBeInstanceOf(Error);
+    expect(() => heartbeat.throwIfFailed()).toThrow("heartbeat-down");
+    await heartbeat.stop();
+  });
+});

openclaw/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.ts

@@ -0,0 +1,580 @@
+import { randomUUID } from "node:crypto";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { z } from "zod";
+
+const DEFAULT_ACQUIRE_TIMEOUT_MS = 90_000;
+const DEFAULT_ENDPOINT_PREFIX = "/qa-credentials/v1";
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000;
+const DEFAULT_HTTP_TIMEOUT_MS = 15_000;
+const DEFAULT_LEASE_TTL_MS = 20 * 60 * 1_000;
+const ALLOW_INSECURE_HTTP_ENV_KEY = "OPENCLAW_QA_ALLOW_INSECURE_HTTP";
+const RETRY_BACKOFF_MS = [500, 1_000, 2_000, 4_000, 5_000] as const;
+const RETRYABLE_ACQUIRE_CODES = new Set(["POOL_EXHAUSTED", "NO_CREDENTIAL_AVAILABLE"]);
+
+const convexAcquireSuccessSchema = z.object({
+  status: z.literal("ok"),
+  credentialId: z.string().min(1),
+  leaseToken: z.string().min(1),
+  payload: z.unknown(),
+  leaseTtlMs: z.number().int().positive().optional(),
+  heartbeatIntervalMs: z.number().int().positive().optional(),
+});
+
+const convexErrorSchema = z.object({
+  status: z.literal("error"),
+  code: z.string().min(1),
+  message: z.string().optional(),
+  retryAfterMs: z.number().int().positive().optional(),
+});
+
+const convexOkSchema = z.object({
+  status: z.literal("ok"),
+});
+
+type ConvexCredentialBrokerConfig = {
+  acquireTimeoutMs: number;
+  acquireUrl: string;
+  authToken: string;
+  heartbeatIntervalMs: number;
+  heartbeatUrl: string;
+  httpTimeoutMs: number;
+  leaseTtlMs: number;
+  ownerId: string;
+  releaseUrl: string;
+  role: QaCredentialRole;
+};
+
+export type QaCredentialLeaseHeartbeat = {
+  getFailure(): Error | null;
+  stop(): Promise<void>;
+  throwIfFailed(): void;
+};
+
+export type QaCredentialRole = "ci" | "maintainer";
+
+export type QaCredentialLeaseSource = "convex" | "env";
+
+export type QaCredentialLease<TPayload> = {
+  credentialId?: string;
+  heartbeat(): Promise<void>;
+  heartbeatIntervalMs: number;
+  kind: string;
+  leaseToken?: string;
+  leaseTtlMs: number;
+  ownerId?: string;
+  payload: TPayload;
+  release(): Promise<void>;
+  role?: QaCredentialRole;
+  source: QaCredentialLeaseSource;
+};
+
+export type AcquireQaCredentialLeaseOptions<TPayload> = {
+  env?: NodeJS.ProcessEnv;
+  fetchImpl?: typeof fetch;
+  kind: string;
+  ownerId?: string;
+  parsePayload: (payload: unknown) => TPayload;
+  randomImpl?: () => number;
+  resolveEnvPayload: () => TPayload;
+  role?: string;
+  sleepImpl?: (ms: number) => Promise<unknown>;
+  source?: string;
+  timeImpl?: () => number;
+};
+
+class QaCredentialBrokerError extends Error {
+  code: string;
+  retryAfterMs?: number;
+
+  constructor(params: { code: string; message: string; retryAfterMs?: number }) {
+    super(params.message);
+    this.name = "QaCredentialBrokerError";
+    this.code = params.code;
+    this.retryAfterMs = params.retryAfterMs;
+  }
+}
+
+function parsePositiveIntegerEnv(env: NodeJS.ProcessEnv, key: string, fallback: number): number {
+  const raw = env[key]?.trim();
+  if (!raw) {
+    return fallback;
+  }
+  const value = Number(raw);
+  if (!Number.isFinite(value) || !Number.isInteger(value) || value < 1) {
+    throw new Error(`${key} must be a positive integer.`);
+  }
+  return value;
+}
+
+function normalizeQaCredentialSource(value: string | undefined): QaCredentialLeaseSource {
+  const normalized = value?.trim().toLowerCase() || "env";
+  if (normalized === "env" || normalized === "convex") {
+    return normalized;
+  }
+  throw new Error(`Credential source must be one of env or convex, got "${value}".`);
+}
+
+function normalizeQaCredentialRole(
+  value: string | undefined,
+  env: NodeJS.ProcessEnv = process.env,
+): QaCredentialRole {
+  const defaultRole = isTruthyOptIn(env.CI) ? "ci" : "maintainer";
+  const normalized = value?.trim().toLowerCase() || defaultRole;
+  if (normalized === "maintainer" || normalized === "ci") {
+    return normalized;
+  }
+  throw new Error(`Credential role must be one of maintainer or ci, got "${value}".`);
+}
+
+function isTruthyOptIn(value: string | undefined) {
+  const normalized = value?.trim().toLowerCase();
+  return normalized === "1" || normalized === "true" || normalized === "yes";
+}
+
+function isLoopbackHostname(hostname: string) {
+  return hostname === "localhost" || hostname === "::1" || hostname.startsWith("127.");
+}
+
+function normalizeConvexSiteUrl(raw: string, env: NodeJS.ProcessEnv): string {
+  let url: URL;
+  try {
+    url = new URL(raw);
+  } catch {
+    throw new Error(`OPENCLAW_QA_CONVEX_SITE_URL must be a valid URL, got "${raw || "<empty>"}".`);
+  }
+  if (url.protocol === "https:") {
+    const text = url.toString();
+    return text.endsWith("/") ? text.slice(0, -1) : text;
+  }
+  if (url.protocol !== "http:") {
+    throw new Error("OPENCLAW_QA_CONVEX_SITE_URL must use https://.");
+  }
+  const allowInsecureHttp = isTruthyOptIn(env[ALLOW_INSECURE_HTTP_ENV_KEY]);
+  if (!allowInsecureHttp || !isLoopbackHostname(url.hostname)) {
+    throw new Error(
+      `OPENCLAW_QA_CONVEX_SITE_URL must use https://. http:// is only allowed for loopback hosts when ${ALLOW_INSECURE_HTTP_ENV_KEY}=1.`,
+    );
+  }
+  const text = url.toString();
+  return text.endsWith("/") ? text.slice(0, -1) : text;
+}
+
+function normalizeEndpointPrefix(value: string | undefined): string {
+  const trimmed = value?.trim();
+  if (!trimmed) {
+    return DEFAULT_ENDPOINT_PREFIX;
+  }
+  const prefixed = trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
+  const normalized = prefixed.endsWith("/") ? prefixed.slice(0, -1) : prefixed;
+  if (!normalized.startsWith("/") || normalized.startsWith("//")) {
+    throw new Error(
+      "OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX must be an absolute path like /qa-credentials/v1.",
+    );
+  }
+  if (normalized.includes("\\") || normalized.split("/").some((segment) => segment === "..")) {
+    throw new Error(
+      "OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX must not contain backslashes or .. path segments.",
+    );
+  }
+  return normalized;
+}
+
+function resolveConvexAuthToken(env: NodeJS.ProcessEnv, role: QaCredentialRole): string {
+  const roleToken =
+    role === "ci"
+      ? env.OPENCLAW_QA_CONVEX_SECRET_CI?.trim()
+      : env.OPENCLAW_QA_CONVEX_SECRET_MAINTAINER?.trim();
+  const token = roleToken;
+  if (token) {
+    return token;
+  }
+  if (role === "ci") {
+    throw new Error("Missing OPENCLAW_QA_CONVEX_SECRET_CI for CI credential access.");
+  }
+  throw new Error("Missing OPENCLAW_QA_CONVEX_SECRET_MAINTAINER for maintainer credential access.");
+}
+
+function joinConvexEndpoint(baseUrl: string, prefix: string, suffix: string): string {
+  const normalizedSuffix = suffix.startsWith("/") ? suffix : `/${suffix}`;
+  const url = new URL(baseUrl);
+  url.pathname = `${prefix}${normalizedSuffix}`.replace(/\/{2,}/gu, "/");
+  url.search = "";
+  url.hash = "";
+  return url.toString();
+}
+
+function resolveConvexCredentialBrokerConfig(params: {
+  env: NodeJS.ProcessEnv;
+  ownerId?: string;
+  role: QaCredentialRole;
+}): ConvexCredentialBrokerConfig {
+  const siteUrl = params.env.OPENCLAW_QA_CONVEX_SITE_URL?.trim();
+  if (!siteUrl) {
+    throw new Error("Missing OPENCLAW_QA_CONVEX_SITE_URL for --credential-source convex.");
+  }
+  const baseUrl = normalizeConvexSiteUrl(siteUrl, params.env);
+  const endpointPrefix = normalizeEndpointPrefix(params.env.OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX);
+  const ownerId =
+    params.ownerId?.trim() ||
+    params.env.OPENCLAW_QA_CREDENTIAL_OWNER_ID?.trim() ||
+    `qa-lab-${params.role}-${process.pid}-${randomUUID().slice(0, 8)}`;
+  return {
+    role: params.role,
+    ownerId,
+    authToken: resolveConvexAuthToken(params.env, params.role),
+    leaseTtlMs: parsePositiveIntegerEnv(
+      params.env,
+      "OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS",
+      DEFAULT_LEASE_TTL_MS,
+    ),
+    heartbeatIntervalMs: parsePositiveIntegerEnv(
+      params.env,
+      "OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS",
+      DEFAULT_HEARTBEAT_INTERVAL_MS,
+    ),
+    acquireTimeoutMs: parsePositiveIntegerEnv(
+      params.env,
+      "OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS",
+      DEFAULT_ACQUIRE_TIMEOUT_MS,
+    ),
+    httpTimeoutMs: parsePositiveIntegerEnv(
+      params.env,
+      "OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS",
+      DEFAULT_HTTP_TIMEOUT_MS,
+    ),
+    acquireUrl: joinConvexEndpoint(baseUrl, endpointPrefix, "acquire"),
+    heartbeatUrl: joinConvexEndpoint(baseUrl, endpointPrefix, "heartbeat"),
+    releaseUrl: joinConvexEndpoint(baseUrl, endpointPrefix, "release"),
+  };
+}
+
+function toBrokerError(params: {
+  payload: unknown;
+  fallback: string;
+}): QaCredentialBrokerError | null {
+  const parsed = convexErrorSchema.safeParse(params.payload);
+  if (!parsed.success) {
+    return null;
+  }
+  return new QaCredentialBrokerError({
+    code: parsed.data.code,
+    message: parsed.data.message?.trim() || params.fallback,
+    retryAfterMs: parsed.data.retryAfterMs,
+  });
+}
+
+async function postConvexBroker(params: {
+  authToken: string;
+  body: Record<string, unknown>;
+  fetchImpl: typeof fetch;
+  timeoutMs: number;
+  url: string;
+}): Promise<unknown> {
+  const response = await params.fetchImpl(params.url, {
+    method: "POST",
+    headers: {
+      authorization: `Bearer ${params.authToken}`,
+      "content-type": "application/json",
+    },
+    body: JSON.stringify(params.body),
+    signal: AbortSignal.timeout(params.timeoutMs),
+  });
+
+  const text = await response.text();
+  const payload: unknown = (() => {
+    if (!text.trim()) {
+      return undefined;
+    }
+    try {
+      return JSON.parse(text) as unknown;
+    } catch {
+      return text;
+    }
+  })();
+
+  const brokerError = toBrokerError({
+    payload,
+    fallback: `Convex credential broker request failed (${response.status}).`,
+  });
+  if (brokerError) {
+    throw brokerError;
+  }
+  if (!response.ok) {
+    throw new Error(
+      `Convex credential broker request to ${params.url} failed with HTTP ${response.status}.`,
+    );
+  }
+  return payload;
+}
+
+function computeAcquireBackoffMs(params: {
+  attempt: number;
+  randomImpl: () => number;
+  retryAfterMs?: number;
+}): number {
+  if (params.retryAfterMs && params.retryAfterMs > 0) {
+    return params.retryAfterMs;
+  }
+  const base = RETRY_BACKOFF_MS[Math.min(RETRY_BACKOFF_MS.length - 1, params.attempt - 1)];
+  const jitter = 0.75 + params.randomImpl() * 0.5;
+  return Math.max(100, Math.round(base * jitter));
+}
+
+function assertConvexOk(payload: unknown, actionLabel: string) {
+  if (payload === undefined) {
+    return;
+  }
+  if (convexOkSchema.safeParse(payload).success) {
+    return;
+  }
+  const brokerError = toBrokerError({
+    payload,
+    fallback: `Convex credential ${actionLabel} failed.`,
+  });
+  if (brokerError) {
+    throw brokerError;
+  }
+  throw new Error(`Convex credential ${actionLabel} failed with an invalid response payload.`);
+}
+
+export async function acquireQaCredentialLease<TPayload>(
+  opts: AcquireQaCredentialLeaseOptions<TPayload>,
+): Promise<QaCredentialLease<TPayload>> {
+  const env = opts.env ?? process.env;
+  const source = normalizeQaCredentialSource(opts.source ?? env.OPENCLAW_QA_CREDENTIAL_SOURCE);
+  if (source === "env") {
+    return {
+      source: "env",
+      kind: opts.kind,
+      payload: opts.resolveEnvPayload(),
+      heartbeatIntervalMs: 0,
+      leaseTtlMs: 0,
+      async heartbeat() {},
+      async release() {},
+    };
+  }
+
+  const role = normalizeQaCredentialRole(opts.role ?? env.OPENCLAW_QA_CREDENTIAL_ROLE, env);
+  const config = resolveConvexCredentialBrokerConfig({
+    env,
+    role,
+    ownerId: opts.ownerId,
+  });
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const sleepImpl =
+    opts.sleepImpl ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
+  const timeImpl = opts.timeImpl ?? (() => Date.now());
+  const randomImpl = opts.randomImpl ?? (() => Math.random());
+  const startedAt = timeImpl();
+  let attempt = 0;
+
+  while (true) {
+    attempt += 1;
+    try {
+      const payload = await postConvexBroker({
+        fetchImpl,
+        timeoutMs: config.httpTimeoutMs,
+        authToken: config.authToken,
+        url: config.acquireUrl,
+        body: {
+          kind: opts.kind,
+          ownerId: config.ownerId,
+          actorRole: config.role,
+          leaseTtlMs: config.leaseTtlMs,
+          heartbeatIntervalMs: config.heartbeatIntervalMs,
+        },
+      });
+      const acquired = convexAcquireSuccessSchema.parse(payload);
+      const releaseLease = async () => {
+        const releasePayload = await postConvexBroker({
+          fetchImpl,
+          timeoutMs: config.httpTimeoutMs,
+          authToken: config.authToken,
+          url: config.releaseUrl,
+          body: {
+            kind: opts.kind,
+            ownerId: config.ownerId,
+            credentialId: acquired.credentialId,
+            leaseToken: acquired.leaseToken,
+            actorRole: config.role,
+          },
+        });
+        assertConvexOk(releasePayload, "release");
+      };
+      let parsedPayload: TPayload;
+      try {
+        parsedPayload = opts.parsePayload(acquired.payload);
+      } catch (error) {
+        try {
+          await releaseLease();
+        } catch (releaseError) {
+          throw new Error(
+            `Convex credential payload validation failed for kind "${opts.kind}" and cleanup release failed: ${formatErrorMessage(error)}; release failed: ${formatErrorMessage(releaseError)}`,
+            { cause: releaseError },
+          );
+        }
+        throw new Error(
+          `Convex credential payload validation failed for kind "${opts.kind}": ${formatErrorMessage(error)}`,
+          { cause: error },
+        );
+      }
+      const leaseTtlMs = acquired.leaseTtlMs ?? config.leaseTtlMs;
+      const heartbeatIntervalMs = acquired.heartbeatIntervalMs ?? config.heartbeatIntervalMs;
+      return {
+        source: "convex",
+        kind: opts.kind,
+        role,
+        ownerId: config.ownerId,
+        credentialId: acquired.credentialId,
+        leaseToken: acquired.leaseToken,
+        leaseTtlMs,
+        heartbeatIntervalMs,
+        payload: parsedPayload,
+        async heartbeat() {
+          const heartbeatPayload = await postConvexBroker({
+            fetchImpl,
+            timeoutMs: config.httpTimeoutMs,
+            authToken: config.authToken,
+            url: config.heartbeatUrl,
+            body: {
+              kind: opts.kind,
+              ownerId: config.ownerId,
+              credentialId: acquired.credentialId,
+              leaseToken: acquired.leaseToken,
+              actorRole: config.role,
+              leaseTtlMs,
+            },
+          });
+          assertConvexOk(heartbeatPayload, "heartbeat");
+        },
+        async release() {
+          await releaseLease();
+        },
+      };
+    } catch (error) {
+      if (error instanceof QaCredentialBrokerError && RETRYABLE_ACQUIRE_CODES.has(error.code)) {
+        const elapsed = timeImpl() - startedAt;
+        if (elapsed >= config.acquireTimeoutMs) {
+          throw new Error(
+            `Convex credential pool exhausted for kind "${opts.kind}" after ${config.acquireTimeoutMs}ms.`,
+            { cause: error },
+          );
+        }
+        const delayMs = Math.min(
+          computeAcquireBackoffMs({
+            attempt,
+            retryAfterMs: error.retryAfterMs,
+            randomImpl,
+          }),
+          Math.max(0, config.acquireTimeoutMs - elapsed),
+        );
+        if (delayMs > 0) {
+          await sleepImpl(delayMs);
+        }
+        continue;
+      }
+      if (error instanceof z.ZodError) {
+        throw new Error(
+          `Convex credential acquire response did not match the expected payload for kind "${opts.kind}": ${error.message}`,
+          { cause: error },
+        );
+      }
+      throw new Error(
+        `Convex credential acquire failed for kind "${opts.kind}": ${formatErrorMessage(error)}`,
+        { cause: error },
+      );
+    }
+  }
+}
+
+export function startQaCredentialLeaseHeartbeat(
+  lease: Pick<QaCredentialLease<unknown>, "heartbeat" | "heartbeatIntervalMs" | "kind" | "source">,
+  opts?: {
+    intervalMs?: number;
+    setTimeoutImpl?: typeof setTimeout;
+    clearTimeoutImpl?: typeof clearTimeout;
+  },
+): QaCredentialLeaseHeartbeat {
+  if (lease.source !== "convex") {
+    return {
+      getFailure: () => null,
+      async stop() {},
+      throwIfFailed() {},
+    };
+  }
+  const intervalMs = opts?.intervalMs ?? lease.heartbeatIntervalMs;
+  if (!Number.isFinite(intervalMs) || intervalMs < 1) {
+    return {
+      getFailure: () => null,
+      async stop() {},
+      throwIfFailed() {},
+    };
+  }
+
+  const setTimeoutImpl = opts?.setTimeoutImpl ?? setTimeout;
+  const clearTimeoutImpl = opts?.clearTimeoutImpl ?? clearTimeout;
+  let failure: Error | null = null;
+  let stopped = false;
+  let timer: ReturnType<typeof setTimeout> | null = null;
+  let inFlight: Promise<void> | null = null;
+
+  const schedule = () => {
+    if (stopped || failure) {
+      return;
+    }
+    timer = setTimeoutImpl(() => {
+      timer = null;
+      if (stopped || failure) {
+        return;
+      }
+      inFlight = (async () => {
+        try {
+          await lease.heartbeat();
+        } catch (error) {
+          failure = new Error(
+            `Credential lease heartbeat failed for kind "${lease.kind}": ${formatErrorMessage(error)}`,
+          );
+          return;
+        } finally {
+          inFlight = null;
+        }
+        schedule();
+      })();
+    }, intervalMs);
+  };
+
+  schedule();
+
+  return {
+    getFailure() {
+      return failure;
+    },
+    throwIfFailed() {
+      if (failure) {
+        throw failure;
+      }
+    },
+    async stop() {
+      stopped = true;
+      if (timer) {
+        clearTimeoutImpl(timer);
+        timer = null;
+      }
+      if (inFlight) {
+        await inFlight.catch(() => {});
+      }
+    },
+  };
+}
+
+export const __testing = {
+  DEFAULT_ACQUIRE_TIMEOUT_MS,
+  DEFAULT_ENDPOINT_PREFIX,
+  DEFAULT_HEARTBEAT_INTERVAL_MS,
+  DEFAULT_LEASE_TTL_MS,
+  computeAcquireBackoffMs,
+  normalizeQaCredentialRole,
+  normalizeQaCredentialSource,
+  parsePositiveIntegerEnv,
+  resolveConvexCredentialBrokerConfig,
+};

openclaw/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts

@@ -0,0 +1,157 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const { startQaGatewayChild, startQaProviderServer } = vi.hoisted(() => ({
+  startQaGatewayChild: vi.fn(),
+  startQaProviderServer: vi.fn(),
+}));
+
+vi.mock("../../gateway-child.js", () => ({
+  startQaGatewayChild,
+}));
+
+vi.mock("../../providers/server-runtime.js", () => ({
+  startQaProviderServer,
+}));
+
+import { startQaLiveLaneGateway } from "./live-gateway.runtime.js";
+
+function createStubTransport(baseUrl = "http://127.0.0.1:43123") {
+  return {
+    requiredPluginIds: ["qa-channel"],
+    createGatewayConfig: () => ({
+      channels: {
+        "qa-channel": {
+          enabled: true,
+          baseUrl,
+          botUserId: "openclaw",
+          botDisplayName: "OpenClaw QA",
+          allowFrom: ["*"],
+          pollTimeoutMs: 250,
+        },
+      },
+      messages: {
+        groupChat: {
+          mentionPatterns: ["\\b@?openclaw\\b"],
+        },
+      },
+    }),
+  };
+}
+
+describe("startQaLiveLaneGateway", () => {
+  const gatewayStop = vi.fn();
+  const gatewayCall = vi.fn();
+  const mockStop = vi.fn();
+
+  beforeEach(() => {
+    gatewayStop.mockReset();
+    gatewayCall.mockReset();
+    mockStop.mockReset();
+    startQaGatewayChild.mockReset();
+    startQaProviderServer.mockReset();
+
+    startQaGatewayChild.mockResolvedValue({
+      call: gatewayCall,
+      cfg: {},
+      stop: gatewayStop,
+    });
+    startQaProviderServer.mockImplementation(async (providerMode: string) =>
+      providerMode === "mock-openai"
+        ? {
+            baseUrl: "http://127.0.0.1:44080",
+            stop: mockStop,
+          }
+        : null,
+    );
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("threads the mock provider base url into the gateway child", async () => {
+    const harness = await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.4",
+      alternateModel: "mock-openai/gpt-5.4-alt",
+      controlUiEnabled: false,
+    });
+
+    expect(startQaProviderServer).toHaveBeenCalledWith("mock-openai");
+    expect(startQaGatewayChild).toHaveBeenCalledWith(
+      expect.objectContaining({
+        transportBaseUrl: "http://127.0.0.1:43123",
+        providerBaseUrl: "http://127.0.0.1:44080/v1",
+        providerMode: "mock-openai",
+      }),
+    );
+
+    await harness.stop();
+    expect(gatewayStop).toHaveBeenCalledTimes(1);
+    expect(mockStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("skips mock bootstrap for live frontier runs", async () => {
+    const harness = await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "live-frontier",
+      primaryModel: "openai/gpt-5.4",
+      alternateModel: "openai/gpt-5.4",
+      controlUiEnabled: false,
+    });
+
+    expect(startQaProviderServer).toHaveBeenCalledWith("live-frontier");
+    expect(startQaGatewayChild).toHaveBeenCalledWith(
+      expect.objectContaining({
+        transportBaseUrl: "http://127.0.0.1:43123",
+        providerBaseUrl: undefined,
+        providerMode: "live-frontier",
+      }),
+    );
+
+    await harness.stop();
+    expect(gatewayStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("still stops the mock server when gateway shutdown fails", async () => {
+    gatewayStop.mockRejectedValueOnce(new Error("gateway down"));
+    const harness = await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.4",
+      alternateModel: "mock-openai/gpt-5.4-alt",
+      controlUiEnabled: false,
+    });
+
+    await expect(harness.stop()).rejects.toThrow(
+      "failed to stop QA live lane resources:\ngateway stop failed: gateway down",
+    );
+    expect(gatewayStop).toHaveBeenCalledTimes(1);
+    expect(mockStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("reports both gateway and mock shutdown failures together", async () => {
+    gatewayStop.mockRejectedValueOnce(new Error("gateway down"));
+    mockStop.mockRejectedValueOnce(new Error("mock down"));
+    const harness = await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.4",
+      alternateModel: "mock-openai/gpt-5.4-alt",
+      controlUiEnabled: false,
+    });
+
+    await expect(harness.stop()).rejects.toThrow(
+      "failed to stop QA live lane resources:\ngateway stop failed: gateway down\nmock provider stop failed: mock down",
+    );
+  });
+});

openclaw/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts

@@ -0,0 +1,77 @@
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
+import { startQaGatewayChild, type QaCliBackendAuthMode } from "../../gateway-child.js";
+import type { QaProviderMode } from "../../model-selection.js";
+import { startQaProviderServer } from "../../providers/server-runtime.js";
+import type { QaThinkingLevel } from "../../qa-gateway-config.js";
+import { appendLiveLaneIssue } from "./live-lane-helpers.js";
+
+async function stopQaLiveLaneResources(resources: {
+  gateway: Awaited<ReturnType<typeof startQaGatewayChild>>;
+  mock: { baseUrl: string; stop(): Promise<void> } | null;
+}) {
+  const errors: string[] = [];
+  try {
+    await resources.gateway.stop();
+  } catch (error) {
+    appendLiveLaneIssue(errors, "gateway stop failed", error);
+  }
+  if (resources.mock) {
+    try {
+      await resources.mock.stop();
+    } catch (error) {
+      appendLiveLaneIssue(errors, "mock provider stop failed", error);
+    }
+  }
+  if (errors.length > 0) {
+    throw new Error(`failed to stop QA live lane resources:\n${errors.join("\n")}`);
+  }
+}
+
+export async function startQaLiveLaneGateway(params: {
+  repoRoot: string;
+  transport: {
+    requiredPluginIds: readonly string[];
+    createGatewayConfig: (params: {
+      baseUrl: string;
+    }) => Pick<OpenClawConfig, "channels" | "messages">;
+  };
+  transportBaseUrl: string;
+  controlUiAllowedOrigins?: string[];
+  providerMode: QaProviderMode;
+  primaryModel: string;
+  alternateModel: string;
+  fastMode?: boolean;
+  thinkingDefault?: QaThinkingLevel;
+  claudeCliAuthMode?: QaCliBackendAuthMode;
+  controlUiEnabled?: boolean;
+  mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
+}) {
+  const mock = await startQaProviderServer(params.providerMode);
+  try {
+    const gateway = await startQaGatewayChild({
+      repoRoot: params.repoRoot,
+      providerBaseUrl: mock ? `${mock.baseUrl}/v1` : undefined,
+      transport: params.transport,
+      transportBaseUrl: params.transportBaseUrl,
+      controlUiAllowedOrigins: params.controlUiAllowedOrigins,
+      providerMode: params.providerMode,
+      primaryModel: params.primaryModel,
+      alternateModel: params.alternateModel,
+      fastMode: params.fastMode,
+      thinkingDefault: params.thinkingDefault,
+      claudeCliAuthMode: params.claudeCliAuthMode,
+      controlUiEnabled: params.controlUiEnabled,
+      mutateConfig: params.mutateConfig,
+    });
+    return {
+      gateway,
+      mock,
+      async stop() {
+        await stopQaLiveLaneResources({ gateway, mock });
+      },
+    };
+  } catch (error) {
+    await mock?.stop().catch(() => {});
+    throw error;
+  }
+}

openclaw/extensions/qa-lab/src/live-transports/shared/live-lane-helpers.ts

@@ -0,0 +1,18 @@
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+
+export function appendLiveLaneIssue(issues: string[], label: string, error: unknown) {
+  issues.push(`${label}: ${formatErrorMessage(error)}`);
+}
+
+export function buildLiveLaneArtifactsError(params: {
+  heading: string;
+  artifacts: Record<string, string>;
+  details?: string[];
+}) {
+  return [
+    params.heading,
+    ...(params.details ?? []),
+    "Artifacts:",
+    ...Object.entries(params.artifacts).map(([label, filePath]) => `- ${label}: ${filePath}`),
+  ].join("\n");
+}

openclaw/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts

@@ -0,0 +1,42 @@
+import path from "node:path";
+import { resolveRepoRelativeOutputDir } from "../../cli-paths.js";
+import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
+import type { QaProviderMode } from "../../run-config.js";
+import { normalizeQaProviderMode } from "../../run-config.js";
+import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
+
+export function resolveLiveTransportQaRunOptions(
+  opts: LiveTransportQaCommandOptions,
+): LiveTransportQaCommandOptions & {
+  repoRoot: string;
+  providerMode: QaProviderMode;
+} {
+  return {
+    repoRoot: path.resolve(opts.repoRoot ?? process.cwd()),
+    outputDir: resolveRepoRelativeOutputDir(
+      path.resolve(opts.repoRoot ?? process.cwd()),
+      opts.outputDir,
+    ),
+    providerMode:
+      opts.providerMode === undefined
+        ? DEFAULT_QA_LIVE_PROVIDER_MODE
+        : normalizeQaProviderMode(opts.providerMode),
+    primaryModel: opts.primaryModel,
+    alternateModel: opts.alternateModel,
+    fastMode: opts.fastMode,
+    allowFailures: opts.allowFailures,
+    scenarioIds: opts.scenarioIds,
+    sutAccountId: opts.sutAccountId,
+    credentialSource: opts.credentialSource?.trim(),
+    credentialRole: opts.credentialRole?.trim(),
+  };
+}
+
+export function printLiveTransportQaArtifacts(
+  laneLabel: string,
+  artifacts: Record<string, string>,
+) {
+  for (const [label, filePath] of Object.entries(artifacts)) {
+    process.stdout.write(`${laneLabel} ${label}: ${filePath}\n`);
+  }
+}

openclaw/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts

@@ -0,0 +1,137 @@
+import type { Command } from "commander";
+import { collectString } from "../../cli-options.js";
+import { DEFAULT_QA_LIVE_PROVIDER_MODE, formatQaProviderModeHelp } from "../../providers/index.js";
+import type { QaProviderModeInput } from "../../run-config.js";
+
+export type LiveTransportQaCommandOptions = {
+  repoRoot?: string;
+  outputDir?: string;
+  providerMode?: QaProviderModeInput;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  allowFailures?: boolean;
+  scenarioIds?: string[];
+  sutAccountId?: string;
+  credentialSource?: string;
+  credentialRole?: string;
+};
+
+type LiveTransportQaCommanderOptions = {
+  repoRoot?: string;
+  outputDir?: string;
+  providerMode?: QaProviderModeInput;
+  model?: string;
+  altModel?: string;
+  scenario?: string[];
+  fast?: boolean;
+  allowFailures?: boolean;
+  sutAccount?: string;
+  credentialSource?: string;
+  credentialRole?: string;
+};
+
+export type LiveTransportQaCliRegistration = {
+  commandName: string;
+  register(qa: Command): void;
+};
+
+export type LiveTransportQaCredentialCliOptions = {
+  sourceDescription?: string;
+  roleDescription?: string;
+};
+
+export function createLazyCliRuntimeLoader<T>(load: () => Promise<T>) {
+  let promise: Promise<T> | null = null;
+  return async () => {
+    promise ??= load();
+    return await promise;
+  };
+}
+
+export function mapLiveTransportQaCommanderOptions(
+  opts: LiveTransportQaCommanderOptions,
+): LiveTransportQaCommandOptions {
+  return {
+    repoRoot: opts.repoRoot,
+    outputDir: opts.outputDir,
+    providerMode: opts.providerMode,
+    primaryModel: opts.model,
+    alternateModel: opts.altModel,
+    fastMode: opts.fast,
+    allowFailures: opts.allowFailures,
+    scenarioIds: opts.scenario,
+    sutAccountId: opts.sutAccount,
+    credentialSource: opts.credentialSource,
+    credentialRole: opts.credentialRole,
+  };
+}
+
+export function registerLiveTransportQaCli(params: {
+  qa: Command;
+  commandName: string;
+  credentialOptions?: LiveTransportQaCredentialCliOptions;
+  description: string;
+  outputDirHelp: string;
+  scenarioHelp: string;
+  sutAccountHelp: string;
+  run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
+}) {
+  const command = params.qa
+    .command(params.commandName)
+    .description(params.description)
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option("--output-dir <path>", params.outputDirHelp)
+    .option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
+    .option("--model <ref>", "Primary provider/model ref")
+    .option("--alt-model <ref>", "Alternate provider/model ref")
+    .option("--scenario <id>", params.scenarioHelp, collectString, [])
+    .option("--fast", "Enable provider fast mode where supported", false)
+    .option(
+      "--allow-failures",
+      "Write artifacts without setting a failing exit code when scenarios fail",
+      false,
+    )
+    .option("--sut-account <id>", params.sutAccountHelp, "sut");
+
+  if (params.credentialOptions) {
+    command.option(
+      "--credential-source <source>",
+      params.credentialOptions.sourceDescription ??
+        "Credential source for live lanes: env or convex (default: env)",
+    );
+    if (params.credentialOptions.roleDescription) {
+      command.option("--credential-role <role>", params.credentialOptions.roleDescription);
+    }
+  }
+
+  command.action(async (opts: LiveTransportQaCommanderOptions) => {
+    await params.run(mapLiveTransportQaCommanderOptions(opts));
+  });
+}
+
+export function createLiveTransportQaCliRegistration(params: {
+  commandName: string;
+  credentialOptions?: LiveTransportQaCredentialCliOptions;
+  description: string;
+  outputDirHelp: string;
+  scenarioHelp: string;
+  sutAccountHelp: string;
+  run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
+}): LiveTransportQaCliRegistration {
+  return {
+    commandName: params.commandName,
+    register(qa: Command) {
+      registerLiveTransportQaCli({
+        qa,
+        commandName: params.commandName,
+        credentialOptions: params.credentialOptions,
+        description: params.description,
+        outputDirHelp: params.outputDirHelp,
+        scenarioHelp: params.scenarioHelp,
+        sutAccountHelp: params.sutAccountHelp,
+        run: params.run,
+      });
+    },
+  };
+}

openclaw/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, it } from "vitest";
+import {
+  LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+  collectLiveTransportStandardScenarioCoverage,
+  findMissingLiveTransportStandardScenarios,
+  selectLiveTransportScenarios,
+} from "./live-transport-scenarios.js";
+
+describe("live transport scenario helpers", () => {
+  it("keeps the repo-wide baseline contract ordered", () => {
+    expect(LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS).toEqual([
+      "canary",
+      "mention-gating",
+      "allowlist-block",
+      "top-level-reply-shape",
+      "restart-resume",
+    ]);
+  });
+
+  it("selects requested scenarios and reports unknown ids with the lane label", () => {
+    const definitions = [
+      { id: "alpha", timeoutMs: 1_000, title: "alpha" },
+      { id: "beta", timeoutMs: 1_000, title: "beta" },
+    ] as const;
+
+    expect(
+      selectLiveTransportScenarios({
+        ids: ["beta"],
+        laneLabel: "Demo",
+        scenarios: definitions,
+      }),
+    ).toEqual([definitions[1]]);
+
+    expect(() =>
+      selectLiveTransportScenarios({
+        ids: ["alpha", "missing"],
+        laneLabel: "Demo",
+        scenarios: definitions,
+      }),
+    ).toThrow("unknown Demo QA scenario id(s): missing");
+  });
+
+  it("dedupes always-on and scenario-backed standard coverage", () => {
+    const covered = collectLiveTransportStandardScenarioCoverage({
+      alwaysOnStandardScenarioIds: ["canary"],
+      scenarios: [
+        {
+          id: "scenario-1",
+          standardId: "mention-gating",
+          timeoutMs: 1_000,
+          title: "mention",
+        },
+        {
+          id: "scenario-2",
+          standardId: "mention-gating",
+          timeoutMs: 1_000,
+          title: "mention again",
+        },
+        {
+          id: "scenario-3",
+          standardId: "restart-resume",
+          timeoutMs: 1_000,
+          title: "restart",
+        },
+      ],
+    });
+
+    expect(covered).toEqual(["canary", "mention-gating", "restart-resume"]);
+    expect(
+      findMissingLiveTransportStandardScenarios({
+        coveredStandardScenarioIds: covered,
+        expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+      }),
+    ).toEqual(["allowlist-block", "top-level-reply-shape"]);
+  });
+});

openclaw/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts

@@ -0,0 +1,149 @@
+export type LiveTransportStandardScenarioId =
+  | "canary"
+  | "mention-gating"
+  | "allowlist-block"
+  | "top-level-reply-shape"
+  | "restart-resume"
+  | "thread-follow-up"
+  | "thread-isolation"
+  | "reaction-observation"
+  | "help-command";
+
+export type LiveTransportScenarioDefinition<TId extends string = string> = {
+  id: TId;
+  standardId?: LiveTransportStandardScenarioId;
+  timeoutMs: number;
+  title: string;
+};
+
+export type LiveTransportStandardScenarioDefinition = {
+  description: string;
+  id: LiveTransportStandardScenarioId;
+  title: string;
+};
+
+export const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] =
+  [
+    {
+      id: "canary",
+      title: "Transport canary",
+      description: "The lane can trigger one known-good reply on the real transport.",
+    },
+    {
+      id: "mention-gating",
+      title: "Mention gating",
+      description: "Messages without the required mention do not trigger a reply.",
+    },
+    {
+      id: "allowlist-block",
+      title: "Sender allowlist block",
+      description: "Non-allowlisted senders do not trigger a reply.",
+    },
+    {
+      id: "top-level-reply-shape",
+      title: "Top-level reply shape",
+      description: "Top-level replies stay top-level when the lane is configured that way.",
+    },
+    {
+      id: "restart-resume",
+      title: "Restart resume",
+      description: "The lane still responds after a gateway restart.",
+    },
+    {
+      id: "thread-follow-up",
+      title: "Thread follow-up",
+      description: "Threaded prompts receive threaded replies with the expected relation metadata.",
+    },
+    {
+      id: "thread-isolation",
+      title: "Thread isolation",
+      description: "Fresh top-level prompts stay out of prior threads.",
+    },
+    {
+      id: "reaction-observation",
+      title: "Reaction observation",
+      description: "Reaction events are observed and normalized correctly.",
+    },
+    {
+      id: "help-command",
+      title: "Help command",
+      description: "The transport-specific help command path replies successfully.",
+    },
+  ] as const;
+
+export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] =
+  [
+    "canary",
+    "mention-gating",
+    "allowlist-block",
+    "top-level-reply-shape",
+    "restart-resume",
+  ] as const;
+
+const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set(
+  LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id),
+);
+
+function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) {
+  for (const id of ids) {
+    if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) {
+      throw new Error(`unknown live transport standard scenario id: ${id}`);
+    }
+  }
+}
+
+export function selectLiveTransportScenarios<TDefinition extends { id: string }>(params: {
+  ids?: string[];
+  laneLabel: string;
+  scenarios: readonly TDefinition[];
+}) {
+  if (!params.ids || params.ids.length === 0) {
+    return [...params.scenarios];
+  }
+  const requested = new Set(params.ids);
+  const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id));
+  const missingIds = [...requested].filter(
+    (id) => !selected.some((scenario) => scenario.id === id),
+  );
+  if (missingIds.length > 0) {
+    throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`);
+  }
+  return selected;
+}
+
+export function collectLiveTransportStandardScenarioCoverage<TId extends string>(params: {
+  alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[];
+  scenarios: readonly LiveTransportScenarioDefinition<TId>[];
+}) {
+  const coverage: LiveTransportStandardScenarioId[] = [];
+  const seen = new Set<LiveTransportStandardScenarioId>();
+  const append = (id: LiveTransportStandardScenarioId | undefined) => {
+    if (!id || seen.has(id)) {
+      return;
+    }
+    seen.add(id);
+    coverage.push(id);
+  };
+
+  assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []);
+  for (const id of params.alwaysOnStandardScenarioIds ?? []) {
+    append(id);
+  }
+  for (const scenario of params.scenarios) {
+    if (scenario.standardId) {
+      assertKnownStandardScenarioIds([scenario.standardId]);
+    }
+    append(scenario.standardId);
+  }
+  return coverage;
+}
+
+export function findMissingLiveTransportStandardScenarios(params: {
+  coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
+  expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
+}) {
+  assertKnownStandardScenarioIds(params.coveredStandardScenarioIds);
+  assertKnownStandardScenarioIds(params.expectedStandardScenarioIds);
+  const covered = new Set(params.coveredStandardScenarioIds);
+  return params.expectedStandardScenarioIds.filter((id) => !covered.has(id));
+}

openclaw/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts

@@ -0,0 +1,22 @@
+import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
+import {
+  printLiveTransportQaArtifacts,
+  resolveLiveTransportQaRunOptions,
+} from "../shared/live-transport-cli.runtime.js";
+import { runTelegramQaLive } from "./telegram-live.runtime.js";
+
+export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) {
+  const runOptions = resolveLiveTransportQaRunOptions(opts);
+  const result = await runTelegramQaLive(runOptions);
+  printLiveTransportQaArtifacts("Telegram QA", {
+    report: result.reportPath,
+    summary: result.summaryPath,
+    "observed messages": result.observedMessagesPath,
+  });
+  if (
+    !runOptions.allowFailures &&
+    result.scenarios.some((scenario) => scenario.status === "fail")
+  ) {
+    process.exitCode = 1;
+  }
+}

openclaw/extensions/qa-lab/src/live-transports/telegram/cli.ts

@@ -0,0 +1,37 @@
+import type { Command } from "commander";
+import {
+  createLazyCliRuntimeLoader,
+  createLiveTransportQaCliRegistration,
+  type LiveTransportQaCliRegistration,
+  type LiveTransportQaCommandOptions,
+} from "../shared/live-transport-cli.js";
+
+type TelegramQaCliRuntime = typeof import("./cli.runtime.js");
+
+const loadTelegramQaCliRuntime = createLazyCliRuntimeLoader<TelegramQaCliRuntime>(
+  () => import("./cli.runtime.js"),
+);
+
+async function runQaTelegram(opts: LiveTransportQaCommandOptions) {
+  const runtime = await loadTelegramQaCliRuntime();
+  await runtime.runQaTelegramCommand(opts);
+}
+
+export const telegramQaCliRegistration: LiveTransportQaCliRegistration =
+  createLiveTransportQaCliRegistration({
+    commandName: "telegram",
+    credentialOptions: {
+      sourceDescription: "Credential source for Telegram QA: env or convex (default: env)",
+      roleDescription:
+        "Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)",
+    },
+    description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness",
+    outputDirHelp: "Telegram QA artifact directory",
+    scenarioHelp: "Run only the named Telegram QA scenario (repeatable)",
+    sutAccountHelp: "Temporary Telegram account id inside the QA gateway config",
+    run: runQaTelegram,
+  });
+
+export function registerTelegramQaCli(qa: Command) {
+  telegramQaCliRegistration.register(qa);
+}

openclaw/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts

@@ -0,0 +1,487 @@
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+  findMissingLiveTransportStandardScenarios,
+} from "../shared/live-transport-scenarios.js";
+import { __testing } from "./telegram-live.runtime.js";
+
+const fetchWithSsrFGuardMock = vi.hoisted(() =>
+  vi.fn(async (params: { url: string; init?: RequestInit; signal?: AbortSignal }) => ({
+    response: await fetch(params.url, {
+      ...params.init,
+      signal: params.signal,
+    }),
+    release: async () => {},
+  })),
+);
+
+vi.mock("openclaw/plugin-sdk/ssrf-runtime", async () => {
+  const actual = await vi.importActual<typeof import("openclaw/plugin-sdk/ssrf-runtime")>(
+    "openclaw/plugin-sdk/ssrf-runtime",
+  );
+  return {
+    ...actual,
+    fetchWithSsrFGuard: fetchWithSsrFGuardMock,
+  };
+});
+
+describe("telegram live qa runtime", () => {
+  afterEach(() => {
+    fetchWithSsrFGuardMock.mockClear();
+    vi.restoreAllMocks();
+    vi.unstubAllGlobals();
+  });
+
+  it("resolves required Telegram QA env vars", () => {
+    expect(
+      __testing.resolveTelegramQaRuntimeEnv({
+        OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123",
+        OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver",
+        OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut",
+      }),
+    ).toEqual({
+      groupId: "-100123",
+      driverToken: "driver",
+      sutToken: "sut",
+    });
+  });
+
+  it("fails when a required Telegram QA env var is missing", () => {
+    expect(() =>
+      __testing.resolveTelegramQaRuntimeEnv({
+        OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123",
+        OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver",
+      }),
+    ).toThrow("OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN");
+  });
+
+  it("fails when the Telegram group id is not numeric", () => {
+    expect(() =>
+      __testing.resolveTelegramQaRuntimeEnv({
+        OPENCLAW_QA_TELEGRAM_GROUP_ID: "qa-group",
+        OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver",
+        OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut",
+      }),
+    ).toThrow("OPENCLAW_QA_TELEGRAM_GROUP_ID must be a numeric Telegram chat id.");
+  });
+
+  it("parses Telegram pooled credential payloads", () => {
+    expect(
+      __testing.parseTelegramQaCredentialPayload({
+        groupId: "-100123",
+        driverToken: "driver",
+        sutToken: "sut",
+      }),
+    ).toEqual({
+      groupId: "-100123",
+      driverToken: "driver",
+      sutToken: "sut",
+    });
+  });
+
+  it("rejects Telegram pooled credential payloads with non-numeric group ids", () => {
+    expect(() =>
+      __testing.parseTelegramQaCredentialPayload({
+        groupId: "qa-group",
+        driverToken: "driver",
+        sutToken: "sut",
+      }),
+    ).toThrow("Telegram credential payload groupId must be a numeric Telegram chat id.");
+  });
+
+  it("injects a temporary Telegram account into the QA gateway config", () => {
+    const baseCfg: OpenClawConfig = {
+      plugins: {
+        allow: ["memory-core", "qa-channel"],
+        entries: {
+          "memory-core": { enabled: true },
+          "qa-channel": { enabled: true },
+        },
+      },
+      channels: {
+        "qa-channel": {
+          enabled: true,
+          baseUrl: "http://127.0.0.1:43123",
+          botUserId: "openclaw",
+          botDisplayName: "OpenClaw QA",
+          allowFrom: ["*"],
+        },
+      },
+    };
+
+    const next = __testing.buildTelegramQaConfig(baseCfg, {
+      groupId: "-100123",
+      sutToken: "sut-token",
+      driverBotId: 42,
+      sutAccountId: "sut",
+    });
+
+    expect(next.plugins?.allow).toContain("telegram");
+    expect(next.plugins?.entries?.telegram).toEqual({ enabled: true });
+    expect(next.channels?.telegram).toEqual({
+      enabled: true,
+      defaultAccount: "sut",
+      accounts: {
+        sut: {
+          enabled: true,
+          botToken: "sut-token",
+          dmPolicy: "disabled",
+          replyToMode: "first",
+          groups: {
+            "-100123": {
+              groupPolicy: "allowlist",
+              allowFrom: ["42"],
+              requireMention: true,
+            },
+          },
+        },
+      },
+    });
+  });
+
+  it("normalizes observed Telegram messages", () => {
+    expect(
+      __testing.normalizeTelegramObservedMessage({
+        update_id: 7,
+        message: {
+          message_id: 9,
+          date: 1_700_000_000,
+          text: "hello",
+          chat: { id: -100123 },
+          from: {
+            id: 42,
+            is_bot: true,
+            username: "driver_bot",
+          },
+          reply_to_message: { message_id: 8 },
+          reply_markup: {
+            inline_keyboard: [[{ text: "Approve" }, { text: "Deny" }]],
+          },
+          photo: [{}],
+        },
+      }),
+    ).toEqual({
+      updateId: 7,
+      messageId: 9,
+      chatId: -100123,
+      senderId: 42,
+      senderIsBot: true,
+      senderUsername: "driver_bot",
+      text: "hello",
+      caption: undefined,
+      replyToMessageId: 8,
+      timestamp: 1_700_000_000_000,
+      inlineButtons: ["Approve", "Deny"],
+      mediaKinds: ["photo"],
+    });
+  });
+
+  it("ignores unrelated sut replies when matching the canary response", () => {
+    expect(
+      __testing.classifyCanaryReply({
+        groupId: "-100123",
+        sutBotId: 88,
+        driverMessageId: 55,
+        message: {
+          updateId: 1,
+          messageId: 9,
+          chatId: -100123,
+          senderId: 88,
+          senderIsBot: true,
+          senderUsername: "sut_bot",
+          text: "other reply",
+          replyToMessageId: 999,
+          timestamp: 1_700_000_000_000,
+          inlineButtons: [],
+          mediaKinds: [],
+        },
+      }),
+    ).toBe("unthreaded");
+    expect(
+      __testing.classifyCanaryReply({
+        groupId: "-100123",
+        sutBotId: 88,
+        driverMessageId: 55,
+        message: {
+          updateId: 2,
+          messageId: 10,
+          chatId: -100123,
+          senderId: 88,
+          senderIsBot: true,
+          senderUsername: "sut_bot",
+          text: "canary reply",
+          replyToMessageId: 55,
+          timestamp: 1_700_000_001_000,
+          inlineButtons: [],
+          mediaKinds: [],
+        },
+      }),
+    ).toBe("match");
+  });
+
+  it("classifies threaded blank sut replies as matches", () => {
+    expect(
+      __testing.classifyCanaryReply({
+        groupId: "-100123",
+        sutBotId: 88,
+        driverMessageId: 55,
+        message: {
+          updateId: 3,
+          messageId: 11,
+          chatId: -100123,
+          senderId: 88,
+          senderIsBot: true,
+          senderUsername: "sut_bot",
+          text: "",
+          replyToMessageId: 55,
+          timestamp: 1_700_000_002_000,
+          inlineButtons: [],
+          mediaKinds: ["photo"],
+        },
+      }),
+    ).toBe("match");
+  });
+
+  it("fails when any requested Telegram scenario id is unknown", () => {
+    expect(() => __testing.findScenario(["telegram-help-command", "typo-scenario"])).toThrow(
+      "unknown Telegram QA scenario id(s): typo-scenario",
+    );
+  });
+
+  it("includes mention gating in the Telegram live scenario catalog", () => {
+    expect(
+      __testing
+        .findScenario([
+          "telegram-help-command",
+          "telegram-commands-command",
+          "telegram-tools-compact-command",
+          "telegram-whoami-command",
+          "telegram-context-command",
+          "telegram-mentioned-message-reply",
+          "telegram-mention-gating",
+        ])
+        .map((scenario) => scenario.id),
+    ).toEqual([
+      "telegram-help-command",
+      "telegram-commands-command",
+      "telegram-tools-compact-command",
+      "telegram-whoami-command",
+      "telegram-context-command",
+      "telegram-mentioned-message-reply",
+      "telegram-mention-gating",
+    ]);
+  });
+
+  it("tracks Telegram live coverage against the shared transport contract", () => {
+    expect(__testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS).toEqual([
+      "canary",
+      "help-command",
+      "mention-gating",
+    ]);
+    expect(
+      findMissingLiveTransportStandardScenarios({
+        coveredStandardScenarioIds: __testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS,
+        expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+      }),
+    ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]);
+  });
+
+  it("matches scenario replies by thread or exact marker", () => {
+    expect(
+      __testing.matchesTelegramScenarioReply({
+        groupId: "-100123",
+        sentMessageId: 55,
+        sutBotId: 88,
+        message: {
+          updateId: 1,
+          messageId: 10,
+          chatId: -100123,
+          senderId: 88,
+          senderIsBot: true,
+          senderUsername: "sut_bot",
+          text: "reply with TELEGRAM_QA_NOMENTION_TOKEN",
+          replyToMessageId: undefined,
+          timestamp: 1_700_000_001_000,
+          inlineButtons: [],
+          mediaKinds: [],
+        },
+        matchText: "TELEGRAM_QA_NOMENTION_TOKEN",
+      }),
+    ).toBe(true);
+    expect(
+      __testing.matchesTelegramScenarioReply({
+        groupId: "-100123",
+        sentMessageId: 55,
+        sutBotId: 88,
+        message: {
+          updateId: 2,
+          messageId: 11,
+          chatId: -100123,
+          senderId: 88,
+          senderIsBot: true,
+          senderUsername: "sut_bot",
+          text: "unrelated chatter",
+          replyToMessageId: undefined,
+          timestamp: 1_700_000_002_000,
+          inlineButtons: [],
+          mediaKinds: [],
+        },
+        matchText: "TELEGRAM_QA_NOMENTION_TOKEN",
+      }),
+    ).toBe(false);
+  });
+
+  it("validates expected Telegram reply markers", () => {
+    expect(() =>
+      __testing.assertTelegramScenarioReply({
+        expectedTextIncludes: ["🧭 Identity", "Channel: telegram"],
+        message: {
+          updateId: 1,
+          messageId: 10,
+          chatId: -100123,
+          senderId: 88,
+          senderIsBot: true,
+          senderUsername: "sut_bot",
+          text: "🧭 Identity\nChannel: telegram\nUser id: 42",
+          replyToMessageId: 55,
+          timestamp: 1_700_000_001_000,
+          inlineButtons: [],
+          mediaKinds: [],
+        },
+      }),
+    ).not.toThrow();
+    expect(() =>
+      __testing.assertTelegramScenarioReply({
+        expectedTextIncludes: ["Use /tools verbose for descriptions."],
+        message: {
+          updateId: 2,
+          messageId: 11,
+          chatId: -100123,
+          senderId: 88,
+          senderIsBot: true,
+          senderUsername: "sut_bot",
+          text: "exec\nbash",
+          replyToMessageId: 55,
+          timestamp: 1_700_000_002_000,
+          inlineButtons: [],
+          mediaKinds: [],
+        },
+      }),
+    ).toThrow("reply message 11 missing expected text: Use /tools verbose for descriptions.");
+  });
+
+  it("adds an abort deadline to Telegram API requests", async () => {
+    const controller = new AbortController();
+    const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal);
+    let signal: AbortSignal | undefined;
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (_input: string | URL | globalThis.Request, init?: RequestInit) => {
+        signal = init?.signal as AbortSignal | undefined;
+        return new Response(JSON.stringify({ ok: true, result: { id: 42 } }), {
+          status: 200,
+          headers: {
+            "content-type": "application/json",
+          },
+        });
+      }),
+    );
+
+    await expect(__testing.callTelegramApi("token", "getMe", undefined, 25)).resolves.toEqual({
+      id: 42,
+    });
+    expect(timeoutSpy).toHaveBeenCalledWith(25);
+    expect(signal).toBe(controller.signal);
+    expect(signal?.aborted).toBe(false);
+    controller.abort();
+    expect(signal?.aborted).toBe(true);
+  });
+
+  it("redacts observed message content by default in artifacts", () => {
+    expect(
+      __testing.buildObservedMessagesArtifact({
+        includeContent: false,
+        observedMessages: [
+          {
+            updateId: 1,
+            messageId: 9,
+            chatId: -100123,
+            senderId: 42,
+            senderIsBot: true,
+            senderUsername: "driver_bot",
+            text: "secret text",
+            caption: "secret caption",
+            replyToMessageId: 8,
+            timestamp: 1_700_000_000_000,
+            inlineButtons: ["Approve"],
+            mediaKinds: ["photo"],
+          },
+        ],
+      }),
+    ).toEqual([
+      {
+        updateId: 1,
+        messageId: 9,
+        chatId: -100123,
+        senderId: 42,
+        senderIsBot: true,
+        senderUsername: "driver_bot",
+        replyToMessageId: 8,
+        timestamp: 1_700_000_000_000,
+        inlineButtons: ["Approve"],
+        mediaKinds: ["photo"],
+      },
+    ]);
+  });
+
+  it("formats phase-specific canary diagnostics with context", () => {
+    const error = new Error(
+      "SUT bot did not send any group reply after the canary command within 30s.",
+    );
+    error.name = "TelegramQaCanaryError";
+    Object.assign(error, {
+      phase: "sut_reply_timeout",
+      context: {
+        driverMessageId: 55,
+        sutBotId: 88,
+      },
+    });
+
+    const message = __testing.canaryFailureMessage({
+      error,
+      groupId: "-100123",
+      driverBotId: 42,
+      driverUsername: "driver_bot",
+      sutBotId: 88,
+      sutUsername: "sut_bot",
+    });
+    expect(message).toContain("Phase: sut_reply_timeout");
+    expect(message).toContain("- driverMessageId: 55");
+    expect(message).not.toContain("- sutBotId: 88\n- sutBotId: 88");
+    expect(message).toContain(
+      "Confirm the SUT bot is present in the target private group and can receive /help@BotUsername commands there.",
+    );
+  });
+
+  it("treats null canary context as a non-canary error", () => {
+    const error = new Error("boom");
+    error.name = "TelegramQaCanaryError";
+    Object.assign(error, {
+      phase: "sut_reply_timeout",
+      context: null,
+    });
+
+    const message = __testing.canaryFailureMessage({
+      error,
+      groupId: "-100123",
+      driverBotId: 42,
+      driverUsername: "driver_bot",
+      sutBotId: 88,
+      sutUsername: "sut_bot",
+    });
+
+    expect(message).toContain("Phase: unknown");
+    expect(message).toContain("boom");
+  });
+});

openclaw/extensions/qa-lab/src/manual-lane.runtime.test.ts

@@ -0,0 +1,132 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const { startQaLabServer, startQaGatewayChild, startQaProviderServer } = vi.hoisted(() => ({
+  startQaLabServer: vi.fn(),
+  startQaGatewayChild: vi.fn(),
+  startQaProviderServer: vi.fn(),
+}));
+
+vi.mock("./lab-server.js", () => ({
+  startQaLabServer,
+}));
+
+vi.mock("./gateway-child.js", () => ({
+  startQaGatewayChild,
+}));
+
+vi.mock("./providers/server-runtime.js", () => ({
+  startQaProviderServer,
+}));
+
+import { runQaManualLane } from "./manual-lane.runtime.js";
+
+describe("runQaManualLane", () => {
+  const gatewayStop = vi.fn();
+  const mockStop = vi.fn();
+  const labStop = vi.fn();
+
+  beforeEach(() => {
+    gatewayStop.mockReset();
+    mockStop.mockReset();
+    labStop.mockReset();
+    startQaLabServer.mockReset();
+    startQaGatewayChild.mockReset();
+    startQaProviderServer.mockReset();
+
+    startQaLabServer.mockResolvedValue({
+      listenUrl: "http://127.0.0.1:43124",
+      baseUrl: "http://127.0.0.1:58000",
+      state: {
+        reset: vi.fn(),
+        addInboundMessage: vi.fn(),
+        addOutboundMessage: vi.fn(),
+        readMessage: vi.fn(),
+        searchMessages: vi.fn(() => []),
+        waitFor: vi.fn(),
+        getSnapshot: () => ({
+          messages: [
+            {
+              direction: "outbound",
+              conversation: { id: "qa-operator" },
+              text: "Protocol note: mock reply.",
+            },
+          ],
+        }),
+      },
+      stop: labStop,
+    });
+
+    startQaGatewayChild.mockResolvedValue({
+      call: vi
+        .fn()
+        .mockResolvedValueOnce({ runId: "run-1" })
+        .mockResolvedValueOnce({ status: "ok" }),
+      stop: gatewayStop,
+    });
+
+    startQaProviderServer.mockImplementation(async (providerMode: string) =>
+      providerMode === "mock-openai"
+        ? {
+            baseUrl: "http://127.0.0.1:44080",
+            stop: mockStop,
+          }
+        : null,
+    );
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("starts the mock provider and threads its base url into the gateway child", async () => {
+    const result = await runQaManualLane({
+      repoRoot: "/tmp/openclaw-repo",
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.4",
+      alternateModel: "mock-openai/gpt-5.4-alt",
+      message: "check the kickoff file",
+      timeoutMs: 5_000,
+    });
+
+    expect(startQaProviderServer).toHaveBeenCalledWith("mock-openai");
+    expect(startQaGatewayChild).toHaveBeenCalledWith(
+      expect.objectContaining({
+        repoRoot: "/tmp/openclaw-repo",
+        providerMode: "mock-openai",
+        providerBaseUrl: "http://127.0.0.1:44080/v1",
+      }),
+    );
+    expect(startQaLabServer).toHaveBeenCalledWith({
+      repoRoot: "/tmp/openclaw-repo",
+      embeddedGateway: "disabled",
+    });
+    expect(result.reply).toBe("Protocol note: mock reply.");
+    expect(gatewayStop).toHaveBeenCalledTimes(1);
+    expect(mockStop).toHaveBeenCalledTimes(1);
+    expect(labStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("skips the mock provider bootstrap for live frontier runs", async () => {
+    const result = await runQaManualLane({
+      repoRoot: "/tmp/openclaw-repo",
+      providerMode: "live-frontier",
+      primaryModel: "openai/gpt-5.4",
+      alternateModel: "openai/gpt-5.4",
+      message: "check the kickoff file",
+      timeoutMs: 5_000,
+    });
+
+    expect(startQaProviderServer).toHaveBeenCalledWith("live-frontier");
+    expect(startQaLabServer).toHaveBeenCalledWith({
+      repoRoot: "/tmp/openclaw-repo",
+      embeddedGateway: "disabled",
+    });
+    expect(startQaGatewayChild).toHaveBeenCalledWith(
+      expect.objectContaining({
+        providerMode: "live-frontier",
+        providerBaseUrl: undefined,
+      }),
+    );
+    expect(result.reply).toBe("Protocol note: mock reply.");
+  });
+});

openclaw/extensions/qa-lab/src/manual-lane.runtime.ts

@@ -0,0 +1,134 @@
+import { randomUUID } from "node:crypto";
+import { setTimeout as sleep } from "node:timers/promises";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { startQaGatewayChild } from "./gateway-child.js";
+import { startQaLabServer } from "./lab-server.js";
+import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
+import type { QaProviderMode } from "./model-selection.js";
+import { startQaProviderServer } from "./providers/server-runtime.js";
+import type { QaThinkingLevel } from "./qa-gateway-config.js";
+import { createQaTransportAdapter, type QaTransportId } from "./qa-transport-registry.js";
+
+type QaManualLaneParams = {
+  repoRoot: string;
+  transportId?: QaTransportId;
+  providerMode: QaProviderMode;
+  primaryModel: string;
+  alternateModel: string;
+  fastMode?: boolean;
+  thinkingDefault?: QaThinkingLevel;
+  message: string;
+  timeoutMs?: number;
+};
+
+function resolveManualLaneTimeoutMs(params: {
+  providerMode: QaProviderMode;
+  primaryModel: string;
+  alternateModel: string;
+  timeoutMs?: number;
+}) {
+  if (
+    typeof params.timeoutMs === "number" &&
+    Number.isFinite(params.timeoutMs) &&
+    params.timeoutMs > 0
+  ) {
+    return params.timeoutMs;
+  }
+  return resolveQaLiveTurnTimeoutMs(
+    {
+      providerMode: params.providerMode,
+      primaryModel: params.primaryModel,
+      alternateModel: params.alternateModel,
+    },
+    120_000,
+    params.primaryModel,
+  );
+}
+
+export async function runQaManualLane(params: QaManualLaneParams) {
+  const sessionSuffix = params.primaryModel.replace(/[^a-z0-9._-]+/gi, "-");
+  const lab = await startQaLabServer({
+    repoRoot: params.repoRoot,
+    embeddedGateway: "disabled",
+  });
+  const transport = createQaTransportAdapter({
+    id: params.transportId ?? "qa-channel",
+    state: lab.state,
+  });
+  const mock = await startQaProviderServer(params.providerMode);
+  const gateway = await startQaGatewayChild({
+    repoRoot: params.repoRoot,
+    providerBaseUrl: mock ? `${mock.baseUrl}/v1` : undefined,
+    transport,
+    transportBaseUrl: lab.listenUrl,
+    providerMode: params.providerMode,
+    primaryModel: params.primaryModel,
+    alternateModel: params.alternateModel,
+    fastMode: params.fastMode,
+    thinkingDefault: params.thinkingDefault,
+    controlUiEnabled: false,
+  });
+
+  const timeoutMs = resolveManualLaneTimeoutMs({
+    providerMode: params.providerMode,
+    primaryModel: params.primaryModel,
+    alternateModel: params.alternateModel,
+    timeoutMs: params.timeoutMs,
+  });
+  try {
+    const delivery = transport.buildAgentDelivery({
+      target: "dm:qa-operator",
+    });
+    const started = (await gateway.call(
+      "agent",
+      {
+        idempotencyKey: randomUUID(),
+        agentId: "qa",
+        sessionKey: `agent:qa:manual:${sessionSuffix}`,
+        message: params.message,
+        deliver: true,
+        channel: delivery.channel,
+        to: "dm:qa-operator",
+        replyChannel: delivery.replyChannel,
+        replyTo: delivery.replyTo,
+      },
+      { timeoutMs: 30_000 },
+    )) as { runId?: string };
+
+    if (!started.runId) {
+      throw new Error(`agent call did not return a runId: ${JSON.stringify(started)}`);
+    }
+
+    const waited = (await gateway.call(
+      "agent.wait",
+      {
+        runId: started.runId,
+        timeoutMs,
+      },
+      { timeoutMs: timeoutMs + 5_000 },
+    )) as { status?: string; error?: string };
+
+    await sleep(500);
+
+    const reply =
+      lab.state
+        .getSnapshot()
+        .messages.findLast(
+          (candidate) =>
+            candidate.direction === "outbound" && candidate.conversation.id === "qa-operator",
+        )?.text ?? null;
+
+    return {
+      model: params.primaryModel,
+      waited,
+      reply,
+      watchUrl: lab.baseUrl,
+    };
+  } catch (error) {
+    throw new Error(formatErrorMessage(error), { cause: error });
+  } finally {
+    await gateway.stop();
+    await mock?.stop();
+    await lab.stop();
+  }
+}

openclaw/extensions/qa-lab/src/model-catalog.runtime.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from "vitest";
+import { selectQaRunnerModelOptions } from "./model-catalog.runtime.js";
+
+describe("qa runner model catalog", () => {
+  it("filters to available rows and prefers gpt-5.4 first", () => {
+    expect(
+      selectQaRunnerModelOptions([
+        {
+          key: "anthropic/claude-sonnet-4-5",
+          name: "Claude Sonnet 4.5",
+          input: "text",
+          available: true,
+          missing: false,
+        },
+        {
+          key: "openai/gpt-5.4",
+          name: "gpt-5.4",
+          input: "text,image",
+          available: true,
+          missing: false,
+        },
+        {
+          key: "openrouter/auto",
+          name: "OpenRouter Auto",
+          input: "text",
+          available: false,
+          missing: false,
+        },
+      ]).map((entry) => entry.key),
+    ).toEqual(["openai/gpt-5.4", "anthropic/claude-sonnet-4-5"]);
+  });
+});

openclaw/extensions/qa-lab/src/model-catalog.runtime.ts

@@ -0,0 +1,197 @@
+import { spawn } from "node:child_process";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
+import { resolveQaNodeExecPath } from "./node-exec.js";
+import {
+  isPreferredQaLiveFrontierCatalogModel,
+  QA_FRONTIER_CATALOG_ALTERNATE_MODEL,
+  QA_FRONTIER_CATALOG_PRIMARY_MODEL,
+  QA_FRONTIER_PROVIDER_IDS,
+} from "./providers/live-frontier/catalog.js";
+import {
+  createQaChannelGatewayConfig,
+  QA_CHANNEL_REQUIRED_PLUGIN_IDS,
+} from "./qa-channel-transport.js";
+import { buildQaGatewayConfig } from "./qa-gateway-config.js";
+
+type ModelRow = {
+  key: string;
+  name: string;
+  input: string;
+  available: boolean | null;
+  missing: boolean;
+};
+
+export type QaRunnerModelOption = {
+  key: string;
+  name: string;
+  provider: string;
+  input: string;
+  preferred: boolean;
+};
+
+function splitModelKey(key: string) {
+  const slash = key.indexOf("/");
+  if (slash <= 0 || slash === key.length - 1) {
+    return null;
+  }
+  return {
+    provider: key.slice(0, slash),
+    model: key.slice(slash + 1),
+  };
+}
+
+export function selectQaRunnerModelOptions(rows: ModelRow[]): QaRunnerModelOption[] {
+  const options = rows
+    .filter((row) => row.available === true && !row.missing)
+    .map((row) => {
+      const parsed = splitModelKey(row.key);
+      return {
+        key: row.key,
+        name: row.name,
+        provider: parsed?.provider ?? "unknown",
+        input: row.input,
+        preferred: isPreferredQaLiveFrontierCatalogModel(row.key),
+      } satisfies QaRunnerModelOption;
+    });
+
+  return options.toSorted((left, right) => {
+    if (left.preferred !== right.preferred) {
+      return left.preferred ? -1 : 1;
+    }
+    const providerCompare = left.provider.localeCompare(right.provider);
+    if (providerCompare !== 0) {
+      return providerCompare;
+    }
+    return left.name.localeCompare(right.name);
+  });
+}
+
+const CATALOG_ABORT_ERROR_MESSAGE = "qa model catalog aborted";
+
+function createCatalogAbortError() {
+  return new Error(CATALOG_ABORT_ERROR_MESSAGE);
+}
+
+function killProcessTree(pid: number | undefined, signal: NodeJS.Signals) {
+  if (pid === undefined) {
+    return;
+  }
+  try {
+    if (process.platform === "win32") {
+      process.kill(pid, signal);
+      return;
+    }
+    process.kill(-pid, signal);
+  } catch {
+    try {
+      process.kill(pid, signal);
+    } catch {
+      // The process already exited.
+    }
+  }
+}
+
+export async function loadQaRunnerModelOptions(params: { repoRoot: string; signal?: AbortSignal }) {
+  const tempRoot = await fs.mkdtemp(
+    path.join(resolvePreferredOpenClawTmpDir(), "openclaw-qa-model-catalog-"),
+  );
+  const workspaceDir = path.join(tempRoot, "workspace");
+  const stateDir = path.join(tempRoot, "state");
+  const homeDir = path.join(tempRoot, "home");
+  const configPath = path.join(tempRoot, "openclaw.json");
+
+  try {
+    await Promise.all([
+      fs.mkdir(workspaceDir, { recursive: true }),
+      fs.mkdir(stateDir, { recursive: true }),
+      fs.mkdir(homeDir, { recursive: true }),
+    ]);
+    const cfg = buildQaGatewayConfig({
+      bind: "loopback",
+      gatewayPort: 0,
+      gatewayToken: "qa-model-catalog",
+      workspaceDir,
+      providerMode: "live-frontier",
+      primaryModel: QA_FRONTIER_CATALOG_PRIMARY_MODEL,
+      alternateModel: QA_FRONTIER_CATALOG_ALTERNATE_MODEL,
+      enabledProviderIds: [...QA_FRONTIER_PROVIDER_IDS],
+      imageGenerationModel: null,
+      controlUiEnabled: false,
+      transportPluginIds: QA_CHANNEL_REQUIRED_PLUGIN_IDS,
+      transportConfig: createQaChannelGatewayConfig({
+        baseUrl: "http://127.0.0.1:9",
+      }),
+    });
+    await fs.writeFile(configPath, `${JSON.stringify(cfg, null, 2)}\n`, "utf8");
+
+    const stdout: Buffer[] = [];
+    const stderr: Buffer[] = [];
+    const nodeExecPath = await resolveQaNodeExecPath();
+    await new Promise<void>((resolve, reject) => {
+      let aborted = params.signal?.aborted === true;
+      let forceKillTimer: NodeJS.Timeout | undefined;
+      const child = spawn(nodeExecPath, ["dist/index.js", "models", "list", "--all", "--json"], {
+        cwd: params.repoRoot,
+        env: {
+          ...process.env,
+          HOME: homeDir,
+          OPENCLAW_HOME: homeDir,
+          OPENCLAW_CONFIG_PATH: configPath,
+          OPENCLAW_STATE_DIR: stateDir,
+          OPENCLAW_OAUTH_DIR: path.join(stateDir, "credentials"),
+          OPENCLAW_CODEX_DISCOVERY_LIVE: "0",
+        },
+        detached: process.platform !== "win32",
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+      const cleanup = () => {
+        params.signal?.removeEventListener("abort", abortCatalogLoad);
+        if (forceKillTimer) {
+          clearTimeout(forceKillTimer);
+        }
+      };
+      const abortCatalogLoad = () => {
+        aborted = true;
+        killProcessTree(child.pid, "SIGTERM");
+        forceKillTimer = setTimeout(() => {
+          killProcessTree(child.pid, "SIGKILL");
+        }, 1_000);
+        forceKillTimer.unref();
+      };
+      if (aborted) {
+        abortCatalogLoad();
+      } else {
+        params.signal?.addEventListener("abort", abortCatalogLoad, { once: true });
+      }
+      child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk)));
+      child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk)));
+      child.once("error", (error) => {
+        cleanup();
+        reject(aborted ? createCatalogAbortError() : error);
+      });
+      child.once("exit", (code) => {
+        cleanup();
+        if (aborted) {
+          reject(createCatalogAbortError());
+          return;
+        }
+        if (code === 0) {
+          resolve();
+          return;
+        }
+        reject(
+          new Error(
+            `qa model catalog failed (${code ?? "unknown"}): ${Buffer.concat(stderr).toString("utf8").trim()}`,
+          ),
+        );
+      });
+    });
+
+    const payload = JSON.parse(Buffer.concat(stdout).toString("utf8")) as { models?: ModelRow[] };
+    return selectQaRunnerModelOptions(payload.models ?? []);
+  } finally {
+    await fs.rm(tempRoot, { recursive: true, force: true });
+  }
+}

openclaw/extensions/qa-lab/src/model-selection.runtime.test.ts

@@ -0,0 +1,71 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const { resolveEnvApiKey, loadAuthProfileStoreForRuntime, listProfilesForProvider } = vi.hoisted(
+  () => ({
+    resolveEnvApiKey: vi.fn(),
+    loadAuthProfileStoreForRuntime: vi.fn(),
+    listProfilesForProvider: vi.fn(),
+  }),
+);
+
+vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
+  resolveEnvApiKey,
+}));
+
+vi.mock("openclaw/plugin-sdk/agent-runtime", () => ({
+  loadAuthProfileStoreForRuntime,
+  listProfilesForProvider,
+}));
+
+import {
+  defaultQaRuntimeModelForMode,
+  resolveQaPreferredLiveModel,
+} from "./model-selection.runtime.js";
+
+describe("qa model selection runtime", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    resolveEnvApiKey.mockReturnValue(undefined);
+    loadAuthProfileStoreForRuntime.mockReturnValue({ profiles: {} });
+    listProfilesForProvider.mockReturnValue([]);
+  });
+
+  it("keeps the OpenAI live default when an API key is configured", () => {
+    resolveEnvApiKey.mockReturnValue({ apiKey: "sk-test" });
+
+    expect(resolveQaPreferredLiveModel()).toBeUndefined();
+    expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.4");
+    expect(loadAuthProfileStoreForRuntime).not.toHaveBeenCalled();
+  });
+
+  it("prefers the Codex OAuth live default when only Codex auth profiles are available", () => {
+    listProfilesForProvider.mockImplementation((_store: unknown, provider: string) =>
+      provider === "openai-codex" ? ["openai-codex:user@example.com"] : [],
+    );
+
+    expect(resolveQaPreferredLiveModel()).toBe("openai-codex/gpt-5.4");
+    expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai-codex/gpt-5.4");
+  });
+
+  it("keeps the OpenAI live default when stored OpenAI profiles are available", () => {
+    listProfilesForProvider.mockImplementation((_store: unknown, provider: string) =>
+      provider === "openai" || provider === "openai-codex" ? [`${provider}:user@example.com`] : [],
+    );
+
+    expect(resolveQaPreferredLiveModel()).toBeUndefined();
+    expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.4");
+  });
+
+  it("leaves mock defaults unchanged", () => {
+    listProfilesForProvider.mockImplementation((_store: unknown, provider: string) =>
+      provider === "openai-codex" ? ["openai-codex:user@example.com"] : [],
+    );
+
+    expect(defaultQaRuntimeModelForMode("mock-openai")).toBe("mock-openai/gpt-5.4");
+    expect(defaultQaRuntimeModelForMode("mock-openai", { alternate: true })).toBe(
+      "mock-openai/gpt-5.4-alt",
+    );
+    expect(defaultQaRuntimeModelForMode("aimock")).toBe("aimock/gpt-5.4");
+    expect(defaultQaRuntimeModelForMode("aimock", { alternate: true })).toBe("aimock/gpt-5.4-alt");
+  });
+});

openclaw/extensions/qa-lab/src/model-selection.runtime.ts

@@ -0,0 +1,29 @@
+import {
+  defaultQaModelForMode,
+  normalizeQaProviderMode,
+  type QaProviderModeInput,
+} from "./model-selection.js";
+import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
+import { resolveQaLiveFrontierPreferredModel } from "./providers/live-frontier/model-selection.runtime.js";
+
+export function resolveQaPreferredLiveModel() {
+  return resolveQaLiveFrontierPreferredModel();
+}
+
+export function defaultQaRuntimeModelForMode(
+  mode: QaProviderModeInput,
+  options?: {
+    alternate?: boolean;
+    preferredLiveModel?: string;
+  },
+) {
+  const preferredLiveModel =
+    options?.preferredLiveModel ??
+    (normalizeQaProviderMode(mode) === DEFAULT_QA_LIVE_PROVIDER_MODE
+      ? resolveQaPreferredLiveModel()
+      : undefined);
+  return defaultQaModelForMode(mode, {
+    ...options,
+    preferredLiveModel,
+  });
+}

openclaw/extensions/qa-lab/src/model-selection.ts

@@ -0,0 +1,46 @@
+import {
+  DEFAULT_QA_LIVE_PROVIDER_MODE,
+  getQaProvider,
+  type QaProviderMode,
+  type QaProviderModeInput,
+} from "./providers/index.js";
+
+export type { QaProviderMode, QaProviderModeInput } from "./providers/index.js";
+
+export type QaModelSelection = {
+  primaryModel: string;
+  alternateModel: string;
+};
+
+export { normalizeQaProviderMode } from "./providers/index.js";
+
+export function defaultQaModelForMode(
+  mode: QaProviderModeInput,
+  options?: {
+    alternate?: boolean;
+    preferredLiveModel?: string;
+  },
+) {
+  return getQaProvider(mode).defaultModel(options);
+}
+
+export function splitQaModelRef(ref: string) {
+  const slash = ref.indexOf("/");
+  if (slash <= 0 || slash === ref.length - 1) {
+    return null;
+  }
+  return {
+    provider: ref.slice(0, slash),
+    model: ref.slice(slash + 1),
+  };
+}
+
+export function isQaFastModeModelRef(ref: string) {
+  return getQaProvider(DEFAULT_QA_LIVE_PROVIDER_MODE).usesFastModeByDefault(ref);
+}
+
+export function isQaFastModeEnabled(selection: QaModelSelection) {
+  return (
+    isQaFastModeModelRef(selection.primaryModel) || isQaFastModeModelRef(selection.alternateModel)
+  );
+}

openclaw/extensions/qa-lab/src/model-switch-eval.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest";
+import { hasModelSwitchContinuityEvidence } from "./model-switch-eval.js";
+
+describe("qa model-switch evaluation", () => {
+  it("accepts direct handoff replies that mention the kickoff task", () => {
+    expect(
+      hasModelSwitchContinuityEvidence(
+        "Handoff confirmed: I reread QA_KICKOFF_TASK.md and switched to gpt.",
+      ),
+    ).toBe(true);
+  });
+
+  it("accepts short mission-oriented switch confirmations", () => {
+    expect(
+      hasModelSwitchContinuityEvidence(
+        "model switch complete. reread the kickoff task; qa mission stays the same.",
+      ),
+    ).toBe(true);
+  });
+
+  it("accepts concise kickoff note confirmations", () => {
+    expect(
+      hasModelSwitchContinuityEvidence(
+        "Handoff clean: after the model switch, I reread the kickoff note.",
+      ),
+    ).toBe(true);
+  });
+
+  it("accepts concise paraphrases of the kickoff task after a handoff", () => {
+    expect(
+      hasModelSwitchContinuityEvidence(
+        "Handoff is clear: after the model switch, read source and docs first, run seeded qa-channel scenarios, and report worked, failed, blocked, and follow-up.",
+      ),
+    ).toBe(true);
+  });
+
+  it("rejects unrelated handoff chatter that never confirms the kickoff reread", () => {
+    expect(
+      hasModelSwitchContinuityEvidence(
+        "subagent-handoff confirmed. qa report update: scenario pass. qa run complete.",
+      ),
+    ).toBe(false);
+  });
+
+  it("rejects over-scoped multi-line wrap-ups even if they mention a switch and the mission", () => {
+    expect(
+      hasModelSwitchContinuityEvidence(
+        `model switch acknowledged. qa mission stays the same.
+
+Final QA tally update: all mandatory scenarios resolved. QA run complete.`,
+      ),
+    ).toBe(false);
+  });
+});

openclaw/extensions/qa-lab/src/model-switch-eval.ts

@@ -0,0 +1,28 @@
+import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
+
+export function hasModelSwitchContinuityEvidence(text: string) {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  const mentionsHandoff =
+    lower.includes("handoff") || lower.includes("model switch") || lower.includes("switched");
+  const mentionsKickoffTask =
+    lower.includes("qa_kickoff_task") ||
+    lower.includes("qa/scenarios/index.md") ||
+    lower.includes("scenario pack") ||
+    lower.includes("kickoff task") ||
+    lower.includes("kickoff note") ||
+    lower.includes("qa mission") ||
+    (lower.includes("source and docs") &&
+      lower.includes("qa-channel scenarios") &&
+      lower.includes("worked") &&
+      lower.includes("blocked") &&
+      lower.includes("follow-up"));
+  const hasScopeLeak =
+    lower.includes("subagent-handoff") ||
+    lower.includes("delegated task") ||
+    lower.includes("final qa tally") ||
+    lower.includes("qa run complete") ||
+    lower.includes("all mandatory scenarios");
+  const looksOverlong =
+    text.length > 280 || text.includes("\n\n") || text.includes("|---") || text.includes("### ");
+  return mentionsHandoff && mentionsKickoffTask && !hasScopeLeak && !looksOverlong;
+}

openclaw/extensions/qa-lab/src/multipass.runtime.test.ts

@@ -0,0 +1,282 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
+import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from "vitest";
+
+const execFileMock = vi.hoisted(() => vi.fn());
+
+function readRootPackageManager() {
+  const packageJson = JSON.parse(
+    fs.readFileSync(path.join(process.cwd(), "package.json"), "utf8"),
+  ) as {
+    packageManager?: string;
+  };
+  return packageJson.packageManager;
+}
+
+vi.mock("node:child_process", async () => {
+  const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
+  return {
+    ...actual,
+    execFile: execFileMock,
+  };
+});
+
+import {
+  createQaMultipassPlan,
+  renderQaMultipassGuestScript,
+  runQaMultipass,
+} from "./multipass.runtime.js";
+
+describe("qa multipass runtime", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    vi.restoreAllMocks();
+  });
+
+  it("rejects output directories outside the mounted repo root", () => {
+    expect(() =>
+      createQaMultipassPlan({
+        repoRoot: process.cwd(),
+        outputDir: "/tmp/qa-out",
+      }),
+    ).toThrow("qa suite --runner multipass requires --output-dir to stay under the repo root");
+  });
+
+  it("rejects repo-local symlink output directories that escape the repo root", () => {
+    const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-multipass-"));
+    const repoRoot = path.join(tempRoot, "repo");
+    const outsideRoot = path.join(tempRoot, "outside");
+    const symlinkPath = path.join(repoRoot, "artifacts-link");
+    fs.mkdirSync(repoRoot, { recursive: true });
+    fs.mkdirSync(outsideRoot, { recursive: true });
+    fs.writeFileSync(
+      path.join(repoRoot, "package.json"),
+      JSON.stringify({ packageManager: "pnpm@10.32.1" }),
+      "utf8",
+    );
+    fs.symlinkSync(outsideRoot, symlinkPath);
+
+    try {
+      expect(() =>
+        createQaMultipassPlan({
+          repoRoot,
+          outputDir: path.join(symlinkPath, "qa-out"),
+        }),
+      ).toThrow("qa suite --runner multipass requires --output-dir to stay under the repo root");
+    } finally {
+      fs.rmSync(tempRoot, { recursive: true, force: true });
+    }
+  });
+
+  it("reuses suite scenario semantics and resolves mounted artifact paths", () => {
+    const repoRoot = process.cwd();
+    const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "multipass-test");
+    const plan = createQaMultipassPlan({
+      repoRoot,
+      outputDir,
+    });
+
+    expect(plan.outputDir).toBe(outputDir);
+    expect(plan.scenarioIds).toEqual([]);
+    expect(plan.qaCommand).not.toContain("--scenario");
+    expect(plan.guestOutputDir).toBe("/workspace/openclaw-host/.artifacts/qa-e2e/multipass-test");
+    expect(plan.reportPath).toBe(path.join(outputDir, "qa-suite-report.md"));
+    expect(plan.summaryPath).toBe(path.join(outputDir, "qa-suite-summary.json"));
+  });
+
+  it("renders a guest script that runs the live qa suite by default", () => {
+    const plan = createQaMultipassPlan({
+      repoRoot: process.cwd(),
+      outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-test"),
+      scenarioIds: ["channel-chat-baseline", "thread-follow-up"],
+    });
+
+    const script = renderQaMultipassGuestScript(plan);
+
+    expect(script).toContain("pnpm install --frozen-lockfile");
+    expect(script).toContain("pnpm build");
+    expect(script).toContain(`corepack prepare '${readRootPackageManager()}' --activate`);
+    expect(script).toContain("'pnpm' 'openclaw' 'qa' 'suite' '--transport' 'qa-channel'");
+    expect(script).toContain("'--provider-mode' 'live-frontier'");
+    expect(script).toContain("'--scenario' 'channel-chat-baseline'");
+    expect(script).toContain("'--scenario' 'thread-follow-up'");
+    expect(script).toContain("/workspace/openclaw-host/.artifacts/qa-e2e/multipass-test");
+  });
+
+  it("carries live suite flags and forwarded auth env into the guest command", () => {
+    vi.stubEnv("OPENAI_API_KEY", "test-openai-key");
+    const plan = createQaMultipassPlan({
+      repoRoot: process.cwd(),
+      outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-live-test"),
+      providerMode: "live-frontier",
+      primaryModel: "openai/gpt-5.4",
+      alternateModel: "openai/gpt-5.4",
+      fastMode: true,
+      scenarioIds: ["channel-chat-baseline"],
+    });
+
+    const script = renderQaMultipassGuestScript(plan);
+
+    expect(plan.qaCommand).toEqual(
+      expect.arrayContaining([
+        "--provider-mode",
+        "live-frontier",
+        "--model",
+        "openai/gpt-5.4",
+        "--alt-model",
+        "openai/gpt-5.4",
+        "--fast",
+      ]),
+    );
+    expect(plan.forwardedEnv.OPENAI_API_KEY).toBe("test-openai-key");
+    expect(script).toContain("OPENAI_API_KEY='test-openai-key'");
+    expect(script).toContain("'pnpm' 'openclaw' 'qa' 'suite' '--transport' 'qa-channel'");
+    expect(script).toContain("'--provider-mode' 'live-frontier'");
+  });
+
+  it("forwards --allow-failures into the guest qa suite command when requested", () => {
+    const plan = createQaMultipassPlan({
+      repoRoot: process.cwd(),
+      outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-allow-failures-test"),
+      allowFailures: true,
+      scenarioIds: ["channel-chat-baseline"],
+    });
+
+    expect(plan.qaCommand).toEqual(expect.arrayContaining(["--allow-failures"]));
+  });
+
+  it("redacts forwarded live secrets in the persisted artifact script", () => {
+    vi.stubEnv("OPENAI_API_KEY", "test-openai-key");
+    const plan = createQaMultipassPlan({
+      repoRoot: process.cwd(),
+      outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-live-test"),
+      providerMode: "live-frontier",
+      scenarioIds: ["channel-chat-baseline"],
+    });
+
+    const redactedScript = renderQaMultipassGuestScript(plan, { redactSecrets: true });
+
+    expect(redactedScript).toContain("OPENAI_API_KEY='<redacted>'");
+    expect(redactedScript).not.toContain("OPENAI_API_KEY='test-openai-key'");
+  });
+
+  it("forwards live key list and numbered key env shapes", () => {
+    vi.stubEnv("OPENCLAW_LIVE_ANTHROPIC_KEYS", "anthropic-a anthropic-b");
+    vi.stubEnv("OPENAI_API_KEY_1", "openai-one");
+    vi.stubEnv("GEMINI_API_KEY_2", "gemini-two");
+    const plan = createQaMultipassPlan({
+      repoRoot: process.cwd(),
+      outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-live-test"),
+      providerMode: "live-frontier",
+      scenarioIds: ["channel-chat-baseline"],
+    });
+
+    expect(plan.forwardedEnv.OPENCLAW_LIVE_ANTHROPIC_KEYS).toBe("anthropic-a anthropic-b");
+    expect(plan.forwardedEnv.OPENAI_API_KEY_1).toBe("openai-one");
+    expect(plan.forwardedEnv.GEMINI_API_KEY_2).toBe("gemini-two");
+  });
+
+  it("skips stale CODEX_HOME values that do not exist on the host", () => {
+    vi.stubEnv("CODEX_HOME", "/tmp/does-not-exist-openclaw-codex-home");
+    const plan = createQaMultipassPlan({
+      repoRoot: process.cwd(),
+      outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-live-test"),
+      providerMode: "live-frontier",
+    });
+
+    expect(plan.forwardedEnv.CODEX_HOME).toBeUndefined();
+    expect(plan.hostCodexHomePath).toBeUndefined();
+    expect(plan.guestCodexHomePath).toBeUndefined();
+  });
+
+  it("falls back to os.homedir() when HOME is unset for CODEX_HOME discovery", () => {
+    const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-multipass-home-"));
+    const fakeHome = path.join(tempRoot, "home");
+    const fakeCodexHome = path.join(fakeHome, ".codex");
+    fs.mkdirSync(fakeCodexHome, { recursive: true });
+    vi.stubEnv("HOME", "");
+    vi.stubEnv("CODEX_HOME", "");
+    vi.spyOn(os, "homedir").mockReturnValue(fakeHome);
+
+    try {
+      const plan = createQaMultipassPlan({
+        repoRoot: process.cwd(),
+        outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-live-test"),
+        providerMode: "live-frontier",
+      });
+
+      expect(plan.forwardedEnv.CODEX_HOME).toBe(fakeCodexHome);
+      expect(plan.hostCodexHomePath).toBe(fakeCodexHome);
+      expect(plan.guestCodexHomePath).toBe("/workspace/openclaw-codex-home");
+    } finally {
+      fs.rmSync(tempRoot, { recursive: true, force: true });
+    }
+  });
+
+  it("does not leave a temp guest transfer script behind when multipass is missing", async () => {
+    const outputDir = path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-missing-test");
+    vi.spyOn(Date, "now").mockReturnValue(1_717_171_717_171);
+    vi.spyOn(Math, "random").mockReturnValue(0.123456789);
+    (execFileMock as unknown as Mock).mockImplementation((...args: unknown[]) => {
+      const callback = args[3] as (error: Error | null, stdout: string, stderr: string) => void;
+      const error = new Error("spawn multipass ENOENT") as NodeJS.ErrnoException;
+      error.code = "ENOENT";
+      callback(error, "", "");
+    });
+
+    const expectedVmName = createQaMultipassPlan({
+      repoRoot: process.cwd(),
+      outputDir,
+      scenarioIds: ["channel-chat-baseline"],
+    }).vmName;
+    const expectedTransferDir = path.join(
+      resolvePreferredOpenClawTmpDir(),
+      `${expectedVmName}-qa-suite-`,
+    );
+
+    await expect(
+      runQaMultipass({
+        repoRoot: process.cwd(),
+        outputDir,
+        scenarioIds: ["channel-chat-baseline"],
+      }),
+    ).rejects.toThrow("Multipass is not installed on this host.");
+
+    const tempEntries = fs
+      .readdirSync(resolvePreferredOpenClawTmpDir())
+      .filter((entry) => entry.startsWith(path.basename(expectedTransferDir)));
+    expect(tempEntries).toEqual([]);
+    fs.rmSync(outputDir, { recursive: true, force: true });
+  });
+
+  it("preserves non-install multipass probe failures", async () => {
+    const outputDir = path.join(
+      process.cwd(),
+      ".artifacts",
+      "qa-e2e",
+      "multipass-probe-error-test",
+    );
+    (execFileMock as unknown as Mock).mockImplementation((...args: unknown[]) => {
+      const callback = args[3] as (error: Error | null, stdout: string, stderr: string) => void;
+      const error = new Error("multipassd is not running") as NodeJS.ErrnoException;
+      error.code = "EACCES";
+      callback(error, "", "multipassd is not running");
+    });
+
+    await expect(
+      runQaMultipass({
+        repoRoot: process.cwd(),
+        outputDir,
+        scenarioIds: ["channel-chat-baseline"],
+      }),
+    ).rejects.toThrow("Unable to verify Multipass availability: multipassd is not running.");
+
+    fs.rmSync(outputDir, { recursive: true, force: true });
+  });
+});

openclaw/extensions/qa-lab/src/multipass.runtime.ts

@@ -0,0 +1,670 @@
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import fs from "node:fs";
+import { access, appendFile, mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
+import type { QaProviderMode } from "./model-selection.js";
+import { resolveQaForwardedLiveEnv, resolveQaLiveProviderConfigPath } from "./providers/env.js";
+import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
+
+const MULTIPASS_MOUNTED_REPO_PATH = "/workspace/openclaw-host";
+const MULTIPASS_GUEST_REPO_PATH = "/workspace/openclaw";
+const MULTIPASS_GUEST_CODEX_HOME_PATH = "/workspace/openclaw-codex-home";
+const MULTIPASS_GUEST_PACKAGES = [
+  "build-essential",
+  "ca-certificates",
+  "curl",
+  "pkg-config",
+  "python3",
+  "rsync",
+  "xz-utils",
+] as const;
+const MULTIPASS_REPO_SYNC_EXCLUDES = [
+  ".git",
+  "node_modules",
+  ".artifacts",
+  ".tmp",
+  ".turbo",
+  "coverage",
+  "*.heapsnapshot",
+] as const;
+const MULTIPASS_EXEC_MAX_BUFFER = 64 * 1024 * 1024;
+const MULTIPASS_GUEST_RUN_TIMEOUT_MS = 60 * 60 * 1000;
+
+export const qaMultipassDefaultResources = {
+  image: "lts",
+  cpus: 2,
+  memory: "4G",
+  disk: "24G",
+} as const;
+
+type ExecResult = {
+  stdout: string;
+  stderr: string;
+};
+
+type ExecFileError = Error & {
+  code?: string;
+};
+
+type ExecFileOptions = {
+  timeoutMs?: number;
+};
+
+export type QaMultipassPlan = {
+  repoRoot: string;
+  outputDir: string;
+  reportPath: string;
+  summaryPath: string;
+  hostLogPath: string;
+  hostBootstrapLogPath: string;
+  hostGuestScriptPath: string;
+  vmName: string;
+  image: string;
+  cpus: number;
+  memory: string;
+  disk: string;
+  pnpmVersion: string;
+  transportId: string;
+  providerMode: QaProviderMode;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  scenarioIds: string[];
+  forwardedEnv: Record<string, string>;
+  hostCodexHomePath?: string;
+  guestCodexHomePath?: string;
+  hostLiveProviderConfigPath?: string;
+  guestLiveProviderConfigPath?: string;
+  guestMountedRepoPath: string;
+  guestRepoPath: string;
+  guestOutputDir: string;
+  guestScriptPath: string;
+  guestBootstrapLogPath: string;
+  qaCommand: string[];
+};
+
+export type QaMultipassRunResult = {
+  outputDir: string;
+  reportPath: string;
+  summaryPath: string;
+  hostLogPath: string;
+  bootstrapLogPath: string;
+  guestScriptPath: string;
+  vmName: string;
+  scenarioIds: string[];
+};
+
+type RenderGuestScriptOptions = {
+  redactSecrets?: boolean;
+};
+
+function shellQuote(value: string) {
+  return `'${value.replaceAll("'", `'"'"'`)}'`;
+}
+
+function createOutputStamp() {
+  return new Date().toISOString().replaceAll(":", "").replaceAll(".", "").replace("T", "-");
+}
+
+function createVmSuffix() {
+  return `${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
+}
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function execFileAsync(file: string, args: string[], options: ExecFileOptions = {}) {
+  return new Promise<ExecResult>((resolve, reject) => {
+    execFile(
+      file,
+      args,
+      {
+        encoding: "utf8",
+        maxBuffer: MULTIPASS_EXEC_MAX_BUFFER,
+        timeout: options.timeoutMs,
+      },
+      (error, stdout, stderr) => {
+        if (error) {
+          const message = stderr.trim() || stdout.trim() || error.message;
+          const wrappedError = new Error(message, { cause: error }) as ExecFileError;
+          wrappedError.code = (error as NodeJS.ErrnoException).code;
+          reject(wrappedError);
+          return;
+        }
+        resolve({ stdout, stderr });
+      },
+    );
+  });
+}
+
+function resolveRealPath(value: string) {
+  return fs.realpathSync.native?.(value) ?? fs.realpathSync(value);
+}
+
+function resolveExistingPath(value: string) {
+  let currentPath = value;
+  while (!fs.existsSync(currentPath)) {
+    const parentPath = path.dirname(currentPath);
+    if (parentPath === currentPath) {
+      throw new Error(`unable to resolve existing path for ${value}`);
+    }
+    currentPath = parentPath;
+  }
+  return currentPath;
+}
+
+function isPathInside(parentPath: string, childPath: string) {
+  const relativePath = path.relative(parentPath, childPath);
+  return !relativePath.startsWith("..") && !path.isAbsolute(relativePath);
+}
+
+function validatePnpmVersion(version: string) {
+  if (!/^[0-9A-Za-z.+_-]+$/u.test(version)) {
+    throw new Error(`unsupported pnpm version in packageManager: ${version}`);
+  }
+  return version;
+}
+
+function resolveMountedOutputPath(repoRoot: string, hostPath: string) {
+  const relativePath = path.relative(repoRoot, hostPath);
+  if (relativePath.startsWith("..") || path.isAbsolute(relativePath) || relativePath.length === 0) {
+    throw new Error(
+      `qa suite --runner multipass requires --output-dir to stay under the repo root (${repoRoot}), got ${hostPath}.`,
+    );
+  }
+
+  const realRepoRoot = resolveRealPath(repoRoot);
+  const existingHostPath = resolveExistingPath(hostPath);
+  const realExistingHostPath = resolveRealPath(existingHostPath);
+  if (!isPathInside(realRepoRoot, realExistingHostPath) && realExistingHostPath !== realRepoRoot) {
+    throw new Error(
+      `qa suite --runner multipass requires --output-dir to stay under the repo root (${repoRoot}), got ${hostPath}.`,
+    );
+  }
+
+  return path.posix.join(MULTIPASS_MOUNTED_REPO_PATH, ...relativePath.split(path.sep));
+}
+
+function resolvePnpmVersion(repoRoot: string) {
+  const packageJsonPath = path.join(repoRoot, "package.json");
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")) as {
+    packageManager?: string;
+  };
+  const packageManager = packageJson.packageManager ?? "";
+  const match = /^pnpm@(.+)$/.exec(packageManager);
+  if (!match?.[1]) {
+    throw new Error(`unable to resolve pnpm version from packageManager in ${packageJsonPath}`);
+  }
+  return match[1];
+}
+
+function resolveMultipassInstallHint() {
+  if (process.platform === "darwin") {
+    return "brew install --cask multipass";
+  }
+  if (process.platform === "win32") {
+    return "winget install Canonical.Multipass";
+  }
+  if (process.platform === "linux") {
+    return "sudo snap install multipass";
+  }
+  return "https://multipass.run/install";
+}
+
+function createQaMultipassOutputDir(repoRoot: string) {
+  return path.join(repoRoot, ".artifacts", "qa-e2e", `multipass-${createOutputStamp()}`);
+}
+
+function resolveGuestMountedPath(repoRoot: string, hostPath: string) {
+  return resolveMountedOutputPath(repoRoot, hostPath);
+}
+
+function appendScenarioArgs(command: string[], scenarioIds: string[]) {
+  for (const scenarioId of scenarioIds) {
+    command.push("--scenario", scenarioId);
+  }
+  return command;
+}
+
+export function createQaMultipassPlan(params: {
+  repoRoot: string;
+  outputDir?: string;
+  transportId?: string;
+  providerMode?: QaProviderMode;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  allowFailures?: boolean;
+  scenarioIds?: string[];
+  concurrency?: number;
+  image?: string;
+  cpus?: number;
+  memory?: string;
+  disk?: string;
+}) {
+  const outputDir = params.outputDir ?? createQaMultipassOutputDir(params.repoRoot);
+  const scenarioIds = [...new Set(params.scenarioIds ?? [])];
+  const transportId = params.transportId?.trim() || "qa-channel";
+  const providerMode = params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE;
+  const provider = getQaProvider(providerMode);
+  const forwardedEnv = provider.appliesLiveEnvAliases ? resolveQaForwardedLiveEnv() : {};
+  const hostCodexHomePath = forwardedEnv.CODEX_HOME;
+  const liveProviderConfig = provider.usesModelProviderPlugins
+    ? resolveQaLiveProviderConfigPath()
+    : undefined;
+  const hostLiveProviderConfigPath =
+    liveProviderConfig && fs.existsSync(liveProviderConfig.path)
+      ? liveProviderConfig.path
+      : undefined;
+  const vmName = `openclaw-qa-${createVmSuffix()}`;
+  const guestOutputDir = resolveGuestMountedPath(params.repoRoot, outputDir);
+  const qaCommand = appendScenarioArgs(
+    [
+      "pnpm",
+      "openclaw",
+      "qa",
+      "suite",
+      "--transport",
+      transportId,
+      "--provider-mode",
+      providerMode,
+      "--output-dir",
+      guestOutputDir,
+      ...(params.primaryModel ? ["--model", params.primaryModel] : []),
+      ...(params.alternateModel ? ["--alt-model", params.alternateModel] : []),
+      ...(params.fastMode ? ["--fast"] : []),
+      ...(params.allowFailures ? ["--allow-failures"] : []),
+      ...(params.concurrency ? ["--concurrency", String(params.concurrency)] : []),
+    ],
+    scenarioIds,
+  );
+
+  return {
+    repoRoot: params.repoRoot,
+    outputDir,
+    reportPath: path.join(outputDir, "qa-suite-report.md"),
+    summaryPath: path.join(outputDir, "qa-suite-summary.json"),
+    hostLogPath: path.join(outputDir, "multipass-host.log"),
+    hostBootstrapLogPath: path.join(outputDir, "multipass-guest-bootstrap.log"),
+    hostGuestScriptPath: path.join(outputDir, "multipass-guest-run.sh"),
+    vmName,
+    image: params.image ?? qaMultipassDefaultResources.image,
+    cpus: params.cpus ?? qaMultipassDefaultResources.cpus,
+    memory: params.memory ?? qaMultipassDefaultResources.memory,
+    disk: params.disk ?? qaMultipassDefaultResources.disk,
+    pnpmVersion: validatePnpmVersion(resolvePnpmVersion(params.repoRoot)),
+    transportId,
+    providerMode,
+    primaryModel: params.primaryModel,
+    alternateModel: params.alternateModel,
+    fastMode: params.fastMode,
+    scenarioIds,
+    forwardedEnv,
+    hostCodexHomePath,
+    guestCodexHomePath: hostCodexHomePath ? MULTIPASS_GUEST_CODEX_HOME_PATH : undefined,
+    hostLiveProviderConfigPath,
+    guestLiveProviderConfigPath: hostLiveProviderConfigPath
+      ? `/tmp/${vmName}-live-provider-config.json`
+      : undefined,
+    guestMountedRepoPath: MULTIPASS_MOUNTED_REPO_PATH,
+    guestRepoPath: MULTIPASS_GUEST_REPO_PATH,
+    guestOutputDir,
+    guestScriptPath: `/tmp/${vmName}-qa-suite.sh`,
+    guestBootstrapLogPath: `/tmp/${vmName}-bootstrap.log`,
+    qaCommand,
+  } satisfies QaMultipassPlan;
+}
+
+export function renderQaMultipassGuestScript(
+  plan: QaMultipassPlan,
+  options: RenderGuestScriptOptions = {},
+) {
+  const redactSecrets = options.redactSecrets ?? false;
+  const rsyncCommand = [
+    "rsync -a --delete",
+    ...MULTIPASS_REPO_SYNC_EXCLUDES.flatMap((value) => ["--exclude", shellQuote(value)]),
+    shellQuote(`${plan.guestMountedRepoPath}/`),
+    shellQuote(`${plan.guestRepoPath}/`),
+  ].join(" ");
+  const qaCommand = [
+    ...Object.entries(plan.forwardedEnv)
+      .filter(
+        ([key]) =>
+          key !== "CODEX_HOME" &&
+          key !== "OPENCLAW_CONFIG_PATH" &&
+          key !== "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH",
+      )
+      .map(([key, value]) => `${key}=${shellQuote(redactSecrets ? "<redacted>" : value)}`),
+    ...(plan.guestCodexHomePath ? [`CODEX_HOME=${shellQuote(plan.guestCodexHomePath)}`] : []),
+    ...(plan.guestLiveProviderConfigPath
+      ? [
+          `OPENCLAW_CONFIG_PATH=${shellQuote(plan.guestLiveProviderConfigPath)}`,
+          `OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH=${shellQuote(plan.guestLiveProviderConfigPath)}`,
+        ]
+      : []),
+    plan.qaCommand.map(shellQuote).join(" "),
+  ].join(" ");
+
+  const lines = [
+    "#!/usr/bin/env bash",
+    "set -euo pipefail",
+    "trap 'status=$?; echo \"guest failure (exit ${status})\" >&2; exit ${status}' ERR",
+    "",
+    "export DEBIAN_FRONTEND=noninteractive",
+    `BOOTSTRAP_LOG=${shellQuote(plan.guestBootstrapLogPath)}`,
+    ': > "$BOOTSTRAP_LOG"',
+    "",
+    "ensure_guest_packages() {",
+    '  sudo -E apt-get update >>"$BOOTSTRAP_LOG" 2>&1',
+    "  sudo -E apt-get install -y \\",
+    ...MULTIPASS_GUEST_PACKAGES.map((value, index) =>
+      index === MULTIPASS_GUEST_PACKAGES.length - 1
+        ? `    ${value} >>"$BOOTSTRAP_LOG" 2>&1`
+        : `    ${value} \\`,
+    ),
+    "}",
+    "",
+    "ensure_node() {",
+    "  if command -v node >/dev/null; then",
+    "    local node_major",
+    '    node_major="$(node -p \'process.versions.node.split(".")[0]\' 2>/dev/null || echo 0)"',
+    '    if [ "${node_major}" -ge 22 ]; then',
+    "      return 0",
+    "    fi",
+    "  fi",
+    "  local node_arch",
+    '  case "$(uname -m)" in',
+    '    x86_64) node_arch="x64" ;;',
+    '    aarch64|arm64) node_arch="arm64" ;;',
+    '    *) echo "unsupported guest architecture for node bootstrap: $(uname -m)" >&2; return 1 ;;',
+    "  esac",
+    "  local node_tmp_dir tarball_name extract_dir base_url",
+    '  node_tmp_dir="$(mktemp -d)"',
+    "  trap 'rm -rf \"${node_tmp_dir}\"' RETURN",
+    '  base_url="https://nodejs.org/dist/latest-v22.x"',
+    '  curl -fsSL "${base_url}/SHASUMS256.txt" -o "${node_tmp_dir}/SHASUMS256.txt" >>"$BOOTSTRAP_LOG" 2>&1',
+    '  tarball_name="$(awk \'/linux-\'"${node_arch}"\'\\.tar\\.xz$/ { print $2; exit }\' "${node_tmp_dir}/SHASUMS256.txt")"',
+    '  [ -n "${tarball_name}" ] || { echo "unable to resolve node tarball for ${node_arch}" >&2; return 1; }',
+    '  curl -fsSL "${base_url}/${tarball_name}" -o "${node_tmp_dir}/${tarball_name}" >>"$BOOTSTRAP_LOG" 2>&1',
+    '  (cd "${node_tmp_dir}" && grep " ${tarball_name}$" SHASUMS256.txt | sha256sum -c -) >>"$BOOTSTRAP_LOG" 2>&1',
+    '  extract_dir="${tarball_name%.tar.xz}"',
+    '  sudo mkdir -p /usr/local/lib/nodejs >>"$BOOTSTRAP_LOG" 2>&1',
+    '  sudo rm -rf "/usr/local/lib/nodejs/${extract_dir}" >>"$BOOTSTRAP_LOG" 2>&1',
+    '  sudo tar -xJf "${node_tmp_dir}/${tarball_name}" -C /usr/local/lib/nodejs >>"$BOOTSTRAP_LOG" 2>&1',
+    '  sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/node" /usr/local/bin/node >>"$BOOTSTRAP_LOG" 2>&1',
+    '  sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/npm" /usr/local/bin/npm >>"$BOOTSTRAP_LOG" 2>&1',
+    '  sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/npx" /usr/local/bin/npx >>"$BOOTSTRAP_LOG" 2>&1',
+    '  sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/corepack" /usr/local/bin/corepack >>"$BOOTSTRAP_LOG" 2>&1',
+    "}",
+    "",
+    "ensure_pnpm() {",
+    '  sudo env PATH="/usr/local/bin:/usr/bin:/bin" corepack enable >>"$BOOTSTRAP_LOG" 2>&1',
+    `  sudo env PATH="/usr/local/bin:/usr/bin:/bin" corepack prepare ${shellQuote(`pnpm@${plan.pnpmVersion}`)} --activate >>"$BOOTSTRAP_LOG" 2>&1`,
+    "}",
+    "",
+    'command -v sudo >/dev/null || { echo "missing sudo in guest" >&2; exit 1; }',
+    "ensure_guest_packages",
+    "ensure_node",
+    "ensure_pnpm",
+    'command -v node >/dev/null || { echo "missing node after guest bootstrap" >&2; exit 1; }',
+    'command -v pnpm >/dev/null || { echo "missing pnpm after guest bootstrap" >&2; exit 1; }',
+    'command -v rsync >/dev/null || { echo "missing rsync after guest bootstrap" >&2; exit 1; }',
+    "",
+    `mkdir -p ${shellQuote(path.posix.dirname(plan.guestRepoPath))}`,
+    `rm -rf ${shellQuote(plan.guestRepoPath)}`,
+    `mkdir -p ${shellQuote(plan.guestRepoPath)}`,
+    `mkdir -p ${shellQuote(plan.guestOutputDir)}`,
+    rsyncCommand,
+    `cd ${shellQuote(plan.guestRepoPath)}`,
+    'pnpm install --frozen-lockfile >>"$BOOTSTRAP_LOG" 2>&1',
+    'pnpm build >>"$BOOTSTRAP_LOG" 2>&1',
+    qaCommand,
+    "",
+  ];
+  return lines.join("\n");
+}
+
+async function appendMultipassLog(logPath: string, message: string) {
+  await appendFile(logPath, message, "utf8");
+}
+
+async function runMultipassCommand(logPath: string, args: string[], options: ExecFileOptions = {}) {
+  await appendMultipassLog(logPath, `$ ${["multipass", ...args].join(" ")}\n`);
+  const result = await execFileAsync("multipass", args, options);
+  if (result.stdout.trim()) {
+    await appendMultipassLog(logPath, `${result.stdout.trim()}\n`);
+  }
+  if (result.stderr.trim()) {
+    await appendMultipassLog(logPath, `${result.stderr.trim()}\n`);
+  }
+  await appendMultipassLog(logPath, "\n");
+  return result;
+}
+
+async function waitForGuestReady(logPath: string, vmName: string) {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= 12; attempt += 1) {
+    try {
+      await runMultipassCommand(logPath, ["exec", vmName, "--", "bash", "-lc", "echo guest-ready"]);
+      return;
+    } catch (error) {
+      lastError = error;
+      await appendMultipassLog(
+        logPath,
+        `guest-ready retry ${attempt}/12: ${error instanceof Error ? error.message : String(error)}\n\n`,
+      );
+      if (attempt < 12) {
+        await sleep(2_000);
+      }
+    }
+  }
+  throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+
+async function mountRepo(logPath: string, repoRoot: string, vmName: string) {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= 5; attempt += 1) {
+    try {
+      await runMultipassCommand(logPath, [
+        "mount",
+        repoRoot,
+        `${vmName}:${MULTIPASS_MOUNTED_REPO_PATH}`,
+      ]);
+      return;
+    } catch (error) {
+      lastError = error;
+      await appendMultipassLog(
+        logPath,
+        `mount retry ${attempt}/5: ${error instanceof Error ? error.message : String(error)}\n\n`,
+      );
+      if (attempt < 5) {
+        await sleep(2_000);
+      }
+    }
+  }
+  throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+
+async function mountCodexHome(logPath: string, hostCodexHomePath: string, vmName: string) {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= 5; attempt += 1) {
+    try {
+      await runMultipassCommand(logPath, [
+        "mount",
+        hostCodexHomePath,
+        `${vmName}:${MULTIPASS_GUEST_CODEX_HOME_PATH}`,
+      ]);
+      return;
+    } catch (error) {
+      lastError = error;
+      await appendMultipassLog(
+        logPath,
+        `codex-home mount retry ${attempt}/5: ${error instanceof Error ? error.message : String(error)}\n\n`,
+      );
+      if (attempt < 5) {
+        await sleep(2_000);
+      }
+    }
+  }
+  throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+
+async function transferLiveProviderConfig(plan: QaMultipassPlan) {
+  if (!plan.hostLiveProviderConfigPath || !plan.guestLiveProviderConfigPath) {
+    return;
+  }
+  await runMultipassCommand(plan.hostLogPath, [
+    "transfer",
+    plan.hostLiveProviderConfigPath,
+    `${plan.vmName}:${plan.guestLiveProviderConfigPath}`,
+  ]);
+}
+
+async function tryCopyGuestBootstrapLog(plan: QaMultipassPlan) {
+  try {
+    await runMultipassCommand(plan.hostLogPath, [
+      "transfer",
+      `${plan.vmName}:${plan.guestBootstrapLogPath}`,
+      plan.hostBootstrapLogPath,
+    ]);
+  } catch (error) {
+    await appendMultipassLog(
+      plan.hostLogPath,
+      `bootstrap log transfer skipped: ${error instanceof Error ? error.message : String(error)}\n\n`,
+    );
+  }
+}
+
+export async function runQaMultipass(params: {
+  repoRoot: string;
+  outputDir?: string;
+  transportId?: string;
+  providerMode?: QaProviderMode;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  allowFailures?: boolean;
+  scenarioIds?: string[];
+  concurrency?: number;
+  image?: string;
+  cpus?: number;
+  memory?: string;
+  disk?: string;
+}) {
+  const plan = createQaMultipassPlan(params);
+  await mkdir(plan.outputDir, { recursive: true });
+  await writeFile(
+    plan.hostLogPath,
+    `# OpenClaw QA Multipass host log\nvmName=${plan.vmName}\noutputDir=${plan.outputDir}\n\n`,
+    "utf8",
+  );
+  await writeFile(
+    plan.hostGuestScriptPath,
+    renderQaMultipassGuestScript(plan, { redactSecrets: true }),
+    {
+      encoding: "utf8",
+      mode: 0o600,
+    },
+  );
+
+  try {
+    await execFileAsync("multipass", ["version"]);
+  } catch (error) {
+    if ((error as ExecFileError).code !== "ENOENT") {
+      throw new Error(
+        `Unable to verify Multipass availability: ${error instanceof Error ? error.message : String(error)}.`,
+        { cause: error },
+      );
+    }
+    throw new Error(
+      `Multipass is not installed on this host. Install it with '${resolveMultipassInstallHint()}', then rerun 'pnpm openclaw qa suite --runner multipass'.`,
+      { cause: error },
+    );
+  }
+
+  const hostTransferDirPath = await fs.promises.mkdtemp(
+    path.join(resolvePreferredOpenClawTmpDir(), `${plan.vmName}-qa-suite-`),
+  );
+  const hostTransferScriptPath = path.join(hostTransferDirPath, "guest-run.sh");
+  await writeFile(hostTransferScriptPath, renderQaMultipassGuestScript(plan), {
+    encoding: "utf8",
+    mode: 0o600,
+  });
+
+  let launched = false;
+  try {
+    await runMultipassCommand(plan.hostLogPath, [
+      "launch",
+      "--name",
+      plan.vmName,
+      "--cpus",
+      String(plan.cpus),
+      "--memory",
+      plan.memory,
+      "--disk",
+      plan.disk,
+      plan.image,
+    ]);
+    launched = true;
+    await waitForGuestReady(plan.hostLogPath, plan.vmName);
+    await mountRepo(plan.hostLogPath, plan.repoRoot, plan.vmName);
+    if (plan.hostCodexHomePath) {
+      await mountCodexHome(plan.hostLogPath, plan.hostCodexHomePath, plan.vmName);
+    }
+    await transferLiveProviderConfig(plan);
+    await runMultipassCommand(plan.hostLogPath, [
+      "transfer",
+      hostTransferScriptPath,
+      `${plan.vmName}:${plan.guestScriptPath}`,
+    ]);
+    await runMultipassCommand(plan.hostLogPath, [
+      "exec",
+      plan.vmName,
+      "--",
+      "chmod",
+      "+x",
+      plan.guestScriptPath,
+    ]);
+    await runMultipassCommand(plan.hostLogPath, ["exec", plan.vmName, "--", plan.guestScriptPath], {
+      timeoutMs: MULTIPASS_GUEST_RUN_TIMEOUT_MS,
+    });
+    await tryCopyGuestBootstrapLog(plan);
+  } catch (error) {
+    if (launched) {
+      await tryCopyGuestBootstrapLog(plan);
+    }
+    throw new Error(
+      `QA Multipass run failed: ${error instanceof Error ? error.message : String(error)}. See ${plan.hostLogPath}.`,
+      { cause: error },
+    );
+  } finally {
+    await fs.promises.rm(hostTransferDirPath, { recursive: true, force: true });
+    if (launched) {
+      try {
+        await runMultipassCommand(plan.hostLogPath, ["delete", "--purge", plan.vmName]);
+      } catch (error) {
+        await appendMultipassLog(
+          plan.hostLogPath,
+          `cleanup error: ${error instanceof Error ? error.message : String(error)}\n\n`,
+        );
+      }
+    }
+  }
+
+  await access(plan.reportPath);
+  await access(plan.summaryPath);
+
+  return {
+    outputDir: plan.outputDir,
+    reportPath: plan.reportPath,
+    summaryPath: plan.summaryPath,
+    hostLogPath: plan.hostLogPath,
+    bootstrapLogPath: plan.hostBootstrapLogPath,
+    guestScriptPath: plan.hostGuestScriptPath,
+    vmName: plan.vmName,
+    scenarioIds: plan.scenarioIds,
+  } satisfies QaMultipassRunResult;
+}

openclaw/extensions/qa-lab/src/node-exec.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest";
+import { resolveQaNodeExecPath } from "./node-exec.js";
+
+describe("resolveQaNodeExecPath", () => {
+  it("reuses the current exec path when already running under Node", async () => {
+    await expect(
+      resolveQaNodeExecPath({
+        execPath: "/opt/homebrew/bin/node",
+        platform: "darwin",
+        versions: { ...process.versions, bun: undefined },
+      }),
+    ).resolves.toBe("/opt/homebrew/bin/node");
+  });
+
+  it("reuses nodejs as a valid current Node executable", async () => {
+    await expect(
+      resolveQaNodeExecPath({
+        execPath: "/usr/bin/nodejs",
+        platform: "linux",
+        versions: { ...process.versions, bun: undefined },
+        execFileImpl: async () => {
+          throw new Error("should not search PATH");
+        },
+      }),
+    ).resolves.toBe("/usr/bin/nodejs");
+  });
+
+  it("resolves node from PATH when the parent runtime is bun", async () => {
+    await expect(
+      resolveQaNodeExecPath({
+        execPath: "/opt/homebrew/bin/bun",
+        platform: "darwin",
+        versions: { ...process.versions, bun: "1.2.3" },
+        execFileImpl: async () => ({
+          stdout: "/usr/local/bin/node\n",
+          stderr: "",
+        }),
+      }),
+    ).resolves.toBe("/usr/local/bin/node");
+  });
+
+  it("throws a clear error when node is unavailable", async () => {
+    await expect(
+      resolveQaNodeExecPath({
+        execPath: "/opt/homebrew/bin/bun",
+        platform: "darwin",
+        versions: { ...process.versions, bun: "1.2.3" },
+        execFileImpl: async () => {
+          throw new Error("missing");
+        },
+      }),
+    ).rejects.toThrow("Node not found in PATH");
+  });
+});

openclaw/extensions/qa-lab/src/node-exec.ts

@@ -0,0 +1,69 @@
+import { execFile } from "node:child_process";
+import path from "node:path";
+import { promisify } from "node:util";
+
+type ExecFileAsync = (
+  file: string,
+  args: readonly string[],
+  options: {
+    encoding: "utf8";
+    env?: NodeJS.ProcessEnv;
+  },
+) => Promise<{ stdout: string; stderr: string }>;
+
+const execFileAsync = promisify(execFile) as unknown as ExecFileAsync;
+
+function isNodeExecPath(execPath: string, platform: NodeJS.Platform): boolean {
+  const pathModule = platform === "win32" ? path.win32 : path.posix;
+  const basename = pathModule.basename(execPath).toLowerCase();
+  return (
+    basename === "node" ||
+    basename === "node.exe" ||
+    basename === "nodejs" ||
+    basename === "nodejs.exe"
+  );
+}
+
+export async function resolveQaNodeExecPath(params?: {
+  execPath?: string;
+  platform?: NodeJS.Platform;
+  versions?: NodeJS.ProcessVersions;
+  env?: NodeJS.ProcessEnv;
+  execFileImpl?: ExecFileAsync;
+}): Promise<string> {
+  const execPath = params?.execPath ?? process.execPath;
+  const platform = params?.platform ?? process.platform;
+  const versions = params?.versions ?? process.versions;
+  if (typeof versions.bun !== "string" && isNodeExecPath(execPath, platform)) {
+    return execPath;
+  }
+
+  const locator = platform === "win32" ? "where" : "which";
+  const execFileImpl = params?.execFileImpl ?? execFileAsync;
+  let stdout = "";
+  try {
+    ({ stdout } = await execFileImpl(locator, ["node"], {
+      encoding: "utf8",
+      env: params?.env,
+    }));
+  } catch {
+    throw new Error(
+      "Node not found in PATH. QA live lanes require Node for child gateway and CLI processes.",
+    );
+  }
+
+  const resolved = stdout
+    .split(/\r?\n/)
+    .map((entry) => entry.trim())
+    .find((entry) => entry.length > 0);
+  if (!resolved) {
+    throw new Error(
+      "Node not found in PATH. QA live lanes require Node for child gateway and CLI processes.",
+    );
+  }
+  return resolved;
+}
+
+export const __testing = {
+  isNodeExecPath,
+};

openclaw/extensions/qa-lab/src/providers/README.md

@@ -0,0 +1,21 @@
+# QA Provider Lanes
+
+QA provider lanes are registered in `index.ts` and implemented in one folder per
+provider. Shared provider contracts and mock-provider helpers live in `shared/`.
+Mock lanes should use `shared/mock-provider-definition.ts` unless they need a
+custom shape.
+
+Each provider definition owns:
+
+- its accepted `providerMode`
+- default primary, alternate, and image-generation model refs
+- gateway `models.providers` config, when the lane needs config injection
+- model runtime params such as SSE transport, fast mode, or thinking defaults
+- optional local server startup for mock lanes
+- optional placeholder auth profile providers for mock lanes
+- whether the lane uses real provider plugins and live environment aliases
+
+Shared suite code should import only `providers/index.ts` and ask the selected
+provider for behavior. Do not add provider-name branches to suite, gateway,
+manual-lane, or live-transport runtime code unless the registry contract is
+missing a needed capability.

openclaw/extensions/qa-lab/src/providers/aimock/index.ts

@@ -0,0 +1,9 @@
+import { createMockQaProviderDefinition } from "../shared/mock-provider-definition.js";
+
+export const aimockProviderDefinition = createMockQaProviderDefinition({
+  mode: "aimock",
+  commandName: "aimock",
+  commandDescription: "Run the local AIMock provider server for QA",
+  serverLabel: "QA AIMock",
+  mockAuthProviders: ["aimock", "openai", "anthropic"],
+});

openclaw/extensions/qa-lab/src/providers/aimock/server.test.ts

@@ -0,0 +1,110 @@
+import { describe, expect, it } from "vitest";
+import { startQaAimockServer } from "./server.js";
+
+function makeResponsesInput(text: string) {
+  return {
+    role: "user",
+    content: [
+      {
+        type: "input_text",
+        text,
+      },
+    ],
+  };
+}
+
+describe("qa aimock server", () => {
+  it("serves OpenAI Responses text replies and debug request snapshots", async () => {
+    const server = await startQaAimockServer({
+      host: "127.0.0.1",
+      port: 0,
+    });
+    try {
+      const response = await fetch(`${server.baseUrl}/v1/responses`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          model: "aimock/gpt-5.4",
+          stream: false,
+          input: [makeResponsesInput("hello aimock")],
+        }),
+      });
+      expect(response.status).toBe(200);
+      expect(await response.json()).toMatchObject({
+        status: "completed",
+        model: "aimock/gpt-5.4",
+      });
+
+      const debug = await fetch(`${server.baseUrl}/debug/last-request`);
+      expect(debug.status).toBe(200);
+      expect(await debug.json()).toMatchObject({
+        prompt: "hello aimock",
+        allInputText: "hello aimock",
+        model: "aimock/gpt-5.4",
+        providerVariant: "openai",
+      });
+    } finally {
+      await server.stop();
+    }
+  });
+
+  it("records the request list for scenario assertions", async () => {
+    const server = await startQaAimockServer({
+      host: "127.0.0.1",
+      port: 0,
+    });
+    try {
+      const response = await fetch(`${server.baseUrl}/v1/responses`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          model: "aimock/gpt-5.4",
+          stream: false,
+          input: [makeResponsesInput("@openclaw explain the QA lab")],
+        }),
+      });
+      expect(response.status).toBe(200);
+      expect(await response.json()).toMatchObject({
+        status: "completed",
+      });
+
+      const debug = await fetch(`${server.baseUrl}/debug/requests`);
+      expect(debug.status).toBe(200);
+      expect(await debug.json()).toEqual([
+        expect.objectContaining({
+          prompt: "@openclaw explain the QA lab",
+        }),
+      ]);
+    } finally {
+      await server.stop();
+    }
+  });
+
+  it("treats OpenAI Codex model refs as OpenAI-compatible snapshots", async () => {
+    const server = await startQaAimockServer({
+      host: "127.0.0.1",
+      port: 0,
+    });
+    try {
+      const response = await fetch(`${server.baseUrl}/v1/responses`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          model: "openai-codex/gpt-5.4",
+          stream: false,
+          input: [makeResponsesInput("hello codex-compatible aimock")],
+        }),
+      });
+      expect(response.status).toBe(200);
+
+      const debug = await fetch(`${server.baseUrl}/debug/last-request`);
+      expect(debug.status).toBe(200);
+      expect(await debug.json()).toMatchObject({
+        model: "openai-codex/gpt-5.4",
+        providerVariant: "openai",
+      });
+    } finally {
+      await server.stop();
+    }
+  });
+});

openclaw/extensions/qa-lab/src/providers/aimock/server.ts

@@ -0,0 +1,203 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import {
+  LLMock,
+  type ChatCompletionRequest,
+  type JournalEntry,
+  type Mountable,
+} from "@copilotkit/aimock";
+
+type AimockRequestSnapshot = {
+  raw: string;
+  body: Record<string, unknown>;
+  prompt: string;
+  allInputText: string;
+  toolOutput: string;
+  model: string;
+  providerVariant: "openai" | "anthropic" | "unknown";
+  imageInputCount: number;
+  plannedToolName?: string;
+};
+
+function writeJson(res: ServerResponse, status: number, body: unknown) {
+  const text = JSON.stringify(body);
+  res.writeHead(status, {
+    "content-type": "application/json; charset=utf-8",
+    "content-length": Buffer.byteLength(text),
+    "cache-control": "no-store",
+  });
+  res.end(text);
+}
+
+function stringifyContent(content: unknown): string {
+  if (typeof content === "string") {
+    return content;
+  }
+  if (Array.isArray(content)) {
+    return content
+      .map((part) => stringifyContent(part))
+      .filter(Boolean)
+      .join("\n");
+  }
+  if (content && typeof content === "object") {
+    const record = content as Record<string, unknown>;
+    if (typeof record.text === "string") {
+      return record.text;
+    }
+    if (typeof record.content === "string") {
+      return record.content;
+    }
+    if (typeof record.output === "string") {
+      return record.output;
+    }
+  }
+  return "";
+}
+
+function requestMessages(body: ChatCompletionRequest | null | undefined) {
+  return Array.isArray(body?.messages) ? body.messages : [];
+}
+
+function extractLastUserText(body: ChatCompletionRequest | null | undefined) {
+  const messages = requestMessages(body);
+  for (let index = messages.length - 1; index >= 0; index -= 1) {
+    const message = messages[index];
+    if (message?.role === "user") {
+      return stringifyContent(message.content);
+    }
+  }
+  return "";
+}
+
+function extractAllInputText(body: ChatCompletionRequest | null | undefined) {
+  return requestMessages(body)
+    .map((message) => stringifyContent(message.content))
+    .filter(Boolean)
+    .join("\n");
+}
+
+function extractToolOutput(body: ChatCompletionRequest | null | undefined) {
+  const messages = requestMessages(body);
+  for (let index = messages.length - 1; index >= 0; index -= 1) {
+    const message = messages[index];
+    if (message?.role === "tool") {
+      return stringifyContent(message.content);
+    }
+  }
+  return "";
+}
+
+function countImageInputs(value: unknown): number {
+  if (Array.isArray(value)) {
+    return value.reduce((sum, entry) => sum + countImageInputs(entry), 0);
+  }
+  if (!value || typeof value !== "object") {
+    return 0;
+  }
+  const record = value as Record<string, unknown>;
+  const type = typeof record.type === "string" ? record.type : "";
+  const imageLikeType =
+    type === "input_image" || type === "image" || type === "image_url" || type === "media";
+  const nested =
+    countImageInputs(record.content) +
+    countImageInputs(record.image_url) +
+    countImageInputs(record.source);
+  return (imageLikeType ? 1 : 0) + nested;
+}
+
+function resolveProviderVariant(model: string): AimockRequestSnapshot["providerVariant"] {
+  const normalized = model.trim().toLowerCase();
+  const provider = /^([^/:]+)[/:]/.exec(normalized)?.[1] ?? normalized;
+  if (provider === "openai" || provider === "aimock" || provider === "openai-codex") {
+    return "openai";
+  }
+  if (provider === "anthropic" || provider === "claude-cli") {
+    return "anthropic";
+  }
+  if (/^(?:gpt-|o1-|openai-)/.test(normalized)) {
+    return "openai";
+  }
+  if (/^(?:claude-|anthropic-)/.test(normalized)) {
+    return "anthropic";
+  }
+  return "unknown";
+}
+
+function extractPlannedToolName(entry: JournalEntry) {
+  const response = entry.response.fixture?.response as
+    | { toolCalls?: Array<{ name?: unknown }> }
+    | undefined;
+  const name = response?.toolCalls?.[0]?.name;
+  return typeof name === "string" && name.length > 0 ? name : undefined;
+}
+
+function toRequestSnapshot(entry: JournalEntry): AimockRequestSnapshot {
+  const body = entry.body ?? null;
+  const model = typeof body?.model === "string" ? body.model : "";
+  return {
+    raw: JSON.stringify(body ?? {}),
+    body: (body ?? {}) as Record<string, unknown>,
+    prompt: extractLastUserText(body),
+    allInputText: extractAllInputText(body),
+    toolOutput: extractToolOutput(body),
+    model,
+    providerVariant: resolveProviderVariant(model),
+    imageInputCount: countImageInputs(requestMessages(body)),
+    plannedToolName: extractPlannedToolName(entry),
+  };
+}
+
+function createDebugMount(mock: LLMock): Mountable {
+  return {
+    async handleRequest(_req: IncomingMessage, res: ServerResponse, pathname: string) {
+      const entries = mock.getRequests();
+      if (pathname === "/last-request") {
+        const lastEntry = entries.at(-1);
+        writeJson(
+          res,
+          200,
+          lastEntry ? toRequestSnapshot(lastEntry) : { ok: false, error: "no request recorded" },
+        );
+        return true;
+      }
+      if (pathname === "/requests") {
+        writeJson(
+          res,
+          200,
+          entries.map((entry) => toRequestSnapshot(entry)),
+        );
+        return true;
+      }
+      if (pathname === "/image-generations") {
+        writeJson(
+          res,
+          200,
+          entries
+            .filter((entry) => entry.path === "/v1/images/generations")
+            .map((entry) => entry.body ?? {}),
+        );
+        return true;
+      }
+      return false;
+    },
+  };
+}
+
+export async function startQaAimockServer(params?: { host?: string; port?: number }) {
+  const mock = new LLMock({
+    host: params?.host ?? "127.0.0.1",
+    port: params?.port ?? 0,
+    strict: false,
+    logLevel: "silent",
+  });
+
+  mock.mount("/debug", createDebugMount(mock));
+  mock.onMessage(/.*/, { content: "AIMOCK_QA_OK" });
+
+  await mock.start();
+  return {
+    baseUrl: mock.baseUrl,
+    async stop() {
+      await mock.stop();
+    },
+  };
+}

openclaw/extensions/qa-lab/src/providers/env.ts

@@ -0,0 +1,229 @@
+import { existsSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type { QaProviderMode } from "./index.js";
+import { getQaProvider } from "./index.js";
+
+const QA_LIVE_ENV_ALIASES = Object.freeze([
+  {
+    liveVar: "OPENCLAW_LIVE_OPENAI_KEY",
+    providerVar: "OPENAI_API_KEY",
+  },
+  {
+    liveVar: "OPENCLAW_LIVE_ANTHROPIC_KEY",
+    providerVar: "ANTHROPIC_API_KEY",
+  },
+  {
+    liveVar: "OPENCLAW_LIVE_GEMINI_KEY",
+    providerVar: "GEMINI_API_KEY",
+  },
+]);
+
+export const QA_LIVE_PROVIDER_CONFIG_PATH_ENV = "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH";
+export const QA_LIVE_CLI_BACKEND_PRESERVE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV";
+export const QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE";
+export type QaCliBackendAuthMode = "auto" | "api-key" | "subscription";
+
+export const QA_PROVIDER_SECRET_ENV_VARS = Object.freeze([
+  "ANTHROPIC_API_KEY",
+  "ANTHROPIC_OAUTH_TOKEN",
+  "AWS_ACCESS_KEY_ID",
+  "AWS_BEARER_TOKEN_BEDROCK",
+  "AWS_SECRET_ACCESS_KEY",
+  "AWS_SESSION_TOKEN",
+  "ANTHROPIC_API_KEYS",
+  "GEMINI_API_KEY",
+  "GEMINI_API_KEYS",
+  "GOOGLE_API_KEY",
+  "MISTRAL_API_KEY",
+  "OPENAI_API_KEY",
+  "OPENAI_API_KEYS",
+  "OPENCLAW_LIVE_ANTHROPIC_KEY",
+  "OPENCLAW_LIVE_ANTHROPIC_KEYS",
+  "OPENCLAW_LIVE_GEMINI_KEY",
+  "OPENCLAW_LIVE_OPENAI_KEY",
+  "VOYAGE_API_KEY",
+]);
+
+const QA_MOCK_BLOCKED_ENV_VARS = Object.freeze([
+  ...QA_PROVIDER_SECRET_ENV_VARS,
+  "AWS_REGION",
+  "OPENAI_BASE_URL",
+  "CODEX_HOME",
+]);
+
+const QA_MOCK_BLOCKED_ENV_KEY_PATTERNS = Object.freeze([
+  /^DISCORD_/i,
+  /^TELEGRAM_/i,
+  /^SLACK_/i,
+  /^MATRIX_/i,
+  /^SIGNAL_/i,
+  /^WHATSAPP_/i,
+  /^IMESSAGE_/i,
+  /^ZALO/i,
+  /^TWILIO_/i,
+  /^PLIVO_/i,
+  /^NGROK_/i,
+]);
+
+const QA_LIVE_ALLOWED_ENV_VARS = Object.freeze([
+  ...QA_PROVIDER_SECRET_ENV_VARS,
+  "AWS_REGION",
+  "OPENAI_BASE_URL",
+  QA_LIVE_PROVIDER_CONFIG_PATH_ENV,
+  "OPENCLAW_CONFIG_PATH",
+]);
+
+const QA_LIVE_ALLOWED_ENV_PATTERNS = Object.freeze([
+  /^[A-Z0-9_]+_API_KEYS$/u,
+  /^[A-Z0-9_]+_API_KEY_[0-9]+$/u,
+  /^OPENCLAW_LIVE_[A-Z0-9_]+_KEYS$/u,
+]);
+
+function resolveUserPath(value: string, env: NodeJS.ProcessEnv = process.env) {
+  if (value === "~") {
+    return env.HOME ?? os.homedir();
+  }
+  if (value.startsWith("~/")) {
+    return path.join(env.HOME ?? os.homedir(), value.slice(2));
+  }
+  return path.resolve(value);
+}
+
+function applyLiveProviderEnvAliases(env: NodeJS.ProcessEnv | Record<string, string>) {
+  for (const { liveVar, providerVar } of QA_LIVE_ENV_ALIASES) {
+    const liveValue = env[liveVar]?.trim();
+    if (!liveValue || env[providerVar]?.trim()) {
+      continue;
+    }
+    env[providerVar] = liveValue;
+  }
+}
+
+function parsePreservedCliEnv(baseEnv: NodeJS.ProcessEnv) {
+  const raw = baseEnv[QA_LIVE_CLI_BACKEND_PRESERVE_ENV]?.trim();
+  if (raw?.startsWith("[")) {
+    try {
+      const parsed = JSON.parse(raw) as unknown;
+      return Array.isArray(parsed)
+        ? parsed.filter((entry): entry is string => typeof entry === "string")
+        : [];
+    } catch {
+      return [];
+    }
+  }
+  return (raw ?? "").split(/[,\s]+/).filter((entry) => entry.length > 0);
+}
+
+function renderPreservedCliEnv(values: string[]) {
+  return JSON.stringify([...new Set(values)]);
+}
+
+export function normalizeQaProviderModeEnv(env: NodeJS.ProcessEnv, providerMode?: QaProviderMode) {
+  const provider = providerMode ? getQaProvider(providerMode) : null;
+  if (provider?.scrubsLiveProviderEnv) {
+    for (const key of QA_MOCK_BLOCKED_ENV_VARS) {
+      delete env[key];
+    }
+    for (const key of Object.keys(env)) {
+      if (QA_MOCK_BLOCKED_ENV_KEY_PATTERNS.some((pattern) => pattern.test(key))) {
+        delete env[key];
+      }
+    }
+    return env;
+  }
+
+  if (provider?.appliesLiveEnvAliases) {
+    applyLiveProviderEnvAliases(env);
+  }
+
+  return env;
+}
+
+export function resolveQaLiveCliAuthEnv(
+  baseEnv: NodeJS.ProcessEnv,
+  opts?: {
+    forwardHostHomeForClaudeCli?: boolean;
+    claudeCliAuthMode?: QaCliBackendAuthMode;
+  },
+) {
+  const authMode = opts?.claudeCliAuthMode ?? "auto";
+  const hasAnthropicKey = Boolean(
+    baseEnv.ANTHROPIC_API_KEY?.trim() || baseEnv.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim(),
+  );
+  if (opts?.forwardHostHomeForClaudeCli && authMode === "api-key" && !hasAnthropicKey) {
+    throw new Error(
+      "Claude CLI API-key QA mode requires ANTHROPIC_API_KEY or OPENCLAW_LIVE_ANTHROPIC_KEY",
+    );
+  }
+  const preserveEnvValues = (() => {
+    if (!opts?.forwardHostHomeForClaudeCli) {
+      return undefined;
+    }
+    const values = parsePreservedCliEnv(baseEnv).filter((entry) => entry !== "ANTHROPIC_API_KEY");
+    if (authMode === "api-key" || (authMode === "auto" && hasAnthropicKey)) {
+      values.push("ANTHROPIC_API_KEY");
+    }
+    return renderPreservedCliEnv(values);
+  })();
+  const claudeCliEnv = opts?.forwardHostHomeForClaudeCli
+    ? {
+        [QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV]: authMode,
+        ...(preserveEnvValues ? { [QA_LIVE_CLI_BACKEND_PRESERVE_ENV]: preserveEnvValues } : {}),
+      }
+    : {};
+  const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
+  if (configuredCodexHome) {
+    return {
+      CODEX_HOME: configuredCodexHome,
+      ...claudeCliEnv,
+      ...(opts?.forwardHostHomeForClaudeCli && baseEnv.HOME?.trim()
+        ? { HOME: baseEnv.HOME.trim() }
+        : {}),
+    };
+  }
+  const hostHome = baseEnv.HOME?.trim();
+  if (!hostHome) {
+    return {};
+  }
+  const codexHome = path.join(hostHome, ".codex");
+  return {
+    ...(existsSync(codexHome) ? { CODEX_HOME: codexHome } : {}),
+    ...claudeCliEnv,
+    ...(opts?.forwardHostHomeForClaudeCli ? { HOME: hostHome } : {}),
+  };
+}
+
+export function resolveQaLiveProviderConfigPath(env: NodeJS.ProcessEnv = process.env) {
+  const explicit =
+    env[QA_LIVE_PROVIDER_CONFIG_PATH_ENV]?.trim() || env.OPENCLAW_CONFIG_PATH?.trim();
+  return explicit
+    ? { path: resolveUserPath(explicit, env), explicit: true }
+    : { path: path.join(os.homedir(), ".openclaw", "openclaw.json"), explicit: false };
+}
+
+export function resolveQaForwardedLiveEnv(baseEnv: NodeJS.ProcessEnv = process.env) {
+  const forwarded: Record<string, string> = {};
+  for (const [key, rawValue] of Object.entries(baseEnv)) {
+    if (
+      !QA_LIVE_ALLOWED_ENV_VARS.includes(key) &&
+      !QA_LIVE_ALLOWED_ENV_PATTERNS.some((pattern) => pattern.test(key))
+    ) {
+      continue;
+    }
+    const value = rawValue?.trim();
+    if (value) {
+      forwarded[key] = value;
+    }
+  }
+  applyLiveProviderEnvAliases(forwarded);
+
+  const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
+  const codexHome = configuredCodexHome
+    ? resolveUserPath(configuredCodexHome, baseEnv)
+    : path.join(baseEnv.HOME?.trim() || os.homedir(), ".codex");
+  if (existsSync(codexHome)) {
+    forwarded.CODEX_HOME = codexHome;
+  }
+  return forwarded;
+}

openclaw/extensions/qa-lab/src/providers/image-generation.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import { buildQaImageGenerationConfigPatch } from "./image-generation.js";
+
+describe("QA provider image generation config", () => {
+  it("uses the selected mock provider for mock-openai image generation", () => {
+    const patch = buildQaImageGenerationConfigPatch({
+      providerMode: "mock-openai",
+      providerBaseUrl: "http://127.0.0.1:44080/v1",
+      requiredPluginIds: ["qa-channel"],
+    });
+
+    expect(patch.plugins.allow).toEqual(["memory-core", "qa-channel"]);
+    expect(patch.agents.defaults.imageGenerationModel.primary).toBe("mock-openai/gpt-image-1");
+    expect(patch.models?.providers["mock-openai"]?.baseUrl).toBe("http://127.0.0.1:44080/v1");
+  });
+
+  it("uses the selected mock provider for AIMock image generation", () => {
+    const patch = buildQaImageGenerationConfigPatch({
+      providerMode: "aimock",
+      providerBaseUrl: "http://127.0.0.1:45080/v1",
+      requiredPluginIds: [],
+    });
+
+    expect(patch.agents.defaults.imageGenerationModel.primary).toBe("aimock/gpt-image-1");
+    expect(patch.models?.providers.aimock?.baseUrl).toBe("http://127.0.0.1:45080/v1");
+    expect(patch.models?.providers["mock-openai"]).toBeUndefined();
+  });
+
+  it("enables the live image provider plugin without replacing live model config", () => {
+    const patch = buildQaImageGenerationConfigPatch({
+      providerMode: "live-frontier",
+      requiredPluginIds: ["qa-channel"],
+    });
+
+    expect(patch.plugins).toEqual({
+      allow: ["memory-core", "openai", "qa-channel"],
+      entries: {
+        openai: {
+          enabled: true,
+        },
+      },
+    });
+    expect(patch.agents.defaults.imageGenerationModel.primary).toBe("openai/gpt-image-1");
+    expect(patch).not.toHaveProperty("models");
+  });
+});

openclaw/extensions/qa-lab/src/providers/image-generation.ts

@@ -0,0 +1,73 @@
+import type { QaProviderMode } from "./index.js";
+import { getQaProvider } from "./index.js";
+
+type QaImageGenerationPatchInput = {
+  providerMode: QaProviderMode;
+  providerBaseUrl?: string;
+  requiredPluginIds: readonly string[];
+};
+
+function splitModelProviderId(modelRef: string) {
+  const slash = modelRef.indexOf("/");
+  return slash > 0 ? modelRef.slice(0, slash) : null;
+}
+
+function uniqueNonEmpty(values: readonly (string | null | undefined)[]) {
+  return [
+    ...new Set(values.map((value) => value?.trim()).filter((value): value is string => !!value)),
+  ];
+}
+
+export function buildQaImageGenerationConfigPatch(input: QaImageGenerationPatchInput) {
+  const provider = getQaProvider(input.providerMode);
+  const imageModelRef = provider.defaultImageGenerationModel({
+    modelProviderIds: provider.defaultImageGenerationProviderIds,
+  });
+  if (!imageModelRef) {
+    throw new Error(
+      `QA provider "${input.providerMode}" does not expose an image generation model`,
+    );
+  }
+  const imageProviderId = splitModelProviderId(imageModelRef);
+  const modelPatch = (() => {
+    if (provider.kind !== "mock") {
+      return null;
+    }
+    if (!input.providerBaseUrl) {
+      throw new Error(`QA provider "${input.providerMode}" requires a mock provider URL`);
+    }
+    return provider.buildGatewayModels({
+      providerBaseUrl: input.providerBaseUrl,
+    });
+  })();
+  const providerPluginIds = provider.usesModelProviderPlugins ? [imageProviderId] : [];
+  const enabledPluginIds = uniqueNonEmpty(providerPluginIds);
+
+  return {
+    plugins: {
+      allow: uniqueNonEmpty(["memory-core", ...enabledPluginIds, ...input.requiredPluginIds]),
+      ...(enabledPluginIds.length > 0
+        ? {
+            entries: Object.fromEntries(
+              enabledPluginIds.map((pluginId) => [pluginId, { enabled: true }]),
+            ),
+          }
+        : {}),
+    },
+    ...(modelPatch
+      ? {
+          models: {
+            mode: modelPatch.mode,
+            providers: modelPatch.providers,
+          },
+        }
+      : {}),
+    agents: {
+      defaults: {
+        imageGenerationModel: {
+          primary: imageModelRef,
+        },
+      },
+    },
+  };
+}

openclaw/extensions/qa-lab/src/providers/index.ts

@@ -0,0 +1,62 @@
+import { aimockProviderDefinition } from "./aimock/index.js";
+import { liveFrontierProviderDefinition } from "./live-frontier/index.js";
+import { mockOpenAiProviderDefinition } from "./mock-openai/index.js";
+import type { QaProviderDefinition, QaProviderMode, QaProviderModeInput } from "./shared/types.js";
+
+export type {
+  QaMockProviderServer,
+  QaProviderDefinition,
+  QaProviderMode,
+  QaProviderModeInput,
+} from "./shared/types.js";
+
+const PROVIDERS = [
+  mockOpenAiProviderDefinition,
+  aimockProviderDefinition,
+  liveFrontierProviderDefinition,
+] as const satisfies readonly QaProviderDefinition[];
+
+export const DEFAULT_QA_PROVIDER_MODE = "mock-openai" satisfies QaProviderMode;
+export const DEFAULT_QA_LIVE_PROVIDER_MODE = "live-frontier" satisfies QaProviderMode;
+
+const PROVIDERS_BY_INPUT = new Map<QaProviderModeInput, QaProviderDefinition>();
+for (const provider of PROVIDERS) {
+  PROVIDERS_BY_INPUT.set(provider.mode, provider);
+}
+
+export function isQaProviderModeInput(input: unknown): input is QaProviderModeInput {
+  return typeof input === "string" && PROVIDERS_BY_INPUT.has(input as QaProviderModeInput);
+}
+
+export function normalizeQaProviderMode(input: QaProviderModeInput): QaProviderMode {
+  return getQaProvider(input).mode;
+}
+
+export function getQaProvider(input: QaProviderModeInput): QaProviderDefinition {
+  const provider = PROVIDERS_BY_INPUT.get(input);
+  if (!provider) {
+    throw new Error(`unknown QA provider mode: ${input}`);
+  }
+  return provider;
+}
+
+export function listQaProviderModes() {
+  return PROVIDERS.map((provider) => provider.mode);
+}
+
+export function formatQaProviderModeHelp() {
+  return `Provider mode: ${listQaProviderModes().join(", ")}`;
+}
+
+export function listQaStandaloneProviderCommands() {
+  return PROVIDERS.flatMap((provider) =>
+    provider.standaloneCommand
+      ? [
+          {
+            providerMode: provider.mode,
+            ...provider.standaloneCommand,
+          },
+        ]
+      : [],
+  );
+}

openclaw/extensions/qa-lab/src/providers/live-frontier/auth.ts

@@ -0,0 +1,60 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
+import {
+  applyAuthProfileConfig,
+  upsertAuthProfile,
+  validateAnthropicSetupToken,
+} from "openclaw/plugin-sdk/provider-auth";
+
+export const QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN";
+export const QA_LIVE_SETUP_TOKEN_VALUE_ENV = "OPENCLAW_LIVE_SETUP_TOKEN_VALUE";
+const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE";
+const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID = "anthropic:qa-setup-token";
+
+function resolveQaLiveAnthropicSetupToken(env: NodeJS.ProcessEnv = process.env) {
+  const token = (
+    env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]?.trim() ||
+    env[QA_LIVE_SETUP_TOKEN_VALUE_ENV]?.trim() ||
+    ""
+  ).replaceAll(/\s+/g, "");
+  if (!token) {
+    return null;
+  }
+  const tokenError = validateAnthropicSetupToken(token);
+  if (tokenError) {
+    throw new Error(`Invalid QA Anthropic setup-token: ${tokenError}`);
+  }
+  const profileId =
+    env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV]?.trim() ||
+    QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID;
+  return { token, profileId };
+}
+
+export async function stageQaLiveAnthropicSetupToken(params: {
+  cfg: OpenClawConfig;
+  stateDir: string;
+  env?: NodeJS.ProcessEnv;
+}): Promise<OpenClawConfig> {
+  const resolved = resolveQaLiveAnthropicSetupToken(params.env);
+  if (!resolved) {
+    return params.cfg;
+  }
+  const agentDir = path.join(params.stateDir, "agents", "main", "agent");
+  await fs.mkdir(agentDir, { recursive: true });
+  upsertAuthProfile({
+    profileId: resolved.profileId,
+    credential: {
+      type: "token",
+      provider: "anthropic",
+      token: resolved.token,
+    },
+    agentDir,
+  });
+  return applyAuthProfileConfig(params.cfg, {
+    profileId: resolved.profileId,
+    provider: "anthropic",
+    mode: "token",
+    displayName: "QA setup-token",
+  });
+}

openclaw/extensions/qa-lab/src/providers/live-frontier/catalog.ts

@@ -0,0 +1,7 @@
+export const QA_FRONTIER_PROVIDER_IDS = ["anthropic", "google", "openai"] as const;
+export const QA_FRONTIER_CATALOG_PRIMARY_MODEL = "openai/gpt-5.4";
+export const QA_FRONTIER_CATALOG_ALTERNATE_MODEL = "anthropic/claude-sonnet-4-6";
+
+export function isPreferredQaLiveFrontierCatalogModel(modelRef: string) {
+  return modelRef === QA_FRONTIER_CATALOG_PRIMARY_MODEL;
+}

openclaw/extensions/qa-lab/src/providers/live-frontier/character-eval.ts

@@ -0,0 +1,36 @@
+import type { QaThinkingLevel } from "../../qa-thinking.js";
+
+type QaFrontierCharacterModelOptions = {
+  thinkingDefault?: QaThinkingLevel;
+  fastMode?: boolean;
+};
+
+export const QA_FRONTIER_CHARACTER_EVAL_MODELS = Object.freeze([
+  "openai/gpt-5.4",
+  "openai/gpt-5.2",
+  "openai/gpt-5",
+  "anthropic/claude-opus-4-6",
+  "anthropic/claude-sonnet-4-6",
+  "zai/glm-5.1",
+  "moonshot/kimi-k2.5",
+  "google/gemini-3.1-pro-preview",
+]);
+
+export const QA_FRONTIER_CHARACTER_THINKING_BY_MODEL: Readonly<Record<string, QaThinkingLevel>> =
+  Object.freeze({
+    "openai/gpt-5.4": "xhigh",
+    "openai/gpt-5.2": "xhigh",
+    "openai/gpt-5": "xhigh",
+  });
+
+export const QA_FRONTIER_CHARACTER_JUDGE_MODELS = Object.freeze([
+  "openai/gpt-5.4",
+  "anthropic/claude-opus-4-6",
+]);
+
+export const QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS: Readonly<
+  Record<string, QaFrontierCharacterModelOptions>
+> = Object.freeze({
+  "openai/gpt-5.4": { thinkingDefault: "xhigh" },
+  "anthropic/claude-opus-4-6": { thinkingDefault: "high" },
+});

openclaw/extensions/qa-lab/src/providers/live-frontier/index.ts

@@ -0,0 +1,61 @@
+import type { QaProviderDefinition } from "../shared/types.js";
+
+function isOpenAiModel(modelRef: string) {
+  return modelRef.startsWith("openai/");
+}
+
+function isAnthropicModel(modelRef: string) {
+  return modelRef.startsWith("anthropic/");
+}
+
+function isQaFastModeModelRef(modelRef: string) {
+  return isOpenAiModel(modelRef);
+}
+
+function isGptFiveModel(modelRef: string) {
+  return isOpenAiModel(modelRef) && modelRef.slice("openai/".length).startsWith("gpt-5");
+}
+
+function isClaudeOpusModel(modelRef: string) {
+  return isAnthropicModel(modelRef) && modelRef.includes("claude-opus");
+}
+
+export const liveFrontierProviderDefinition: QaProviderDefinition = {
+  mode: "live-frontier",
+  kind: "live",
+  defaultModel: (options) => options?.preferredLiveModel ?? "openai/gpt-5.4",
+  defaultImageGenerationProviderIds: ["openai"],
+  defaultImageGenerationModel: ({ modelProviderIds }) =>
+    modelProviderIds.includes("openai") ? "openai/gpt-image-1" : null,
+  usesFastModeByDefault: isQaFastModeModelRef,
+  resolveModelParams: ({ modelRef, fastMode, thinkingDefault }) => ({
+    transport: "sse",
+    openaiWsWarmup: false,
+    ...(fastMode === true || isQaFastModeModelRef(modelRef) ? { fastMode: true } : {}),
+    ...(thinkingDefault ? { thinking: thinkingDefault } : {}),
+  }),
+  resolveTurnTimeoutMs: ({ fallbackMs, modelRef }) => {
+    if (isClaudeOpusModel(modelRef)) {
+      return Math.max(fallbackMs, 240_000);
+    }
+    if (isAnthropicModel(modelRef)) {
+      return Math.max(fallbackMs, 180_000);
+    }
+    if (isGptFiveModel(modelRef)) {
+      return Math.max(fallbackMs, 360_000);
+    }
+    return Math.max(fallbackMs, 120_000);
+  },
+  buildGatewayModels: ({ liveProviderConfigs }) => {
+    const providers = liveProviderConfigs ?? {};
+    return Object.keys(providers).length > 0
+      ? {
+          mode: "merge",
+          providers,
+        }
+      : null;
+  },
+  usesModelProviderPlugins: true,
+  scrubsLiveProviderEnv: false,
+  appliesLiveEnvAliases: true,
+};

openclaw/extensions/qa-lab/src/providers/live-frontier/model-selection.runtime.ts

@@ -0,0 +1,27 @@
+import {
+  listProfilesForProvider,
+  loadAuthProfileStoreForRuntime,
+} from "openclaw/plugin-sdk/agent-runtime";
+import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth";
+
+const QA_CODEX_OAUTH_LIVE_MODEL = "openai-codex/gpt-5.4";
+
+export function resolveQaLiveFrontierPreferredModel() {
+  if (resolveEnvApiKey("openai")?.apiKey) {
+    return undefined;
+  }
+  try {
+    const store = loadAuthProfileStoreForRuntime(undefined, {
+      readOnly: true,
+      allowKeychainPrompt: false,
+    });
+    if (listProfilesForProvider(store, "openai").length > 0) {
+      return undefined;
+    }
+    return listProfilesForProvider(store, "openai-codex").length > 0
+      ? QA_CODEX_OAUTH_LIVE_MODEL
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}

openclaw/extensions/qa-lab/src/providers/live-frontier/parity.ts

@@ -0,0 +1,2 @@
+export const QA_FRONTIER_PARITY_CANDIDATE_LABEL = "openai/gpt-5.4";
+export const QA_FRONTIER_PARITY_BASELINE_LABEL = "anthropic/claude-opus-4-6";

openclaw/extensions/qa-lab/src/providers/mock-openai/index.ts

@@ -0,0 +1,9 @@
+import { createMockQaProviderDefinition } from "../shared/mock-provider-definition.js";
+
+export const mockOpenAiProviderDefinition = createMockQaProviderDefinition({
+  mode: "mock-openai",
+  commandName: "mock-openai",
+  commandDescription: "Run the local mock OpenAI Responses API server for QA",
+  serverLabel: "QA mock OpenAI",
+  mockAuthProviders: ["openai", "anthropic"],
+});

openclaw/extensions/qa-lab/src/providers/server-runtime.ts

@@ -0,0 +1,36 @@
+import { getQaProvider, type QaMockProviderServer, type QaProviderModeInput } from "./index.js";
+
+type QaProviderServerParams = {
+  host: string;
+  port: number;
+};
+
+async function startMockOpenAiProviderServer(params: QaProviderServerParams) {
+  const { startQaMockOpenAiServer } = await import("./mock-openai/server.js");
+  return await startQaMockOpenAiServer(params);
+}
+
+async function startAimockProviderServer(params: QaProviderServerParams) {
+  const { startQaAimockServer } = await import("./aimock/server.js");
+  return await startQaAimockServer(params);
+}
+
+export async function startQaProviderServer(
+  input: QaProviderModeInput,
+  params?: { host?: string; port?: number },
+): Promise<QaMockProviderServer | null> {
+  const provider = getQaProvider(input);
+  const serverParams = {
+    host: params?.host ?? "127.0.0.1",
+    port: params?.port ?? 0,
+  };
+  switch (provider.mode) {
+    case "mock-openai":
+      return await startMockOpenAiProviderServer(serverParams);
+    case "aimock":
+      return await startAimockProviderServer(serverParams);
+    case "live-frontier":
+    default:
+      return null;
+  }
+}

openclaw/extensions/qa-lab/src/providers/shared/mock-auth.ts

@@ -0,0 +1,70 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
+import { applyAuthProfileConfig, upsertAuthProfile } from "openclaw/plugin-sdk/provider-auth";
+
+/** Providers the mock harness stages placeholder credentials for by default. */
+export const QA_MOCK_AUTH_PROVIDERS = Object.freeze(["openai", "anthropic"] as const);
+
+/** Agent IDs the mock harness stages credentials under. */
+export const QA_MOCK_AUTH_AGENT_IDS = Object.freeze(["main", "qa"] as const);
+
+export function buildQaMockProfileId(provider: string): string {
+  return `qa-mock-${provider}`;
+}
+
+/**
+ * In mock provider modes the qa suite runs against an embedded mock server
+ * instead of a real provider API. The mock does not validate credentials, but
+ * the agent auth layer still needs a matching `api_key` auth profile in
+ * `auth-profiles.json` before it will route the request through
+ * `providerBaseUrl`. Without this staging step, every scenario fails with
+ * `FailoverError: No API key found for provider "openai"` before the mock
+ * server ever sees a request.
+ *
+ * Stages a placeholder `api_key` profile per provider in each of the agent
+ * dirs the qa suite uses (`main` for the runtime config, `qa` for scenario
+ * runs) and returns a config with matching `auth.profiles` entries so the
+ * runtime accepts the profile on the first lookup.
+ *
+ * The placeholder value `qa-mock-not-a-real-key` is intentionally not
+ * shaped like a real API key (no `sk-` prefix that would trip secret
+ * scanners). It only needs to be non-empty to pass the credential
+ * serializer; anything beyond that is ignored by the mock.
+ */
+export async function stageQaMockAuthProfiles(params: {
+  cfg: OpenClawConfig;
+  stateDir: string;
+  agentIds?: readonly string[];
+  providers?: readonly string[];
+}): Promise<OpenClawConfig> {
+  const agentIds = [...new Set(params.agentIds ?? QA_MOCK_AUTH_AGENT_IDS)];
+  const providers = [...new Set(params.providers ?? QA_MOCK_AUTH_PROVIDERS)];
+  let next = params.cfg;
+  for (const agentId of agentIds) {
+    const agentDir = path.join(params.stateDir, "agents", agentId, "agent");
+    await fs.mkdir(agentDir, { recursive: true });
+    for (const provider of providers) {
+      const profileId = buildQaMockProfileId(provider);
+      upsertAuthProfile({
+        profileId,
+        credential: {
+          type: "api_key",
+          provider,
+          key: "qa-mock-not-a-real-key",
+          displayName: `QA mock ${provider} credential`,
+        },
+        agentDir,
+      });
+    }
+  }
+  for (const provider of providers) {
+    next = applyAuthProfileConfig(next, {
+      profileId: buildQaMockProfileId(provider),
+      provider,
+      mode: "api_key",
+      displayName: `QA mock ${provider} credential`,
+    });
+  }
+  return next;
+}