mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-10 02:30:46 +08:00
重构主控编排与运行时预热链路,统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为(含 wiki worker)、更新 Admin 可观测性与相关测试,降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
169
openclaw/scripts/lib/arg-utils.mjs
Normal file
169
openclaw/scripts/lib/arg-utils.mjs
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
export function readEnvNumber(name, env = process.env) {
|
||||
const raw = env[name]?.trim();
|
||||
if (!raw) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Number.parseFloat(raw);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
export function consumeStringFlag(argv, index, flag, currentValue) {
|
||||
if (argv[index] !== flag) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
nextIndex: index + 1,
|
||||
value: argv[index + 1] ?? currentValue,
|
||||
};
|
||||
}
|
||||
|
||||
export function consumeStringListFlag(argv, index, flag) {
|
||||
if (argv[index] !== flag) {
|
||||
return null;
|
||||
}
|
||||
const value = argv[index + 1];
|
||||
return {
|
||||
nextIndex: index + 1,
|
||||
value: typeof value === "string" && value.length > 0 ? value : null,
|
||||
};
|
||||
}
|
||||
|
||||
export function consumeIntFlag(argv, index, flag, currentValue, options = {}) {
|
||||
if (argv[index] !== flag) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Number.parseInt(argv[index + 1] ?? "", 10);
|
||||
const min = options.min ?? Number.NEGATIVE_INFINITY;
|
||||
return {
|
||||
nextIndex: index + 1,
|
||||
value: Number.isFinite(parsed) && parsed >= min ? parsed : currentValue,
|
||||
};
|
||||
}
|
||||
|
||||
export function consumeFloatFlag(argv, index, flag, currentValue, options = {}) {
|
||||
if (argv[index] !== flag) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Number.parseFloat(argv[index + 1] ?? "");
|
||||
const min = options.min ?? Number.NEGATIVE_INFINITY;
|
||||
const includeMin = options.includeMin ?? true;
|
||||
const isValid = Number.isFinite(parsed) && (includeMin ? parsed >= min : parsed > min);
|
||||
return {
|
||||
nextIndex: index + 1,
|
||||
value: isValid ? parsed : currentValue,
|
||||
};
|
||||
}
|
||||
|
||||
export function stringFlag(flag, key) {
|
||||
return {
|
||||
consume(argv, index, args) {
|
||||
const option = consumeStringFlag(argv, index, flag, args[key]);
|
||||
if (!option) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
nextIndex: option.nextIndex,
|
||||
apply(target) {
|
||||
target[key] = option.value;
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function stringListFlag(flag, key) {
|
||||
return {
|
||||
consume(argv, index) {
|
||||
const option = consumeStringListFlag(argv, index, flag);
|
||||
if (!option) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
nextIndex: option.nextIndex,
|
||||
apply(target) {
|
||||
if (option.value) {
|
||||
target[key].push(option.value);
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createAssignedValueFlag(consumeOption) {
|
||||
return {
|
||||
consume(argv, index, args) {
|
||||
const option = consumeOption(argv, index, args);
|
||||
if (!option) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
nextIndex: option.nextIndex,
|
||||
apply(target) {
|
||||
target[option.key] = option.value;
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function intFlag(flag, key, options) {
|
||||
return createAssignedValueFlag((argv, index, args) => {
|
||||
const option = consumeIntFlag(argv, index, flag, args[key], options);
|
||||
return option ? { ...option, key } : null;
|
||||
});
|
||||
}
|
||||
|
||||
export function floatFlag(flag, key, options) {
|
||||
return createAssignedValueFlag((argv, index, args) => {
|
||||
const option = consumeFloatFlag(argv, index, flag, args[key], options);
|
||||
return option ? { ...option, key } : null;
|
||||
});
|
||||
}
|
||||
|
||||
export function booleanFlag(flag, key, value = true) {
|
||||
return {
|
||||
consume(argv, index) {
|
||||
if (argv[index] !== flag) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
nextIndex: index,
|
||||
apply(target) {
|
||||
target[key] = value;
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function parseFlagArgs(argv, args, specs, options = {}) {
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i];
|
||||
if (arg === "--" && options.ignoreDoubleDash) {
|
||||
continue;
|
||||
}
|
||||
let handled = false;
|
||||
for (const spec of specs) {
|
||||
const option = spec.consume(argv, i, args);
|
||||
if (!option) {
|
||||
continue;
|
||||
}
|
||||
option.apply(args);
|
||||
i = option.nextIndex;
|
||||
handled = true;
|
||||
break;
|
||||
}
|
||||
if (handled) {
|
||||
continue;
|
||||
}
|
||||
const fallbackResult = options.onUnhandledArg?.(arg, args);
|
||||
if (fallbackResult === "handled") {
|
||||
continue;
|
||||
}
|
||||
if (!options.allowUnknownOptions && arg.startsWith("-")) {
|
||||
throw new Error(`Unknown option: ${arg}`);
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
44
openclaw/scripts/lib/bundled-extension-manifest.ts
Normal file
44
openclaw/scripts/lib/bundled-extension-manifest.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import { validateMinHostVersion } from "../../src/plugins/min-host-version.ts";
|
||||
import { isRecord } from "../../src/utils.js";
|
||||
|
||||
export type ExtensionPackageJson = {
|
||||
name?: string;
|
||||
version?: string;
|
||||
dependencies?: Record<string, string>;
|
||||
optionalDependencies?: Record<string, string>;
|
||||
openclaw?: {
|
||||
install?: unknown;
|
||||
releaseChecks?: unknown;
|
||||
};
|
||||
};
|
||||
|
||||
export type BundledExtension = { id: string; packageJson: ExtensionPackageJson };
|
||||
|
||||
export function collectBundledExtensionManifestErrors(extensions: BundledExtension[]): string[] {
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const extension of extensions) {
|
||||
const install = extension.packageJson.openclaw?.install;
|
||||
if (install !== undefined && !isRecord(install)) {
|
||||
errors.push(
|
||||
`bundled extension '${extension.id}' manifest invalid | openclaw.install must be an object`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const hasNpmSpec = isRecord(install) && "npmSpec" in install;
|
||||
if (
|
||||
hasNpmSpec &&
|
||||
(!install.npmSpec || typeof install.npmSpec !== "string" || !install.npmSpec.trim())
|
||||
) {
|
||||
errors.push(
|
||||
`bundled extension '${extension.id}' manifest invalid | openclaw.install.npmSpec must be a non-empty string`,
|
||||
);
|
||||
}
|
||||
const minHostVersionError = validateMinHostVersion(install?.minHostVersion);
|
||||
if (minHostVersionError) {
|
||||
errors.push(`bundled extension '${extension.id}' manifest invalid | ${minHostVersionError}`);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
23
openclaw/scripts/lib/bundled-plugin-build-entries-types.d.ts
vendored
Normal file
23
openclaw/scripts/lib/bundled-plugin-build-entries-types.d.ts
vendored
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
export type BundledPluginBuildEntry = {
|
||||
id: string;
|
||||
hasPackageJson: boolean;
|
||||
packageJson: unknown;
|
||||
sourceEntries: string[];
|
||||
};
|
||||
|
||||
export type BundledPluginBuildEntryParams = {
|
||||
cwd?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
};
|
||||
|
||||
export const NON_PACKAGED_BUNDLED_PLUGIN_DIRS: Set<string>;
|
||||
export function collectBundledPluginBuildEntries(
|
||||
params?: BundledPluginBuildEntryParams,
|
||||
): BundledPluginBuildEntry[];
|
||||
export function listBundledPluginBuildEntries(
|
||||
params?: BundledPluginBuildEntryParams,
|
||||
): Record<string, string>;
|
||||
export function listBundledPluginPackArtifacts(params?: BundledPluginBuildEntryParams): string[];
|
||||
export function listBundledPluginRuntimeDependencies(
|
||||
params?: BundledPluginBuildEntryParams,
|
||||
): string[];
|
||||
1
openclaw/scripts/lib/bundled-plugin-build-entries.d.mts
Normal file
1
openclaw/scripts/lib/bundled-plugin-build-entries.d.mts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export * from "./bundled-plugin-build-entries-types.js";
|
||||
1
openclaw/scripts/lib/bundled-plugin-build-entries.d.ts
vendored
Normal file
1
openclaw/scripts/lib/bundled-plugin-build-entries.d.ts
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
export * from "./bundled-plugin-build-entries-types.js";
|
||||
188
openclaw/scripts/lib/bundled-plugin-build-entries.mjs
Normal file
188
openclaw/scripts/lib/bundled-plugin-build-entries.mjs
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import {
|
||||
BUNDLED_PLUGIN_ROOT_DIR,
|
||||
bundledDistPluginFile,
|
||||
bundledPluginFile,
|
||||
} from "./bundled-plugin-paths.mjs";
|
||||
import { shouldBuildBundledCluster } from "./optional-bundled-clusters.mjs";
|
||||
|
||||
const TOP_LEVEL_PUBLIC_SURFACE_EXTENSIONS = new Set([".ts", ".js", ".mts", ".cts", ".mjs", ".cjs"]);
|
||||
export const NON_PACKAGED_BUNDLED_PLUGIN_DIRS = new Set(["qa-channel", "qa-lab", "qa-matrix"]);
|
||||
const toPosixPath = (value) => value.replaceAll("\\", "/");
|
||||
|
||||
function readBundledPluginPackageJson(packageJsonPath) {
|
||||
if (!fs.existsSync(packageJsonPath)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isManifestlessBundledRuntimeSupportPackage(params) {
|
||||
const packageName = typeof params.packageJson?.name === "string" ? params.packageJson.name : "";
|
||||
if (packageName !== `@openclaw/${params.dirName}`) {
|
||||
return false;
|
||||
}
|
||||
return params.topLevelPublicSurfaceEntries.length > 0;
|
||||
}
|
||||
|
||||
function collectPluginSourceEntries(packageJson) {
|
||||
let packageEntries = Array.isArray(packageJson?.openclaw?.extensions)
|
||||
? packageJson.openclaw.extensions.filter(
|
||||
(entry) => typeof entry === "string" && entry.trim().length > 0,
|
||||
)
|
||||
: [];
|
||||
const setupEntry =
|
||||
typeof packageJson?.openclaw?.setupEntry === "string" &&
|
||||
packageJson.openclaw.setupEntry.trim().length > 0
|
||||
? packageJson.openclaw.setupEntry
|
||||
: undefined;
|
||||
if (setupEntry) {
|
||||
packageEntries = Array.from(new Set([...packageEntries, setupEntry]));
|
||||
}
|
||||
return packageEntries.length > 0 ? packageEntries : ["./index.ts"];
|
||||
}
|
||||
|
||||
function shouldStageBundledPluginRuntimeDependencies(packageJson) {
|
||||
return packageJson?.openclaw?.bundle?.stageRuntimeDependencies === true;
|
||||
}
|
||||
|
||||
function collectTopLevelPublicSurfaceEntries(pluginDir) {
|
||||
if (!fs.existsSync(pluginDir)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return fs
|
||||
.readdirSync(pluginDir, { withFileTypes: true })
|
||||
.flatMap((dirent) => {
|
||||
if (!dirent.isFile()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const ext = path.extname(dirent.name);
|
||||
if (!TOP_LEVEL_PUBLIC_SURFACE_EXTENSIONS.has(ext)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const normalizedName = dirent.name.toLowerCase();
|
||||
if (
|
||||
normalizedName.endsWith(".d.ts") ||
|
||||
/^config-api\.(?:[cm]?[jt]s)$/u.test(normalizedName) ||
|
||||
normalizedName.includes(".test.") ||
|
||||
normalizedName.includes(".spec.") ||
|
||||
normalizedName.includes(".fixture.") ||
|
||||
normalizedName.includes(".snap")
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [`./${dirent.name}`];
|
||||
})
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export function collectBundledPluginBuildEntries(params = {}) {
|
||||
const cwd = params.cwd ?? process.cwd();
|
||||
const env = params.env ?? process.env;
|
||||
const extensionsRoot = path.join(cwd, BUNDLED_PLUGIN_ROOT_DIR);
|
||||
const entries = [];
|
||||
|
||||
for (const dirent of fs.readdirSync(extensionsRoot, { withFileTypes: true })) {
|
||||
if (!dirent.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const pluginDir = path.join(extensionsRoot, dirent.name);
|
||||
const manifestPath = path.join(pluginDir, "openclaw.plugin.json");
|
||||
const hasManifest = fs.existsSync(manifestPath);
|
||||
const packageJsonPath = path.join(pluginDir, "package.json");
|
||||
const packageJson = readBundledPluginPackageJson(packageJsonPath);
|
||||
const topLevelPublicSurfaceEntries = collectTopLevelPublicSurfaceEntries(pluginDir);
|
||||
if (
|
||||
!hasManifest &&
|
||||
!isManifestlessBundledRuntimeSupportPackage({
|
||||
dirName: dirent.name,
|
||||
packageJson,
|
||||
topLevelPublicSurfaceEntries,
|
||||
})
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (!shouldBuildBundledCluster(dirent.name, env, { packageJson })) {
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.push({
|
||||
id: dirent.name,
|
||||
hasManifest,
|
||||
hasPackageJson: packageJson !== null,
|
||||
packageJson,
|
||||
sourceEntries: Array.from(
|
||||
new Set([
|
||||
...(hasManifest ? collectPluginSourceEntries(packageJson) : []),
|
||||
...topLevelPublicSurfaceEntries,
|
||||
]),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
export function listBundledPluginBuildEntries(params = {}) {
|
||||
return Object.fromEntries(
|
||||
collectBundledPluginBuildEntries(params).flatMap(({ id, sourceEntries }) =>
|
||||
sourceEntries.map((entry) => {
|
||||
const normalizedEntry = entry.replace(/^\.\//, "");
|
||||
const entryKey = bundledPluginFile(id, normalizedEntry.replace(/\.[^.]+$/u, ""));
|
||||
return [entryKey, toPosixPath(path.join(BUNDLED_PLUGIN_ROOT_DIR, id, normalizedEntry))];
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function listBundledPluginPackArtifacts(params = {}) {
|
||||
const entries = collectBundledPluginBuildEntries(params).filter(
|
||||
({ id }) => !NON_PACKAGED_BUNDLED_PLUGIN_DIRS.has(id),
|
||||
);
|
||||
const artifacts = new Set();
|
||||
|
||||
for (const { id, hasManifest, hasPackageJson, sourceEntries } of entries) {
|
||||
if (hasManifest) {
|
||||
artifacts.add(bundledDistPluginFile(id, "openclaw.plugin.json"));
|
||||
}
|
||||
if (hasPackageJson) {
|
||||
artifacts.add(bundledDistPluginFile(id, "package.json"));
|
||||
}
|
||||
for (const entry of sourceEntries) {
|
||||
const normalizedEntry = entry.replace(/^\.\//, "").replace(/\.[^.]+$/u, "");
|
||||
artifacts.add(bundledDistPluginFile(id, `${normalizedEntry}.js`));
|
||||
}
|
||||
}
|
||||
|
||||
return [...artifacts].toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export function listBundledPluginRuntimeDependencies(params = {}) {
|
||||
const runtimeDependencies = new Set();
|
||||
|
||||
for (const { packageJson } of collectBundledPluginBuildEntries(params)) {
|
||||
if (!shouldStageBundledPluginRuntimeDependencies(packageJson)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const dependencyName of Object.keys(packageJson?.dependencies ?? {})) {
|
||||
runtimeDependencies.add(dependencyName);
|
||||
}
|
||||
|
||||
for (const dependencyName of Object.keys(packageJson?.optionalDependencies ?? {})) {
|
||||
runtimeDependencies.add(dependencyName);
|
||||
}
|
||||
}
|
||||
|
||||
return [...runtimeDependencies].toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
25
openclaw/scripts/lib/bundled-plugin-paths.mjs
Normal file
25
openclaw/scripts/lib/bundled-plugin-paths.mjs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
export const BUNDLED_PLUGIN_ROOT_DIR = "extensions";
|
||||
export const BUNDLED_PLUGIN_PATH_PREFIX = `${BUNDLED_PLUGIN_ROOT_DIR}/`;
|
||||
export const BUNDLED_PLUGIN_TEST_GLOB = `${BUNDLED_PLUGIN_ROOT_DIR}/**/*.test.ts`;
|
||||
export const BUNDLED_PLUGIN_E2E_TEST_GLOB = `${BUNDLED_PLUGIN_ROOT_DIR}/**/*.e2e.test.ts`;
|
||||
export const BUNDLED_PLUGIN_LIVE_TEST_GLOB = `${BUNDLED_PLUGIN_ROOT_DIR}/**/*.live.test.ts`;
|
||||
|
||||
export function bundledPluginRoot(pluginId) {
|
||||
return `${BUNDLED_PLUGIN_PATH_PREFIX}${pluginId}`;
|
||||
}
|
||||
|
||||
export function bundledPluginFile(pluginId, relativePath) {
|
||||
return `${bundledPluginRoot(pluginId)}/${relativePath}`;
|
||||
}
|
||||
|
||||
export function bundledDistPluginRoot(pluginId) {
|
||||
return `dist/${bundledPluginRoot(pluginId)}`;
|
||||
}
|
||||
|
||||
export function bundledDistPluginFile(pluginId, relativePath) {
|
||||
return `${bundledDistPluginRoot(pluginId)}/${relativePath}`;
|
||||
}
|
||||
|
||||
export function bundledPluginCallsite(pluginId, relativePath, line) {
|
||||
return `${bundledPluginFile(pluginId, relativePath)}:${line}`;
|
||||
}
|
||||
208
openclaw/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs
Normal file
208
openclaw/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const JS_EXTENSIONS = new Set([".cjs", ".js", ".mjs"]);
|
||||
const CURATED_ROOT_RUNTIME_MIRRORS = new Set([
|
||||
"@matrix-org/matrix-sdk-crypto-nodejs",
|
||||
"@matrix-org/matrix-sdk-crypto-wasm",
|
||||
]);
|
||||
|
||||
export function collectRuntimeDependencySpecs(packageJson = {}) {
|
||||
return new Map(
|
||||
[
|
||||
...Object.entries(packageJson.dependencies ?? {}),
|
||||
...Object.entries(packageJson.optionalDependencies ?? {}),
|
||||
].filter((entry) => typeof entry[1] === "string" && entry[1].length > 0),
|
||||
);
|
||||
}
|
||||
|
||||
export function packageNameFromSpecifier(specifier) {
|
||||
if (
|
||||
typeof specifier !== "string" ||
|
||||
specifier.startsWith(".") ||
|
||||
specifier.startsWith("/") ||
|
||||
specifier.startsWith("node:") ||
|
||||
specifier.startsWith("#")
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const [first, second] = specifier.split("/");
|
||||
if (!first) {
|
||||
return null;
|
||||
}
|
||||
return first.startsWith("@") && second ? `${first}/${second}` : first;
|
||||
}
|
||||
|
||||
function readJson(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function collectPackageJsonPaths(rootDir) {
|
||||
if (!fs.existsSync(rootDir)) {
|
||||
return [];
|
||||
}
|
||||
return fs
|
||||
.readdirSync(rootDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => path.join(rootDir, entry.name, "package.json"))
|
||||
.filter((packageJsonPath) => fs.existsSync(packageJsonPath))
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function usesStagedRuntimeDependencies(packageJson) {
|
||||
return packageJson?.openclaw?.bundle?.stageRuntimeDependencies === true;
|
||||
}
|
||||
|
||||
function dependencySentinelPath(packageRoot, dependencyName) {
|
||||
return path.join(packageRoot, "node_modules", ...dependencyName.split("/"), "package.json");
|
||||
}
|
||||
|
||||
function pluginIdFromPackageJsonPath(packageJsonPath) {
|
||||
return path.basename(path.dirname(packageJsonPath));
|
||||
}
|
||||
|
||||
export function collectBundledPluginRuntimeDependencySpecs(bundledPluginsDir) {
|
||||
const specs = new Map();
|
||||
|
||||
for (const packageJsonPath of collectPackageJsonPaths(bundledPluginsDir)) {
|
||||
const packageJson = readJson(packageJsonPath);
|
||||
const pluginId = path.basename(path.dirname(packageJsonPath));
|
||||
for (const [name, spec] of collectRuntimeDependencySpecs(packageJson)) {
|
||||
const existing = specs.get(name);
|
||||
if (existing) {
|
||||
if (existing.spec !== spec) {
|
||||
existing.conflicts.push({ pluginId, spec });
|
||||
} else if (!existing.pluginIds.includes(pluginId)) {
|
||||
existing.pluginIds.push(pluginId);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
specs.set(name, { conflicts: [], pluginIds: [pluginId], spec });
|
||||
}
|
||||
}
|
||||
|
||||
return specs;
|
||||
}
|
||||
|
||||
export function collectBuiltBundledPluginStagedRuntimeDependencyErrors(params) {
|
||||
const errors = [];
|
||||
|
||||
for (const packageJsonPath of collectPackageJsonPaths(params.bundledPluginsDir)) {
|
||||
const packageJson = readJson(packageJsonPath);
|
||||
if (!usesStagedRuntimeDependencies(packageJson)) {
|
||||
continue;
|
||||
}
|
||||
const pluginId = pluginIdFromPackageJsonPath(packageJsonPath);
|
||||
const pluginRoot = path.dirname(packageJsonPath);
|
||||
|
||||
for (const [dependencyName, spec] of collectRuntimeDependencySpecs(packageJson)) {
|
||||
if (!fs.existsSync(dependencySentinelPath(pluginRoot, dependencyName))) {
|
||||
const specText = String(spec);
|
||||
errors.push(
|
||||
`built bundled plugin '${pluginId}' is missing staged runtime dependency '${dependencyName}: ${specText}' under dist/extensions/${pluginId}/node_modules.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errors.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function walkJavaScriptFiles(rootDir) {
|
||||
const files = [];
|
||||
if (!fs.existsSync(rootDir)) {
|
||||
return files;
|
||||
}
|
||||
const queue = [rootDir];
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
|
||||
const fullPath = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name === "node_modules") {
|
||||
continue;
|
||||
}
|
||||
queue.push(fullPath);
|
||||
continue;
|
||||
}
|
||||
if (entry.isFile() && JS_EXTENSIONS.has(path.extname(entry.name))) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
return files.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function extractModuleSpecifiers(source) {
|
||||
const specifiers = new Set();
|
||||
const patterns = [
|
||||
/\bfrom\s*["']([^"']+)["']/g,
|
||||
/\bimport\s*["']([^"']+)["']/g,
|
||||
/\bimport\s*\(\s*["']([^"']+)["']\s*\)/g,
|
||||
/\brequire\s*\(\s*["']([^"']+)["']\s*\)/g,
|
||||
];
|
||||
for (const pattern of patterns) {
|
||||
for (const match of source.matchAll(pattern)) {
|
||||
if (match[1]) {
|
||||
specifiers.add(match[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
return specifiers;
|
||||
}
|
||||
|
||||
export function collectRootDistBundledRuntimeMirrors(params) {
|
||||
const distDir = params.distDir;
|
||||
const bundledSpecs = params.bundledRuntimeDependencySpecs;
|
||||
const mirrors = new Map();
|
||||
|
||||
for (const dependencyName of CURATED_ROOT_RUNTIME_MIRRORS) {
|
||||
const bundledSpec = bundledSpecs.get(dependencyName);
|
||||
if (!bundledSpec) {
|
||||
continue;
|
||||
}
|
||||
mirrors.set(dependencyName, {
|
||||
importers: new Set(["<curated root runtime surface>"]),
|
||||
pluginIds: bundledSpec.pluginIds,
|
||||
spec: bundledSpec.spec,
|
||||
});
|
||||
}
|
||||
|
||||
for (const filePath of walkJavaScriptFiles(distDir)) {
|
||||
const source = fs.readFileSync(filePath, "utf8");
|
||||
const relativePath = path.relative(distDir, filePath).replaceAll(path.sep, "/");
|
||||
for (const specifier of extractModuleSpecifiers(source)) {
|
||||
const dependencyName = packageNameFromSpecifier(specifier);
|
||||
if (!dependencyName || !bundledSpecs.has(dependencyName)) {
|
||||
continue;
|
||||
}
|
||||
const bundledSpec = bundledSpecs.get(dependencyName);
|
||||
const existing = mirrors.get(dependencyName);
|
||||
if (existing) {
|
||||
existing.importers.add(relativePath);
|
||||
continue;
|
||||
}
|
||||
mirrors.set(dependencyName, {
|
||||
importers: new Set([relativePath]),
|
||||
pluginIds: bundledSpec.pluginIds,
|
||||
spec: bundledSpec.spec,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return mirrors;
|
||||
}
|
||||
|
||||
export function collectBundledPluginRootRuntimeMirrorErrors(params) {
|
||||
const errors = [];
|
||||
|
||||
for (const [dependencyName, record] of params.bundledRuntimeDependencySpecs) {
|
||||
for (const conflict of record.conflicts) {
|
||||
errors.push(
|
||||
`bundled runtime dependency '${dependencyName}' has conflicting plugin specs: ${record.pluginIds.join(", ")} use '${record.spec}', ${conflict.pluginId} uses '${conflict.spec}'.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
51
openclaw/scripts/lib/bundled-plugin-source-utils.mjs
Normal file
51
openclaw/scripts/lib/bundled-plugin-source-utils.mjs
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
export function readIfExists(filePath) {
|
||||
try {
|
||||
return fs.readFileSync(filePath, "utf8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function collectBundledPluginSources(params = {}) {
|
||||
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
|
||||
const extensionsRoot = path.join(repoRoot, "extensions");
|
||||
if (!fs.existsSync(extensionsRoot)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const requirePackageJson = params.requirePackageJson === true;
|
||||
const entries = [];
|
||||
for (const dirent of fs.readdirSync(extensionsRoot, { withFileTypes: true })) {
|
||||
if (!dirent.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const pluginDir = path.join(extensionsRoot, dirent.name);
|
||||
const manifestPath = path.join(pluginDir, "openclaw.plugin.json");
|
||||
const packageJsonPath = path.join(pluginDir, "package.json");
|
||||
if (!fs.existsSync(manifestPath)) {
|
||||
continue;
|
||||
}
|
||||
if (requirePackageJson && !fs.existsSync(packageJsonPath)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.push({
|
||||
dirName: dirent.name,
|
||||
pluginDir,
|
||||
manifestPath,
|
||||
manifest: JSON.parse(fs.readFileSync(manifestPath, "utf8")),
|
||||
...(fs.existsSync(packageJsonPath)
|
||||
? {
|
||||
packageJsonPath,
|
||||
packageJson: JSON.parse(fs.readFileSync(packageJsonPath, "utf8")),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
return entries.toSorted((left, right) => left.dirName.localeCompare(right.dirName));
|
||||
}
|
||||
39
openclaw/scripts/lib/bundled-runtime-sidecar-paths.json
Normal file
39
openclaw/scripts/lib/bundled-runtime-sidecar-paths.json
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
[
|
||||
"dist/extensions/acpx/runtime-api.js",
|
||||
"dist/extensions/bluebubbles/runtime-api.js",
|
||||
"dist/extensions/browser/runtime-api.js",
|
||||
"dist/extensions/copilot-proxy/runtime-api.js",
|
||||
"dist/extensions/diffs/runtime-api.js",
|
||||
"dist/extensions/discord/runtime-api.js",
|
||||
"dist/extensions/feishu/runtime-api.js",
|
||||
"dist/extensions/google/runtime-api.js",
|
||||
"dist/extensions/googlechat/runtime-api.js",
|
||||
"dist/extensions/imessage/runtime-api.js",
|
||||
"dist/extensions/irc/runtime-api.js",
|
||||
"dist/extensions/line/runtime-api.js",
|
||||
"dist/extensions/lmstudio/runtime-api.js",
|
||||
"dist/extensions/lobster/runtime-api.js",
|
||||
"dist/extensions/matrix/helper-api.js",
|
||||
"dist/extensions/matrix/runtime-api.js",
|
||||
"dist/extensions/matrix/thread-bindings-runtime.js",
|
||||
"dist/extensions/mattermost/runtime-api.js",
|
||||
"dist/extensions/memory-core/runtime-api.js",
|
||||
"dist/extensions/msteams/runtime-api.js",
|
||||
"dist/extensions/nextcloud-talk/runtime-api.js",
|
||||
"dist/extensions/nostr/runtime-api.js",
|
||||
"dist/extensions/ollama/runtime-api.js",
|
||||
"dist/extensions/open-prose/runtime-api.js",
|
||||
"dist/extensions/qqbot/runtime-api.js",
|
||||
"dist/extensions/signal/runtime-api.js",
|
||||
"dist/extensions/slack/runtime-api.js",
|
||||
"dist/extensions/telegram/runtime-api.js",
|
||||
"dist/extensions/tlon/runtime-api.js",
|
||||
"dist/extensions/twitch/runtime-api.js",
|
||||
"dist/extensions/voice-call/runtime-api.js",
|
||||
"dist/extensions/webhooks/runtime-api.js",
|
||||
"dist/extensions/whatsapp/light-runtime-api.js",
|
||||
"dist/extensions/whatsapp/runtime-api.js",
|
||||
"dist/extensions/zai/runtime-api.js",
|
||||
"dist/extensions/zalo/runtime-api.js",
|
||||
"dist/extensions/zalouser/runtime-api.js"
|
||||
]
|
||||
45
openclaw/scripts/lib/callsite-guard.mjs
Normal file
45
openclaw/scripts/lib/callsite-guard.mjs
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import {
|
||||
collectTypeScriptFilesFromRoots,
|
||||
resolveRepoRoot,
|
||||
resolveSourceRoots,
|
||||
} from "./ts-guard-utils.mjs";
|
||||
|
||||
export async function runCallsiteGuard(params) {
|
||||
const repoRoot = resolveRepoRoot(params.importMetaUrl);
|
||||
const sourceRoots = resolveSourceRoots(repoRoot, params.sourceRoots);
|
||||
const files = await collectTypeScriptFilesFromRoots(sourceRoots, {
|
||||
extraTestSuffixes: params.extraTestSuffixes,
|
||||
});
|
||||
const violations = [];
|
||||
|
||||
for (const filePath of files) {
|
||||
const relPath = path.relative(repoRoot, filePath).replaceAll(path.sep, "/");
|
||||
if (params.skipRelativePath?.(relPath)) {
|
||||
continue;
|
||||
}
|
||||
const content = await fs.readFile(filePath, "utf8");
|
||||
for (const line of params.findCallLines(content, filePath)) {
|
||||
const callsite = `${relPath}:${line}`;
|
||||
if (params.allowCallsite?.(callsite)) {
|
||||
continue;
|
||||
}
|
||||
violations.push(callsite);
|
||||
}
|
||||
}
|
||||
|
||||
if (violations.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(params.header);
|
||||
const output = params.sortViolations === false ? violations : violations.toSorted();
|
||||
for (const violation of output) {
|
||||
console.error(`- ${violation}`);
|
||||
}
|
||||
if (params.footer) {
|
||||
console.error(params.footer);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
136
openclaw/scripts/lib/changed-extensions.mjs
Normal file
136
openclaw/scripts/lib/changed-extensions.mjs
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { BUNDLED_PLUGIN_PATH_PREFIX, BUNDLED_PLUGIN_ROOT_DIR } from "./bundled-plugin-paths.mjs";
|
||||
|
||||
const repoRoot = path.resolve(import.meta.dirname, "..", "..");
|
||||
|
||||
function runGit(args, options = {}) {
|
||||
return execFileSync("git", args, {
|
||||
cwd: repoRoot,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
encoding: "utf8",
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeRelative(inputPath) {
|
||||
return inputPath.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function hasGitCommit(ref) {
|
||||
if (!ref || /^0+$/.test(ref)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
runGit(["rev-parse", "--verify", `${ref}^{commit}`]);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveChangedPathsBase(params = {}) {
|
||||
const base = params.base;
|
||||
const head = params.head ?? "HEAD";
|
||||
const fallbackBaseRef = params.fallbackBaseRef;
|
||||
|
||||
if (hasGitCommit(base)) {
|
||||
return base;
|
||||
}
|
||||
|
||||
if (fallbackBaseRef) {
|
||||
const remoteBaseRef = fallbackBaseRef.startsWith("origin/")
|
||||
? fallbackBaseRef
|
||||
: `origin/${fallbackBaseRef}`;
|
||||
if (hasGitCommit(remoteBaseRef)) {
|
||||
const mergeBase = runGit(["merge-base", remoteBaseRef, head]).trim();
|
||||
if (hasGitCommit(mergeBase)) {
|
||||
return mergeBase;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!base) {
|
||||
throw new Error("A git base revision is required to list changed extensions.");
|
||||
}
|
||||
|
||||
throw new Error(`Git base revision is unavailable locally: ${base}`);
|
||||
}
|
||||
|
||||
function listChangedPaths(base, head = "HEAD") {
|
||||
if (!base) {
|
||||
throw new Error("A git base revision is required to list changed extensions.");
|
||||
}
|
||||
|
||||
return runGit(["diff", "--name-only", base, head])
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0);
|
||||
}
|
||||
|
||||
function hasExtensionPackage(extensionId) {
|
||||
return fs.existsSync(path.join(repoRoot, BUNDLED_PLUGIN_ROOT_DIR, extensionId, "package.json"));
|
||||
}
|
||||
|
||||
export function listAvailableExtensionIds() {
|
||||
const extensionsDir = path.join(repoRoot, BUNDLED_PLUGIN_ROOT_DIR);
|
||||
if (!fs.existsSync(extensionsDir)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return fs
|
||||
.readdirSync(extensionsDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.filter((extensionId) => hasExtensionPackage(extensionId))
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export function detectChangedExtensionIds(changedPaths) {
|
||||
const extensionIds = new Set();
|
||||
|
||||
for (const rawPath of changedPaths) {
|
||||
const relativePath = normalizeRelative(String(rawPath).trim());
|
||||
if (!relativePath) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const extensionMatch = relativePath.match(
|
||||
new RegExp(`^${BUNDLED_PLUGIN_PATH_PREFIX.replace("/", "\\/")}([^/]+)(?:/|$)`),
|
||||
);
|
||||
if (extensionMatch) {
|
||||
const extensionId = extensionMatch[1];
|
||||
if (hasExtensionPackage(extensionId)) {
|
||||
extensionIds.add(extensionId);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const pairedCoreMatch = relativePath.match(/^src\/([^/]+)(?:\/|$)/);
|
||||
if (pairedCoreMatch && hasExtensionPackage(pairedCoreMatch[1])) {
|
||||
extensionIds.add(pairedCoreMatch[1]);
|
||||
}
|
||||
}
|
||||
|
||||
return [...extensionIds].toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export function listChangedExtensionIds(params = {}) {
|
||||
const head = params.head ?? "HEAD";
|
||||
const unavailableBaseBehavior = params.unavailableBaseBehavior ?? "error";
|
||||
|
||||
try {
|
||||
const base = resolveChangedPathsBase(params);
|
||||
return detectChangedExtensionIds(listChangedPaths(base, head));
|
||||
} catch (error) {
|
||||
if (unavailableBaseBehavior === "all") {
|
||||
return listAvailableExtensionIds();
|
||||
}
|
||||
if (unavailableBaseBehavior === "empty") {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
95
openclaw/scripts/lib/ci-node-test-plan.mjs
Normal file
95
openclaw/scripts/lib/ci-node-test-plan.mjs
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import { fullSuiteVitestShards } from "../../test/vitest/vitest.test-shards.mjs";
|
||||
|
||||
const EXCLUDED_FULL_SUITE_SHARDS = new Set([
|
||||
"test/vitest/vitest.full-core-contracts.config.ts",
|
||||
"test/vitest/vitest.full-core-bundled.config.ts",
|
||||
"test/vitest/vitest.full-extensions.config.ts",
|
||||
]);
|
||||
|
||||
const EXCLUDED_PROJECT_CONFIGS = new Set(["test/vitest/vitest.channels.config.ts"]);
|
||||
const SPLIT_NODE_SHARDS = new Map([
|
||||
[
|
||||
"agentic",
|
||||
[
|
||||
{
|
||||
shardName: "agentic-control-plane",
|
||||
configs: [
|
||||
"test/vitest/vitest.gateway-core.config.ts",
|
||||
"test/vitest/vitest.gateway-client.config.ts",
|
||||
"test/vitest/vitest.gateway-methods.config.ts",
|
||||
"test/vitest/vitest.gateway-server.config.ts",
|
||||
"test/vitest/vitest.daemon.config.ts",
|
||||
],
|
||||
requiresDist: false,
|
||||
},
|
||||
{
|
||||
shardName: "agentic-commands",
|
||||
configs: [
|
||||
"test/vitest/vitest.cli.config.ts",
|
||||
"test/vitest/vitest.commands-light.config.ts",
|
||||
"test/vitest/vitest.commands.config.ts",
|
||||
],
|
||||
requiresDist: false,
|
||||
},
|
||||
{
|
||||
shardName: "agentic-agents-plugins",
|
||||
configs: [
|
||||
"test/vitest/vitest.agents.config.ts",
|
||||
"test/vitest/vitest.plugin-sdk-light.config.ts",
|
||||
"test/vitest/vitest.plugin-sdk.config.ts",
|
||||
"test/vitest/vitest.plugins.config.ts",
|
||||
],
|
||||
requiresDist: true,
|
||||
},
|
||||
],
|
||||
],
|
||||
]);
|
||||
const DIST_DEPENDENT_NODE_SHARD_NAMES = new Set(["core-support-boundary", "core-runtime"]);
|
||||
|
||||
function formatNodeTestShardCheckName(shardName) {
|
||||
const normalizedShardName = shardName.startsWith("core-unit-")
|
||||
? `core-${shardName.slice("core-unit-".length)}`
|
||||
: shardName;
|
||||
return `checks-node-${normalizedShardName}`;
|
||||
}
|
||||
|
||||
export function createNodeTestShards() {
|
||||
return fullSuiteVitestShards.flatMap((shard) => {
|
||||
if (EXCLUDED_FULL_SUITE_SHARDS.has(shard.config)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const configs = shard.projects.filter((config) => !EXCLUDED_PROJECT_CONFIGS.has(config));
|
||||
if (configs.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const splitShards = SPLIT_NODE_SHARDS.get(shard.name);
|
||||
if (splitShards) {
|
||||
return splitShards.flatMap((splitShard) => {
|
||||
const splitConfigs = splitShard.configs.filter((config) => configs.includes(config));
|
||||
if (splitConfigs.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
checkName: formatNodeTestShardCheckName(splitShard.shardName),
|
||||
shardName: splitShard.shardName,
|
||||
configs: splitConfigs,
|
||||
requiresDist: splitShard.requiresDist,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
checkName: formatNodeTestShardCheckName(shard.name),
|
||||
shardName: shard.name,
|
||||
configs,
|
||||
requiresDist: DIST_DEPENDENT_NODE_SHARD_NAMES.has(shard.name),
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
30
openclaw/scripts/lib/copy-assets.ts
Normal file
30
openclaw/scripts/lib/copy-assets.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export type BuildCopyContext = {
|
||||
prefix: string;
|
||||
projectRoot: string;
|
||||
verbose: boolean;
|
||||
};
|
||||
|
||||
export function resolveBuildCopyContext(importMetaUrl: string): BuildCopyContext {
|
||||
const filePath = fileURLToPath(importMetaUrl);
|
||||
return {
|
||||
prefix: `[${path.basename(filePath, path.extname(filePath))}]`,
|
||||
projectRoot: path.resolve(path.dirname(filePath), ".."),
|
||||
verbose: process.env.OPENCLAW_BUILD_VERBOSE === "1",
|
||||
};
|
||||
}
|
||||
|
||||
export function ensureDirectory(dirPath: string): void {
|
||||
if (!fs.existsSync(dirPath)) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
export function logVerboseCopy(context: BuildCopyContext, message: string): void {
|
||||
if (context.verbose) {
|
||||
console.log(`${context.prefix} ${message}`);
|
||||
}
|
||||
}
|
||||
14
openclaw/scripts/lib/docker-e2e-logs.sh
Normal file
14
openclaw/scripts/lib/docker-e2e-logs.sh
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
run_logged() {
|
||||
local label="$1"
|
||||
shift
|
||||
local log_file
|
||||
log_file="$(mktemp "${TMPDIR:-/tmp}/openclaw-${label}.XXXXXX.log")"
|
||||
if ! "$@" >"$log_file" 2>&1; then
|
||||
cat "$log_file"
|
||||
rm -f "$log_file"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$log_file"
|
||||
}
|
||||
6
openclaw/scripts/lib/error-format.mjs
Normal file
6
openclaw/scripts/lib/error-format.mjs
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
export function formatErrorMessage(error) {
|
||||
if (error instanceof Error) {
|
||||
return error.message || error.name || "Error";
|
||||
}
|
||||
return String(error);
|
||||
}
|
||||
104
openclaw/scripts/lib/extension-import-boundary-checker.mjs
Normal file
104
openclaw/scripts/lib/extension-import-boundary-checker.mjs
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import ts from "typescript";
|
||||
import { BUNDLED_PLUGIN_PATH_PREFIX } from "./bundled-plugin-paths.mjs";
|
||||
import {
|
||||
collectTypeScriptInventory,
|
||||
createCachedAsync,
|
||||
formatGroupedInventoryHuman,
|
||||
normalizeRepoPath,
|
||||
resolveRepoSpecifier,
|
||||
visitModuleSpecifiers,
|
||||
writeLine,
|
||||
} from "./guard-inventory-utils.mjs";
|
||||
import {
|
||||
collectTypeScriptFilesFromRoots,
|
||||
resolveRepoRoot,
|
||||
resolveSourceRoots,
|
||||
toLine,
|
||||
} from "./ts-guard-utils.mjs";
|
||||
|
||||
const repoRoot = resolveRepoRoot(import.meta.url);
|
||||
|
||||
function compareEntries(left, right) {
|
||||
return (
|
||||
left.file.localeCompare(right.file) ||
|
||||
left.line - right.line ||
|
||||
left.kind.localeCompare(right.kind) ||
|
||||
left.specifier.localeCompare(right.specifier) ||
|
||||
left.reason.localeCompare(right.reason)
|
||||
);
|
||||
}
|
||||
|
||||
function classifyResolvedExtensionReason(kind, boundaryLabel) {
|
||||
const verb =
|
||||
kind === "export"
|
||||
? "re-exports"
|
||||
: kind === "dynamic-import"
|
||||
? "dynamically imports"
|
||||
: "imports";
|
||||
return `${verb} bundled plugin file from ${boundaryLabel} boundary`;
|
||||
}
|
||||
|
||||
function scanImportBoundaryViolations(sourceFile, filePath, boundaryLabel) {
|
||||
const entries = [];
|
||||
const relativeFile = normalizeRepoPath(repoRoot, filePath);
|
||||
|
||||
visitModuleSpecifiers(ts, sourceFile, ({ kind, specifier, specifierNode }) => {
|
||||
const resolvedPath = resolveRepoSpecifier(repoRoot, specifier, filePath);
|
||||
if (!resolvedPath?.startsWith(BUNDLED_PLUGIN_PATH_PREFIX)) {
|
||||
return;
|
||||
}
|
||||
entries.push({
|
||||
file: relativeFile,
|
||||
line: toLine(sourceFile, specifierNode),
|
||||
kind,
|
||||
specifier,
|
||||
resolvedPath,
|
||||
reason: classifyResolvedExtensionReason(kind, boundaryLabel),
|
||||
});
|
||||
});
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
export function createExtensionImportBoundaryChecker(params) {
|
||||
const scanRoots = resolveSourceRoots(repoRoot, params.roots);
|
||||
|
||||
const collectInventory = createCachedAsync(async () => {
|
||||
const files = (await collectTypeScriptFilesFromRoots(scanRoots))
|
||||
.filter((filePath) => !params.shouldSkipFile?.(normalizeRepoPath(repoRoot, filePath)))
|
||||
.toSorted((left, right) =>
|
||||
normalizeRepoPath(repoRoot, left).localeCompare(normalizeRepoPath(repoRoot, right)),
|
||||
);
|
||||
return await collectTypeScriptInventory({
|
||||
ts,
|
||||
files,
|
||||
compareEntries,
|
||||
collectEntries(sourceFile, filePath) {
|
||||
return scanImportBoundaryViolations(sourceFile, filePath, params.boundaryLabel);
|
||||
},
|
||||
shouldParseSource: params.skipSourcesWithoutBundledPluginPrefix
|
||||
? (source) => source.includes(BUNDLED_PLUGIN_PATH_PREFIX)
|
||||
: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
async function main(argv = process.argv.slice(2), io) {
|
||||
const streams = io ?? { stdout: process.stdout, stderr: process.stderr };
|
||||
const json = argv.includes("--json");
|
||||
const inventory = await collectInventory();
|
||||
|
||||
if (json) {
|
||||
writeLine(streams.stdout, JSON.stringify(inventory, null, 2));
|
||||
} else {
|
||||
writeLine(streams.stdout, formatGroupedInventoryHuman(params, inventory));
|
||||
writeLine(
|
||||
streams.stdout,
|
||||
inventory.length === 0 ? "Boundary is clean." : "Boundary has violations.",
|
||||
);
|
||||
}
|
||||
|
||||
return inventory.length === 0 ? 0 : 1;
|
||||
}
|
||||
|
||||
return { collectInventory, main };
|
||||
}
|
||||
218
openclaw/scripts/lib/extension-package-boundary.ts
Normal file
218
openclaw/scripts/lib/extension-package-boundary.ts
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
||||
import { join, posix, resolve } from "node:path";
|
||||
|
||||
export const EXTENSION_PACKAGE_BOUNDARY_BASE_CONFIG =
|
||||
"extensions/tsconfig.package-boundary.base.json" as const;
|
||||
|
||||
export const EXTENSION_PACKAGE_BOUNDARY_INCLUDE = ["./*.ts", "./src/**/*.ts"] as const;
|
||||
export const EXTENSION_PACKAGE_BOUNDARY_EXCLUDE = [
|
||||
"./**/*.test.ts",
|
||||
"./dist/**",
|
||||
"./node_modules/**",
|
||||
"./src/test-support/**",
|
||||
"./src/**/*test-helpers.ts",
|
||||
"./src/**/*test-harness.ts",
|
||||
"./src/**/*test-support.ts",
|
||||
] as const;
|
||||
export const EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS = {
|
||||
"openclaw/extension-api": ["../src/extensionAPI.ts"],
|
||||
"openclaw/plugin-sdk": ["../dist/plugin-sdk/src/plugin-sdk/index.d.ts"],
|
||||
"openclaw/plugin-sdk/*": ["../dist/plugin-sdk/src/plugin-sdk/*.d.ts"],
|
||||
"openclaw/plugin-sdk/account-id": ["../dist/plugin-sdk/src/plugin-sdk/account-id.d.ts"],
|
||||
"openclaw/plugin-sdk/channel-entry-contract": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/channel-entry-contract.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/browser-maintenance": [
|
||||
"../packages/plugin-sdk/dist/extensions/browser/browser-maintenance.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/browser-config-runtime": [
|
||||
"../dist/plugin-sdk/src/plugin-sdk/browser-config-runtime.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/browser-node-runtime": [
|
||||
"../dist/plugin-sdk/src/plugin-sdk/browser-node-runtime.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/browser-setup-tools": [
|
||||
"../dist/plugin-sdk/src/plugin-sdk/browser-setup-tools.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/browser-security-runtime": [
|
||||
"../dist/plugin-sdk/src/plugin-sdk/browser-security-runtime.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/channel-secret-basic-runtime": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/channel-secret-basic-runtime.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/channel-secret-runtime": [
|
||||
"../dist/plugin-sdk/src/plugin-sdk/channel-secret-runtime.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/channel-secret-tts-runtime": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/channel-secret-tts-runtime.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/channel-streaming": [
|
||||
"../dist/plugin-sdk/src/plugin-sdk/channel-streaming.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/error-runtime": ["../dist/plugin-sdk/src/plugin-sdk/error-runtime.d.ts"],
|
||||
"openclaw/plugin-sdk/provider-catalog-shared": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/provider-catalog-shared.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/provider-entry": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/provider-entry.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/secret-ref-runtime": [
|
||||
"../dist/plugin-sdk/src/plugin-sdk/secret-ref-runtime.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/ssrf-runtime": ["../dist/plugin-sdk/src/plugin-sdk/ssrf-runtime.d.ts"],
|
||||
"@openclaw/qa-channel/api.js": ["../dist/plugin-sdk/extensions/qa-channel/api.d.ts"],
|
||||
"@openclaw/*.js": ["../packages/plugin-sdk/dist/extensions/*.d.ts", "../extensions/*"],
|
||||
"@openclaw/*": ["../packages/plugin-sdk/dist/extensions/*", "../extensions/*"],
|
||||
"@openclaw/plugin-sdk/*": ["../dist/plugin-sdk/src/plugin-sdk/*.d.ts"],
|
||||
} as const;
|
||||
|
||||
function prefixExtensionPackageBoundaryPaths(
|
||||
paths: Record<string, readonly string[]>,
|
||||
prefix: string,
|
||||
): Record<string, readonly string[]> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(paths).map(([key, values]) => [
|
||||
key,
|
||||
values.map((value) => posix.join(prefix, value)),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
export const EXTENSION_PACKAGE_BOUNDARY_XAI_PATHS = {
|
||||
...prefixExtensionPackageBoundaryPaths(
|
||||
(({
|
||||
"openclaw/plugin-sdk/channel-secret-basic-runtime": _omitBasic,
|
||||
"openclaw/plugin-sdk/channel-secret-tts-runtime": _omitTts,
|
||||
...rest
|
||||
}) => rest)(EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS),
|
||||
"../",
|
||||
),
|
||||
"openclaw/plugin-sdk/channel-entry-contract": [
|
||||
"../../dist/plugin-sdk/src/plugin-sdk/channel-entry-contract.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/browser-maintenance": [
|
||||
"../../dist/plugin-sdk/src/plugin-sdk/browser-maintenance.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/cli-runtime": ["../../dist/plugin-sdk/src/plugin-sdk/cli-runtime.d.ts"],
|
||||
"openclaw/plugin-sdk/provider-catalog-shared": [
|
||||
"../../dist/plugin-sdk/src/plugin-sdk/provider-catalog-shared.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/provider-env-vars": [
|
||||
"../../dist/plugin-sdk/src/plugin-sdk/provider-env-vars.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/provider-entry": [
|
||||
"../../dist/plugin-sdk/src/plugin-sdk/provider-entry.d.ts",
|
||||
],
|
||||
"openclaw/plugin-sdk/provider-web-search-contract": [
|
||||
"../../dist/plugin-sdk/src/plugin-sdk/provider-web-search-contract.d.ts",
|
||||
],
|
||||
"@openclaw/qa-channel/api.js": ["../../dist/plugin-sdk/extensions/qa-channel/api.d.ts"],
|
||||
"@openclaw/*.js": ["../../packages/plugin-sdk/dist/extensions/*.d.ts", "../*"],
|
||||
"@openclaw/*": ["../*"],
|
||||
"@openclaw/plugin-sdk/*": ["../../dist/plugin-sdk/src/plugin-sdk/*.d.ts"],
|
||||
"@openclaw/anthropic-vertex/api.js": ["./.boundary-stubs/anthropic-vertex-api.d.ts"],
|
||||
"@openclaw/ollama/api.js": ["./.boundary-stubs/ollama-api.d.ts"],
|
||||
"@openclaw/ollama/runtime-api.js": ["./.boundary-stubs/ollama-runtime-api.d.ts"],
|
||||
"@openclaw/speech-core/runtime-api.js": ["./.boundary-stubs/speech-core-runtime-api.d.ts"],
|
||||
} as const;
|
||||
|
||||
export type ExtensionPackageBoundaryTsConfigJson = {
|
||||
extends?: unknown;
|
||||
compilerOptions?: {
|
||||
rootDir?: unknown;
|
||||
paths?: unknown;
|
||||
};
|
||||
include?: unknown;
|
||||
exclude?: unknown;
|
||||
};
|
||||
|
||||
export type ExtensionPackageBoundaryPackageJson = {
|
||||
devDependencies?: Record<string, string>;
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- Boundary helper lets callers ascribe JSON file shape.
|
||||
function readJsonFile<T>(filePath: string): T {
|
||||
return JSON.parse(readFileSync(filePath, "utf8")) as T;
|
||||
}
|
||||
|
||||
export function collectBundledExtensionIds(rootDir = resolve(".")): string[] {
|
||||
return readdirSync(join(rootDir, "extensions"), { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.toSorted();
|
||||
}
|
||||
|
||||
export function resolveExtensionTsconfigPath(extensionId: string, rootDir = resolve(".")): string {
|
||||
return join(rootDir, "extensions", extensionId, "tsconfig.json");
|
||||
}
|
||||
|
||||
export function resolveExtensionPackageJsonPath(
|
||||
extensionId: string,
|
||||
rootDir = resolve("."),
|
||||
): string {
|
||||
return join(rootDir, "extensions", extensionId, "package.json");
|
||||
}
|
||||
|
||||
export function readExtensionPackageBoundaryTsconfig(
|
||||
extensionId: string,
|
||||
rootDir = resolve("."),
|
||||
): ExtensionPackageBoundaryTsConfigJson {
|
||||
return readJsonFile<ExtensionPackageBoundaryTsConfigJson>(
|
||||
resolveExtensionTsconfigPath(extensionId, rootDir),
|
||||
);
|
||||
}
|
||||
|
||||
export function readExtensionPackageBoundaryPackageJson(
|
||||
extensionId: string,
|
||||
rootDir = resolve("."),
|
||||
): ExtensionPackageBoundaryPackageJson {
|
||||
return readJsonFile<ExtensionPackageBoundaryPackageJson>(
|
||||
resolveExtensionPackageJsonPath(extensionId, rootDir),
|
||||
);
|
||||
}
|
||||
|
||||
export function isOptInExtensionPackageBoundaryTsconfig(
|
||||
tsconfig: ExtensionPackageBoundaryTsConfigJson,
|
||||
): boolean {
|
||||
return tsconfig.extends === "../tsconfig.package-boundary.base.json";
|
||||
}
|
||||
|
||||
export function collectExtensionsWithTsconfig(rootDir = resolve(".")): string[] {
|
||||
return collectBundledExtensionIds(rootDir).filter((extensionId) =>
|
||||
existsSync(resolveExtensionTsconfigPath(extensionId, rootDir)),
|
||||
);
|
||||
}
|
||||
|
||||
export function collectOptInExtensionPackageBoundaries(rootDir = resolve(".")): string[] {
|
||||
return collectExtensionsWithTsconfig(rootDir).filter((extensionId) =>
|
||||
isOptInExtensionPackageBoundaryTsconfig(
|
||||
readExtensionPackageBoundaryTsconfig(extensionId, rootDir),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function renderExtensionPackageBoundaryTsconfig(params?: {
|
||||
paths?: Record<string, readonly string[]>;
|
||||
}): {
|
||||
extends: "../tsconfig.package-boundary.base.json";
|
||||
compilerOptions: { rootDir: "."; paths?: Record<string, readonly string[]> };
|
||||
include: typeof EXTENSION_PACKAGE_BOUNDARY_INCLUDE;
|
||||
exclude: typeof EXTENSION_PACKAGE_BOUNDARY_EXCLUDE;
|
||||
} {
|
||||
return {
|
||||
extends: "../tsconfig.package-boundary.base.json",
|
||||
compilerOptions: {
|
||||
rootDir: ".",
|
||||
...(params?.paths
|
||||
? {
|
||||
paths: {
|
||||
...EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS,
|
||||
...params.paths,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
include: EXTENSION_PACKAGE_BOUNDARY_INCLUDE,
|
||||
exclude: EXTENSION_PACKAGE_BOUNDARY_EXCLUDE,
|
||||
};
|
||||
}
|
||||
14
openclaw/scripts/lib/extension-source-classifier.d.mts
Normal file
14
openclaw/scripts/lib/extension-source-classifier.d.mts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
export type BundledExtensionSourceClassification = {
|
||||
normalizedPath: string;
|
||||
isCodeFile: boolean;
|
||||
isRuntimeApiBarrel: boolean;
|
||||
isPublicApiBarrel: boolean;
|
||||
isTestLike: boolean;
|
||||
isInfraArtifact: boolean;
|
||||
isProductionSource: boolean;
|
||||
};
|
||||
|
||||
export function normalizeExtensionSourcePath(filePath: string): string;
|
||||
export function classifyBundledExtensionSourcePath(
|
||||
filePath: string,
|
||||
): BundledExtensionSourceClassification;
|
||||
44
openclaw/scripts/lib/extension-source-classifier.mjs
Normal file
44
openclaw/scripts/lib/extension-source-classifier.mjs
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
const CODE_FILE_RE = /\.(?:[cm]?ts|[cm]?js|tsx|jsx)$/u;
|
||||
const DECLARATION_FILE_RE = /\.d\.ts$/u;
|
||||
const RUNTIME_API_BARREL_RE = /(^|\/)runtime-api\.(?:[cm]?ts|[cm]?js|tsx|jsx)$/u;
|
||||
const PUBLIC_API_BARREL_RE = /(^|\/)api\.(?:[cm]?ts|[cm]?js|tsx|jsx)$/u;
|
||||
const TEST_LIKE_SEGMENT_RE =
|
||||
/(^|\/)(?:__tests__|fixtures|test|tests|test-fixtures|test-support|test-utils)(?:\/|$)/u;
|
||||
const TEST_LIKE_FILENAME_RE =
|
||||
/(^|\/)[^/]*test-(?:support|helpers|fixtures|harness)\.(?:[cm]?ts|[cm]?js|tsx|jsx)$/u;
|
||||
const TEST_SHARED_FILENAME_RE = /(^|\/)[^/]*\.test-[^/]*\.(?:[cm]?ts|[cm]?js|tsx|jsx)$/u;
|
||||
const TEST_CANARY_FILENAME_RE = /(^|\/)__rootdir_boundary_canary__\.(?:[cm]?ts|[cm]?js|tsx|jsx)$/u;
|
||||
const SNAPSHOT_FILE_RE = /\.snap$/u;
|
||||
const SUFFIX_SKIP_RE = /\.(?:test|spec|fixture)\./u;
|
||||
const INFRA_DIR_RE = /(^|\/)(?:coverage|dist|node_modules)(?:\/|$)/u;
|
||||
const INFRA_NAME_RE = /(test-harness|test-support|test-helpers|test-fixtures)/u;
|
||||
|
||||
export function normalizeExtensionSourcePath(filePath) {
|
||||
return filePath.replaceAll("\\", "/");
|
||||
}
|
||||
|
||||
export function classifyBundledExtensionSourcePath(filePath) {
|
||||
const normalizedPath = normalizeExtensionSourcePath(filePath);
|
||||
const isCodeFile = CODE_FILE_RE.test(normalizedPath) && !DECLARATION_FILE_RE.test(normalizedPath);
|
||||
const isRuntimeApiBarrel = RUNTIME_API_BARREL_RE.test(normalizedPath);
|
||||
const isPublicApiBarrel = PUBLIC_API_BARREL_RE.test(normalizedPath);
|
||||
const isTestLike =
|
||||
TEST_LIKE_SEGMENT_RE.test(normalizedPath) ||
|
||||
TEST_LIKE_FILENAME_RE.test(normalizedPath) ||
|
||||
TEST_SHARED_FILENAME_RE.test(normalizedPath) ||
|
||||
TEST_CANARY_FILENAME_RE.test(normalizedPath) ||
|
||||
SUFFIX_SKIP_RE.test(normalizedPath) ||
|
||||
SNAPSHOT_FILE_RE.test(normalizedPath) ||
|
||||
INFRA_NAME_RE.test(normalizedPath);
|
||||
const isInfraArtifact = INFRA_DIR_RE.test(normalizedPath);
|
||||
|
||||
return {
|
||||
normalizedPath,
|
||||
isCodeFile,
|
||||
isRuntimeApiBarrel,
|
||||
isPublicApiBarrel,
|
||||
isTestLike,
|
||||
isInfraArtifact,
|
||||
isProductionSource: isCodeFile && !isRuntimeApiBarrel && !isTestLike && !isInfraArtifact,
|
||||
};
|
||||
}
|
||||
289
openclaw/scripts/lib/extension-test-plan.mjs
Normal file
289
openclaw/scripts/lib/extension-test-plan.mjs
Normal file
|
|
@ -0,0 +1,289 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { channelTestRoots } from "../../test/vitest/vitest.channel-paths.mjs";
|
||||
import { isAcpxExtensionRoot } from "../../test/vitest/vitest.extension-acpx-paths.mjs";
|
||||
import { isBlueBubblesExtensionRoot } from "../../test/vitest/vitest.extension-bluebubbles-paths.mjs";
|
||||
import { isDiffsExtensionRoot } from "../../test/vitest/vitest.extension-diffs-paths.mjs";
|
||||
import { isFeishuExtensionRoot } from "../../test/vitest/vitest.extension-feishu-paths.mjs";
|
||||
import { isIrcExtensionRoot } from "../../test/vitest/vitest.extension-irc-paths.mjs";
|
||||
import { isMatrixExtensionRoot } from "../../test/vitest/vitest.extension-matrix-paths.mjs";
|
||||
import { isMattermostExtensionRoot } from "../../test/vitest/vitest.extension-mattermost-paths.mjs";
|
||||
import { isMemoryExtensionRoot } from "../../test/vitest/vitest.extension-memory-paths.mjs";
|
||||
import { isMessagingExtensionRoot } from "../../test/vitest/vitest.extension-messaging-paths.mjs";
|
||||
import { isMsTeamsExtensionRoot } from "../../test/vitest/vitest.extension-msteams-paths.mjs";
|
||||
import { isProviderExtensionRoot } from "../../test/vitest/vitest.extension-provider-paths.mjs";
|
||||
import { isTelegramExtensionRoot } from "../../test/vitest/vitest.extension-telegram-paths.mjs";
|
||||
import { isVoiceCallExtensionRoot } from "../../test/vitest/vitest.extension-voice-call-paths.mjs";
|
||||
import { isWhatsAppExtensionRoot } from "../../test/vitest/vitest.extension-whatsapp-paths.mjs";
|
||||
import { isZaloExtensionRoot } from "../../test/vitest/vitest.extension-zalo-paths.mjs";
|
||||
import { BUNDLED_PLUGIN_PATH_PREFIX, BUNDLED_PLUGIN_ROOT_DIR } from "./bundled-plugin-paths.mjs";
|
||||
import { listAvailableExtensionIds } from "./changed-extensions.mjs";
|
||||
|
||||
const repoRoot = path.resolve(import.meta.dirname, "..", "..");
|
||||
export const DEFAULT_EXTENSION_TEST_SHARD_COUNT = 6;
|
||||
const EXTENSION_TEST_COST_MULTIPLIERS = {
|
||||
"test/vitest/vitest.extension-feishu.config.ts": 1.6,
|
||||
"test/vitest/vitest.extension-msteams.config.ts": 1.6,
|
||||
// This shared config is comparatively cheap per file, so raw file count
|
||||
// overstates its real wall-clock cost during CI shard planning.
|
||||
"test/vitest/vitest.extensions.config.ts": 0.45,
|
||||
};
|
||||
|
||||
function normalizeRelative(inputPath) {
|
||||
return inputPath.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function countTestFiles(rootPath) {
|
||||
let total = 0;
|
||||
const stack = [rootPath];
|
||||
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop();
|
||||
if (!current || !fs.existsSync(current)) {
|
||||
continue;
|
||||
}
|
||||
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
|
||||
const fullPath = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name === "node_modules" || entry.name === "dist") {
|
||||
continue;
|
||||
}
|
||||
stack.push(fullPath);
|
||||
continue;
|
||||
}
|
||||
if (entry.isFile() && (fullPath.endsWith(".test.ts") || fullPath.endsWith(".test.tsx"))) {
|
||||
total += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
function estimatePlanCost(config, testFileCount) {
|
||||
const multiplier = EXTENSION_TEST_COST_MULTIPLIERS[config] ?? 1;
|
||||
return Math.max(1, Math.ceil(testFileCount * multiplier));
|
||||
}
|
||||
|
||||
function resolveExtensionDirectory(targetArg, cwd = process.cwd()) {
|
||||
if (targetArg) {
|
||||
const asGiven = path.resolve(cwd, targetArg);
|
||||
if (fs.existsSync(path.join(asGiven, "package.json"))) {
|
||||
return asGiven;
|
||||
}
|
||||
|
||||
const byName = path.join(repoRoot, BUNDLED_PLUGIN_ROOT_DIR, targetArg);
|
||||
if (fs.existsSync(path.join(byName, "package.json"))) {
|
||||
return byName;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Unknown extension target "${targetArg}". Use a plugin name like "slack" or a path inside the bundled plugin workspace tree.`,
|
||||
);
|
||||
}
|
||||
|
||||
let current = cwd;
|
||||
while (true) {
|
||||
if (
|
||||
normalizeRelative(path.relative(repoRoot, current)).startsWith(BUNDLED_PLUGIN_PATH_PREFIX) &&
|
||||
fs.existsSync(path.join(current, "package.json"))
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
const parent = path.dirname(current);
|
||||
if (parent === current) {
|
||||
break;
|
||||
}
|
||||
current = parent;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"No extension target provided, and current working directory is not inside the bundled plugin workspace tree.",
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveExtensionTestPlan(params = {}) {
|
||||
const cwd = params.cwd ?? process.cwd();
|
||||
const targetArg = params.targetArg;
|
||||
const extensionDir = resolveExtensionDirectory(targetArg, cwd);
|
||||
const extensionId = path.basename(extensionDir);
|
||||
const relativeExtensionDir = normalizeRelative(path.relative(repoRoot, extensionDir));
|
||||
|
||||
const roots = [relativeExtensionDir];
|
||||
|
||||
const usesChannelConfig = roots.some((root) => channelTestRoots.includes(root));
|
||||
const usesAcpxConfig = roots.some((root) => isAcpxExtensionRoot(root));
|
||||
const usesDiffsConfig = roots.some((root) => isDiffsExtensionRoot(root));
|
||||
const usesBlueBubblesConfig = roots.some((root) => isBlueBubblesExtensionRoot(root));
|
||||
const usesFeishuConfig = roots.some((root) => isFeishuExtensionRoot(root));
|
||||
const usesIrcConfig = roots.some((root) => isIrcExtensionRoot(root));
|
||||
const usesMattermostConfig = roots.some((root) => isMattermostExtensionRoot(root));
|
||||
const usesTelegramConfig = roots.some((root) => isTelegramExtensionRoot(root));
|
||||
const usesVoiceCallConfig = roots.some((root) => isVoiceCallExtensionRoot(root));
|
||||
const usesWhatsAppConfig = roots.some((root) => isWhatsAppExtensionRoot(root));
|
||||
const usesZaloConfig = roots.some((root) => isZaloExtensionRoot(root));
|
||||
const usesMatrixConfig = roots.some((root) => isMatrixExtensionRoot(root));
|
||||
const usesMemoryConfig = roots.some((root) => isMemoryExtensionRoot(root));
|
||||
const usesMsTeamsConfig = roots.some((root) => isMsTeamsExtensionRoot(root));
|
||||
const usesMessagingConfig = roots.some((root) => isMessagingExtensionRoot(root));
|
||||
const usesProviderConfig = roots.some((root) => isProviderExtensionRoot(root));
|
||||
const config = usesChannelConfig
|
||||
? "test/vitest/vitest.extension-channels.config.ts"
|
||||
: usesAcpxConfig
|
||||
? "test/vitest/vitest.extension-acpx.config.ts"
|
||||
: usesDiffsConfig
|
||||
? "test/vitest/vitest.extension-diffs.config.ts"
|
||||
: usesBlueBubblesConfig
|
||||
? "test/vitest/vitest.extension-bluebubbles.config.ts"
|
||||
: usesFeishuConfig
|
||||
? "test/vitest/vitest.extension-feishu.config.ts"
|
||||
: usesIrcConfig
|
||||
? "test/vitest/vitest.extension-irc.config.ts"
|
||||
: usesMattermostConfig
|
||||
? "test/vitest/vitest.extension-mattermost.config.ts"
|
||||
: usesMatrixConfig
|
||||
? "test/vitest/vitest.extension-matrix.config.ts"
|
||||
: usesTelegramConfig
|
||||
? "test/vitest/vitest.extension-telegram.config.ts"
|
||||
: usesVoiceCallConfig
|
||||
? "test/vitest/vitest.extension-voice-call.config.ts"
|
||||
: usesWhatsAppConfig
|
||||
? "test/vitest/vitest.extension-whatsapp.config.ts"
|
||||
: usesZaloConfig
|
||||
? "test/vitest/vitest.extension-zalo.config.ts"
|
||||
: usesMemoryConfig
|
||||
? "test/vitest/vitest.extension-memory.config.ts"
|
||||
: usesMsTeamsConfig
|
||||
? "test/vitest/vitest.extension-msteams.config.ts"
|
||||
: usesMessagingConfig
|
||||
? "test/vitest/vitest.extension-messaging.config.ts"
|
||||
: usesProviderConfig
|
||||
? "test/vitest/vitest.extension-providers.config.ts"
|
||||
: "test/vitest/vitest.extensions.config.ts";
|
||||
const testFileCount = roots.reduce(
|
||||
(sum, root) => sum + countTestFiles(path.join(repoRoot, root)),
|
||||
0,
|
||||
);
|
||||
const estimatedCost = estimatePlanCost(config, testFileCount);
|
||||
|
||||
return {
|
||||
config,
|
||||
estimatedCost,
|
||||
extensionDir: relativeExtensionDir,
|
||||
extensionId,
|
||||
hasTests: testFileCount > 0,
|
||||
roots,
|
||||
testFileCount,
|
||||
};
|
||||
}
|
||||
|
||||
function mergeTestPlans(plans) {
|
||||
const groupsByConfig = new Map();
|
||||
|
||||
for (const plan of plans) {
|
||||
const current = groupsByConfig.get(plan.config) ?? {
|
||||
config: plan.config,
|
||||
extensionIds: [],
|
||||
roots: [],
|
||||
estimatedCost: 0,
|
||||
testFileCount: 0,
|
||||
};
|
||||
|
||||
current.extensionIds.push(plan.extensionId);
|
||||
current.roots.push(...plan.roots);
|
||||
current.estimatedCost += plan.estimatedCost;
|
||||
current.testFileCount += plan.testFileCount;
|
||||
groupsByConfig.set(plan.config, current);
|
||||
}
|
||||
|
||||
const planGroups = [...groupsByConfig.values()]
|
||||
.map((group) =>
|
||||
Object.assign({}, group, {
|
||||
extensionIds: group.extensionIds.toSorted((left, right) => left.localeCompare(right)),
|
||||
roots: [...new Set(group.roots)],
|
||||
}),
|
||||
)
|
||||
.toSorted((left, right) => left.config.localeCompare(right.config));
|
||||
|
||||
return {
|
||||
extensionCount: plans.length,
|
||||
extensionIds: plans
|
||||
.map((plan) => plan.extensionId)
|
||||
.toSorted((left, right) => left.localeCompare(right)),
|
||||
estimatedCost: plans.reduce((sum, plan) => sum + plan.estimatedCost, 0),
|
||||
hasTests: plans.length > 0,
|
||||
planGroups,
|
||||
testFileCount: plans.reduce((sum, plan) => sum + plan.testFileCount, 0),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveExtensionBatchPlan(params = {}) {
|
||||
const cwd = params.cwd ?? process.cwd();
|
||||
const extensionIds = params.extensionIds ?? listAvailableExtensionIds();
|
||||
const plans = extensionIds
|
||||
.map((extensionId) => resolveExtensionTestPlan({ cwd, targetArg: extensionId }))
|
||||
.filter((plan) => plan.hasTests);
|
||||
|
||||
return mergeTestPlans(plans);
|
||||
}
|
||||
|
||||
function pickLeastLoadedShard(shards) {
|
||||
return shards.reduce((bestIndex, shard, index) => {
|
||||
if (bestIndex === -1) {
|
||||
return index;
|
||||
}
|
||||
const best = shards[bestIndex];
|
||||
if (shard.estimatedCost !== best.estimatedCost) {
|
||||
return shard.estimatedCost < best.estimatedCost ? index : bestIndex;
|
||||
}
|
||||
if (shard.testFileCount !== best.testFileCount) {
|
||||
return shard.testFileCount < best.testFileCount ? index : bestIndex;
|
||||
}
|
||||
if (shard.plans.length !== best.plans.length) {
|
||||
return shard.plans.length < best.plans.length ? index : bestIndex;
|
||||
}
|
||||
return index < bestIndex ? index : bestIndex;
|
||||
}, -1);
|
||||
}
|
||||
|
||||
export function createExtensionTestShards(params = {}) {
|
||||
const cwd = params.cwd ?? process.cwd();
|
||||
const extensionIds = params.extensionIds ?? listAvailableExtensionIds();
|
||||
const shardCount = Math.max(1, Number.parseInt(String(params.shardCount ?? ""), 10) || 1);
|
||||
const plans = extensionIds
|
||||
.map((extensionId) => resolveExtensionTestPlan({ cwd, targetArg: extensionId }))
|
||||
.filter((plan) => plan.hasTests)
|
||||
.toSorted((left, right) => {
|
||||
if (left.estimatedCost !== right.estimatedCost) {
|
||||
return right.estimatedCost - left.estimatedCost;
|
||||
}
|
||||
if (left.testFileCount !== right.testFileCount) {
|
||||
return right.testFileCount - left.testFileCount;
|
||||
}
|
||||
return left.extensionId.localeCompare(right.extensionId);
|
||||
});
|
||||
|
||||
const effectiveShardCount = Math.min(shardCount, Math.max(1, plans.length));
|
||||
const shards = Array.from({ length: effectiveShardCount }, () => ({
|
||||
estimatedCost: 0,
|
||||
plans: [],
|
||||
testFileCount: 0,
|
||||
}));
|
||||
|
||||
for (const plan of plans) {
|
||||
const targetIndex = pickLeastLoadedShard(shards);
|
||||
shards[targetIndex].plans.push(plan);
|
||||
shards[targetIndex].estimatedCost += plan.estimatedCost;
|
||||
shards[targetIndex].testFileCount += plan.testFileCount;
|
||||
}
|
||||
|
||||
return shards
|
||||
.map((shard, index) =>
|
||||
Object.assign(
|
||||
{},
|
||||
{ index, checkName: `checks-node-extensions-shard-${index + 1}` },
|
||||
mergeTestPlans(shard.plans),
|
||||
),
|
||||
)
|
||||
.filter((shard) => shard.hasTests);
|
||||
}
|
||||
41
openclaw/scripts/lib/format-generated-module.mjs
Normal file
41
openclaw/scripts/lib/format-generated-module.mjs
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
export function formatGeneratedModule(source, { repoRoot, outputPath, errorLabel }) {
|
||||
const resolvedRepoRoot = path.resolve(repoRoot);
|
||||
const resolvedOutputPath = path.resolve(
|
||||
resolvedRepoRoot,
|
||||
path.isAbsolute(outputPath) ? path.relative(resolvedRepoRoot, outputPath) : outputPath,
|
||||
);
|
||||
const directFormatterPath = path.join(resolvedRepoRoot, "node_modules", ".bin", "oxfmt");
|
||||
const useDirectFormatter = process.platform !== "win32" && fs.existsSync(directFormatterPath);
|
||||
const command = useDirectFormatter ? directFormatterPath : "pnpm";
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-generated-format-"));
|
||||
const tempOutputPath = path.join(tempDir, path.basename(resolvedOutputPath));
|
||||
|
||||
try {
|
||||
fs.writeFileSync(tempOutputPath, source, "utf8");
|
||||
const args = useDirectFormatter
|
||||
? ["--write", tempOutputPath]
|
||||
: ["exec", "oxfmt", "--write", tempOutputPath];
|
||||
const formatter = spawnSync(command, args, {
|
||||
cwd: resolvedRepoRoot,
|
||||
encoding: "utf8",
|
||||
// Windows requires a shell to launch package-manager shim scripts reliably.
|
||||
...(process.platform === "win32" ? { shell: true } : {}),
|
||||
});
|
||||
if (formatter.status !== 0) {
|
||||
const details =
|
||||
formatter.stderr?.trim() ||
|
||||
formatter.stdout?.trim() ||
|
||||
formatter.error?.message ||
|
||||
"unknown formatter failure";
|
||||
throw new Error(`failed to format generated ${errorLabel}: ${details}`);
|
||||
}
|
||||
return fs.readFileSync(tempOutputPath, "utf8");
|
||||
} finally {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
45
openclaw/scripts/lib/generated-output-utils.mjs
Normal file
45
openclaw/scripts/lib/generated-output-utils.mjs
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { writeTextFileIfChanged } from "../runtime-postbuild-shared.mjs";
|
||||
import { readIfExists } from "./bundled-plugin-source-utils.mjs";
|
||||
|
||||
export function writeGeneratedOutput(params) {
|
||||
const outputPath = path.resolve(params.repoRoot, params.outputPath);
|
||||
const current = readIfExists(outputPath);
|
||||
const changed = current !== params.next;
|
||||
|
||||
if (params.check) {
|
||||
return {
|
||||
changed,
|
||||
wrote: false,
|
||||
outputPath,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
changed,
|
||||
wrote: writeTextFileIfChanged(outputPath, params.next),
|
||||
outputPath,
|
||||
};
|
||||
}
|
||||
|
||||
export function reportGeneratedOutputCli(params) {
|
||||
if (params.importMetaUrl !== pathToFileURL(process.argv[1] ?? "").href) {
|
||||
return;
|
||||
}
|
||||
|
||||
const check = process.argv.includes("--check");
|
||||
const result = params.run({ check });
|
||||
if (!result.changed) {
|
||||
return;
|
||||
}
|
||||
|
||||
const relativeOutputPath = path.relative(process.cwd(), result.outputPath);
|
||||
if (check) {
|
||||
console.error(`[${params.label}] stale generated output at ${relativeOutputPath}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[${params.label}] wrote ${relativeOutputPath}`);
|
||||
}
|
||||
175
openclaw/scripts/lib/guard-inventory-utils.mjs
Normal file
175
openclaw/scripts/lib/guard-inventory-utils.mjs
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const parsedTypeScriptSourceCache = new Map();
|
||||
|
||||
export function normalizeRepoPath(repoRoot, filePath) {
|
||||
return path.relative(repoRoot, filePath).split(path.sep).join("/");
|
||||
}
|
||||
|
||||
export function resolveRepoSpecifier(repoRoot, specifier, importerFile) {
|
||||
if (specifier.startsWith(".")) {
|
||||
return normalizeRepoPath(repoRoot, path.resolve(path.dirname(importerFile), specifier));
|
||||
}
|
||||
if (specifier.startsWith("/")) {
|
||||
return normalizeRepoPath(repoRoot, specifier);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function visitModuleSpecifiers(ts, sourceFile, visit) {
|
||||
function walk(node) {
|
||||
if (ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier)) {
|
||||
visit({
|
||||
kind: "import",
|
||||
node,
|
||||
specifier: node.moduleSpecifier.text,
|
||||
specifierNode: node.moduleSpecifier,
|
||||
});
|
||||
} else if (
|
||||
ts.isExportDeclaration(node) &&
|
||||
node.moduleSpecifier &&
|
||||
ts.isStringLiteral(node.moduleSpecifier)
|
||||
) {
|
||||
visit({
|
||||
kind: "export",
|
||||
node,
|
||||
specifier: node.moduleSpecifier.text,
|
||||
specifierNode: node.moduleSpecifier,
|
||||
});
|
||||
} else if (
|
||||
ts.isCallExpression(node) &&
|
||||
node.expression.kind === ts.SyntaxKind.ImportKeyword &&
|
||||
node.arguments.length === 1 &&
|
||||
ts.isStringLiteral(node.arguments[0])
|
||||
) {
|
||||
visit({
|
||||
kind: "dynamic-import",
|
||||
node,
|
||||
specifier: node.arguments[0].text,
|
||||
specifierNode: node.arguments[0],
|
||||
});
|
||||
}
|
||||
|
||||
ts.forEachChild(node, walk);
|
||||
}
|
||||
|
||||
walk(sourceFile);
|
||||
}
|
||||
|
||||
export function diffInventoryEntries(expected, actual, compareEntries) {
|
||||
const expectedKeys = new Set(expected.map((entry) => JSON.stringify(entry)));
|
||||
const actualKeys = new Set(actual.map((entry) => JSON.stringify(entry)));
|
||||
return {
|
||||
missing: expected
|
||||
.filter((entry) => !actualKeys.has(JSON.stringify(entry)))
|
||||
.toSorted(compareEntries),
|
||||
unexpected: actual
|
||||
.filter((entry) => !expectedKeys.has(JSON.stringify(entry)))
|
||||
.toSorted(compareEntries),
|
||||
};
|
||||
}
|
||||
|
||||
export function writeLine(stream, text) {
|
||||
stream.write(`${text}\n`);
|
||||
}
|
||||
|
||||
export function createCachedAsync(factory) {
|
||||
let cachedPromise = null;
|
||||
return async function getCachedValue() {
|
||||
if (cachedPromise) {
|
||||
return cachedPromise;
|
||||
}
|
||||
|
||||
cachedPromise = factory();
|
||||
try {
|
||||
return await cachedPromise;
|
||||
} catch (error) {
|
||||
cachedPromise = null;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function formatGroupedInventoryHuman(params, inventory) {
|
||||
if (inventory.length === 0) {
|
||||
return `${params.rule}\n${params.cleanMessage}`;
|
||||
}
|
||||
|
||||
const lines = [params.rule, params.inventoryTitle];
|
||||
let activeFile = "";
|
||||
for (const entry of inventory) {
|
||||
if (entry.file !== activeFile) {
|
||||
activeFile = entry.file;
|
||||
lines.push(activeFile);
|
||||
}
|
||||
lines.push(` - line ${entry.line} [${entry.kind}] ${entry.reason}`);
|
||||
lines.push(` specifier: ${entry.specifier}`);
|
||||
lines.push(` resolved: ${entry.resolvedPath}`);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export async function collectTypeScriptInventory(params) {
|
||||
const inventory = [];
|
||||
const scriptKind = params.scriptKind ?? params.ts.ScriptKind.TS;
|
||||
|
||||
for (const filePath of params.files) {
|
||||
const cacheKey = `${scriptKind}:${filePath}`;
|
||||
let sourceFile = parsedTypeScriptSourceCache.get(cacheKey);
|
||||
if (!sourceFile) {
|
||||
const source = await fs.readFile(filePath, "utf8");
|
||||
if (params.shouldParseSource && !params.shouldParseSource(source, filePath)) {
|
||||
continue;
|
||||
}
|
||||
sourceFile = params.ts.createSourceFile(
|
||||
filePath,
|
||||
source,
|
||||
params.ts.ScriptTarget.Latest,
|
||||
true,
|
||||
scriptKind,
|
||||
);
|
||||
parsedTypeScriptSourceCache.set(cacheKey, sourceFile);
|
||||
}
|
||||
inventory.push(...params.collectEntries(sourceFile, filePath));
|
||||
}
|
||||
|
||||
return inventory.toSorted(params.compareEntries);
|
||||
}
|
||||
|
||||
export async function runBaselineInventoryCheck(params) {
|
||||
const streams = params.io ?? { stdout: process.stdout, stderr: process.stderr };
|
||||
const json = params.argv.includes("--json");
|
||||
const actual = await params.collectActual();
|
||||
const expected = await params.readExpected();
|
||||
const { missing, unexpected } = params.diffInventory(expected, actual);
|
||||
const matchesBaseline = missing.length === 0 && unexpected.length === 0;
|
||||
|
||||
if (json) {
|
||||
writeLine(streams.stdout, JSON.stringify(actual, null, 2));
|
||||
} else {
|
||||
writeLine(streams.stdout, params.formatInventoryHuman(actual));
|
||||
writeLine(
|
||||
streams.stdout,
|
||||
matchesBaseline
|
||||
? `Baseline matches (${actual.length} entries).`
|
||||
: `Baseline mismatch (${unexpected.length} unexpected, ${missing.length} missing).`,
|
||||
);
|
||||
if (!matchesBaseline) {
|
||||
if (unexpected.length > 0) {
|
||||
writeLine(streams.stderr, "Unexpected entries:");
|
||||
for (const entry of unexpected) {
|
||||
writeLine(streams.stderr, `- ${params.formatEntry(entry)}`);
|
||||
}
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
writeLine(streams.stderr, "Missing baseline entries:");
|
||||
for (const entry of missing) {
|
||||
writeLine(streams.stderr, `- ${params.formatEntry(entry)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matchesBaseline ? 0 : 1;
|
||||
}
|
||||
218
openclaw/scripts/lib/ios-version.ts
Normal file
218
openclaw/scripts/lib/ios-version.ts
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
export const IOS_VERSION_FILE = "apps/ios/version.json";
|
||||
export const IOS_CHANGELOG_FILE = "apps/ios/CHANGELOG.md";
|
||||
export const IOS_VERSION_XCCONFIG_FILE = "apps/ios/Config/Version.xcconfig";
|
||||
export const IOS_RELEASE_NOTES_FILE = "apps/ios/fastlane/metadata/en-US/release_notes.txt";
|
||||
|
||||
const PINNED_IOS_VERSION_PATTERN = /^(\d{4}\.\d{1,2}\.\d{1,2})$/u;
|
||||
const GATEWAY_VERSION_PATTERN = /^(\d{4}\.\d{1,2}\.\d{1,2})(?:-(?:beta\.\d+|\d+))?$/u;
|
||||
|
||||
export type IosVersionManifest = {
|
||||
version: string;
|
||||
};
|
||||
|
||||
export type ResolvedIosVersion = {
|
||||
canonicalVersion: string;
|
||||
marketingVersion: string;
|
||||
buildVersion: string;
|
||||
versionFilePath: string;
|
||||
changelogPath: string;
|
||||
versionXcconfigPath: string;
|
||||
releaseNotesPath: string;
|
||||
};
|
||||
|
||||
export type SyncIosVersioningMode = "check" | "write";
|
||||
|
||||
function normalizeTrailingNewline(value: string): string {
|
||||
return value.endsWith("\n") ? value : `${value}\n`;
|
||||
}
|
||||
|
||||
export function normalizePinnedIosVersion(rawVersion: string): string {
|
||||
const trimmed = rawVersion.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error(`Missing iOS version in ${IOS_VERSION_FILE}.`);
|
||||
}
|
||||
|
||||
const match = PINNED_IOS_VERSION_PATTERN.exec(trimmed);
|
||||
if (!match) {
|
||||
throw new Error(`Invalid iOS version '${rawVersion}'. Expected pinned CalVer like 2026.4.6.`);
|
||||
}
|
||||
|
||||
return match[1] ?? trimmed;
|
||||
}
|
||||
|
||||
export function normalizeGatewayVersionToPinnedIosVersion(rawVersion: string): string {
|
||||
const trimmed = rawVersion.trim().replace(/^v/u, "");
|
||||
if (!trimmed) {
|
||||
throw new Error("Missing root package.json version.");
|
||||
}
|
||||
|
||||
const match = GATEWAY_VERSION_PATTERN.exec(trimmed);
|
||||
if (!match) {
|
||||
throw new Error(
|
||||
`Invalid gateway version '${rawVersion}'. Expected YYYY.M.D, YYYY.M.D-beta.N, or YYYY.M.D-N.`,
|
||||
);
|
||||
}
|
||||
|
||||
return match[1] ?? trimmed;
|
||||
}
|
||||
|
||||
export function readRootPackageVersion(rootDir = path.resolve(".")): string {
|
||||
const packageJsonPath = path.join(rootDir, "package.json");
|
||||
const parsed = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { version?: unknown };
|
||||
const version = typeof parsed.version === "string" ? parsed.version.trim() : "";
|
||||
if (!version) {
|
||||
throw new Error(`Missing package.json version in ${packageJsonPath}.`);
|
||||
}
|
||||
return version;
|
||||
}
|
||||
|
||||
export function resolveGatewayVersionForIosRelease(rootDir = path.resolve(".")): {
|
||||
packageVersion: string;
|
||||
pinnedIosVersion: string;
|
||||
} {
|
||||
const packageVersion = readRootPackageVersion(rootDir);
|
||||
return {
|
||||
packageVersion,
|
||||
pinnedIosVersion: normalizeGatewayVersionToPinnedIosVersion(packageVersion),
|
||||
};
|
||||
}
|
||||
|
||||
export function readIosVersionManifest(rootDir = path.resolve(".")): IosVersionManifest {
|
||||
const versionFilePath = path.join(rootDir, IOS_VERSION_FILE);
|
||||
return JSON.parse(readFileSync(versionFilePath, "utf8")) as IosVersionManifest;
|
||||
}
|
||||
|
||||
export function writeIosVersionManifest(version: string, rootDir = path.resolve(".")): string {
|
||||
const versionFilePath = path.join(rootDir, IOS_VERSION_FILE);
|
||||
const normalizedVersion = normalizePinnedIosVersion(version);
|
||||
const nextContent = `${JSON.stringify({ version: normalizedVersion }, null, 2)}\n`;
|
||||
writeFileSync(versionFilePath, nextContent, "utf8");
|
||||
return versionFilePath;
|
||||
}
|
||||
|
||||
export function resolveIosVersion(rootDir = path.resolve(".")): ResolvedIosVersion {
|
||||
const versionFilePath = path.join(rootDir, IOS_VERSION_FILE);
|
||||
const changelogPath = path.join(rootDir, IOS_CHANGELOG_FILE);
|
||||
const versionXcconfigPath = path.join(rootDir, IOS_VERSION_XCCONFIG_FILE);
|
||||
const releaseNotesPath = path.join(rootDir, IOS_RELEASE_NOTES_FILE);
|
||||
const manifest = readIosVersionManifest(rootDir);
|
||||
const canonicalVersion = normalizePinnedIosVersion(manifest.version ?? "");
|
||||
|
||||
return {
|
||||
canonicalVersion,
|
||||
marketingVersion: canonicalVersion,
|
||||
buildVersion: "1",
|
||||
versionFilePath,
|
||||
changelogPath,
|
||||
versionXcconfigPath,
|
||||
releaseNotesPath,
|
||||
};
|
||||
}
|
||||
|
||||
export function renderIosVersionXcconfig(version: ResolvedIosVersion): string {
|
||||
return `// Shared iOS version defaults.\n// Source of truth: apps/ios/version.json\n// Generated by scripts/ios-sync-versioning.ts.\n\nOPENCLAW_IOS_VERSION = ${version.canonicalVersion}\nOPENCLAW_MARKETING_VERSION = ${version.marketingVersion}\nOPENCLAW_BUILD_VERSION = ${version.buildVersion}\n\n#include? "../build/Version.xcconfig"\n`;
|
||||
}
|
||||
|
||||
function matchChangelogHeading(line: string, heading: string): boolean {
|
||||
const normalized = line.trim();
|
||||
return normalized === `## ${heading}` || normalized.startsWith(`## ${heading} - `);
|
||||
}
|
||||
|
||||
export function extractChangelogSection(content: string, heading: string): string | null {
|
||||
const lines = content.split(/\r?\n/);
|
||||
const startIndex = lines.findIndex((line) => matchChangelogHeading(line, heading));
|
||||
if (startIndex === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let endIndex = lines.length;
|
||||
for (let index = startIndex + 1; index < lines.length; index += 1) {
|
||||
if (lines[index]?.startsWith("## ")) {
|
||||
endIndex = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const body = lines
|
||||
.slice(startIndex + 1, endIndex)
|
||||
.join("\n")
|
||||
.trim();
|
||||
return body || null;
|
||||
}
|
||||
|
||||
export function renderIosReleaseNotes(
|
||||
version: ResolvedIosVersion,
|
||||
changelogContent: string,
|
||||
): string {
|
||||
const candidateHeadings = [version.canonicalVersion, "Unreleased"];
|
||||
|
||||
for (const heading of candidateHeadings) {
|
||||
const body = extractChangelogSection(changelogContent, heading);
|
||||
if (body) {
|
||||
return `${body}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Unable to find iOS changelog notes for ${version.canonicalVersion}. Add a matching section to ${IOS_CHANGELOG_FILE}.`,
|
||||
);
|
||||
}
|
||||
|
||||
function syncFile(params: {
|
||||
mode: SyncIosVersioningMode;
|
||||
path: string;
|
||||
nextContent: string;
|
||||
label: string;
|
||||
}): boolean {
|
||||
const nextContent = normalizeTrailingNewline(params.nextContent);
|
||||
const currentContent = readFileSync(params.path, "utf8");
|
||||
if (currentContent === nextContent) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (params.mode === "check") {
|
||||
throw new Error(`${params.label} is stale: ${path.relative(process.cwd(), params.path)}`);
|
||||
}
|
||||
|
||||
writeFileSync(params.path, nextContent, "utf8");
|
||||
return true;
|
||||
}
|
||||
|
||||
export function syncIosVersioning(params?: { mode?: SyncIosVersioningMode; rootDir?: string }): {
|
||||
updatedPaths: string[];
|
||||
} {
|
||||
const mode = params?.mode ?? "write";
|
||||
const rootDir = path.resolve(params?.rootDir ?? ".");
|
||||
const version = resolveIosVersion(rootDir);
|
||||
const changelogContent = readFileSync(version.changelogPath, "utf8");
|
||||
const nextVersionXcconfig = renderIosVersionXcconfig(version);
|
||||
const nextReleaseNotes = renderIosReleaseNotes(version, changelogContent);
|
||||
const updatedPaths: string[] = [];
|
||||
|
||||
if (
|
||||
syncFile({
|
||||
mode,
|
||||
path: version.versionXcconfigPath,
|
||||
nextContent: nextVersionXcconfig,
|
||||
label: "iOS version xcconfig",
|
||||
})
|
||||
) {
|
||||
updatedPaths.push(version.versionXcconfigPath);
|
||||
}
|
||||
|
||||
if (
|
||||
syncFile({
|
||||
mode,
|
||||
path: version.releaseNotesPath,
|
||||
nextContent: nextReleaseNotes,
|
||||
label: "iOS release notes",
|
||||
})
|
||||
) {
|
||||
updatedPaths.push(version.releaseNotesPath);
|
||||
}
|
||||
|
||||
return { updatedPaths };
|
||||
}
|
||||
205
openclaw/scripts/lib/live-docker-auth.sh
Normal file
205
openclaw/scripts/lib/live-docker-auth.sh
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
OPENCLAW_DOCKER_LIVE_AUTH_ALL=(.gemini .minimax)
|
||||
OPENCLAW_DOCKER_LIVE_AUTH_FILES_ALL=(
|
||||
.codex/auth.json
|
||||
.codex/config.toml
|
||||
.claude.json
|
||||
.claude/.credentials.json
|
||||
.claude/settings.json
|
||||
.claude/settings.local.json
|
||||
.gemini/settings.json
|
||||
)
|
||||
|
||||
openclaw_live_trim() {
|
||||
local value="${1:-}"
|
||||
value="${value#"${value%%[![:space:]]*}"}"
|
||||
value="${value%"${value##*[![:space:]]}"}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
openclaw_live_validate_relative_home_path() {
|
||||
local value
|
||||
value="$(openclaw_live_trim "${1:-}")"
|
||||
[[ -n "$value" ]] || {
|
||||
echo "ERROR: empty auth path." >&2
|
||||
return 1
|
||||
}
|
||||
case "$value" in
|
||||
/* | *..* | *\\* | *:*)
|
||||
echo "ERROR: invalid auth path '$value'." >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
openclaw_live_normalize_auth_dir() {
|
||||
local value
|
||||
value="$(openclaw_live_trim "${1:-}")"
|
||||
[[ -n "$value" ]] || return 1
|
||||
if [[ "$value" != .* ]]; then
|
||||
value=".$value"
|
||||
fi
|
||||
value="$(openclaw_live_validate_relative_home_path "$value")" || return 1
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
openclaw_live_should_include_auth_dir_for_provider() {
|
||||
local provider
|
||||
provider="$(openclaw_live_trim "${1:-}")"
|
||||
case "$provider" in
|
||||
gemini | gemini-cli | google-gemini-cli)
|
||||
printf '%s\n' ".gemini"
|
||||
;;
|
||||
minimax | minimax-portal)
|
||||
printf '%s\n' ".minimax"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
openclaw_live_should_include_auth_file_for_provider() {
|
||||
local provider
|
||||
provider="$(openclaw_live_trim "${1:-}")"
|
||||
case "$provider" in
|
||||
codex-cli | openai-codex)
|
||||
printf '%s\n' ".codex/auth.json"
|
||||
printf '%s\n' ".codex/config.toml"
|
||||
;;
|
||||
anthropic | claude-cli)
|
||||
printf '%s\n' ".claude.json"
|
||||
printf '%s\n' ".claude/.credentials.json"
|
||||
printf '%s\n' ".claude/settings.json"
|
||||
printf '%s\n' ".claude/settings.local.json"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
openclaw_live_collect_auth_dirs_from_csv() {
|
||||
local raw="${1:-}"
|
||||
local token normalized
|
||||
[[ -n "$(openclaw_live_trim "$raw")" ]] || return 0
|
||||
IFS=',' read -r -a tokens <<<"$raw"
|
||||
for token in "${tokens[@]}"; do
|
||||
while IFS= read -r normalized; do
|
||||
printf '%s\n' "$normalized"
|
||||
done < <(openclaw_live_should_include_auth_dir_for_provider "$token")
|
||||
done | awk 'NF && !seen[$0]++'
|
||||
}
|
||||
|
||||
openclaw_live_collect_auth_dirs_from_override() {
|
||||
local raw token normalized
|
||||
raw="$(openclaw_live_trim "${OPENCLAW_DOCKER_AUTH_DIRS:-}")"
|
||||
[[ -n "$raw" ]] || return 1
|
||||
case "$raw" in
|
||||
all)
|
||||
printf '%s\n' "${OPENCLAW_DOCKER_LIVE_AUTH_ALL[@]}"
|
||||
return 0
|
||||
;;
|
||||
none)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
IFS=',' read -r -a tokens <<<"$raw"
|
||||
for token in "${tokens[@]}"; do
|
||||
normalized="$(openclaw_live_normalize_auth_dir "$token")" || continue
|
||||
printf '%s\n' "$normalized"
|
||||
done | awk '!seen[$0]++'
|
||||
return 0
|
||||
}
|
||||
|
||||
openclaw_live_collect_auth_dirs() {
|
||||
if openclaw_live_collect_auth_dirs_from_override; then
|
||||
return 0
|
||||
fi
|
||||
printf '%s\n' "${OPENCLAW_DOCKER_LIVE_AUTH_ALL[@]}"
|
||||
}
|
||||
|
||||
openclaw_live_collect_auth_files_from_csv() {
|
||||
local raw="${1:-}"
|
||||
local token normalized
|
||||
[[ -n "$(openclaw_live_trim "$raw")" ]] || return 0
|
||||
IFS=',' read -r -a tokens <<<"$raw"
|
||||
for token in "${tokens[@]}"; do
|
||||
while IFS= read -r normalized; do
|
||||
printf '%s\n' "$normalized"
|
||||
done < <(openclaw_live_should_include_auth_file_for_provider "$token")
|
||||
done | awk 'NF && !seen[$0]++'
|
||||
}
|
||||
|
||||
openclaw_live_collect_auth_files_from_override() {
|
||||
local raw
|
||||
raw="$(openclaw_live_trim "${OPENCLAW_DOCKER_AUTH_DIRS:-}")"
|
||||
[[ -n "$raw" ]] || return 1
|
||||
case "$raw" in
|
||||
all)
|
||||
printf '%s\n' "${OPENCLAW_DOCKER_LIVE_AUTH_FILES_ALL[@]}"
|
||||
return 0
|
||||
;;
|
||||
none)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
openclaw_live_collect_auth_files() {
|
||||
if openclaw_live_collect_auth_files_from_override; then
|
||||
return 0
|
||||
fi
|
||||
printf '%s\n' "${OPENCLAW_DOCKER_LIVE_AUTH_FILES_ALL[@]}"
|
||||
}
|
||||
|
||||
openclaw_live_join_csv() {
|
||||
local first=1 value
|
||||
for value in "$@"; do
|
||||
[[ -n "$value" ]] || continue
|
||||
if (( first )); then
|
||||
printf '%s' "$value"
|
||||
first=0
|
||||
else
|
||||
printf ',%s' "$value"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
openclaw_live_stage_auth_into_home() {
|
||||
local dest_home="${1:?destination home directory required}"
|
||||
shift
|
||||
|
||||
local mode="dirs"
|
||||
local relative_path source_path dest_path
|
||||
|
||||
mkdir -p "$dest_home"
|
||||
chmod u+rwx "$dest_home" || true
|
||||
|
||||
while (($# > 0)); do
|
||||
case "$1" in
|
||||
--files)
|
||||
mode="files"
|
||||
shift
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
relative_path="$(openclaw_live_validate_relative_home_path "$1")" || return 1
|
||||
source_path="$HOME/$relative_path"
|
||||
dest_path="$dest_home/$relative_path"
|
||||
|
||||
if [[ "$mode" == "dirs" ]]; then
|
||||
if [[ -d "$source_path" ]]; then
|
||||
mkdir -p "$dest_path"
|
||||
cp -R "$source_path"/. "$dest_path"
|
||||
chmod -R u+rwX "$dest_path" || true
|
||||
fi
|
||||
else
|
||||
if [[ -f "$source_path" ]]; then
|
||||
mkdir -p "$(dirname "$dest_path")"
|
||||
cp "$source_path" "$dest_path"
|
||||
chmod u+rw "$dest_path" || true
|
||||
fi
|
||||
fi
|
||||
|
||||
shift
|
||||
done
|
||||
}
|
||||
91
openclaw/scripts/lib/live-docker-stage.sh
Normal file
91
openclaw/scripts/lib/live-docker-stage.sh
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
openclaw_live_stage_source_tree() {
|
||||
local dest_dir="${1:?destination directory required}"
|
||||
|
||||
set +e
|
||||
tar -C /src \
|
||||
--warning=no-file-changed \
|
||||
--ignore-failed-read \
|
||||
--exclude=.git \
|
||||
--exclude=node_modules \
|
||||
--exclude=dist \
|
||||
--exclude=ui/dist \
|
||||
--exclude=ui/node_modules \
|
||||
--exclude=.pnpm-store \
|
||||
--exclude=.tmp \
|
||||
--exclude=.tmp-precommit-venv \
|
||||
--exclude=.worktrees \
|
||||
--exclude=__openclaw_vitest__ \
|
||||
--exclude=relay.sock \
|
||||
--exclude='*.sock' \
|
||||
--exclude='*/*.sock' \
|
||||
--exclude='apps/*/.build' \
|
||||
--exclude='apps/*/*.bun-build' \
|
||||
--exclude='apps/*/.gradle' \
|
||||
--exclude='apps/*/.kotlin' \
|
||||
--exclude='apps/*/build' \
|
||||
-cf - . | tar -C "$dest_dir" -xf -
|
||||
local status=$?
|
||||
set -e
|
||||
if [ "$status" -gt 1 ]; then
|
||||
return "$status"
|
||||
fi
|
||||
}
|
||||
|
||||
openclaw_live_link_runtime_tree() {
|
||||
local dest_dir="${1:?destination directory required}"
|
||||
|
||||
ln -s /app/node_modules "$dest_dir/node_modules"
|
||||
ln -s /app/dist "$dest_dir/dist"
|
||||
if [ -d /app/dist-runtime/extensions ]; then
|
||||
export OPENCLAW_BUNDLED_PLUGINS_DIR=/app/dist-runtime/extensions
|
||||
elif [ -d /app/dist/extensions ]; then
|
||||
export OPENCLAW_BUNDLED_PLUGINS_DIR=/app/dist/extensions
|
||||
fi
|
||||
}
|
||||
|
||||
openclaw_live_stage_state_dir() {
|
||||
local dest_dir="${1:?destination directory required}"
|
||||
local source_dir="${HOME}/.openclaw"
|
||||
|
||||
mkdir -p "$dest_dir"
|
||||
if [ -d "$source_dir" ]; then
|
||||
# Sandbox workspaces can accumulate root-owned artifacts from prior Docker
|
||||
# runs. They are not needed for live-test auth/config staging and can make
|
||||
# temp-dir cleanup fail on exit, so keep them out of the staged state copy.
|
||||
set +e
|
||||
tar -C "$source_dir" \
|
||||
--warning=no-file-changed \
|
||||
--ignore-failed-read \
|
||||
--exclude=workspace \
|
||||
--exclude=sandboxes \
|
||||
--exclude=relay.sock \
|
||||
--exclude='*.sock' \
|
||||
--exclude='*/*.sock' \
|
||||
-cf - . | tar -C "$dest_dir" -xf -
|
||||
local status=$?
|
||||
set -e
|
||||
if [ "$status" -gt 1 ]; then
|
||||
return "$status"
|
||||
fi
|
||||
chmod -R u+rwX "$dest_dir" || true
|
||||
if [ -d "$source_dir/workspace" ] && [ ! -e "$dest_dir/workspace" ]; then
|
||||
ln -s "$source_dir/workspace" "$dest_dir/workspace"
|
||||
fi
|
||||
fi
|
||||
|
||||
export OPENCLAW_STATE_DIR="$dest_dir"
|
||||
export OPENCLAW_CONFIG_PATH="$dest_dir/openclaw.json"
|
||||
}
|
||||
|
||||
openclaw_live_prepare_staged_config() {
|
||||
if [ ! -f "${OPENCLAW_CONFIG_PATH:-}" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
(
|
||||
cd /app
|
||||
node --import tsx /src/scripts/live-docker-normalize-config.ts
|
||||
)
|
||||
}
|
||||
395
openclaw/scripts/lib/local-heavy-check-runtime.mjs
Normal file
395
openclaw/scripts/lib/local-heavy-check-runtime.mjs
Normal file
|
|
@ -0,0 +1,395 @@
|
|||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const GIB = 1024 ** 3;
|
||||
const DEFAULT_LOCAL_GO_GC = "30";
|
||||
const DEFAULT_LOCAL_GO_MEMORY_LIMIT = "3GiB";
|
||||
const DEFAULT_LOCAL_TSGO_BUILD_INFO_FILE = ".artifacts/tsgo-cache/root.tsbuildinfo";
|
||||
const DEFAULT_LOCK_TIMEOUT_MS = 10 * 60 * 1000;
|
||||
const DEFAULT_LOCK_POLL_MS = 500;
|
||||
const DEFAULT_LOCK_PROGRESS_MS = 15 * 1000;
|
||||
const DEFAULT_STALE_LOCK_MS = 30 * 1000;
|
||||
const DEFAULT_FAST_LOCAL_CHECK_MIN_MEMORY_BYTES = 48 * GIB;
|
||||
const DEFAULT_FAST_LOCAL_CHECK_MIN_CPUS = 12;
|
||||
const SLEEP_BUFFER = new Int32Array(new SharedArrayBuffer(4));
|
||||
|
||||
export function isLocalCheckEnabled(env) {
|
||||
const raw = env.OPENCLAW_LOCAL_CHECK?.trim().toLowerCase();
|
||||
return raw !== "0" && raw !== "false";
|
||||
}
|
||||
|
||||
export function hasFlag(args, name) {
|
||||
return args.some((arg) => arg === name || arg.startsWith(`${name}=`));
|
||||
}
|
||||
|
||||
export function applyLocalTsgoPolicy(args, env, hostResources) {
|
||||
const nextEnv = { ...env };
|
||||
const nextArgs = [...args];
|
||||
const defaultProjectRun = nextArgs.length === 0;
|
||||
|
||||
if (!hasFlag(nextArgs, "--declaration") && !nextArgs.includes("-d")) {
|
||||
insertBeforeSeparator(nextArgs, "--declaration", "false");
|
||||
}
|
||||
|
||||
if (!isLocalCheckEnabled(nextEnv)) {
|
||||
return { env: nextEnv, args: nextArgs };
|
||||
}
|
||||
|
||||
if (defaultProjectRun) {
|
||||
insertBeforeSeparator(nextArgs, "--incremental");
|
||||
insertBeforeSeparator(
|
||||
nextArgs,
|
||||
"--tsBuildInfoFile",
|
||||
nextEnv.OPENCLAW_TSGO_BUILD_INFO_FILE ?? DEFAULT_LOCAL_TSGO_BUILD_INFO_FILE,
|
||||
);
|
||||
}
|
||||
|
||||
if (shouldThrottleLocalHeavyChecks(nextEnv, hostResources)) {
|
||||
insertBeforeSeparator(nextArgs, "--singleThreaded");
|
||||
insertBeforeSeparator(nextArgs, "--checkers", "1");
|
||||
|
||||
if (!nextEnv.GOGC) {
|
||||
nextEnv.GOGC = DEFAULT_LOCAL_GO_GC;
|
||||
}
|
||||
if (!nextEnv.GOMEMLIMIT) {
|
||||
nextEnv.GOMEMLIMIT = DEFAULT_LOCAL_GO_MEMORY_LIMIT;
|
||||
}
|
||||
}
|
||||
if (nextEnv.OPENCLAW_TSGO_PPROF_DIR && !hasFlag(nextArgs, "--pprofDir")) {
|
||||
insertBeforeSeparator(nextArgs, "--pprofDir", nextEnv.OPENCLAW_TSGO_PPROF_DIR);
|
||||
}
|
||||
|
||||
return { env: nextEnv, args: nextArgs };
|
||||
}
|
||||
|
||||
export function applyLocalOxlintPolicy(args, env, hostResources) {
|
||||
const nextEnv = { ...env };
|
||||
const nextArgs = [...args];
|
||||
|
||||
insertBeforeSeparator(nextArgs, "--type-aware");
|
||||
insertBeforeSeparator(nextArgs, "--tsconfig", "tsconfig.oxlint.json");
|
||||
if (
|
||||
!hasFlag(nextArgs, "--report-unused-disable-directives") &&
|
||||
!hasFlag(nextArgs, "--report-unused-disable-directives-severity")
|
||||
) {
|
||||
insertBeforeSeparator(nextArgs, "--report-unused-disable-directives-severity", "error");
|
||||
}
|
||||
|
||||
if (shouldThrottleLocalHeavyChecks(nextEnv, hostResources)) {
|
||||
insertBeforeSeparator(nextArgs, "--threads=1");
|
||||
}
|
||||
|
||||
return { env: nextEnv, args: nextArgs };
|
||||
}
|
||||
|
||||
export function shouldAcquireLocalHeavyCheckLockForOxlint(
|
||||
args,
|
||||
{ cwd = process.cwd(), env = process.env } = {},
|
||||
) {
|
||||
if (env.OPENCLAW_OXLINT_FORCE_LOCK === "1") {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
args.some(
|
||||
(arg) =>
|
||||
arg === "--help" ||
|
||||
arg === "-h" ||
|
||||
arg === "--version" ||
|
||||
arg === "-V" ||
|
||||
arg === "--rules" ||
|
||||
arg === "--print-config" ||
|
||||
arg === "--init",
|
||||
)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const separatorIndex = args.indexOf("--");
|
||||
const candidateArgs = (() => {
|
||||
if (separatorIndex !== -1) {
|
||||
return args.slice(separatorIndex + 1);
|
||||
}
|
||||
const firstFlagIndex = args.findIndex((arg) => arg.startsWith("-"));
|
||||
return firstFlagIndex === -1 ? args : args.slice(0, firstFlagIndex);
|
||||
})();
|
||||
const explicitTargets = candidateArgs.filter((arg) => arg.length > 0 && !arg.startsWith("-"));
|
||||
if (explicitTargets.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return !explicitTargets.every((target) => {
|
||||
try {
|
||||
return fs.statSync(path.resolve(cwd, target)).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function shouldAcquireLocalHeavyCheckLockForTsgo(args, env = process.env) {
|
||||
if (env.OPENCLAW_TSGO_FORCE_LOCK === "1") {
|
||||
return true;
|
||||
}
|
||||
|
||||
return !args.some(
|
||||
(arg) =>
|
||||
arg === "--help" ||
|
||||
arg === "-h" ||
|
||||
arg === "--version" ||
|
||||
arg === "-v" ||
|
||||
arg === "--init" ||
|
||||
arg === "--showConfig",
|
||||
);
|
||||
}
|
||||
|
||||
export function shouldThrottleLocalHeavyChecks(env, hostResources) {
|
||||
if (!isLocalCheckEnabled(env)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const mode = readLocalCheckMode(env);
|
||||
if (mode === "throttled") {
|
||||
return true;
|
||||
}
|
||||
if (mode === "full") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const resolvedHostResources = resolveHostResources(hostResources);
|
||||
return (
|
||||
resolvedHostResources.totalMemoryBytes < DEFAULT_FAST_LOCAL_CHECK_MIN_MEMORY_BYTES ||
|
||||
resolvedHostResources.logicalCpuCount < DEFAULT_FAST_LOCAL_CHECK_MIN_CPUS
|
||||
);
|
||||
}
|
||||
|
||||
export function acquireLocalHeavyCheckLockSync(params) {
|
||||
const env = params.env ?? process.env;
|
||||
|
||||
if (!isLocalCheckEnabled(env)) {
|
||||
return () => {};
|
||||
}
|
||||
|
||||
const commonDir = resolveGitCommonDir(params.cwd);
|
||||
const locksDir = path.join(commonDir, "openclaw-local-checks");
|
||||
const lockDir = path.join(locksDir, `${params.lockName ?? "heavy-check"}.lock`);
|
||||
const ownerPath = path.join(lockDir, "owner.json");
|
||||
const timeoutMs = readPositiveInt(
|
||||
env.OPENCLAW_HEAVY_CHECK_LOCK_TIMEOUT_MS,
|
||||
DEFAULT_LOCK_TIMEOUT_MS,
|
||||
);
|
||||
const pollMs = readPositiveInt(env.OPENCLAW_HEAVY_CHECK_LOCK_POLL_MS, DEFAULT_LOCK_POLL_MS);
|
||||
const progressMs = readPositiveInt(
|
||||
env.OPENCLAW_HEAVY_CHECK_LOCK_PROGRESS_MS,
|
||||
DEFAULT_LOCK_PROGRESS_MS,
|
||||
);
|
||||
const staleLockMs = readPositiveInt(
|
||||
env.OPENCLAW_HEAVY_CHECK_STALE_LOCK_MS,
|
||||
DEFAULT_STALE_LOCK_MS,
|
||||
);
|
||||
const startedAt = Date.now();
|
||||
let waitingLogged = false;
|
||||
let lastProgressAt = 0;
|
||||
|
||||
fs.mkdirSync(locksDir, { recursive: true });
|
||||
if (!params.lockName) {
|
||||
cleanupLegacyLockDirs(locksDir, staleLockMs);
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
try {
|
||||
fs.mkdirSync(lockDir);
|
||||
writeOwnerFile(ownerPath, {
|
||||
pid: process.pid,
|
||||
tool: params.toolName,
|
||||
cwd: params.cwd,
|
||||
hostname: os.hostname(),
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
return () => {
|
||||
fs.rmSync(lockDir, { recursive: true, force: true });
|
||||
};
|
||||
} catch (error) {
|
||||
if (!isAlreadyExistsError(error)) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
const owner = readOwnerFile(ownerPath);
|
||||
if (shouldReclaimLock({ owner, lockDir, staleLockMs })) {
|
||||
fs.rmSync(lockDir, { recursive: true, force: true });
|
||||
continue;
|
||||
}
|
||||
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
if (elapsedMs >= timeoutMs) {
|
||||
const ownerLabel = describeOwner(owner);
|
||||
throw new Error(
|
||||
`[${params.toolName}] timed out waiting for the local heavy-check lock at ${lockDir}${
|
||||
ownerLabel ? ` (${ownerLabel})` : ""
|
||||
}. If no local heavy checks are still running, remove the stale lock and retry.`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
if (!waitingLogged) {
|
||||
const ownerLabel = describeOwner(owner);
|
||||
console.error(
|
||||
`[${params.toolName}] queued behind the local heavy-check lock${
|
||||
ownerLabel ? ` held by ${ownerLabel}` : ""
|
||||
}...`,
|
||||
);
|
||||
waitingLogged = true;
|
||||
lastProgressAt = Date.now();
|
||||
} else if (Date.now() - lastProgressAt >= progressMs) {
|
||||
const ownerLabel = describeOwner(owner);
|
||||
console.error(
|
||||
`[${params.toolName}] still waiting ${formatElapsedMs(elapsedMs)} for the local heavy-check lock${
|
||||
ownerLabel ? ` held by ${ownerLabel}` : ""
|
||||
}...`,
|
||||
);
|
||||
lastProgressAt = Date.now();
|
||||
}
|
||||
|
||||
sleepSync(pollMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveGitCommonDir(cwd) {
|
||||
const result = spawnSync("git", ["rev-parse", "--git-common-dir"], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
});
|
||||
|
||||
if (result.status === 0) {
|
||||
const raw = result.stdout.trim();
|
||||
if (raw.length > 0) {
|
||||
return path.resolve(cwd, raw);
|
||||
}
|
||||
}
|
||||
|
||||
return path.join(cwd, ".git");
|
||||
}
|
||||
|
||||
function cleanupLegacyLockDirs(locksDir, staleLockMs) {
|
||||
for (const legacyLockName of ["test"]) {
|
||||
const legacyLockDir = path.join(locksDir, `${legacyLockName}.lock`);
|
||||
if (!fs.existsSync(legacyLockDir)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const owner = readOwnerFile(path.join(legacyLockDir, "owner.json"));
|
||||
if (shouldReclaimLock({ owner, lockDir: legacyLockDir, staleLockMs })) {
|
||||
fs.rmSync(legacyLockDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function insertBeforeSeparator(args, ...items) {
|
||||
if (items.length > 0 && hasFlag(args, items[0])) {
|
||||
return;
|
||||
}
|
||||
|
||||
const separatorIndex = args.indexOf("--");
|
||||
const insertIndex = separatorIndex === -1 ? args.length : separatorIndex;
|
||||
args.splice(insertIndex, 0, ...items);
|
||||
}
|
||||
|
||||
function readLocalCheckMode(env) {
|
||||
const raw = env.OPENCLAW_LOCAL_CHECK_MODE?.trim().toLowerCase();
|
||||
if (raw === "throttled" || raw === "low-memory") {
|
||||
return "throttled";
|
||||
}
|
||||
if (raw === "full" || raw === "fast") {
|
||||
return "full";
|
||||
}
|
||||
// Keep local heavy checks conservative by default. Developers can still opt
|
||||
// into full-speed runs explicitly with OPENCLAW_LOCAL_CHECK_MODE=full.
|
||||
return "throttled";
|
||||
}
|
||||
|
||||
function resolveHostResources(hostResources) {
|
||||
if (hostResources) {
|
||||
return hostResources;
|
||||
}
|
||||
|
||||
return {
|
||||
totalMemoryBytes: os.totalmem(),
|
||||
logicalCpuCount:
|
||||
typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length,
|
||||
};
|
||||
}
|
||||
|
||||
function readPositiveInt(rawValue, fallback) {
|
||||
const parsed = Number.parseInt(rawValue ?? "", 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function writeOwnerFile(ownerPath, owner) {
|
||||
fs.writeFileSync(ownerPath, `${JSON.stringify(owner, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function readOwnerFile(ownerPath) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(ownerPath, "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isAlreadyExistsError(error) {
|
||||
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EEXIST");
|
||||
}
|
||||
|
||||
function shouldReclaimLock({ owner, lockDir, staleLockMs }) {
|
||||
if (owner && typeof owner.pid === "number") {
|
||||
return !isProcessAlive(owner.pid);
|
||||
}
|
||||
|
||||
try {
|
||||
const stats = fs.statSync(lockDir);
|
||||
return Date.now() - stats.mtimeMs >= staleLockMs;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM");
|
||||
}
|
||||
}
|
||||
|
||||
function describeOwner(owner) {
|
||||
if (!owner || typeof owner !== "object") {
|
||||
return "";
|
||||
}
|
||||
|
||||
const tool = typeof owner.tool === "string" ? owner.tool : "unknown-tool";
|
||||
const pid = typeof owner.pid === "number" ? `pid ${owner.pid}` : "unknown pid";
|
||||
const cwd = typeof owner.cwd === "string" ? owner.cwd : "unknown cwd";
|
||||
return `${tool}, ${pid}, cwd ${cwd}`;
|
||||
}
|
||||
|
||||
function formatElapsedMs(elapsedMs) {
|
||||
if (elapsedMs < 1000) {
|
||||
return `${elapsedMs}ms`;
|
||||
}
|
||||
const seconds = elapsedMs / 1000;
|
||||
if (seconds < 60) {
|
||||
return `${seconds.toFixed(seconds >= 10 ? 0 : 1)}s`;
|
||||
}
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainderSeconds = Math.round(seconds % 60);
|
||||
return `${minutes}m ${remainderSeconds}s`;
|
||||
}
|
||||
|
||||
function sleepSync(ms) {
|
||||
Atomics.wait(SLEEP_BUFFER, 0, 0, ms);
|
||||
}
|
||||
22
openclaw/scripts/lib/npm-pack-budget.d.mts
Normal file
22
openclaw/scripts/lib/npm-pack-budget.d.mts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
export type NpmPackBudgetResult = {
|
||||
filename?: string;
|
||||
unpackedSize?: number;
|
||||
};
|
||||
|
||||
export declare const NPM_PACK_UNPACKED_SIZE_BUDGET_BYTES: number;
|
||||
|
||||
export declare function formatMiB(bytes: number): string;
|
||||
|
||||
export declare function formatPackUnpackedSizeBudgetError(params: {
|
||||
budgetBytes?: number;
|
||||
label: string;
|
||||
unpackedSize: number;
|
||||
}): string;
|
||||
|
||||
export declare function collectPackUnpackedSizeErrors(
|
||||
results: Iterable<NpmPackBudgetResult>,
|
||||
options?: {
|
||||
budgetBytes?: number;
|
||||
missingDataMessage?: string;
|
||||
},
|
||||
): string[];
|
||||
55
openclaw/scripts/lib/npm-pack-budget.mjs
Normal file
55
openclaw/scripts/lib/npm-pack-budget.mjs
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
// 2026.3.12 ballooned to ~213.6 MiB unpacked and correlated with low-memory
|
||||
// startup/doctor OOM reports. 2026.4.12 intentionally stages Matrix runtime
|
||||
// dependencies, including crypto wasm, so packaged installs do not miss Docker
|
||||
// and gateway runtime dependencies. Keep the budget below the 2026.3.12 bloat
|
||||
// level while allowing that mirrored runtime surface.
|
||||
export const NPM_PACK_UNPACKED_SIZE_BUDGET_BYTES = 202 * 1024 * 1024;
|
||||
|
||||
export function formatMiB(bytes) {
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`;
|
||||
}
|
||||
|
||||
function resolvePackResultLabel(entry, index) {
|
||||
return entry.filename?.trim() || `pack result #${index + 1}`;
|
||||
}
|
||||
|
||||
export function formatPackUnpackedSizeBudgetError(params) {
|
||||
const budgetBytes = params.budgetBytes ?? NPM_PACK_UNPACKED_SIZE_BUDGET_BYTES;
|
||||
return [
|
||||
`${params.label} unpackedSize ${params.unpackedSize} bytes (${formatMiB(params.unpackedSize)}) exceeds budget ${budgetBytes} bytes (${formatMiB(budgetBytes)}).`,
|
||||
"Investigate duplicate channel shims, copied extension trees, or other accidental pack bloat before release.",
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
export function collectPackUnpackedSizeErrors(results, options = {}) {
|
||||
const entries = Array.from(results);
|
||||
const errors = [];
|
||||
const budgetBytes = options.budgetBytes ?? NPM_PACK_UNPACKED_SIZE_BUDGET_BYTES;
|
||||
let checkedCount = 0;
|
||||
|
||||
for (const [index, entry] of entries.entries()) {
|
||||
if (typeof entry.unpackedSize !== "number" || !Number.isFinite(entry.unpackedSize)) {
|
||||
continue;
|
||||
}
|
||||
checkedCount += 1;
|
||||
if (entry.unpackedSize <= budgetBytes) {
|
||||
continue;
|
||||
}
|
||||
errors.push(
|
||||
formatPackUnpackedSizeBudgetError({
|
||||
budgetBytes,
|
||||
label: resolvePackResultLabel(entry, index),
|
||||
unpackedSize: entry.unpackedSize,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (entries.length > 0 && checkedCount === 0) {
|
||||
errors.push(
|
||||
options.missingDataMessage ??
|
||||
"npm pack --dry-run produced no unpackedSize data; pack size budget was not verified.",
|
||||
);
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
223
openclaw/scripts/lib/npm-publish-plan.mjs
Normal file
223
openclaw/scripts/lib/npm-publish-plan.mjs
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
const STABLE_VERSION_REGEX = /^(?<year>\d{4})\.(?<month>[1-9]\d?)\.(?<day>[1-9]\d?)$/;
|
||||
const BETA_VERSION_REGEX =
|
||||
/^(?<year>\d{4})\.(?<month>[1-9]\d?)\.(?<day>[1-9]\d?)-beta\.(?<beta>[1-9]\d*)$/;
|
||||
const CORRECTION_VERSION_REGEX =
|
||||
/^(?<year>\d{4})\.(?<month>[1-9]\d?)\.(?<day>[1-9]\d?)-(?<correction>[1-9]\d*)$/;
|
||||
|
||||
/**
|
||||
* @typedef {object} ParsedReleaseVersion
|
||||
* @property {string} version
|
||||
* @property {string} baseVersion
|
||||
* @property {"stable" | "beta"} channel
|
||||
* @property {number} year
|
||||
* @property {number} month
|
||||
* @property {number} day
|
||||
* @property {number | undefined} [betaNumber]
|
||||
* @property {number | undefined} [correctionNumber]
|
||||
* @property {Date} date
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {object} NpmPublishPlan
|
||||
* @property {"stable" | "beta"} channel
|
||||
* @property {"latest" | "beta"} publishTag
|
||||
* @property {("latest" | "beta")[]} mirrorDistTags
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {object} NpmDistTagMirrorAuth
|
||||
* @property {boolean} hasAuth
|
||||
* @property {"node-auth-token" | "npm-token" | "none"} source
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {"--dry-run" | "--publish"} NpmPublishMode
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {string} version
|
||||
* @param {Record<string, string | undefined>} groups
|
||||
* @param {"stable" | "beta"} channel
|
||||
* @returns {ParsedReleaseVersion | null}
|
||||
*/
|
||||
function parseDateParts(version, groups, channel) {
|
||||
const year = Number.parseInt(groups.year ?? "", 10);
|
||||
const month = Number.parseInt(groups.month ?? "", 10);
|
||||
const day = Number.parseInt(groups.day ?? "", 10);
|
||||
const betaNumber = channel === "beta" ? Number.parseInt(groups.beta ?? "", 10) : undefined;
|
||||
|
||||
if (
|
||||
!Number.isInteger(year) ||
|
||||
!Number.isInteger(month) ||
|
||||
!Number.isInteger(day) ||
|
||||
month < 1 ||
|
||||
month > 12 ||
|
||||
day < 1 ||
|
||||
day > 31
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
if (channel === "beta" && (!Number.isInteger(betaNumber) || (betaNumber ?? 0) < 1)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const date = new Date(Date.UTC(year, month - 1, day));
|
||||
if (
|
||||
date.getUTCFullYear() !== year ||
|
||||
date.getUTCMonth() !== month - 1 ||
|
||||
date.getUTCDate() !== day
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
version,
|
||||
baseVersion: `${year}.${month}.${day}`,
|
||||
channel,
|
||||
year,
|
||||
month,
|
||||
day,
|
||||
betaNumber,
|
||||
date,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} version
|
||||
* @returns {ParsedReleaseVersion | null}
|
||||
*/
|
||||
export function parseReleaseVersion(version) {
|
||||
const trimmed = version.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const stableMatch = STABLE_VERSION_REGEX.exec(trimmed);
|
||||
if (stableMatch?.groups) {
|
||||
return parseDateParts(trimmed, stableMatch.groups, "stable");
|
||||
}
|
||||
|
||||
const betaMatch = BETA_VERSION_REGEX.exec(trimmed);
|
||||
if (betaMatch?.groups) {
|
||||
return parseDateParts(trimmed, betaMatch.groups, "beta");
|
||||
}
|
||||
|
||||
const correctionMatch = CORRECTION_VERSION_REGEX.exec(trimmed);
|
||||
if (correctionMatch?.groups) {
|
||||
const parsedCorrection = parseDateParts(trimmed, correctionMatch.groups, "stable");
|
||||
const correctionNumber = Number.parseInt(correctionMatch.groups.correction ?? "", 10);
|
||||
if (parsedCorrection === null || !Number.isInteger(correctionNumber) || correctionNumber < 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...parsedCorrection,
|
||||
correctionNumber,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} left
|
||||
* @param {string} right
|
||||
* @returns {number | null}
|
||||
*/
|
||||
export function compareReleaseVersions(left, right) {
|
||||
const parsedLeft = parseReleaseVersion(left);
|
||||
const parsedRight = parseReleaseVersion(right);
|
||||
if (parsedLeft === null || parsedRight === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const dateDelta = parsedLeft.date.getTime() - parsedRight.date.getTime();
|
||||
if (dateDelta !== 0) {
|
||||
return Math.sign(dateDelta);
|
||||
}
|
||||
|
||||
if (parsedLeft.channel !== parsedRight.channel) {
|
||||
return parsedLeft.channel === "stable" ? 1 : -1;
|
||||
}
|
||||
|
||||
if (parsedLeft.channel === "beta" && parsedRight.channel === "beta") {
|
||||
return Math.sign((parsedLeft.betaNumber ?? 0) - (parsedRight.betaNumber ?? 0));
|
||||
}
|
||||
|
||||
return Math.sign((parsedLeft.correctionNumber ?? 0) - (parsedRight.correctionNumber ?? 0));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} version
|
||||
* @param {string | null} [currentBetaVersion]
|
||||
* @returns {NpmPublishPlan}
|
||||
*/
|
||||
export function resolveNpmPublishPlan(version, currentBetaVersion) {
|
||||
const parsedVersion = parseReleaseVersion(version);
|
||||
if (parsedVersion === null) {
|
||||
throw new Error(`Unsupported release version "${version}".`);
|
||||
}
|
||||
|
||||
if (parsedVersion.channel === "beta") {
|
||||
return {
|
||||
channel: "beta",
|
||||
publishTag: "beta",
|
||||
mirrorDistTags: [],
|
||||
};
|
||||
}
|
||||
|
||||
const normalizedCurrentBeta = currentBetaVersion?.trim();
|
||||
if (normalizedCurrentBeta) {
|
||||
const betaVsStable = compareReleaseVersions(normalizedCurrentBeta, version);
|
||||
if (betaVsStable !== null && betaVsStable > 0) {
|
||||
return {
|
||||
channel: "stable",
|
||||
publishTag: "latest",
|
||||
mirrorDistTags: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
channel: "stable",
|
||||
publishTag: "latest",
|
||||
mirrorDistTags: ["beta"],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{
|
||||
* nodeAuthToken?: string | null | undefined;
|
||||
* npmToken?: string | null | undefined;
|
||||
* }} [params]
|
||||
* @returns {NpmDistTagMirrorAuth}
|
||||
*/
|
||||
export function resolveNpmDistTagMirrorAuth(params = {}) {
|
||||
const nodeAuthToken = params.nodeAuthToken?.trim();
|
||||
if (nodeAuthToken) {
|
||||
return { hasAuth: true, source: "node-auth-token" };
|
||||
}
|
||||
|
||||
const npmToken = params.npmToken?.trim();
|
||||
if (npmToken) {
|
||||
return { hasAuth: true, source: "npm-token" };
|
||||
}
|
||||
|
||||
return { hasAuth: false, source: "none" };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{
|
||||
* mode: NpmPublishMode;
|
||||
* mirrorDistTags: string[] | readonly string[];
|
||||
* hasAuth: boolean;
|
||||
* }} params
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function shouldRequireNpmDistTagMirrorAuth(params) {
|
||||
return (
|
||||
params.mode === "--publish" &&
|
||||
params.mirrorDistTags.some((distTag) => distTag.trim().length > 0) &&
|
||||
!params.hasAuth
|
||||
);
|
||||
}
|
||||
11
openclaw/scripts/lib/optional-bundled-clusters-types.d.ts
vendored
Normal file
11
openclaw/scripts/lib/optional-bundled-clusters-types.d.ts
vendored
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
export const optionalBundledClusters: string[];
|
||||
export const optionalBundledClusterSet: Set<string>;
|
||||
export const OPTIONAL_BUNDLED_BUILD_ENV: "OPENCLAW_INCLUDE_OPTIONAL_BUNDLED";
|
||||
export function isOptionalBundledCluster(cluster: string): boolean;
|
||||
export function shouldIncludeOptionalBundledClusters(env?: NodeJS.ProcessEnv): boolean;
|
||||
export function hasReleasedBundledInstall(packageJson: unknown): boolean;
|
||||
export function shouldBuildBundledCluster(
|
||||
cluster: string,
|
||||
env?: NodeJS.ProcessEnv,
|
||||
options?: { packageJson?: unknown },
|
||||
): boolean;
|
||||
1
openclaw/scripts/lib/optional-bundled-clusters.d.mts
Normal file
1
openclaw/scripts/lib/optional-bundled-clusters.d.mts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export * from "./optional-bundled-clusters-types.js";
|
||||
1
openclaw/scripts/lib/optional-bundled-clusters.d.ts
vendored
Normal file
1
openclaw/scripts/lib/optional-bundled-clusters.d.ts
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
export * from "./optional-bundled-clusters-types.js";
|
||||
43
openclaw/scripts/lib/optional-bundled-clusters.mjs
Normal file
43
openclaw/scripts/lib/optional-bundled-clusters.mjs
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
export const optionalBundledClusters = [
|
||||
"acpx",
|
||||
"diagnostics-otel",
|
||||
"diffs",
|
||||
"googlechat",
|
||||
"matrix",
|
||||
"memory-lancedb",
|
||||
"msteams",
|
||||
"nostr",
|
||||
"tlon",
|
||||
"twitch",
|
||||
"ui",
|
||||
"whatsapp",
|
||||
"zalouser",
|
||||
];
|
||||
|
||||
export const optionalBundledClusterSet = new Set(optionalBundledClusters);
|
||||
|
||||
export const OPTIONAL_BUNDLED_BUILD_ENV = "OPENCLAW_INCLUDE_OPTIONAL_BUNDLED";
|
||||
|
||||
export function isOptionalBundledCluster(cluster) {
|
||||
return optionalBundledClusterSet.has(cluster);
|
||||
}
|
||||
|
||||
export function shouldIncludeOptionalBundledClusters(env = process.env) {
|
||||
// Release artifacts should preserve the last shipped upgrade surface by
|
||||
// default. Specific size-sensitive lanes can still opt out explicitly.
|
||||
return env[OPTIONAL_BUNDLED_BUILD_ENV] !== "0";
|
||||
}
|
||||
|
||||
export function hasReleasedBundledInstall(packageJson) {
|
||||
return (
|
||||
typeof packageJson?.openclaw?.install?.npmSpec === "string" &&
|
||||
packageJson.openclaw.install.npmSpec.trim().length > 0
|
||||
);
|
||||
}
|
||||
|
||||
export function shouldBuildBundledCluster(cluster, env = process.env, options = {}) {
|
||||
if (hasReleasedBundledInstall(options.packageJson)) {
|
||||
return true;
|
||||
}
|
||||
return shouldIncludeOptionalBundledClusters(env) || !isOptionalBundledCluster(cluster);
|
||||
}
|
||||
23
openclaw/scripts/lib/package-root-args.mjs
Normal file
23
openclaw/scripts/lib/package-root-args.mjs
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const defaultPackageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
|
||||
|
||||
export function parsePackageRootArg(argv, envName) {
|
||||
let packageRoot = process.env[envName];
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--package-root") {
|
||||
packageRoot = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg?.startsWith("--package-root=")) {
|
||||
packageRoot = arg.slice("--package-root=".length);
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unknown argument: ${arg}`);
|
||||
}
|
||||
return { packageRoot: path.resolve(packageRoot ?? defaultPackageRoot) };
|
||||
}
|
||||
13
openclaw/scripts/lib/pairing-guard-context.mjs
Normal file
13
openclaw/scripts/lib/pairing-guard-context.mjs
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
import path from "node:path";
|
||||
import { resolveRepoRoot, resolveSourceRoots } from "./ts-guard-utils.mjs";
|
||||
|
||||
export function createPairingGuardContext(importMetaUrl) {
|
||||
const repoRoot = resolveRepoRoot(importMetaUrl);
|
||||
const sourceRoots = resolveSourceRoots(repoRoot, ["src", "extensions"]);
|
||||
return {
|
||||
repoRoot,
|
||||
sourceRoots,
|
||||
resolveFromRepo: (relativePath) =>
|
||||
path.join(repoRoot, ...relativePath.split("/").filter(Boolean)),
|
||||
};
|
||||
}
|
||||
371
openclaw/scripts/lib/plugin-clawhub-release.ts
Normal file
371
openclaw/scripts/lib/plugin-clawhub-release.ts
Normal file
|
|
@ -0,0 +1,371 @@
|
|||
import { execFileSync } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
import { validateExternalCodePluginPackageJson } from "../../packages/plugin-package-contract/src/index.ts";
|
||||
import {
|
||||
collectExtensionPackageJsonCandidates,
|
||||
collectChangedPathsFromGitRange,
|
||||
collectChangedExtensionIdsFromPaths,
|
||||
collectPublishablePluginPackageErrors,
|
||||
parsePluginReleaseArgs,
|
||||
parsePluginReleaseSelection,
|
||||
parsePluginReleaseSelectionMode,
|
||||
resolvePublishablePluginVersion,
|
||||
resolveGitCommitSha,
|
||||
resolveChangedPublishablePluginPackages,
|
||||
resolveSelectedPublishablePluginPackages,
|
||||
type GitRangeSelection,
|
||||
type ParsedPluginReleaseArgs,
|
||||
type PluginReleaseSelectionMode,
|
||||
} from "./plugin-npm-release.ts";
|
||||
|
||||
export {
|
||||
collectChangedExtensionIdsFromPaths,
|
||||
parsePluginReleaseArgs,
|
||||
parsePluginReleaseSelection,
|
||||
parsePluginReleaseSelectionMode,
|
||||
resolveChangedPublishablePluginPackages,
|
||||
resolveSelectedPublishablePluginPackages,
|
||||
type GitRangeSelection,
|
||||
type ParsedPluginReleaseArgs,
|
||||
type PluginReleaseSelectionMode,
|
||||
};
|
||||
|
||||
export type PluginPackageJson = {
|
||||
name?: string;
|
||||
version?: string;
|
||||
private?: boolean;
|
||||
openclaw?: {
|
||||
extensions?: string[];
|
||||
install?: {
|
||||
npmSpec?: string;
|
||||
};
|
||||
compat?: {
|
||||
pluginApi?: string;
|
||||
minGatewayVersion?: string;
|
||||
};
|
||||
build?: {
|
||||
openclawVersion?: string;
|
||||
pluginSdkVersion?: string;
|
||||
};
|
||||
release?: {
|
||||
publishToClawHub?: boolean;
|
||||
publishToNpm?: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
export type PublishablePluginPackage = {
|
||||
extensionId: string;
|
||||
packageDir: string;
|
||||
packageName: string;
|
||||
version: string;
|
||||
channel: "stable" | "beta";
|
||||
publishTag: "latest" | "beta";
|
||||
};
|
||||
|
||||
export type PluginReleasePlanItem = PublishablePluginPackage & {
|
||||
alreadyPublished: boolean;
|
||||
};
|
||||
|
||||
export type PluginReleasePlan = {
|
||||
all: PluginReleasePlanItem[];
|
||||
candidates: PluginReleasePlanItem[];
|
||||
skippedPublished: PluginReleasePlanItem[];
|
||||
};
|
||||
|
||||
const CLAWHUB_DEFAULT_REGISTRY = "https://clawhub.ai";
|
||||
const SAFE_EXTENSION_ID_RE = /^[a-z0-9][a-z0-9._-]*$/;
|
||||
const CLAWHUB_SHARED_RELEASE_INPUT_PATHS = [
|
||||
".github/workflows/plugin-clawhub-release.yml",
|
||||
".github/actions/setup-node-env",
|
||||
"package.json",
|
||||
"pnpm-lock.yaml",
|
||||
"packages/plugin-package-contract/src/index.ts",
|
||||
"scripts/lib/npm-publish-plan.mjs",
|
||||
"scripts/lib/plugin-npm-release.ts",
|
||||
"scripts/lib/plugin-clawhub-release.ts",
|
||||
"scripts/openclaw-npm-release-check.ts",
|
||||
"scripts/plugin-clawhub-publish.sh",
|
||||
"scripts/plugin-clawhub-release-check.ts",
|
||||
"scripts/plugin-clawhub-release-plan.ts",
|
||||
] as const;
|
||||
|
||||
function getRegistryBaseUrl(explicit?: string) {
|
||||
return (
|
||||
explicit?.trim() ||
|
||||
process.env.CLAWHUB_REGISTRY?.trim() ||
|
||||
process.env.CLAWHUB_SITE?.trim() ||
|
||||
CLAWHUB_DEFAULT_REGISTRY
|
||||
);
|
||||
}
|
||||
|
||||
export function collectClawHubPublishablePluginPackages(
|
||||
rootDir = resolve("."),
|
||||
): PublishablePluginPackage[] {
|
||||
const publishable: PublishablePluginPackage[] = [];
|
||||
const validationErrors: string[] = [];
|
||||
|
||||
for (const candidate of collectExtensionPackageJsonCandidates(rootDir)) {
|
||||
const { extensionId, packageDir, packageJson } = candidate;
|
||||
if (packageJson.openclaw?.release?.publishToClawHub !== true) {
|
||||
continue;
|
||||
}
|
||||
if (!SAFE_EXTENSION_ID_RE.test(extensionId)) {
|
||||
validationErrors.push(
|
||||
`${extensionId}: extension directory name must match ^[a-z0-9][a-z0-9._-]*$ for ClawHub publish.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const errors = collectPublishablePluginPackageErrors({
|
||||
extensionId,
|
||||
packageDir,
|
||||
packageJson,
|
||||
});
|
||||
if (errors.length > 0) {
|
||||
validationErrors.push(...errors.map((error) => `${extensionId}: ${error}`));
|
||||
continue;
|
||||
}
|
||||
const contractValidation = validateExternalCodePluginPackageJson(packageJson);
|
||||
if (contractValidation.issues.length > 0) {
|
||||
validationErrors.push(
|
||||
...contractValidation.issues.map((issue) => `${extensionId}: ${issue.message}`),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const resolvedVersion = resolvePublishablePluginVersion({
|
||||
extensionId,
|
||||
packageJson,
|
||||
validationErrors,
|
||||
});
|
||||
if (!resolvedVersion) {
|
||||
continue;
|
||||
}
|
||||
const { version, parsedVersion } = resolvedVersion;
|
||||
|
||||
publishable.push({
|
||||
extensionId,
|
||||
packageDir,
|
||||
packageName: packageJson.name!.trim(),
|
||||
version,
|
||||
channel: parsedVersion.channel,
|
||||
publishTag: parsedVersion.channel === "beta" ? "beta" : "latest",
|
||||
});
|
||||
}
|
||||
|
||||
if (validationErrors.length > 0) {
|
||||
throw new Error(
|
||||
`Publishable ClawHub plugin metadata validation failed:\n${validationErrors.map((error) => `- ${error}`).join("\n")}`,
|
||||
);
|
||||
}
|
||||
|
||||
return publishable.toSorted((left, right) => left.packageName.localeCompare(right.packageName));
|
||||
}
|
||||
|
||||
export function collectPluginClawHubReleasePathsFromGitRange(params: {
|
||||
rootDir?: string;
|
||||
gitRange: GitRangeSelection;
|
||||
}): string[] {
|
||||
return collectPluginClawHubReleasePathsFromGitRangeForPathspecs(params, ["extensions"]);
|
||||
}
|
||||
|
||||
function collectPluginClawHubRelevantPathsFromGitRange(params: {
|
||||
rootDir?: string;
|
||||
gitRange: GitRangeSelection;
|
||||
}): string[] {
|
||||
return collectPluginClawHubReleasePathsFromGitRangeForPathspecs(params, [
|
||||
"extensions",
|
||||
...CLAWHUB_SHARED_RELEASE_INPUT_PATHS,
|
||||
]);
|
||||
}
|
||||
|
||||
function collectPluginClawHubReleasePathsFromGitRangeForPathspecs(
|
||||
params: {
|
||||
rootDir?: string;
|
||||
gitRange: GitRangeSelection;
|
||||
},
|
||||
pathspecs: readonly string[],
|
||||
): string[] {
|
||||
return collectChangedPathsFromGitRange({
|
||||
rootDir: params.rootDir,
|
||||
gitRange: params.gitRange,
|
||||
pathspecs,
|
||||
});
|
||||
}
|
||||
|
||||
function hasSharedClawHubReleaseInputChanges(changedPaths: readonly string[]) {
|
||||
return changedPaths.some((path) =>
|
||||
CLAWHUB_SHARED_RELEASE_INPUT_PATHS.some(
|
||||
(sharedPath) => path === sharedPath || path.startsWith(`${sharedPath}/`),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveChangedClawHubPublishablePluginPackages(params: {
|
||||
plugins: PublishablePluginPackage[];
|
||||
changedPaths: readonly string[];
|
||||
}): PublishablePluginPackage[] {
|
||||
return resolveChangedPublishablePluginPackages({
|
||||
plugins: params.plugins,
|
||||
changedExtensionIds: collectChangedExtensionIdsFromPaths(params.changedPaths),
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveSelectedClawHubPublishablePluginPackages(params: {
|
||||
plugins: PublishablePluginPackage[];
|
||||
selection?: string[];
|
||||
selectionMode?: PluginReleaseSelectionMode;
|
||||
gitRange?: GitRangeSelection;
|
||||
rootDir?: string;
|
||||
}): PublishablePluginPackage[] {
|
||||
if (params.selectionMode === "all-publishable") {
|
||||
return params.plugins;
|
||||
}
|
||||
if (params.selection && params.selection.length > 0) {
|
||||
return resolveSelectedPublishablePluginPackages({
|
||||
plugins: params.plugins,
|
||||
selection: params.selection,
|
||||
});
|
||||
}
|
||||
if (params.gitRange) {
|
||||
const changedPaths = collectPluginClawHubRelevantPathsFromGitRange({
|
||||
rootDir: params.rootDir,
|
||||
gitRange: params.gitRange,
|
||||
});
|
||||
if (hasSharedClawHubReleaseInputChanges(changedPaths)) {
|
||||
return params.plugins;
|
||||
}
|
||||
return resolveChangedClawHubPublishablePluginPackages({
|
||||
plugins: params.plugins,
|
||||
changedPaths,
|
||||
});
|
||||
}
|
||||
return params.plugins;
|
||||
}
|
||||
|
||||
function readPackageManifestAtGitRef(params: {
|
||||
rootDir?: string;
|
||||
ref: string;
|
||||
packageDir: string;
|
||||
}): PluginPackageJson | null {
|
||||
const rootDir = params.rootDir ?? resolve(".");
|
||||
const commitSha = resolveGitCommitSha(rootDir, params.ref, "ref");
|
||||
try {
|
||||
const raw = execFileSync("git", ["show", `${commitSha}:${params.packageDir}/package.json`], {
|
||||
cwd: rootDir,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
return JSON.parse(raw) as PluginPackageJson;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function collectClawHubVersionGateErrors(params: {
|
||||
plugins: PublishablePluginPackage[];
|
||||
gitRange: GitRangeSelection;
|
||||
rootDir?: string;
|
||||
}): string[] {
|
||||
const changedPaths = collectPluginClawHubReleasePathsFromGitRange({
|
||||
rootDir: params.rootDir,
|
||||
gitRange: params.gitRange,
|
||||
});
|
||||
const changedPlugins = resolveChangedClawHubPublishablePluginPackages({
|
||||
plugins: params.plugins,
|
||||
changedPaths,
|
||||
});
|
||||
|
||||
const errors: string[] = [];
|
||||
for (const plugin of changedPlugins) {
|
||||
const baseManifest = readPackageManifestAtGitRef({
|
||||
rootDir: params.rootDir,
|
||||
ref: params.gitRange.baseRef,
|
||||
packageDir: plugin.packageDir,
|
||||
});
|
||||
if (baseManifest?.openclaw?.release?.publishToClawHub !== true) {
|
||||
continue;
|
||||
}
|
||||
const baseVersion =
|
||||
typeof baseManifest.version === "string" && baseManifest.version.trim()
|
||||
? baseManifest.version.trim()
|
||||
: null;
|
||||
if (baseVersion === null || baseVersion !== plugin.version) {
|
||||
continue;
|
||||
}
|
||||
errors.push(
|
||||
`${plugin.packageName}@${plugin.version}: changed publishable plugin still has the same version in package.json.`,
|
||||
);
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
export async function isPluginVersionPublishedOnClawHub(
|
||||
packageName: string,
|
||||
version: string,
|
||||
options: {
|
||||
fetchImpl?: typeof fetch;
|
||||
registryBaseUrl?: string;
|
||||
} = {},
|
||||
): Promise<boolean> {
|
||||
const fetchImpl = options.fetchImpl ?? fetch;
|
||||
const url = new URL(
|
||||
`/api/v1/packages/${encodeURIComponent(packageName)}/versions/${encodeURIComponent(version)}`,
|
||||
getRegistryBaseUrl(options.registryBaseUrl),
|
||||
);
|
||||
const response = await fetchImpl(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (response.status === 404) {
|
||||
return false;
|
||||
}
|
||||
if (response.ok) {
|
||||
return true;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Failed to query ClawHub for ${packageName}@${version}: ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function collectPluginClawHubReleasePlan(params?: {
|
||||
rootDir?: string;
|
||||
selection?: string[];
|
||||
selectionMode?: PluginReleaseSelectionMode;
|
||||
gitRange?: GitRangeSelection;
|
||||
registryBaseUrl?: string;
|
||||
fetchImpl?: typeof fetch;
|
||||
}): Promise<PluginReleasePlan> {
|
||||
const allPublishable = collectClawHubPublishablePluginPackages(params?.rootDir);
|
||||
const selectedPublishable = resolveSelectedClawHubPublishablePluginPackages({
|
||||
plugins: allPublishable,
|
||||
selection: params?.selection,
|
||||
selectionMode: params?.selectionMode,
|
||||
gitRange: params?.gitRange,
|
||||
rootDir: params?.rootDir,
|
||||
});
|
||||
|
||||
const all = await Promise.all(
|
||||
selectedPublishable.map(async (plugin) =>
|
||||
Object.assign({}, plugin, {
|
||||
alreadyPublished: await isPluginVersionPublishedOnClawHub(
|
||||
plugin.packageName,
|
||||
plugin.version,
|
||||
{ registryBaseUrl: params?.registryBaseUrl, fetchImpl: params?.fetchImpl },
|
||||
),
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
all,
|
||||
candidates: all.filter((plugin) => !plugin.alreadyPublished),
|
||||
skippedPublished: all.filter((plugin) => plugin.alreadyPublished),
|
||||
};
|
||||
}
|
||||
476
openclaw/scripts/lib/plugin-npm-release.ts
Normal file
476
openclaw/scripts/lib/plugin-npm-release.ts
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { normalizeOptionalString } from "../../src/shared/string-coerce.ts";
|
||||
import { parseReleaseVersion } from "../openclaw-npm-release-check.ts";
|
||||
import { resolveNpmPublishPlan } from "./npm-publish-plan.mjs";
|
||||
|
||||
export type PluginPackageJson = {
|
||||
name?: string;
|
||||
version?: string;
|
||||
private?: boolean;
|
||||
openclaw?: {
|
||||
extensions?: string[];
|
||||
install?: {
|
||||
npmSpec?: string;
|
||||
};
|
||||
release?: {
|
||||
publishToNpm?: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
export type PublishablePluginPackage = {
|
||||
extensionId: string;
|
||||
packageDir: string;
|
||||
packageName: string;
|
||||
version: string;
|
||||
channel: "stable" | "beta";
|
||||
publishTag: "latest" | "beta";
|
||||
installNpmSpec?: string;
|
||||
};
|
||||
|
||||
export type PluginReleasePlanItem = PublishablePluginPackage & {
|
||||
alreadyPublished: boolean;
|
||||
};
|
||||
|
||||
export type PluginReleasePlan = {
|
||||
all: PluginReleasePlanItem[];
|
||||
candidates: PluginReleasePlanItem[];
|
||||
skippedPublished: PluginReleasePlanItem[];
|
||||
};
|
||||
|
||||
export type PluginReleaseSelectionMode = "selected" | "all-publishable";
|
||||
|
||||
export type GitRangeSelection = {
|
||||
baseRef: string;
|
||||
headRef: string;
|
||||
};
|
||||
|
||||
export type ParsedPluginReleaseArgs = {
|
||||
selection: string[];
|
||||
selectionMode?: PluginReleaseSelectionMode;
|
||||
pluginsFlagProvided: boolean;
|
||||
baseRef?: string;
|
||||
headRef?: string;
|
||||
};
|
||||
|
||||
export type PublishablePluginPackageCandidate<
|
||||
TPackageJson extends PluginPackageJson = PluginPackageJson,
|
||||
> = {
|
||||
extensionId: string;
|
||||
packageDir: string;
|
||||
packageJson: TPackageJson;
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- Release helper preserves caller-specific package.json shape.
|
||||
function readPluginPackageJson<TPackageJson extends PluginPackageJson = PluginPackageJson>(
|
||||
path: string,
|
||||
): TPackageJson {
|
||||
return JSON.parse(readFileSync(path, "utf8")) as TPackageJson;
|
||||
}
|
||||
|
||||
export function collectExtensionPackageJsonCandidates<
|
||||
TPackageJson extends PluginPackageJson = PluginPackageJson,
|
||||
>(rootDir = resolve(".")): PublishablePluginPackageCandidate<TPackageJson>[] {
|
||||
const extensionsDir = join(rootDir, "extensions");
|
||||
const dirs = readdirSync(extensionsDir, { withFileTypes: true }).filter((entry) =>
|
||||
entry.isDirectory(),
|
||||
);
|
||||
|
||||
const candidates: PublishablePluginPackageCandidate<TPackageJson>[] = [];
|
||||
for (const dir of dirs) {
|
||||
const packageDir = `extensions/${dir.name}`;
|
||||
const absolutePackageDir = join(extensionsDir, dir.name);
|
||||
const packageJsonPath = join(absolutePackageDir, "package.json");
|
||||
try {
|
||||
candidates.push({
|
||||
extensionId: dir.name,
|
||||
packageDir,
|
||||
packageJson: readPluginPackageJson<TPackageJson>(packageJsonPath),
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
export function resolvePublishablePluginVersion(params: {
|
||||
extensionId: string;
|
||||
packageJson: Pick<PluginPackageJson, "version">;
|
||||
validationErrors: string[];
|
||||
}): { version: string; parsedVersion: NonNullable<ReturnType<typeof parseReleaseVersion>> } | null {
|
||||
const version = params.packageJson.version?.trim() ?? "";
|
||||
const parsedVersion = parseReleaseVersion(version);
|
||||
if (parsedVersion === null) {
|
||||
params.validationErrors.push(
|
||||
`${params.extensionId}: package.json version must match YYYY.M.D, YYYY.M.D-N, or YYYY.M.D-beta.N; found "${version}".`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
return { version, parsedVersion };
|
||||
}
|
||||
|
||||
export function normalizeGitDiffPath(path: string): string {
|
||||
return path.trim().replaceAll("\\", "/");
|
||||
}
|
||||
|
||||
export function parsePluginReleaseSelection(value: string | undefined): string[] {
|
||||
if (!value?.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
...new Set(
|
||||
value
|
||||
.split(/[,\s]+/)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean),
|
||||
),
|
||||
].toSorted();
|
||||
}
|
||||
|
||||
export function parsePluginReleaseSelectionMode(
|
||||
value: string | undefined,
|
||||
): PluginReleaseSelectionMode {
|
||||
if (value === "selected" || value === "all-publishable") {
|
||||
return value;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Unknown selection mode: ${value ?? "<missing>"}. Expected "selected" or "all-publishable".`,
|
||||
);
|
||||
}
|
||||
|
||||
export function parsePluginReleaseArgs(argv: string[]): ParsedPluginReleaseArgs {
|
||||
let selection: string[] = [];
|
||||
let selectionMode: PluginReleaseSelectionMode | undefined;
|
||||
let pluginsFlagProvided = false;
|
||||
let baseRef: string | undefined;
|
||||
let headRef: string | undefined;
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--") {
|
||||
continue;
|
||||
}
|
||||
if (arg === "--plugins") {
|
||||
selection = parsePluginReleaseSelection(argv[index + 1]);
|
||||
pluginsFlagProvided = true;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--selection-mode") {
|
||||
selectionMode = parsePluginReleaseSelectionMode(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--base-ref") {
|
||||
baseRef = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--head-ref") {
|
||||
headRef = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
|
||||
if (pluginsFlagProvided && selection.length === 0) {
|
||||
throw new Error("`--plugins` must include at least one package name.");
|
||||
}
|
||||
if (selectionMode === "selected" && !pluginsFlagProvided) {
|
||||
throw new Error("`--selection-mode selected` requires `--plugins`.");
|
||||
}
|
||||
if (selectionMode === "all-publishable" && pluginsFlagProvided) {
|
||||
throw new Error("`--selection-mode all-publishable` must not be combined with `--plugins`.");
|
||||
}
|
||||
if (selection.length > 0 && (baseRef || headRef)) {
|
||||
throw new Error("Use either --plugins or --base-ref/--head-ref, not both.");
|
||||
}
|
||||
if (selectionMode && (baseRef || headRef)) {
|
||||
throw new Error("Use either --selection-mode or --base-ref/--head-ref, not both.");
|
||||
}
|
||||
if ((baseRef && !headRef) || (!baseRef && headRef)) {
|
||||
throw new Error("Both --base-ref and --head-ref are required together.");
|
||||
}
|
||||
|
||||
return { selection, selectionMode, pluginsFlagProvided, baseRef, headRef };
|
||||
}
|
||||
|
||||
export function collectPublishablePluginPackageErrors(
|
||||
candidate: PublishablePluginPackageCandidate,
|
||||
): string[] {
|
||||
const { packageJson } = candidate;
|
||||
const errors: string[] = [];
|
||||
const packageName = packageJson.name?.trim() ?? "";
|
||||
const packageVersion = packageJson.version?.trim() ?? "";
|
||||
const extensions = packageJson.openclaw?.extensions ?? [];
|
||||
|
||||
if (!packageName.startsWith("@openclaw/")) {
|
||||
errors.push(
|
||||
`package name must start with "@openclaw/"; found "${packageName || "<missing>"}".`,
|
||||
);
|
||||
}
|
||||
if (packageJson.private === true) {
|
||||
errors.push("package.json private must not be true.");
|
||||
}
|
||||
if (!packageVersion) {
|
||||
errors.push("package.json version must be non-empty.");
|
||||
} else if (parseReleaseVersion(packageVersion) === null) {
|
||||
errors.push(
|
||||
`package.json version must match YYYY.M.D, YYYY.M.D-N, or YYYY.M.D-beta.N; found "${packageVersion}".`,
|
||||
);
|
||||
}
|
||||
if (!Array.isArray(extensions) || extensions.length === 0) {
|
||||
errors.push("openclaw.extensions must contain at least one entry.");
|
||||
}
|
||||
if (extensions.some((entry) => typeof entry !== "string" || !entry.trim())) {
|
||||
errors.push("openclaw.extensions must contain only non-empty strings.");
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function collectPublishablePluginPackages(
|
||||
rootDir = resolve("."),
|
||||
): PublishablePluginPackage[] {
|
||||
const publishable: PublishablePluginPackage[] = [];
|
||||
const validationErrors: string[] = [];
|
||||
|
||||
for (const candidate of collectExtensionPackageJsonCandidates(rootDir)) {
|
||||
const { extensionId, packageDir, packageJson } = candidate;
|
||||
if (packageJson.openclaw?.release?.publishToNpm !== true) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const errors = collectPublishablePluginPackageErrors(candidate);
|
||||
if (errors.length > 0) {
|
||||
validationErrors.push(...errors.map((error) => `${extensionId}: ${error}`));
|
||||
continue;
|
||||
}
|
||||
|
||||
const resolvedVersion = resolvePublishablePluginVersion({
|
||||
extensionId,
|
||||
packageJson,
|
||||
validationErrors,
|
||||
});
|
||||
if (!resolvedVersion) {
|
||||
continue;
|
||||
}
|
||||
const { version, parsedVersion } = resolvedVersion;
|
||||
|
||||
publishable.push({
|
||||
extensionId,
|
||||
packageDir,
|
||||
packageName: packageJson.name!.trim(),
|
||||
version,
|
||||
channel: parsedVersion.channel,
|
||||
publishTag: resolveNpmPublishPlan(version).publishTag,
|
||||
installNpmSpec: normalizeOptionalString(packageJson.openclaw?.install?.npmSpec),
|
||||
});
|
||||
}
|
||||
|
||||
if (validationErrors.length > 0) {
|
||||
throw new Error(
|
||||
`Publishable plugin metadata validation failed:\n${validationErrors.map((error) => `- ${error}`).join("\n")}`,
|
||||
);
|
||||
}
|
||||
|
||||
return publishable.toSorted((left, right) => left.packageName.localeCompare(right.packageName));
|
||||
}
|
||||
|
||||
export function resolveSelectedPublishablePluginPackages(params: {
|
||||
plugins: PublishablePluginPackage[];
|
||||
selection: string[];
|
||||
}): PublishablePluginPackage[] {
|
||||
if (params.selection.length === 0) {
|
||||
return params.plugins;
|
||||
}
|
||||
|
||||
const byName = new Map(params.plugins.map((plugin) => [plugin.packageName, plugin]));
|
||||
const selected: PublishablePluginPackage[] = [];
|
||||
const missing: string[] = [];
|
||||
|
||||
for (const packageName of params.selection) {
|
||||
const plugin = byName.get(packageName);
|
||||
if (!plugin) {
|
||||
missing.push(packageName);
|
||||
continue;
|
||||
}
|
||||
selected.push(plugin);
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`Unknown or non-publishable plugin package selection: ${missing.join(", ")}.`);
|
||||
}
|
||||
|
||||
return selected;
|
||||
}
|
||||
|
||||
export function collectChangedExtensionIdsFromPaths(paths: readonly string[]): string[] {
|
||||
const extensionIds = new Set<string>();
|
||||
|
||||
for (const path of paths) {
|
||||
const normalized = path.trim().replaceAll("\\", "/");
|
||||
const match = /^extensions\/([^/]+)\//.exec(normalized);
|
||||
if (match?.[1]) {
|
||||
extensionIds.add(match[1]);
|
||||
}
|
||||
}
|
||||
|
||||
return [...extensionIds].toSorted();
|
||||
}
|
||||
|
||||
export function isNullGitRef(ref: string | undefined): boolean {
|
||||
return !ref || /^0+$/.test(ref);
|
||||
}
|
||||
|
||||
function assertSafeGitRef(ref: string, label: string): string {
|
||||
const trimmed = ref.trim();
|
||||
if (!trimmed || isNullGitRef(trimmed)) {
|
||||
throw new Error(`${label} is required.`);
|
||||
}
|
||||
if (
|
||||
trimmed.startsWith("-") ||
|
||||
trimmed.includes("\u0000") ||
|
||||
trimmed.includes("\r") ||
|
||||
trimmed.includes("\n")
|
||||
) {
|
||||
throw new Error(`${label} must be a normal git ref or commit SHA.`);
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
export function resolveGitCommitSha(rootDir: string, ref: string, label: string): string {
|
||||
const safeRef = assertSafeGitRef(ref, label);
|
||||
try {
|
||||
return execFileSync("git", ["rev-parse", "--verify", "--quiet", `${safeRef}^{commit}`], {
|
||||
cwd: rootDir,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
}).trim();
|
||||
} catch {
|
||||
throw new Error(`${label} is not a valid git commit ref: ${safeRef}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function collectChangedPathsFromGitRange(params: {
|
||||
rootDir?: string;
|
||||
gitRange: GitRangeSelection;
|
||||
pathspecs: readonly string[];
|
||||
}): string[] {
|
||||
const rootDir = params.rootDir ?? resolve(".");
|
||||
const { baseRef, headRef } = params.gitRange;
|
||||
|
||||
if (isNullGitRef(baseRef) || isNullGitRef(headRef)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const baseSha = resolveGitCommitSha(rootDir, baseRef, "baseRef");
|
||||
const headSha = resolveGitCommitSha(rootDir, headRef, "headRef");
|
||||
|
||||
return execFileSync(
|
||||
"git",
|
||||
["diff", "--name-only", "--diff-filter=ACMR", baseSha, headSha, "--", ...params.pathspecs],
|
||||
{
|
||||
cwd: rootDir,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
)
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((path) => normalizeGitDiffPath(path));
|
||||
}
|
||||
|
||||
export function collectChangedExtensionIdsFromGitRange(params: {
|
||||
rootDir?: string;
|
||||
gitRange: GitRangeSelection;
|
||||
}): string[] {
|
||||
return collectChangedExtensionIdsFromPaths(
|
||||
collectChangedPathsFromGitRange({
|
||||
rootDir: params.rootDir,
|
||||
gitRange: params.gitRange,
|
||||
pathspecs: ["extensions"],
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveChangedPublishablePluginPackages(params: {
|
||||
plugins: PublishablePluginPackage[];
|
||||
changedExtensionIds: readonly string[];
|
||||
}): PublishablePluginPackage[] {
|
||||
if (params.changedExtensionIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const changed = new Set(params.changedExtensionIds);
|
||||
return params.plugins.filter((plugin) => changed.has(plugin.extensionId));
|
||||
}
|
||||
|
||||
export function isPluginVersionPublished(packageName: string, version: string): boolean {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "openclaw-plugin-npm-view-"));
|
||||
const userconfigPath = join(tempDir, "npmrc");
|
||||
writeFileSync(userconfigPath, "");
|
||||
|
||||
try {
|
||||
execFileSync(
|
||||
"npm",
|
||||
["view", `${packageName}@${version}`, "version", "--userconfig", userconfigPath],
|
||||
{
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export function collectPluginReleasePlan(params?: {
|
||||
rootDir?: string;
|
||||
selection?: string[];
|
||||
selectionMode?: PluginReleaseSelectionMode;
|
||||
gitRange?: GitRangeSelection;
|
||||
}): PluginReleasePlan {
|
||||
const allPublishable = collectPublishablePluginPackages(params?.rootDir);
|
||||
const selectedPublishable =
|
||||
params?.selectionMode === "all-publishable"
|
||||
? allPublishable
|
||||
: params?.selection && params.selection.length > 0
|
||||
? resolveSelectedPublishablePluginPackages({
|
||||
plugins: allPublishable,
|
||||
selection: params.selection,
|
||||
})
|
||||
: params?.gitRange
|
||||
? resolveChangedPublishablePluginPackages({
|
||||
plugins: allPublishable,
|
||||
changedExtensionIds: collectChangedExtensionIdsFromGitRange({
|
||||
rootDir: params.rootDir,
|
||||
gitRange: params.gitRange,
|
||||
}),
|
||||
})
|
||||
: allPublishable;
|
||||
|
||||
const all = selectedPublishable.map((plugin) =>
|
||||
Object.assign({}, plugin, {
|
||||
alreadyPublished: isPluginVersionPublished(plugin.packageName, plugin.version),
|
||||
}),
|
||||
);
|
||||
|
||||
return {
|
||||
all,
|
||||
candidates: all.filter((plugin) => !plugin.alreadyPublished),
|
||||
skippedPublished: all.filter((plugin) => plugin.alreadyPublished),
|
||||
};
|
||||
}
|
||||
114
openclaw/scripts/lib/plugin-sdk-doc-metadata.ts
Normal file
114
openclaw/scripts/lib/plugin-sdk-doc-metadata.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
export type PluginSdkDocCategory =
|
||||
| "channel"
|
||||
| "core"
|
||||
| "legacy"
|
||||
| "provider"
|
||||
| "runtime"
|
||||
| "utilities";
|
||||
|
||||
export type PluginSdkDocMetadata = {
|
||||
category: PluginSdkDocCategory;
|
||||
};
|
||||
|
||||
export const pluginSdkDocMetadata = {
|
||||
index: {
|
||||
category: "legacy",
|
||||
},
|
||||
"channel-runtime": {
|
||||
category: "legacy",
|
||||
},
|
||||
core: {
|
||||
category: "core",
|
||||
},
|
||||
"approval-runtime": {
|
||||
category: "runtime",
|
||||
},
|
||||
"approval-auth-runtime": {
|
||||
category: "runtime",
|
||||
},
|
||||
"approval-client-runtime": {
|
||||
category: "runtime",
|
||||
},
|
||||
"approval-delivery-runtime": {
|
||||
category: "runtime",
|
||||
},
|
||||
"approval-native-runtime": {
|
||||
category: "runtime",
|
||||
},
|
||||
"approval-reply-runtime": {
|
||||
category: "runtime",
|
||||
},
|
||||
"plugin-entry": {
|
||||
category: "core",
|
||||
},
|
||||
"channel-actions": {
|
||||
category: "channel",
|
||||
},
|
||||
"channel-config-schema": {
|
||||
category: "channel",
|
||||
},
|
||||
"channel-contract": {
|
||||
category: "channel",
|
||||
},
|
||||
"channel-pairing": {
|
||||
category: "channel",
|
||||
},
|
||||
"channel-reply-pipeline": {
|
||||
category: "channel",
|
||||
},
|
||||
"channel-setup": {
|
||||
category: "channel",
|
||||
},
|
||||
"command-auth": {
|
||||
category: "channel",
|
||||
},
|
||||
"command-status": {
|
||||
category: "channel",
|
||||
},
|
||||
"command-status-runtime": {
|
||||
category: "runtime",
|
||||
},
|
||||
"secret-input": {
|
||||
category: "channel",
|
||||
},
|
||||
"webhook-ingress": {
|
||||
category: "channel",
|
||||
},
|
||||
"provider-onboard": {
|
||||
category: "provider",
|
||||
},
|
||||
opencode: {
|
||||
category: "provider",
|
||||
},
|
||||
"runtime-store": {
|
||||
category: "runtime",
|
||||
},
|
||||
"allow-from": {
|
||||
category: "utilities",
|
||||
},
|
||||
"reply-payload": {
|
||||
category: "utilities",
|
||||
},
|
||||
testing: {
|
||||
category: "utilities",
|
||||
},
|
||||
} as const satisfies Record<string, PluginSdkDocMetadata>;
|
||||
|
||||
export type PluginSdkDocEntrypoint = keyof typeof pluginSdkDocMetadata;
|
||||
|
||||
export const pluginSdkDocCategories = [
|
||||
"core",
|
||||
"channel",
|
||||
"provider",
|
||||
"runtime",
|
||||
"utilities",
|
||||
"legacy",
|
||||
] as const satisfies readonly PluginSdkDocCategory[];
|
||||
|
||||
export const pluginSdkDocEntrypoints = Object.keys(
|
||||
pluginSdkDocMetadata,
|
||||
) as PluginSdkDocEntrypoint[];
|
||||
|
||||
export function resolvePluginSdkDocImportSpecifier(entrypoint: PluginSdkDocEntrypoint): string {
|
||||
return entrypoint === "index" ? "openclaw/plugin-sdk" : `openclaw/plugin-sdk/${entrypoint}`;
|
||||
}
|
||||
13
openclaw/scripts/lib/plugin-sdk-entries.d.mts
Normal file
13
openclaw/scripts/lib/plugin-sdk-entries.d.mts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
export const pluginSdkEntrypoints: string[];
|
||||
export const pluginSdkSubpaths: string[];
|
||||
|
||||
export function buildPluginSdkEntrySources(): Record<string, string>;
|
||||
export function buildPluginSdkSpecifiers(): string[];
|
||||
export function buildPluginSdkPackageExports(): Record<
|
||||
string,
|
||||
{
|
||||
types: string;
|
||||
default: string;
|
||||
}
|
||||
>;
|
||||
export function listPluginSdkDistArtifacts(): string[];
|
||||
36
openclaw/scripts/lib/plugin-sdk-entries.mjs
Normal file
36
openclaw/scripts/lib/plugin-sdk-entries.mjs
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import pluginSdkEntryList from "./plugin-sdk-entrypoints.json" with { type: "json" };
|
||||
|
||||
export const pluginSdkEntrypoints = [...pluginSdkEntryList];
|
||||
|
||||
export const pluginSdkSubpaths = pluginSdkEntrypoints.filter((entry) => entry !== "index");
|
||||
|
||||
export function buildPluginSdkEntrySources() {
|
||||
return Object.fromEntries(
|
||||
pluginSdkEntrypoints.map((entry) => [entry, `src/plugin-sdk/${entry}.ts`]),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildPluginSdkSpecifiers() {
|
||||
return pluginSdkEntrypoints.map((entry) =>
|
||||
entry === "index" ? "openclaw/plugin-sdk" : `openclaw/plugin-sdk/${entry}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function buildPluginSdkPackageExports() {
|
||||
return Object.fromEntries(
|
||||
pluginSdkEntrypoints.map((entry) => [
|
||||
entry === "index" ? "./plugin-sdk" : `./plugin-sdk/${entry}`,
|
||||
{
|
||||
types: `./dist/plugin-sdk/${entry}.d.ts`,
|
||||
default: `./dist/plugin-sdk/${entry}.js`,
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
export function listPluginSdkDistArtifacts() {
|
||||
return pluginSdkEntrypoints.flatMap((entry) => [
|
||||
`dist/plugin-sdk/${entry}.js`,
|
||||
`dist/plugin-sdk/${entry}.d.ts`,
|
||||
]);
|
||||
}
|
||||
291
openclaw/scripts/lib/plugin-sdk-entrypoints.json
Normal file
291
openclaw/scripts/lib/plugin-sdk-entrypoints.json
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
[
|
||||
"index",
|
||||
"core",
|
||||
"lmstudio",
|
||||
"lmstudio-runtime",
|
||||
"provider-setup",
|
||||
"sandbox",
|
||||
"self-hosted-provider-setup",
|
||||
"routing",
|
||||
"runtime",
|
||||
"runtime-doctor",
|
||||
"runtime-env",
|
||||
"proxy-capture",
|
||||
"runtime-secret-resolution",
|
||||
"setup",
|
||||
"setup-adapter-runtime",
|
||||
"setup-runtime",
|
||||
"channel-setup",
|
||||
"channel-streaming",
|
||||
"setup-tools",
|
||||
"approval-auth-runtime",
|
||||
"approval-client-runtime",
|
||||
"approval-delivery-runtime",
|
||||
"approval-gateway-runtime",
|
||||
"approval-handler-adapter-runtime",
|
||||
"approval-handler-runtime",
|
||||
"channel-runtime-context",
|
||||
"approval-native-runtime",
|
||||
"approval-reply-runtime",
|
||||
"approval-runtime",
|
||||
"config-runtime",
|
||||
"config-schema",
|
||||
"reply-runtime",
|
||||
"reply-dispatch-runtime",
|
||||
"reply-reference",
|
||||
"reply-chunking",
|
||||
"reply-payload",
|
||||
"agent-media-payload",
|
||||
"inbound-reply-dispatch",
|
||||
"inbound-envelope",
|
||||
"channel-reply-pipeline",
|
||||
"channel-reply-options-runtime",
|
||||
"channel-runtime",
|
||||
"interactive-runtime",
|
||||
"outbound-media",
|
||||
"outbound-runtime",
|
||||
"poll-runtime",
|
||||
"infra-runtime",
|
||||
"runtime-config-snapshot",
|
||||
"runtime-group-policy",
|
||||
"ssrf-policy",
|
||||
"ssrf-runtime",
|
||||
"media-runtime",
|
||||
"media-mime",
|
||||
"media-generation-runtime",
|
||||
"conversation-binding-runtime",
|
||||
"conversation-runtime",
|
||||
"matrix-runtime-heavy",
|
||||
"matrix-runtime-shared",
|
||||
"thread-bindings-runtime",
|
||||
"thread-bindings-session-runtime",
|
||||
"text-runtime",
|
||||
"text-chunking",
|
||||
"agent-runtime",
|
||||
"simple-completion-runtime",
|
||||
"speech-core",
|
||||
"plugin-runtime",
|
||||
"channel-secret-basic-runtime",
|
||||
"channel-secret-runtime",
|
||||
"channel-secret-tts-runtime",
|
||||
"secret-ref-runtime",
|
||||
"secret-file-runtime",
|
||||
"security-runtime",
|
||||
"gateway-runtime",
|
||||
"github-copilot-login",
|
||||
"github-copilot-token",
|
||||
"cli-runtime",
|
||||
"cli-backend",
|
||||
"agent-harness",
|
||||
"hook-runtime",
|
||||
"host-runtime",
|
||||
"process-runtime",
|
||||
"windows-spawn",
|
||||
"acp-runtime",
|
||||
"acp-binding-runtime",
|
||||
"acp-binding-resolve-runtime",
|
||||
"lazy-runtime",
|
||||
"testing",
|
||||
"temp-path",
|
||||
"logging-core",
|
||||
"markdown-table-runtime",
|
||||
"account-helpers",
|
||||
"account-core",
|
||||
"account-id",
|
||||
"account-resolution",
|
||||
"account-resolution-runtime",
|
||||
"agent-config-primitives",
|
||||
"allow-from",
|
||||
"allowlist-config-edit",
|
||||
"bluebubbles",
|
||||
"bluebubbles-policy",
|
||||
"browser-cdp",
|
||||
"browser-config",
|
||||
"browser-config-runtime",
|
||||
"browser-config-support",
|
||||
"browser-control-auth",
|
||||
"browser-node-runtime",
|
||||
"browser-profiles",
|
||||
"browser-security-runtime",
|
||||
"browser-setup-tools",
|
||||
"browser-support",
|
||||
"boolean-param",
|
||||
"dangerous-name-runtime",
|
||||
"command-auth",
|
||||
"command-auth-native",
|
||||
"command-gating",
|
||||
"command-status",
|
||||
"command-status-runtime",
|
||||
"command-detection",
|
||||
"command-surface",
|
||||
"collection-runtime",
|
||||
"compat",
|
||||
"direct-dm",
|
||||
"direct-dm-access",
|
||||
"direct-dm-guard-policy",
|
||||
"device-bootstrap",
|
||||
"diagnostic-runtime",
|
||||
"diagnostics-otel",
|
||||
"diffs",
|
||||
"error-runtime",
|
||||
"extension-shared",
|
||||
"channel-config-helpers",
|
||||
"channel-config-writes",
|
||||
"channel-config-primitives",
|
||||
"channel-config-schema",
|
||||
"channel-actions",
|
||||
"channel-plugin-common",
|
||||
"channel-core",
|
||||
"channel-entry-contract",
|
||||
"channel-contract",
|
||||
"channel-feedback",
|
||||
"channel-inbound",
|
||||
"channel-inbound-roots",
|
||||
"channel-logging",
|
||||
"channel-location",
|
||||
"channel-mention-gating",
|
||||
"channel-lifecycle",
|
||||
"channel-pairing",
|
||||
"channel-pairing-paths",
|
||||
"channel-policy",
|
||||
"channel-send-result",
|
||||
"channel-targets",
|
||||
"context-visibility-runtime",
|
||||
"feishu",
|
||||
"feishu-conversation",
|
||||
"feishu-setup",
|
||||
"file-lock",
|
||||
"fetch-runtime",
|
||||
"runtime-fetch",
|
||||
"response-limit-runtime",
|
||||
"session-binding-runtime",
|
||||
"session-key-runtime",
|
||||
"session-store-runtime",
|
||||
"ssrf-dispatcher",
|
||||
"string-coerce-runtime",
|
||||
"group-access",
|
||||
"global-singleton",
|
||||
"directory-config-runtime",
|
||||
"directory-runtime",
|
||||
"googlechat",
|
||||
"googlechat-runtime-shared",
|
||||
"media-generation-runtime-shared",
|
||||
"image-generation",
|
||||
"image-generation-runtime",
|
||||
"image-generation-core",
|
||||
"music-generation",
|
||||
"music-generation-core",
|
||||
"video-generation",
|
||||
"video-generation-runtime",
|
||||
"video-generation-core",
|
||||
"irc",
|
||||
"irc-surface",
|
||||
"reply-history",
|
||||
"realtime-transcription",
|
||||
"realtime-voice",
|
||||
"media-understanding",
|
||||
"media-understanding-runtime",
|
||||
"messaging-targets",
|
||||
"request-url",
|
||||
"runtime-store",
|
||||
"json-store",
|
||||
"persistent-dedupe",
|
||||
"keyed-async-queue",
|
||||
"line",
|
||||
"line-core",
|
||||
"line-runtime",
|
||||
"line-surface",
|
||||
"llm-task",
|
||||
"matrix",
|
||||
"matrix-helper",
|
||||
"matrix-runtime-surface",
|
||||
"matrix-surface",
|
||||
"matrix-thread-bindings",
|
||||
"qa-runner-runtime",
|
||||
"mattermost",
|
||||
"mattermost-policy",
|
||||
"memory-core",
|
||||
"memory-core-engine-runtime",
|
||||
"memory-core-host-engine-embeddings",
|
||||
"memory-core-host-engine-foundation",
|
||||
"memory-core-host-engine-qmd",
|
||||
"memory-core-host-engine-storage",
|
||||
"memory-core-host-multimodal",
|
||||
"memory-core-host-query",
|
||||
"memory-core-host-secret",
|
||||
"memory-core-host-events",
|
||||
"memory-core-host-status",
|
||||
"memory-core-host-runtime-cli",
|
||||
"memory-core-host-runtime-core",
|
||||
"memory-core-host-runtime-files",
|
||||
"memory-host-core",
|
||||
"memory-host-events",
|
||||
"memory-host-files",
|
||||
"memory-host-markdown",
|
||||
"memory-host-search",
|
||||
"memory-host-status",
|
||||
"memory-lancedb",
|
||||
"msteams",
|
||||
"models-provider-runtime",
|
||||
"skill-commands-runtime",
|
||||
"native-command-config-runtime",
|
||||
"native-command-registry",
|
||||
"nextcloud-talk",
|
||||
"nostr",
|
||||
"qa-channel",
|
||||
"provider-auth",
|
||||
"provider-auth-runtime",
|
||||
"provider-auth-api-key",
|
||||
"provider-auth-result",
|
||||
"provider-auth-login",
|
||||
"plugin-entry",
|
||||
"provider-catalog-shared",
|
||||
"provider-entry",
|
||||
"provider-env-vars",
|
||||
"provider-http",
|
||||
"provider-model-types",
|
||||
"provider-model-shared",
|
||||
"opencode",
|
||||
"volc-model-catalog-shared",
|
||||
"provider-onboard",
|
||||
"provider-stream-family",
|
||||
"provider-stream-shared",
|
||||
"provider-transport-runtime",
|
||||
"provider-stream",
|
||||
"provider-tools",
|
||||
"provider-usage",
|
||||
"provider-web-fetch-contract",
|
||||
"provider-web-fetch",
|
||||
"provider-web-search-config-contract",
|
||||
"provider-web-search-contract",
|
||||
"provider-web-search",
|
||||
"retry-runtime",
|
||||
"run-command",
|
||||
"param-readers",
|
||||
"provider-zai-endpoint",
|
||||
"secret-input",
|
||||
"secret-input-runtime",
|
||||
"channel-status",
|
||||
"status-helpers",
|
||||
"speech",
|
||||
"string-normalization-runtime",
|
||||
"state-paths",
|
||||
"target-resolver-runtime",
|
||||
"telegram-command-config",
|
||||
"text-autolink-runtime",
|
||||
"thread-ownership",
|
||||
"tlon",
|
||||
"tool-payload",
|
||||
"tool-send",
|
||||
"twitch",
|
||||
"webhook-ingress",
|
||||
"webhook-targets",
|
||||
"webhook-request-guards",
|
||||
"webhook-path",
|
||||
"web-media",
|
||||
"voice-call",
|
||||
"zalo",
|
||||
"zalo-setup",
|
||||
"zalouser",
|
||||
"zod"
|
||||
]
|
||||
|
|
@ -0,0 +1 @@
|
|||
["qa-lab", "qa-runtime"]
|
||||
7
openclaw/scripts/lib/record-shared.mjs
Normal file
7
openclaw/scripts/lib/record-shared.mjs
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
export function isRecord(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function trimString(value) {
|
||||
return typeof value === "string" ? value.trim() : "";
|
||||
}
|
||||
146
openclaw/scripts/lib/run-extension-oxlint.mjs
Normal file
146
openclaw/scripts/lib/run-extension-oxlint.mjs
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
acquireLocalHeavyCheckLockSync,
|
||||
applyLocalOxlintPolicy,
|
||||
} from "./local-heavy-check-runtime.mjs";
|
||||
|
||||
export function runExtensionOxlint(params) {
|
||||
const repoRoot = process.cwd();
|
||||
const oxlintPath = path.resolve("node_modules", ".bin", "oxlint");
|
||||
const releaseLock = acquireLocalHeavyCheckLockSync({
|
||||
cwd: repoRoot,
|
||||
env: process.env,
|
||||
toolName: params.toolName,
|
||||
lockName: params.lockName,
|
||||
});
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), params.tempDirPrefix));
|
||||
const tempConfigPath = path.join(tempDir, "oxlint.json");
|
||||
let exitCode = 0;
|
||||
|
||||
try {
|
||||
prepareExtensionPackageBoundaryArtifacts(repoRoot);
|
||||
|
||||
const extensionFiles = params.roots.flatMap((root) =>
|
||||
collectTypeScriptFiles(path.resolve(repoRoot, root)),
|
||||
);
|
||||
|
||||
if (extensionFiles.length === 0) {
|
||||
console.error(params.emptyMessage);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
writeTempOxlintConfig(repoRoot, tempConfigPath);
|
||||
|
||||
const baseArgs = ["-c", tempConfigPath, ...process.argv.slice(2), ...extensionFiles];
|
||||
const { args: finalArgs, env } = applyLocalOxlintPolicy(baseArgs, process.env);
|
||||
const result = spawnSync(oxlintPath, finalArgs, {
|
||||
stdio: "inherit",
|
||||
env,
|
||||
shell: process.platform === "win32",
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
exitCode = result.status ?? 1;
|
||||
} finally {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
releaseLock();
|
||||
}
|
||||
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
|
||||
function prepareExtensionPackageBoundaryArtifacts(repoRoot) {
|
||||
const releaseLock = acquireLocalHeavyCheckLockSync({
|
||||
cwd: repoRoot,
|
||||
env: process.env,
|
||||
toolName: "extension-package-boundary-artifacts",
|
||||
lockName: "extension-package-boundary-artifacts",
|
||||
});
|
||||
let exitCode = 0;
|
||||
|
||||
try {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[path.resolve(repoRoot, "scripts", "prepare-extension-package-boundary-artifacts.mjs")],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
stdio: "inherit",
|
||||
env: process.env,
|
||||
},
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
exitCode = result.status ?? 1;
|
||||
} finally {
|
||||
releaseLock();
|
||||
}
|
||||
|
||||
if (exitCode !== 0) {
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
}
|
||||
|
||||
function writeTempOxlintConfig(repoRoot, configPath) {
|
||||
const config = JSON.parse(fs.readFileSync(path.resolve(repoRoot, ".oxlintrc.json"), "utf8"));
|
||||
|
||||
delete config.$schema;
|
||||
|
||||
if (Array.isArray(config.ignorePatterns)) {
|
||||
const extensionsIgnorePattern = config.ignorePatterns.find((pattern) =>
|
||||
isTopLevelExtensionsIgnorePattern(pattern),
|
||||
);
|
||||
if (extensionsIgnorePattern) {
|
||||
throw new Error(
|
||||
`Refusing to run extension oxlint with .oxlintrc.json ignore pattern ${JSON.stringify(
|
||||
extensionsIgnorePattern,
|
||||
)}. Remove the top-level extensions ignore so root and focused lint agree.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function isTopLevelExtensionsIgnorePattern(pattern) {
|
||||
const normalized = pattern
|
||||
.trim()
|
||||
.replaceAll("\\", "/")
|
||||
.replaceAll(/^\.?\//g, "");
|
||||
return (
|
||||
normalized === "extensions" || normalized === "extensions/" || normalized === "extensions/**"
|
||||
);
|
||||
}
|
||||
|
||||
function collectTypeScriptFiles(directoryPath) {
|
||||
const entries = fs.readdirSync(directoryPath, { withFileTypes: true });
|
||||
const files = [];
|
||||
|
||||
for (const entry of entries.toSorted((a, b) => a.name.localeCompare(b.name))) {
|
||||
const entryPath = path.join(directoryPath, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...collectTypeScriptFiles(entryPath));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.isFile()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.name.endsWith(".ts") && !entry.name.endsWith(".tsx")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
files.push(path.relative(process.cwd(), entryPath).split(path.sep).join("/"));
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
80
openclaw/scripts/lib/source-file-scan-cache.mjs
Normal file
80
openclaw/scripts/lib/source-file-scan-cache.mjs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const scanCache = new Map();
|
||||
|
||||
function normalizeRepoPath(repoRoot, filePath) {
|
||||
return path.relative(repoRoot, filePath).split(path.sep).join("/");
|
||||
}
|
||||
|
||||
async function walkFiles(params, rootDir) {
|
||||
const out = [];
|
||||
let entries = [];
|
||||
try {
|
||||
entries = await fs.readdir(rootDir, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
|
||||
return out;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
entries.sort((left, right) => left.name.localeCompare(right.name));
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(rootDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (!params.ignoredDirNames.has(entry.name)) {
|
||||
out.push(...(await walkFiles(params, entryPath)));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (entry.isFile() && params.scanExtensions.has(path.extname(entry.name))) {
|
||||
out.push(entryPath);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function collectSourceFileContents(params) {
|
||||
const cacheKey = JSON.stringify({
|
||||
repoRoot: params.repoRoot,
|
||||
scanRoots: params.scanRoots,
|
||||
scanExtensions: [...params.scanExtensions].toSorted((left, right) => left.localeCompare(right)),
|
||||
ignoredDirNames: [...params.ignoredDirNames].toSorted((left, right) =>
|
||||
left.localeCompare(right),
|
||||
),
|
||||
});
|
||||
const cached = scanCache.get(cacheKey);
|
||||
if (cached) {
|
||||
return await cached;
|
||||
}
|
||||
|
||||
const promise = (async () => {
|
||||
const files = (
|
||||
await Promise.all(
|
||||
params.scanRoots.map(async (root) => walkFiles(params, path.join(params.repoRoot, root))),
|
||||
)
|
||||
)
|
||||
.flat()
|
||||
.toSorted((left, right) =>
|
||||
normalizeRepoPath(params.repoRoot, left).localeCompare(
|
||||
normalizeRepoPath(params.repoRoot, right),
|
||||
),
|
||||
);
|
||||
|
||||
return await Promise.all(
|
||||
files.map(async (filePath) => ({
|
||||
filePath,
|
||||
relativeFile: normalizeRepoPath(params.repoRoot, filePath),
|
||||
content: await fs.readFile(filePath, "utf8"),
|
||||
})),
|
||||
);
|
||||
})();
|
||||
|
||||
scanCache.set(cacheKey, promise);
|
||||
try {
|
||||
return await promise;
|
||||
} catch (error) {
|
||||
scanCache.delete(cacheKey);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
486
openclaw/scripts/lib/test-group-report.mjs
Normal file
486
openclaw/scripts/lib/test-group-report.mjs
Normal file
|
|
@ -0,0 +1,486 @@
|
|||
import path from "node:path";
|
||||
import { collectVitestFileDurations, normalizeTrackedRepoPath } from "../test-report-utils.mjs";
|
||||
import { formatMs } from "./vitest-report-cli-utils.mjs";
|
||||
|
||||
export function formatBytesAsMb(valueBytes) {
|
||||
return valueBytes === null || valueBytes === undefined
|
||||
? "n/a"
|
||||
: `${(valueBytes / 1024 / 1024).toFixed(1)}MB`;
|
||||
}
|
||||
|
||||
export function formatSignedMs(value, digits = 1) {
|
||||
return `${value > 0 ? "+" : ""}${formatMs(value, digits)}`;
|
||||
}
|
||||
|
||||
export function formatSignedBytesAsMb(valueBytes) {
|
||||
return valueBytes === null || valueBytes === undefined
|
||||
? "n/a"
|
||||
: `${valueBytes > 0 ? "+" : ""}${formatBytesAsMb(valueBytes)}`;
|
||||
}
|
||||
|
||||
export function normalizeConfigLabel(config) {
|
||||
return config.replace(/^test\/vitest\/vitest\./u, "").replace(/\.config\.ts$/u, "");
|
||||
}
|
||||
|
||||
export function resolveTestArea(file) {
|
||||
const normalized = normalizeTrackedRepoPath(file);
|
||||
const parts = normalized.split("/");
|
||||
if (parts[0] === "extensions" && parts[1]) {
|
||||
return `extensions/${parts[1]}`;
|
||||
}
|
||||
if (parts[0] === "src" && parts[1]) {
|
||||
return `src/${parts[1]}`;
|
||||
}
|
||||
if (parts[0] === "packages" && parts[1]) {
|
||||
return `packages/${parts[1]}`;
|
||||
}
|
||||
if (parts[0] === "apps" && parts[1]) {
|
||||
return `apps/${parts[1]}`;
|
||||
}
|
||||
if (parts[0] === "ui") {
|
||||
return parts[3] ? `ui/${parts[3]}` : "ui";
|
||||
}
|
||||
if (parts[0] === "test" && parts[1]) {
|
||||
return `test/${parts[1]}`;
|
||||
}
|
||||
return parts[0] || normalized;
|
||||
}
|
||||
|
||||
export function resolveTestFolder(file, depth = 2) {
|
||||
const normalized = normalizeTrackedRepoPath(file);
|
||||
const dir = path.posix.dirname(normalized);
|
||||
if (dir === ".") {
|
||||
return normalized;
|
||||
}
|
||||
return dir.split("/").slice(0, Math.max(1, depth)).join("/");
|
||||
}
|
||||
|
||||
export function resolveGroupKey(file, mode = "area") {
|
||||
if (mode === "folder") {
|
||||
return resolveTestFolder(file, 3);
|
||||
}
|
||||
if (mode === "top") {
|
||||
return normalizeTrackedRepoPath(file).split("/")[0] || file;
|
||||
}
|
||||
return resolveTestArea(file);
|
||||
}
|
||||
|
||||
function createCounter(key) {
|
||||
return {
|
||||
key,
|
||||
durationMs: 0,
|
||||
fileCount: 0,
|
||||
testCount: 0,
|
||||
configs: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
function addFileEntry(target, entry, config) {
|
||||
target.durationMs += entry.durationMs;
|
||||
target.fileCount += 1;
|
||||
target.testCount += entry.testCount;
|
||||
target.configs.add(config);
|
||||
}
|
||||
|
||||
function finalizeCounter(counter) {
|
||||
return {
|
||||
key: counter.key,
|
||||
durationMs: counter.durationMs,
|
||||
fileCount: counter.fileCount,
|
||||
testCount: counter.testCount,
|
||||
configs: [...counter.configs].toSorted((left, right) => left.localeCompare(right)),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildGroupedTestReport(params) {
|
||||
const byGroup = new Map();
|
||||
const byConfig = new Map();
|
||||
const files = [];
|
||||
|
||||
for (const input of params.reports) {
|
||||
const config = normalizeConfigLabel(input.config);
|
||||
const fileEntries = collectVitestFileDurations(input.report, normalizeTrackedRepoPath);
|
||||
const configCounter = byConfig.get(config) ?? createCounter(config);
|
||||
byConfig.set(config, configCounter);
|
||||
|
||||
for (const entry of fileEntries) {
|
||||
const groupKey = resolveGroupKey(entry.file, params.groupBy);
|
||||
const groupCounter = byGroup.get(groupKey) ?? createCounter(groupKey);
|
||||
byGroup.set(groupKey, groupCounter);
|
||||
addFileEntry(groupCounter, entry, config);
|
||||
addFileEntry(configCounter, entry, config);
|
||||
files.push({ ...entry, config, group: groupKey });
|
||||
}
|
||||
}
|
||||
|
||||
const sortByDuration = (left, right) =>
|
||||
right.durationMs - left.durationMs || left.key.localeCompare(right.key);
|
||||
const sortFilesByDuration = (left, right) =>
|
||||
right.durationMs - left.durationMs || left.file.localeCompare(right.file);
|
||||
|
||||
const groups = [...byGroup.values()].map(finalizeCounter).toSorted(sortByDuration);
|
||||
const configs = [...byConfig.values()].map(finalizeCounter).toSorted(sortByDuration);
|
||||
const topFiles = files.toSorted(sortFilesByDuration);
|
||||
const totals = groups.reduce(
|
||||
(acc, group) => ({
|
||||
durationMs: acc.durationMs + group.durationMs,
|
||||
fileCount: acc.fileCount + group.fileCount,
|
||||
testCount: acc.testCount + group.testCount,
|
||||
}),
|
||||
{ durationMs: 0, fileCount: 0, testCount: 0 },
|
||||
);
|
||||
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
groupBy: params.groupBy,
|
||||
totals,
|
||||
groups,
|
||||
configs,
|
||||
topFiles,
|
||||
};
|
||||
}
|
||||
|
||||
function percentDelta(beforeValue, afterValue) {
|
||||
if (beforeValue === 0) {
|
||||
return afterValue === 0 ? 0 : null;
|
||||
}
|
||||
return ((afterValue - beforeValue) / beforeValue) * 100;
|
||||
}
|
||||
|
||||
function formatPercent(value) {
|
||||
if (value === null || value === undefined) {
|
||||
return "new";
|
||||
}
|
||||
return `${value > 0 ? "+" : ""}${value.toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function normalizeCounter(item) {
|
||||
return {
|
||||
durationMs: item?.durationMs ?? 0,
|
||||
fileCount: item?.fileCount ?? 0,
|
||||
testCount: item?.testCount ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
function compareStatus(beforeItem, afterItem) {
|
||||
if (beforeItem && afterItem) {
|
||||
return "changed";
|
||||
}
|
||||
return beforeItem ? "removed" : "added";
|
||||
}
|
||||
|
||||
function compareCounters(beforeItems = [], afterItems = []) {
|
||||
const beforeByKey = new Map(beforeItems.map((item) => [item.key, item]));
|
||||
const afterByKey = new Map(afterItems.map((item) => [item.key, item]));
|
||||
const keys = new Set([...beforeByKey.keys(), ...afterByKey.keys()]);
|
||||
|
||||
return [...keys]
|
||||
.map((key) => {
|
||||
const beforeItem = beforeByKey.get(key);
|
||||
const afterItem = afterByKey.get(key);
|
||||
const before = normalizeCounter(beforeItem);
|
||||
const after = normalizeCounter(afterItem);
|
||||
return {
|
||||
key,
|
||||
status: compareStatus(beforeItem, afterItem),
|
||||
before,
|
||||
after,
|
||||
delta: {
|
||||
durationMs: after.durationMs - before.durationMs,
|
||||
fileCount: after.fileCount - before.fileCount,
|
||||
testCount: after.testCount - before.testCount,
|
||||
},
|
||||
percent: {
|
||||
durationMs: percentDelta(before.durationMs, after.durationMs),
|
||||
},
|
||||
};
|
||||
})
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
Math.abs(right.delta.durationMs) - Math.abs(left.delta.durationMs) ||
|
||||
left.key.localeCompare(right.key),
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeFileCounter(item) {
|
||||
return {
|
||||
durationMs: item?.durationMs ?? 0,
|
||||
testCount: item?.testCount ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
function fileKey(item) {
|
||||
return `${item.config}\0${item.file}`;
|
||||
}
|
||||
|
||||
function compareFiles(beforeFiles = [], afterFiles = []) {
|
||||
const beforeByKey = new Map(beforeFiles.map((item) => [fileKey(item), item]));
|
||||
const afterByKey = new Map(afterFiles.map((item) => [fileKey(item), item]));
|
||||
const keys = new Set([...beforeByKey.keys(), ...afterByKey.keys()]);
|
||||
|
||||
return [...keys]
|
||||
.map((key) => {
|
||||
const beforeItem = beforeByKey.get(key);
|
||||
const afterItem = afterByKey.get(key);
|
||||
const before = normalizeFileCounter(beforeItem);
|
||||
const after = normalizeFileCounter(afterItem);
|
||||
const source = afterItem ?? beforeItem;
|
||||
return {
|
||||
key,
|
||||
config: source.config,
|
||||
file: source.file,
|
||||
group: source.group,
|
||||
status: compareStatus(beforeItem, afterItem),
|
||||
before,
|
||||
after,
|
||||
delta: {
|
||||
durationMs: after.durationMs - before.durationMs,
|
||||
testCount: after.testCount - before.testCount,
|
||||
},
|
||||
percent: {
|
||||
durationMs: percentDelta(before.durationMs, after.durationMs),
|
||||
},
|
||||
};
|
||||
})
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
Math.abs(right.delta.durationMs) - Math.abs(left.delta.durationMs) ||
|
||||
left.file.localeCompare(right.file) ||
|
||||
left.config.localeCompare(right.config),
|
||||
);
|
||||
}
|
||||
|
||||
function runKey(run) {
|
||||
return normalizeConfigLabel(run.config);
|
||||
}
|
||||
|
||||
function compareOptionalNumber(beforeValue, afterValue) {
|
||||
if (typeof beforeValue !== "number" || typeof afterValue !== "number") {
|
||||
return null;
|
||||
}
|
||||
return afterValue - beforeValue;
|
||||
}
|
||||
|
||||
function normalizeRun(run) {
|
||||
return run
|
||||
? {
|
||||
elapsedMs: typeof run.elapsedMs === "number" ? run.elapsedMs : null,
|
||||
maxRssBytes: typeof run.maxRssBytes === "number" ? run.maxRssBytes : null,
|
||||
status: typeof run.status === "number" ? run.status : null,
|
||||
}
|
||||
: {
|
||||
elapsedMs: null,
|
||||
maxRssBytes: null,
|
||||
status: null,
|
||||
};
|
||||
}
|
||||
|
||||
function compareRuns(beforeRuns = [], afterRuns = []) {
|
||||
const beforeByKey = new Map(beforeRuns.map((run) => [runKey(run), run]));
|
||||
const afterByKey = new Map(afterRuns.map((run) => [runKey(run), run]));
|
||||
const keys = new Set([...beforeByKey.keys(), ...afterByKey.keys()]);
|
||||
|
||||
return [...keys]
|
||||
.map((key) => {
|
||||
const beforeRun = beforeByKey.get(key);
|
||||
const afterRun = afterByKey.get(key);
|
||||
const before = normalizeRun(beforeRun);
|
||||
const after = normalizeRun(afterRun);
|
||||
return {
|
||||
key,
|
||||
status: compareStatus(beforeRun, afterRun),
|
||||
before,
|
||||
after,
|
||||
delta: {
|
||||
elapsedMs: compareOptionalNumber(before.elapsedMs, after.elapsedMs),
|
||||
maxRssBytes: compareOptionalNumber(before.maxRssBytes, after.maxRssBytes),
|
||||
},
|
||||
};
|
||||
})
|
||||
.toSorted((left, right) => {
|
||||
const leftMagnitude = Math.abs(left.delta.elapsedMs ?? left.delta.maxRssBytes ?? 0);
|
||||
const rightMagnitude = Math.abs(right.delta.elapsedMs ?? right.delta.maxRssBytes ?? 0);
|
||||
return rightMagnitude - leftMagnitude || left.key.localeCompare(right.key);
|
||||
});
|
||||
}
|
||||
|
||||
export function buildGroupedTestComparison(params) {
|
||||
const before = params.before;
|
||||
const after = params.after;
|
||||
const beforeTotals = normalizeCounter(before.totals);
|
||||
const afterTotals = normalizeCounter(after.totals);
|
||||
const warnings = [];
|
||||
|
||||
if (before.groupBy !== after.groupBy) {
|
||||
warnings.push(`groupBy differs: before=${before.groupBy} after=${after.groupBy}`);
|
||||
}
|
||||
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
command: "test-group-report:compare",
|
||||
groupBy: after.groupBy ?? before.groupBy,
|
||||
warnings,
|
||||
totals: {
|
||||
before: beforeTotals,
|
||||
after: afterTotals,
|
||||
delta: {
|
||||
durationMs: afterTotals.durationMs - beforeTotals.durationMs,
|
||||
fileCount: afterTotals.fileCount - beforeTotals.fileCount,
|
||||
testCount: afterTotals.testCount - beforeTotals.testCount,
|
||||
},
|
||||
percent: {
|
||||
durationMs: percentDelta(beforeTotals.durationMs, afterTotals.durationMs),
|
||||
},
|
||||
},
|
||||
groups: compareCounters(before.groups, after.groups),
|
||||
configs: compareCounters(before.configs, after.configs),
|
||||
files: compareFiles(before.topFiles, after.topFiles),
|
||||
runs: compareRuns(before.runs, after.runs),
|
||||
inputs: {
|
||||
before: params.beforePath ?? null,
|
||||
after: params.afterPath ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatCountDelta(value) {
|
||||
return `${value > 0 ? "+" : ""}${value}`;
|
||||
}
|
||||
|
||||
function formatOptionalMs(value) {
|
||||
return typeof value === "number" ? formatMs(value) : "n/a";
|
||||
}
|
||||
|
||||
function formatOptionalSignedMs(value) {
|
||||
return typeof value === "number" ? formatSignedMs(value) : "n/a";
|
||||
}
|
||||
|
||||
function formatOptionalBytes(value) {
|
||||
return typeof value === "number" ? formatBytesAsMb(value) : "n/a";
|
||||
}
|
||||
|
||||
function formatOptionalSignedBytes(value) {
|
||||
return typeof value === "number" ? formatSignedBytesAsMb(value) : "n/a";
|
||||
}
|
||||
|
||||
function pushChangeRows(lines, entries, options) {
|
||||
const selected = entries.slice(0, options.limit);
|
||||
if (selected.length === 0) {
|
||||
lines.push(" (none)");
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [index, entry] of selected.entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatSignedMs(entry.delta.durationMs).padStart(11, " ")} (${formatPercent(entry.percent.durationMs).padStart(7, " ")}) | before=${formatMs(entry.before.durationMs).padStart(10, " ")} after=${formatMs(entry.after.durationMs).padStart(10, " ")} | files=${formatCountDelta(entry.delta.fileCount ?? 0).padStart(4, " ")} tests=${formatCountDelta(entry.delta.testCount ?? 0).padStart(5, " ")} | ${entry.key}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function pushFileChangeRows(lines, entries, options) {
|
||||
const selected = entries.slice(0, options.limit);
|
||||
if (selected.length === 0) {
|
||||
lines.push(" (none)");
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [index, entry] of selected.entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatSignedMs(entry.delta.durationMs).padStart(11, " ")} (${formatPercent(entry.percent.durationMs).padStart(7, " ")}) | before=${formatMs(entry.before.durationMs).padStart(10, " ")} after=${formatMs(entry.after.durationMs).padStart(10, " ")} | tests=${formatCountDelta(entry.delta.testCount).padStart(4, " ")} | ${entry.config} | ${entry.file}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function renderGroupedTestComparison(comparison, options = {}) {
|
||||
const limit = options.limit ?? 25;
|
||||
const topFiles = options.topFiles ?? 25;
|
||||
const groupRegressions = comparison.groups.filter((entry) => entry.delta.durationMs > 0);
|
||||
const groupGains = comparison.groups.filter((entry) => entry.delta.durationMs < 0);
|
||||
const fileRegressions = comparison.files.filter((entry) => entry.delta.durationMs > 0);
|
||||
const fileGains = comparison.files.filter((entry) => entry.delta.durationMs < 0);
|
||||
const addedFiles = comparison.files.filter((entry) => entry.status === "added").length;
|
||||
const removedFiles = comparison.files.filter((entry) => entry.status === "removed").length;
|
||||
const lines = [
|
||||
`[test-group-report:compare] groupBy=${comparison.groupBy} file-sum=${formatMs(comparison.totals.before.durationMs)} -> ${formatMs(comparison.totals.after.durationMs)} (${formatSignedMs(comparison.totals.delta.durationMs)}, ${formatPercent(comparison.totals.percent.durationMs)}) files=${comparison.totals.before.fileCount}->${comparison.totals.after.fileCount} (${formatCountDelta(comparison.totals.delta.fileCount)}) tests=${comparison.totals.before.testCount}->${comparison.totals.after.testCount} (${formatCountDelta(comparison.totals.delta.testCount)}) addedFiles=${addedFiles} removedFiles=${removedFiles}`,
|
||||
];
|
||||
|
||||
for (const warning of comparison.warnings) {
|
||||
lines.push(`[test-group-report:compare] warning: ${warning}`);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Top group regressions (${Math.min(limit, groupRegressions.length)} of ${groupRegressions.length})`,
|
||||
);
|
||||
pushChangeRows(lines, groupRegressions, { limit });
|
||||
|
||||
lines.push("", `Top group gains (${Math.min(limit, groupGains.length)} of ${groupGains.length})`);
|
||||
pushChangeRows(lines, groupGains, { limit });
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Config duration deltas (${Math.min(limit, comparison.configs.length)} of ${comparison.configs.length})`,
|
||||
);
|
||||
pushChangeRows(lines, comparison.configs, { limit });
|
||||
|
||||
if (comparison.runs.length > 0) {
|
||||
lines.push(
|
||||
"",
|
||||
`Config wall/RSS deltas (${Math.min(limit, comparison.runs.length)} of ${comparison.runs.length})`,
|
||||
);
|
||||
for (const [index, run] of comparison.runs.slice(0, limit).entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. wall=${formatOptionalSignedMs(run.delta.elapsedMs).padStart(11, " ")} before=${formatOptionalMs(run.before.elapsedMs).padStart(10, " ")} after=${formatOptionalMs(run.after.elapsedMs).padStart(10, " ")} | rss=${formatOptionalSignedBytes(run.delta.maxRssBytes).padStart(10, " ")} before=${formatOptionalBytes(run.before.maxRssBytes).padStart(9, " ")} after=${formatOptionalBytes(run.after.maxRssBytes).padStart(9, " ")} | status=${run.before.status ?? "n/a"}->${run.after.status ?? "n/a"} | ${run.key}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Top file regressions (${Math.min(topFiles, fileRegressions.length)} of ${fileRegressions.length})`,
|
||||
);
|
||||
pushFileChangeRows(lines, fileRegressions, { limit: topFiles });
|
||||
|
||||
lines.push("", `Top file gains (${Math.min(topFiles, fileGains.length)} of ${fileGains.length})`);
|
||||
pushFileChangeRows(lines, fileGains, { limit: topFiles });
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function renderGroupedTestReport(report, options = {}) {
|
||||
const limit = options.limit ?? 25;
|
||||
const topFiles = options.topFiles ?? 25;
|
||||
const lines = [
|
||||
`[test-group-report] groupBy=${report.groupBy} files=${report.totals.fileCount} tests=${report.totals.testCount} file-sum=${formatMs(report.totals.durationMs)}`,
|
||||
"",
|
||||
`Top groups (${Math.min(limit, report.groups.length)} of ${report.groups.length})`,
|
||||
];
|
||||
|
||||
for (const [index, group] of report.groups.slice(0, limit).entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatMs(group.durationMs).padStart(10, " ")} | files=${String(group.fileCount).padStart(4, " ")} | tests=${String(group.testCount).padStart(5, " ")} | ${group.key}`,
|
||||
);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Top configs (${Math.min(limit, report.configs.length)} of ${report.configs.length})`,
|
||||
);
|
||||
for (const [index, config] of report.configs.slice(0, limit).entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatMs(config.durationMs).padStart(10, " ")} | files=${String(config.fileCount).padStart(4, " ")} | tests=${String(config.testCount).padStart(5, " ")} | ${config.key}`,
|
||||
);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Top files (${Math.min(topFiles, report.topFiles.length)} of ${report.topFiles.length})`,
|
||||
);
|
||||
for (const [index, file] of report.topFiles.slice(0, topFiles).entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatMs(file.durationMs).padStart(10, " ")} | tests=${String(file.testCount).padStart(4, " ")} | ${file.config} | ${file.file}`,
|
||||
);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
194
openclaw/scripts/lib/ts-guard-utils.mjs
Normal file
194
openclaw/scripts/lib/ts-guard-utils.mjs
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
import { existsSync, promises as fs } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
let tsCache;
|
||||
|
||||
function getTypeScript() {
|
||||
tsCache ??= require("typescript");
|
||||
return tsCache;
|
||||
}
|
||||
|
||||
const baseTestSuffixes = [".test.ts", ".test-utils.ts", ".test-harness.ts", ".e2e-harness.ts"];
|
||||
|
||||
export function resolveRepoRoot(importMetaUrl) {
|
||||
// Walk up from the caller's directory until we find the repo root (.git).
|
||||
// This handles callers at any depth (scripts/*.mjs, scripts/lib/*.mjs, etc.)
|
||||
// instead of assuming a fixed number of parent traversals.
|
||||
let dir = path.dirname(fileURLToPath(importMetaUrl));
|
||||
const { root } = path.parse(dir);
|
||||
while (dir !== root) {
|
||||
if (existsSync(path.join(dir, ".git"))) {
|
||||
return dir;
|
||||
}
|
||||
dir = path.dirname(dir);
|
||||
}
|
||||
// Fallback: two levels up (original behavior).
|
||||
return path.resolve(path.dirname(fileURLToPath(importMetaUrl)), "..", "..");
|
||||
}
|
||||
|
||||
export function resolveSourceRoots(repoRoot, relativeRoots) {
|
||||
return relativeRoots.map((root) => path.join(repoRoot, ...root.split("/").filter(Boolean)));
|
||||
}
|
||||
|
||||
export function isTestLikeTypeScriptFile(filePath, options = {}) {
|
||||
const extraTestSuffixes = options.extraTestSuffixes ?? [];
|
||||
return [...baseTestSuffixes, ...extraTestSuffixes].some((suffix) => filePath.endsWith(suffix));
|
||||
}
|
||||
|
||||
export async function collectTypeScriptFiles(targetPath, options = {}) {
|
||||
const includeTests = options.includeTests ?? false;
|
||||
const extraTestSuffixes = options.extraTestSuffixes ?? [];
|
||||
const skipNodeModules = options.skipNodeModules ?? true;
|
||||
const ignoreMissing = options.ignoreMissing ?? false;
|
||||
|
||||
let stat;
|
||||
try {
|
||||
stat = await fs.stat(targetPath);
|
||||
} catch (error) {
|
||||
if (
|
||||
ignoreMissing &&
|
||||
error &&
|
||||
typeof error === "object" &&
|
||||
"code" in error &&
|
||||
error.code === "ENOENT"
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (stat.isFile()) {
|
||||
if (!targetPath.endsWith(".ts")) {
|
||||
return [];
|
||||
}
|
||||
if (!includeTests && isTestLikeTypeScriptFile(targetPath, { extraTestSuffixes })) {
|
||||
return [];
|
||||
}
|
||||
return [targetPath];
|
||||
}
|
||||
|
||||
const entries = await fs.readdir(targetPath, { withFileTypes: true });
|
||||
const out = [];
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(targetPath, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (skipNodeModules && entry.name === "node_modules") {
|
||||
continue;
|
||||
}
|
||||
out.push(...(await collectTypeScriptFiles(entryPath, options)));
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile() || !entryPath.endsWith(".ts")) {
|
||||
continue;
|
||||
}
|
||||
if (!includeTests && isTestLikeTypeScriptFile(entryPath, { extraTestSuffixes })) {
|
||||
continue;
|
||||
}
|
||||
out.push(entryPath);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function collectTypeScriptFilesFromRoots(sourceRoots, options = {}) {
|
||||
return (
|
||||
await Promise.all(
|
||||
sourceRoots.map(
|
||||
async (root) =>
|
||||
await collectTypeScriptFiles(root, {
|
||||
ignoreMissing: true,
|
||||
...options,
|
||||
}),
|
||||
),
|
||||
)
|
||||
).flat();
|
||||
}
|
||||
|
||||
export async function collectFileViolations(params) {
|
||||
const files = await collectTypeScriptFilesFromRoots(params.sourceRoots, {
|
||||
extraTestSuffixes: params.extraTestSuffixes,
|
||||
});
|
||||
|
||||
const violations = [];
|
||||
for (const filePath of files) {
|
||||
if (params.skipFile?.(filePath)) {
|
||||
continue;
|
||||
}
|
||||
const content = await fs.readFile(filePath, "utf8");
|
||||
const fileViolations = params.findViolations(content, filePath);
|
||||
for (const violation of fileViolations) {
|
||||
violations.push({
|
||||
path: path.relative(params.repoRoot, filePath),
|
||||
...violation,
|
||||
});
|
||||
}
|
||||
}
|
||||
return violations;
|
||||
}
|
||||
|
||||
export function toLine(sourceFile, node) {
|
||||
return sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)).line + 1;
|
||||
}
|
||||
|
||||
export function getPropertyNameText(name) {
|
||||
const ts = getTypeScript();
|
||||
if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNumericLiteral(name)) {
|
||||
return name.text;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function unwrapExpression(expression) {
|
||||
const ts = getTypeScript();
|
||||
let current = expression;
|
||||
while (true) {
|
||||
if (ts.isParenthesizedExpression(current)) {
|
||||
current = current.expression;
|
||||
continue;
|
||||
}
|
||||
if (ts.isAsExpression(current) || ts.isTypeAssertionExpression(current)) {
|
||||
current = current.expression;
|
||||
continue;
|
||||
}
|
||||
if (ts.isNonNullExpression(current)) {
|
||||
current = current.expression;
|
||||
continue;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
}
|
||||
|
||||
export function collectCallExpressionLines(ts, sourceFile, resolveLineNode) {
|
||||
const lines = [];
|
||||
const visit = (node) => {
|
||||
if (ts.isCallExpression(node)) {
|
||||
const lineNode = resolveLineNode(node);
|
||||
if (lineNode) {
|
||||
lines.push(toLine(sourceFile, lineNode));
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
visit(sourceFile);
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function isDirectExecution(importMetaUrl) {
|
||||
const entry = process.argv[1];
|
||||
if (!entry) {
|
||||
return false;
|
||||
}
|
||||
return path.resolve(entry) === fileURLToPath(importMetaUrl);
|
||||
}
|
||||
|
||||
export function runAsScript(importMetaUrl, main) {
|
||||
if (!isDirectExecution(importMetaUrl)) {
|
||||
return;
|
||||
}
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
417
openclaw/scripts/lib/ts-topology/analyze.ts
Normal file
417
openclaw/scripts/lib/ts-topology/analyze.ts
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
import path from "node:path";
|
||||
import ts from "typescript";
|
||||
import {
|
||||
canonicalSymbolInfo,
|
||||
countIdentifierUsages,
|
||||
countNamespacePropertyUsages,
|
||||
createProgramContext,
|
||||
getRepoRevision,
|
||||
} from "./context.js";
|
||||
import type {
|
||||
ProgramContext,
|
||||
PublicEntrypoint,
|
||||
RankedCandidates,
|
||||
ReferenceEvent,
|
||||
TopologyEnvelope,
|
||||
TopologyRecord,
|
||||
TopologyReportName,
|
||||
TopologyScope,
|
||||
} from "./types.js";
|
||||
|
||||
function pushUnique(values: string[], next: string | null | undefined) {
|
||||
if (!next) {
|
||||
return;
|
||||
}
|
||||
if (!values.includes(next)) {
|
||||
values.push(next);
|
||||
}
|
||||
}
|
||||
|
||||
function sortUnique(values: string[]) {
|
||||
values.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function clampScore(value: number): number {
|
||||
return Math.max(0, Math.min(100, Math.round(value)));
|
||||
}
|
||||
|
||||
function isTypeOnlyCandidate(record: Pick<TopologyRecord, "kind">): boolean {
|
||||
return record.kind === "interface" || record.kind === "type";
|
||||
}
|
||||
|
||||
function computeSharednessScore(record: TopologyRecord): number {
|
||||
const extensionWeight = record.productionExtensions.length * 30;
|
||||
const packageWeight = record.productionPackages.length * 20;
|
||||
const internalWeight = record.internalRefCount > 0 ? 10 : 0;
|
||||
const publicSpecifierWeight = Math.min(record.publicSpecifiers.length, 4) * 5;
|
||||
const typeWeight = record.isTypeOnlyCandidate ? 10 : 0;
|
||||
const testOnlyPenalty = record.productionRefCount === 0 && record.testRefCount > 0 ? 25 : 0;
|
||||
return clampScore(
|
||||
extensionWeight +
|
||||
packageWeight +
|
||||
internalWeight +
|
||||
publicSpecifierWeight +
|
||||
typeWeight -
|
||||
testOnlyPenalty,
|
||||
);
|
||||
}
|
||||
|
||||
function computeMoveBackToOwnerScore(record: TopologyRecord): number {
|
||||
const singleExtensionWeight = record.productionExtensions.length === 1 ? 45 : 0;
|
||||
const noNonExtensionOwnersWeight = record.productionPackages.length === 0 ? 20 : 0;
|
||||
const runtimeWeight = record.isTypeOnlyCandidate ? 0 : 10;
|
||||
const usedWeight = record.productionRefCount > 0 ? 10 : 0;
|
||||
const publicSpecifierWeight = record.publicSpecifiers.length > 1 ? 5 : 0;
|
||||
const multiOwnerPenalty = record.productionOwners.length > 1 ? 35 : 0;
|
||||
const packagePenalty = record.productionPackages.length > 0 ? 25 : 0;
|
||||
return clampScore(
|
||||
singleExtensionWeight +
|
||||
noNonExtensionOwnersWeight +
|
||||
runtimeWeight +
|
||||
usedWeight +
|
||||
publicSpecifierWeight -
|
||||
multiOwnerPenalty -
|
||||
packagePenalty,
|
||||
);
|
||||
}
|
||||
|
||||
function createRecord(info: ReturnType<typeof canonicalSymbolInfo>): TopologyRecord {
|
||||
return {
|
||||
canonicalKey: info.canonicalKey,
|
||||
declarationPath: info.declarationPath,
|
||||
declarationLine: info.declarationLine,
|
||||
kind: info.kind,
|
||||
aliasName: info.aliasName,
|
||||
entrypoints: [],
|
||||
exportNames: [],
|
||||
publicSpecifiers: [],
|
||||
internalRefCount: 0,
|
||||
productionRefCount: 0,
|
||||
testRefCount: 0,
|
||||
internalImportCount: 0,
|
||||
productionImportCount: 0,
|
||||
testImportCount: 0,
|
||||
internalConsumers: [],
|
||||
productionConsumers: [],
|
||||
testConsumers: [],
|
||||
productionExtensions: [],
|
||||
productionPackages: [],
|
||||
productionOwners: [],
|
||||
isTypeOnlyCandidate: info.kind === "interface" || info.kind === "type",
|
||||
sharednessScore: 0,
|
||||
moveBackToOwnerScore: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function bucketConsumer(record: TopologyRecord, event: ReferenceEvent) {
|
||||
if (event.bucket === "internal") {
|
||||
record.internalImportCount += event.importCount;
|
||||
record.internalRefCount += event.usageCount;
|
||||
pushUnique(record.internalConsumers, event.consumerPath);
|
||||
return;
|
||||
}
|
||||
if (event.bucket === "test") {
|
||||
record.testImportCount += event.importCount;
|
||||
record.testRefCount += event.usageCount;
|
||||
pushUnique(record.testConsumers, event.consumerPath);
|
||||
return;
|
||||
}
|
||||
record.productionImportCount += event.importCount;
|
||||
record.productionRefCount += event.usageCount;
|
||||
pushUnique(record.productionConsumers, event.consumerPath);
|
||||
pushUnique(record.productionExtensions, event.extensionId);
|
||||
pushUnique(record.productionPackages, event.packageOwner);
|
||||
pushUnique(record.productionOwners, event.owner);
|
||||
}
|
||||
|
||||
function addEntrypointMetadata(
|
||||
record: TopologyRecord,
|
||||
entrypoint: PublicEntrypoint,
|
||||
exportName: string,
|
||||
aliasName?: string,
|
||||
) {
|
||||
pushUnique(record.entrypoints, entrypoint.entrypoint);
|
||||
pushUnique(record.exportNames, exportName);
|
||||
pushUnique(record.publicSpecifiers, entrypoint.importSpecifier);
|
||||
if (aliasName) {
|
||||
pushUnique(record.exportNames, aliasName);
|
||||
}
|
||||
}
|
||||
|
||||
function buildScopeMaps(context: ProgramContext, scope: TopologyScope) {
|
||||
const recordByCanonicalKey = new Map<string, TopologyRecord>();
|
||||
const recordBySpecifierAndExportName = new Map<string, Map<string, TopologyRecord>>();
|
||||
|
||||
for (const entrypoint of scope.entrypoints) {
|
||||
const absolutePath = path.join(context.repoRoot, entrypoint.sourcePath);
|
||||
const sourceFile = context.program.getSourceFile(absolutePath);
|
||||
if (!sourceFile) {
|
||||
continue;
|
||||
}
|
||||
const moduleSymbol = context.checker.getSymbolAtLocation(sourceFile);
|
||||
if (!moduleSymbol) {
|
||||
continue;
|
||||
}
|
||||
const exportMap = new Map<string, TopologyRecord>();
|
||||
for (const exportedSymbol of context.checker.getExportsOfModule(moduleSymbol)) {
|
||||
const info = canonicalSymbolInfo(context, exportedSymbol);
|
||||
let record = recordByCanonicalKey.get(info.canonicalKey);
|
||||
if (!record) {
|
||||
record = createRecord(info);
|
||||
recordByCanonicalKey.set(info.canonicalKey, record);
|
||||
}
|
||||
addEntrypointMetadata(record, entrypoint, exportedSymbol.getName(), info.aliasName);
|
||||
exportMap.set(exportedSymbol.getName(), record);
|
||||
}
|
||||
recordBySpecifierAndExportName.set(entrypoint.importSpecifier, exportMap);
|
||||
}
|
||||
|
||||
return { recordByCanonicalKey, recordBySpecifierAndExportName };
|
||||
}
|
||||
|
||||
function collectReferenceEvents(
|
||||
context: ProgramContext,
|
||||
scope: TopologyScope,
|
||||
recordBySpecifierAndExportName: Map<string, Map<string, TopologyRecord>>,
|
||||
includeTests: boolean,
|
||||
): ReferenceEvent[] {
|
||||
const events: ReferenceEvent[] = [];
|
||||
for (const sourceFile of context.program.getSourceFiles()) {
|
||||
if (sourceFile.isDeclarationFile) {
|
||||
continue;
|
||||
}
|
||||
const normalizedFileName = context.normalizePath(sourceFile.fileName);
|
||||
if (!normalizedFileName.startsWith(context.normalizePath(context.repoRoot))) {
|
||||
continue;
|
||||
}
|
||||
const relPath = context.relativeToRepo(sourceFile.fileName);
|
||||
const bucket = scope.classifyUsageBucket(relPath);
|
||||
if (!includeTests && bucket === "test") {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const statement of sourceFile.statements) {
|
||||
if (!ts.isImportDeclaration(statement) || !ts.isStringLiteral(statement.moduleSpecifier)) {
|
||||
continue;
|
||||
}
|
||||
const importSpecifier = statement.moduleSpecifier.text.trim();
|
||||
if (!scope.importFilter(importSpecifier)) {
|
||||
continue;
|
||||
}
|
||||
const recordMap = recordBySpecifierAndExportName.get(importSpecifier);
|
||||
if (!recordMap) {
|
||||
continue;
|
||||
}
|
||||
const clause = statement.importClause;
|
||||
if (!clause?.namedBindings) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ts.isNamedImports(clause.namedBindings)) {
|
||||
for (const element of clause.namedBindings.elements) {
|
||||
const importedName = element.propertyName?.text ?? element.name.text;
|
||||
const record = recordMap.get(importedName);
|
||||
if (!record) {
|
||||
continue;
|
||||
}
|
||||
const localSymbol = context.checker.getSymbolAtLocation(element.name);
|
||||
if (!localSymbol) {
|
||||
continue;
|
||||
}
|
||||
events.push({
|
||||
canonicalKey: record.canonicalKey,
|
||||
bucket,
|
||||
consumerPath: relPath,
|
||||
usageCount: countIdentifierUsages(context, sourceFile, localSymbol, element.name.text),
|
||||
importCount: 1,
|
||||
importSpecifier,
|
||||
owner: bucket === "production" ? scope.ownerForPath(relPath) : null,
|
||||
extensionId: bucket === "production" ? scope.extensionForPath(relPath) : null,
|
||||
packageOwner: bucket === "production" ? scope.packageOwnerForPath(relPath) : null,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ts.isNamespaceImport(clause.namedBindings)) {
|
||||
const namespaceSymbol = context.checker.getSymbolAtLocation(clause.namedBindings.name);
|
||||
if (!namespaceSymbol) {
|
||||
continue;
|
||||
}
|
||||
for (const [exportedName, record] of recordMap.entries()) {
|
||||
const usageCount = countNamespacePropertyUsages(
|
||||
context,
|
||||
sourceFile,
|
||||
namespaceSymbol,
|
||||
exportedName,
|
||||
);
|
||||
if (usageCount <= 0) {
|
||||
continue;
|
||||
}
|
||||
events.push({
|
||||
canonicalKey: record.canonicalKey,
|
||||
bucket,
|
||||
consumerPath: relPath,
|
||||
usageCount,
|
||||
importCount: 1,
|
||||
importSpecifier,
|
||||
owner: bucket === "production" ? scope.ownerForPath(relPath) : null,
|
||||
extensionId: bucket === "production" ? scope.extensionForPath(relPath) : null,
|
||||
packageOwner: bucket === "production" ? scope.packageOwnerForPath(relPath) : null,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
function finalizeRecords(records: TopologyRecord[]) {
|
||||
for (const record of records) {
|
||||
sortUnique(record.entrypoints);
|
||||
sortUnique(record.exportNames);
|
||||
sortUnique(record.publicSpecifiers);
|
||||
sortUnique(record.internalConsumers);
|
||||
sortUnique(record.productionConsumers);
|
||||
sortUnique(record.testConsumers);
|
||||
sortUnique(record.productionExtensions);
|
||||
sortUnique(record.productionPackages);
|
||||
sortUnique(record.productionOwners);
|
||||
record.isTypeOnlyCandidate = isTypeOnlyCandidate(record);
|
||||
record.sharednessScore = computeSharednessScore(record);
|
||||
record.moveBackToOwnerScore = computeMoveBackToOwnerScore(record);
|
||||
}
|
||||
return records.toSorted((left, right) => {
|
||||
const byRefs =
|
||||
right.productionRefCount +
|
||||
right.testRefCount +
|
||||
right.internalRefCount -
|
||||
(left.productionRefCount + left.testRefCount + left.internalRefCount);
|
||||
if (byRefs !== 0) {
|
||||
return byRefs;
|
||||
}
|
||||
return (
|
||||
left.publicSpecifiers[0].localeCompare(right.publicSpecifiers[0]) ||
|
||||
left.exportNames[0].localeCompare(right.exportNames[0])
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function buildRankedCandidates(records: TopologyRecord[], limit: number): RankedCandidates {
|
||||
return {
|
||||
candidateToMove: records
|
||||
.filter(
|
||||
(record) =>
|
||||
record.productionOwners.length === 1 &&
|
||||
record.productionExtensions.length === 1 &&
|
||||
record.productionRefCount > 0,
|
||||
)
|
||||
.toSorted((left, right) => right.moveBackToOwnerScore - left.moveBackToOwnerScore)
|
||||
.slice(0, limit),
|
||||
duplicatedPublicExports: records
|
||||
.filter((record) => record.publicSpecifiers.length > 1)
|
||||
.toSorted((left, right) => right.publicSpecifiers.length - left.publicSpecifiers.length)
|
||||
.slice(0, limit),
|
||||
singleOwnerShared: records
|
||||
.filter((record) => record.productionOwners.length === 1 && record.productionImportCount > 0)
|
||||
.toSorted((left, right) => right.productionRefCount - left.productionRefCount)
|
||||
.slice(0, limit),
|
||||
};
|
||||
}
|
||||
|
||||
export function analyzeTopology(options: {
|
||||
repoRoot: string;
|
||||
scope: TopologyScope;
|
||||
report: TopologyReportName;
|
||||
includeTests?: boolean;
|
||||
limit?: number;
|
||||
tsconfigName?: string;
|
||||
}): TopologyEnvelope {
|
||||
const includeTests = options.includeTests ?? true;
|
||||
const limit = options.limit ?? 25;
|
||||
const context = createProgramContext(options.repoRoot, options.tsconfigName);
|
||||
const { recordByCanonicalKey, recordBySpecifierAndExportName } = buildScopeMaps(
|
||||
context,
|
||||
options.scope,
|
||||
);
|
||||
const events = collectReferenceEvents(
|
||||
context,
|
||||
options.scope,
|
||||
recordBySpecifierAndExportName,
|
||||
includeTests,
|
||||
);
|
||||
for (const event of events) {
|
||||
const record = recordByCanonicalKey.get(event.canonicalKey);
|
||||
if (record) {
|
||||
bucketConsumer(record, event);
|
||||
}
|
||||
}
|
||||
const allRecords = finalizeRecords([...recordByCanonicalKey.values()]);
|
||||
const filteredRecords = filterRecordsForReport(allRecords, options.report);
|
||||
|
||||
return {
|
||||
metadata: {
|
||||
tool: "ts-topology",
|
||||
version: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
repoRevision: getRepoRevision(options.repoRoot),
|
||||
tsconfigPath: context.tsconfigPath,
|
||||
},
|
||||
scope: {
|
||||
id: options.scope.id,
|
||||
description: options.scope.description,
|
||||
repoRoot: options.repoRoot,
|
||||
entrypoints: options.scope.entrypoints,
|
||||
includeTests,
|
||||
},
|
||||
report: options.report,
|
||||
totals: {
|
||||
exports: allRecords.length,
|
||||
usedByProduction: allRecords.filter((record) => record.productionImportCount > 0).length,
|
||||
usedByTests: allRecords.filter((record) => record.testImportCount > 0).length,
|
||||
usedInternally: allRecords.filter((record) => record.internalImportCount > 0).length,
|
||||
singleOwnerShared: allRecords.filter(
|
||||
(record) => record.productionOwners.length === 1 && record.productionImportCount > 0,
|
||||
).length,
|
||||
unused: allRecords.filter(
|
||||
(record) =>
|
||||
record.productionImportCount === 0 &&
|
||||
record.testImportCount === 0 &&
|
||||
record.internalImportCount === 0,
|
||||
).length,
|
||||
},
|
||||
rankedCandidates: buildRankedCandidates(allRecords, limit),
|
||||
records: filteredRecords,
|
||||
};
|
||||
}
|
||||
|
||||
export function filterRecordsForReport(
|
||||
records: TopologyRecord[],
|
||||
report: TopologyReportName,
|
||||
): TopologyRecord[] {
|
||||
switch (report) {
|
||||
case "owner-map":
|
||||
return records.filter((record) => record.productionImportCount > 0);
|
||||
case "single-owner-shared":
|
||||
return records.filter(
|
||||
(record) => record.productionOwners.length === 1 && record.productionImportCount > 0,
|
||||
);
|
||||
case "unused-public-surface":
|
||||
return records.filter(
|
||||
(record) =>
|
||||
record.productionImportCount === 0 &&
|
||||
record.testImportCount === 0 &&
|
||||
record.internalImportCount === 0,
|
||||
);
|
||||
case "consumer-topology":
|
||||
return records.filter(
|
||||
(record) =>
|
||||
record.productionImportCount > 0 ||
|
||||
record.testImportCount > 0 ||
|
||||
record.internalImportCount > 0,
|
||||
);
|
||||
case "public-surface-usage":
|
||||
return records;
|
||||
}
|
||||
throw new Error("Unsupported topology report");
|
||||
}
|
||||
175
openclaw/scripts/lib/ts-topology/context.ts
Normal file
175
openclaw/scripts/lib/ts-topology/context.ts
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
import { execFileSync } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import ts from "typescript";
|
||||
import type { CanonicalSymbol, ProgramContext, SymbolKind } from "./types.js";
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizePath(filePath: string): string {
|
||||
return filePath.split(path.sep).join(path.posix.sep);
|
||||
}
|
||||
|
||||
export function createProgramContext(
|
||||
repoRoot: string,
|
||||
tsconfigName = "tsconfig.json",
|
||||
): ProgramContext {
|
||||
const configPath = ts.findConfigFile(
|
||||
repoRoot,
|
||||
(candidate) => ts.sys.fileExists(candidate),
|
||||
tsconfigName,
|
||||
);
|
||||
assert(configPath, `Could not find ${tsconfigName}`);
|
||||
const configFile = ts.readConfigFile(configPath, (candidate) => ts.sys.readFile(candidate));
|
||||
if (configFile.error) {
|
||||
throw new Error(ts.flattenDiagnosticMessageText(configFile.error.messageText, "\n"));
|
||||
}
|
||||
const parsed = ts.parseJsonConfigFileContent(configFile.config, ts.sys, repoRoot);
|
||||
const program = ts.createProgram(parsed.fileNames, parsed.options);
|
||||
return {
|
||||
repoRoot,
|
||||
tsconfigPath: normalizePath(path.relative(repoRoot, configPath)),
|
||||
program,
|
||||
checker: program.getTypeChecker(),
|
||||
normalizePath,
|
||||
relativeToRepo(filePath: string) {
|
||||
return normalizePath(path.relative(repoRoot, filePath));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function comparableSymbol(
|
||||
checker: ts.TypeChecker,
|
||||
symbol: ts.Symbol | undefined,
|
||||
): ts.Symbol | undefined {
|
||||
if (!symbol) {
|
||||
return undefined;
|
||||
}
|
||||
return symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol;
|
||||
}
|
||||
|
||||
function symbolKind(symbol: ts.Symbol, declaration: ts.Declaration | undefined): SymbolKind {
|
||||
if (declaration) {
|
||||
switch (declaration.kind) {
|
||||
case ts.SyntaxKind.FunctionDeclaration:
|
||||
return "function";
|
||||
case ts.SyntaxKind.ClassDeclaration:
|
||||
return "class";
|
||||
case ts.SyntaxKind.InterfaceDeclaration:
|
||||
return "interface";
|
||||
case ts.SyntaxKind.TypeAliasDeclaration:
|
||||
return "type";
|
||||
case ts.SyntaxKind.EnumDeclaration:
|
||||
return "enum";
|
||||
case ts.SyntaxKind.VariableDeclaration:
|
||||
return "variable";
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (symbol.flags & ts.SymbolFlags.Function) {
|
||||
return "function";
|
||||
}
|
||||
if (symbol.flags & ts.SymbolFlags.Class) {
|
||||
return "class";
|
||||
}
|
||||
if (symbol.flags & ts.SymbolFlags.Interface) {
|
||||
return "interface";
|
||||
}
|
||||
if (symbol.flags & ts.SymbolFlags.TypeAlias) {
|
||||
return "type";
|
||||
}
|
||||
if (symbol.flags & ts.SymbolFlags.Enum) {
|
||||
return "enum";
|
||||
}
|
||||
if (symbol.flags & ts.SymbolFlags.Variable) {
|
||||
return "variable";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
export function canonicalSymbolInfo(context: ProgramContext, symbol: ts.Symbol): CanonicalSymbol {
|
||||
const resolved = comparableSymbol(context.checker, symbol) ?? symbol;
|
||||
const declaration =
|
||||
resolved.getDeclarations()?.find((candidate) => candidate.kind !== ts.SyntaxKind.SourceFile) ??
|
||||
symbol.getDeclarations()?.find((candidate) => candidate.kind !== ts.SyntaxKind.SourceFile);
|
||||
assert(declaration, `Missing declaration for symbol ${symbol.getName()}`);
|
||||
const sourceFile = declaration.getSourceFile();
|
||||
const declarationPath = context.relativeToRepo(sourceFile.fileName);
|
||||
const declarationLine = sourceFile.getLineAndCharacterOfPosition(declaration.getStart()).line + 1;
|
||||
return {
|
||||
canonicalKey: `${declarationPath}:${declarationLine}:${resolved.getName()}`,
|
||||
declarationPath,
|
||||
declarationLine,
|
||||
kind: symbolKind(resolved, declaration),
|
||||
aliasName: symbol.getName() !== resolved.getName() ? symbol.getName() : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function countIdentifierUsages(
|
||||
context: ProgramContext,
|
||||
sourceFile: ts.SourceFile,
|
||||
importedSymbol: ts.Symbol,
|
||||
localName: string,
|
||||
): number {
|
||||
const targetSymbol = comparableSymbol(context.checker, importedSymbol);
|
||||
let count = 0;
|
||||
const visit = (node: ts.Node) => {
|
||||
if (ts.isIdentifier(node) && node.text === localName) {
|
||||
const symbol = comparableSymbol(context.checker, context.checker.getSymbolAtLocation(node));
|
||||
if (
|
||||
symbol === targetSymbol &&
|
||||
!ts.isImportClause(node.parent) &&
|
||||
!ts.isImportSpecifier(node.parent)
|
||||
) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
ts.forEachChild(sourceFile, visit);
|
||||
return count;
|
||||
}
|
||||
|
||||
export function countNamespacePropertyUsages(
|
||||
context: ProgramContext,
|
||||
sourceFile: ts.SourceFile,
|
||||
namespaceSymbol: ts.Symbol,
|
||||
exportedName: string,
|
||||
): number {
|
||||
const targetSymbol = comparableSymbol(context.checker, namespaceSymbol);
|
||||
let count = 0;
|
||||
const visit = (node: ts.Node) => {
|
||||
if (
|
||||
ts.isPropertyAccessExpression(node) &&
|
||||
ts.isIdentifier(node.expression) &&
|
||||
node.name.text === exportedName
|
||||
) {
|
||||
const symbol = comparableSymbol(
|
||||
context.checker,
|
||||
context.checker.getSymbolAtLocation(node.expression),
|
||||
);
|
||||
if (symbol === targetSymbol) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
ts.forEachChild(sourceFile, visit);
|
||||
return count;
|
||||
}
|
||||
|
||||
export function getRepoRevision(repoRoot: string): string | null {
|
||||
try {
|
||||
return execFileSync("git", ["rev-parse", "HEAD"], {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
}).trim();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
114
openclaw/scripts/lib/ts-topology/reports.ts
Normal file
114
openclaw/scripts/lib/ts-topology/reports.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
import type { ReportModule, TopologyEnvelope, TopologyRecord } from "./types.js";
|
||||
|
||||
function canonicalExportName(record: TopologyRecord): string {
|
||||
const finalColon = record.canonicalKey.lastIndexOf(":");
|
||||
return finalColon >= 0
|
||||
? record.canonicalKey.slice(finalColon + 1)
|
||||
: (record.exportNames[0] ?? "<unknown>");
|
||||
}
|
||||
|
||||
function primarySymbol(record: TopologyRecord): string {
|
||||
return `${record.publicSpecifiers[0] ?? "<unknown>"}:${canonicalExportName(record)}`;
|
||||
}
|
||||
|
||||
function formatRecordLine(record: TopologyRecord): string {
|
||||
return (
|
||||
`- ${primarySymbol(record)} -> ${record.declarationPath}:${record.declarationLine} ` +
|
||||
`(prodRefs=${record.productionRefCount}, owners=${record.productionOwners.join(",") || "-"}, ` +
|
||||
`sharedness=${record.sharednessScore}, move=${record.moveBackToOwnerScore})`
|
||||
);
|
||||
}
|
||||
|
||||
const reportModules: Record<ReportModule["name"], ReportModule> = {
|
||||
"public-surface-usage": {
|
||||
name: "public-surface-usage",
|
||||
describe(envelope, limit) {
|
||||
const candidates = envelope.rankedCandidates?.candidateToMove ?? [];
|
||||
const duplicateExports = envelope.rankedCandidates?.duplicatedPublicExports ?? [];
|
||||
return [
|
||||
`Scope: ${envelope.scope.id}`,
|
||||
`Public exports analyzed: ${envelope.totals.exports}`,
|
||||
`Production-used exports: ${envelope.totals.usedByProduction}`,
|
||||
`Single-owner shared exports: ${envelope.totals.singleOwnerShared}`,
|
||||
`Unused public exports: ${envelope.totals.unused}`,
|
||||
"",
|
||||
`Top ${Math.min(limit, candidates.length)} candidate-to-move exports:`,
|
||||
...candidates.slice(0, limit).map(formatRecordLine),
|
||||
"",
|
||||
`Top ${Math.min(limit, duplicateExports.length)} duplicated public exports:`,
|
||||
...duplicateExports
|
||||
.slice(0, limit)
|
||||
.map(
|
||||
(record) =>
|
||||
`- ${primarySymbol(record)} via ${record.publicSpecifiers.join(", ")} ` +
|
||||
`(${record.declarationPath}:${record.declarationLine})`,
|
||||
),
|
||||
].join("\n");
|
||||
},
|
||||
},
|
||||
"owner-map": {
|
||||
name: "owner-map",
|
||||
describe(envelope, limit) {
|
||||
return [
|
||||
`Scope: ${envelope.scope.id}`,
|
||||
`Production-owned records: ${envelope.records.length}`,
|
||||
"",
|
||||
`Top ${Math.min(limit, envelope.records.length)} owner-map records:`,
|
||||
...envelope.records
|
||||
.slice(0, limit)
|
||||
.map(
|
||||
(record) =>
|
||||
`- ${primarySymbol(record)} owners=${record.productionOwners.join(",")} ` +
|
||||
`extensions=${record.productionExtensions.join(",") || "-"} ` +
|
||||
`packages=${record.productionPackages.join(",") || "-"}`,
|
||||
),
|
||||
].join("\n");
|
||||
},
|
||||
},
|
||||
"single-owner-shared": {
|
||||
name: "single-owner-shared",
|
||||
describe(envelope, limit) {
|
||||
return [
|
||||
`Scope: ${envelope.scope.id}`,
|
||||
`Single-owner shared exports: ${envelope.records.length}`,
|
||||
"",
|
||||
`Top ${Math.min(limit, envelope.records.length)} single-owner shared exports:`,
|
||||
...envelope.records.slice(0, limit).map(formatRecordLine),
|
||||
].join("\n");
|
||||
},
|
||||
},
|
||||
"unused-public-surface": {
|
||||
name: "unused-public-surface",
|
||||
describe(envelope, limit) {
|
||||
return [
|
||||
`Scope: ${envelope.scope.id}`,
|
||||
`Unused public exports: ${envelope.records.length}`,
|
||||
"",
|
||||
`Top ${Math.min(limit, envelope.records.length)} unused exports:`,
|
||||
...envelope.records.slice(0, limit).map(formatRecordLine),
|
||||
].join("\n");
|
||||
},
|
||||
},
|
||||
"consumer-topology": {
|
||||
name: "consumer-topology",
|
||||
describe(envelope, limit) {
|
||||
return [
|
||||
`Scope: ${envelope.scope.id}`,
|
||||
`Records with consumers: ${envelope.records.length}`,
|
||||
"",
|
||||
`Top ${Math.min(limit, envelope.records.length)} consumer-topology records:`,
|
||||
...envelope.records
|
||||
.slice(0, limit)
|
||||
.map(
|
||||
(record) =>
|
||||
`- ${primarySymbol(record)} prod=${record.productionConsumers.length} ` +
|
||||
`test=${record.testConsumers.length} internal=${record.internalConsumers.length}`,
|
||||
),
|
||||
].join("\n");
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export function renderTextReport(envelope: TopologyEnvelope, limit: number): string {
|
||||
return reportModules[envelope.report].describe(envelope, limit);
|
||||
}
|
||||
162
openclaw/scripts/lib/ts-topology/scope.ts
Normal file
162
openclaw/scripts/lib/ts-topology/scope.ts
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { BUNDLED_PLUGIN_PATH_PREFIX } from "../bundled-plugin-paths.mjs";
|
||||
import { pluginSdkEntrypoints } from "../plugin-sdk-entries.mjs";
|
||||
import type { ConsumerScope, PublicEntrypoint, TopologyScope, UsageBucket } from "./types.js";
|
||||
|
||||
function isTestFile(relPath: string): boolean {
|
||||
return (
|
||||
relPath.startsWith("test/") ||
|
||||
relPath.includes("/__tests__/") ||
|
||||
relPath.includes(".test.") ||
|
||||
relPath.includes(".spec.") ||
|
||||
relPath.includes(".e2e.") ||
|
||||
relPath.includes(".suite.") ||
|
||||
relPath.includes("test-harness") ||
|
||||
relPath.includes("test-support") ||
|
||||
relPath.includes("test-helper") ||
|
||||
relPath.includes("test-utils")
|
||||
);
|
||||
}
|
||||
|
||||
function classifyScope(relPath: string): ConsumerScope {
|
||||
if (relPath.startsWith(BUNDLED_PLUGIN_PATH_PREFIX)) {
|
||||
return "extension";
|
||||
}
|
||||
if (relPath.startsWith("packages/")) {
|
||||
return "package";
|
||||
}
|
||||
if (relPath.startsWith("apps/")) {
|
||||
return "app";
|
||||
}
|
||||
if (relPath.startsWith("ui/")) {
|
||||
return "ui";
|
||||
}
|
||||
if (relPath.startsWith("scripts/")) {
|
||||
return "script";
|
||||
}
|
||||
if (relPath.startsWith("src/")) {
|
||||
return "src";
|
||||
}
|
||||
if (relPath.startsWith("test/")) {
|
||||
return "test";
|
||||
}
|
||||
return "other";
|
||||
}
|
||||
|
||||
function classifyUsageBucketForRoots(internalRoots: string[], relPath: string): UsageBucket {
|
||||
if (internalRoots.some((root) => relPath === root || relPath.startsWith(`${root}/`))) {
|
||||
return "internal";
|
||||
}
|
||||
return isTestFile(relPath) ? "test" : "production";
|
||||
}
|
||||
|
||||
function extractOwner(relPath: string): string | null {
|
||||
const scope = classifyScope(relPath);
|
||||
const parts = relPath.split("/");
|
||||
switch (scope) {
|
||||
case "extension":
|
||||
return parts[1] ? `extension:${parts[1]}` : "extension";
|
||||
case "package":
|
||||
return parts[1] ? `package:${parts[1]}` : "package";
|
||||
case "app":
|
||||
return parts[1] ? `app:${parts[1]}` : "app";
|
||||
case "src":
|
||||
return "src";
|
||||
case "ui":
|
||||
return "ui";
|
||||
case "script":
|
||||
return "scripts";
|
||||
case "other":
|
||||
return parts[0] || "other";
|
||||
case "test":
|
||||
return null;
|
||||
}
|
||||
throw new Error("Unsupported topology scope");
|
||||
}
|
||||
|
||||
function extractExtensionId(relPath: string): string | null {
|
||||
if (!relPath.startsWith(BUNDLED_PLUGIN_PATH_PREFIX)) {
|
||||
return null;
|
||||
}
|
||||
const parts = relPath.split("/");
|
||||
return parts[1] ?? null;
|
||||
}
|
||||
|
||||
function extractPackageOwner(relPath: string): string | null {
|
||||
const owner = extractOwner(relPath);
|
||||
return owner?.startsWith("extension:") ? null : owner;
|
||||
}
|
||||
|
||||
function buildScopeFromEntrypoints(
|
||||
id: string,
|
||||
description: string,
|
||||
entrypoints: PublicEntrypoint[],
|
||||
): TopologyScope {
|
||||
const internalRoots = [
|
||||
...new Set(entrypoints.map((entrypoint) => path.posix.dirname(entrypoint.sourcePath))),
|
||||
];
|
||||
const publicSpecifiers = new Set(entrypoints.map((entrypoint) => entrypoint.importSpecifier));
|
||||
return {
|
||||
id,
|
||||
description,
|
||||
entrypoints,
|
||||
importFilter(specifier: string) {
|
||||
return publicSpecifiers.has(specifier);
|
||||
},
|
||||
classifyUsageBucket(relPath: string) {
|
||||
return classifyUsageBucketForRoots(internalRoots, relPath);
|
||||
},
|
||||
classifyScope,
|
||||
ownerForPath(relPath: string) {
|
||||
return extractOwner(relPath);
|
||||
},
|
||||
extensionForPath(relPath: string) {
|
||||
return extractExtensionId(relPath);
|
||||
},
|
||||
packageOwnerForPath(relPath: string) {
|
||||
return extractPackageOwner(relPath);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createPluginSdkScope(_repoRoot: string): TopologyScope {
|
||||
const entrypoints = pluginSdkEntrypoints.map((entrypoint) => ({
|
||||
entrypoint,
|
||||
sourcePath: `src/plugin-sdk/${entrypoint}.ts`,
|
||||
importSpecifier:
|
||||
entrypoint === "index" ? "openclaw/plugin-sdk" : `openclaw/plugin-sdk/${entrypoint}`,
|
||||
}));
|
||||
return buildScopeFromEntrypoints("plugin-sdk", "OpenClaw plugin-sdk public surface", entrypoints);
|
||||
}
|
||||
|
||||
export function createFilesystemPublicSurfaceScope(
|
||||
repoRoot: string,
|
||||
options: {
|
||||
id: string;
|
||||
description?: string;
|
||||
entrypointRoot: string;
|
||||
importPrefix: string;
|
||||
},
|
||||
): TopologyScope {
|
||||
const absoluteRoot = path.join(repoRoot, options.entrypointRoot);
|
||||
const entries = fs
|
||||
.readdirSync(absoluteRoot, { withFileTypes: true })
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith(".ts"))
|
||||
.map((entry) => entry.name)
|
||||
.toSorted();
|
||||
const publicEntrypoints = entries.map((fileName) => {
|
||||
const entrypoint = fileName.replace(/\.ts$/, "");
|
||||
return {
|
||||
entrypoint,
|
||||
sourcePath: path.posix.join(options.entrypointRoot, fileName),
|
||||
importSpecifier:
|
||||
entrypoint === "index" ? options.importPrefix : `${options.importPrefix}/${entrypoint}`,
|
||||
};
|
||||
});
|
||||
return buildScopeFromEntrypoints(
|
||||
options.id,
|
||||
options.description ?? `Public surface rooted at ${options.entrypointRoot}`,
|
||||
publicEntrypoints,
|
||||
);
|
||||
}
|
||||
137
openclaw/scripts/lib/ts-topology/types.ts
Normal file
137
openclaw/scripts/lib/ts-topology/types.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import type ts from "typescript";
|
||||
|
||||
export type UsageBucket = "internal" | "production" | "test";
|
||||
|
||||
export type ConsumerScope =
|
||||
| "src"
|
||||
| "extension"
|
||||
| "package"
|
||||
| "app"
|
||||
| "ui"
|
||||
| "script"
|
||||
| "test"
|
||||
| "other";
|
||||
|
||||
export type TopologyReportName =
|
||||
| "public-surface-usage"
|
||||
| "owner-map"
|
||||
| "single-owner-shared"
|
||||
| "unused-public-surface"
|
||||
| "consumer-topology";
|
||||
|
||||
export type SymbolKind =
|
||||
| "function"
|
||||
| "class"
|
||||
| "interface"
|
||||
| "type"
|
||||
| "enum"
|
||||
| "variable"
|
||||
| "unknown";
|
||||
|
||||
export type ProgramContext = {
|
||||
repoRoot: string;
|
||||
tsconfigPath: string;
|
||||
program: ts.Program;
|
||||
checker: ts.TypeChecker;
|
||||
normalizePath: (filePath: string) => string;
|
||||
relativeToRepo: (filePath: string) => string;
|
||||
};
|
||||
|
||||
export type CanonicalSymbol = {
|
||||
canonicalKey: string;
|
||||
declarationPath: string;
|
||||
declarationLine: number;
|
||||
kind: SymbolKind;
|
||||
aliasName?: string;
|
||||
};
|
||||
|
||||
export type PublicEntrypoint = {
|
||||
entrypoint: string;
|
||||
sourcePath: string;
|
||||
importSpecifier: string;
|
||||
};
|
||||
|
||||
export type ReferenceEvent = {
|
||||
canonicalKey: string;
|
||||
bucket: UsageBucket;
|
||||
consumerPath: string;
|
||||
usageCount: number;
|
||||
importCount: number;
|
||||
importSpecifier: string;
|
||||
owner: string | null;
|
||||
extensionId: string | null;
|
||||
packageOwner: string | null;
|
||||
};
|
||||
|
||||
export type TopologyRecord = CanonicalSymbol & {
|
||||
entrypoints: string[];
|
||||
exportNames: string[];
|
||||
publicSpecifiers: string[];
|
||||
internalRefCount: number;
|
||||
productionRefCount: number;
|
||||
testRefCount: number;
|
||||
internalImportCount: number;
|
||||
productionImportCount: number;
|
||||
testImportCount: number;
|
||||
internalConsumers: string[];
|
||||
productionConsumers: string[];
|
||||
testConsumers: string[];
|
||||
productionExtensions: string[];
|
||||
productionPackages: string[];
|
||||
productionOwners: string[];
|
||||
isTypeOnlyCandidate: boolean;
|
||||
sharednessScore: number;
|
||||
moveBackToOwnerScore: number;
|
||||
};
|
||||
|
||||
export type TopologyScope = {
|
||||
id: string;
|
||||
description: string;
|
||||
entrypoints: PublicEntrypoint[];
|
||||
importFilter: (specifier: string) => boolean;
|
||||
classifyUsageBucket: (relPath: string) => UsageBucket;
|
||||
classifyScope: (relPath: string) => ConsumerScope;
|
||||
ownerForPath: (relPath: string) => string | null;
|
||||
extensionForPath: (relPath: string) => string | null;
|
||||
packageOwnerForPath: (relPath: string) => string | null;
|
||||
};
|
||||
|
||||
export type RankedCandidates = {
|
||||
candidateToMove: TopologyRecord[];
|
||||
duplicatedPublicExports: TopologyRecord[];
|
||||
singleOwnerShared: TopologyRecord[];
|
||||
};
|
||||
|
||||
export type TopologyEnvelope = {
|
||||
metadata: {
|
||||
tool: "ts-topology";
|
||||
version: 1;
|
||||
generatedAt: string;
|
||||
repoRevision: string | null;
|
||||
tsconfigPath: string;
|
||||
};
|
||||
scope: {
|
||||
id: string;
|
||||
description: string;
|
||||
repoRoot: string;
|
||||
entrypoints: PublicEntrypoint[];
|
||||
includeTests: boolean;
|
||||
};
|
||||
report: TopologyReportName;
|
||||
totals: {
|
||||
exports: number;
|
||||
usedByProduction: number;
|
||||
usedByTests: number;
|
||||
usedInternally: number;
|
||||
singleOwnerShared: number;
|
||||
unused: number;
|
||||
};
|
||||
rankedCandidates?: RankedCandidates;
|
||||
records: TopologyRecord[];
|
||||
};
|
||||
|
||||
export type ReportModule = {
|
||||
name: TopologyReportName;
|
||||
describe: (envelope: TopologyEnvelope, limit: number) => string;
|
||||
filterRecords?: (record: TopologyRecord) => boolean;
|
||||
};
|
||||
47
openclaw/scripts/lib/vitest-batch-runner.mjs
Normal file
47
openclaw/scripts/lib/vitest-batch-runner.mjs
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import { spawn } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import {
|
||||
installVitestProcessGroupCleanup,
|
||||
shouldUseDetachedVitestProcessGroup,
|
||||
} from "../vitest-process-group.mjs";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const repoRoot = path.resolve(__dirname, "../..");
|
||||
const pnpm = "pnpm";
|
||||
|
||||
export async function runVitestBatch(params) {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const child = spawn(
|
||||
pnpm,
|
||||
["exec", "vitest", "run", "--config", params.config, ...params.targets, ...params.args],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
detached: shouldUseDetachedVitestProcessGroup(),
|
||||
stdio: "inherit",
|
||||
shell: process.platform === "win32",
|
||||
env: params.env,
|
||||
},
|
||||
);
|
||||
const teardownChildCleanup = installVitestProcessGroupCleanup({ child });
|
||||
|
||||
child.on("error", (error) => {
|
||||
teardownChildCleanup();
|
||||
reject(error);
|
||||
});
|
||||
child.on("exit", (code, signal) => {
|
||||
teardownChildCleanup();
|
||||
if (signal) {
|
||||
process.kill(process.pid, signal);
|
||||
return;
|
||||
}
|
||||
resolve(code ?? 1);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function isDirectScriptRun(metaUrl) {
|
||||
const entryHref = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : "";
|
||||
return metaUrl === entryHref;
|
||||
}
|
||||
40
openclaw/scripts/lib/vitest-local-scheduling.d.mts
Normal file
40
openclaw/scripts/lib/vitest-local-scheduling.d.mts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
export type VitestHostInfo = {
|
||||
cpuCount?: number;
|
||||
loadAverage1m?: number;
|
||||
totalMemoryBytes?: number;
|
||||
};
|
||||
|
||||
export type LocalVitestScheduling = {
|
||||
maxWorkers: number;
|
||||
fileParallelism: boolean;
|
||||
throttledBySystem: boolean;
|
||||
};
|
||||
|
||||
export const DEFAULT_LOCAL_FULL_SUITE_PARALLELISM: number;
|
||||
export const LARGE_LOCAL_FULL_SUITE_PARALLELISM: number;
|
||||
export const DEFAULT_LOCAL_FULL_SUITE_VITEST_WORKERS: number;
|
||||
export const LARGE_LOCAL_FULL_SUITE_VITEST_WORKERS: number;
|
||||
|
||||
export function isCiLikeEnv(env?: Record<string, string | undefined>): boolean;
|
||||
export function detectVitestHostInfo(): Required<VitestHostInfo>;
|
||||
export function resolveLocalVitestMaxWorkers(
|
||||
env?: Record<string, string | undefined>,
|
||||
system?: VitestHostInfo,
|
||||
pool?: "forks" | "threads",
|
||||
): number;
|
||||
export function resolveLocalVitestScheduling(
|
||||
env?: Record<string, string | undefined>,
|
||||
system?: VitestHostInfo,
|
||||
pool?: "forks" | "threads",
|
||||
): LocalVitestScheduling;
|
||||
export function shouldUseLargeLocalFullSuiteProfile(
|
||||
env?: Record<string, string | undefined>,
|
||||
system?: VitestHostInfo,
|
||||
): boolean;
|
||||
export function resolveLocalFullSuiteProfile(
|
||||
env?: Record<string, string | undefined>,
|
||||
system?: VitestHostInfo,
|
||||
): {
|
||||
shardParallelism: number;
|
||||
vitestMaxWorkers: number;
|
||||
};
|
||||
162
openclaw/scripts/lib/vitest-local-scheduling.mjs
Normal file
162
openclaw/scripts/lib/vitest-local-scheduling.mjs
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
/** @typedef {{ cpuCount?: number, loadAverage1m?: number, totalMemoryBytes?: number }} VitestHostInfo */
|
||||
/** @typedef {{ maxWorkers: number, fileParallelism: boolean, throttledBySystem: boolean }} LocalVitestScheduling */
|
||||
|
||||
import os from "node:os";
|
||||
|
||||
export const DEFAULT_LOCAL_FULL_SUITE_PARALLELISM = 4;
|
||||
export const LARGE_LOCAL_FULL_SUITE_PARALLELISM = 10;
|
||||
export const DEFAULT_LOCAL_FULL_SUITE_VITEST_WORKERS = 1;
|
||||
export const LARGE_LOCAL_FULL_SUITE_VITEST_WORKERS = 2;
|
||||
|
||||
const clamp = (value, min, max) => Math.max(min, Math.min(max, value));
|
||||
|
||||
function parsePositiveInt(value) {
|
||||
const parsed = Number.parseInt(value ?? "", 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
||||
}
|
||||
|
||||
function isSystemThrottleDisabled(env) {
|
||||
const normalized = env.OPENCLAW_VITEST_DISABLE_SYSTEM_THROTTLE?.trim().toLowerCase();
|
||||
return normalized === "1" || normalized === "true";
|
||||
}
|
||||
|
||||
export function isCiLikeEnv(env = process.env) {
|
||||
return env.CI === "true" || env.GITHUB_ACTIONS === "true";
|
||||
}
|
||||
|
||||
export function detectVitestHostInfo() {
|
||||
return {
|
||||
cpuCount:
|
||||
typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length,
|
||||
loadAverage1m: os.loadavg()[0] ?? 0,
|
||||
totalMemoryBytes: os.totalmem(),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveLocalVitestMaxWorkers(
|
||||
env = process.env,
|
||||
system = detectVitestHostInfo(),
|
||||
pool = "threads",
|
||||
) {
|
||||
return resolveLocalVitestScheduling(env, system, pool).maxWorkers;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Record<string, string | undefined>} env
|
||||
* @param {VitestHostInfo} system
|
||||
* @param {"forks" | "threads"} pool
|
||||
* @returns {LocalVitestScheduling}
|
||||
*/
|
||||
export function resolveLocalVitestScheduling(
|
||||
env = process.env,
|
||||
system = detectVitestHostInfo(),
|
||||
pool = "threads",
|
||||
) {
|
||||
const override = parsePositiveInt(env.OPENCLAW_VITEST_MAX_WORKERS ?? env.OPENCLAW_TEST_WORKERS);
|
||||
if (override !== null) {
|
||||
const maxWorkers = clamp(override, 1, 16);
|
||||
return {
|
||||
maxWorkers,
|
||||
fileParallelism: maxWorkers > 1,
|
||||
throttledBySystem: false,
|
||||
};
|
||||
}
|
||||
|
||||
const cpuCount = Math.max(1, system.cpuCount ?? 1);
|
||||
const loadAverage1m = Math.max(0, system.loadAverage1m ?? 0);
|
||||
const totalMemoryGb = (system.totalMemoryBytes ?? 0) / 1024 ** 3;
|
||||
|
||||
let inferred =
|
||||
cpuCount <= 2
|
||||
? 1
|
||||
: cpuCount <= 4
|
||||
? 2
|
||||
: cpuCount <= 8
|
||||
? 4
|
||||
: Math.max(1, Math.floor(cpuCount * 0.75));
|
||||
|
||||
if (totalMemoryGb <= 16) {
|
||||
inferred = Math.min(inferred, 2);
|
||||
} else if (totalMemoryGb <= 32) {
|
||||
inferred = Math.min(inferred, 4);
|
||||
} else if (totalMemoryGb <= 64) {
|
||||
inferred = Math.min(inferred, 6);
|
||||
} else if (totalMemoryGb <= 128) {
|
||||
inferred = Math.min(inferred, 8);
|
||||
} else if (totalMemoryGb <= 256) {
|
||||
inferred = Math.min(inferred, 12);
|
||||
} else {
|
||||
inferred = Math.min(inferred, 16);
|
||||
}
|
||||
|
||||
const loadRatio = loadAverage1m > 0 ? loadAverage1m / cpuCount : 0;
|
||||
if (loadRatio >= 1) {
|
||||
inferred = Math.max(1, Math.floor(inferred / 2));
|
||||
} else if (loadRatio >= 0.75) {
|
||||
inferred = Math.max(1, inferred - 2);
|
||||
} else if (loadRatio >= 0.5) {
|
||||
inferred = Math.max(1, inferred - 1);
|
||||
}
|
||||
|
||||
if (pool === "forks") {
|
||||
inferred = Math.min(inferred, 8);
|
||||
}
|
||||
|
||||
inferred = clamp(inferred, 1, 16);
|
||||
|
||||
if (isSystemThrottleDisabled(env)) {
|
||||
return {
|
||||
maxWorkers: inferred,
|
||||
fileParallelism: true,
|
||||
throttledBySystem: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (loadRatio >= 1) {
|
||||
const maxWorkers = Math.max(1, Math.floor(inferred / 2));
|
||||
return {
|
||||
maxWorkers,
|
||||
fileParallelism: maxWorkers > 1,
|
||||
throttledBySystem: maxWorkers < inferred,
|
||||
};
|
||||
}
|
||||
|
||||
if (loadRatio >= 0.75) {
|
||||
const maxWorkers = Math.max(2, Math.ceil(inferred * 0.75));
|
||||
return {
|
||||
maxWorkers,
|
||||
fileParallelism: true,
|
||||
throttledBySystem: maxWorkers < inferred,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
maxWorkers: inferred,
|
||||
fileParallelism: true,
|
||||
throttledBySystem: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function shouldUseLargeLocalFullSuiteProfile(
|
||||
env = process.env,
|
||||
system = detectVitestHostInfo(),
|
||||
) {
|
||||
if (isCiLikeEnv(env)) {
|
||||
return false;
|
||||
}
|
||||
const scheduling = resolveLocalVitestScheduling(env, system, "threads");
|
||||
return scheduling.maxWorkers >= 5 && !scheduling.throttledBySystem;
|
||||
}
|
||||
|
||||
export function resolveLocalFullSuiteProfile(env = process.env, system = detectVitestHostInfo()) {
|
||||
if (shouldUseLargeLocalFullSuiteProfile(env, system)) {
|
||||
return {
|
||||
shardParallelism: LARGE_LOCAL_FULL_SUITE_PARALLELISM,
|
||||
vitestMaxWorkers: LARGE_LOCAL_FULL_SUITE_VITEST_WORKERS,
|
||||
};
|
||||
}
|
||||
return {
|
||||
shardParallelism: DEFAULT_LOCAL_FULL_SUITE_PARALLELISM,
|
||||
vitestMaxWorkers: DEFAULT_LOCAL_FULL_SUITE_VITEST_WORKERS,
|
||||
};
|
||||
}
|
||||
31
openclaw/scripts/lib/vitest-report-cli-utils.mjs
Normal file
31
openclaw/scripts/lib/vitest-report-cli-utils.mjs
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import { readJsonFile, runVitestJsonReport } from "../test-report-utils.mjs";
|
||||
import { intFlag, parseFlagArgs, stringFlag } from "./arg-utils.mjs";
|
||||
|
||||
export function parseVitestReportArgs(argv, defaults) {
|
||||
return parseFlagArgs(
|
||||
argv,
|
||||
{
|
||||
config: defaults.config,
|
||||
limit: defaults.limit,
|
||||
reportPath: defaults.reportPath ?? "",
|
||||
},
|
||||
[
|
||||
stringFlag("--config", "config"),
|
||||
intFlag("--limit", "limit", { min: 1 }),
|
||||
stringFlag("--report", "reportPath"),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
export function loadVitestReportFromArgs(args, prefix) {
|
||||
const reportPath = runVitestJsonReport({
|
||||
config: args.config,
|
||||
reportPath: args.reportPath,
|
||||
prefix,
|
||||
});
|
||||
return readJsonFile(reportPath);
|
||||
}
|
||||
|
||||
export function formatMs(value, digits = 1) {
|
||||
return `${value.toFixed(digits)}ms`;
|
||||
}
|
||||
112
openclaw/scripts/lib/workspace-bootstrap-smoke.mjs
Normal file
112
openclaw/scripts/lib/workspace-bootstrap-smoke.mjs
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdtempSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
export const WORKSPACE_TEMPLATE_PACK_PATHS = [
|
||||
"docs/reference/templates/AGENTS.md",
|
||||
"docs/reference/templates/SOUL.md",
|
||||
"docs/reference/templates/TOOLS.md",
|
||||
"docs/reference/templates/IDENTITY.md",
|
||||
"docs/reference/templates/USER.md",
|
||||
"docs/reference/templates/HEARTBEAT.md",
|
||||
"docs/reference/templates/BOOTSTRAP.md",
|
||||
];
|
||||
|
||||
const REQUIRED_BOOTSTRAP_WORKSPACE_FILES = [
|
||||
"AGENTS.md",
|
||||
"SOUL.md",
|
||||
"TOOLS.md",
|
||||
"IDENTITY.md",
|
||||
"USER.md",
|
||||
"HEARTBEAT.md",
|
||||
"BOOTSTRAP.md",
|
||||
];
|
||||
|
||||
function collectMissingBootstrapWorkspaceFiles(workspaceDir) {
|
||||
return REQUIRED_BOOTSTRAP_WORKSPACE_FILES.filter(
|
||||
(filename) => !existsSync(join(workspaceDir, filename)),
|
||||
);
|
||||
}
|
||||
|
||||
function describeExecFailure(error) {
|
||||
if (!(error instanceof Error)) {
|
||||
return String(error);
|
||||
}
|
||||
const stdout =
|
||||
typeof error.stdout === "string"
|
||||
? error.stdout.trim()
|
||||
: error.stdout instanceof Uint8Array
|
||||
? Buffer.from(error.stdout).toString("utf8").trim()
|
||||
: "";
|
||||
const stderr =
|
||||
typeof error.stderr === "string"
|
||||
? error.stderr.trim()
|
||||
: error.stderr instanceof Uint8Array
|
||||
? Buffer.from(error.stderr).toString("utf8").trim()
|
||||
: "";
|
||||
return [error.message, stdout, stderr].filter(Boolean).join(" | ");
|
||||
}
|
||||
|
||||
export function runInstalledWorkspaceBootstrapSmoke(params) {
|
||||
const tempRoot = mkdtempSync(join(tmpdir(), "openclaw-workspace-bootstrap-smoke-"));
|
||||
const homeDir = join(tempRoot, "home");
|
||||
const cwd = join(tempRoot, "cwd");
|
||||
mkdirSync(homeDir, { recursive: true });
|
||||
mkdirSync(cwd, { recursive: true });
|
||||
|
||||
let combinedOutput = "";
|
||||
try {
|
||||
try {
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
join(params.packageRoot, "openclaw.mjs"),
|
||||
"agent",
|
||||
"--message",
|
||||
"workspace bootstrap smoke",
|
||||
"--session-id",
|
||||
"workspace-bootstrap-smoke",
|
||||
"--local",
|
||||
"--timeout",
|
||||
"1",
|
||||
"--json",
|
||||
],
|
||||
{
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 1024 * 1024 * 16,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: homeDir,
|
||||
OPENCLAW_HOME: homeDir,
|
||||
OPENCLAW_SUPPRESS_NOTES: "1",
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
combinedOutput = describeExecFailure(error);
|
||||
}
|
||||
|
||||
if (combinedOutput.includes("Missing workspace template:")) {
|
||||
throw new Error(
|
||||
`installed workspace bootstrap failed before agent execution: ${combinedOutput}`,
|
||||
);
|
||||
}
|
||||
|
||||
const workspaceDir = join(homeDir, ".openclaw", "workspace");
|
||||
const missingFiles = collectMissingBootstrapWorkspaceFiles(workspaceDir);
|
||||
if (missingFiles.length > 0) {
|
||||
throw new Error(
|
||||
`installed workspace bootstrap did not create required files in ${workspaceDir}: ${missingFiles.join(", ")}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
} catch {
|
||||
// best effort cleanup only
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue