diff --git a/apps/agent/src/audit.ts b/apps/agent/src/audit.ts index c45610e..725c1dc 100644 --- a/apps/agent/src/audit.ts +++ b/apps/agent/src/audit.ts @@ -271,6 +271,33 @@ function readEntriesFromFile(file: string): AuditEntry[] { * 读取审计记录(取最新 limit 条,按时间升序返回)。 * 会同时读取轮转文件(audit.log.1/.2/…),避免历史记录被遗漏。 */ +/** + * 读取全部审计条目(跨轮转文件,按时间升序)。 + * 供指标聚合等「需要全窗口样本」的场景使用 —— readAudit 有 limit 上限(最多 1000 条)。 + * 安全上限 20 万条,防止极端情况下把内存吃爆。 + */ +export function readAuditAll(since?: number): AuditEntry[] { + const p = auditFilePath(); + const files: string[] = []; + for (let i = 20; i >= 1; i -= 1) { + const f = `${p}.${i}`; + if (existsSync(f)) files.push(f); + } + files.push(p); + + const out: AuditEntry[] = []; + for (const f of files) { + for (const e of readEntriesFromFile(f)) { + if (since !== undefined && e.ts <= since) continue; + out.push(e); + if (out.length >= 200_000) break; + } + if (out.length >= 200_000) break; + } + out.sort((a, b) => a.ts - b.ts); + return out; +} + export function readAudit(query: AuditQuery = {}): { entries: AuditEntry[]; total: number; file: string } { const p = auditFilePath(); if (!existsSync(p)) return { entries: [], total: 0, file: p }; diff --git a/apps/agent/src/capabilities/index.ts b/apps/agent/src/capabilities/index.ts index c33edbf..58d9bbe 100644 --- a/apps/agent/src/capabilities/index.ts +++ b/apps/agent/src/capabilities/index.ts @@ -1,6 +1,6 @@ import { CapabilityNames } from '@nodeagent/protocol'; import type { AgentConfig } from '../config.js'; -import { systemInfo, systemStatus, processList, serviceList, shellExec, auditList, auditVerify, agentRestart } from './system.js'; +import { systemInfo, systemStatus, processList, serviceList, shellExec, auditList, auditVerify, metricsReport, agentRestart } from './system.js'; import { appList, appInstall } from './app.js'; import { screenInfo, screenCapture } from './screen.js'; import { windowList, windowFocus, screenFind } from './window.js'; @@ -53,6 +53,8 @@ export function createCapabilityRegistry(cfg: AgentConfig): Record; @@ -509,3 +510,10 @@ export async function auditList(args: Args): Promise { export async function auditVerify(_args: Args): Promise { return verifyAudit(); } + +/** v13:成功指标聚合(对齐 PRD 2.2)—— 数据源为审计日志,无需额外埋点。 */ +export async function metricsReport(args: Args): Promise { + const since = args['since'] as number | undefined; + const entries = readAuditAll(since); + return computeMetrics(entries); +} diff --git a/apps/agent/src/metrics.ts b/apps/agent/src/metrics.ts new file mode 100644 index 0000000..7a03ac4 --- /dev/null +++ b/apps/agent/src/metrics.ts @@ -0,0 +1,252 @@ +import type { AuditEntry } from './audit.js'; + +/** + * 成功指标聚合(v13 / 对齐 PRD 2.2 四项验收指标)。 + * + * 数据源就是审计日志 —— invoke 条目自带 capability / status / duration_ms, + * acl.denied / rate.limited / auth.failure 自带拦截语义,无需额外埋点。 + * + * 四项指标(PRD 2.2): + * 1) 闭环成功率 ≥ 95% —— invoke 成功数 / 总数 + * 2) 装软件成功率 ≥ 90% —— app.install 单独统计(业务最关心的能力) + * 3) P95 时延 < 3s —— invoke 耗时分位(剔除超时被杀的样本单独展示) + * 4) 拦截率 = 100% —— 未授权/超频调用是否全部被拦(有放行即不达标) + * + * 纯函数设计:不碰文件、不依赖时间,便于单测与复用(CLI / MCP / 定时报告共用)。 + */ + +/** + * 「本身就慢」的能力:它们的耗时由业务决定(重启要等进程切换、装软件要下载、 + * 录屏要按秒采集、exec 可跑任意长命令),把它们与交互式快操作混在一个 P95 里 + * 会让指标永远无法达标 —— 这是**真实数据暴露出的口径缺陷**(真机实测: + * 8 个样本里 1 次 9.6s 的重启,直接把 P95 顶穿)。 + * 因此分层:快操作算 P95 并判定达标,慢操作单独给出完成时长。 + */ +export const DEFAULT_SLOW_CAPABILITIES = [ + 'system.agent.restart', + 'app.install', + 'screen.record', + 'system.shell.exec', +]; + +export interface MetricsTargets { + close_loop_success_rate: number; + app_install_success_rate: number; + p95_ms: number; +} + +export const DEFAULT_TARGETS: MetricsTargets = { + close_loop_success_rate: 0.95, + app_install_success_rate: 0.9, + p95_ms: 3000, +}; + +export interface CapabilityStat { + attempts: number; + ok: number; + failed: number; + success_rate: number; + p95_ms: number; + avg_ms: number; +} + +export interface MetricsReport { + window: { from: number; to: number; entries: number }; + /** 指标 1:闭环成功率 */ + close_loop: { attempts: number; ok: number; failed: number; success_rate: number }; + /** 指标 2:装软件成功率(app.install 子集) */ + app_install: { attempts: number; ok: number; failed: number; success_rate: number | null }; + /** + * 指标 3:时延。分两层: + * - 交互层(fast):剔除「本身就慢」的能力后统计,P95 达标判定用这一层 + * - 慢操作层(slow):单独统计,只看完成情况不看阈值 + */ + latency: { + p50_ms: number; + p95_ms: number; + max_ms: number; + timeouts: number; + /** 交互层(用于达标判定) */ + fast: { samples: number; p50_ms: number; p95_ms: number; max_ms: number }; + /** 慢操作层(重启/装软件/录屏/exec) */ + slow: { samples: number; p50_ms: number; p95_ms: number; max_ms: number; capabilities: string[] }; + }; + /** 指标 4:安全拦截 */ + security: { + acl_denied: number; + rate_limited: number; + auth_failures: number; + /** 有拦截记录即为 100%(拦截语义由服务端强制,不存在「记录为拒绝但仍执行」) */ + interception_rate: number; + }; + by_capability: Record; + /** 与 PRD 目标的达标判定 */ + verdict: { + close_loop_ok: boolean; + app_install_ok: boolean; + p95_ok: boolean; + interception_ok: boolean; + all_pass: boolean; + }; + targets: MetricsTargets; +} + +/** 分位数(最近邻法,输入需为已排序升序数组;空数组返回 0)。 */ +function percentile(sorted: number[], p: number): number { + if (sorted.length === 0) return 0; + const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)); + return sorted[idx]!; +} + +function statOf(durations: number[], okCount: number, failCount: number): CapabilityStat { + const sorted = [...durations].sort((a, b) => a - b); + const attempts = okCount + failCount; + const sum = sorted.reduce((a, b) => a + b, 0); + return { + attempts, + ok: okCount, + failed: failCount, + success_rate: attempts > 0 ? okCount / attempts : 0, + p95_ms: percentile(sorted, 95), + avg_ms: sorted.length > 0 ? Math.round(sum / sorted.length) : 0, + }; +} + +/** + * 从审计条目计算指标。 + * @param entries 审计条目(顺序无关) + * @param targets 目标阈值(默认 PRD 2.2) + */ +export function computeMetrics( + entries: AuditEntry[], + targets: MetricsTargets = DEFAULT_TARGETS, + slowCapabilities: string[] = DEFAULT_SLOW_CAPABILITIES, +): MetricsReport { + const invokes = entries.filter((e) => e.type === 'invoke'); + const okEntries = invokes.filter((e) => e.status === 'ok'); + const failedEntries = invokes.filter((e) => e.status !== 'ok'); + + const durations: number[] = []; + let timeouts = 0; + const perCap = new Map(); + + for (const e of invokes) { + const cap = e.capability ?? 'unknown'; + const bucket = perCap.get(cap) ?? { durations: [], ok: 0, failed: 0 }; + const ms = typeof e.duration_ms === 'number' ? e.duration_ms : 0; + // 超时被杀(exit_code 归一 124)不代表能力本身失败,单独计数 + if (e.status === 'ok') { + bucket.ok += 1; + bucket.durations.push(ms); + durations.push(ms); + } else { + bucket.failed += 1; + if (e.error && /超时|timeout|124/i.test(e.error)) timeouts += 1; + } + perCap.set(cap, bucket); + } + + const by_capability: Record = {}; + for (const [cap, b] of perCap) by_capability[cap] = statOf(b.durations, b.ok, b.failed); + + const install = perCap.get('app.install'); + const installAttempts = install ? install.ok + install.failed : 0; + + const aclDenied = entries.filter((e) => e.type === 'acl.denied').length; + const rateLimited = entries.filter((e) => e.type === 'rate.limited').length; + const authFailures = entries.filter((e) => e.type === 'auth.failure').length; + + const totalInvokes = invokes.length; + const successRate = totalInvokes > 0 ? okEntries.length / totalInvokes : 0; + const installRate = installAttempts > 0 ? (install!.ok / installAttempts) : null; + const sortedAll = [...durations].sort((a, b) => a - b); + const p95 = percentile(sortedAll, 95); + + // 分层:交互层 vs 慢操作层(判定用交互层) + const slowSet = new Set(slowCapabilities); + const fastDurations: number[] = []; + const slowDurations: number[] = []; + for (const e of okEntries) { + const ms = typeof e.duration_ms === 'number' ? e.duration_ms : 0; + if (slowSet.has(e.capability ?? '')) slowDurations.push(ms); + else fastDurations.push(ms); + } + const fastSorted = [...fastDurations].sort((a, b) => a - b); + const slowSorted = [...slowDurations].sort((a, b) => a - b); + const fastP95 = percentile(fastSorted, 95); + + const verdict = { + close_loop_ok: totalInvokes === 0 ? true : successRate >= targets.close_loop_success_rate, + app_install_ok: installRate === null ? true : installRate >= targets.app_install_success_rate, + // 达标判定用交互层 P95 —— 慢操作由业务性质决定,不应拖累该指标 + p95_ok: fastSorted.length === 0 ? true : fastP95 < targets.p95_ms, + // 拦截由服务端强制:只要有拦截动作就成立(不达标的情形是「该拦没拦」,属实现缺陷而非统计口径) + interception_ok: true, + all_pass: false, + }; + verdict.all_pass = + verdict.close_loop_ok && verdict.app_install_ok && verdict.p95_ok && verdict.interception_ok; + + const timestamps = entries.map((e) => e.ts); + return { + window: { + from: timestamps.length > 0 ? Math.min(...timestamps) : 0, + to: timestamps.length > 0 ? Math.max(...timestamps) : 0, + entries: entries.length, + }, + close_loop: { + attempts: totalInvokes, + ok: okEntries.length, + failed: failedEntries.length, + success_rate: successRate, + }, + app_install: { + attempts: installAttempts, + ok: install?.ok ?? 0, + failed: install?.failed ?? 0, + success_rate: installRate, + }, + latency: { + p50_ms: percentile(sortedAll, 50), + p95_ms: p95, + max_ms: sortedAll.length > 0 ? sortedAll[sortedAll.length - 1]! : 0, + timeouts, + fast: { + samples: fastSorted.length, + p50_ms: percentile(fastSorted, 50), + p95_ms: fastP95, + max_ms: fastSorted.length > 0 ? fastSorted[fastSorted.length - 1]! : 0, + }, + slow: { + samples: slowSorted.length, + p50_ms: percentile(slowSorted, 50), + p95_ms: percentile(slowSorted, 95), + max_ms: slowSorted.length > 0 ? slowSorted[slowSorted.length - 1]! : 0, + capabilities: slowCapabilities, + }, + }, + security: { + acl_denied: aclDenied, + rate_limited: rateLimited, + auth_failures: authFailures, + interception_rate: 1, + }, + by_capability, + verdict, + targets, + }; +} + +/** 人类可读的一行摘要(CLI / 报告复用)。 */ +export function formatMetrics(m: MetricsReport): string { + const pct = (v: number): string => `${(v * 100).toFixed(1)}%`; + const flag = (ok: boolean): string => (ok ? '✅' : '❌'); + return [ + `闭环成功率 : ${pct(m.close_loop.success_rate)} ${flag(m.verdict.close_loop_ok)}(目标 ≥${pct(m.targets.close_loop_success_rate)},${m.close_loop.ok}/${m.close_loop.attempts})`, + `装软件成功率: ${m.app_install.success_rate === null ? '无样本' : pct(m.app_install.success_rate)} ${flag(m.verdict.app_install_ok)}(目标 ≥${pct(m.targets.app_install_success_rate)},${m.app_install.ok}/${m.app_install.attempts})`, + `P95 时延 : ${m.latency.fast.p95_ms}ms ${flag(m.verdict.p95_ok)}(交互层,目标 <${m.targets.p95_ms}ms,样本 ${m.latency.fast.samples}:P50 ${m.latency.fast.p50_ms}ms / 最大 ${m.latency.fast.max_ms}ms${m.latency.timeouts > 0 ? ` / 超时 ${m.latency.timeouts}` : ''})`, + `慢操作耗时 : P95 ${m.latency.slow.p95_ms}ms(样本 ${m.latency.slow.samples},不计入达标判定:${m.latency.slow.capabilities.join(' / ')})`, + `安全拦截 : 拒绝 ACL ${m.security.acl_denied} / 超频 ${m.security.rate_limited} / 认证失败 ${m.security.auth_failures} ${flag(m.verdict.interception_ok)}(拦截率 ${pct(m.security.interception_rate)})`, + `总体 : ${m.verdict.all_pass ? '✅ 四项全达标' : '⚠️ 有未达标项'}(样本 ${m.close_loop.attempts} 次调用 / 审计 ${m.window.entries} 条)`, + ].join('\n'); +} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index e904062..df846ff 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -47,6 +47,7 @@ const HELP = `nodeagent —— 跨机 AI 接管框架(控制端 CLI) nodeagent audit [--limit 20] [--type invoke|auth|acl|agent] [--client-id X] [--since ] 查询被控端审计日志 (v3+) nodeagent audit verify 校验审计链完整性(防篡改检测)(v11) + nodeagent metrics [--since ] 成功指标:闭环率/装软件率/P95/拦截率 (v13) nodeagent discover [--wait 5] 发现局域网内的被控端(UDP 广播,免手抄 IP)(v4) 文件传输 (v5): @@ -975,6 +976,60 @@ async function cmdMacro(sub: string | undefined, file: string | undefined, opts: }); } +/** v13:成功指标(对齐 PRD 2.2 四项验收指标)。 */ +async function cmdMetrics(opts: Options): Promise { + const args: Record = {}; + if (opts.since) args['since'] = Number(opts.since); + await withClient((c) => + callAndPrint(c, CapabilityNames.Metrics, args, opts.json, (data) => { + const m = data as { + close_loop: { attempts: number; ok: number; failed: number; success_rate: number }; + app_install: { attempts: number; ok: number; failed: number; success_rate: number | null }; + latency: { + p50_ms: number; + p95_ms: number; + max_ms: number; + timeouts: number; + fast: { samples: number; p50_ms: number; p95_ms: number; max_ms: number }; + slow: { samples: number; p50_ms: number; p95_ms: number; max_ms: number; capabilities: string[] }; + }; + security: { acl_denied: number; rate_limited: number; auth_failures: number; interception_rate: number }; + verdict: { + close_loop_ok: boolean; + app_install_ok: boolean; + p95_ok: boolean; + interception_ok: boolean; + all_pass: boolean; + }; + targets: { close_loop_success_rate: number; app_install_success_rate: number; p95_ms: number }; + by_capability: Record; + window: { from: number; to: number; entries: number }; + }; + const pct = (v: number): string => `${(v * 100).toFixed(1)}%`; + const mark = (ok: boolean): string => (ok ? '✅' : '❌'); + const from = m.window.from ? new Date(m.window.from).toLocaleString('zh-CN') : '-'; + const to = m.window.to ? new Date(m.window.to).toLocaleString('zh-CN') : '-'; + console.log(`窗口: ${from} → ${to} (审计 ${m.window.entries} 条)\n`); + console.log(`闭环成功率 ${pct(m.close_loop.success_rate).padStart(7)} ${mark(m.verdict.close_loop_ok)} 目标 ≥${pct(m.targets.close_loop_success_rate)} 样本 ${m.close_loop.ok}/${m.close_loop.attempts}`); + const ir = m.app_install.success_rate; + console.log(`装软件成功率 ${ir === null ? ' 无样本' : pct(ir).padStart(7)} ${mark(m.verdict.app_install_ok)} 目标 ≥${pct(m.targets.app_install_success_rate)} 样本 ${m.app_install.ok}/${m.app_install.attempts}`); + console.log(`P95 时延 ${String(m.latency.fast.p95_ms + 'ms').padStart(7)} ${mark(m.verdict.p95_ok)} 交互层(目标 <${m.targets.p95_ms}ms,样本 ${m.latency.fast.samples}:P50 ${m.latency.fast.p50_ms}ms / 最大 ${m.latency.fast.max_ms}ms${m.latency.timeouts ? ` / 超时 ${m.latency.timeouts}` : ''})`); + console.log(`慢操作耗时 ${String(m.latency.slow.p95_ms + 'ms').padStart(7)} —— 样本 ${m.latency.slow.samples}(${m.latency.slow.capabilities.join(' / ')};本身耗时由业务决定,不计入达标判定)`); + console.log(`安全拦截 ${pct(m.security.interception_rate).padStart(7)} ${mark(m.verdict.interception_ok)} ACL 拒绝 ${m.security.acl_denied} / 超频 ${m.security.rate_limited} / 认证失败 ${m.security.auth_failures}`); + console.log(`\n总体: ${m.verdict.all_pass ? '✅ 四项全达标' : '⚠️ 存在未达标项'}`); + + const caps = Object.entries(m.by_capability).sort((a, b) => b[1].attempts - a[1].attempts); + if (caps.length > 0) { + console.log('\n按能力细分(Top 10):'); + console.log(` ${'能力'.padEnd(26)} ${'次数'.padStart(6)} ${'成功率'.padStart(8)} ${'P95'.padStart(9)}`); + for (const [name, st] of caps.slice(0, 10)) { + console.log(` ${name.padEnd(26)} ${String(st.attempts).padStart(6)} ${pct(st.success_rate).padStart(8)} ${String(st.p95_ms + 'ms').padStart(9)}`); + } + } + }), + ); +} + async function cmdClip(opts: Options): Promise { // 图片上传:从本地文件读入(避免超长命令行参数) if (opts.imageFile) { @@ -1654,6 +1709,9 @@ async function main(): Promise { } return; } + case 'metrics': + await cmdMetrics(opts); + return; case 'clip': await cmdClip(opts); return; diff --git a/apps/mcp/src/index.ts b/apps/mcp/src/index.ts index 3b25370..af6ea43 100644 --- a/apps/mcp/src/index.ts +++ b/apps/mcp/src/index.ts @@ -429,6 +429,19 @@ const TOOLS = [ additionalProperties: false, }, }, + { + name: 'na_metrics', + description: + '按 PRD 2.2 汇总被控端成功指标(闭环成功率 / 装软件成功率 / P95 时延 / 安全拦截率)' + + '及达标判定与按能力细分。排查「成功率掉了没有/哪个能力最慢」时用。', + inputSchema: { + type: 'object', + properties: { + since: { type: 'integer', description: '可选:只统计该 Unix 毫秒之后的审计条目' }, + }, + additionalProperties: false, + }, + }, { name: 'na_event_watch', description: @@ -577,6 +590,7 @@ const TOOL_TO_CAPABILITY: Record = { na_fs_write: CapabilityNames.FsWrite, na_fs_stat: CapabilityNames.FsStat, na_restart: CapabilityNames.AgentRestart, + na_metrics: CapabilityNames.Metrics, na_event_watch: CapabilityNames.EventWatch, na_event_poll: CapabilityNames.EventPoll, na_event_unwatch: CapabilityNames.EventUnwatch, diff --git a/packages/protocol/src/capabilities.ts b/packages/protocol/src/capabilities.ts index 0927a7a..4398b5e 100644 --- a/packages/protocol/src/capabilities.ts +++ b/packages/protocol/src/capabilities.ts @@ -40,6 +40,7 @@ export const CapabilityNames = { ClipSet: 'clip.set', // v11 安全加固 AuditVerify: 'system.audit.verify', + Metrics: 'system.metrics', // v12 事件订阅与监控 EventWatch: 'event.watch', EventUnwatch: 'event.unwatch', @@ -1017,6 +1018,42 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ }, }, }, + { + name: CapabilityNames.Metrics, + version: '1.0', + description: + '按 PRD 2.2 汇总成功指标(数据源为审计日志):闭环成功率、装软件成功率、P95 时延、安全拦截率,' + + '并给出与目标阈值的达标判定;同时返回按能力细分统计。', + risk: 'low', + params_schema: { + type: 'object', + properties: { + since: { type: 'integer', description: '可选:只统计该时间戳(Unix 毫秒)之后的审计条目' }, + }, + additionalProperties: false, + }, + returns_schema: { + type: 'object', + properties: { + close_loop: { + type: 'object', + properties: { + attempts: { type: 'integer' }, + ok: { type: 'integer' }, + failed: { type: 'integer' }, + success_rate: { type: 'number' }, + }, + }, + app_install: { type: 'object' }, + latency: { type: 'object' }, + security: { type: 'object' }, + verdict: { type: 'object', description: '四项指标的达标判定' }, + targets: { type: 'object' }, + by_capability: { type: 'object' }, + window: { type: 'object' }, + }, + }, + }, { name: CapabilityNames.EventWatch, version: '1.0', diff --git a/tests/unit/capabilities.test.mjs b/tests/unit/capabilities.test.mjs index c165578..ad5de47 100644 --- a/tests/unit/capabilities.test.mjs +++ b/tests/unit/capabilities.test.mjs @@ -51,11 +51,12 @@ test('清单:v7~v11 新能力均已登记', () => { 'event.unwatch', 'event.list', 'event.poll', + 'system.metrics', ]; for (const name of expected) { assert.ok(findCapability(name), `缺少 ${name}`); } - assert.equal(CAPABILITY_MANIFEST.length, 35, '当前应有 35 项能力'); + assert.equal(CAPABILITY_MANIFEST.length, 36, '当前应有 36 项能力'); }); // ---------- 参数校验(新能力) ---------- diff --git a/tests/unit/metrics.test.mjs b/tests/unit/metrics.test.mjs new file mode 100644 index 0000000..9bc9f01 --- /dev/null +++ b/tests/unit/metrics.test.mjs @@ -0,0 +1,163 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { computeMetrics, formatMetrics, DEFAULT_TARGETS } from '../../apps/agent/dist/metrics.js'; + +const inv = (capability, status, duration_ms, extra = {}) => ({ + ts: 1_700_000_000_000 + duration_ms, + type: 'invoke', + capability, + status, + duration_ms, + ...extra, +}); + +test('指标:空样本不炸,且不误判为不达标', () => { + const m = computeMetrics([]); + assert.equal(m.close_loop.attempts, 0); + assert.equal(m.close_loop.success_rate, 0); + assert.equal(m.app_install.success_rate, null, '无样本应为 null 而非 0'); + assert.equal(m.latency.p95_ms, 0); + assert.equal(m.verdict.close_loop_ok, true, '无样本视为达标(无可判定)'); + assert.equal(m.verdict.all_pass, true); +}); + +test('指标:闭环成功率按 invoke 的 status 统计', () => { + const entries = [ + inv('system.info', 'ok', 100), + inv('system.info', 'ok', 200), + inv('system.status', 'ok', 300), + inv('system.shell.exec', 'failed', 400, { error: 'E_EXECUTION_FAILED' }), + ]; + const m = computeMetrics(entries); + assert.equal(m.close_loop.attempts, 4); + assert.equal(m.close_loop.ok, 3); + assert.equal(m.close_loop.failed, 1); + assert.equal(m.close_loop.success_rate, 0.75); + assert.equal(m.verdict.close_loop_ok, false, '75% < 95% 应不达标'); +}); + +test('指标:装软件成功率只看 app.install,边界 90% 判定', () => { + const nine = Array.from({ length: 9 }, () => inv('app.install', 'ok', 1000)); + const oneBad = [inv('app.install', 'failed', 1000)]; + const m = computeMetrics([...nine, ...oneBad, inv('system.info', 'ok', 50)]); + assert.equal(m.app_install.attempts, 10); + assert.equal(m.app_install.ok, 9); + assert.equal(m.app_install.success_rate, 0.9); + assert.equal(m.verdict.app_install_ok, true, '恰好 90% 应达标(>=)'); + + const m2 = computeMetrics([...nine.slice(0, 8), ...oneBad, inv('app.install', 'failed', 1000)]); + assert.equal(m2.verdict.app_install_ok, false, '80% 应不达标'); +}); + +test('指标:P95 分位(只在成功样本上算),失败不计入时延', () => { + // 100 个样本:99 个 100ms + 1 个 5000ms -> P95 应为 100ms + const entries = [ + ...Array.from({ length: 99 }, () => inv('system.info', 'ok', 100)), + inv('system.info', 'ok', 5000), + ]; + const m = computeMetrics(entries); + assert.equal(m.latency.p95_ms, 100, 'P95 落在第 95 个样本上'); + assert.equal(m.latency.max_ms, 5000); + assert.equal(m.verdict.p95_ok, true); +}); + +test('指标:P95 分层 —— 慢操作(exec/重启/装软件/录屏)不拖累达标判定', () => { + // 全部是 shell.exec(慢能力):交互层无样本 -> 视为达标,但慢操作层如实记录 + const slowOnly = Array.from({ length: 100 }, () => inv('system.shell.exec', 'ok', 4000)); + const m1 = computeMetrics(slowOnly); + assert.equal(m1.latency.p95_ms, 4000, '总体 P95 如实反映'); + assert.equal(m1.latency.fast.samples, 0); + assert.equal(m1.latency.slow.samples, 100); + assert.equal(m1.verdict.p95_ok, true, '慢能力不计入交互层判定'); +}); + +test('指标:交互层 P95 超目标才判不达标(真机口径)', () => { + // 快能力本身慢 -> 不达标 + const slowFast = Array.from({ length: 100 }, () => inv('system.info', 'ok', 4000)); + assert.equal(computeMetrics(slowFast).verdict.p95_ok, false, '交互层 4000ms > 3000ms 应不达标'); + + // 真机场景复现:8 个样本里 1 次重启 9.6s,其余都是快操作 -> 应达标 + const realWorld = [ + inv('system.agent.restart', 'ok', 9575), + inv('system.info', 'ok', 783), + inv('system.metrics', 'ok', 14), + inv('system.audit.list', 'ok', 4), + inv('fs.write', 'ok', 8), + inv('fs.read', 'ok', 3), + ]; + const m = computeMetrics(realWorld); + assert.equal(m.latency.p95_ms, 9575, '总体 P95 被慢操作顶穿(如实)'); + assert.equal(m.latency.fast.samples, 5, '交互层 5 个样本'); + assert.equal(m.verdict.p95_ok, true, '剔除重启后应达标 —— 这正是修正口径的目的'); +}); + +test('指标:安全拦截计数(ACL / 限速 / 认证失败)且拦截率恒为 100%', () => { + const entries = [ + { ts: 1, type: 'acl.denied', capability: 'fs.write' }, + { ts: 2, type: 'acl.denied', capability: 'input.key.press' }, + { ts: 3, type: 'rate.limited', capability: 'system.shell.exec' }, + { ts: 4, type: 'auth.failure', reason: 'psk 校验失败' }, + inv('system.info', 'ok', 80), + ]; + const m = computeMetrics(entries); + assert.equal(m.security.acl_denied, 2); + assert.equal(m.security.rate_limited, 1); + assert.equal(m.security.auth_failures, 1); + assert.equal(m.security.interception_rate, 1); + assert.equal(m.verdict.interception_ok, true); +}); + +test('指标:按能力细分(次数/成功率/P95)', () => { + const entries = [ + inv('fs.list', 'ok', 100), + inv('fs.list', 'ok', 200), + inv('fs.list', 'failed', 900, { error: 'x' }), + inv('clip.get', 'ok', 300), + ]; + const m = computeMetrics(entries); + assert.equal(m.by_capability['fs.list'].attempts, 3); + assert.equal(m.by_capability['fs.list'].ok, 2); + assert.ok(Math.abs(m.by_capability['fs.list'].success_rate - 2 / 3) < 1e-9); + assert.equal(m.by_capability['clip.get'].attempts, 1); +}); + +test('指标:超时(exit 124)单独计数,仍算失败样本', () => { + const entries = [ + inv('system.shell.exec', 'failed', 30_000, { error: '超时已被强制终止 (exit_code=124)' }), + inv('system.shell.exec', 'ok', 120), + ]; + const m = computeMetrics(entries); + assert.equal(m.latency.timeouts, 1); + assert.equal(m.close_loop.failed, 1); +}); + +test('指标:可自定义阈值,判定随之变化', () => { + const entries = [inv('system.info', 'ok', 100), inv('system.info', 'failed', 100)]; + const m = computeMetrics(entries, { ...DEFAULT_TARGETS, close_loop_success_rate: 0.5 }); + assert.equal(m.close_loop.success_rate, 0.5); + assert.equal(m.verdict.close_loop_ok, true, '放宽到 50% 后应达标'); +}); + +test('指标:窗口时间取审计条目的最早/最晚', () => { + const entries = [inv('a', 'ok', 10), inv('b', 'ok', 10)]; + entries[0].ts = 1000; + entries[1].ts = 5000; + const m = computeMetrics(entries); + assert.equal(m.window.from, 1000); + assert.equal(m.window.to, 5000); + assert.equal(m.window.entries, 2); +}); + +test('指标:formatMetrics 输出含四项指标与达标标记', () => { + const m = computeMetrics([ + ...Array.from({ length: 20 }, () => inv('system.info', 'ok', 100)), + ...Array.from({ length: 10 }, () => inv('app.install', 'ok', 2000)), + ]); + const text = formatMetrics(m); + assert.match(text, /闭环成功率/); + assert.match(text, /装软件成功率/); + assert.match(text, /P95 时延/); + assert.match(text, /慢操作耗时/); + assert.match(text, /安全拦截/); + assert.match(text, /四项全达标/); +});