import fs from 'node:fs'; import crypto from 'node:crypto'; import { pathToFileURL } from 'node:url'; export const sha256 = value => crypto.createHash('sha256').update(value).digest('hex'); export function checkResponse(raw, inputs) { const problems = []; let rows; try { rows = JSON.parse(raw); } catch { return { passed: false, problems: ['回复不能直接解析为 JSON'], rows: null }; } if (!Array.isArray(rows)) return { passed: false, problems: ['顶层不是数组'], rows: null }; if (rows.length !== inputs.length) problems.push(`应有 ${inputs.length} 项,实际 ${rows.length} 项`); rows.forEach((row, index) => { if (!row || typeof row !== 'object' || Array.isArray(row)) { problems.push(`第 ${index + 1} 项不是对象`); return; } if (Object.keys(row).sort().join(',') !== 'id,label,reason') problems.push(`第 ${index + 1} 项字段不符`); if (row.id !== inputs[index]?.id) problems.push(`第 ${index + 1} 项编号或顺序不符`); if (!['物流', '商品', '其他', '待确认'].includes(row.label)) problems.push(`第 ${index + 1} 项标签不符`); if (typeof row.reason !== 'string' || !row.reason.trim()) problems.push(`第 ${index + 1} 项解释为空或非文本`); }); return { passed: !problems.length, problems, rows }; } export function checkComparison(record) { const integrityProblems = []; if (new Set(record.protocol.inputs.map(input => input.id)).size !== record.protocol.inputs.length) integrityProblems.push('协议中的样本编号重复'); if (JSON.stringify(record.runs.map(run => run.id)) !== JSON.stringify(record.protocol.sequence)) integrityProblems.push('实际记录与计划轮次/顺序不一致'); const runs = record.runs.map(run => ({ id: run.id, status: run.status, promptHashMatches: sha256(run.submittedPrompt) === run.submittedPromptSha256, responseHashMatches: run.rawResponse === null ? null : sha256(run.rawResponse) === run.rawResponseSha256, structural: run.rawResponse === null ? { passed: false, problems: ['没有实际回复'], rows: null } : checkResponse(run.rawResponse, record.protocol.inputs), })); const labelMatrix = record.protocol.inputs.map(input => ({ id: input.id, cohort: input.cohort, labels: Object.fromEntries(runs.map(run => [run.id, run.structural.rows?.find(row => row?.id === input.id)?.label ?? null])), })); const changedIds = labelMatrix.filter(row => new Set(Object.values(row.labels).filter(Boolean)).size > 1).map(row => row.id); return { plannedRuns: record.protocol.sequence.length, recordedRuns: runs.length, integrityProblems, structurallyPassedRuns: runs.filter(run => run.structural.passed).length, runs, labelMatrix, changedIds, limitation: '只重算文本哈希、JSON结构和标签变化;不验证解释语义、模型真实身份、人工参考答案或业务准确率。', }; } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { if (!process.argv[2]) throw Error('用法:node check-ai-comparison.mjs <完整记录.json>'); const result = checkComparison(JSON.parse(fs.readFileSync(process.argv[2], 'utf8'))); console.log(JSON.stringify(result, null, 2)); if (result.integrityProblems.length || result.runs.some(run => !run.promptHashMatches || run.responseHashMatches === false)) process.exitCode = 1; }