diff --git a/.changeset/patch-large-record-mask-redaction.md b/.changeset/patch-large-record-mask-redaction.md new file mode 100644 index 00000000000..d0cb5b774fb --- /dev/null +++ b/.changeset/patch-large-record-mask-redaction.md @@ -0,0 +1,13 @@ +--- +"gh-aw": patch +--- + +Avoid regular-expression stack exhaustion when redacting runtime masks from multi-megabyte JSON and mixed-log artifacts, while preserving fail-closed cleanup for genuine redaction failures. + +Keep malformed-token recovery linear, preserve unrelated plain-text escapes when masking encoded values, and match configured secrets together with runtime masks to prevent fragmentation or encoding changes from bypassing redaction. + +Sanitize malformed ordinary JSON artifacts as whole-file text, matching raw and escaped credentials together even inside recoverable quoted spans or across line breaks. + +Preserve bundled safe-output schema vocabulary during redaction so short masks do not corrupt review message fields, dispatch types, or collector configuration. Continue redacting payload data and fail closed on malformed executable records, unsafe custom routing changes, or conflicts with control data. + +Coalesce overlapping built-in, configured, and runtime credentials in executable safe-output payloads without reparsing their already-decoded free-form strings. diff --git a/actions/setup/js/add_mask_redaction.cjs b/actions/setup/js/add_mask_redaction.cjs index 6c2cde4ba7c..fef182d1ee9 100644 --- a/actions/setup/js/add_mask_redaction.cjs +++ b/actions/setup/js/add_mask_redaction.cjs @@ -14,8 +14,8 @@ /** Matches an `::add-mask::` workflow command and captures its payload. */ const ADD_MASK_COMMAND_RE = /::add-mask::(.*)$/; -/** Replacement used for masked values, matching the runner's own rendering. */ -const MASK_REPLACEMENT = "***"; +const { MASK_REPLACEMENT, redactMaskedValues, maskedValueMatches, overlappingMatchGroups } = require("./mask_value_matcher.cjs"); +const { transformJSONStrings } = require("./json_string_scanner.cjs"); /** * Decode a GitHub Actions workflow command payload. @@ -68,74 +68,206 @@ function collectAddMaskedValues(logContent) { } /** - * Replace all occurrences of the masked values with `***`. + * Remove `::add-mask::` command lines and redact every masked value from the text. * + * @param {string} text - Text about to be rendered into generated content + * @param {string[]} maskedValues - Values collected via {@link collectAddMaskedValues} + * @param {(value: string) => string} [replacementForValue] + * @returns {string} + */ +function applyAddMaskRedaction(text, maskedValues, replacementForValue) { + if (!text) return text; + const withoutCommands = text + .split("\n") + .filter(line => !isAddMaskCommandLine(line)) + .join("\n"); + return redactMaskedValues(withoutCommands, maskedValues, replacementForValue); +} + +/** + * Decode ambiguous escapes for matching only, mapping matches back to source + * spans so unrelated plain-text paths and literal escapes keep their bytes. * @param {string} text * @param {string[]} maskedValues + * @param {(value: string) => string} replacementForValue * @returns {string} */ -function redactMaskedValues(text, maskedValues) { - if (!text || !maskedValues || maskedValues.length === 0) return text; - const pending = []; - for (let order = 0; order < maskedValues.length; order++) { - const value = maskedValues[order]; - if (!value) continue; - const index = text.indexOf(value); - if (index !== -1) pending.push({ value, index, order }); - } - const precedes = (a, b) => a.index < b.index || (a.index === b.index && a.order < b.order); - const siftDown = start => { - let parent = start; - while (parent * 2 + 1 < pending.length) { - let child = parent * 2 + 1; - if (child + 1 < pending.length && precedes(pending[child + 1], pending[child])) child++; - if (!precedes(pending[child], pending[parent])) break; - [pending[parent], pending[child]] = [pending[child], pending[parent]]; - parent = child; +function redactAmbiguousMaskedValues(text, maskedValues, replacementForValue) { + const escapes = { '"': '"', "\\": "\\", "/": "/", b: "\b", f: "\f", n: "\n", r: "\r", t: "\t" }; + const pattern = /\\(["\\/bfnrt]|u[0-9a-fA-F]{4})/g; + const unescaped = text.replace(pattern, (_, escape) => { + if (escape[0] === "u") return String.fromCharCode(parseInt(escape.slice(1), 16)); + return escapes[escape]; + }); + let nextEscape = pattern.exec(text); + let offset = 0; + const sourceOffset = position => { + while (nextEscape && nextEscape.index - offset < position) { + offset += nextEscape[0].length - 1; + nextEscape = pattern.exec(text); } + return position + offset; }; - for (let i = Math.floor(pending.length / 2) - 1; i >= 0; i--) siftDown(i); const parts = []; let cursor = 0; - let replacements = 0; - while (pending.length && cursor < text.length) { - const next = pending[0]; - if (next.index >= cursor) { - if (next.index > cursor) { - if (replacements) parts.push(MASK_REPLACEMENT.repeat(replacements)); - replacements = 0; - parts.push(text.slice(cursor, next.index)); - } - replacements++; - cursor = next.index + next.value.length; - } - // Only refresh consumed or overlapping occurrences; later candidates stay cached. - next.index = text.indexOf(next.value, cursor); - if (next.index === -1) { - const last = pending.pop(); - if (pending.length) pending[0] = last; - } - siftDown(0); + for (const match of maskedValueMatches(unescaped, maskedValues)) { + const start = sourceOffset(match.start); + const end = sourceOffset(match.end); + parts.push(text.slice(cursor, start), replacementForValue(match.value)); + cursor = end; } - if (replacements) parts.push(MASK_REPLACEMENT.repeat(replacements)); - parts.push(text.slice(cursor)); - return parts.join(""); + // Ambiguous records retain their JSON-string wrapper, but its decoded value + // now changes only at the source spans that actually matched a mask. + return cursor ? JSON.stringify(parts.join("") + text.slice(cursor)) : text; } /** - * Remove `::add-mask::` command lines and redact every masked value from the text. - * - * @param {string} text - Text about to be rendered into generated content - * @param {string[]} maskedValues - Values collected via {@link collectAddMaskedValues} + * @param {string} encoded * @returns {string} */ -function applyAddMaskRedaction(text, maskedValues) { - if (!text) return text; - const withoutCommands = text - .split("\n") - .filter(line => !isAddMaskCommandLine(line)) - .join("\n"); - return redactMaskedValues(withoutCommands, maskedValues); +function decodeJSONString(encoded) { + try { + return JSON.parse(encoded); + } catch (error) { + throw new Error("Unable to decode JSON log string during secret redaction", { cause: error }); + } +} + +/** + * Resolve raw and encoded matches together before a malformed record can be + * rewritten. Quoted-token interiors use decoded semantics; raw matches spanning + * token boundaries remain eligible so a shorter mask cannot fragment them. + * @param {string} text + * @param {string[]} maskedValues + * @param {(value: string) => string} replacementForValue + * @param {boolean} [matchRawText] - Include raw matches inside quoted spans of invalid artifacts + * @returns {string} + */ +function redactMixedMaskedValues(text, maskedValues, replacementForValue, matchRawText = false) { + const commandLines = new Set(); + if (matchRawText) { + const lines = text.split("\n"); + for (let index = 0; index < lines.length; index++) { + if (isAddMaskCommandLine(lines[index])) commandLines.add(lines[index] + (index < lines.length - 1 ? "\n" : "")); + } + maskedValues = [...maskedValues, ...commandLines]; + } + maskedValues = [...maskedValues].sort((a, b) => b.length - a.length); + const tokens = []; + let scanned = 0; + transformJSONStrings( + text, + token => { + decodeJSONString(token); + tokens.push(scanned, scanned + token.length); + scanned += token.length; + return token; + }, + plain => { + scanned += plain.length; + return plain; + } + ); + const tokenAt = position => { + let low = 0; + let high = tokens.length / 2; + while (low < high) { + const mid = (low + high) >>> 1; + if (tokens[mid * 2] <= position) low = mid + 1; + else high = mid; + } + return (low - 1) * 2; + }; + const escapeStarts = []; + const escapeOffsets = []; + const escapes = { '"': '"', "\\": "\\", "/": "/", b: "\b", f: "\f", n: "\n", r: "\r", t: "\t" }; + let offset = 0; + const unescaped = text.replace(/\\(["\\/bfnrt]|u[0-9a-fA-F]{4})/g, (match, escape, index) => { + escapeStarts.push(index - offset); + offset += match.length - 1; + escapeOffsets.push(offset); + return escape[0] === "u" ? String.fromCharCode(parseInt(escape.slice(1), 16)) : escapes[escape]; + }); + const sourceOffset = position => { + let low = 0; + let high = escapeStarts.length; + while (low < high) { + const mid = (low + high) >>> 1; + if (escapeStarts[mid] < position) low = mid + 1; + else high = mid; + } + return position + (low ? escapeOffsets[low - 1] : 0); + }; + const toSource = match => ({ start: sourceOffset(match.start), end: sourceOffset(match.end), value: match.value }); + const rawMatches = maskedValueMatches( + text, + maskedValues, + match => { + if (matchRawText) return true; + const token = tokenAt(match.start); + return token < 0 || match.end > tokens[token + 1]; + }, + true + ); + const decodedMatches = maskedValueMatches( + unescaped, + maskedValues, + match => { + const source = toSource(match); + const token = tokenAt(source.start); + return token < 0 || source.end > tokens[token + 1] || (source.start > tokens[token] && source.end < tokens[token + 1]); + }, + true + ); + function* sourceMatches() { + let raw = rawMatches.next(); + let decoded = decodedMatches.next(); + let decodedSource = decoded.done ? undefined : toSource(decoded.value); + while (!raw.done || !decoded.done) { + const useRaw = !decodedSource || (!raw.done && (raw.value.start < decodedSource.start || (raw.value.start === decodedSource.start && raw.value.end >= decodedSource.end))); + const match = useRaw && !raw.done ? raw.value : decodedSource; + if (!match) throw new Error("Unable to select log spans during secret redaction"); + const token = tokenAt(match.start); + const insideToken = token >= 0 && match.start > tokens[token] && match.end < tokens[token + 1]; + const overlapsToken = (token >= 0 && match.start < tokens[token + 1]) || (token + 2 < tokens.length && tokens[token + 2] < match.end); + yield { ...match, ambiguous: !insideToken && (overlapsToken || text.slice(match.start, match.end) !== match.value) }; + if (useRaw) raw = rawMatches.next(); + else { + decoded = decodedMatches.next(); + decodedSource = decoded.done ? undefined : toSource(decoded.value); + } + } + } + const parts = []; + let cursor = 0; + let ambiguous = false; + let repeated = ""; + let repetitions = 0; + for (const match of overlappingMatchGroups(sourceMatches())) { + const token = tokenAt(match.start); + const insideToken = token >= 0 && match.start > tokens[token] && match.end < tokens[token + 1]; + const wholeLine = commandLines.has(match.value) || ((match.start === 0 || text[match.start - 1] === "\n") && (match.end === text.length || text[match.end] === "\n") && maskedValues.includes(text.slice(match.start, match.end))); + ambiguous ||= match.ambiguous && !wholeLine; + const marker = commandLines.has(match.value) ? "" : replacementForValue(match.value); + const replacement = insideToken ? JSON.stringify(marker).slice(1, -1) : marker; + if (match.start > cursor || replacement !== repeated) { + if (repetitions) parts.push(repeated.repeat(repetitions)); + repetitions = 0; + if (match.start > cursor) parts.push(text.slice(cursor, match.start)); + } + repeated = replacement; + repetitions++; + cursor = match.end; + } + if (repetitions) parts.push(repeated.repeat(repetitions)); + parts.push(text.slice(cursor)); + const redacted = transformJSONStrings(parts.join(""), encoded => { + const value = decodeJSONString(encoded); + const withoutCommands = applyAddMaskRedaction(value, []); + return withoutCommands === value ? encoded : JSON.stringify(withoutCommands); + }); + // A registered raw whole-line value removes all of the original syntax. + return ambiguous && !maskedValues.includes(text) ? JSON.stringify(redacted) : redacted; } /** @@ -145,32 +277,79 @@ function applyAddMaskRedaction(text, maskedValues) { * byte-for-byte. * @param {string} content * @param {string[]} maskedValues + * @param {(value: any) => any} [redactJSONValue] - Structured protocol redactor for valid JSON records + * @param {(value: string) => string} [replacementForValue] - Marker for each matched registered value + * @param {{textMode?: boolean}} [options] - Match invalid artifacts as raw and escaped text without splitting multiline secrets * @returns {string} */ -function redactArtifactMaskedValues(content, maskedValues) { - if (!content || !maskedValues.length) return content; +function redactArtifactMaskedValues(content, maskedValues, redactJSONValue, replacementForValue, options = {}) { + if (!content || (!maskedValues.length && !redactJSONValue)) return content; + const preserveLexemes = !!replacementForValue && !redactJSONValue; + const replacement = replacementForValue || (() => MASK_REPLACEMENT); + if (options.textMode && preserveLexemes) { + // Only multiline credentials can overlap a removed command-line boundary. + if (!maskedValues.some(value => /[\r\n]/.test(value))) { + content = content + .split("\n") + .filter(line => !isAddMaskCommandLine(line)) + .join("\n"); + } + return redactMixedMaskedValues(content, maskedValues, replacement, true); + } const redactValue = value => { - if (typeof value === "string") return applyAddMaskRedaction(value, maskedValues); + if (typeof value === "string") return applyAddMaskRedaction(value, maskedValues, replacementForValue); if (Array.isArray(value)) return value.map(redactValue); - if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, nested]) => [redactMaskedValues(key, maskedValues), redactValue(nested)])); + if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, nested]) => [redactMaskedValues(key, maskedValues, replacementForValue), redactValue(nested)])); return value; }; + const redactString = (encoded, isKey = false) => { + const value = decodeJSONString(encoded); + const redacted = isKey ? redactMaskedValues(value, maskedValues, replacementForValue) : redactValue(value); + return redacted === value ? encoded : JSON.stringify(redacted); + }; const redactJson = text => { + let value; try { - let value = JSON.parse(text); - // Skip quoted strings and match original numeric tokens at every depth so - // rounding or exponent notation cannot hide a mask. - const numericRedacted = text.replace(/"(?:[^"\\]|\\.)*"|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/g, token => { - if (token.startsWith('"')) return token; - const redacted = redactMaskedValues(token, maskedValues); - return redacted === token ? token : JSON.stringify(redacted); - }); - if (numericRedacted !== text) value = JSON.parse(numericRedacted); - const redacted = JSON.stringify(redactValue(value)); - return redacted === JSON.stringify(value) ? numericRedacted : text.match(/^\s*/)[0] + redacted + text.match(/\s*$/)[0]; + value = JSON.parse(text); } catch { return undefined; } + // Skip quoted strings and match original numeric tokens at every depth so + // rounding or exponent notation cannot hide a mask. + const numericRedacted = transformJSONStrings( + text, + token => token, + plain => + plain.replace(/-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/g, token => { + const redacted = redactMaskedValues(token, maskedValues, replacementForValue); + return redacted === token ? token : JSON.stringify(redacted); + }) + ); + if (numericRedacted !== text) { + try { + value = JSON.parse(numericRedacted); + } catch (error) { + throw new Error("Unable to decode redacted JSON log record during secret redaction", { cause: error }); + } + } + if (preserveLexemes) { + let cursor = 0; + return transformJSONStrings( + numericRedacted, + token => { + cursor += token.length; + let next = cursor; + while (next < numericRedacted.length && " \t\r\n".includes(numericRedacted[next])) next++; + return redactString(token, numericRedacted[next] === ":"); + }, + plain => { + cursor += plain.length; + return plain; + } + ); + } + const redacted = JSON.stringify((redactJSONValue || redactValue)(value)); + return redacted === JSON.stringify(value) ? numericRedacted : text.match(/^\s*/)[0] + redacted + text.match(/\s*$/)[0]; }; const json = redactJson(content); if (json !== undefined) return json; @@ -180,17 +359,11 @@ function redactArtifactMaskedValues(content, maskedValues) { .map(line => { const jsonLine = redactJson(line); if (jsonLine !== undefined) return jsonLine; + if (preserveLexemes) return redactMixedMaskedValues(line, maskedValues, replacement); // Mixed or malformed logs can still contain recoverable JSON strings. - const decoded = line.replace(/"(?:[^"\\]|\\.)*"/g, encoded => { - try { - const value = JSON.parse(encoded); - const redacted = redactValue(value); - return redacted === value ? encoded : JSON.stringify(redacted); - } catch { - return encoded; - } - }); - return redactMaskedValues(decoded, maskedValues); + const decoded = transformJSONStrings(line, redactString); + const redacted = redactMaskedValues(decoded, maskedValues, replacementForValue); + return redactAmbiguousMaskedValues(redacted, maskedValues, replacement); }) .join("\n"); } @@ -203,5 +376,6 @@ module.exports = { isAddMaskCommandLine, redactArtifactMaskedValues, redactMaskedValues, + transformJSONStrings, unescapeWorkflowCommandValue, }; diff --git a/actions/setup/js/add_mask_redaction.test.cjs b/actions/setup/js/add_mask_redaction.test.cjs index 81355292823..c872a95e947 100644 --- a/actions/setup/js/add_mask_redaction.test.cjs +++ b/actions/setup/js/add_mask_redaction.test.cjs @@ -3,9 +3,145 @@ import { describe, it, expect, vi } from "vitest"; import { createRequire } from "module"; const require = createRequire(import.meta.url); -const { applyAddMaskRedaction, collectAddMaskedValues, isAddMaskCommandLine, redactArtifactMaskedValues, redactMaskedValues, unescapeWorkflowCommandValue } = require("./add_mask_redaction.cjs"); +const { applyAddMaskRedaction, collectAddMaskedValues, isAddMaskCommandLine, redactArtifactMaskedValues, transformJSONStrings, unescapeWorkflowCommandValue } = require("./add_mask_redaction.cjs"); +const { redactMaskedValues } = require("./mask_value_matcher.cjs"); describe("add_mask_redaction", () => { + describe("seeded parser fuzz tests", () => { + const seeds = [1, 42, 2026, 0xdeadbeef, 0xffffffff]; + const alphabet = ["a", "z", "0", '"', "\\", "/", "\b", "\f", "\n", "\r", "\t", "\u0000", "\u001f", "\u2028", "\u2029", "\ud800", "\udfff", "\ud83d\ude80"]; + + function randomFor(seed) { + let state = seed; + return max => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return Math.floor((state / 0x100000000) * max); + }; + } + + function randomText(random) { + return Array.from({ length: random(24) }, () => alphabet[random(alphabet.length)]).join(""); + } + + function encodeString(value, random) { + let encoded = '"'; + for (let index = 0; index < value.length; index++) { + const character = value[index]; + encoded += random(2) ? `\\u${value.charCodeAt(index).toString(16).padStart(4, "0")}` : JSON.stringify(character).slice(1, -1); + } + return encoded + '"'; + } + + it.each(seeds)("matches an independent bounded token grammar after random insertions, deletions, and truncations (seed %i)", seed => { + const random = randomFor(seed); + const fragments = ['"', "\\", "\\u", "\\u0000", "\\u12gg", "\\q", '""', "\t", "\n", "\r", "plain", ":", ","]; + // The regex oracle only sees short fuzz records, never the large production inputs. + const tokenPattern = /"(?:[^"\\\x00-\x1f]|\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4}))*"/g; + for (let iteration = 0; iteration < 500; iteration++) { + let content = Array.from({ length: 1 + random(8) }, () => (random(2) ? encodeString(randomText(random), random) : fragments[random(fragments.length)])).join(""); + const position = random(content.length + 1); + switch (random(3)) { + case 0: + content = content.slice(0, position) + String.fromCharCode(random(0x10000)) + content.slice(position); + break; + case 1: + content = content.slice(0, position) + content.slice(position + 1); + break; + case 2: + content = content.slice(0, position); + break; + } + const tokens = []; + const spans = []; + const result = transformJSONStrings( + content, + token => { + expect(typeof JSON.parse(token), `seed ${seed}, iteration ${iteration}`).toBe("string"); + tokens.push(token); + spans.push(token); + return token; + }, + plain => { + spans.push(plain); + return plain; + } + ); + const context = `seed ${seed}, iteration ${iteration}, input ${JSON.stringify(content)}`; + expect(tokens, context).toEqual(Array.from(content.matchAll(tokenPattern), match => match[0])); + expect(spans.join(""), context).toBe(content); + expect(result, context).toBe(content); + expect(redactArtifactMaskedValues(content, ["ABSENT_RUNTIME_MASK"]), context).toBe(content); + } + }); + + it.each(seeds)("redacts randomly escaped masks in nested JSON, JSONL, and mixed records against a decoded oracle (seed %i)", seed => { + const random = randomFor(seed); + for (let iteration = 0; iteration < 300; iteration++) { + const mask = `MASK-${seed}-${iteration}-` + randomText(random); + const left = randomText(random); + const right = randomText(random); + const value = left + mask + right + mask; + const expected = value.replaceAll(mask, "***"); + const key = `key-${mask}`; + const encodedValue = encodeString(value, random); + const encodedKey = encodeString(key, random); + const nested = `{${encodedKey}:[${encodedValue},{"value":${encodedValue}},null,true,false,42]}`; + const context = `seed ${seed}, iteration ${iteration}`; + const expectedNested = { "key-***": [expected, { value: expected }, null, true, false, 42] }; + expect(JSON.parse(redactArtifactMaskedValues(nested, [mask])), context).toEqual(expectedNested); + const jsonl = ` \t${nested}\r\n${encodedValue}\n`; + const redactedLines = redactArtifactMaskedValues(jsonl, [mask]).trim().split("\n"); + expect( + redactedLines.map(line => JSON.parse(line)), + context + ).toEqual([expectedNested, expected]); + const mixed = `output: ${encodedValue} end`; + expect(redactArtifactMaskedValues(mixed, [mask]), context).toBe(`output: ${JSON.stringify(expected)} end`); + const selected = redactArtifactMaskedValues(mixed, [mask], undefined, () => "***"); + expect(JSON.parse(selected.slice("output: ".length, -" end".length)), context).toBe(expected); + expect(redactArtifactMaskedValues(nested, ["ABSENT_RUNTIME_MASK"]), context).toBe(nested); + expect(redactArtifactMaskedValues(jsonl, ["ABSENT_RUNTIME_MASK"]), context).toBe(jsonl); + expect(redactArtifactMaskedValues(mixed, ["ABSENT_RUNTIME_MASK"]), context).toBe(mixed); + expect( + redactArtifactMaskedValues(mixed, ["ABSENT_RUNTIME_MASK"], undefined, () => "***"), + context + ).toBe(mixed); + } + }); + + it.each(seeds)("finds randomly encoded masks after ambiguous malformed quotes without rewriting unmatched records (seed %i)", seed => { + const random = randomFor(seed); + const prefixes = ['"bad\\q"', '"bad\\u12gg"', '"bad\\u123"', '"bad\t"', '"bad\u0000"']; + for (let iteration = 0; iteration < 300; iteration++) { + const prefix = prefixes[random(prefixes.length)]; + const before = (random(2) ? String.raw`C:\temp\file.txt \u0041 \\u006fpaque ` : "") + "a".repeat(random(16)); + const after = "z".repeat(random(16)) + (random(2) ? String.raw` D:\new\report.txt \u0042` : ""); + const encodedMask = "\\u006f" + encodeString("paque", random).slice(1, -1); + const content = `${prefix} "${before}${encodedMask}${after}"`; + const context = `seed ${seed}, iteration ${iteration}, input ${JSON.stringify(content)}`; + expect(JSON.parse(redactArtifactMaskedValues(content, ["opaque"])), context).toBe(`${prefix} "${before}***${after}"`); + expect(JSON.parse(redactArtifactMaskedValues(content, ["opaque"], undefined, () => "***")), context).toBe(`${prefix} "${before}***${after}"`); + expect(redactArtifactMaskedValues(content, ["ABSENT_RUNTIME_MASK"]), context).toBe(content); + } + }); + + it.each(seeds)("preserves original numeric lexemes while masking generated high-precision tokens (seed %i)", seed => { + const random = randomFor(seed); + for (let iteration = 0; iteration < 300; iteration++) { + const integer = String(1 + random(9)) + Array.from({ length: 20 + random(20) }, () => String(random(10))).join(""); + const fraction = random(2) ? "." + String(random(10000)) : ""; + const exponent = random(2) ? `e${random(2) ? "+" : "-"}${random(1000)}` : ""; + const token = (random(2) ? "-" : "") + integer + fraction + exponent; + const content = ` \t[${token},-0,1e+09,9007199254740993,1e400]\r\n`; + const context = `seed ${seed}, iteration ${iteration}, token ${token}`; + const redacted = redactArtifactMaskedValues(content, [token]); + expect(redacted, context).toBe(' \t["***",-0,1e+09,9007199254740993,1e400]\r\n'); + expect(JSON.parse(redacted)[0], context).toBe("***"); + expect(redactArtifactMaskedValues(content, ["ABSENT_RUNTIME_MASK"]), context).toBe(content); + } + }); + }); + describe("isAddMaskCommandLine", () => { it("detects add-mask command lines", () => { expect(isAddMaskCommandLine("::add-mask::secret")).toBe(true); @@ -55,85 +191,291 @@ describe("add_mask_redaction", () => { }); }); - describe("redactMaskedValues", () => { - it("replaces every occurrence with ***", () => { - expect(redactMaskedValues("token=abc and abc again", ["abc"])).toBe("token=*** and *** again"); + describe("applyAddMaskRedaction", () => { + it("strips add-mask command lines and redacts masked values", () => { + const log = ["output using s3cr3t", "::add-mask::s3cr3t", "more output"].join("\n"); + const maskedValues = collectAddMaskedValues(log); + expect(applyAddMaskRedaction(log, maskedValues)).toBe(["output using ***", "more output"].join("\n")); }); - it("handles regex metacharacters in masked values", () => { - expect(redactMaskedValues("value a.c here", ["a.c"])).toBe("value *** here"); - expect(redactMaskedValues("value abc here", ["a.c"])).toBe("value abc here"); + it("returns falsy input unchanged", () => { + expect(applyAddMaskRedaction("", ["x"])).toBe(""); }); - it("returns the text unchanged when there are no masked values", () => { - expect(redactMaskedValues("plain", [])).toBe("plain"); - }); + describe("redactArtifactMaskedValues", () => { + it.each([ + ['{"configured":["configured",12345678],"live":"runtime","safe":42}', '{"***REDACTED***":["***REDACTED***","***REDACTED***"],"live":"***","safe":42}'], + ['plain configured runtime "\\u0063onfigured" "\\u0072untime"', 'plain ***REDACTED*** *** "***REDACTED***" "***"'], + ['"bad\t" "\\u0063onfigured \\u0072untime" C:\\temp\\file.txt', JSON.stringify('"bad\t" "***REDACTED*** ***" C:\\temp\\file.txt')], + ])("supports per-value replacement markers throughout encoding-aware redaction (%j)", (content, expected) => { + const replacement = value => (value === "runtime" ? "***" : "***REDACTED***"); + expect(redactArtifactMaskedValues(content, ["configured", "12345678", "runtime"], undefined, replacement)).toBe(expected); + }); - it("redacts masks exceeding the regular expression size limit", () => { - const mask = "x".repeat(200000); - expect(redactMaskedValues(`before ${mask} after ${mask}`, [mask])).toBe("before *** after ***"); - expect(redactMaskedValues("unrelated output", [mask])).toBe("unrelated output"); - }); + it("preserves original JSON whitespace, safe escapes, and numeric lexemes with explicit replacement markers", () => { + const content = ' \t{ "configured" : "\\u0063onfigured", "safe" : "\\u0041", "values" : [12345678,-0,1e+09,9007199254740993,1e400] }\r\n'; + const expected = ' \t{ "***REDACTED***" : "***REDACTED***", "safe" : "\\u0041", "values" : ["***REDACTED***",-0,1e+09,9007199254740993,1e400] }\r\n'; + expect(redactArtifactMaskedValues(content, ["configured", "12345678"], undefined, () => "***REDACTED***")).toBe(expected); + }); - it("preserves leftmost matching and supplied mask priority for overlaps", () => { - expect(redactMaskedValues("abc ab", ["abc", "ab"])).toBe("*** ***"); - expect(redactMaskedValues("abc", ["ab", "abc"])).toBe("***c"); - expect(redactMaskedValues("zabc", ["abc", "za"])).toBe("***bc"); - }); + it.each([ + ["synthetic\\n", "synthetic\npassword", JSON.stringify("synthetic\npassword")], + ["synthetic\\u006e", "syntheticnpassword", '"synthetic\\u006epassword"'], + ['"password', '"passwordtail', JSON.stringify('"passwordtail')], + ])("resolves longer encoded masks before raw configured prefixes (%j)", (configured, runtime, encoded) => { + const masks = [configured, runtime].sort((a, b) => b.length - a.length); + const replacement = value => (value === configured ? "***REDACTED***" : "***"); + expect(redactArtifactMaskedValues("output: " + encoded, masks, undefined, replacement)).toBe('output: "***"'); + expect(JSON.parse(redactArtifactMaskedValues('"bad\t" ' + encoded, masks, undefined, replacement))).toBe('"bad\t" "***"'); + }); - it("does not rescan replacements or create matches across removed values", () => { - expect(redactMaskedValues("secret *", ["secret", "*"])).toBe("*** ***"); - expect(redactMaskedValues("axb", ["x", "ab"])).toBe("a***b"); - }); + it("resolves raw configured masks spanning recoverable token boundaries before shorter decoded masks", () => { + const configured = 'synthetic"\\\tpassword'; + const content = '"bad\t" "\\u006fpaque" token=' + configured; + const replacement = value => (value === configured ? "***REDACTED***" : "***"); + expect(JSON.parse(redactArtifactMaskedValues(content, [configured, "synthetic", "opaque"], undefined, replacement))).toBe('"bad\t" "***" token=***REDACTED***'); + }); - it("matches ordered regex semantics while refreshing overlapping cached candidates", () => { - const values = ["ab", "aba", "ba", "b", "a", "", "ab"]; - for (let offset = 0; offset < values.length; offset++) { - const masks = [...values.slice(offset), ...values.slice(0, offset)]; - const pattern = new RegExp(masks.filter(Boolean).join("|"), "g"); - for (let length = 0; length <= 6; length++) { - for (let bits = 0; bits < 2 ** length; bits++) { - const text = Array.from({ length }, (_, index) => (bits & (1 << index) ? "a" : "b")).join(""); - expect(redactMaskedValues(text, masks)).toBe(text.replace(pattern, "***")); - } + it.each(["***", "***REDACTED***"])("keeps a fully masked raw shell command as a plain marker (%s)", marker => { + const content = 'export PAYLOAD_DIR="/synthetic/session/payload"'; + expect(redactArtifactMaskedValues(content, [content], undefined, () => marker)).toBe(marker); + expect(redactArtifactMaskedValues(content + "\n", [content], undefined, () => marker)).toBe(marker + "\n"); + }); + + it("coalesces raw and encoded matches that overlap at different source positions", () => { + const content = String.raw`\u0061bc\nf`; + expect(JSON.parse(redactArtifactMaskedValues(content, [String.raw`u0061bc\nf`, String.raw`c\nf`, "ab"], undefined, () => "***"))).toBe("***"); + }); + + it.each([ + ["long-configured-secret", "ablong", "ablong-configured-secret"], + ["12345678", "9123", "912345678"], + ])("coalesces shorter earlier runtime overlaps across text, JSON strings, keys, and numeric lexemes (%j)", (configured, runtime, content) => { + const replacement = value => (value === configured ? "***REDACTED***" : "***"); + const masks = [configured, runtime]; + expect(redactMaskedValues(content, masks, replacement)).toBe("***REDACTED***"); + expect(redactArtifactMaskedValues(content, masks, undefined, replacement)).toBe(/^\d+$/.test(content) ? '"***REDACTED***"' : "***REDACTED***"); + expect(redactArtifactMaskedValues(JSON.stringify({ [content]: content }), masks, undefined, replacement)).toBe('{"***REDACTED***":"***REDACTED***"}'); + expect(redactArtifactMaskedValues("output: " + JSON.stringify(content), masks, undefined, replacement)).toBe('output: "***REDACTED***"'); + }); + + it("keeps literal Unicode escapes distinct with explicit replacement markers", () => { + const content = '"bad\t" ' + JSON.stringify("\\u006fpaque"); + expect(redactArtifactMaskedValues(content, ["opaque"], undefined, () => "***")).toBe(content); + }); + + it("fails closed if decoding an original validated token unexpectedly fails with explicit replacement markers", () => { + const encoded = '"\\u006fpaque"'; + const failure = new Error("synthetic decoding failure"); + const original = JSON.parse; + const spy = vi.spyOn(JSON, "parse").mockImplementation(value => { + if (value === encoded) throw failure; + return original(value); + }); + try { + expect(() => redactArtifactMaskedValues("output: " + encoded, ["opaque"], undefined, () => "***")).toThrow("Unable to decode JSON log string during secret redaction"); + } finally { + spy.mockRestore(); } - } - }); + }); - it("does not re-search later masks for every frequent match", () => { - const masks = Array.from({ length: 500 }, (_, index) => `mask-${index.toString().padStart(4, "0")}`); - const repeats = 2000; - const suffix = masks.join(" "); - const text = "hot".repeat(repeats) + " ".repeat(2 * 1024 * 1024) + suffix; - const indexOf = String.prototype.indexOf; - let searches = 0; - const spy = vi.spyOn(String.prototype, "indexOf").mockImplementation(function (value, from) { - if (this === text) searches++; - return indexOf.call(this, value, from); - }); - let result; - try { - result = redactMaskedValues(text, [...masks, "hot"]); - } finally { - spy.mockRestore(); - } - expect(result).toBe("***".repeat(repeats) + " ".repeat(2 * 1024 * 1024) + masks.map(() => "***").join(" ")); - expect(searches).toBeLessThanOrEqual(masks.length * 2 + repeats + 1); - }); - }); + it("keeps the structured JSON visitor authoritative and propagates its failures", () => { + const visitor = vi.fn(value => ({ wrapped: value })); + expect(redactArtifactMaskedValues('{"safe":42}', [], visitor, () => "***REDACTED***")).toBe('{"wrapped":{"safe":42}}'); + expect(visitor).toHaveBeenCalledExactlyOnceWith({ safe: 42 }); + const error = new Error("synthetic visitor failure"); + expect(() => + redactArtifactMaskedValues('{"safe":42}', [], () => { + throw error; + }) + ).toThrow(error); + }); - describe("applyAddMaskRedaction", () => { - it("strips add-mask command lines and redacts masked values", () => { - const log = ["output using s3cr3t", "::add-mask::s3cr3t", "more output"].join("\n"); - const maskedValues = collectAddMaskedValues(log); - expect(applyAddMaskRedaction(log, maskedValues)).toBe(["output using ***", "more output"].join("\n")); - }); + describe("JSON string scanner boundaries", () => { + const encodedMask = '"\\u006fpaque"'; + const validStrings = [ + "", + "ordinary ASCII", + 'quote"then opaque', + "backslash\\then opaque", + "slash/then opaque", + "backspace\bthen opaque", + "formfeed\fthen opaque", + "newline\nthen opaque", + "return\rthen opaque", + "tab\tthen opaque", + "\u0000opaque\u001f", + "caf\u00e9 opaque \u2603", + "\ud83d\ude80opaque", + "\ud800opaque\udfff", + "\u2028opaque\u2029", + '""opaque""', + ]; + + it.each(validStrings.map(value => [JSON.stringify(value), value]))("matches decoded-string masking for valid token %s", (encoded, value) => { + const expected = JSON.stringify(value.replaceAll("opaque", "***")); + expect(redactArtifactMaskedValues(encoded, ["opaque"])).toBe(expected); + expect(redactArtifactMaskedValues(`before ${encoded} after`, ["opaque"])).toBe(`before ${expected} after`); + }); + + it.each(Array.from({ length: 17 }, (_, count) => count))("recognizes closing quotes after %i pairs of escaped backslashes", count => { + const safe = JSON.stringify("\\".repeat(count)); + const escapedQuote = JSON.stringify("\\".repeat(count) + '"opaque'); + const content = `prefix ${safe} ${encodedMask} ${escapedQuote} suffix`; + const expected = `prefix ${safe} "***" ${JSON.stringify("\\".repeat(count) + '"***')} suffix`; + expect(redactArtifactMaskedValues(content, ["opaque"])).toBe(expected); + }); + + it.each(["\\u006fpaque", "op\\u0061que", "\\u006F\\u0070\\u0061\\u0071\\u0075\\u0065"])("decodes Unicode escapes at every mask position (%s)", encoded => { + expect(redactArtifactMaskedValues(`log "${encoded}"`, ["opaque"])).toBe('log "***"'); + }); + + it.each(Array.from({ length: 32 }, (_, code) => code))("recovers after unescaped control character U+%i", code => { + const prefix = '"invalid' + String.fromCharCode(code) + " "; + expect(redactArtifactMaskedValues(prefix + encodedMask, ["opaque"])).toBe(prefix + '"***"'); + }); + + it.each(["a", "c", "e", "v", "x", "0", "1", "'", "U", " ", "\t", "\r", "\n", "\u2028", "\u2029"])("recovers after illegal escaped character %j", escaped => { + const prefix = '"invalid\\' + escaped + " "; + expect(redactArtifactMaskedValues(prefix + encodedMask, ["opaque"])).toBe(prefix + '"***"'); + }); + + it.each(["", "0", "00", "000", "gggg", "000g", "00g0", "0g00", "g000", "+000", "-000", " 000"])("recovers after malformed Unicode escape %j", digits => { + const prefix = '"invalid\\u' + digits + " "; + expect(redactArtifactMaskedValues(prefix + encodedMask, ["opaque"])).toBe(prefix + '"***"'); + }); + + it.each(['"bad\t"', '"bad\u0001"', '"bad\\q"', '"bad\\u12gg"'])("conservatively redacts alternate quote pairings after terminated malformed tokens (%j)", prefix => { + const content = `${prefix} ${encodedMask}`; + const redacted = redactArtifactMaskedValues(content, ["opaque"]); + expect(JSON.parse(redacted)).toBe(`${prefix} "***"`); + expect(redacted).not.toContain("\\u006fpaque"); + expect(redactArtifactMaskedValues(content, ["absent"])).toBe(content); + }); + + it("conservatively redacts escapes before invalid controls and in unterminated strings", () => { + for (const content of ['"\\u006fpaque\t"', '"\\u006fpaque', '"\\u006fpaque\\']) { + const redacted = redactArtifactMaskedValues(content, ["opaque"]); + expect(JSON.parse(redacted)).toBe(content.replace("\\u006fpaque", "***")); + } + }); + + it("redacts encoded masks crossing quote boundaries in an alternate pairing", () => { + const content = '"bad\t" "foo\\"bar"'; + expect(JSON.parse(redactArtifactMaskedValues(content, ['foo"bar']))).toBe('"bad\t" "***"'); + }); + + it.each(["opaque", 'foo"bar', "a\tb", "a\\b", "\ud83d\ude80"])("only rewrites source spans participating in conservative matches (%j)", mask => { + const encodedMask = Array.from({ length: mask.length }, (_, index) => `\\u${mask.charCodeAt(index).toString(16).padStart(4, "0")}`).join(""); + const prefix = String.raw`before C:\temp\file.txt \\server\folder \n \t \b \f \r \" \/ \u0041 "bad` + '\t" "'; + const suffix = String.raw`" after D:\new\report.txt \\server\share \u0042 \\u006fpaque`; + const content = prefix + encodedMask + suffix; + expect(JSON.parse(redactArtifactMaskedValues(content, [mask]))).toBe(prefix + "***" + suffix); + expect(redactArtifactMaskedValues(content, ["absent"])).toBe(content); + }); + + it("preserves unmatched escapes between multiple conservative matches and overlapping candidates", () => { + const content = '"bad\t" ' + String.raw`"\u006fpaque" C:\temp\file.txt "\u006fpaque-tail" \\u006fpaque`; + expect(JSON.parse(redactArtifactMaskedValues(content, ["opaque-tail", "opaque"]))).toBe('"bad\t" ' + String.raw`"***" C:\temp\file.txt "***" \\u006fpaque`); + }); + + it("does not recursively decode literal Unicode escapes after malformed prefixes", () => { + const content = '"bad\t" ' + JSON.stringify("\\u006fpaque"); + expect(redactArtifactMaskedValues(content, ["opaque"])).toBe(content); + }); + + it.each(["", "plain", '"', '"unterminated', '"ends\\', '"ends\\u', '"ends\\u123', '"bad\\q', '""', '"" ""', '"safe"tail', '"safe" "unterminated'])("preserves unmatched or truncated input byte-for-byte (%j)", content => { + expect(redactArtifactMaskedValues(content, ["absent"])).toBe(content); + }); + + it.each(['"unterminated opaque', '"ends opaque\\', '"bad\\q opaque', 'opaque "safe" opaque', '"safe" opaque "unterminated'])("still applies raw masks outside recoverable tokens (%j)", content => { + expect(redactArtifactMaskedValues(content, ["opaque"])).toBe(content.replaceAll("opaque", "***")); + }); + + it("handles adjacent strings, empty strings, invalid prefixes, and trailing text together", () => { + const content = `start ""${encodedMask}""${encodedMask} "bad\\q ${encodedMask} trailing`; + const redactedToken = JSON.stringify("***"); + expect(redactArtifactMaskedValues(content, ["opaque"])).toBe(`start ""${redactedToken}""${redactedToken} "bad\\q ${redactedToken} trailing`); + }); + + it("keeps literal escaped Unicode distinct from an actual encoded mask", () => { + const literal = JSON.stringify("\\u006fpaque"); + expect(redactArtifactMaskedValues(`log ${literal} ${encodedMask}`, ["opaque"])).toBe(`log ${literal} "***"`); + }); + + it("keeps numeric-looking string tokens separate from original numeric tokens", () => { + const content = ' \t{"escaped":"\\u0031e400","quoted":"9007199254740993","ids":[1e400,9007199254740993,-0,1e+09]}\r\n'; + const redacted = redactArtifactMaskedValues(content, ["1e400", "9007199254740993"]); + expect(JSON.parse(redacted)).toEqual({ escaped: "***", quoted: "***", ids: ["***", "***", 0, 1000000000] }); + expect(redacted.startsWith(" \t")).toBe(true); + expect(redacted.endsWith("\r\n")).toBe(true); + }); + + it("matches a deterministic decoded-string oracle across generated escape combinations", () => { + const alphabet = ["a", '"', "\\", "/", "\b", "\f", "\n", "\r", "\t", "\u0000", "\u001f", "\u2028", "\ud800", "\ud83d\ude80"]; + for (let index = 0; index < 256; index++) { + const left = alphabet[index % alphabet.length]; + const right = alphabet[Math.floor(index / alphabet.length) % alphabet.length]; + const value = left + right + "opaque" + right + left; + const encoded = JSON.stringify(value); + const expected = JSON.stringify(value.replaceAll("opaque", "***")); + expect(redactArtifactMaskedValues(`record ${encoded} end`, ["opaque"]), `generated record ${index}`).toBe(`record ${expected} end`); + const untouched = `record ${encoded} end`; + expect(redactArtifactMaskedValues(untouched, ["absent"]), `unchanged record ${index}`).toBe(untouched); + } + }); - it("returns falsy input unchanged", () => { - expect(applyAddMaskRedaction("", ["x"])).toBe(""); - }); + it("redacts thousands of adjacent tokens without losing delimiters or unchanged bytes", () => { + const safe = '"safe\\\\\\"value"'; + const record = `prefix ${safe}:${encodedMask}:`; + expect(redactArtifactMaskedValues(record.repeat(4096) + "tail", ["opaque"])).toBe(`prefix ${safe}:"***":`.repeat(4096) + "tail"); + }); + }); + + it.each(["json", "mixed"])("redacts multi-megabyte %s records without exhausting the regex stack", shape => { + const mask = 'synthetic-"quoted"\\mask'; + const value = "x".repeat(10 * 1024 * 1024) + mask; + const record = JSON.stringify({ text: value, number: shape === "mixed" ? 42 : 12345678 }); + const prefix = shape === "mixed" ? "tool output: " : ""; + const redacted = redactArtifactMaskedValues(prefix + record, [mask, "12345678"]); + expect(redacted.includes(mask)).toBe(false); + expect(redacted.includes(JSON.stringify(mask).slice(1, -1))).toBe(false); + const parsed = JSON.parse(redacted.slice(prefix.length)); + expect(parsed.text.length).toBe(10 * 1024 * 1024 + 3); + expect(parsed.text.endsWith("***")).toBe(true); + expect(parsed.number).toBe(shape === "mixed" ? 42 : "***"); + }); + + it("handles many escaped quotes and both odd and even backslash runs", () => { + const mask = 'synthetic-"quoted"\\mask'; + const value = '\\"'.repeat(1024 * 1024) + mask + "\\\\"; + const record = JSON.stringify({ text: value, safe: "12345678" }); + const redacted = redactArtifactMaskedValues(record, [mask]); + const parsed = JSON.parse(redacted); + expect(parsed.text.length).toBe(2 * 1024 * 1024 + 5); + expect(parsed.text.endsWith("***\\\\")).toBe(true); + expect(parsed.safe).toBe("12345678"); + }); + + it.each([false, true])("redacts a multi-megabyte malformed escaped-quote record without rescanning its rejected prefix (explicit selector %j)", selector => { + const prefix = '"' + '\\"'.repeat(1024 * 1024); + const suffix = String.raw`\q "safe" C:\temp\file.txt`; + const content = prefix + String.raw`\u006fpaque` + suffix; + const replacement = selector ? () => "***" : undefined; + expect(JSON.parse(redactArtifactMaskedValues(content, ["opaque"], undefined, replacement))).toBe(prefix + "***" + suffix); + expect(redactArtifactMaskedValues(content, ["absent"], undefined, replacement)).toBe(content); + }); + + it.each(["\n", "\r", "\u2028", "\u2029"])("recovers encoded masks after an invalid escaped line terminator", terminator => { + const mask = 'synthetic-"quoted"\\mask'; + const prefix = '"invalid\\' + terminator; + expect(redactArtifactMaskedValues(prefix + JSON.stringify(mask), [mask])).toBe(prefix + '"***"'); + }); + + it.each(['"bad\\q ', '"bad\\u12gg ', '"bad\\u12 ', '"bad\u0001 ', '"bad\t ', '"bad\\v '])("recovers Unicode-encoded masks after invalid JSON string prefixes (%j)", prefix => { + expect(redactArtifactMaskedValues(prefix + '"\\u006fpaque"', ["opaque"])).toBe(prefix + '"***"'); + }); - describe("redactArtifactMaskedValues", () => { it("redacts decoded JSON strings and keys without mutating syntax or primitives", () => { const secret = 'quote"\\percent%0A'; const value = { [secret]: [secret, 0, false, null], nested: { output: "first\nsecond" } }; diff --git a/actions/setup/js/json_string_scanner.cjs b/actions/setup/js/json_string_scanner.cjs new file mode 100644 index 00000000000..9421f5a8c50 --- /dev/null +++ b/actions/setup/js/json_string_scanner.cjs @@ -0,0 +1,61 @@ +// @ts-check + +/** + * Visit quoted JSON tokens without a repeated regex alternation, which can + * exhaust the RegExp stack on multi-megabyte log records. + * Malformed prefixes are preserved as text; their quote pairing is ambiguous. + * @param {string} text + * @param {(token: string) => string} transformString + * @param {(text: string) => string} [transformText] + * @returns {string} + */ +function transformJSONStrings(text, transformString, transformText = value => value) { + const parts = []; + let cursor = 0; + while (cursor < text.length) { + const start = text.indexOf('"', cursor); + if (start === -1) { + parts.push(transformText(text.slice(cursor))); + break; + } + let end = start + 1; + let invalidString = false; + while (end < text.length && text[end] !== '"') { + if (text.charCodeAt(end) < 0x20) { + invalidString = true; + break; + } + if (text[end] === "\\") { + end++; + const escaped = text[end]; + if (escaped === "u") { + if (!/^[0-9a-fA-F]{4}$/.test(text.slice(end + 1, end + 5))) { + invalidString = true; + break; + } + end += 4; + } else if (!escaped || !'"\\/bfnrt'.includes(escaped)) { + invalidString = true; + break; + } + } + end++; + } + if (invalidString) { + // Do not rescan escaped quotes in the rejected prefix. + parts.push(transformText(text.slice(cursor, end))); + cursor = end; + continue; + } + if (end >= text.length) { + parts.push(transformText(text.slice(cursor))); + break; + } + parts.push(transformText(text.slice(cursor, start))); + parts.push(transformString(text.slice(start, end + 1))); + cursor = end + 1; + } + return parts.join(""); +} + +module.exports = { transformJSONStrings }; diff --git a/actions/setup/js/json_string_scanner.test.cjs b/actions/setup/js/json_string_scanner.test.cjs new file mode 100644 index 00000000000..932e7a53a22 --- /dev/null +++ b/actions/setup/js/json_string_scanner.test.cjs @@ -0,0 +1,92 @@ +// @ts-check +import { describe, expect, it, vi } from "vitest"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const { transformJSONStrings } = require("./json_string_scanner.cjs"); + +describe("JSON string scanner", () => { + it("passes complete quoted tokens and plain spans to separate callbacks", () => { + const strings = vi.fn(token => `[${token}]`); + const plain = vi.fn(text => `{${text}}`); + expect(transformJSONStrings('head "one" between "two" tail', strings, plain)).toBe('{head }["one"]{ between }["two"]{ tail}'); + expect(strings.mock.calls.map(([token]) => token)).toEqual(['"one"', '"two"']); + expect(plain.mock.calls.map(([text]) => text)).toEqual(["head ", " between ", " tail"]); + }); + + it("visits adjacent and empty tokens without separators or trailing text", () => { + const strings = vi.fn(token => token); + const content = '"""one""two"""'; + expect(transformJSONStrings(content, strings)).toBe(content); + expect(strings.mock.calls.map(([token]) => token)).toEqual(['""', '"one"', '"two"', '""']); + }); + + it.each(['"unterminated', '"ends\\', '"ends\\u', '"ends\\u123', 'plain "unterminated', '"safe" "ends\\'])("preserves truncated input and never transforms incomplete strings (%j)", content => { + const strings = vi.fn(token => token); + const plain = vi.fn(text => text); + expect(transformJSONStrings(content, strings, plain)).toBe(content); + expect(strings.mock.calls.map(([token]) => token)).toEqual(content.startsWith('"safe"') ? ['"safe"'] : []); + expect(plain.mock.calls.map(([text]) => text).join("")).toBe(content.startsWith('"safe"') ? content.slice(6) : content); + }); + + it.each(['"bad\t ', '"bad\\q ', '"bad\\u12gg '])("recovers valid tokens after invalid unterminated prefixes without losing bytes (%j)", prefix => { + const strings = vi.fn(token => token); + const plain = vi.fn(text => text); + expect(transformJSONStrings(prefix + '"\\u006fpaque" tail', strings, plain)).toBe(prefix + '"\\u006fpaque" tail'); + expect(strings.mock.calls.map(([token]) => token)).toEqual(['"\\u006fpaque"']); + expect(plain.mock.calls.map(([text]) => text).join("")).toBe(prefix + " tail"); + }); + + it("preserves bytes even when malformed terminated input makes quote pairing ambiguous", () => { + const strings = vi.fn(token => token); + const content = '"bad\t" "\\u006fpaque"'; + expect(transformJSONStrings(content, strings)).toBe(content); + expect(strings.mock.calls.map(([token]) => token)).toEqual(['" "']); + }); + + it.each([1024, 2048])("bounds character inspections after %i malformed escaped quotes", count => { + const prefix = '"' + '\\"'.repeat(count) + "\\q "; + const content = prefix + '"\\u006fpaque" tail'; + const strings = vi.fn(token => token); + const plain = vi.fn(text => text); + const original = String.prototype.charCodeAt; + let inspections = 0; + const spy = vi.spyOn(String.prototype, "charCodeAt").mockImplementation(function (index) { + inspections++; + return original.call(this, index); + }); + let result; + try { + result = transformJSONStrings(content, strings, plain); + } finally { + spy.mockRestore(); + } + expect(inspections).toBeLessThanOrEqual(content.length); + expect(result).toBe(content); + expect(strings.mock.calls.map(([token]) => token)).toEqual(['"\\u006fpaque"']); + expect(plain.mock.calls.map(([text]) => text).join("")).toBe(prefix + " tail"); + }); + + it("continues scanning when a callback removes tokens and adjacent spans", () => { + const content = 'before "" "remove" after "keep"'; + expect(transformJSONStrings(content, token => (token === '"remove"' ? "" : token))).toBe('before "" after "keep"'); + }); + + it.each(["string", "plain"])("propagates %s callback failures without adding source text", callback => { + const error = new Error("synthetic callback failure"); + const fail = () => { + throw error; + }; + expect(() => transformJSONStrings('plain "value"', callback === "string" ? fail : token => token, callback === "plain" ? fail : text => text)).toThrow(error); + }); + + it.each(["", "ordinary", 'quote"slash\\', "\u0000\b\f\n\r\t\u001f", "\ud800\udfff", "\\u006fpaque", '\\"'.repeat(32)])("only transforms a complete token at every possible truncation boundary (%j)", value => { + const encoded = JSON.stringify(value); + for (let length = 0; length <= encoded.length; length++) { + const strings = vi.fn(token => `[${token}]`); + const content = `prefix ${encoded.slice(0, length)}`; + expect(transformJSONStrings(content, strings)).toBe(length === encoded.length ? `prefix [${encoded}]` : content); + expect(strings.mock.calls.map(([token]) => token)).toEqual(length === encoded.length ? [encoded] : []); + } + }); +}); diff --git a/actions/setup/js/mask_value_matcher.cjs b/actions/setup/js/mask_value_matcher.cjs new file mode 100644 index 00000000000..07bc6942106 --- /dev/null +++ b/actions/setup/js/mask_value_matcher.cjs @@ -0,0 +1,132 @@ +// @ts-check + +/** Replacement used for masked values, matching the runner's own rendering. */ +const MASK_REPLACEMENT = "***"; + +/** + * Yield non-overlapping matches, breaking equal-position ties by mask order. + * @param {string} text + * @param {string[]} maskedValues + * @param {(match: {start: number, end: number, value: string}) => boolean} [allowed] + * @param {boolean} [overlapping] + * @yields {{start: number, end: number, value: string}} + */ +function* maskedValueMatches(text, maskedValues, allowed = () => true, overlapping = false) { + const pending = []; + for (let order = 0; order < maskedValues.length; order++) { + const value = maskedValues[order]; + if (!value) continue; + const index = text.indexOf(value); + if (index !== -1) pending.push({ value, index, order }); + } + const precedes = (a, b) => a.index < b.index || (a.index === b.index && a.order < b.order); + const siftDown = start => { + let parent = start; + while (parent * 2 + 1 < pending.length) { + let child = parent * 2 + 1; + if (child + 1 < pending.length && precedes(pending[child + 1], pending[child])) child++; + if (!precedes(pending[child], pending[parent])) break; + [pending[parent], pending[child]] = [pending[child], pending[parent]]; + parent = child; + } + }; + for (let i = Math.floor(pending.length / 2) - 1; i >= 0; i--) siftDown(i); + let cursor = 0; + while (pending.length && cursor < text.length) { + const next = pending[0]; + let searchStart = cursor; + if (next.index >= cursor) { + const match = { start: next.index, end: next.index + next.value.length, value: next.value }; + if (allowed(match)) { + yield match; + cursor = overlapping ? match.start + 1 : match.end; + searchStart = cursor; + } else { + searchStart = next.index + 1; + } + } + // Only refresh consumed or overlapping occurrences; later candidates stay cached. + next.index = text.indexOf(next.value, searchStart); + if (next.index === -1) { + const last = pending.pop(); + if (pending.length) pending[0] = last; + } + siftDown(0); + } +} + +/** + * @param {Iterable<{start: number, end: number, value: string, ambiguous?: boolean}>} matches + */ +function* overlappingMatchGroups(matches) { + let group; + for (const match of matches) { + if (group && match.start < group.end) { + group.end = Math.max(group.end, match.end); + if (match.value.length > group.value.length) group.value = match.value; + group.ambiguous ||= match.ambiguous; + } else { + if (group) yield group; + group = { ...match }; + } + } + if (group) yield group; +} + +/** + * Coalesce overlapping credentials before choosing their replacement marker. + * @param {string} text + * @param {string[]} maskedValues - Longest credentials first + * @param {(value: string) => string} replacementForValue + * @returns {string} + */ +function redactOverlappingMaskedValues(text, maskedValues, replacementForValue) { + const parts = []; + let cursor = 0; + for (const match of overlappingMatchGroups(maskedValueMatches(text, maskedValues, undefined, true))) { + parts.push(text.slice(cursor, match.start), replacementForValue(match.value)); + cursor = match.end; + } + parts.push(text.slice(cursor)); + return parts.join(""); +} + +/** + * Replace all occurrences of the masked values, defaulting to `***`. + * + * @param {string} text + * @param {string[]} maskedValues + * @param {(value: string) => string} [replacementForValue] - Explicit markers coalesce overlapping credentials + * @returns {string} + */ +function redactMaskedValues(text, maskedValues, replacementForValue) { + if (!text || !maskedValues || maskedValues.length === 0) return text; + if (replacementForValue) + return redactOverlappingMaskedValues( + text, + [...maskedValues].sort((a, b) => b.length - a.length), + replacementForValue + ); + const parts = []; + let cursor = 0; + let replacements = 0; + for (const match of maskedValueMatches(text, maskedValues)) { + if (match.start > cursor) { + if (replacements) parts.push(MASK_REPLACEMENT.repeat(replacements)); + replacements = 0; + parts.push(text.slice(cursor, match.start)); + } + replacements++; + cursor = match.end; + } + if (replacements) parts.push(MASK_REPLACEMENT.repeat(replacements)); + parts.push(text.slice(cursor)); + return parts.join(""); +} + +module.exports = { + MASK_REPLACEMENT, + maskedValueMatches, + overlappingMatchGroups, + redactMaskedValues, +}; diff --git a/actions/setup/js/mask_value_matcher.test.cjs b/actions/setup/js/mask_value_matcher.test.cjs new file mode 100644 index 00000000000..7a4fd9629c1 --- /dev/null +++ b/actions/setup/js/mask_value_matcher.test.cjs @@ -0,0 +1,88 @@ +// @ts-check +import { describe, expect, it, vi } from "vitest"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const { redactMaskedValues } = require("./mask_value_matcher.cjs"); + +describe("mask value matcher", () => { + it("preserves per-value replacement markers across adjacent and overlapping matches", () => { + const replacement = value => (value === "configured" ? "***REDACTED***" : "***"); + expect(redactMaskedValues("configuredruntimeconfiguredconfigured runtime", ["configured", "runtime", "figure"], replacement)).toBe("***REDACTED***" + "***" + "***REDACTED***".repeat(2) + " ***"); + expect(redactMaskedValues("ababa", ["aba"])).toBe("***ba"); + expect(redactMaskedValues("ababa", ["aba"], replacement)).toBe("***"); + }); + + it("replaces every occurrence with the default marker", () => { + expect(redactMaskedValues("token=abc and abc again", ["abc"])).toBe("token=*** and *** again"); + }); + + it("treats mask text literally instead of as a regular expression", () => { + expect(redactMaskedValues("value a.c here", ["a.c"])).toBe("value *** here"); + expect(redactMaskedValues("value abc here", ["a.c"])).toBe("value abc here"); + }); + + it("ignores empty masks and empty input", () => { + expect(redactMaskedValues("plain", [])).toBe("plain"); + expect(redactMaskedValues("plain", ["", ""])).toBe("plain"); + expect(redactMaskedValues("", ["secret"])).toBe(""); + }); + + it("redacts masks exceeding the regular expression size limit", () => { + const mask = "x".repeat(200000); + expect(redactMaskedValues(`before ${mask} after ${mask}`, [mask])).toBe("before *** after ***"); + expect(redactMaskedValues("unrelated output", [mask])).toBe("unrelated output"); + }); + + it("preserves leftmost matching and supplied mask priority for overlaps", () => { + expect(redactMaskedValues("abc ab", ["abc", "ab"])).toBe("*** ***"); + expect(redactMaskedValues("abc", ["ab", "abc"])).toBe("***c"); + expect(redactMaskedValues("zabc", ["abc", "za"])).toBe("***bc"); + }); + + it("does not rescan replacements or create matches across removed values", () => { + expect(redactMaskedValues("secret *", ["secret", "*"])).toBe("*** ***"); + expect(redactMaskedValues("axb", ["x", "ab"])).toBe("a***b"); + }); + + it("matches ordered regex semantics while refreshing overlapping cached candidates", () => { + const values = ["ab", "aba", "ba", "b", "a", "", "ab"]; + for (let offset = 0; offset < values.length; offset++) { + const masks = [...values.slice(offset), ...values.slice(0, offset)]; + const pattern = new RegExp(masks.filter(Boolean).join("|"), "g"); + for (let length = 0; length <= 6; length++) { + for (let bits = 0; bits < 2 ** length; bits++) { + const text = Array.from({ length }, (_, index) => (bits & (1 << index) ? "a" : "b")).join(""); + expect(redactMaskedValues(text, masks)).toBe(text.replace(pattern, "***")); + } + } + } + }); + + it("does not mutate mask ordering when choosing overlap replacements", () => { + const masks = ["ab", "abc"]; + expect(redactMaskedValues("abc", masks, value => `[${value}]`)).toBe("[abc]"); + expect(masks).toEqual(["ab", "abc"]); + }); + + it("does not re-search later masks for every frequent match", () => { + const masks = Array.from({ length: 500 }, (_, index) => `mask-${index.toString().padStart(4, "0")}`); + const repeats = 2000; + const suffix = masks.join(" "); + const text = "hot".repeat(repeats) + " ".repeat(2 * 1024 * 1024) + suffix; + const indexOf = String.prototype.indexOf; + let searches = 0; + const spy = vi.spyOn(String.prototype, "indexOf").mockImplementation(function (value, from) { + if (this === text) searches++; + return indexOf.call(this, value, from); + }); + let result; + try { + result = redactMaskedValues(text, [...masks, "hot"]); + } finally { + spy.mockRestore(); + } + expect(result).toBe("***".repeat(repeats) + " ".repeat(2 * 1024 * 1024) + masks.map(() => "***").join(" ")); + expect(searches).toBeLessThanOrEqual(masks.length * 2 + repeats + 1); + }); +}); diff --git a/actions/setup/js/redact_secrets.cjs b/actions/setup/js/redact_secrets.cjs index f1d6e4c7c18..f7f21e6dc39 100644 --- a/actions/setup/js/redact_secrets.cjs +++ b/actions/setup/js/redact_secrets.cjs @@ -10,7 +10,9 @@ const fs = require("fs"); const path = require("path"); const { getErrorMessage } = require("./error_helpers.cjs"); const { ERR_VALIDATION } = require("./error_codes.cjs"); -const { collectAddMaskedValues, redactArtifactMaskedValues } = require("./add_mask_redaction.cjs"); +const { applyAddMaskRedaction, collectAddMaskedValues, isAddMaskCommandLine, redactArtifactMaskedValues } = require("./add_mask_redaction.cjs"); +const { maskedValueMatches, overlappingMatchGroups } = require("./mask_value_matcher.cjs"); +const { redactSafeOutputContent } = require("./safe_output_redaction.cjs"); const { redactPiSessionHTML } = require("./pi_session_redaction.cjs"); const { getOTLPSecretValues } = require("./otlp_header_helpers.cjs"); const { redactSensitiveConfig } = require("./safe_outputs_config_redact.cjs"); @@ -303,6 +305,99 @@ function redactSecrets(content, secretValues) { return { content: redacted, redactionCount }; } +/** + * @param {string} content + * @returns {boolean} + */ +function isValidJSONArtifact(content) { + try { + JSON.parse(content); + return true; + } catch { + const records = content.split(/\r?\n/).filter(line => line.trim()); + return ( + records.length > 0 && + records.every(record => { + try { + JSON.parse(record); + return true; + } catch { + return false; + } + }) + ); + } +} + +/** + * Match all credential sources before any replacement or JSON serialization can + * fragment a longer secret or change its spelling. + * @param {string} content + * @param {string[]} secretValues + * @param {string[]} maskedValues + * @param {{decodedText?: boolean}} [options] - Treat already-decoded protocol payloads as literal text + * @returns {{content: string, redactionCount: number}} + */ +function redactArtifactContent(content, secretValues, maskedValues, options = {}) { + const configured = new Set(secretValues.filter(value => value && value.length >= 6)); + /** @type {Map} */ + const builtInValues = new Map(); + // Discovery only: include escaped built-in credentials without rewriting the + // original numeric lexemes, malformed records, or unchanged JSON strings. + const escapes = { '"': '"', "\\": "\\", "/": "/", b: "\b", f: "\f", n: "\n", r: "\r", t: "\t" }; + const decoded = options.decodedText ? content : content.replace(/\\(["\\/bfnrt]|u[0-9a-fA-F]{4})/g, (_, escape) => (escape[0] === "u" ? String.fromCharCode(parseInt(escape.slice(1), 16)) : escapes[escape])); + for (const text of new Set([content, decoded])) { + for (const { name, pattern } of BUILT_IN_PATTERNS) { + for (const match of text.matchAll(pattern)) builtInValues.set(match[0], name); + } + } + const values = [...new Set([...configured, ...builtInValues.keys(), ...maskedValues])].sort((a, b) => b.length - a.length); + let redactionCount = 0; + const builtInCounts = new Map(); + let configuredCount = 0; + const replacement = value => { + redactionCount++; + const name = builtInValues.get(value); + if (name) builtInCounts.set(name, (builtInCounts.get(name) || 0) + 1); + else if (configured.has(value)) configuredCount++; + return name || configured.has(value) ? "***REDACTED***" : "***"; + }; + // Protocol callbacks already receive decoded strings, not serialized artifacts. + const redacted = options.decodedText ? redactDecodedCredentialText(content, values, replacement) : redactArtifactMaskedValues(content, values, undefined, replacement, { textMode: !isValidJSONArtifact(content) }); + for (const [name, count] of builtInCounts) global.core?.info?.(`Redacted ${count} occurrence(s) of ${name}`); + if (configuredCount) global.core?.info?.(`Redacted ${configuredCount} occurrence(s) of a secret`); + return { content: redacted, redactionCount: redacted === content ? 0 : Math.max(1, redactionCount) }; +} + +/** + * Keep command lines intact until removal, unless a credential union crosses + * their boundary and must be replaced before removing any original text. + * @param {string} content + * @param {string[]} values + * @param {(value: string) => string} replacement + * @returns {string} + */ +function redactDecodedCredentialText(content, values, replacement) { + const commands = []; + let offset = 0; + for (const line of content.split("\n")) { + if (isAddMaskCommandLine(line)) commands.push({ start: offset, end: offset + line.length }); + offset += line.length + 1; + } + const parts = []; + let cursor = 0; + let commandIndex = 0; + for (const match of overlappingMatchGroups(maskedValueMatches(content, values, undefined, true))) { + while (commandIndex < commands.length && commands[commandIndex].end < match.start) commandIndex++; + const command = commands[commandIndex]; + if (command && match.start >= command.start && match.end <= command.end) continue; + parts.push(content.slice(cursor, match.start), replacement(match.value)); + cursor = match.end; + } + parts.push(content.slice(cursor)); + return applyAddMaskRedaction(parts.join(""), []); +} + /** * Validate and read the host-only AWF early configuration artifact. The exact * path contract and metadata checks prevent an environment variable from @@ -498,26 +593,29 @@ function processFile(filePath, secretValues, maskedValues = [], prepareForCustom let stage = "read"; try { const content = fs.readFileSync(filePath, "utf8"); + const inputBytes = Buffer.byteLength(content, "utf8"); + core.info(`Secret redaction input: bytes=${inputBytes}`); stage = "redact"; - const encodedResult = - path.extname(filePath).toLowerCase() === ".html" - ? redactPiSessionHTML(content, text => { - const masked = redactArtifactMaskedValues(text, maskedValues); - const builtIn = redactBuiltInPatterns(masked); - const custom = redactSecrets(builtIn.content, secretValues); - return { content: custom.content, redactionCount: builtIn.redactionCount + custom.redactionCount + (masked !== text ? 1 : 0) }; - }) - : { content, redactionCount: 0 }; - const runtimeRedacted = redactArtifactMaskedValues(encodedResult.content, maskedValues); - // First, redact built-in patterns - const builtInResult = redactBuiltInPatterns(runtimeRedacted); - let redacted = builtInResult.content; - let totalRedactions = encodedResult.redactionCount + builtInResult.redactionCount + (runtimeRedacted !== encodedResult.content ? 1 : 0); - - // Then, redact custom secrets - const customResult = redactSecrets(redacted, secretValues); - redacted = customResult.content; - totalRedactions += customResult.redactionCount; + const structured = redactSafeOutputContent( + filePath, + content, + [...maskedValues, ...secretValues.filter(value => value && value.length >= 6)].sort((a, b) => b.length - a.length), + text => redactArtifactContent(text, secretValues, maskedValues, { decodedText: true }).content + ); + if (structured !== null) { + const redactionCount = structured === content ? 0 : 1; + if (redactionCount || prepareForCustomMasking) { + stage = "write"; + writeProcessedFile(filePath, structured, Boolean(redactionCount)); + } + if (redactionCount) core.info(`Processed ${filePath}: ${redactionCount} redaction(s)`); + core.info(`Secret redaction sizes: input_bytes=${inputBytes} output_bytes=${Buffer.byteLength(structured, "utf8")} format=structured`); + return redactionCount; + } + const encodedResult = path.extname(filePath).toLowerCase() === ".html" ? redactPiSessionHTML(content, text => redactArtifactContent(text, secretValues, maskedValues)) : { content, redactionCount: 0 }; + const genericResult = redactArtifactContent(encodedResult.content, secretValues, maskedValues); + const redacted = genericResult.content; + const totalRedactions = encodedResult.redactionCount + genericResult.redactionCount; if (totalRedactions > 0 || prepareForCustomMasking) { stage = "write"; @@ -526,6 +624,7 @@ function processFile(filePath, secretValues, maskedValues = [], prepareForCustom if (totalRedactions > 0) { core.info(`Processed ${filePath}: ${totalRedactions} redaction(s)`); } + core.info(`Secret redaction sizes: input_bytes=${inputBytes} output_bytes=${Buffer.byteLength(redacted, "utf8")} format=text`); return totalRedactions; } catch (error) { if (isErrnoCode(error, "ENOENT") && !fs.existsSync(filePath)) { diff --git a/actions/setup/js/redact_secrets.test.cjs b/actions/setup/js/redact_secrets.test.cjs index ad0adf21f58..a65f8cc7318 100644 --- a/actions/setup/js/redact_secrets.test.cjs +++ b/actions/setup/js/redact_secrets.test.cjs @@ -59,7 +59,7 @@ describe("redact_secrets.cjs", () => { assertRedacted(fs.readFileSync(file, "utf8")); const logContent = fs.readFileSync(log, "utf8"); for (const value of [...passwords, ...encoded, ...headers]) expect(logContent).not.toContain(value); - expect(JSON.parse(logContent.split("\n")[2])).toBe("***"); + expect(logContent.split("\n")[2]).toBe("***"); expect(JSON.parse(fs.readFileSync(scalarFile, "utf8"))).toBe("***"); expect(mockCore.setFailed).not.toHaveBeenCalled(); @@ -127,6 +127,429 @@ describe("redact_secrets.cjs", () => { } }), describe("file handling", () => { + it.each(["ordinary.json", "safeoutputs.jsonl"])("logs UTF-8 input and output byte sizes without payload data for %s", filename => { + const { processFile } = require("./redact_secrets.cjs"); + const secret = "synthetic-private-value"; + const file = path.join(tempDir, filename); + const content = JSON.stringify({ type: "add_comment", body: "\u00e9 \ud83d\ude80 " + secret }); + fs.writeFileSync(file, content); + + expect(processFile(file, [secret])).toBeGreaterThan(0); + + const output = fs.readFileSync(file, "utf8"); + const format = filename === "safeoutputs.jsonl" ? "structured" : "text"; + expect(Buffer.byteLength(content, "utf8")).toBeGreaterThan(content.length); + expect(mockCore.info).toHaveBeenCalledWith(`Secret redaction input: bytes=${Buffer.byteLength(content, "utf8")}`); + expect(mockCore.info).toHaveBeenCalledWith(`Secret redaction sizes: input_bytes=${Buffer.byteLength(content, "utf8")} output_bytes=${Buffer.byteLength(output, "utf8")} format=${format}`); + expect(mockCore.info.mock.calls.flat().join("\n")).not.toContain(secret); + expect(mockCore.info.mock.calls.flat().join("\n")).not.toContain("\u00e9"); + }); + + it("logs equal byte sizes for unchanged artifacts", () => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, "ordinary.log"); + fs.writeFileSync(file, "public"); + + expect(processFile(file, [])).toBe(0); + + expect(mockCore.info).toHaveBeenCalledWith("Secret redaction sizes: input_bytes=6 output_bytes=6 format=text"); + }); + + it("logs input size before a fail-closed redaction error without reporting successful output", () => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, "safeoutputs.jsonl"); + const content = '{"body":"synthetic-private-value"'; + fs.writeFileSync(file, content); + + expect(processFile(file, ["synthetic-private-value"])).toBe(0); + + expect(fs.existsSync(file)).toBe(false); + expect(mockCore.info).toHaveBeenCalledWith(`Secret redaction input: bytes=${Buffer.byteLength(content, "utf8")}`); + expect(mockCore.info.mock.calls.flat().some(message => message.startsWith("Secret redaction sizes:"))).toBe(false); + expect(mockCore.info.mock.calls.flat().join("\n")).not.toContain("synthetic-private-value"); + }); + + it.each(["safeoutputs.jsonl", "agent_output.json"])("coalesces all credential sources in executable %s payloads", filename => { + const { processFile } = require("./redact_secrets.cjs"); + const token = "ghs_" + "A".repeat(40); + const cases = [ + { text: "SECRET_AB_LONGVALUE", secrets: ["AB_LONGVALUE"], masks: ["SECRET_AB"] }, + { text: "prefix-" + token + "-configured", secrets: ["prefix-" + token + "-configured"], masks: [] }, + { text: "before" + token, secrets: [], masks: ["beforeghs_"] }, + ]; + const file = path.join(tempDir, filename); + for (const { text, secrets, masks } of cases) { + const message = { type: "submit_pull_request_review", event: "COMMENT", pull_request_number: 12, body: `before [${text}] after` }; + const envelope = filename === "agent_output.json"; + fs.writeFileSync(file, JSON.stringify(envelope ? { items: [message], errors: [] } : message) + "\n"); + + expect(processFile(file, secrets, [...masks, "body", "COMMENT", "type", "items"])).toBeGreaterThan(0); + + const output = JSON.parse(fs.readFileSync(file, "utf8")); + const expected = { ...message, body: "before [***REDACTED***] after" }; + expect(output).toEqual(envelope ? { items: [expected], errors: [] } : expected); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + } + }); + + it.each(["safeoutputs.jsonl", "agent_output.json"])("keeps decoded free-form strings literal while redacting %s", filename => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, filename); + const values = [ + ['{ "key" : "SECRET_AB_LONGVALUE" }', '{ "key" : "***REDACTED***" }'], + ['"SECRET_AB_LONGVALUE"', '"***REDACTED***"'], + ["12345678", "***REDACTED***"], + [String.raw`literal \u0053ECRET_WORD \n C:\new\path`, String.raw`literal \u0053ECRET_WORD \n C:\new\path`], + ["first\nSECRET_AB_LONGVALUE\nlast", "first\n***REDACTED***\nlast"], + ['{ "key" : "public" }', '{ "key" : "public" }'], + [String.raw`before synthetic\npassword after`, "before ***REDACTED*** after"], + ["before synthetic\n::add-mask::middle\npassword after", "before ***REDACTED*** after"], + ["before\n::add-mask::unrelated\nafter", "before\nafter"], + ["before\n::add-mask::SECRET_AB_LONGVALUE\nafter", "before\nafter"], + ["before\n::add-mask::whole-command\nafter", "before\nafter"], + ["before runtime\n::add-mask::middle\npassword after", "before *** after"], + ]; + const record = Object.fromEntries(values.map(([value], index) => [`field-${index}`, value])); + record["SECRET_AB_LONGVALUE"] = "public"; + const message = { type: "ledger_append", record }; + const expected = { type: "ledger_append", record: { ...Object.fromEntries(values.map(([, value], index) => [`field-${index}`, value])), "***REDACTED***": "public" } }; + const envelope = filename === "agent_output.json"; + fs.writeFileSync(file, JSON.stringify(envelope ? { items: [message], errors: [] } : message)); + + expect( + processFile(file, ["AB_LONGVALUE", "12345678", "SECRET_WORD", String.raw`synthetic\npassword`, "synthetic\n::add-mask::middle\npassword"], ["SECRET_AB", "::add-mask::whole-command", "runtime\n::add-mask::middle\npassword"]) + ).toBeGreaterThan(0); + + expect(JSON.parse(fs.readFileSync(file, "utf8"))).toEqual(envelope ? { items: [expected], errors: [] } : expected); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + }); + + it.each(['export SYNTHETIC_DIR="/example/synthetic"', 'mkdir -p "/example/synthetic"'])("keeps complete-line quoted shell masks as plain redaction markers (%s)", line => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, "shell-mask.log"); + for (const configured of [false, true]) { + fs.writeFileSync(file, line + "\n"); + + expect(processFile(file, configured ? [line] : [], configured ? [] : [line])).toBeGreaterThan(0); + + expect(fs.readFileSync(file, "utf8")).toBe((configured ? "***REDACTED***" : "***") + "\n"); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + } + }); + it.each(['synthetic"password', "synthetic\\password", "synthetic\\npassword", "synthetic\tpassword", "synthetic\npassword", 'synthetic"\\\tpassword'])( + "redacts configured secrets before runtime masking can serialize malformed records (%j)", + secret => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, "combined.log"); + fs.writeFileSync(file, '"bad\t" "\\u006fpaque" token=' + secret); + + expect(processFile(file, [secret], ["opaque", "synthetic"])).toBeGreaterThan(0); + + const output = fs.readFileSync(file, "utf8"); + const decoded = output + .split("\n") + .map(line => { + try { + return JSON.parse(line); + } catch { + return line; + } + }) + .join("\n"); + expect(decoded).not.toContain(secret); + expect(decoded).not.toContain("password"); + expect(decoded).not.toContain("opaque"); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + expect(mockCore.info.mock.calls.flat().join("\n")).not.toContain(secret); + } + ); + + it("redacts invalid JSON as text while retaining encoded runtime-mask recovery", () => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, "invalid-json.log"); + const builtIn = "ghs_" + "A".repeat(40); + const content = `{"configured":"configured-secret","runtime":"opaque","builtIn":"${builtIn}", invalid}`; + fs.writeFileSync(file, content); + + expect(processFile(file, ["configured-secret"], ["opaque"])).toBeGreaterThan(0); + + expect(fs.readFileSync(file, "utf8")).toBe('{"configured":"***REDACTED***","runtime":"***","builtIn":"***REDACTED***", invalid}'); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + + fs.writeFileSync(file, '"bad\\q "\\u006fpaque"'); + expect(processFile(file, [], ["opaque"])).toBeGreaterThan(0); + expect(fs.readFileSync(file, "utf8")).toBe('"bad\\q "***"'); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + }); + + it.each(["configured", "runtime"])("redacts raw %s secrets inside recoverable strings in malformed JSON", source => { + const { processFile } = require("./redact_secrets.cjs"); + const secret = String.raw`synthetic\npassword`; + const file = path.join(tempDir, "malformed.json"); + const content = `{"token":"${secret}", "other":"\\u0041", broken`; + fs.writeFileSync(file, content, { mode: 0o600 }); + + expect(processFile(file, source === "configured" ? [secret] : [], source === "runtime" ? [secret] : [])).toBeGreaterThan(0); + + const output = fs.readFileSync(file, "utf8"); + expect(output).not.toContain(secret); + expect(output).not.toContain("password"); + expect(output).toContain('"other":"\\u0041"'); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + expect(mockCore.info.mock.calls.flat().join("\n")).not.toContain(secret); + }); + + it("redacts encoded credentials across malformed multiline strings without fragmenting overlapping secrets", () => { + const { processFile } = require("./redact_secrets.cjs"); + const secret = "synthetic\npassword"; + const runtime = "before" + secret; + const file = path.join(tempDir, "malformed.json"); + fs.writeFileSync(file, '{"token":"before\\u0073ynthetic\npassword", broken'); + + expect(processFile(file, [secret], [runtime])).toBeGreaterThan(0); + + const output = fs.readFileSync(file, "utf8"); + expect(output).not.toContain("password"); + expect(output).not.toContain("ynthetic"); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + }); + + it("matches multiline credentials before removing embedded add-mask command lines", () => { + const { processFile } = require("./redact_secrets.cjs"); + const secret = "synthetic\n::add-mask::ignored\npassword"; + const file = path.join(tempDir, "malformed.json"); + fs.writeFileSync(file, '{"token":"' + secret + '", broken'); + + expect(processFile(file, [secret])).toBeGreaterThan(0); + + const output = fs.readFileSync(file, "utf8"); + expect(output).not.toContain("synthetic"); + expect(output).not.toContain("password"); + expect(output).not.toContain("::add-mask::"); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + }); + + it.each(["configured", "runtime", "built-in"])("sanitizes %s credentials across malformed JSON mutations and encodings", source => { + const { processFile } = require("./redact_secrets.cjs"); + const secret = source === "built-in" ? "ghs_" + "A".repeat(40) : 'synthetic"\\\t\npassword'; + const unicode = Array.from(secret, char => "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")).join(""); + const spellings = [secret, JSON.stringify(secret).slice(1, -1), unicode]; + const file = path.join(tempDir, "malformed.jsonl"); + const mutations = [ + token => `{"token":"${token}"`, + token => `{"token":"${token}`, + token => `{"token":"${token}",}`, + token => `{"token":"${token}", "broken":"\\q"}`, + token => `"bad\t" {"token":"${token}"}`, + token => `{"safe":true}\n{"token":"${token}", broken`, + token => `{"safe":true}\n"\\u0070ublic"\n{"token":"${token}", broken`, + ]; + for (const spelling of spellings) { + for (const mutate of mutations) { + fs.writeFileSync(file, mutate(spelling), { mode: 0o600 }); + + expect(processFile(file, source === "configured" ? [secret] : [], source === "runtime" ? [secret] : [])).toBeGreaterThan(0); + + const output = fs.readFileSync(file, "utf8"); + let record = output; + try { + record = JSON.parse(output); + } catch { + // Sanitized malformed records need not become valid JSON. + } + const decoded = record.replace(/\\(["\\/bfnrt]|u[0-9a-fA-F]{4})/g, escape => JSON.parse('"' + escape + '"')); + expect(output).not.toContain(secret); + expect(output).not.toContain(spelling); + expect(decoded).not.toContain(secret); + expect(decoded).not.toContain(source === "built-in" ? "A".repeat(40) : "password"); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + expect(mockCore.warning).not.toHaveBeenCalled(); + } + } + }); + + it.each(['synthetic"password', "synthetic\\password", "synthetic\tpassword", "synthetic\npassword"])("redacts decoded configured secrets alongside overlapping runtime masks in JSON and mixed records (%j)", secret => { + const { processFile } = require("./redact_secrets.cjs"); + const record = JSON.stringify({ [secret]: secret, safe: "public" }); + const encoded = record.replace(/synthetic/g, "\\u0073ynthetic"); + for (const [name, input] of [ + ["valid.json", encoded], + ["mixed.log", "output: " + encoded], + ]) { + const file = path.join(tempDir, name); + fs.writeFileSync(file, input); + + expect(processFile(file, [secret], ["synthetic"])).toBeGreaterThan(0); + + const output = fs.readFileSync(file, "utf8"); + const decoded = JSON.parse(name === "valid.json" ? output : output.slice("output: ".length)); + expect(Object.keys(decoded)).toEqual(["***REDACTED***", "safe"]); + expect(Object.values(decoded)).toEqual(["***REDACTED***", "public"]); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + } + }); + + it.each([false, true])("removes full built-in and configured credentials before overlapping runtime masks (encoded=%s)", encoded => { + const { processFile } = require("./redact_secrets.cjs"); + const token = "ghs_" + "A".repeat(40); + const secret = "prefix-" + token + "-configured"; + const file = path.join(tempDir, "overlapping.json"); + const record = JSON.stringify({ token, secret, safe: "public" }); + fs.writeFileSync(file, encoded ? record.replace(/ghs_/g, "\\u0067hs_") : record); + + expect(processFile(file, [secret], ["ghs_", "prefix"])).toBeGreaterThan(0); + + expect(JSON.parse(fs.readFileSync(file, "utf8"))).toEqual({ token: "***REDACTED***", secret: "***REDACTED***", safe: "public" }); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + }); + + it.each([ + ["configured", "long-configured-secret", "ablong", "ab", "-configured-secret"], + ["built-in", "ghs_" + "A".repeat(40), "beforeghs_", "before", "A".repeat(40)], + ["numeric", "12345678", "9123", "9", "45678"], + ["multiline", "long\nconfigured-secret", "ablong\n", "ab", "configured-secret"], + ])("removes full %s credentials when a shorter runtime mask starts before their overlapping prefix", (kind, secret, runtime, prefix, suffix) => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, "crossing-overlap.log"); + const text = prefix + secret; + for (const content of [text, JSON.stringify({ [text]: text }), "output: " + JSON.stringify(text)]) { + fs.writeFileSync(file, content); + + expect(processFile(file, kind === "built-in" ? [] : [secret], [runtime])).toBeGreaterThan(0); + + const output = fs.readFileSync(file, "utf8"); + expect(output).not.toContain(suffix); + expect(output).toContain("***"); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + } + }); + + it.each([ + ["synthetic\\n", "synthetic\npassword", false], + ["synthetic\\u006e", "syntheticnpassword", true], + ['"password', '"passwordtail', false], + ])("does not fragment longer encoded runtime masks with raw configured escape prefixes (%j)", (secret, runtime, unicode) => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, "encoded-overlap.log"); + const encoded = JSON.stringify(runtime); + for (const prefix of ["output: ", '"bad\t" ']) { + fs.writeFileSync(file, prefix + (unicode ? encoded.replace("syntheticn", "synthetic\\u006e") : encoded)); + + expect(processFile(file, [secret], [runtime])).toBeGreaterThan(0); + + const output = fs.readFileSync(file, "utf8"); + let decoded = output; + try { + decoded = JSON.parse(output); + } catch { + // A mixed record can remain text instead of a quoted JSON string. + } + expect(decoded).toBe(prefix + '"***"'); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + } + }); + + it("redacts original configured numeric lexemes before a shorter runtime mask", () => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, "configured-numeric.json"); + fs.writeFileSync(file, '{"ids":[9007199254740993,12345678,1.2345678e+8],"safe":42}'); + + expect(processFile(file, ["9007199254740993", "12345678", "1.2345678e+8"], ["123", "900"])).toBeGreaterThan(0); + + expect(JSON.parse(fs.readFileSync(file, "utf8"))).toEqual({ ids: ["***REDACTED***", "***REDACTED***", "***REDACTED***"], safe: 42 }); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + }); + + it("preserves generic JSON formatting and untouched numeric lexemes when configured values change", () => { + const { processFile } = require("./redact_secrets.cjs"); + const file = path.join(tempDir, "formatted.json"); + fs.writeFileSync(file, '{\n "text": "synthetic-password",\n "ids": [9007199254740993, 1e2],\n "escaped": "\\u0070ublic"\n}\n'); + + expect(processFile(file, ["synthetic-password"])).toBeGreaterThan(0); + + expect(fs.readFileSync(file, "utf8")).toBe('{\n "text": "***REDACTED***",\n "ids": [9007199254740993, 1e2],\n "escaped": "\\u0070ublic"\n}\n'); + }); + + it("redacts decoded HTML session payloads and surrounding mixed text with one combined mask set", () => { + const { processFile } = require("./redact_secrets.cjs"); + const secret = 'synthetic"\\\tpassword'; + const malformed = '"bad\t" "\\u006fpaque" token=' + secret; + const payload = Buffer.from(JSON.stringify({ text: malformed, [secret]: secret }), "utf8").toString("base64"); + const file = path.join(tempDir, "session.html"); + fs.writeFileSync(file, `
${malformed}
`); + + expect(processFile(file, [secret], ["opaque", "synthetic"])).toBeGreaterThan(0); + + const raw = fs.readFileSync(file, "utf8"); + const output = raw.startsWith('"') ? JSON.parse(raw) : raw; + const session = JSON.parse(Buffer.from(output.match(/