Fix runtime-mask parsing and preserve safe-output schemas

actions/setup/js/json_string_scanner.cjs

@@ -0,0 +1,61 @@
+// @ts-check
+
+/**
+ * Visit quoted JSON tokens without a repeated regex alternation, which can
+ * exhaust the RegExp stack on multi-megabyte log records.
+ * Malformed prefixes are preserved as text; their quote pairing is ambiguous.
+ * @param {string} text
+ * @param {(token: string) => string} transformString
+ * @param {(text: string) => string} [transformText]
+ * @returns {string}
+ */
+function transformJSONStrings(text, transformString, transformText = value => value) {
+  const parts = [];
+  let cursor = 0;
+  while (cursor < text.length) {
+    const start = text.indexOf('"', cursor);
+    if (start === -1) {
+      parts.push(transformText(text.slice(cursor)));
+      break;
+    }
+    let end = start + 1;
+    let invalidString = false;
+    while (end < text.length && text[end] !== '"') {
+      if (text.charCodeAt(end) < 0x20) {
+        invalidString = true;
+        break;
+      }
+      if (text[end] === "\\") {
+        end++;
+        const escaped = text[end];
+        if (escaped === "u") {
+          if (!/^[0-9a-fA-F]{4}$/.test(text.slice(end + 1, end + 5))) {
+            invalidString = true;
+            break;
+          }
+          end += 4;
+        } else if (!escaped || !'"\\/bfnrt'.includes(escaped)) {
+          invalidString = true;
+          break;
+        }
+      }
+      end++;
+    }
+    if (invalidString) {
+      // Do not rescan escaped quotes in the rejected prefix.
+      parts.push(transformText(text.slice(cursor, end)));
+      cursor = end;
+      continue;
+    }
+    if (end >= text.length) {
+      parts.push(transformText(text.slice(cursor)));
+      break;
+    }
+    parts.push(transformText(text.slice(cursor, start)));
+    parts.push(transformString(text.slice(start, end + 1)));
+    cursor = end + 1;
+  }
+  return parts.join("");
+}
+
+module.exports = { transformJSONStrings };

actions/setup/js/json_string_scanner.test.cjs

@@ -0,0 +1,92 @@
+// @ts-check
+import { describe, expect, it, vi } from "vitest";
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+const { transformJSONStrings } = require("./json_string_scanner.cjs");
+
+describe("JSON string scanner", () => {
+  it("passes complete quoted tokens and plain spans to separate callbacks", () => {
+    const strings = vi.fn(token => `[${token}]`);
+    const plain = vi.fn(text => `{${text}}`);
+    expect(transformJSONStrings('head "one" between "two" tail', strings, plain)).toBe('{head }["one"]{ between }["two"]{ tail}');
+    expect(strings.mock.calls.map(([token]) => token)).toEqual(['"one"', '"two"']);
+    expect(plain.mock.calls.map(([text]) => text)).toEqual(["head ", " between ", " tail"]);
+  });
+
+  it("visits adjacent and empty tokens without separators or trailing text", () => {
+    const strings = vi.fn(token => token);
+    const content = '"""one""two"""';
+    expect(transformJSONStrings(content, strings)).toBe(content);
+    expect(strings.mock.calls.map(([token]) => token)).toEqual(['""', '"one"', '"two"', '""']);
+  });
+
+  it.each(['"unterminated', '"ends\\', '"ends\\u', '"ends\\u123', 'plain "unterminated', '"safe" "ends\\'])("preserves truncated input and never transforms incomplete strings (%j)", content => {
+    const strings = vi.fn(token => token);
+    const plain = vi.fn(text => text);
+    expect(transformJSONStrings(content, strings, plain)).toBe(content);
+    expect(strings.mock.calls.map(([token]) => token)).toEqual(content.startsWith('"safe"') ? ['"safe"'] : []);
+    expect(plain.mock.calls.map(([text]) => text).join("")).toBe(content.startsWith('"safe"') ? content.slice(6) : content);
+  });
+
+  it.each(['"bad\t ', '"bad\\q ', '"bad\\u12gg '])("recovers valid tokens after invalid unterminated prefixes without losing bytes (%j)", prefix => {
+    const strings = vi.fn(token => token);
+    const plain = vi.fn(text => text);
+    expect(transformJSONStrings(prefix + '"\\u006fpaque" tail', strings, plain)).toBe(prefix + '"\\u006fpaque" tail');
+    expect(strings.mock.calls.map(([token]) => token)).toEqual(['"\\u006fpaque"']);
+    expect(plain.mock.calls.map(([text]) => text).join("")).toBe(prefix + " tail");
+  });
+
+  it("preserves bytes even when malformed terminated input makes quote pairing ambiguous", () => {
+    const strings = vi.fn(token => token);
+    const content = '"bad\t" "\\u006fpaque"';
+    expect(transformJSONStrings(content, strings)).toBe(content);
+    expect(strings.mock.calls.map(([token]) => token)).toEqual(['" "']);
+  });
+
+  it.each([1024, 2048])("bounds character inspections after %i malformed escaped quotes", count => {
+    const prefix = '"' + '\\"'.repeat(count) + "\\q ";
+    const content = prefix + '"\\u006fpaque" tail';
+    const strings = vi.fn(token => token);
+    const plain = vi.fn(text => text);
+    const original = String.prototype.charCodeAt;
+    let inspections = 0;
+    const spy = vi.spyOn(String.prototype, "charCodeAt").mockImplementation(function (index) {
+      inspections++;
+      return original.call(this, index);
+    });
+    let result;
+    try {
+      result = transformJSONStrings(content, strings, plain);
+    } finally {
+      spy.mockRestore();
+    }
+    expect(inspections).toBeLessThanOrEqual(content.length);
+    expect(result).toBe(content);
+    expect(strings.mock.calls.map(([token]) => token)).toEqual(['"\\u006fpaque"']);
+    expect(plain.mock.calls.map(([text]) => text).join("")).toBe(prefix + " tail");
+  });
+
+  it("continues scanning when a callback removes tokens and adjacent spans", () => {
+    const content = 'before "" "remove" after "keep"';
+    expect(transformJSONStrings(content, token => (token === '"remove"' ? "" : token))).toBe('before ""  after "keep"');
+  });
+
+  it.each(["string", "plain"])("propagates %s callback failures without adding source text", callback => {
+    const error = new Error("synthetic callback failure");
+    const fail = () => {
+      throw error;
+    };
+    expect(() => transformJSONStrings('plain "value"', callback === "string" ? fail : token => token, callback === "plain" ? fail : text => text)).toThrow(error);
+  });
+
+  it.each(["", "ordinary", 'quote"slash\\', "\u0000\b\f\n\r\t\u001f", "\ud800\udfff", "\\u006fpaque", '\\"'.repeat(32)])("only transforms a complete token at every possible truncation boundary (%j)", value => {
+    const encoded = JSON.stringify(value);
+    for (let length = 0; length <= encoded.length; length++) {
+      const strings = vi.fn(token => `[${token}]`);
+      const content = `prefix ${encoded.slice(0, length)}`;
+      expect(transformJSONStrings(content, strings)).toBe(length === encoded.length ? `prefix [${encoded}]` : content);
+      expect(strings.mock.calls.map(([token]) => token)).toEqual(length === encoded.length ? [encoded] : []);
+    }
+  });
+});

actions/setup/js/mask_value_matcher.cjs

@@ -0,0 +1,132 @@
+// @ts-check
+
+/** Replacement used for masked values, matching the runner's own rendering. */
+const MASK_REPLACEMENT = "***";
+
+/**
+ * Yield non-overlapping matches, breaking equal-position ties by mask order.
+ * @param {string} text
+ * @param {string[]} maskedValues
+ * @param {(match: {start: number, end: number, value: string}) => boolean} [allowed]
+ * @param {boolean} [overlapping]
+ * @yields {{start: number, end: number, value: string}}
+ */
+function* maskedValueMatches(text, maskedValues, allowed = () => true, overlapping = false) {
+  const pending = [];
+  for (let order = 0; order < maskedValues.length; order++) {
+    const value = maskedValues[order];
+    if (!value) continue;
+    const index = text.indexOf(value);
+    if (index !== -1) pending.push({ value, index, order });
+  }
+  const precedes = (a, b) => a.index < b.index || (a.index === b.index && a.order < b.order);
+  const siftDown = start => {
+    let parent = start;
+    while (parent * 2 + 1 < pending.length) {
+      let child = parent * 2 + 1;
+      if (child + 1 < pending.length && precedes(pending[child + 1], pending[child])) child++;
+      if (!precedes(pending[child], pending[parent])) break;
+      [pending[parent], pending[child]] = [pending[child], pending[parent]];
+      parent = child;
+    }
+  };
+  for (let i = Math.floor(pending.length / 2) - 1; i >= 0; i--) siftDown(i);
+  let cursor = 0;
+  while (pending.length && cursor < text.length) {
+    const next = pending[0];
+    let searchStart = cursor;
+    if (next.index >= cursor) {
+      const match = { start: next.index, end: next.index + next.value.length, value: next.value };
+      if (allowed(match)) {
+        yield match;
+        cursor = overlapping ? match.start + 1 : match.end;
+        searchStart = cursor;
+      } else {
+        searchStart = next.index + 1;
+      }
+    }
+    // Only refresh consumed or overlapping occurrences; later candidates stay cached.
+    next.index = text.indexOf(next.value, searchStart);
+    if (next.index === -1) {
+      const last = pending.pop();
+      if (pending.length) pending[0] = last;
+    }
+    siftDown(0);
+  }
+}
+
+/**
+ * @param {Iterable<{start: number, end: number, value: string, ambiguous?: boolean}>} matches
+ */
+function* overlappingMatchGroups(matches) {
+  let group;
+  for (const match of matches) {
+    if (group && match.start < group.end) {
+      group.end = Math.max(group.end, match.end);
+      if (match.value.length > group.value.length) group.value = match.value;
+      group.ambiguous ||= match.ambiguous;
+    } else {
+      if (group) yield group;
+      group = { ...match };
+    }
+  }
+  if (group) yield group;
+}
+
+/**
+ * Coalesce overlapping credentials before choosing their replacement marker.
+ * @param {string} text
+ * @param {string[]} maskedValues - Longest credentials first
+ * @param {(value: string) => string} replacementForValue
+ * @returns {string}
+ */
+function redactOverlappingMaskedValues(text, maskedValues, replacementForValue) {
+  const parts = [];
+  let cursor = 0;
+  for (const match of overlappingMatchGroups(maskedValueMatches(text, maskedValues, undefined, true))) {
+    parts.push(text.slice(cursor, match.start), replacementForValue(match.value));
+    cursor = match.end;
+  }
+  parts.push(text.slice(cursor));
+  return parts.join("");
+}
+
+/**
+ * Replace all occurrences of the masked values, defaulting to `***`.
+ *
+ * @param {string} text
+ * @param {string[]} maskedValues
+ * @param {(value: string) => string} [replacementForValue] - Explicit markers coalesce overlapping credentials
+ * @returns {string}
+ */
+function redactMaskedValues(text, maskedValues, replacementForValue) {
+  if (!text || !maskedValues || maskedValues.length === 0) return text;
+  if (replacementForValue)
+    return redactOverlappingMaskedValues(
+      text,
+      [...maskedValues].sort((a, b) => b.length - a.length),
+      replacementForValue
+    );
+  const parts = [];
+  let cursor = 0;
+  let replacements = 0;
+  for (const match of maskedValueMatches(text, maskedValues)) {
+    if (match.start > cursor) {
+      if (replacements) parts.push(MASK_REPLACEMENT.repeat(replacements));
+      replacements = 0;
+      parts.push(text.slice(cursor, match.start));
+    }
+    replacements++;
+    cursor = match.end;
+  }
+  if (replacements) parts.push(MASK_REPLACEMENT.repeat(replacements));
+  parts.push(text.slice(cursor));
+  return parts.join("");
+}
+
+module.exports = {
+  MASK_REPLACEMENT,
+  maskedValueMatches,
+  overlappingMatchGroups,
+  redactMaskedValues,
+};

actions/setup/js/mask_value_matcher.test.cjs

@@ -0,0 +1,88 @@
+// @ts-check
+import { describe, expect, it, vi } from "vitest";
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+const { redactMaskedValues } = require("./mask_value_matcher.cjs");
+
+describe("mask value matcher", () => {
+  it("preserves per-value replacement markers across adjacent and overlapping matches", () => {
+    const replacement = value => (value === "configured" ? "***REDACTED***" : "***");
+    expect(redactMaskedValues("configuredruntimeconfiguredconfigured runtime", ["configured", "runtime", "figure"], replacement)).toBe("***REDACTED***" + "***" + "***REDACTED***".repeat(2) + " ***");
+    expect(redactMaskedValues("ababa", ["aba"])).toBe("***ba");
+    expect(redactMaskedValues("ababa", ["aba"], replacement)).toBe("***");
+  });
+
+  it("replaces every occurrence with the default marker", () => {
+    expect(redactMaskedValues("token=abc and abc again", ["abc"])).toBe("token=*** and *** again");
+  });
+
+  it("treats mask text literally instead of as a regular expression", () => {
+    expect(redactMaskedValues("value a.c here", ["a.c"])).toBe("value *** here");
+    expect(redactMaskedValues("value abc here", ["a.c"])).toBe("value abc here");
+  });
+
+  it("ignores empty masks and empty input", () => {
+    expect(redactMaskedValues("plain", [])).toBe("plain");
+    expect(redactMaskedValues("plain", ["", ""])).toBe("plain");
+    expect(redactMaskedValues("", ["secret"])).toBe("");
+  });
+
+  it("redacts masks exceeding the regular expression size limit", () => {
+    const mask = "x".repeat(200000);
+    expect(redactMaskedValues(`before ${mask} after ${mask}`, [mask])).toBe("before *** after ***");
+    expect(redactMaskedValues("unrelated output", [mask])).toBe("unrelated output");
+  });
+
+  it("preserves leftmost matching and supplied mask priority for overlaps", () => {
+    expect(redactMaskedValues("abc ab", ["abc", "ab"])).toBe("*** ***");
+    expect(redactMaskedValues("abc", ["ab", "abc"])).toBe("***c");
+    expect(redactMaskedValues("zabc", ["abc", "za"])).toBe("***bc");
+  });
+
+  it("does not rescan replacements or create matches across removed values", () => {
+    expect(redactMaskedValues("secret *", ["secret", "*"])).toBe("*** ***");
+    expect(redactMaskedValues("axb", ["x", "ab"])).toBe("a***b");
+  });
+
+  it("matches ordered regex semantics while refreshing overlapping cached candidates", () => {
+    const values = ["ab", "aba", "ba", "b", "a", "", "ab"];
+    for (let offset = 0; offset < values.length; offset++) {
+      const masks = [...values.slice(offset), ...values.slice(0, offset)];
+      const pattern = new RegExp(masks.filter(Boolean).join("|"), "g");
+      for (let length = 0; length <= 6; length++) {
+        for (let bits = 0; bits < 2 ** length; bits++) {
+          const text = Array.from({ length }, (_, index) => (bits & (1 << index) ? "a" : "b")).join("");
+          expect(redactMaskedValues(text, masks)).toBe(text.replace(pattern, "***"));
+        }
+      }
+    }
+  });
+
+  it("does not mutate mask ordering when choosing overlap replacements", () => {
+    const masks = ["ab", "abc"];
+    expect(redactMaskedValues("abc", masks, value => `[${value}]`)).toBe("[abc]");
+    expect(masks).toEqual(["ab", "abc"]);
+  });
+
+  it("does not re-search later masks for every frequent match", () => {
+    const masks = Array.from({ length: 500 }, (_, index) => `mask-${index.toString().padStart(4, "0")}`);
+    const repeats = 2000;
+    const suffix = masks.join(" ");
+    const text = "hot".repeat(repeats) + " ".repeat(2 * 1024 * 1024) + suffix;
+    const indexOf = String.prototype.indexOf;
+    let searches = 0;
+    const spy = vi.spyOn(String.prototype, "indexOf").mockImplementation(function (value, from) {
+      if (this === text) searches++;
+      return indexOf.call(this, value, from);
+    });
+    let result;
+    try {
+      result = redactMaskedValues(text, [...masks, "hot"]);
+    } finally {
+      spy.mockRestore();
+    }
+    expect(result).toBe("***".repeat(repeats) + " ".repeat(2 * 1024 * 1024) + masks.map(() => "***").join(" "));
+    expect(searches).toBeLessThanOrEqual(masks.length * 2 + repeats + 1);
+  });
+});