zap-extract.js

// Zap form extractor — JavaScript port of zap/extract.py (keep the two in lockstep).
// Runs on a live DOM (browser extension content script) or jsdom.
//   const forms = zapExtract(document, location.href);
//   forms[i] = { root, elements, ctx, fields, text }   // `text` is what the Zap model reads
(function (global) {
  "use strict";

  const SKIP_INPUT_TYPES = new Set(["hidden", "submit", "button", "reset", "image", "file", "checkbox", "radio", "range", "color"]);
  const FIELD_TAGS = new Set(["input", "select", "textarea"]);
  const NO_TEXT_TAGS = new Set(["script", "style", "noscript", "template", "svg", "select", "textarea", "option", "head"]);
  const HEADING_TAGS = new Set(["h1", "h2", "h3", "h4", "h5", "h6", "legend"]);
  const DATA_ATTRS = ["data-testid", "data-test", "data-qa", "data-cy", "formcontrolname", "data-field", "data-name", "data-type"];
  const MAX_FIELDS = 40;
  const MAX_FORMS = 10;
  const CTX_LIMITS = { title: 60, url: 60, form: 60, head: 60, btn: 60, link: 80, txt: 120, hid: 40 };
  const FIELD_LIMITS = { name: 40, id: 40, class: 30, ac: 30, ph: 50, lbl: 60, aria: 50, tip: 40, near: 50, after: 30, mode: 12, max: 6, opt: 60, data: 30 };
  const CTX_ORDER = [["title", "[TITLE]"], ["url", "[URL]"], ["form", "[FORM]"], ["head", "[HEAD]"], ["btn", "[BTN]"], ["link", "[LINK]"], ["txt", "[TXT]"], ["hid", "[HID]"]];
  const FIELD_ORDER = [["type", "[TYPE]"], ["name", "[NAME]"], ["id", "[ID]"], ["class", "[CLASS]"], ["ac", "[AC]"], ["ph", "[PH]"], ["lbl", "[LBL]"], ["aria", "[ARIA]"], ["tip", "[TIP]"], ["near", "[NEAR]"], ["after", "[AFTER]"], ["mode", "[MODE]"], ["max", "[MAX]"], ["req", "[REQ]"], ["opt", "[OPT]"], ["data", "[DATA]"]];

  const ELEMENT = 1, TEXT = 3, CDATA = 4;

  function norm(s) {
    if (!s) return "";
    return s.normalize("NFKC").replace(/\s+/g, " ").trim();
  }
  function cut(s, n) {
    const cps = Array.from(s);
    return cps.length <= n ? s.trimEnd() : cps.slice(0, n).join("").trimEnd();
  }
  function words(s) {
    if (!s) return "";
    s = s.normalize("NFKC");
    s = s.replace(/([a-z])([A-Z])/g, "$1 $2");
    s = s.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
    s = s.replace(/([A-Za-z])([0-9])/g, "$1 $2");
    s = s.replace(/([0-9])([A-Za-z])/g, "$1 $2");
    s = s.replace(/[^\p{L}\p{N}\p{M}]/gu, " ");
    return s.replace(/\s+/g, " ").trim().toLowerCase();
  }
  function tag(el) { return el && el.nodeType === ELEMENT ? el.localName.toLowerCase() : ""; }
  function attr(el, name) { const v = el.getAttribute(name); return v === null ? "" : v; }
  function has(el, name) { return el.hasAttribute(name); }
  function unquote(p) { try { return decodeURIComponent(p); } catch (e) { return p; } }

  function text(el) {
    const parts = [];
    (function walk(e) {
      if (NO_TEXT_TAGS.has(tag(e))) return;
      for (let c = e.firstChild; c; c = c.nextSibling) {
        if (c.nodeType === ELEMENT) walk(c);
        else if (c.nodeType === TEXT || c.nodeType === CDATA) parts.push(c.data);
      }
    })(el);
    return norm(parts.join(" "));
  }
  function inputType(el) {
    const t = tag(el);
    if (t === "input") return attr(el, "type").trim().toLowerCase() || "text";
    return t;
  }
  function hiddenAttr(el) {
    if (has(el, "hidden") || attr(el, "aria-hidden").trim().toLowerCase() === "true") return true;
    const style = attr(el, "style").replace(/ /g, "").toLowerCase();
    return style.includes("display:none") || style.includes("visibility:hidden");
  }
  function isField(el) {
    const t = tag(el);
    if (!FIELD_TAGS.has(t)) return false;
    if (t === "input" && SKIP_INPUT_TYPES.has(inputType(el))) return false;
    return !has(el, "disabled");
  }
  function visibleWithin(el, root) {
    for (let n = el; n; n = n.parentElement) {
      if (hiddenAttr(n)) return false;
      if (n === root) return true;
    }
    return true;
  }
  function containsField(el) {
    return FIELD_TAGS.has(tag(el)) || !!el.querySelector("input,select,textarea");
  }

  // Previous / next siblings in DOM order, nearest first: ["t", string] or ["e", element]
  function* prevNodes(el) {
    for (let n = el.previousSibling; n; n = n.previousSibling) {
      if (n.nodeType === TEXT || n.nodeType === CDATA) yield ["t", n.data];
      else if (n.nodeType === ELEMENT) yield ["e", n];
    }
  }
  function* nextNodes(el) {
    for (let n = el.nextSibling; n; n = n.nextSibling) {
      if (n.nodeType === TEXT || n.nodeType === CDATA) yield ["t", n.data];
      else if (n.nodeType === ELEMENT) yield ["e", n];
    }
  }
  function nearText(el, root) {
    let node = el;
    for (let level = 0; level < 3; level++) {
      let seen = 0;
      for (const [kind, v] of prevNodes(node)) {
        let t;
        if (kind === "t") t = norm(v);
        else { if (containsField(v)) return ""; t = text(v); }
        if (t) return t;
        if (kind === "e") { seen++; if (seen >= 3) break; }
      }
      if (node === root) break;
      node = node.parentElement;
      if (!node || node === root) break;
    }
    return "";
  }
  function afterText(el) {
    let seen = 0;
    for (const [kind, v] of nextNodes(el)) {
      let t;
      if (kind === "t") t = norm(v);
      else { if (containsField(v)) return ""; t = text(v); }
      if (t) return t;
      if (kind === "e") { seen++; if (seen >= 2) break; }
    }
    return "";
  }

  function makePage(doc, url) {
    const byId = new Map(), labelFor = new Map();
    for (const e of doc.getElementsByTagName("*")) {
      const i = e.getAttribute("id");
      if (i && !byId.has(i)) byId.set(i, e);
      if (tag(e) === "label" && e.getAttribute("for")) {
        const f = e.getAttribute("for");
        if (!labelFor.has(f)) labelFor.set(f, []);
        labelFor.get(f).push(e);
      }
    }
    const t = doc.querySelector("title");
    return { doc, url, byId, labelFor, title: t ? norm(t.textContent) : "" };
  }
  function labelText(el, page) {
    const out = [];
    for (const ref of attr(el, "aria-labelledby").split(/\s+/).filter(Boolean)) {
      const e = page.byId.get(ref);
      if (e) out.push(text(e));
    }
    const i = el.getAttribute("id");
    if (i) for (const lab of page.labelFor.get(i) || []) out.push(text(lab));
    for (let n = el.parentElement; n; n = n.parentElement) {
      if (tag(n) === "label") { out.push(text(n)); break; }
    }
    const uniq = [];
    for (const t of out) if (t && !uniq.includes(t)) uniq.push(t);
    return uniq.join(" | ");
  }

  function isButtonish(e) {
    const t = tag(e);
    if (t === "button") return true;
    if (t === "input" && ["submit", "button", "image"].includes(inputType(e))) return true;
    return attr(e, "role").toLowerCase() === "button";
  }
  function isHeading(e) { return HEADING_TAGS.has(tag(e)) || attr(e, "role").toLowerCase() === "heading"; }
  function* iterSelf(root) { yield root; yield* root.getElementsByTagName("*"); }

  function heading(root) {
    for (const e of iterSelf(root)) {
      if (isHeading(e)) { const t = text(e); if (t) return t; }
    }
    let node = root;
    for (let level = 0; level < 4; level++) {
      let n = 0;
      for (const [kind, sib] of prevNodes(node)) {
        if (kind !== "e") continue;
        n++;
        if (n > 5) break;
        if (isHeading(sib)) { const t = text(sib); if (t) return t; }
        const hs = Array.from(sib.getElementsByTagName("*")).filter(isHeading);
        for (let k = hs.length - 1; k >= 0; k--) { const t = text(hs[k]); if (t) return t; }
      }
      node = node.parentElement;
      if (!node || tag(node) === "body" || tag(node) === "html") break;
    }
    return "";
  }
  function buttonTexts(root) {
    const out = [];
    for (const e of iterSelf(root)) {
      if (!isButtonish(e)) continue;
      const t = text(e) || norm(attr(e, "value")) || norm(attr(e, "aria-label")) || norm(attr(e, "alt")) || norm(attr(e, "title"));
      if (t && !out.includes(t)) out.push(cut(t, 30));
      if (out.length >= 3) break;
    }
    return out.join(" | ");
  }
  function linkTexts(root) {
    const out = [];
    for (const e of iterSelf(root)) {
      if (tag(e) !== "a") continue;
      const t = text(e) || norm(attr(e, "aria-label")) || norm(attr(e, "title"));
      if (t && !out.includes(t)) out.push(cut(t, 30));
      if (out.length >= 4) break;
    }
    return out.join(" | ");
  }
  function hiddenNames(root) {
    const out = [];
    for (const e of iterSelf(root)) {
      if (tag(e) === "input" && inputType(e) === "hidden") {
        const w = words(attr(e, "name"));
        if (w && !out.includes(w)) out.push(w);
        if (out.length >= 4) break;
      }
    }
    return out.join(" ");
  }
  function urlPath(u, base) {
    try { const x = base ? new URL(u, base) : new URL(u); return { host: x.hostname, path: x.pathname }; } catch (e) { return null; }
  }
  function urlWords(url) {
    const u = urlPath(url);
    if (!u) return "";
    const host = u.host.split(".");
    const lead = host.length >= 3 && host[0] !== "www" ? host[0] : "";
    return words(lead + " " + unquote(u.path));
  }
  function formWords(root) {
    const parts = [attr(root, "id"), attr(root, "name"), attr(root, "class")];
    if (tag(root) === "form") {
      const a = attr(root, "action");
      const u = a ? urlPath(a, "http://x/") : null;
      parts.push(u ? unquote(u.path) : "");
    } else parts.push(attr(root, "role"));
    return (words(parts.join(" ")) + " " + norm(attr(root, "aria-label")).toLowerCase()).trim();
  }

  function fieldFeatures(el, root, page) {
    const t = tag(el);
    const f = { type: t !== "input" ? t : "input " + inputType(el) };
    f.name = words(attr(el, "name"));
    f.id = words(attr(el, "id"));
    f.class = words(attr(el, "class"));
    f.ac = norm(attr(el, "autocomplete")).toLowerCase();
    f.ph = norm(attr(el, "placeholder"));
    f.lbl = labelText(el, page);
    f.aria = norm(attr(el, "aria-label"));
    f.tip = norm(attr(el, "title"));
    const near = nearText(el, root);
    f.near = near === f.lbl || near === f.ph || near === f.aria ? "" : near;
    f.after = afterText(el);
    f.mode = norm(attr(el, "inputmode")).toLowerCase();
    f.max = norm(attr(el, "maxlength"));
    f.req = has(el, "required") || attr(el, "aria-required").toLowerCase() === "true";
    if (t === "select") {
      const opts = Array.from(el.getElementsByTagName("option")).map((o) => norm(o.textContent)).filter(Boolean);
      f.opt = opts.length ? opts.length + ": " + opts.slice(0, 6).join(" | ") : "";
    } else f.opt = "";
    const data = DATA_ATTRS.filter((a) => attr(el, a).trim()).map((a) => norm(attr(el, a)));
    f.data = words(data.slice(0, 2).join(" "));
    for (const [k, n] of Object.entries(FIELD_LIMITS)) if (typeof f[k] === "string") f[k] = cut(f[k], n);
    return f;
  }
  function formContext(root, page) {
    const c = { title: page.title, url: urlWords(page.url), form: formWords(root), head: heading(root),
      btn: buttonTexts(root), link: linkTexts(root), txt: text(root), hid: hiddenNames(root) };
    for (const k of Object.keys(c)) c[k] = cut(c[k], CTX_LIMITS[k]);
    return c;
  }
  function groupRoot(el, body) {
    let node = el.parentElement;
    for (let level = 0; level < 6; level++) {
      if (!node || node === body || tag(node) === "body" || tag(node) === "html") break;
      for (const d of iterSelf(node)) {
        if (isButtonish(d) && !d.closest("form")) return node;
      }
      node = node.parentElement;
    }
    node = el;
    for (let level = 0; level < 3; level++) {
      const p = node.parentElement;
      if (!p || p === body || tag(p) === "body" || tag(p) === "html") break;
      node = p;
    }
    return node;
  }
  function findForms(doc) {
    const body = doc.body || doc.documentElement;
    const groups = new Map();
    for (const el of body.getElementsByTagName("*")) {
      if (!isField(el)) continue;
      let root = el.parentElement ? el.parentElement.closest("form") : null;
      if (!root) root = groupRoot(el, body);
      if (!visibleWithin(el, root)) continue;
      if (!groups.has(root)) groups.set(root, []);
      const g = groups.get(root);
      if (g.length < MAX_FIELDS) g.push(el);
    }
    return Array.from(groups.entries()).slice(0, MAX_FORMS);
  }

  function serializeCtx(ctx) {
    return CTX_ORDER.filter(([k]) => ctx[k]).map(([k, key]) => key + " " + ctx[k]).join(" ");
  }
  function serializeField(f) {
    const out = ["[FLD]"];
    for (const [k, key] of FIELD_ORDER) {
      const v = f[k];
      if (k === "req") { if (v) out.push("[REQ]"); } else if (v) out.push(key + " " + v);
    }
    return out.join(" ");
  }
  function serialize(ctx, fields) {
    return [serializeCtx(ctx)].concat(fields.map(serializeField)).join(" ");
  }

  function zapExtract(doc, url) {
    const page = makePage(doc, url);
    return findForms(doc).map(([root, elements]) => {
      const fields = elements.map((el) => fieldFeatures(el, root, page));
      const ctx = formContext(root, page);
      return { kind: tag(root) === "form" ? "form" : "group", root, elements, ctx, fields, text: serialize(ctx, fields) };
    });
  }

  const api = { zapExtract, serialize, serializeCtx, serializeField, words, norm };
  if (typeof module !== "undefined" && module.exports) module.exports = api;
  else global.Zap = Object.assign(global.Zap || {}, api);
})(typeof globalThis !== "undefined" ? globalThis : this);