lib/onpage.js

// On-page SEO extraction from a full HTML document (net-new — wp_to_markdown.js
// only ever saw WP body HTML, never a <head>). Returns structured intel + a clean
// Markdown copy of the main content.

const cheerio = require("cheerio");
const TurndownService = require("turndown");
const { gfm } = require("turndown-plugin-gfm");

const turndown = new TurndownService({
  headingStyle: "atx",
  codeBlockStyle: "fenced",
  bulletListMarker: "-",
  hr: "---",
});
turndown.use(gfm);
turndown.addRule("dropImages", {
  filter: "img",
  replacement: (_c, node) => {
    const alt = (node.getAttribute("alt") || "").trim();
    return alt.length > 3 ? ` (image: ${alt}) ` : "";
  },
});

const STOP = new Set(
  ("a an the and or but of to in on for with at by from as is are was were be been being this that these those you your we our it its their they them how what why when where which who can do does will more most other new get make use using help need want take know like just dont do not vs amp nbsp about into out over under has have had i if so no than then there here also may might should could would each any all one two").split(
    /\s+/,
  ),
);

function toMarkdown(html) {
  try {
    return turndown
      .turndown(html || "")
      .replace(/[ \t]+\n/g, "\n")
      .replace(/\n{3,}/g, "\n\n")
      .trim();
  } catch {
    return cheerio.load(html || "").text().trim();
  }
}

function tokenize(text) {
  return (text.toLowerCase().match(/\b[a-z][a-z'-]+\b/g) || []).filter(
    (w) => w.length > 2 && !STOP.has(w),
  );
}

// Density of a phrase (1+ words) in a token stream, as a percentage of total tokens.
function phraseDensity(phraseTokens, tokens) {
  if (!phraseTokens.length || !tokens.length) return 0;
  if (phraseTokens.length === 1) {
    const c = tokens.filter((t) => t === phraseTokens[0]).length;
    return +((100 * c) / tokens.length).toFixed(3);
  }
  let count = 0;
  for (let i = 0; i <= tokens.length - phraseTokens.length; i++) {
    let ok = true;
    for (let j = 0; j < phraseTokens.length; j++) {
      if (tokens[i + j] !== phraseTokens[j]) {
        ok = false;
        break;
      }
    }
    if (ok) count++;
  }
  return +((100 * count) / tokens.length).toFixed(3);
}

function topTerms(tokens, n = 25) {
  const uni = new Map();
  const bi = new Map();
  for (let i = 0; i < tokens.length; i++) {
    uni.set(tokens[i], (uni.get(tokens[i]) || 0) + 1);
    if (i < tokens.length - 1) {
      const b = tokens[i] + " " + tokens[i + 1];
      bi.set(b, (bi.get(b) || 0) + 1);
    }
  }
  const top = (m) =>
    [...m.entries()]
      .sort((a, b) => b[1] - a[1])
      .slice(0, n)
      .map(([term, count]) => ({ term, count }));
  return { unigrams: top(uni), bigrams: top(bi) };
}

// Extract structured on-page intel. `queries` = the brief's target keyword phrases,
// so we can report how strongly each competitor targets them.
function analyze(html, { url = null, queries = [] } = {}) {
  const $ = cheerio.load(html || "");

  const titleTag = ($("head > title").first().text() || "").trim();
  const metaDescription =
    $('meta[name="description"]').attr("content")?.trim() ||
    $('meta[property="og:description"]').attr("content")?.trim() ||
    null;
  const canonical = $('link[rel="canonical"]').attr("href") || null;

  const h1s = $("h1")
    .map((_, el) => $(el).text().trim())
    .get()
    .filter(Boolean);

  // Heading outline (H1-H3 in document order).
  const outline = $("h1, h2, h3")
    .map((_, el) => ({
      level: Number(el.tagName.slice(1)),
      text: $(el).text().trim(),
    }))
    .get()
    .filter((h) => h.text);

  // JSON-LD schema types present.
  const schemaTypes = [];
  $('script[type="application/ld+json"]').each((_, el) => {
    try {
      const data = JSON.parse($(el).text());
      const arr = Array.isArray(data) ? data : [data];
      for (const node of arr) {
        const t = node["@type"];
        if (t) schemaTypes.push(...[].concat(t));
      }
    } catch {
      /* ignore malformed JSON-LD */
    }
  });

  // Main-content markdown + body text. Strip chrome before extracting.
  const $body = cheerio.load(html || "");
  $body("script,style,noscript,svg,iframe,form,nav,footer,header,aside").remove();
  const mainHtml =
    $body("main").html() ||
    $body("article").html() ||
    $body("body").html() ||
    html ||
    "";
  const markdown = toMarkdown(mainHtml);
  const bodyText = cheerio.load(`<div>${mainHtml}</div>`).text();
  const tokens = tokenize(bodyText);
  const wordCount = (bodyText.match(/\b[\w'-]+\b/g) || []).length;

  const titleTokens = tokenize(titleTag);
  const h1Tokens = tokenize(h1s.join(" "));

  const keywordTargeting = queries.map((q) => {
    const qt = tokenize(q);
    return {
      query: q,
      inTitle: phraseDensity(qt, titleTokens) > 0,
      inH1: phraseDensity(qt, h1Tokens) > 0,
      bodyDensityPct: phraseDensity(qt, tokens),
      bodyCount: Math.round((phraseDensity(qt, tokens) / 100) * tokens.length),
    };
  });

  return {
    url,
    titleTag,
    titleLength: titleTag.length,
    metaDescription,
    metaDescriptionLength: metaDescription ? metaDescription.length : 0,
    canonical,
    h1s,
    headingCounts: {
      h1: outline.filter((h) => h.level === 1).length,
      h2: outline.filter((h) => h.level === 2).length,
      h3: outline.filter((h) => h.level === 3).length,
    },
    outline,
    schemaTypes: [...new Set(schemaTypes)],
    wordCount,
    keywordTargeting,
    topTerms: topTerms(tokens),
    markdown,
  };
}

module.exports = { analyze, toMarkdown, tokenize };