seo_intel.js

// Stage 0 — SEO intel. For each brief's primary target keyword:
//   SerpApi (localized to Chicago, IL) -> top-5 organic -> Puppeteer scrape
//   (Bright Data fallback) -> on-page analysis. Writes per-brief intel bundles.
//
// Usage:
//   node seo_intel.js --slug <slug>      # one brief (dry-run)
//   node seo_intel.js --limit 1          # first N briefs
//   node seo_intel.js                    # all 15

const fs = require("fs");
const path = require("path");
const { parseBriefs } = require("./lib/briefs");
const { serpSearch, organicUrls, LOCATION, sleep } = require("./lib/serp");
const { scrape } = require("./lib/scrape");
const { analyze } = require("./lib/onpage");

const OUT_DIR = path.join(__dirname, "seo");
const LOG_FILE = path.join(__dirname, "seo_run.log");

function log(msg) {
  const line = `[${new Date().toISOString()}] ${msg}`;
  console.log(line);
  fs.appendFileSync(LOG_FILE, line + "\n");
}

function arg(name) {
  const i = process.argv.indexOf(`--${name}`);
  return i >= 0 ? process.argv[i + 1] : null;
}

async function processBrief(brief) {
  const dir = path.join(OUT_DIR, brief.slug);
  fs.mkdirSync(dir, { recursive: true });
  const query = brief.targetKeyword || brief.queries[0];
  log(`[${brief.slug}] query="${query}" (location: ${LOCATION})`);

  let serpJson;
  try {
    serpJson = await serpSearch(query);
  } catch (e) {
    log(`[${brief.slug}] SERP FAILED: ${e.message}`);
    return { slug: brief.slug, ok: false, error: `serp: ${e.message}` };
  }
  fs.writeFileSync(
    path.join(dir, "serp.json"),
    JSON.stringify(serpJson, null, 2),
  );

  const organics = organicUrls(serpJson, 5);
  log(`[${brief.slug}] top organic: ${organics.length} urls`);

  const competitors = [];
  for (let i = 0; i < organics.length; i++) {
    const { url } = organics[i];
    const n = i + 1;
    const res = await scrape(url);
    if (!res.ok) {
      log(`[${brief.slug}]  #${n} ${url} -> SKIP (${res.error})`);
      competitors.push({ rank: n, url, ok: false, error: res.error, via: res.via });
      continue;
    }
    const intel = analyze(res.html, { url, queries: brief.queries });
    fs.writeFileSync(
      path.join(dir, `competitor_${n}.md`),
      `# ${intel.titleTag || url}\n\n> source: ${url} (via ${res.via})\n\n${intel.markdown}\n`,
    );
    // Don't store the giant markdown twice inside intel.json.
    const { markdown, ...intelMeta } = intel;
    competitors.push({
      rank: n,
      url,
      ok: true,
      via: res.via,
      serpTitle: organics[i].title,
      ...intelMeta,
    });
    log(
      `[${brief.slug}]  #${n} ${url} -> OK (${res.via}, ${intel.wordCount}w, title="${(intel.titleTag || "").slice(0, 60)}")`,
    );
    await sleep(800);
  }

  const bundle = {
    slug: brief.slug,
    workingTitle: brief.workingTitle,
    targetKeyword: query,
    queries: brief.queries,
    location: LOCATION,
    generated_at: new Date().toISOString(),
    competitors,
  };
  fs.writeFileSync(path.join(dir, "intel.json"), JSON.stringify(bundle, null, 2));
  const okCount = competitors.filter((c) => c.ok).length;
  return { slug: brief.slug, ok: true, scraped: okCount, total: organics.length };
}

async function main() {
  fs.mkdirSync(OUT_DIR, { recursive: true });
  const { briefs } = parseBriefs();
  let targets = briefs;
  const slug = arg("slug");
  const limit = arg("limit");
  if (slug) targets = briefs.filter((b) => b.slug === slug);
  else if (limit) targets = briefs.slice(0, Number(limit));
  if (!targets.length) {
    log(`No briefs matched (slug=${slug}). Available: ${briefs.map((b) => b.slug).join(", ")}`);
    return;
  }

  log(`=== Stage 0 SEO intel: ${targets.length} brief(s) ===`);
  const results = [];
  for (const b of targets) {
    results.push(await processBrief(b));
  }
  fs.writeFileSync(
    path.join(OUT_DIR, "_stats.json"),
    JSON.stringify(
      { generated_at: new Date().toISOString(), location: LOCATION, results },
      null,
      2,
    ),
  );
  const ok = results.filter((r) => r.ok).length;
  const scraped = results.reduce((s, r) => s + (r.scraped || 0), 0);
  log(`=== done: ${ok}/${results.length} briefs, ${scraped} competitor pages scraped ===`);
}

main().catch((e) => {
  log(`FATAL: ${e.stack || e}`);
  process.exit(1);
});