lib/scrape.js

// Fetch a page's full HTML with Puppeteer. Tries a direct headless fetch first;
// on block/timeout/error retries through the Bright Data proxy (if creds are in
// .env). Returns { ok, html, via, status, error }. Never throws — callers log+skip.

const { env, envAny } = require("./env");

const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36";

// Bright Data's standard super-proxy endpoint (used by residential / datacenter /
// Web-Unlocker zones; the zone is encoded in the username). Override with
// BRIGHTDATA_HOST / BRIGHTDATA_PORT or a full BRIGHTDATA_PROXY URL if your zone differs.
const BD_DEFAULT_HOST = "brd.superproxy.io";
const BD_DEFAULT_PORT = "33335";

// Resolve Bright Data proxy config from .env. Accepts a full BRIGHTDATA_PROXY URL,
// or username/password (your keys: Brightdata_Username / Brightdata_Password) + an
// optional host/port override.
function brightDataProxy() {
  const single = env("BRIGHTDATA_PROXY");
  if (single) {
    try {
      const u = new URL(single);
      return {
        server: `${u.protocol}//${u.host}`,
        username: decodeURIComponent(u.username || ""),
        password: decodeURIComponent(u.password || ""),
      };
    } catch {
      return null;
    }
  }
  let username = envAny(["Brightdata_Username", "BRIGHTDATA_USER", "BRIGHTDATA_USERNAME"]);
  const password = envAny(["Brightdata_Password", "BRIGHTDATA_PASS", "BRIGHTDATA_PASSWORD"]);
  if (!username || !password) return null;
  // Route through a US exit node so pages render as a US visitor sees them
  // (faithful to the Chicago, IL intent). Append -country-xx only if the username
  // doesn't already pin a country. Set BRIGHTDATA_COUNTRY="" to disable.
  const country = env("BRIGHTDATA_COUNTRY") ?? "us";
  if (country && !/-country-/.test(username)) {
    username = `${username}-country-${country}`;
  }
  const host = env("BRIGHTDATA_HOST") || BD_DEFAULT_HOST;
  const port = env("BRIGHTDATA_PORT") || BD_DEFAULT_PORT;
  return { server: `http://${host}:${port}`, username, password };
}

async function fetchWith(puppeteer, url, proxy) {
  const args = ["--no-sandbox", "--disable-setuid-sandbox"];
  if (proxy) args.push(`--proxy-server=${proxy.server}`);
  const browser = await puppeteer.launch({ headless: "new", args });
  try {
    const page = await browser.newPage();
    if (proxy && proxy.username) {
      await page.authenticate({
        username: proxy.username,
        password: proxy.password,
      });
    }
    await page.setUserAgent(UA);
    await page.setExtraHTTPHeaders({ "Accept-Language": "en-US,en;q=0.9" });
    const resp = await page.goto(url, {
      waitUntil: "networkidle2",
      timeout: 60000,
    });
    const status = resp ? resp.status() : 0;
    // Wait for client-rendered pages to actually populate body text before capture.
    await page
      .waitForFunction(
        () => document.body && document.body.innerText.trim().length > 500,
        { timeout: 8000 },
      )
      .catch(() => {});
    const html = await page.content();
    const text = await page
      .evaluate(() => (document.body ? document.body.innerText : ""))
      .catch(() => "");
    return { status, html, text };
  } finally {
    await browser.close().catch(() => {});
  }
}

// Judge by EXTRACTED body text, not raw HTML size (raw size misfires both ways:
// huge framework shells look "fine", and content-rich pages like Wikipedia were
// wrongly flagged). A real article has substantial visible text.
function looksBlocked(status, html, text) {
  if (status === 403 || status === 429 || status === 503) return true;
  if (!text || text.trim().length < 600) return true; // ~100+ words of real content
  return /are you a human|cf-browser-verification|just a moment\.\.\.|attention required|enable javascript and cookies|verifying you are human/i.test(
    text.slice(0, 4000),
  );
}

async function scrape(url) {
  let puppeteer;
  try {
    puppeteer = require("puppeteer");
  } catch {
    return { ok: false, error: "puppeteer not installed", via: null };
  }

  // 1) Direct.
  try {
    const { status, html, text } = await fetchWith(puppeteer, url, null);
    if (!looksBlocked(status, html, text)) {
      return { ok: true, html, status, via: "direct" };
    }
    // 2) Bright Data fallback.
    const proxy = brightDataProxy();
    if (!proxy) {
      return {
        ok: false,
        status,
        via: "direct",
        error: `blocked (HTTP ${status}); no Bright Data creds for fallback`,
      };
    }
    try {
      const r2 = await fetchWith(puppeteer, url, proxy);
      if (!looksBlocked(r2.status, r2.html, r2.text)) {
        return { ok: true, html: r2.html, status: r2.status, via: "brightdata" };
      }
      return {
        ok: false,
        status: r2.status,
        via: "brightdata",
        error: `blocked via proxy (HTTP ${r2.status})`,
      };
    } catch (e) {
      return { ok: false, via: "brightdata", error: String(e.message || e) };
    }
  } catch (e) {
    // Direct fetch threw — try proxy if available.
    const proxy = brightDataProxy();
    if (proxy) {
      try {
        const r2 = await fetchWith(puppeteer, url, proxy);
        if (!looksBlocked(r2.status, r2.html, r2.text)) {
          return { ok: true, html: r2.html, status: r2.status, via: "brightdata" };
        }
      } catch (e2) {
        return { ok: false, via: "brightdata", error: String(e2.message || e2) };
      }
    }
    return { ok: false, via: "direct", error: String(e.message || e) };
  }
}

module.exports = { scrape, brightDataProxy };