lib/serp.js

// SerpApi Google search localized to Chicago, IL — the nearest major metro to Northern
// Illinois Recovery's Crystal Lake (McHenry County) campus and the market it competes
// in. Reuses the request shape from enrich-carf-serp.js, adds retry/backoff (that script
// had none), and returns the top organic results for a query.

const axios = require("axios");
const { env } = require("./env");

const LOCATION = "Chicago, Illinois, United States";

function sleep(ms) {
  return new Promise((r) => setTimeout(r, ms));
}

// One SERP call with retry on network/5xx/429. `extraParams` merges into the
// request (e.g. { start: 10 } for page 2 — Google dropped &num=100 in 2025, so
// depth crawls paginate 10 at a time).
async function serpSearch(q, { location = LOCATION, retries = 4, extraParams = {} } = {}) {
  const apiKey = env("SERP_API", { required: true });
  let lastErr = null;
  for (let attempt = 1; attempt <= retries; attempt++) {
    try {
      const res = await axios.get("https://serpapi.com/search.json", {
        params: {
          engine: "google",
          q,
          location,
          google_domain: "google.com",
          hl: "en",
          gl: "us",
          api_key: apiKey,
          ...extraParams,
        },
        timeout: 60000, // SerpApi p95 can exceed 30s under load
        validateStatus: () => true,
      });
      if (res.status === 429 || res.status >= 500) {
        lastErr = new Error(`SERP HTTP ${res.status}`);
        await sleep(1500 * attempt);
        continue;
      }
      if (res.data && res.data.error) {
        // SerpApi returns 200 with an { error } body for some failures.
        lastErr = new Error(`SERP error: ${res.data.error}`);
        // A genuine query-level error (e.g. "hasn't returned any results") is not
        // retryable — bail out and let the caller record an empty result.
        if (/no results|hasn't returned/i.test(res.data.error)) break;
        await sleep(1500 * attempt);
        continue;
      }
      return res.data;
    } catch (e) {
      lastErr = e;
      await sleep(1500 * attempt);
    }
  }
  throw lastErr || new Error("SERP failed");
}

// Top-N organic result URLs (deduped, http(s) only).
function organicUrls(serpJson, n = 5) {
  const out = [];
  const seen = new Set();
  for (const o of serpJson.organic_results || []) {
    const url = o.link;
    if (!url || !/^https?:\/\//i.test(url)) continue;
    if (seen.has(url)) continue;
    seen.add(url);
    out.push({
      position: o.position,
      url,
      title: o.title || null,
      snippet: o.snippet || null,
      displayed_link: o.displayed_link || null,
    });
    if (out.length >= n) break;
  }
  return out;
}

module.exports = { serpSearch, organicUrls, LOCATION, sleep };