/**
 * Same-domain website crawler. Aims for COMPLETE coverage of a site:
 *   1. Seeds the queue from sitemap.xml / sitemap_index.xml (catches inner pages
 *      that aren't linked from the nav).
 *   2. BFS over every same-domain link found on each page.
 * Links are extracted from the FULL page (before boilerplate is stripped) so
 * menus/footers are followed and no page is missed. No page-count limit by
 * default (a high safety ceiling prevents runaway crawls).
 */
import * as cheerio from "cheerio";

const SKIP_EXT = /\.(pdf|docx?|xlsx?|pptx?|zip|gz|rar|7z|jpe?g|png|gif|svg|webp|ico|bmp|mp4|webm|mp3|wav|avi|mov|css|js|mjs|json|rss|txt|woff2?|ttf|eot)($|\?)/i;
const SAFETY_CEILING = 3000; // stops truly pathological/infinite crawls

function normalize(href) {
  try {
    const u = new URL(href);
    u.hash = "";
    let s = u.href;
    if (u.pathname !== "/" && s.endsWith("/")) s = s.slice(0, -1);
    return s;
  } catch { return href; }
}

function dedupeLinks(links) {
  const seen = new Set(), out = [];
  for (const l of links) { if (seen.has(l.url)) continue; seen.add(l.url); out.push(l); }
  return out;
}

async function fetchText(url, accept) {
  const res = await fetch(url, {
    headers: { "User-Agent": "W3care-ChatWidget-Crawler/1.0", Accept: accept },
    redirect: "follow",
    signal: AbortSignal.timeout(20000),
  });
  return { res, url: res.url || url };
}

// Collect every <loc> from sitemap.xml / sitemap indexes (one level of nesting).
async function collectSitemapUrls(origin) {
  const found = new Set();
  const candidates = [origin + "/sitemap.xml", origin + "/sitemap_index.xml"];
  async function readSitemap(url, depth) {
    let xml;
    try {
      const { res } = await fetchText(url, "application/xml,text/xml");
      if (!res.ok) return;
      xml = await res.text();
    } catch { return; }
    const locs = [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)].map((m) => m[1].trim());
    const isIndex = /<sitemapindex/i.test(xml);
    if (isIndex && depth < 2) {
      for (const child of locs.slice(0, 50)) await readSitemap(child, depth + 1);
    } else {
      for (const u of locs) found.add(u);
    }
  }
  for (const c of candidates) await readSitemap(c, 0);
  return [...found];
}

export async function crawlSite(startUrl, maxPages, onProgress) {
  const start = new URL(startUrl);
  const origin = start.origin;
  const limit = maxPages && maxPages > 0 ? Math.min(maxPages, SAFETY_CEILING) : SAFETY_CEILING;

  const seen = new Set();
  const queue = [normalize(start.href)];

  // Seed from sitemaps for complete coverage.
  try {
    for (const u of await collectSitemapUrls(origin)) {
      let parsed; try { parsed = new URL(u); } catch { continue; }
      if (parsed.origin !== origin || SKIP_EXT.test(parsed.pathname)) continue;
      const n = normalize(u);
      if (!queue.includes(n)) queue.push(n);
    }
  } catch { /* sitemap optional */ }

  const pages = [];
  while (queue.length && pages.length < limit) {
    const url = queue.shift();
    if (seen.has(url)) continue;
    seen.add(url);

    let html, finalUrl = url;
    try {
      const r = await fetchText(url, "text/html");
      finalUrl = r.url;
      const ct = r.res.headers.get("content-type") || "";
      if (!r.res.ok || !ct.includes("text/html")) continue;
      html = await r.res.text();
    } catch { continue; }

    const $ = cheerio.load(html);

    // 1) Discover links from the WHOLE document first.
    const pageLinks = [];
    $("a[href]").each((_, el) => {
      const raw = ($(el).attr("href") || "").trim();
      if (!raw || raw.startsWith("#") || /^(mailto:|tel:|javascript:)/i.test(raw)) return;
      let u; try { u = new URL(raw, finalUrl); } catch { return; }
      if (u.protocol !== "http:" && u.protocol !== "https:") return;
      const abs = normalize(u.href);
      const text = $(el).text().replace(/\s+/g, " ").trim();
      if (u.origin === origin && !SKIP_EXT.test(u.pathname)) {
        if (!seen.has(abs) && !queue.includes(abs)) queue.push(abs);
      }
      if (text) pageLinks.push({ text: text.slice(0, 120), url: abs });
    });

    // 2) Extract clean readable text (strip chrome AFTER links are captured).
    $("script, style, noscript, svg, template, iframe, nav, footer, header, form").remove();
    const title = ($("title").first().text() || $("h1").first().text() || "").replace(/\s+/g, " ").trim();
    const scope = $("main").length ? $("main") : $("article").length ? $("article") : $("body");
    const text = scope.text().replace(/[ \t ]+/g, " ").replace(/\n[ \t]*\n[ \t]*\n+/g, "\n\n").trim();

    if (text && text.length > 40) {
      pages.push({ url: finalUrl, title, text, links: dedupeLinks(pageLinks).slice(0, 60) });
      if (onProgress) onProgress(pages.length, finalUrl);
    }
  }

  return pages;
}

/** Combine crawled pages into one text document, URLs and links included. */
export function pagesToText(pages) {
  return pages
    .map((p) => {
      const links = (p.links || []).length
        ? "\n\nLinks on this page:\n" + p.links.map((l) => `- ${l.text} → ${l.url}`).join("\n")
        : "";
      return `PAGE TITLE: ${p.title || p.url}\nPAGE URL: ${p.url}\n\n${p.text}${links}`;
    })
    .join("\n\n========================================\n\n");
}
