#!/usr/bin/env node
/**
 * Enhanced Index Generator for The Fold Within
 * FIXED: Uses frontmatter date as primary source
 */

import { promises as fs } from "fs";
import path from "path";
import pdf from "pdf-parse";

const ROOT = "public";
const BASE_URL = "https://thefoldwithin.earth";
const OUT_JSON = path.join(ROOT, "index.json");
const OUT_SITEMAP = path.join(ROOT, "sitemap.xml");
const OUT_ROBOTS = path.join(ROOT, "robots.txt");
const OUT_FEED = path.join(ROOT, "feed.xml");
const OUT_SCHEMA = path.join(ROOT, "schema.jsonld");
const EXCERPT_LENGTH = 400;

function extractFrontmatterDate(content) {
  const fmMatch = content.match(/^---\n([\s\S]*?)
---/);
  if (fmMatch) {
    const fm = fmMatch[1];
    const dateMatch = fm.match(/^date:\s*(\d{4}-\d{2}-\d{2})/m);
    if (dateMatch) return new Date(dateMatch[1]).getTime();
  }
  return null;
}

function dateFromName(name) {
  const m = name.match(/^(\d{4}-\d{2}-\d{2})/);
  return m ? new Date(m[0]).getTime() : null;
}

async function readHead(abs, full = false) {
  const fh = await fs.open(abs, "r");
  const size = full ? await fs.stat(abs).then(s => Math.min(s.size, EXCERPT_LENGTH * 2)) : 64 * 1024;
  const buf = Buffer.alloc(size);
  const { bytesRead } = await fh.read(buf, 0, size, 0);
  await fh.close();
  return buf.slice(0, bytesRead).toString("utf8");
}

function parseTitle(raw, ext) {
  if (ext === ".md") return raw.match(/^\s*#\s+(.+?)\s*$/m)?.[1].trim();
  if (ext === ".html") return raw.match(/<title[^>]*>([^<]+)<\/title>/i)?.[1].trim();
  return null;
}

function extractExcerpt(raw, ext) {
  if (ext === ".md") raw = raw.replace(/^#.*\n/, '').trim();
  if (ext === ".html") raw = raw.replace(/<head>[\s\S]*<\/head>/i, '').replace(/<[^>]+>/g, ' ').trim();
  return raw.replace(/\s+/g, ' ').slice(0, EXCERPT_LENGTH);
}

function extractTags(raw, ext, pdfData) {
  let tags = [];
  if (ext === ".md") {
    const m = raw.match(/^\s*tags:\s*(.+)$/im);
    if (m) tags = m[1].split(',').map(t => t.trim().toLowerCase());
  }
  return tags;
}

function generateSitemap(flat) {
  let xml = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`;
  const staticPages = ["", "/about", "/about/solaria", "/about/mark", "/about/initiatives", "/fieldnotes"];
  for (const page of staticPages) {
    xml += `  <url>\n    <loc>${BASE_URL}${page}/</loc>\n    <changefreq>weekly</changefreq>\n    <priority>${page === "" ? "1.0" : "0.8"}</priority>\n  </url>\n`;
  }
  for (const f of flat.filter(x => !x.isIndex)) {
    const urlPath = f.path.replace(/\.(md|html|pdf)$/, "/").replace("//", "/");
    const date = f.originalDate ? new Date(f.originalDate).toISOString().split('T')[0] : new Date(f.mtime).toISOString().split('T')[0];
    xml += `  <url>\n    <loc>${BASE_URL}/${urlPath}</loc>\n    <lastmod>${date}</lastmod>\n    <changefreq>monthly</changefreq>\n  </url>\n`;
  }
  return xml + "</urlset>";
}

function generateRobots() {
  return `# robots.txt for The Fold Within Earth\nSitemap: ${BASE_URL}/sitemap.xml\n`;
}

function generateFeed(flat) {
  const items = flat.filter(f => !f.isIndex && f.originalDate).sort((a, b) => b.originalDate - a.originalDate).slice(0, 20);
  let xml = `<?xml version="1.0" encoding="UTF-8"?>\n<rss version="2.0">\n<channel>\n<title>The Fold Within Earth</title>\n<link>${BASE_URL}</link>`;
  for (const f of items) {
    const urlPath = f.path.replace(/\.(md|html|pdf)$/, "/").replace("//", "/");
    xml += `  <item>\n    <title>${f.title || f.name}</title>\n    <link>${BASE_URL}/${urlPath}</link>\n    <pubDate>${new Date(f.originalDate).toUTCString()}</pubDate>\n  </item>\n`;
  }
  return xml + "</channel>\n</rss>";
}

async function collectFiles(relBase = "", flat = []) {
  const abs = path.join(ROOT, relBase);
  const entries = await fs.readdir(abs, { withFileTypes: true });
  for (const e of entries) {
    if (e.name.startsWith(".")) continue;
    const rel = path.posix.join(relBase, e.name);
    const absPath = path.join(ROOT, rel);
    if (rel.toLowerCase() === "index.html" || rel.toLowerCase() === "index.md") continue;
    if (e.isDirectory()) { await collectFiles(rel, flat); continue; }
    const ext = path.posix.extname(e.name).toLowerCase();
    if (![".md", ".html", ".pdf"].includes(ext)) continue;
    const st = await fs.stat(absPath);
    let raw = ext === ".pdf" ? (await pdf(await fs.readFile(absPath))).text : await readHead(absPath, true);
    const title = parseTitle(raw, ext) || e.name.replace(new RegExp(`\\${ext}$`), "").trim();
    const originalDate = ext === ".md" ? extractFrontmatterDate(raw) : null;
    const ctime = st.birthtimeMs || st.mtimeMs || dateFromName(e.name) || st.mtimeMs;
    const mtime = dateFromName(e.name) ?? st.mtimeMs;
    flat.push({ type: "file", name: e.name, title, path: rel, ext, ctime, mtime, originalDate, excerpt: extractExcerpt(raw, ext), tags: extractTags(raw, ext), isIndex: e.name.toLowerCase().startsWith("index.") });
  }
  return flat;
}

(async () => {
  try {
    console.log("Crawling...");
    const flat = await collectFiles();
    const sections = [...new Set(flat.filter(f => !f.isIndex).map(f => f.path.split("/")[0]))].sort();
    const allTags = [...new Set(flat.flatMap(f => f.tags))].sort();
    await fs.writeFile(OUT_JSON, JSON.stringify({ flat, sections, tags: allTags, generated: new Date().toISOString() }, null, 2));
    await fs.writeFile(OUT_SITEMAP, generateSitemap(flat));
    await fs.writeFile(OUT_ROBOTS, generateRobots());
    await fs.writeFile(OUT_FEED, generateFeed(flat));
    console.log(`Done! ${flat.length} files indexed with original dates from frontmatter.`);
  } catch (e) { console.error("Failed:", e); process.exit(1); }
})();
