crawl.ts

import pLimit from 'p-limit';
import { JSDOM } from 'jsdom';
import { ExtractedPageData } from './report';

// --- Helper Functions ---

export const normalizeURL = (url: string) => {
    try {
        const urlObject = new URL(url);
        // Consistentie: geen trailing slash
        const hostPath = `${urlObject.origin}${urlObject.pathname}`;
        if (hostPath.length > 0 && hostPath.slice(-1) === '/') {
            return hostPath.slice(0, -1);
        }
        return hostPath;
    } catch (error) {
        // console.error(`Error normalizing URL ${url}: ${error}`); // Soms handig om te dempen bij tests
        return null;
    }
};

function getURLsFromHTML(htmlBody: string, baseURL: string): string[] {
    const urls: string[] = [];
    const dom = new JSDOM(htmlBody);
    const linkElements = dom.window.document.querySelectorAll('a');

    for (const linkElement of linkElements) {
        if (linkElement.href.slice(0, 1) === '/') {
            // Relatief
            try {
                const urlObj = new URL(`${baseURL}${linkElement.href}`);
                urls.push(urlObj.href);
            } catch (err) {}
        } else {
            // Absoluut
            try {
                const urlObj = new URL(linkElement.href);
                urls.push(urlObj.href);
            } catch (err) {}
        }
    }
    return urls;
}

// --- ConcurrentCrawler Class ---

class ConcurrentCrawler {
    baseURL: string;
    maxPages: number;
    // LET OP: Type veranderd van number naar ExtractedPageData
    pages: Record<string, ExtractedPageData>;
    limit: any;
    shouldStop: boolean;
    allTasks: Set<Promise<void>>;
    abortController: AbortController;

    constructor(baseURL: string, maxConcurrency: number = 3, maxPages: number = 10) {
        this.baseURL = baseURL;
        this.maxPages = maxPages;
        this.pages = {};
        this.limit = pLimit(maxConcurrency);
        this.shouldStop = false;
        this.allTasks = new Set();
        this.abortController = new AbortController();
    }

    // Aangepaste helper om data te extracten
    private extractDataFromHTML(htmlBody: string, currentURL: string): ExtractedPageData {
        const dom = new JSDOM(htmlBody);
        const doc = dom.window.document;

        // Extract H1
        const h1 = doc.querySelector('h1')?.textContent?.trim() || '';

        // Extract eerste paragraaf
        const first_paragraph = doc.querySelector('p')?.textContent?.trim() || '';

        // Extract links (zoals je eerder deed, maar nu opslaan)
        const linkElements = doc.querySelectorAll('a');
        const outgoing_links: string[] = [];
        for (const link of linkElements) {
            try {
                // Maak absolute URLs
                const href = link.href.startsWith('/') ? `${this.baseURL}${link.href}` : link.href;
                outgoing_links.push(new URL(href).href);
            } catch (e) {}
        }

        // Extract images
        const imgElements = doc.querySelectorAll('img');
        const image_urls: string[] = [];
        for (const img of imgElements) {
            try {
                const src = img.src.startsWith('/') ? `${this.baseURL}${img.src}` : img.src;
                image_urls.push(new URL(src).href);
            } catch (e) {}
        }

        return {
            url: currentURL,
            h1,
            first_paragraph,
            outgoing_links,
            image_urls,
        };
    }

    private addPageVisit(normalizedURL: string, pageData: ExtractedPageData): boolean {
        if (this.shouldStop) return false;
        if (this.pages[normalizedURL]) return false; // Al bezocht

        // Sla nu het hele object op in plaats van een nummer
        this.pages[normalizedURL] = pageData;

        if (Object.keys(this.pages).length >= this.maxPages) {
            this.shouldStop = true;
            console.log('Reached maximum number of pages to crawl.');
            this.abortController.abort();
            return false;
        }

        return true;
    }

    private async getHTML(currentURL: string): Promise<string> {
        return await this.limit(async () => {
            // Als we al moeten stoppen, doe dan geen nieuwe fetch meer
            if (this.shouldStop) throw new Error('Crawler stopped');

            try {
                const response = await fetch(currentURL, {
                    headers: { 'User-Agent': 'BootCrawler/1.0' },
                    signal: this.abortController.signal, // Koppel de abort controller
                });

                if (!response.ok) {
                    throw new Error(`HTTP error! status: ${response.status}`);
                }
                return await response.text();
            } catch (error) {
                // Als het een AbortError is, is dat verwacht gedrag bij het stoppen
                if (error.name === 'AbortError') {
                    throw new Error('Request aborted');
                }
                throw error;
            }
        });
    }

    private async crawlPage(currentURL: string): Promise<void> {
        if (this.shouldStop) return;

        const normalizedCurrentURL = normalizeURL(currentURL);
        const normalizedBaseURL = normalizeURL(this.baseURL);

        // Check of we deze URL al hebben (om dubbel werk te voorkomen voordat we fetchen)
        if (this.pages[normalizedCurrentURL]) return;

        if (!normalizedCurrentURL || !normalizedCurrentURL.startsWith(normalizedBaseURL)) return;

        console.log(`Crawling ${currentURL}`);

        try {
            const html = await this.getHTML(currentURL);

            // Nu extracten we ALLE data
            const pageData = this.extractDataFromHTML(html, currentURL);

            // En voegen we het toe (inclusief limiet check)
            if (!this.addPageVisit(normalizedCurrentURL, pageData)) {
                return;
            }

            // Gebruik de links die we net geëxtraheerd hebben voor de volgende stap
            const promises = pageData.outgoing_links.map((url) => {
                const task = this.crawlPage(url);
                this.allTasks.add(task);
                task.finally(() => this.allTasks.delete(task));
                return task;
            });

            await Promise.all(promises);
        } catch (error) {
            if (error.message !== 'Request aborted' && error.message !== 'Crawler stopped') {
                // console.error(`Error processing ${currentURL}: ${error.message}`);
            }
        }
    }

    // Update return type
    public async crawl(): Promise<Record<string, ExtractedPageData>> {
        const initialTask = this.crawlPage(this.baseURL);
        this.allTasks.add(initialTask);
        initialTask.finally(() => this.allTasks.delete(initialTask));
        await initialTask;
        return this.pages;
    }
}

// Update crawlSiteAsync return type
export async function crawlSiteAsync(
    baseURL: string,
    maxConcurrency: number = 3,
    maxPages: number = 10,
): Promise<Record<string, ExtractedPageData>> {
    const crawler = new ConcurrentCrawler(baseURL, maxConcurrency, maxPages);
    return await crawler.crawl();
}