webscraping_nodejs_crawlee_puppeteer/webscraping_nodejs_crawlee_puppeteer.js at main · worldsoft998/webscraping_nodejs_crawlee_puppeteer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
// JavaScript Document
//https://docs.apify.com/sdk/js/docs/examples/puppeteer-recursive-crawl
//https://docs.apify.com/sdk/js/docs/examples/puppeteer-recursive-crawl
//https://crawlee.dev/docs/examples/puppeteer-recursive-crawl
//puppeteer-recursive-crawl
//Piece of code to find page containing certain keywords or
//list of keywords in a list of websites and
//print out the page list with keywords in front of the links.

import { Actor } from 'apify';
import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
import fs from 'fs';

await Actor.init();

//keywords file, here you shall modify it to your file location
const keywordslist= fs.readFileSync('C:\\XXXBackup\\zzzzsoftware\\000webcrawlerproject\\zzzdata\\wordlist.txt', 'utf-8');
const arrkeywords = keywordslist.split(/\r?\n/);
//console.log(arrkeywords);
console.info(arrkeywords);

// url file, here you shall modify it to your file location
const start_urls = fs.readFileSync('C:\\XXXBackup\\zzzzsoftware\\000webcrawlerproject\\zzzdata\\start_urls.txt', 'utf-8');
const arrurls = start_urls.split(/\r?\n/);
//console.log(arrurls);
console.info(arrurls);

//proxy file, here you shall modify it to your file location
const proxylist= fs.readFileSync('C:\\XXXBackup\\00WEBITSettings\\004Emails\\proxylist4mails\\000000proxylist4emailscraping.txt', 'utf-8');
const arrproxy = proxylist.split(/\r?\n/);
//console.log(arrproxy);
console.info(arrproxy);

const proxyConfiguration = new ProxyConfiguration({

    proxyUrls: arrproxy,

});


const crawler = new PuppeteerCrawler({

    proxyConfiguration,

    launchContext: {
        // Here you can set options that are passed to the playwright .launch() function.
        launchOptions: {
            headless: true,
        },
    },

    async requestHandler({ request, page, enqueueLinks }) {
        const title = await page.title();
        console.log(`Title of ${request.url}: ${title}`);

        const html = await page.content();

        for (const keywords of arrkeywords) {

            if (html.includes(keywords)) console.info(`${keywords} is found at ${request.loadedUrl}`);

        //console.log(`The title of "${request.url}" is: ${title}.`);
        }

        await enqueueLinks();

    },
    //maxRequestsPerCrawl: 100,
});


await crawler.addRequests(arrurls);

await crawler.run();

await Actor.exit();