research/news-crawling.py at master · faisalburhanudin/research · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
# %%
import uuid
import pickledb
from pathlib import Path
from typing import List
import newspaper
import requests
from tqdm import tqdm

# %%
# list news will be crawled
news = [
    ("detik.com", "https://www.detik.com/"),
    ("kontan.co.id", "https://kontan.co.id/"),
    ("cnnindonesia.com", "https://www.cnnindonesia.com/"),
    ("antaranews.com", "https://www.antaranews.com/"),
    ("bisnis.com", "https://www.bisnis.com/"),
    ("kompas.com", "https://www.kompas.com/"),
    ("republika.co.id", "https://republika.co.id"),
    ("liputan6.com", "https://www.liputan6.com/"),
    ("suara.com", "https://www.suara.com/"),
    ("tribunnews.com", "https://www.tribunnews.com"),
]


# %%
# get all article url
def generate_link(news_url: str) -> List[str]:
    """Generate link from news

    Args:
        news_url: url news ex: detik.com, kontan.co.id

    Returns:
        List of article
    """
    source = newspaper.build(news_url, memoize_articles=False)
    return source.article_urls()


article_url = []
for domain, url_article in tqdm(news):
    link = generate_link(url_article)
    article_url.append((domain, link))

# %%
# print total article fetch
for domain, item in article_url:
    print(f"{domain}: {len(item)}")

# %%
# get page info
session = requests.Session()


def get_html(url: str) -> str:
    """read url and return html

    Args:
        url: url will be get

    Returns:
        str html
    """
    r = session.get(url)
    if r.status_code != 200:
        return ''
    return r.text


def write_html(html: str, directory="news") -> str:
    """write html to file

    Args:
        html: str html will be write
        directory: target directory for save file

    Returns:
        filename generated
    """
    filename = str(uuid.uuid4())
    path = Path(directory) / filename
    with path.open("w") as file:
        file.write(html)
    return filename


db = pickledb.load('mapping.db', False)


def save_and_write(url: str, rewrite=True):
    if not rewrite:
        exist = db.get(url)
        if exist:
            return

    html = get_html(url)
    if not html:
        return

    filename = write_html(html)
    db.set(url, filename)


for d, item in article_url[:1]:
    for i in tqdm(item):
        save_and_write(i)