newsdiffs/parsers/telegraaf.py at fa13f8c3cbfad72772522e1972d403489ce6a842 · ecprice/newsdiffs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
from baseparser import BaseParser
from BeautifulSoup import BeautifulSoup, Tag, Comment


class TelegraafParser(BaseParser):
    domains = ['www.telegraaf.nl']

    feeder_base = 'http://www.telegraaf.nl'
    feeder_pat  = '^http://www.telegraaf.nl/\w+/\d+/'

    def _parse(self, html):
        soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES,
                             fromEncoding='utf-8')

        self.meta = soup.findAll('meta')

        article = soup.find('div', id = 'artikel')
        title = article.find('h1')

        self.title = ''
        for i in title.childGenerator():
            # Skip comments
            if isinstance(i, Comment):
                continue

            self.title += i.lstrip()

        self.byline = ''

        date = article.find('div', 'artDatePostings')
        self.date = date.find('span', 'datum').getText()

        article_column = soup.find('div', id = 'artikelKolom')

        self.body = ''
        for i in article_column.childGenerator():
            if not isinstance(i, Tag):
                continue
            if not i.name == 'p':
                continue
            self.body += i.getText() + '\n\n'