diff options
| -rw-r--r-- | newsdiff/core/parsers/haaretz.py | 23 | ||||
| -rw-r--r-- | newsdiff/core/parsers/ynet.py | 5 | ||||
| -rw-r--r-- | newsdiff/core/tasks.py | 7 |
3 files changed, 20 insertions, 15 deletions
diff --git a/newsdiff/core/parsers/haaretz.py b/newsdiff/core/parsers/haaretz.py index 7e32f17..c4d7cad 100644 --- a/newsdiff/core/parsers/haaretz.py +++ b/newsdiff/core/parsers/haaretz.py @@ -24,16 +24,19 @@ class HaaretzParser(HtmlSoupParser): return '{}/{}'.format(self.BASE_URL, article_id) def parse_article(self, url, soup): - haaretz_id = re.findall(self.ARTICLE_ID_PATTERN, url)[0] - title = soup.find('h1', class_='mainTitle').text.strip() - subtitle = soup.find('h2', class_='subtitle').text.strip() - author_bar = soup.find('ul', class_='author-bar') - author = author_bar.find(class_=re.compile('autorBar(Anchor|Writers)')).text.strip() - date = author_bar.find_all('li')[1].text.strip() - time = author_bar.find_all('li')[2].text.strip() - article_date = self.TIMEZONE.localize(datetime.strptime(' '.join([date, time]), '%d.%m.%Y %H:%M')) - article_body = soup.find('div', id='article-box').find_all('p') - article_text = '\n\n'.join([p.text.strip() for p in article_body]) + try: + haaretz_id = re.findall(self.ARTICLE_ID_PATTERN, url)[0] + title = soup.find('h1', class_='mainTitle').text.strip() + subtitle = soup.find('h2', class_='subtitle').text.strip() + author_bar = soup.find('ul', class_='author-bar') + author = author_bar.find(class_=re.compile('autorBar(Anchor|Writers)')).text.strip() + date = author_bar.find_all('li')[1].text.strip() + time = author_bar.find_all('li')[2].text.strip() + article_date = self.TIMEZONE.localize(datetime.strptime(' '.join([date, time]), '%d.%m.%Y %H:%M')) + article_body = soup.find('div', id='article-box').find_all('p') + article_text = '\n\n'.join([p.text.strip() for p in article_body]) + except AttributeError: + return with transaction.atomic(), reversion.create_revision(): try: diff --git a/newsdiff/core/parsers/ynet.py b/newsdiff/core/parsers/ynet.py index 995e7a2..6791e4e 100644 --- a/newsdiff/core/parsers/ynet.py +++ b/newsdiff/core/parsers/ynet.py @@ -26,10 +26,7 @@ class YnetParser(HtmlSoupParser): def parse_article(self, url, soup): ynet_id = re.findall(self.ARTICLE_ID_PATTERN, url)[0] title = soup.find('div', class_='art_header_title').text.strip().encode('iso-8859-1') - try: - subtitle = soup.find('div', class_='art_header_sub_title').text.strip().encode('iso-8859-1') - except: - subtitle = '' + subtitle = soup.find('div', class_='art_header_sub_title').text.strip().encode('iso-8859-1') author_bar = soup.find('span', class_='art_header_footer_author') author = author_bar.find('a').text.strip().encode('iso-8859-1') date = re.findall(r'\d\d.\d\d.\d\d', author_bar.text)[0] diff --git a/newsdiff/core/tasks.py b/newsdiff/core/tasks.py index 4646089..a923aa3 100644 --- a/newsdiff/core/tasks.py +++ b/newsdiff/core/tasks.py @@ -27,7 +27,12 @@ def process_ynet_homepage(): @app.task(rate_limit='12/m') def process_ynet_article(url): yp = YnetParser() - yp.process_article(url) + try: + yp.process_article(url) + except: + # such ynet + # much errors + pass @app.task() def preload_thumbnail(filename, size): |