nologcz/headline
2
1
Fork 0
mirror of https://git.nolog.cz/NoLog.cz/headline.git synced 2025-01-31 11:53:35 +01:00
Code Issues Projects Releases Packages Wiki Activity

add dirty exceptions to skip broken RSS feeds

Browse source
This commit is contained in:
mdivecky 2023-08-16 12:41:29 +02:00
parent 9800a0825d
commit 9918da68eb
1 changed files with 34 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
processor/app.py
View file

@ -112,30 +112,41 @@ def process_item(article, rc):
for feed in config['feeds']: for feed in config['feeds']:
rss_source = str(feed['rss_source']) try:
unique_tag = str(feed['unique_tag']) rss_source = str(feed['rss_source'])
name = str(feed['name']) unique_tag = str(feed['unique_tag'])
name = str(feed['name'])
rss = feedparser.parse(rss_source) rss = feedparser.parse(rss_source)
for item in rss['entries']: for item in rss['entries']:
rss_id = item[unique_tag] try:
title = item['title'] rss_id = item[unique_tag]
#description = item['description'] ## Don't store description for now, as we don't need it and it's big. title = item['title']
published = time.strftime('%Y:%m:%d %H:%M:%S %Z %z', item['published_parsed']) #description = item['description'] ## Don't store description for now, as we don't need it and it's big.
link = item['link'] published = time.strftime('%Y:%m:%d %H:%M:%S %Z %z', item['published_parsed'])
article_data = { link = item['link']
'title' : title, article_data = {
#'description': description, 'title' : title,
'published' : published, #'description': description,
'link' : link, 'published' : published,
'medium' : name 'link' : link,
} 'medium' : name
article = { }
'rss_id' : rss_id, article = {
'content' : article_data 'rss_id' : rss_id,
} 'content' : article_data
article_count += 1 }
process_item(article, rc) article_count += 1
process_item(article, rc)
except Exception as e:
print("Parsing article failed")
print(e)
print(item)
except Exception as e:
print("Parsing feed failed.")
print(e)
print(feed)
pass
print("Processed articles: " + str(article_count)) print("Processed articles: " + str(article_count))