nologcz/headline
2
1
Fork 0
mirror of https://git.nolog.cz/NoLog.cz/headline.git synced 2025-01-31 11:53:35 +01:00
Code Issues Projects Releases Packages Wiki Activity

add dirty exceptions to skip broken RSS feeds

Browse source
This commit is contained in:
mdivecky 2023-08-16 12:41:29 +02:00
parent 9800a0825d
commit 9918da68eb
1 changed files with 34 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
processor/app.py
View file

@ -112,30 +112,41 @@ def process_item(article, rc):
for feed in config['feeds']:
rss_source = str(feed['rss_source'])
unique_tag = str(feed['unique_tag'])
name = str(feed['name'])
try:
rss_source = str(feed['rss_source'])
unique_tag = str(feed['unique_tag'])
name = str(feed['name'])
rss = feedparser.parse(rss_source)
rss = feedparser.parse(rss_source)
for item in rss['entries']:
rss_id = item[unique_tag]
title = item['title']
#description = item['description'] ## Don't store description for now, as we don't need it and it's big.
published = time.strftime('%Y:%m:%d %H:%M:%S %Z %z', item['published_parsed'])
link = item['link']
article_data = {
'title' : title,
#'description': description,
'published' : published,
'link' : link,
'medium' : name
}
article = {
'rss_id' : rss_id,
'content' : article_data
}
article_count += 1
process_item(article, rc)
for item in rss['entries']:
try:
rss_id = item[unique_tag]
title = item['title']
#description = item['description'] ## Don't store description for now, as we don't need it and it's big.
published = time.strftime('%Y:%m:%d %H:%M:%S %Z %z', item['published_parsed'])
link = item['link']
article_data = {
'title' : title,
#'description': description,
'published' : published,
'link' : link,
'medium' : name
}
article = {
'rss_id' : rss_id,
'content' : article_data
}
article_count += 1
process_item(article, rc)
except Exception as e:
print("Parsing article failed")
print(e)
print(item)
except Exception as e:
print("Parsing feed failed.")
print(e)
print(feed)
pass
print("Processed articles: " + str(article_count))