headline/processor/app.py

#!/usr/bin/python3

import feedparser
import confuse
import redis
import time
import json
import sqlite3
import hashlib

from diff_match_patch import diff_match_patch


#
# Idea block:
#
#	Můžeme zkusit ke každému ID článku přidat taky hash obsahu, s tím že v začátku budeme kontrolovat jenom změnu hashe a až pak obsah stringů. 
#	Ale nevím jestli to bude reálně efektivnější
#
#	Teď budeme kontrolovat jen titulky, ale postupně můžeme přidat i description článku

CONFIG_FILE = "../data/config.yaml"
REDIS_ARTICLE_EXPIRE_SEC = 604800

config = confuse.Configuration('headline', __name__)
config.set_file(CONFIG_FILE)

dmp = diff_match_patch()

rc = redis.Redis(host='redis', port=6379, db=0)

db_con = sqlite3.connect("../data/diffs.db")
db = db_con.cursor()

db.executescript("""
PRAGMA journal_mode=WAL;

CREATE TABLE IF NOT EXISTS diffs (
	diff_id INTEGER PRIMARY KEY,
	article_id TEXT,
	feed_name TEXT NOT NULL,
	article_url TEXT NOT NULL,
	title_orig TEXT NOT NULL,
	title_new TEXT NOT NULL,
    diff_html TEXT NOT NULL,
    diff_time TEXT
);

CREATE VIRTUAL TABLE IF NOT EXISTS diffs_fts USING fts5(
    title_orig,
    title_new,
    content="diffs",
    content_rowid="diff_id",
    tokenize="trigram case_sensitive 0"
);
-- rebuild search index
-- useful when creating the table, or when it is externally updated
INSERT INTO diffs_fts(diffs_fts) VALUES ('rebuild');

CREATE TRIGGER IF NOT EXISTS diffs_ainsert AFTER INSERT ON diffs
    BEGIN
        INSERT INTO diffs_fts (rowid, title_orig, title_new)
        VALUES (new.diff_id, new.title_orig, new.title_new);
    END;
CREATE TRIGGER IF NOT EXISTS diffs_adelete AFTER DELETE ON diffs
    BEGIN
        INSERT INTO diffs_fts (diffs_fts, rowid, title_orig, title_new)
        VALUES ('delete', old.diff_id, old.title_orig, old.title_new);
    END;
CREATE TRIGGER IF NOT EXISTS diffs_aupdate AFTER UPDATE ON diffs
    BEGIN
        INSERT INTO diffs_fts (diffs_fts, rowid, title_orig, title_new)
        VALUES ('delete', old.diff_id, old.title_orig, old.title_new);
        INSERT INTO diffs_fts (rowid, title_orig, title_new)
        VALUES (new.diff_id, new.title_orig, new.title_new);
    END;
""")
article_count = 0

def write_article(article, rc):
	rval = json.dumps(article['content'])
	rc.set(article['rss_id'], rval, ex=REDIS_ARTICLE_EXPIRE_SEC)

def process_diff(old, new, rss_id):
	diff = dmp.diff_main(old['title'], new['title'])
	dmp.diff_cleanupSemantic(diff)
	html_diff = dmp.diff_prettyHtml(diff)
	# print(old['link'])
	# print(diff)

	sql = "INSERT INTO diffs(article_id, feed_name, article_url, title_orig, title_new, diff_html, diff_time) VALUES (?,?,?,?,?,?,datetime('now', 'localtime'))"
	sql_data = (new['article_id'], old['medium'], old['link'], old['title'], new['title'], html_diff)
	db.execute(sql, sql_data)
	db_con.commit()

	return(True)


def process_item(article, rc):
	if rc.exists(article['rss_id']):
		old = json.loads(rc.get(article['rss_id']))
		new = article['content']
		if old['title'] != new['title']:
			# print('Article changed. World is fucked.')
			diff = process_diff(old, new, article['rss_id'])
			write_article(article, rc)
			return(True)
		else:
			# Article is the same. All good!
			return(True)
	else:
		# Article is new, just create it and exit
		write_article(article, rc)

def create_article_id(uid, feed):
	# Create a unique ID from RSS unique tag and feed name to reference the article in database
	id_string = str(uid) + str(feed)
	id_bytes = id_string.encode('utf-8')
	article_id = hashlib.sha256(id_bytes).hexdigest()
	return(article_id)


for feed in config['feeds']:
	try:
		rss_source = str(feed['rss_source'])
		unique_tag = str(feed['unique_tag'])
		name = str(feed['name'])

		rss = feedparser.parse(rss_source)

		for item in rss['entries']:
			try:
				rss_id = item[unique_tag]
				title = item['title']
				article_id = create_article_id(rss_id, name)
				#description = item['description'] ## Don't store description for now, as we don't need it and it's big.
				published = time.strftime('%Y:%m:%d %H:%M:%S %Z %z', item['published_parsed'])
				link = item['link']
				article_data = {
					'title'		:	title,
					'article_id':	article_id,
					#'description':	description,
					'published'	:	published,
					'link'		:	link,
					'medium'	:	name
				}
				article = {
					'rss_id'	: rss_id, 
					'content'	: article_data
				}
				article_count += 1
				process_item(article, rc)
			except Exception as e:
				print("Parsing article failed")
				print(e)
				print(item)
	except Exception as e:
		print("Parsing feed failed.")
		print(e)
		print(feed)
		pass

print("Processed articles: " + str(article_count))