flask UI and dockerization:) Sorry.

.gitignore

@@ -0,0 +1,3 @@
+data/proc_log.txt
+data/diffs.db
+*.pyc

docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3.8"
+services:
+  view:
+    build: ./view/
+    command: python app.py
+    ports:
+      - "5000:5000"
+    volumes:
+      - ./view:/app
+      - ./data:/data
+    container_name: view
+    restart: unless-stopped
+
+  processor:
+    build: ./processor/
+    volumes:
+      - ./processor:/app
+      - ./data:/data
+    container_name: processor
+    restart: unless-stopped
+  
+  redis:
+    image: redis:latest
+    restart: unless-stopped
+    container_name: redis_db
+

processor/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.9-alpine
+ADD . /app/
+WORKDIR /app
+RUN pip install -r requirements.txt
+COPY crontab /etc/cron.d/crontab
+RUN crontab /etc/cron.d/crontab
+CMD ["crond", "-f"]

processor/app.py

@@ -5,13 +5,10 @@ import confuse
 import redis
 import time
 import json
-import imgkit
+import sqlite3
 
 from diff_match_patch import diff_match_patch
 
-from pprint import pprint
-import hashlib
-
 
 #
 # Idea block:
@@ -26,7 +23,20 @@ config.set_file('config.yaml')
 
 dmp = diff_match_patch()
 
-rc = redis.Redis(host='localhost', port=6379, db=0)
+rc = redis.Redis(host='redis', port=6379, db=0)
+
+db_con = sqlite3.connect("../data/diffs.db")
+db = db_con.cursor()
+
+db.execute("""CREATE TABLE IF NOT EXISTS diffs (
+	diff_id INTEGER PRIMARY KEY,
+	feed_name TEXT NOT NULL,
+	article_url TEXT NOT NULL,
+	title_orig TEXT NOT NULL,
+	title_new TEXT NOT NULL,
+    diff_html TEXT NOT NULL,
+    diff_time TEXT
+);""")
 
 article_count = 0
 
@@ -39,9 +49,13 @@ def process_diff(old, new, rss_id):
 	dmp.diff_cleanupSemantic(diff)
 	html_diff = dmp.diff_prettyHtml(diff)
 	print(old['link'])
-	#print(diff)
+	print(diff)
-	#filename = "./img/" + hashlib.md5(rss_id.encode()).hexdigest() + ".jpg"
+
-	#image = imgkit.from_string(html_diff, filename, options = {'width': '450', 'quiet': ''})
+	sql = "INSERT INTO diffs(feed_name, article_url, title_orig, title_new, diff_html, diff_time) VALUES (?,?,?,?,?,datetime('now', 'localtime'))"
+	sql_data = (old['medium'], old['link'], old['title'], new['title'], html_diff)
+	db.execute(sql, sql_data)
+	db_con.commit()
+
 	return(True)
 
 
@@ -52,7 +66,7 @@ def process_item(article, rc):
 		if old['title'] != new['title']:
 			print('Article changed. World is fucked.')
 			diff = process_diff(old, new, article['rss_id'])
-			#write_article(article, rc)
+			write_article(article, rc)
 			return(True)
 		else:
 			# Article is the same. All good!

processor/crontab

@@ -0,0 +1 @@
+*/1 * * * * cd /app && /usr/local/bin/python3 app.py >> /data/proc_log.txt 2>&1

processor/requirements.txt

@@ -2,4 +2,3 @@ feedparser
 confuse
 redis
 diff-match-patch
-imgkit

view/Dockerfile

@@ -0,0 +1,4 @@
+FROM python:slim-bullseye
+ADD . /app/
+WORKDIR /app
+RUN pip install -r requirements.txt

view/app.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python3
+
+import sqlite3
+from flask import Flask, request, render_template
+from flask import g
+from flask_paginate import Pagination, get_page_parameter
+
+
+DATABASE = "../data/diffs.db"
+
+app = Flask(__name__)
+
+
+
+
+def get_db():
+    db = getattr(g, '_database', None)
+    if db is None:
+        db = g._database = sqlite3.connect(DATABASE)
+    db.row_factory = sqlite3.Row
+    return db
+
+@app.teardown_appcontext
+def close_connection(exception):
+    db = getattr(g, '_database', None)
+    if db is not None:
+        db.close()
+
+
+
+@app.route('/')
+def index():
+    db = get_db().cursor()
+    db.execute('SELECT count(diff_id) FROM diffs')
+    diff_count = db.fetchall()[0][0]
+
+
+    #flask-paginate
+    page = request.args.get(get_page_parameter(), type=int, default=1)
+
+    pagination = Pagination(page=page, total=diff_count, record_name='diffs')
+
+    
+    page_start = pagination.skip
+    page_stop = page_start + 10
+    db.execute("SELECT * FROM diffs ORDER BY diff_id DESC LIMIT ?,?", (page_start,page_stop))
+    diffs = db.fetchall()
+
+    return render_template('./index.html',
+                           diffs=diffs,
+                           pagination=pagination,
+                           )
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0")

view/requirements.txt

@@ -0,0 +1,2 @@
+flask
+flask-paginate

view/templates/index.html

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <title>Headliner</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <style>
+
+/* .expanded {
+    width: 100%;
+} */
+/* Longer name hidden by default  */
+span.long{
+    display:none;
+}
+/* On hover, hide the short name */
+.expanded:hover span.short{
+    display:none;
+}
+/* On hover, display the longer name.  */
+.expanded:hover span.long{
+    display:block;
+}
+
+</style>
+
+  </head>
+  <body>
+    <div class="container">
+      <div class="table-responsive">
+        <table class="table table-hover">
+          <thead>
+            <tr>
+              <th>#</th>
+              <th>Detection time</th>
+              <th>Source</th>
+              <th>Diff</th>
+              <th>Original title</th>
+              <th>Changed title</th>
+            </tr>
+          </thead>
+          <tbody>
+            {% for diff in diffs %}
+            <tr>
+              <td>{{ diff.diff_id }}</td>
+              <td>{{ diff.diff_time }}</td>
+              <td><a href='{{ diff.article_url }}' target="_blank">{{ diff.feed_name }}</a></td>
+              <td>{{ diff.diff_html|safe }}</td>
+              <td class="expanded">
+                <span class="short">{{ diff.title_orig|truncate(15) }} </span>
+                <span class="long">{{ diff.title_orig }} </span>
+              </td>
+              <td class="expanded">
+                <span class="short">{{ diff.title_new|truncate(15) }} </span>
+                <span class="long">{{ diff.title_new}} </span>
+              </td>
+            </tr>
+            {% endfor %}
+          </tbody>
+        </table>
+      </div>
+      {{ pagination.links }}
+    </div>
+  </body>
+</html>