Added scraping whispa for Q&As

5 files changed, 52 insertions, 11 deletions

diff --git a/app.py b/app.py
index 2f38fb6..50d5b5b 100755
--- a/app.py
+++ b/app.py
@@ -225,6 +225,7 @@ def serve_questions():
         return flask.render_template(
             "questions.html.j2",
             **get_template_items("questions and answers", db),
+            qnas_link = CONFIG.get("qnas", "url"),
             qnas = db.get_qnas()
         )
 
diff --git a/database.py b/database.py
index d5327e4..cd15cec 100644
--- a/database.py
+++ b/database.py
@@ -224,15 +224,20 @@ class Database:
                 cursor.execute("SELECT curiouscat_id FROM qnas WHERE curiouscat_id = %s;", (qna["id"], ))
                 if cursor.fetchone() is None:
 
-                    cursor.execute("INSERT INTO `qnas` VALUES (%s, %s, %s, %s, %s);", (
-                        qna["id"], qna["link"], qna["datetime"], qna["question"], qna["answer"]
+                    cursor.execute("INSERT INTO `qnas` VALUES (%s, %s, %s, %s, %s, %s);", (
+                        qna["id"], qna["link"], qna["datetime"], qna["question"], qna["answer"], qna["host"]
                     ))
-                    print("Appended question with timestamp %s" % datetime.datetime.fromtimestamp(qna["id"]).isoformat())
+                    print("Appended question with timestamp %s" % qna["datetime"].isoformat())
 
                 else:
-                    print("Skipped question with timestamp %s" % datetime.datetime.fromtimestamp(qna["id"]).isoformat())
+                    print("Skipped question with timestamp %s" % qna["datetime"].isoformat())
         self.__connection.commit()
 
+    def get_oldest_qna(self):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT MAX(timestamp) FROM qnas;")
+            return cursor.fetchone()[0]
+
     def get_qnas(self):
         with self.__connection.cursor() as cursor:
             cursor.execute("SELECT * FROM qnas;")
@@ -243,12 +248,12 @@ def update_cache():
     with Database() as db:
         db.update_commit_cache(services.request_recent_commits(since = db.get_last_commit_time()))
         print("Finished adding github commits...")
+        db.append_qnas(services.scrape_whispa(db.config.get("qnas", "url"), since = db.get_oldest_qna()))
+        print("Finished parsing Q&As...")
 
 
 if __name__ == "__main__":
     update_cache()
-    import json
-    with Database() as db:
-        print(db.get_cached_commits()[0])
+    
 
 
diff --git a/services.py b/services.py
index 075d533..b56a07d 100755
--- a/services.py
+++ b/services.py
@@ -6,6 +6,7 @@ import multiprocessing
 import pihole as ph
 import qbittorrent
 import configparser
+import math as maths
 import requests
 import datetime
 import urllib
@@ -242,6 +243,30 @@ def parse_tweet(tweet_url):
 
     return dt, replying_to, text, images
 
+def scrape_whispa(whispa_url, since):
+    tree = html.fromstring(requests.get(whispa_url).content.decode())
+    qnas = []
+    # we're not doing proper HTML scraping here really... since the site uses client side rendering
+    # we rather parse the JS scripts to get the JSON payload of useful information... sadly this looks horrible
+    for i, script in enumerate(tree.xpath("/html/body/script"), 0):
+        js = str(script.text)
+        if "receivedFeedback" in js:
+            # my god this is horrible...
+            for j in json.loads(json.loads(js[19:-1])[1][2:])[0][3]["loadedUser"]["receivedFeedback"]:
+                dt = datetime.datetime.fromisoformat(j["childFeedback"][0]["createdAt"][:-1])
+            
+                qnas.append({
+                    # "id": int(str(maths.modf(maths.log(int(j["id"], 16)))[0])[2:]),
+                    "id": int(dt.timestamp()),
+                    "link": None,
+                    "datetime": dt,
+                    "question": j["content"],
+                    "answer": j["childFeedback"][0]["content"],
+                    "host": "whispa.sh"
+                })
+    return qnas
+
+
 if __name__ == "__main__":
     # print(get_trans_stats())
 
@@ -250,4 +275,6 @@ if __name__ == "__main__":
 
     # print(parse_tweet("https://nitter.net/HONMISGENDERER/status/1694231618443981161#m"))
 
-    print(request_recent_commits(since = datetime.datetime.now() - datetime.timedelta(days=30)))
+    # print(request_recent_commits(since = datetime.datetime.now() - datetime.timedelta(days=30)))
+
+    print(scrape_whispa(CONFIG.get("qnas", "url"), datetime.datetime.fromtimestamp(0.0)))
diff --git a/static/style.css b/static/style.css
index 4a47ea7..6069ebf 100755
--- a/static/style.css
+++ b/static/style.css
@@ -147,6 +147,10 @@ aside {
     margin-top: 10px;
 }
 
+.qnaheader {
+    font-weight: bold;
+}
+
 blockquote span {
     color: #789922;
 }
diff --git a/templates/questions.html.j2 b/templates/questions.html.j2
index 97edd38..eb58380 100644
--- a/templates/questions.html.j2
+++ b/templates/questions.html.j2
@@ -1,9 +1,13 @@
 {% extends "template.html.j2" %}
 {% block content %}
-    <h4><a href="https://whispa.sh/@boymoderology">ask a question!</a></h4>
+    <h4><a href="{{ qnas_link }}">ask a question!</a></h4>
     <dl>
-        {% for id_, link, dt, question, answer in qnas %}
-            <dt><a href="{{ link }}">{{ dt.isoformat() }}</a></dt>
+        {% for id_, link, dt, question, answer, host in qnas %}
+            {% if host == "curiouscat" %}
+                <dt class="qnaheader"><a href="{{ link }}">{{ dt.isoformat() }}</a> - {{ host }}</dt>
+            {% else %}
+                <dt class="qnaheader">{{ dt.isoformat() }} - {{ host }}</dt>
+            {% endif %}
             <dd>
                 <dt class="question"><p>{{ question }}</p></dt>
                 <dd class="answer"><p>{{ answer }}</p></dd>