Merge branch 'master' of git.eda.gay:eda.gay

7 files changed, 137 insertions, 18 deletions

diff --git a/app.py b/app.py
index 7847914..81cfad9 100755
--- a/app.py
+++ b/app.py
@@ -240,6 +240,16 @@ def serve_random():
             localimg = "/img/random.jpg?seed=%i" % random.randint(0, 9999)
         )
 
+@app.route("/questions")
+def serve_questions():
+    with database.Database() as db:
+        return flask.render_template(
+            "questions.html.j2",
+            **get_template_items("questions and answers", db),
+            curiouscat_username = db.get_curiouscat_username(),
+            qnas = db.get_curiouscat_qnas()
+        )
+
 if __name__ == "__main__":
     try:
         if sys.argv[1] == "--production":
diff --git a/curiouscat.py b/curiouscat.py
new file mode 100644
index 0000000..531f08d
--- /dev/null
+++ b/curiouscat.py
@@ -0,0 +1,47 @@
+import datetime
+import requests
+import json
+
+def get_curiouscat_qnas_after(name, last_timestamp = None):
+    if last_timestamp is None:
+        url = "https://curiouscat.live/api/v2/profile?username=%s&count=100" % (name)
+    else:
+        url = "https://curiouscat.live/api/v2/profile?username=%s&count=100&max_timestamp=%d" % (name, last_timestamp)
+
+    req = requests.get(url)
+    return req.json()["posts"]
+
+def get_all_curiouscat_qnas(name):
+    out = []
+    period = get_curiouscat_qnas_after(name)
+    out += period
+    while len(period) == 100:       
+        oldest = min([i["timestamp"] for i in period])
+        period = get_curiouscat_qnas_after("jwnskanzkwk", last_timestamp = oldest - 1)
+
+        out += period
+
+    return post_process(out, name)
+
+def get_all_curiouscat_qnas_before(name, min_dt):
+    url = "https://curiouscat.live/api/v2/profile?username=%s&count=100&min_timestamp=%d" % (name, int(min_dt.timestamp()) + 1)
+    req = requests.get(url)
+    return post_process(req.json()["posts"], name)
+
+def post_process(cc, name):
+    return [
+        {
+            "id": i["id"],
+            "link": "https://curiouscat.me/%s/post/%d" % (name, i["id"]),
+            "datetime": datetime.datetime.fromtimestamp(i["timestamp"]),
+            "question": i["comment"],
+            "answer": i["reply"]
+        }
+        for i in cc
+    ]
+
+if __name__ == "__main__":
+    import database
+
+    with database.Database() as db:
+        print(db.append_curiouscat_qnas(get_all_curiouscat_qnas_before("jwnskanzkwk", db.get_biggest_curiouscat_timestamp())))
diff --git a/database.py b/database.py
index 78cf237..1877ab7 100755
--- a/database.py
+++ b/database.py
@@ -3,7 +3,9 @@ from dataclasses import dataclass
 from github import Github
 from lxml import html
 import configparser
+import curiouscat
 import threading
+import operator
 import datetime
 import requests
 import twython
@@ -225,7 +227,9 @@ class Database:
         threading.Thread(target = update_cache).start()
         out = {}
         with self.__connection.cursor() as cursor:
-            cursor.execute("SELECT tweet_id, tweeted_at, tweet FROM diary WHERE replying_to IS NULL ORDER BY tweeted_at DESC;")
+            # cursor.execute("SELECT tweet_id, tweeted_at, tweet FROM diary WHERE replying_to IS NULL ORDER BY tweeted_at DESC;")
+            # attempt to ignore curiouscat automatic tweets by comparing with the q&a table
+            cursor.execute("SELECT tweet_id, tweeted_at, tweet FROM diary WHERE replying_to IS NULL AND tweet_id NOT IN (SELECT tweet_id FROM diary INNER JOIN qnas ON SUBSTRING(tweet, 1, 16) = SUBSTRING(question, 1, 16)) ORDER BY tweeted_at DESC;")
             for tweet_id, tweeted_at, tweet_text in cursor.fetchall():
                 # print(tweet_id, tweeted_at, tweet_text)
                 out[tweeted_at] = [{
@@ -304,9 +308,45 @@ class Database:
 
         self.__connection.commit()
 
+    def get_curiouscat_username(self):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT link FROM headerLinks WHERE name = 'curiouscat';")
+        return urlparse(cursor.fetchone()[0]).path.split("/")[1]
+
+    def append_curiouscat_qnas(self, qnas):
+        with self.__connection.cursor() as cursor:
+            for qna in qnas:
+                cursor.execute("SELECT curiouscat_id FROM qnas WHERE curiouscat_id = %s;", (qna["id"], ))
+                if cursor.fetchone() is None:
+
+                    cursor.execute("INSERT INTO `qnas` VALUES (%s, %s, %s, %s, %s);", (
+                        qna["id"], qna["link"], qna["datetime"], qna["question"], qna["answer"]
+                    ))
+                    print("Appended question with timestamp %s" % datetime.datetime.fromtimestamp(qna["id"]).isoformat())
+
+                else:
+                    print("Skipped question with timestamp %s" % datetime.datetime.fromtimestamp(qna["id"]).isoformat())
+        self.__connection.commit()
+
+    def get_biggest_curiouscat_timestamp(self):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT MAX(`timestamp`) FROM `qnas`;")
+            return cursor.fetchone()[0]
+
+    def get_curiouscat_qnas(self):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT * FROM qnas;")
+            return sorted(cursor.fetchall(), key = operator.itemgetter(2), reverse = True)
+
 def update_cache():
     # print("updating cache...")
     with Database() as db:
+        db.append_curiouscat_qnas(
+            curiouscat.get_all_curiouscat_qnas_before(
+                db.get_curiouscat_username(), 
+                db.get_biggest_curiouscat_timestamp()
+            )
+        )
         db.fetch_diary()
         db.update_twitter_cache(request_recent_tweets(10000))
         # print("Done updating twitter cache...")
@@ -339,23 +379,26 @@ def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta(
     out = []
     for repo in g.get_user().get_repos():
         # print(repo.name, list(repo.get_branches()))
-        for commit in repo.get_commits(since = since):
-            out.append({
-                "repo": repo.name,
-                "message": commit.commit.message,
-                "url": commit.html_url,
-                "datetime": commit.commit.author.date,
-                "stats": {
-                    "additions": commit.stats.additions,
-                    "deletions": commit.stats.deletions,
-                    "total": commit.stats.total
-                }
-            })
+        try:
+            for commit in repo.get_commits(since = since):
+                out.append({
+                    "repo": repo.name,
+                    "message": commit.commit.message,
+                    "url": commit.html_url,
+                    "datetime": commit.commit.author.date,
+                    "stats": {
+                        "additions": commit.stats.additions,
+                        "deletions": commit.stats.deletions,
+                        "total": commit.stats.total
+                    }
+                })
+        except Exception as e:
+            print(e)
+
     return sorted(out, key = lambda a: a["datetime"], reverse = True) 
 
 
 if __name__ == "__main__":
-    import app
+    # print(request_recent_commits())
     with Database() as db:
-        print(app.get_sidebar_img(db))
-        # print(db.get_sidebar_images())
+        print(db.get_curiouscat_qnas())
diff --git a/static/style.css b/static/style.css
index 32dc92d..4a47ea7 100755
--- a/static/style.css
+++ b/static/style.css
@@ -125,6 +125,10 @@ aside {
     font-size: xx-small;
 }
 
+#sidebarImage {
+    transform: translateX(10px);
+}
+
 .header_linker {
     font-size: x-small;
 }
diff --git a/templates/diary.html.j2 b/templates/diary.html.j2
index 15cbe71..d7c363b 100755
--- a/templates/diary.html.j2
+++ b/templates/diary.html.j2
@@ -1,5 +1,7 @@
 {% extends "template.html.j2" %}
 {% block content %}
+    <h4>this page might not be up-to-date if my diary account is search banned</h4>
+    <p>if in doubt check my <a href="https://twitter.com/FORMER_SHOTA">diary account</a> <br> <s>serves me right for using the official API instead of HTML scraping like i did with the <a href="/">recent tweets</a> thing</s> <br> <a href="https://shadowban.yuzurisa.com/FORMER_SHOTA">check if i'm currently search banned</a></p>
     <dl>
         {% for dt, entries in diary.items() %}
             <dt><a href="{{ entries[0]['link'] }}">{{ dt }}</a></dt>
@@ -21,4 +23,4 @@
             </dd>
         {% endfor %}
     </dl>
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/templates/questions.html.j2 b/templates/questions.html.j2
new file mode 100644
index 0000000..2d0eaf2
--- /dev/null
+++ b/templates/questions.html.j2
@@ -0,0 +1,13 @@
+{% extends "template.html.j2" %}
+{% block content %}
+    <h4><a href="https://curiouscat.live/{{ curiouscat_username }}">ask a question!</a></h4>
+    <dl>
+        {% for id_, link, dt, question, answer in qnas %}
+            <dt><a href="{{ link }}">{{ dt.isoformat() }}</a></dt>
+            <dd>
+                <dt class="question"><p>{{ question }}</p></dt>
+                <dd class="answer"><p>{{ answer }}</p></dd>
+            </dd>
+        {% endfor %}
+    </dl>
+{% endblock %}
diff --git a/templates/template.html.j2 b/templates/template.html.j2
index b0bf62c..f80155d 100755
--- a/templates/template.html.j2
+++ b/templates/template.html.j2
@@ -33,7 +33,7 @@
                             </ul>
                         </nav>
                     </div>
-                    <a href="/">
+                    <a id=sidebarImage href="/">
                         <img alt="{{image[0]}}" src="{{image[1]}}">
                     </a>         
                 </div>