diff options
| -rwxr-xr-x | app.py | 10 | ||||
| -rw-r--r-- | curiouscat.py | 47 | ||||
| -rwxr-xr-x | database.py | 46 | ||||
| -rw-r--r-- | templates/questions.html.j2 | 13 |
4 files changed, 111 insertions, 5 deletions
@@ -236,6 +236,16 @@ def serve_random(): localimg = "/img/random.jpg?seed=%i" % random.randint(0, 9999) ) +@app.route("/questions") +def serve_questions(): + with database.Database() as db: + return flask.render_template( + "questions.html.j2", + **get_template_items("random image", db), + curiouscat_username = db.get_curiouscat_username(), + qnas = db.get_curiouscat_qnas() + ) + if __name__ == "__main__": try: if sys.argv[1] == "--production": diff --git a/curiouscat.py b/curiouscat.py new file mode 100644 index 0000000..90cdaf6 --- /dev/null +++ b/curiouscat.py @@ -0,0 +1,47 @@ +import datetime +import requests +import json + +def get_curiouscat_qnas_after(name, last_timestamp = None): + if last_timestamp is None: + url = "https://curiouscat.live/api/v2/profile?username=%s&count=100" % (name) + else: + url = "https://curiouscat.live/api/v2/profile?username=%s&count=100&max_timestamp=%d" % (name, last_timestamp) + + req = requests.get(url) + return req.json()["posts"] + +def get_all_curiouscat_qnas(name): + out = [] + period = get_curiouscat_qnas_after(name) + out += period + while len(period) == 100: + oldest = min([i["timestamp"] for i in period]) + period = get_curiouscat_qnas_after("jwnskanzkwk", last_timestamp = oldest - 1) + + out += period + + return post_process(out, name) + +def get_all_curiouscat_qnas_before(name, min_dt): + url = "https://curiouscat.live/api/v2/profile?username=%s&count=100&min_timestamp=%d" % (name, int(min_dt.timestamp()) + 1) + req = requests.get(url) + return post_process(req.json()["posts"], "name") + +def post_process(cc, name): + return [ + { + "id": i["id"], + "link": "https://curiouscat.me/%s/post/%d" % (name, i["id"]), + "datetime": datetime.datetime.fromtimestamp(i["timestamp"]), + "question": i["comment"], + "answer": i["reply"] + } + for i in cc + ] + +if __name__ == "__main__": + import database + + with database.Database() as db: + print(db.append_curiouscat_qnas(get_all_curiouscat_qnas_before("jwnskanzkwk", db.get_biggest_curiouscat_timestamp())))
\ No newline at end of file diff --git a/database.py b/database.py index c8bf5ff..8829615 100755 --- a/database.py +++ b/database.py @@ -3,7 +3,9 @@ from dataclasses import dataclass from github import Github from lxml import html import configparser +import curiouscat import threading +import operator import datetime import requests import twython @@ -304,9 +306,45 @@ class Database: self.__connection.commit() + def get_curiouscat_username(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT link FROM headerLinks WHERE name = 'curiouscat';") + return urlparse(cursor.fetchone()[0]).path.split("/")[1] + + def append_curiouscat_qnas(self, qnas): + with self.__connection.cursor() as cursor: + for qna in qnas: + cursor.execute("SELECT curiouscat_id FROM qnas WHERE curiouscat_id = %s;", (qna["id"], )) + if cursor.fetchone() is None: + + cursor.execute("INSERT INTO `qnas` VALUES (%s, %s, %s, %s, %s);", ( + qna["id"], qna["link"], qna["datetime"], qna["question"], qna["answer"] + )) + print("Appended question with timestamp %s" % datetime.datetime.fromtimestamp(qna["id"]).isoformat()) + + else: + print("Skipped question with timestamp %s" % datetime.datetime.fromtimestamp(qna["id"]).isoformat()) + self.__connection.commit() + + def get_biggest_curiouscat_timestamp(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT MAX(`timestamp`) FROM `qnas`;") + return cursor.fetchone()[0] + + def get_curiouscat_qnas(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT * FROM qnas;") + return sorted(cursor.fetchall(), key = operator.itemgetter(2), reverse = True) + def update_cache(): # print("updating cache...") with Database() as db: + db.append_curiouscat_qnas( + curiouscat.get_all_curiouscat_qnas_before( + db.get_curiouscat_username(), + db.get_biggest_curiouscat_timestamp() + ) + ) db.fetch_diary() db.update_twitter_cache(request_recent_tweets(10000)) # print("Done updating twitter cache...") @@ -359,8 +397,6 @@ def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta( if __name__ == "__main__": - print(request_recent_commits()) - #import app - #with Database() as db: - # print(app.get_sidebar_img(db)) - # # print(db.get_sidebar_images()) + # print(request_recent_commits()) + with Database() as db: + print(db.get_curiouscat_qnas()) diff --git a/templates/questions.html.j2 b/templates/questions.html.j2 new file mode 100644 index 0000000..fd53120 --- /dev/null +++ b/templates/questions.html.j2 @@ -0,0 +1,13 @@ +{% extends "template.html.j2" %} +{% block content %} + <h4><a href="https://curiouscat.live/{{ curiouscat_username }}">ask a question!</a></h4> + <dl> + {% for id_, link, dt, question, answer in qnas %} + <dt><a href="{{ link }}">{{ dt.isoformat() }}</a></dt> + <dd> + <dt class="question">{{ question }}</dt> + <dd class="answer">{{ answer }}</dd> + </dd> + {% endfor %} + </dl> +{% endblock %}
\ No newline at end of file |