From cc2aede0c79ffab0ea6ece346225095a85377269 Mon Sep 17 00:00:00 2001 From: Kira Date: Thu, 9 Feb 2023 18:51:35 +0000 Subject: Update sidebar id --- templates/template.html.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/template.html.j2 b/templates/template.html.j2 index b0bf62c..f80155d 100755 --- a/templates/template.html.j2 +++ b/templates/template.html.j2 @@ -33,7 +33,7 @@ - + {{image[0]}} -- cgit v1.2.3 From 655895f25c78fe7e0750e89c1e1d79fd2f7a29ef Mon Sep 17 00:00:00 2001 From: Kira Date: Thu, 9 Feb 2023 18:51:46 +0000 Subject: Update styling to fix sidebar --- static/style.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/static/style.css b/static/style.css index 32dc92d..4a47ea7 100755 --- a/static/style.css +++ b/static/style.css @@ -125,6 +125,10 @@ aside { font-size: xx-small; } +#sidebarImage { + transform: translateX(10px); +} + .header_linker { font-size: x-small; } -- cgit v1.2.3 From 988c90ce774d163a5aec17e18fd9c9c3c8bfb3b0 Mon Sep 17 00:00:00 2001 From: jwansek Date: Sun, 12 Feb 2023 21:34:35 +0000 Subject: Added exception handling for when a github repo is empty --- database.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/database.py b/database.py index cf51922..c8bf5ff 100755 --- a/database.py +++ b/database.py @@ -339,23 +339,28 @@ def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta( out = [] for repo in g.get_user().get_repos(): # print(repo.name, list(repo.get_branches())) - for commit in repo.get_commits(since = since): - out.append({ - "repo": repo.name, - "message": commit.commit.message, - "url": commit.html_url, - "datetime": commit.commit.author.date, - "stats": { - "additions": commit.stats.additions, - "deletions": commit.stats.deletions, - "total": commit.stats.total - } - }) + try: + for commit in repo.get_commits(since = since): + out.append({ + "repo": repo.name, + "message": commit.commit.message, + "url": commit.html_url, + "datetime": commit.commit.author.date, + "stats": { + "additions": commit.stats.additions, + "deletions": commit.stats.deletions, + "total": commit.stats.total + } + }) + except Exception as e: + print(e) + return sorted(out, key = lambda a: a["datetime"], reverse = True) if __name__ == "__main__": - import app - with Database() as db: - print(app.get_sidebar_img(db)) - # print(db.get_sidebar_images()) + print(request_recent_commits()) + #import app + #with Database() as db: + # print(app.get_sidebar_img(db)) + # # print(db.get_sidebar_images()) -- cgit v1.2.3 From 2831eb8af07b1c387f17cf4d9a4b3a1b1aa47bb5 Mon Sep 17 00:00:00 2001 From: jwansek Date: Sat, 25 Feb 2023 23:18:49 +0000 Subject: Added disclaimer to diary page --- templates/diary.html.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/diary.html.j2 b/templates/diary.html.j2 index 15cbe71..d7c363b 100755 --- a/templates/diary.html.j2 +++ b/templates/diary.html.j2 @@ -1,5 +1,7 @@ {% extends "template.html.j2" %} {% block content %} +

this page might not be up-to-date if my diary account is search banned

+

if in doubt check my diary account
serves me right for using the official API instead of HTML scraping like i did with the recent tweets thing
check if i'm currently search banned

{% for dt, entries in diary.items() %}
{{ dt }}
@@ -21,4 +23,4 @@ {% endfor %}
-{% endblock %} \ No newline at end of file +{% endblock %} -- cgit v1.2.3 From 7c53f89ccdd92e308d14d069e2e070f7a1c2da26 Mon Sep 17 00:00:00 2001 From: jwansek Date: Mon, 27 Feb 2023 00:05:41 +0000 Subject: Added q&a page --- app.py | 10 ++++++++++ curiouscat.py | 47 +++++++++++++++++++++++++++++++++++++++++++++ database.py | 46 +++++++++++++++++++++++++++++++++++++++----- templates/questions.html.j2 | 13 +++++++++++++ 4 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 curiouscat.py create mode 100644 templates/questions.html.j2 diff --git a/app.py b/app.py index 75558f8..556d61a 100755 --- a/app.py +++ b/app.py @@ -236,6 +236,16 @@ def serve_random(): localimg = "/img/random.jpg?seed=%i" % random.randint(0, 9999) ) +@app.route("/questions") +def serve_questions(): + with database.Database() as db: + return flask.render_template( + "questions.html.j2", + **get_template_items("random image", db), + curiouscat_username = db.get_curiouscat_username(), + qnas = db.get_curiouscat_qnas() + ) + if __name__ == "__main__": try: if sys.argv[1] == "--production": diff --git a/curiouscat.py b/curiouscat.py new file mode 100644 index 0000000..90cdaf6 --- /dev/null +++ b/curiouscat.py @@ -0,0 +1,47 @@ +import datetime +import requests +import json + +def get_curiouscat_qnas_after(name, last_timestamp = None): + if last_timestamp is None: + url = "https://curiouscat.live/api/v2/profile?username=%s&count=100" % (name) + else: + url = "https://curiouscat.live/api/v2/profile?username=%s&count=100&max_timestamp=%d" % (name, last_timestamp) + + req = requests.get(url) + return req.json()["posts"] + +def get_all_curiouscat_qnas(name): + out = [] + period = get_curiouscat_qnas_after(name) + out += period + while len(period) == 100: + oldest = min([i["timestamp"] for i in period]) + period = get_curiouscat_qnas_after("jwnskanzkwk", last_timestamp = oldest - 1) + + out += period + + return post_process(out, name) + +def get_all_curiouscat_qnas_before(name, min_dt): + url = "https://curiouscat.live/api/v2/profile?username=%s&count=100&min_timestamp=%d" % (name, int(min_dt.timestamp()) + 1) + req = requests.get(url) + return post_process(req.json()["posts"], "name") + +def post_process(cc, name): + return [ + { + "id": i["id"], + "link": "https://curiouscat.me/%s/post/%d" % (name, i["id"]), + "datetime": datetime.datetime.fromtimestamp(i["timestamp"]), + "question": i["comment"], + "answer": i["reply"] + } + for i in cc + ] + +if __name__ == "__main__": + import database + + with database.Database() as db: + print(db.append_curiouscat_qnas(get_all_curiouscat_qnas_before("jwnskanzkwk", db.get_biggest_curiouscat_timestamp()))) \ No newline at end of file diff --git a/database.py b/database.py index c8bf5ff..8829615 100755 --- a/database.py +++ b/database.py @@ -3,7 +3,9 @@ from dataclasses import dataclass from github import Github from lxml import html import configparser +import curiouscat import threading +import operator import datetime import requests import twython @@ -304,9 +306,45 @@ class Database: self.__connection.commit() + def get_curiouscat_username(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT link FROM headerLinks WHERE name = 'curiouscat';") + return urlparse(cursor.fetchone()[0]).path.split("/")[1] + + def append_curiouscat_qnas(self, qnas): + with self.__connection.cursor() as cursor: + for qna in qnas: + cursor.execute("SELECT curiouscat_id FROM qnas WHERE curiouscat_id = %s;", (qna["id"], )) + if cursor.fetchone() is None: + + cursor.execute("INSERT INTO `qnas` VALUES (%s, %s, %s, %s, %s);", ( + qna["id"], qna["link"], qna["datetime"], qna["question"], qna["answer"] + )) + print("Appended question with timestamp %s" % datetime.datetime.fromtimestamp(qna["id"]).isoformat()) + + else: + print("Skipped question with timestamp %s" % datetime.datetime.fromtimestamp(qna["id"]).isoformat()) + self.__connection.commit() + + def get_biggest_curiouscat_timestamp(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT MAX(`timestamp`) FROM `qnas`;") + return cursor.fetchone()[0] + + def get_curiouscat_qnas(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT * FROM qnas;") + return sorted(cursor.fetchall(), key = operator.itemgetter(2), reverse = True) + def update_cache(): # print("updating cache...") with Database() as db: + db.append_curiouscat_qnas( + curiouscat.get_all_curiouscat_qnas_before( + db.get_curiouscat_username(), + db.get_biggest_curiouscat_timestamp() + ) + ) db.fetch_diary() db.update_twitter_cache(request_recent_tweets(10000)) # print("Done updating twitter cache...") @@ -359,8 +397,6 @@ def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta( if __name__ == "__main__": - print(request_recent_commits()) - #import app - #with Database() as db: - # print(app.get_sidebar_img(db)) - # # print(db.get_sidebar_images()) + # print(request_recent_commits()) + with Database() as db: + print(db.get_curiouscat_qnas()) diff --git a/templates/questions.html.j2 b/templates/questions.html.j2 new file mode 100644 index 0000000..fd53120 --- /dev/null +++ b/templates/questions.html.j2 @@ -0,0 +1,13 @@ +{% extends "template.html.j2" %} +{% block content %} +

ask a question!

+
+ {% for id_, link, dt, question, answer in qnas %} +
{{ dt.isoformat() }}
+
+
{{ question }}
+
{{ answer }}
+ + {% endfor %} +
+{% endblock %} \ No newline at end of file -- cgit v1.2.3 From 96090364a7ebcfcf35f924cdd9f24f6898257e6a Mon Sep 17 00:00:00 2001 From: jwansek Date: Mon, 27 Feb 2023 00:09:28 +0000 Subject: Added

tags on q&a page --- templates/questions.html.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/questions.html.j2 b/templates/questions.html.j2 index fd53120..6f9c1eb 100644 --- a/templates/questions.html.j2 +++ b/templates/questions.html.j2 @@ -5,8 +5,8 @@ {% for id_, link, dt, question, answer in qnas %}

{{ dt.isoformat() }}
-
{{ question }}
-
{{ answer }}
+

{{ question }}

+

{{ answer }}

{% endfor %} -- cgit v1.2.3 From a617a0fa5a47428b08a294ddda2bf9b2499c14de Mon Sep 17 00:00:00 2001 From: jwansek Date: Mon, 27 Feb 2023 01:13:52 +0000 Subject: fixed issue with page header --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 556d61a..035f734 100755 --- a/app.py +++ b/app.py @@ -241,7 +241,7 @@ def serve_questions(): with database.Database() as db: return flask.render_template( "questions.html.j2", - **get_template_items("random image", db), + **get_template_items("questions and answers", db), curiouscat_username = db.get_curiouscat_username(), qnas = db.get_curiouscat_qnas() ) -- cgit v1.2.3 From 966fe80c956fec5dd2fb3ee27b488a8e1453eafa Mon Sep 17 00:00:00 2001 From: jwansek Date: Thu, 2 Mar 2023 16:44:41 +0000 Subject: Updated fetching for the diary page, attempting to ignore automatic curiouscat tweets buy comparing the first 16 characters with the curiouscat table --- database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/database.py b/database.py index 8829615..1a02cea 100755 --- a/database.py +++ b/database.py @@ -227,7 +227,9 @@ class Database: threading.Thread(target = update_cache).start() out = {} with self.__connection.cursor() as cursor: - cursor.execute("SELECT tweet_id, tweeted_at, tweet FROM diary WHERE replying_to IS NULL ORDER BY tweeted_at DESC;") + # cursor.execute("SELECT tweet_id, tweeted_at, tweet FROM diary WHERE replying_to IS NULL ORDER BY tweeted_at DESC;") + # attempt to ignore curiouscat automatic tweets by comparing with the q&a table + cursor.execute("SELECT tweet_id, tweeted_at, tweet FROM diary WHERE replying_to IS NULL AND tweet_id NOT IN (SELECT tweet_id FROM diary INNER JOIN qnas ON SUBSTRING(tweet, 1, 16) = SUBSTRING(question, 1, 16)) ORDER BY tweeted_at DESC;") for tweet_id, tweeted_at, tweet_text in cursor.fetchall(): # print(tweet_id, tweeted_at, tweet_text) out[tweeted_at] = [{ -- cgit v1.2.3 From cfc2bfbec443ea24f4af0027a077267771b0698f Mon Sep 17 00:00:00 2001 From: jwansek Date: Sun, 5 Mar 2023 13:40:17 +0000 Subject: Fixed bug with curiouscat URLs --- curiouscat.py | 4 ++-- templates/questions.html.j2 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/curiouscat.py b/curiouscat.py index 90cdaf6..531f08d 100644 --- a/curiouscat.py +++ b/curiouscat.py @@ -26,7 +26,7 @@ def get_all_curiouscat_qnas(name): def get_all_curiouscat_qnas_before(name, min_dt): url = "https://curiouscat.live/api/v2/profile?username=%s&count=100&min_timestamp=%d" % (name, int(min_dt.timestamp()) + 1) req = requests.get(url) - return post_process(req.json()["posts"], "name") + return post_process(req.json()["posts"], name) def post_process(cc, name): return [ @@ -44,4 +44,4 @@ if __name__ == "__main__": import database with database.Database() as db: - print(db.append_curiouscat_qnas(get_all_curiouscat_qnas_before("jwnskanzkwk", db.get_biggest_curiouscat_timestamp()))) \ No newline at end of file + print(db.append_curiouscat_qnas(get_all_curiouscat_qnas_before("jwnskanzkwk", db.get_biggest_curiouscat_timestamp()))) diff --git a/templates/questions.html.j2 b/templates/questions.html.j2 index 6f9c1eb..2d0eaf2 100644 --- a/templates/questions.html.j2 +++ b/templates/questions.html.j2 @@ -10,4 +10,4 @@ {% endfor %} -{% endblock %} \ No newline at end of file +{% endblock %} -- cgit v1.2.3