diff options
| -rw-r--r-- | app.py | 18 | ||||
| -rw-r--r-- | database.py | 118 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | services.py | 29 | ||||
| -rw-r--r-- | templates/index.html | 10 |
5 files changed, 145 insertions, 31 deletions
@@ -54,7 +54,8 @@ def index(): **get_template_items("eden's site :3", db), markdown = parser.parse_text(f.read()), featured_thoughts = db.get_featured_thoughts(), - tweets = services.get_recent_tweets(6) + tweets = db.get_cached_tweets(7) + [("view all tweets...", db.get_my_twitter())], + commits = db.get_cached_commits(since = datetime.datetime.now() - datetime.timedelta(days = 7)) ) @app.route("/discord") @@ -131,6 +132,21 @@ def serve_image(filename): else: flask.abort(404) +@app.route("/api/<infoRequest>") +def serve_api_request(infoRequest): + if infoRequest == "commits": + try: + return flask.jsonify(services.request_recent_commits(since = datetime.datetime.fromtimestamp(int(flask.request.args['since'])))) + except (ValueError, KeyError): + flask.abort(400) + elif infoRequest == "tweets": + try: + return flask.jsonify(services.request_recent_tweets(int(flask.request.args['toGet']))) + except (ValueError, KeyError): + flask.abort(400) + else: + flask.abort(404) + @app.route("/preview") def preview(): diff --git a/database.py b/database.py index 5e50f82..0edcad7 100644 --- a/database.py +++ b/database.py @@ -1,7 +1,14 @@ +from urllib.parse import urlparse from dataclasses import dataclass +from github import Github +from lxml import html import configparser +import threading +import datetime +import requests import pymysql import random +import os @dataclass class Database: @@ -107,6 +114,115 @@ class Database: """) return cursor.fetchall() + def get_cached_tweets(self, numToGet = None, recurse = True): + with self.__connection.cursor() as cursor: + if numToGet is not None: + cursor.execute("SELECT text, url FROM twitterCache ORDER BY appended DESC LIMIT %s;", (numToGet, )) + else: + cursor.execute("SELECT text, url FROM twitterCache ORDER BY appended DESC;") + if recurse: + threading.Thread(target = update_cache).start() + return list(cursor.fetchall()) + + def update_twitter_cache(self, requested): + urls = [i[1] for i in self.get_cached_tweets(recurse = False)] + with self.__connection.cursor() as cursor: + for url, text in requested: + if url not in urls: + cursor.execute("INSERT INTO twitterCache (text, url) VALUES (%s, %s);", (text, url)) + self.__connection.commit() + + def get_cached_commits(self, since = None, recurse = True): + with self.__connection.cursor() as cursor: + if since is not None: + cursor.execute("SELECT message, url, commitTime, additions, deletions, total FROM commitCache WHERE commitTime > %s ORDER BY commitTime DESC;", (since, )) + else: + cursor.execute("SELECT message, url, commitTime, additions, deletions, total FROM commitCache ORDER BY commitTime DESC;") + return [{ + "repo": urlparse(i[1]).path.split("/")[2], + "message": i[0], + "url": i[1], + "datetime": i[2], + "stats": { + "additions": i[3], + "deletions": i[4], + "total": i[5] + } + } for i in cursor.fetchall()] + + def update_commit_cache(self, requested): + urls = [i["url"] for i in self.get_cached_commits(recurse = False)] + with self.__connection.cursor() as cursor: + for commit in requested: + if commit["url"] not in urls: + cursor.execute(""" + INSERT INTO commitCache (message, url, commitTime, additions, deletions, total) + VALUES (%s, %s, %s, %s, %s, %s)""", + (commit["message"], commit["url"], commit["datetime"], commit["stats"]["additions"], commit["stats"]["deletions"], commit["stats"]["total"]) + ) + self.__connection.commit() + + def get_last_commit_time(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT MAX(commitTime) FROM commitCache;") + return cursor.fetchone()[0] + + def get_my_twitter(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT link FROM headerLinks WHERE name = 'twitter';") + return cursor.fetchone()[0] + +def update_cache(): + # print("updating cache...") + with Database() as db: + db.update_twitter_cache(request_recent_tweets(10000)) + # print("Done updating twitter cache...") + db.update_commit_cache(request_recent_commits(since = db.get_last_commit_time())) + # print("Done updating commit cache...") + +CONFIG = configparser.ConfigParser() +CONFIG.read("edaweb.conf") + +def request_recent_tweets(numToGet): + tweets = [] + domain = "http://" + CONFIG.get("nitter", "domain") + with Database() as db: + for title, url in db.get_header_links(): + if title == "twitter": + break + tree = html.fromstring(requests.get(url).content) + for i, tweetUrlElement in enumerate(tree.xpath('//*[@class="tweet-link"]'), 0): + if i > 0: + tweets.append(( + domain + tweetUrlElement.get("href"), + tweetUrlElement.getparent().find_class("tweet-content media-body")[0].text + )) + if len(tweets) >= numToGet: + break + return tweets + +def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta(days=7)): + g = Github(CONFIG.get("github", "access_code")) + out = [] + for repo in g.get_user().get_repos(): + # print(repo.name, list(repo.get_branches())) + for commit in repo.get_commits(since = since): + out.append({ + "repo": repo.name, + "message": commit.commit.message, + "url": commit.html_url, + "datetime": commit.commit.author.date, + "stats": { + "additions": commit.stats.additions, + "deletions": commit.stats.deletions, + "total": commit.stats.total + } + }) + return sorted(out, key = lambda a: a["datetime"], reverse = True) + if __name__ == "__main__": + import datetime + start = datetime.datetime.now() with Database() as db: - print(db.get_header_articles())
\ No newline at end of file + print(db.get_cached_tweets()) + print("Took: ", (datetime.datetime.now() - start))
\ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 32044b1..3644402 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ docker PiHole-api Pillow==8.1.0 python-qbittorrent==0.4.2 +PyGithub diff --git a/services.py b/services.py index 2cfe394..4f014d8 100644 --- a/services.py +++ b/services.py @@ -1,6 +1,5 @@ from dataclasses import dataclass from io import StringIO -from lxml import html import multiprocessing import pihole as ph import qbittorrent @@ -112,30 +111,4 @@ def get_pihole_stats(): "domains": pihole.domain_count, "last_updated": str(datetime.datetime.fromtimestamp(pihole.gravity_last_updated["absolute"])) } - -# @timeout -def get_recent_tweets(numToGet): - tweets = [] - domain = "http://" + app.CONFIG.get("nitter", "domain") - with app.database.Database() as db: - for title, url in db.get_header_links(): - if title == "twitter": - break - tree = html.fromstring(requests.get(url).content) - for i, tweetUrlElement in enumerate(tree.xpath('//*[@class="tweet-link"]'), 0): - if i > 0: - tweets.append(( - domain + tweetUrlElement.get("href"), - tweetUrlElement.getparent().find_class("tweet-content media-body")[0].text - )) - if len(tweets) >= numToGet: - break - return tweets + [(url, "view all tweets...")] - - - -if __name__ == "__main__": - for tweet in get_recent_tweets(): - print(tweet.get_url()) - print(tweet.get_text()) - print()
\ No newline at end of file + diff --git a/templates/index.html b/templates/index.html index d79743b..fd0bae1 100644 --- a/templates/index.html +++ b/templates/index.html @@ -15,7 +15,7 @@ <section id="recent_tweets"> <h1>recent tweets</h1> <ul> - {% for url, text in tweets %} + {% for text, url in tweets %} {% if text == None %} <li><a href={{url}}>[image only]</a></li> {% else %} @@ -25,4 +25,12 @@ </ul> </section> {% endif %} + <section id="recent commits"> + <h1>recent git commits:</h4> + <ul> + {% for commit in commits %} + <li><a href={{commit["url"]}}>{{"[%s] %s {+%i;-%i}" % (commit["repo"], commit["message"], commit["stats"]["additions"], commit["stats"]["deletions"])}}</a></li> + {% endfor %} + </ul> + </section> {% endblock %}
\ No newline at end of file |