added caching tweets, added git commits

1 files changed, 117 insertions, 1 deletions

diff --git a/database.py b/database.py
index 5e50f82..0edcad7 100644
--- a/database.py
+++ b/database.py
@@ -1,7 +1,14 @@
+from urllib.parse import urlparse
 from dataclasses import dataclass
+from github import Github
+from lxml import html
 import configparser
+import threading
+import datetime
+import requests
 import pymysql
 import random
+import os
 
 @dataclass
 class Database:
@@ -107,6 +114,115 @@ class Database:
             """)
             return cursor.fetchall()
 
+    def get_cached_tweets(self, numToGet = None, recurse = True):
+        with self.__connection.cursor() as cursor:
+            if numToGet is not None:
+                cursor.execute("SELECT text, url FROM twitterCache ORDER BY appended DESC LIMIT %s;", (numToGet, ))
+            else:
+                cursor.execute("SELECT text, url FROM twitterCache ORDER BY appended DESC;")
+            if recurse:
+                threading.Thread(target = update_cache).start()
+            return list(cursor.fetchall())
+
+    def update_twitter_cache(self, requested):
+        urls = [i[1] for i in self.get_cached_tweets(recurse = False)]
+        with self.__connection.cursor() as cursor:
+            for url, text in requested:
+                if url not in urls:
+                    cursor.execute("INSERT INTO twitterCache (text, url) VALUES (%s, %s);", (text, url))
+        self.__connection.commit()
+
+    def get_cached_commits(self, since = None, recurse = True):
+        with self.__connection.cursor() as cursor:
+            if since is not None:
+                cursor.execute("SELECT message, url, commitTime, additions, deletions, total FROM commitCache WHERE commitTime > %s ORDER BY commitTime DESC;", (since, ))
+            else:
+                cursor.execute("SELECT message, url, commitTime, additions, deletions, total FROM commitCache ORDER BY commitTime DESC;")
+            return [{
+                "repo": urlparse(i[1]).path.split("/")[2],
+                "message": i[0],
+                "url": i[1],
+                "datetime": i[2],
+                "stats": {
+                    "additions": i[3],
+                    "deletions": i[4],
+                    "total": i[5]
+                }
+            } for i in cursor.fetchall()]
+
+    def update_commit_cache(self, requested):
+        urls = [i["url"] for i in self.get_cached_commits(recurse = False)]
+        with self.__connection.cursor() as cursor:
+            for commit in requested:
+                if commit["url"] not in urls:
+                    cursor.execute("""
+                    INSERT INTO commitCache (message, url, commitTime, additions, deletions, total)
+                    VALUES (%s, %s, %s, %s, %s, %s)""",
+                    (commit["message"], commit["url"], commit["datetime"], commit["stats"]["additions"], commit["stats"]["deletions"], commit["stats"]["total"])
+                )
+        self.__connection.commit()
+
+    def get_last_commit_time(self):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT MAX(commitTime) FROM commitCache;")
+            return cursor.fetchone()[0]
+
+    def get_my_twitter(self):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT link FROM headerLinks WHERE name = 'twitter';")
+            return cursor.fetchone()[0]
+
+def update_cache():
+    # print("updating cache...")
+    with Database() as db:
+        db.update_twitter_cache(request_recent_tweets(10000))
+        # print("Done updating twitter cache...")
+        db.update_commit_cache(request_recent_commits(since = db.get_last_commit_time()))
+        # print("Done updating commit cache...")
+
+CONFIG = configparser.ConfigParser()
+CONFIG.read("edaweb.conf")
+
+def request_recent_tweets(numToGet):
+    tweets = []
+    domain = "http://" + CONFIG.get("nitter", "domain")
+    with Database() as db:
+        for title, url in db.get_header_links():
+            if title == "twitter":
+                break
+    tree = html.fromstring(requests.get(url).content)
+    for i, tweetUrlElement in enumerate(tree.xpath('//*[@class="tweet-link"]'), 0):
+        if i > 0:
+            tweets.append((
+                domain + tweetUrlElement.get("href"),
+                tweetUrlElement.getparent().find_class("tweet-content media-body")[0].text
+            ))
+        if len(tweets) >= numToGet:
+            break
+    return tweets
+            
+def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta(days=7)):
+    g = Github(CONFIG.get("github", "access_code"))
+    out = []
+    for repo in g.get_user().get_repos():
+        # print(repo.name, list(repo.get_branches()))
+        for commit in repo.get_commits(since = since):
+            out.append({
+                "repo": repo.name,
+                "message": commit.commit.message,
+                "url": commit.html_url,
+                "datetime": commit.commit.author.date,
+                "stats": {
+                    "additions": commit.stats.additions,
+                    "deletions": commit.stats.deletions,
+                    "total": commit.stats.total
+                }
+            })
+    return sorted(out, key = lambda a: a["datetime"], reverse = True) 
+
 if __name__ == "__main__":
+    import datetime
+    start = datetime.datetime.now()
     with Database() as db:
-        print(db.get_header_articles())
\ No newline at end of file
+        print(db.get_cached_tweets())
+        print("Took: ", (datetime.datetime.now() - start))
\ No newline at end of file