diff options
| -rwxr-xr-x | database.py | 83 | ||||
| -rwxr-xr-x | edaweb.conf.example | 6 | ||||
| -rwxr-xr-x | requirements.txt | 1 |
3 files changed, 87 insertions, 3 deletions
diff --git a/database.py b/database.py index ee3e3f0..52cfe39 100755 --- a/database.py +++ b/database.py @@ -6,9 +6,11 @@ import configparser import threading import datetime import requests +import twython import pymysql import random import os +import re @dataclass class Database: @@ -202,9 +204,85 @@ class Database: self.__connection.commit() return id_ + def append_diary(self, tweet_id, tweeted_at, replying_to, tweet): + with self.__connection.cursor() as cursor: + cursor.execute("INSERT INTO diary VALUES (%s, %s, %s, %s);", (tweet_id, tweeted_at, replying_to, tweet)) + print("Appended diary with tweet '%s'" % tweet) + + def append_diary_images(self, tweet_id, imurl): + with self.__connection.cursor() as cursor: + cursor.execute("INSERT INTO diaryimages (tweet_id, link) VALUES (%s, %s);", (tweet_id, imurl)) + + def get_diary(self, twitteracc = "FUCKEDUPTRANNY"): + threading.Thread(target = update_cache).start() + out = {} + with self.__connection.cursor() as cursor: + cursor.execute("SELECT tweet_id, tweeted_at, tweet FROM diary WHERE replying_to IS NULL ORDER BY tweeted_at DESC;") + for tweet_id, tweeted_at, tweet_text in cursor.fetchall(): + # print(tweet_id, tweeted_at, tweet_text) + out[tweeted_at] = [{ + "text": tweet_text, + "images": self.get_diary_image(tweet_id), + "link": "https://%s/%s/status/%d" % (self.config.get("nitter", "domain"), twitteracc, tweet_id) + }] + + next_tweet = self.get_child_tweets(tweet_id) + while next_tweet is not None: + tweet, id_ = next_tweet + out[tweeted_at].append(tweet) + next_tweet = self.get_child_tweets(id_) + + return out + + def get_diary_image(self, tweet_id): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT link FROM diaryimages WHERE tweet_id = %s;", (tweet_id, )) + return [i[0] for i in cursor.fetchall()] + + def get_child_tweets(self, parent_id, twitteracc = "FUCKEDUPTRANNY"): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT tweet_id, tweet FROM diary WHERE replying_to = %s;", (parent_id, )) + out = cursor.fetchall() + if out == (): + return None + + out = out[0] + id_ = out[0] + return {"text": out[1], "images": self.get_diary_image(id_), "link": "https://%s/%s/status/%d" % (self.config.get("nitter", "domain"), twitteracc, id_)}, id_ + + def get_newest_diary_tweet_id(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT MAX(tweet_id) FROM diary;") + return cursor.fetchone()[0] + + def fetch_diary(self, twitteracc = "FUCKEDUPTRANNY"): + twitter = twython.Twython(*dict(dict(self.config)["twitter"]).values()) + for tweet in twitter.search(q = "(from:%s)" % twitteracc, since_id = self.get_newest_diary_tweet_id())["statuses"]: + tweet_id = tweet["id"] + tweeted_at = datetime.datetime.strptime(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y") + replying_to = tweet["in_reply_to_status_id"] + tweet_text = re.sub(r"https://t\.co/\w{10}", "", tweet["text"], 0, re.MULTILINE) + + if tweet["in_reply_to_screen_name"] == twitteracc or tweet["in_reply_to_screen_name"] is None: + self.append_diary(tweet_id, tweeted_at, replying_to, tweet_text) + + if "media" in tweet["entities"].keys(): + associated_images = [ + i["media_url_https"].replace("pbs.twimg.com", self.config.get("nitter", "domain") + "/pic") + for i in tweet["entities"]["media"] + ] + for im in associated_images: + self.append_diary_images(tweet_id, im) + + self.__connection.commit() + print("Fetch completed.") + + + def update_cache(): # print("updating cache...") with Database() as db: + db.fetch_diary() db.update_twitter_cache(request_recent_tweets(10000)) # print("Done updating twitter cache...") db.update_commit_cache(request_recent_commits(since = db.get_last_commit_time())) @@ -250,8 +328,7 @@ def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta( }) return sorted(out, key = lambda a: a["datetime"], reverse = True) + if __name__ == "__main__": - import parser with Database() as db: - # print(db.get_similar_thoughts("about me", 5)) - print(db.get_iso_cd_options()) + print(db.get_diary()) diff --git a/edaweb.conf.example b/edaweb.conf.example index 074a3f3..6f5b3e3 100755 --- a/edaweb.conf.example +++ b/edaweb.conf.example @@ -35,3 +35,9 @@ access_code = ******************************* [cds] location = /media/ISOs/ maxsize = 700000000 + +[twitter] +consumer_key = *************** +consumer_secret = ********************************** +access_token = ***************************************** +access_token_secret = ***************************************** diff --git a/requirements.txt b/requirements.txt index b09cc1c..7aee926 100755 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ waitress houdini.py Pygments mistune +twython==3.8.2 |