From 4bcba9bfeff003e081833b4d6290290271aae967 Mon Sep 17 00:00:00 2001 From: jwansek Date: Tue, 17 Jun 2025 14:28:49 +0100 Subject: Updated cron scripts --- .gitignore | 1 + config.json.example | 38 +++++++++++++++++++ cron/Dockerfile | 12 ++++++ cron/daily.py | 94 +++++++++++++++++++++++++++++++++++++++++++++++ cron/entrypoint.sh | 4 ++ cron/graph.py | 31 ++++++++++++++++ cron/hourly.py | 43 ++++++++++++++++++++++ cron/requirements.txt | 2 + docker-compose.yml | 4 +- exampleconfig.json | 37 ------------------- onceaday/Dockerfile | 10 ----- onceaday/crontab | 1 - onceaday/graph.py | 31 ---------------- onceaday/onceaday.py | 94 ----------------------------------------------- onceaday/requirements.txt | 1 - 15 files changed, 227 insertions(+), 176 deletions(-) create mode 100755 config.json.example create mode 100755 cron/Dockerfile create mode 100755 cron/daily.py create mode 100644 cron/entrypoint.sh create mode 100755 cron/graph.py create mode 100644 cron/hourly.py create mode 100755 cron/requirements.txt delete mode 100755 exampleconfig.json delete mode 100755 onceaday/Dockerfile delete mode 100755 onceaday/crontab delete mode 100755 onceaday/graph.py delete mode 100755 onceaday/onceaday.py delete mode 100755 onceaday/requirements.txt diff --git a/.gitignore b/.gitignore index 2107721..96694a3 100755 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ api.log config.json *.log *.db +*.sql* # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/config.json.example b/config.json.example new file mode 100755 index 0000000..6349b60 --- /dev/null +++ b/config.json.example @@ -0,0 +1,38 @@ +{ + "discord_webhook": "https://discord.com/api/webhooks/*********************/******************************************************", + "redditapi": + { + "client_id": "***************", + "client_secret": "************************", + "user_agent": "SmallYTChannelBot", + "username": "SmallYTChannelBot", + "password": "*******************" + }, + "imgurapi": + { + "client_id": "***************", + "client_secret": "********************************" + }, + "youtubeapi": + { + "developer_key": "******************************" + }, + "subreddit": "SmallYTChannel", + "comment_tail": "\n\n\n ^/u/SmallYTChannelBot ^*made* ^*by* ^/u/jwnskanzkwk. ^*For* ^*more* ^*information,* ^*read* ^*the* ^[FAQ.](https://www.reddit.com/user/SmallYTChannelBot/comments/a4u7qj/smallytchannelbot_faq/)", + "free_flairs": + [ + "Discussion", + "Meta", + "Collab" + ], + "mysql": + { + "host": "mysql", + "port": 3306, + "user": "root", + "passwd": "******************", + "database": "SmallYTChannel" + }, + "min_comment_len": 120, + "lambda_cost": 1 +} diff --git a/cron/Dockerfile b/cron/Dockerfile new file mode 100755 index 0000000..21aa4e4 --- /dev/null +++ b/cron/Dockerfile @@ -0,0 +1,12 @@ +FROM reg.reaweb.uk/smallytchannelbot +MAINTAINER Eden Attenborough "eddie.atten.ea29@gmail.com" +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get install -y tzdata cron mysql-client +COPY . /app +WORKDIR /app +RUN pip3 install -r cron/requirements.txt + +RUN echo "@daily root python3 /app/cron/daily.py > /proc/1/fd/1 2>/proc/1/fd/2" > /etc/crontab +RUN echo "@hourly root python3 /app/cron/hourly.py > /proc/1/fd/1 2>/proc/1/fd/2" >> /etc/crontab +ENTRYPOINT ["bash"] +CMD ["entrypoint.sh"] diff --git a/cron/daily.py b/cron/daily.py new file mode 100755 index 0000000..0f93d23 --- /dev/null +++ b/cron/daily.py @@ -0,0 +1,94 @@ +import os +import sys + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), "..")) + +from operator import itemgetter +import subreddit +import database +import datetime +import graph + +def main(): + subreddit.display("Starting every day program...") + subreddit.display("Updating database statistics...") + with database.Database() as db: + db.update_stats() + subreddit.display("Posting and updating wiki...") + update_tables(db.get_scores(), db.get_stats()) + + subreddit.display("Formatting leaderboard...") + leaderboard = format_monthly_leaderboard() + subreddit.display("Made the following leaderboard:\n======================\n%s\n======================\n" % leaderboard) + subreddit.display("Updating sidebar...") + #it'd be cool to find a way to access this directly without iteration + for widget in subreddit.SUBREDDIT.widgets.sidebar: + if widget.shortName == "Monthly Lambda Leaderboard": + widget.mod.update(text = leaderboard) + subreddit.display("Updated in new reddit...") + + sidebar = subreddit.SUBREDDIT.mod.settings()["description"] + oldtable = sidebar.split("------")[-1] + subreddit.SUBREDDIT.wiki['config/sidebar'].edit(content = sidebar.replace(oldtable, "\n\n## Monthly Lambda Leaderboard\n\n" + leaderboard)) + subreddit.display("Updated in old reddit...") + subreddit.display("Completed.") + + subreddit.logging.info("Called OAD prog @ %s" % subreddit.get_time()) + +def update_tables(scores, data): + # really ought to switch to jinja for this... + content = "" + date = str(datetime.date.today()) + mods = get_mods() + imagepath = graph.make_graph(data) + imageurl = upload_image(imagepath, date) + bylambda = [i for i in sorted(scores, key = itemgetter(1), reverse = True) if i[0] not in mods][:10] + byhelps = sorted(scores, key = itemgetter(2), reverse = True)[:10] + + content += "\n\n##/r/SmallYTChannel lambda tables: %s" % date + + content += "\n\n###By lambda:" + content += "\n\nUsername|Lambda|Help given\n:--|:--|:--" + for line in bylambda: + content += "\n/u/%s|%i|%i" % (line[0], line[1], line[2]) + + content += "\n\n###By Help given:" + content += "\n\nUsername|Lambda|Help given\n:--|:--|:--" + for line in byhelps: + λ = str(line[1]) + if line[0] in mods: + λ = "∞" + content += "\n/u/%s|%s|%i" % (line[0], λ, line[2]) + + content += "\n\n##Statistics from %s:\n\nIf you're looking at this through the wiki, not through the bot's profile, then" % (date) + content += "the most up-to-date graph will be shown below. To see the graph at this date, follow [this link.](%s)" % (imageurl) + content += "\n\n![](%%%%wikigraph%%%%)\n\nTotal λ in circulation|Useful advice given|Unique users\n:--|:--|:--\n%i|%i|%i" % (data[-1][1], data[-1][2], data[-1][3]) + + subreddit.REDDIT.subreddit("u_SmallYTChannelBot").submit("/r/SmallYTChannel Statistics: %s" % date, url = imageurl).reply(content) + + +def get_mods(): + return [str(i) for i in subreddit.SUBREDDIT.moderator()] + ["AutoModerator"] + +def format_monthly_leaderboard(): + with database.Database() as db: + leaderboard = db.get_lambda_leaderboard() + out = "**Username**|**Medal**|**Times Helped**|**Lambda**\n:-|:-|:-|:-\n" + for username, times_helped, λ in leaderboard: + out += "/u/%s|%1s|%s|%sλ\n" % (username, subreddit.get_medal(λ)[:-1], times_helped, λ) + return out + "\nLast updated: %s" % subreddit.get_time() + +def upload_image(path, date): + config = { + 'album': None, + 'name': 'SmallYTChannelBot Statistics graph: %s' % date, + 'title': 'SmallYTChannelBot Statistics graph: %s' % date, + 'description': 'SmallYTChannelBot Statistics graph: %s' % date + } + + image = subreddit.IMGUR.upload_from_path(path, config = config) + + return "https://i.imgur.com/%s.png" % image["id"] + +if __name__ == "__main__": + main() diff --git a/cron/entrypoint.sh b/cron/entrypoint.sh new file mode 100644 index 0000000..610bf84 --- /dev/null +++ b/cron/entrypoint.sh @@ -0,0 +1,4 @@ +# https://stackoverflow.com/questions/27771781/how-can-i-access-docker-set-environment-variables-from-a-cron-job/35088810#35088810 +printenv | grep -v "no_proxy" >> /etc/environment + +cron -f \ No newline at end of file diff --git a/cron/graph.py b/cron/graph.py new file mode 100755 index 0000000..723997b --- /dev/null +++ b/cron/graph.py @@ -0,0 +1,31 @@ +from mpl_toolkits.axes_grid1 import host_subplot +import mpl_toolkits.axisartist as AA +import matplotlib.pyplot as plt +import matplotlib +import datetime + +def make_graph(data): + fig = plt.figure() + + lambdaCount = [i[1] for i in data] + helpGiven = [i[2] for i in data] + uniqueUsers = [i[3] for i in data] + date = [datetime.datetime.strptime(i[4], "%Y-%m-%d") for i in data] + + fig, ax1 = plt.subplots() + ax1.plot(date, lambdaCount, label = "Total λ in circulation", color = "r") + ax1.set_ylabel("Total λ / help given") + + ax1.plot(date, helpGiven, label = "Times help given", color = "g") + + ax2 = ax1.twinx() + ax2.plot(date, uniqueUsers, label = "Unique users") + ax2.set_ylabel("No. Unique Users") + + ax1.legend() + ax2.legend(loc = 4) + fig.autofmt_xdate() + + filepath = "graph.png" + fig.savefig(filepath) + return filepath \ No newline at end of file diff --git a/cron/hourly.py b/cron/hourly.py new file mode 100644 index 0000000..daf46ec --- /dev/null +++ b/cron/hourly.py @@ -0,0 +1,43 @@ +import os +import sys + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), "..")) + +from discord_webhook import DiscordWebhook +from operator import itemgetter +import subprocess +import subreddit +import database +import datetime + +def dump(): + subprocess.run(["rm", "-fv", "/tmp/*.sql*"]) + proc1 = subprocess.Popen( + [ + "mysqldump", subreddit.CONFIG["mysql"]["database"], + "--ignore-table", "SmallYTChannel.log", "--verbose", + "-u", subreddit.CONFIG["mysql"]["user"], + "-h", subreddit.CONFIG["mysql"]["host"], + "-p%s" % subreddit.CONFIG["mysql"]["passwd"] + ], + stdout = subprocess.PIPE + ) + proc2 = subprocess.Popen("gzip > /tmp/sytc_nolog.sql.gz", shell = True, stdin = proc1.stdout, stdout = subprocess.PIPE) + output = proc2.communicate() + +def push(fp = "/tmp/sytc_nolog.sql.gz"): + webhook = DiscordWebhook( + url = subreddit.CONFIG["discord_webhook"], + content = "Hourly /u/SmallYTChannelBot database dump from %s" % datetime.datetime.now().astimezone().isoformat() + ) + + with open(fp, "rb") as f: + webhook.add_file(file = f.read(), filename = os.path.split(fp)[-1]) + + response = webhook.execute() + subreddit.display(str(response)) + +if __name__ == "__main__": + dump() + push() + diff --git a/cron/requirements.txt b/cron/requirements.txt new file mode 100755 index 0000000..465e808 --- /dev/null +++ b/cron/requirements.txt @@ -0,0 +1,2 @@ +matplotlib==3.3.4 +discord-webhook diff --git a/docker-compose.yml b/docker-compose.yml index deb7375..5af0778 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,8 +15,8 @@ services: cron: build: context: . - dockerfile: ./onceaday/Dockerfile - image: reg.reaweb.uk/smallytchannelbot_oad + dockerfile: ./cron/Dockerfile + image: reg.reaweb.uk/smallytchannelbot_cron networks: - db-network external_links: diff --git a/exampleconfig.json b/exampleconfig.json deleted file mode 100755 index 0ad4134..0000000 --- a/exampleconfig.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "redditapi": - { - "client_id": "xxxxxxxxxxxxxxx", - "client_secret": "xxxxxxxxxxxxxxxxxxxxxxxx", - "user_agent": "SmallYTChannelBot", - "username": "SmallYTChannelBot", - "password": "xxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - "imgurapi": - { - "client_id": "xxxxxxxxxxxxxxxx", - "client_secret": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - "youtubeapi": - { - "developer_key": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - "subreddit": "SmallYTChannel", - "comment_tail": "\n\n\n ^/u/SmallYTChannelBot ^*made* ^*by* ^/u/jwnskanzkwk. ^*PM* ^*for* ^*bug* ^*reports.* ^*For* ^*more* ^*information,* ^*read* ^*the* ^[FAQ.](https://www.reddit.com/user/SmallYTChannelBot/comments/a4u7qj/smallytchannelbot_faq/)", - "free_flairs": - [ - "Discussion", - "Meta", - "Collab" - ], - "mysql": - { - "host": "localhost", - "port": 3306, - "user": "root", - "passwd": "xxxxxxxxxxxxxxxx", - "database": "SmallYTChannel" - }, - "min_comment_len": 120, - "lambda_cost": 2 -} diff --git a/onceaday/Dockerfile b/onceaday/Dockerfile deleted file mode 100755 index 6f733db..0000000 --- a/onceaday/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM reg.reaweb.uk/smallytchannelbot -MAINTAINER Eden Attenborough "eddie.atten.ea29@gmail.com" -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get install -y tzdata cron -COPY . /app -WORKDIR /app -COPY onceaday/crontab /etc/cron.d/oad-crontab -RUN chmod 0644 /etc/cron.d/oad-crontab && crontab /etc/cron.d/oad-crontab -RUN pip3 install -r onceaday/requirements.txt -ENTRYPOINT ["cron", "-f"] diff --git a/onceaday/crontab b/onceaday/crontab deleted file mode 100755 index c8ce32a..0000000 --- a/onceaday/crontab +++ /dev/null @@ -1 +0,0 @@ -@daily python3 /app/onceaday/onceaday.py /app/onceaday diff --git a/onceaday/graph.py b/onceaday/graph.py deleted file mode 100755 index 723997b..0000000 --- a/onceaday/graph.py +++ /dev/null @@ -1,31 +0,0 @@ -from mpl_toolkits.axes_grid1 import host_subplot -import mpl_toolkits.axisartist as AA -import matplotlib.pyplot as plt -import matplotlib -import datetime - -def make_graph(data): - fig = plt.figure() - - lambdaCount = [i[1] for i in data] - helpGiven = [i[2] for i in data] - uniqueUsers = [i[3] for i in data] - date = [datetime.datetime.strptime(i[4], "%Y-%m-%d") for i in data] - - fig, ax1 = plt.subplots() - ax1.plot(date, lambdaCount, label = "Total λ in circulation", color = "r") - ax1.set_ylabel("Total λ / help given") - - ax1.plot(date, helpGiven, label = "Times help given", color = "g") - - ax2 = ax1.twinx() - ax2.plot(date, uniqueUsers, label = "Unique users") - ax2.set_ylabel("No. Unique Users") - - ax1.legend() - ax2.legend(loc = 4) - fig.autofmt_xdate() - - filepath = "graph.png" - fig.savefig(filepath) - return filepath \ No newline at end of file diff --git a/onceaday/onceaday.py b/onceaday/onceaday.py deleted file mode 100755 index 7762d05..0000000 --- a/onceaday/onceaday.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -import sys - -os.chdir(sys.argv[1]) -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - -from operator import itemgetter -import subreddit -import database -import datetime -import graph - -def main(): - subreddit.display("Starting every day program...") - subreddit.display("Updating database statistics...") - with database.Database() as db: - db.update_stats() - subreddit.display("Posting and updating wiki...") - update_tables(db.get_scores(), db.get_stats()) - - subreddit.display("Formatting leaderboard...") - leaderboard = format_monthly_leaderboard() - subreddit.display("Made the following leaderboard:\n======================\n%s\n======================\n" % leaderboard) - subreddit.display("Updating sidebar...") - #it'd be cool to find a way to access this directly without iteration - for widget in subreddit.SUBREDDIT.widgets.sidebar: - if widget.shortName == "Monthly Lambda Leaderboard": - widget.mod.update(text = leaderboard) - subreddit.display("Updated in new reddit...") - - sidebar = subreddit.SUBREDDIT.mod.settings()["description"] - oldtable = sidebar.split("------")[-1] - subreddit.SUBREDDIT.wiki['config/sidebar'].edit(content = sidebar.replace(oldtable, "\n\n## Monthly Lambda Leaderboard\n\n" + leaderboard)) - subreddit.display("Updated in old reddit...") - subreddit.display("Completed.") - - subreddit.logging.info("Called OAD prog @ %s" % subreddit.get_time()) - -def update_tables(scores, data): - content = "" - date = str(datetime.date.today()) - mods = get_mods() - imagepath = graph.make_graph(data) - imageurl = upload_image(imagepath, date) - bylambda = [i for i in sorted(scores, key = itemgetter(1), reverse = True) if i[0] not in mods][:10] - byhelps = sorted(scores, key = itemgetter(2), reverse = True)[:10] - - content += "\n\n##/r/SmallYTChannel lambda tables: %s" % date - - content += "\n\n###By lambda:" - content += "\n\nUsername|Lambda|Help given\n:--|:--|:--" - for line in bylambda: - content += "\n/u/%s|%i|%i" % (line[0], line[1], line[2]) - - content += "\n\n###By Help given:" - content += "\n\nUsername|Lambda|Help given\n:--|:--|:--" - for line in byhelps: - λ = str(line[1]) - if line[0] in mods: - λ = "∞" - content += "\n/u/%s|%s|%i" % (line[0], λ, line[2]) - - content += "\n\n##Statistics from %s:\n\nIf you're looking at this through the wiki, not through the bot's profile, then" % (date) - content += "the most up-to-date graph will be shown below. To see the graph at this date, follow [this link.](%s)" % (imageurl) - content += "\n\n![](%%%%wikigraph%%%%)\n\nTotal λ in circulation|Useful advice given|Unique users\n:--|:--|:--\n%i|%i|%i" % (data[-1][1], data[-1][2], data[-1][3]) - - subreddit.REDDIT.subreddit("u_SmallYTChannelBot").submit("/r/SmallYTChannel Statistics: %s" % date, url = imageurl).reply(content) - - -def get_mods(): - return [str(i) for i in subreddit.SUBREDDIT.moderator()] + ["AutoModerator"] - -def format_monthly_leaderboard(): - with database.Database() as db: - leaderboard = db.get_lambda_leaderboard() - out = "**Username**|**Medal**|**Times Helped**|**Lambda**\n:-|:-|:-|:-\n" - for username, times_helped, λ in leaderboard: - out += "/u/%s|%1s|%s|%sλ\n" % (username, subreddit.get_medal(λ)[:-1], times_helped, λ) - return out + "\nLast updated: %s" % subreddit.get_time() - -def upload_image(path, date): - config = { - 'album': None, - 'name': 'SmallYTChannelBot Statistics graph: %s' % date, - 'title': 'SmallYTChannelBot Statistics graph: %s' % date, - 'description': 'SmallYTChannelBot Statistics graph: %s' % date - } - - image = subreddit.IMGUR.upload_from_path(path, config = config) - - return "https://i.imgur.com/%s.png" % image["id"] - -if __name__ == "__main__": - main() diff --git a/onceaday/requirements.txt b/onceaday/requirements.txt deleted file mode 100755 index e3a623c..0000000 --- a/onceaday/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -matplotlib==3.3.4 -- cgit v1.2.3