A lot of fixes and refactoring

7 files changed, 208 insertions, 130 deletions

diff --git a/.dockerignore b/.dockerignore
index 26ceb08..2107721 100644..100755
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,2 +1,115 @@
+logs/
+login.py
+graph.png
+pid.txt
+ch.py
+chblacklist.txt
+api.log
 config.json
+*.log
+*.db
 
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.vs/
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
diff --git a/api_plotter.py b/api_plotter.py
deleted file mode 100755
index ba197e2..0000000
--- a/api_plotter.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import matplotlib.pyplot as plt
-import datetime
-import math
-import os
-import re
-
-def round_to_min(dt: datetime.datetime):
-    return datetime.datetime(
-        year = dt.year,
-        month = dt.month,
-        day = dt.day,
-        hour = dt.hour,
-        minute = math.floor(dt.minute)
-    )
-
-with open(os.path.join("logs", "api.log"), "r") as f:
-    s = f.read().split("\n")
-
-timestamps = set()
-
-for l in s:
-    x = re.search(r"^.*\tResponse: 200", l)
-    if x is not None:
-        timestamps.add(datetime.datetime.strptime(x.group()[1:24], "%Y-%m-%d %H:%M:%S,%f"))
-
-d = {}
-for timestamp in timestamps:
-    nearest = round_to_min(timestamp)
-
-    try:
-        d[nearest] += 1
-    except KeyError:
-        d[nearest] = 1
-
-d_sorted = {k: v for k, v in sorted(d.items(), key=lambda x: x[0])}
-
-fig, ax = plt.subplots()
-ax.plot(list(d_sorted.keys()), list(d_sorted.values()))
-
-plt.show()
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index e9cdf67..deb7375 100755
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ services:
         build:
             context: .
             dockerfile: Dockerfile
-        image: smallytchannelbot
+        image: reg.reaweb.uk/smallytchannelbot
         networks:
             - db-network
         external_links:
@@ -16,7 +16,7 @@ services:
         build:
             context: .
             dockerfile: ./onceaday/Dockerfile
-        image: smallytchannelbot_oad
+        image: reg.reaweb.uk/smallytchannelbot_oad
         networks:
             - db-network
         external_links:
diff --git a/onceaday/Dockerfile b/onceaday/Dockerfile
index 626fe53..6f733db 100755
--- a/onceaday/Dockerfile
+++ b/onceaday/Dockerfile
@@ -1,4 +1,4 @@
-FROM smallytchannelbot
+FROM reg.reaweb.uk/smallytchannelbot
 MAINTAINER Eden Attenborough "eddie.atten.ea29@gmail.com"
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get install -y tzdata cron
diff --git a/requirements.txt b/requirements.txt
index 7f1d56a..958ab64 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,3 +5,4 @@ js2py
 praw
 sqlite3-to-mysql
 pymysql
+jinja2
diff --git a/subreddit.py b/subreddit.py
index 510771b..8e00915 100755
--- a/subreddit.py
+++ b/subreddit.py
@@ -1,8 +1,13 @@
 from imgurpython import ImgurClient
 from operator import itemgetter
+
+import praw.models
+import praw.models.reddit
+import praw.models.reddit.comment
 import database
 import datetime
 import logging
+import jinja2
 import ytapi
 # import graph
 import time
@@ -28,7 +33,7 @@ FREE_FLAIRS = CONFIG["free_flairs"]
 IMGUR = ImgurClient(**CONFIG["imgurapi"])
 
 logging.basicConfig( 
-    format = "%(process)s\t[%(asctime)s]\t%(message)s", 
+    format = "%(message)s", 
     level = logging.INFO,
     handlers=[
         logging.FileHandler("actions.log"),
@@ -62,9 +67,9 @@ def get_lambda_from_flair(s):
     else:
         return ""
 
-def update_users_flair_from_comment(comment, reddit):
+def update_users_flair_from_comment(comment):
     #implemented only for legacy
-    update_users_flair(str(comment.author), reddit)
+    update_users_flair(str(comment.author))
 
 def get_medal(actualscore):
     if actualscore >= 10 and actualscore < 25:
@@ -78,14 +83,14 @@ def get_medal(actualscore):
     else:
         return ""
 
-def update_users_flair(username, reddit):
-    flairtext = next(reddit.subreddit(CONFIG["subreddit"]).flair(redditor=username))["flair_text"]
+def update_users_flair(username):
+    flairtext = next(REDDIT.subreddit(CONFIG["subreddit"]).flair(redditor=username))["flair_text"]
     if flairtext is None:
         flairtext = ""
     else:
         flairscore = get_lambda_from_flair(flairtext)
         flairtext = str(flairtext.replace("[%s] " % flairscore, ""))
-    if username in get_mods(reddit):
+    if username in get_mods():
         newflair = "[🏆 ∞λ] %s" % (flairtext)
     else:
         with database.Database() as db:
@@ -93,16 +98,16 @@ def update_users_flair(username, reddit):
         newflair = "[%s%iλ] %s" % (get_medal(actualscore), actualscore, flairtext)
 
     logging.info("/u/%s had their flair updated" % username)
-    reddit.subreddit(CONFIG["subreddit"]).flair.set(redditor = username, text = newflair)
+    REDDIT.subreddit(CONFIG["subreddit"]).flair.set(redditor = username, text = newflair)
 
-def get_mods(reddit):
-    return [str(i) for i in reddit.subreddit(CONFIG["subreddit"]).moderator()] + ["AutoModerator"]
+def get_mods():
+    return [str(i) for i in REDDIT.subreddit(CONFIG["subreddit"]).moderator()] + ["AutoModerator"]
 
-def handle_mylambda(comment, reddit):
+def handle_mylambda(comment: praw.models.reddit.comment):
     author = str(comment.author)
     with database.Database() as db:
         λ, links = db.get_lambda(author)
-    if author in get_mods(reddit):
+    if author in get_mods():
         text = "/u/%s is a moderator, and therefore has ∞λ." % author
     else:
         text = "/u/%s currently has %iλ, and has helped helping the following posts:" % (author, λ)
@@ -119,10 +124,10 @@ def handle_mylambda(comment, reddit):
                 text += "\n\n[%i more...]" % len(links) - count
                 break
 
-    update_users_flair_from_comment(comment, reddit)
+    update_users_flair_from_comment(comment)
     return text
 
-def handle_givelambda(comment, reddit):
+def handle_givelambda(comment: praw.models.reddit.comment):
     submission = comment.submission
     parentauthour = str(comment.parent().author)
     op = str(comment.author)
@@ -131,7 +136,7 @@ def handle_givelambda(comment, reddit):
             text = "You cannot give yourself λ."
         elif parentauthour == "SmallYTChannelBot":
             text = "Please only give lambda to humans."
-        elif str(comment.author) in get_mods(reddit):
+        elif str(comment.author) in get_mods():
             text = "The moderator /u/%s has given /u/%s 1λ. /u/%s now has %iλ." % (str(comment.author), parentauthour, parentauthour, db.get_lambda(parentauthour)[0] + 1)
             db.give_lambda(parentauthour, submission.permalink, timestamp = int(submission.created_utc)) 
             display(text, concerning=comment.permalink)
@@ -158,10 +163,10 @@ def handle_givelambda(comment, reddit):
             db.give_lambda(parentauthour, submission.permalink, timestamp = int(submission.created_utc))
     
     # update_users_flair_from_comment(comment)
-    update_users_flair_from_comment(comment.parent(), reddit)
+    update_users_flair_from_comment(comment.parent())
     return text
 
-def handle_setlambda(comment, reddit):
+def handle_setlambda(comment: praw.models.reddit.comment):
     try:
         splitted = comment.body.split()
         user = splitted[1].replace("/u/", "").replace("u/", "")
@@ -170,17 +175,18 @@ def handle_setlambda(comment, reddit):
         with database.Database() as db:
             text = "/u/%s had their lambda set to %iλ by a moderator." % (user, toset)
             db.set_lambda(user, toset)
-        display("A moderator set /u/%s λ to %d" % (user, toset), concerning=comment.permalink)
+        display("A moderator set /u/%s λ to %d" % (user, toset), concerning = comment.permalink)
     except Exception as e:
-        display("{ERROR while setting λ} %s" % e, concerning=comment.permalink)
+        display("{ERROR while setting λ} %s" % e, concerning = comment.permalink)
         text = r"An error was encountered. Please use the syntax `!setlambda <user> <set to {integer}>`" + "\n\nThe error was:\n\n" + str(e)
 
-    update_users_flair(user, reddit)
+    update_users_flair(user)
     return text
 
-def handle_submission(submission, reddit):
+def handle_submission(submission: praw.models.reddit.submission):
     with database.Database() as db:
         score = db.get_lambda(str(submission.author))[0]
+
     if submission.link_flair_text in FREE_FLAIRS:
         if "youtube.com" in str(submission.url) or "youtu.be" in str(submission.url):
             text = "Your post has been removed because it has the wrong flair. [Discussion], [Meta] and [Collab] flairs are only for text submissions."
@@ -201,76 +207,54 @@ def handle_submission(submission, reddit):
             with database.Database() as db:
                 db.change_lambda(str(submission.author), -CONFIG["lambda_cost"])
 
-            try:
-                ytid = ytapi.get_videoId_from_url(submission.url)
-                if "/" not in ytid:
+            ytid = ytapi.get_videoId_from_url(submission.url)
+            if "/" not in ytid: # why is this necessary?
+                try:
                     ytdata = ytapi.get_video_data(ytid)
-
-                    text += """
-\n\n\n##Video data:
-
-**Field**|**Data**
-:-|:-
-Title|%s
-Thumbnail|[Link](%s)
-Views|%s
-Length|%s
-Likes|%s
-Comments|%s
-Description|%s
-
-##Channel Data:
-
-**Field**|**Data**
-:-|:-
-Name|%s
-Thumbnail|[Link](%s)
-Subscribers|%s
-Videos|%s
-Views|%s
-                    """ % (
-                        ytdata["title"],
-                        ytdata["thumbnail"],
-                        ytdata["views"],
-                        ytdata["length"],
-                        ytdata["likes"],
-                        ytdata["comments"],
-                        ytdata["description"],
-                        ytdata["channel"],
-                        ytdata["channelThumb"],
-                        ytdata["subscribers"],
-                        ytdata["videos"],
-                        ytdata["channelViews"]
-                    )
-
-                    curflair = submission.link_flair_text
-                    if str(curflair) != "None":
-                        submission.mod.flair(" %s | %s | :youtube: %s" % (curflair, ytdata["length"], ytdata["channel"]))
-                    else:    
-                        submission.mod.flair("%s | :youtube: %s" % (ytdata["length"], ytdata["channel"]))
-            except:
-                pass
-
-    update_users_flair(str(submission.author), reddit)
-    return text
-
-def handle_comment(comment, reddit):
+                except Exception as e:
+                    ytdata = ytapi.ERROR_DICT
+                    display("{ERROR WITH YOUTUBE} %s" % e, concerning = submission.permalink)
+
+                jinja_env = jinja2.Environment(loader = jinja2.FileSystemLoader(os.path.dirname(__file__)))
+                template = jinja_env.get_template("youtubeInfoComment.j2")
+                text += template.render(ytdata)
+
+                curflair = submission.link_flair_text
+                if str(curflair) != "None":
+                    submission.mod.flair(text = " %s | %s | :youtube: %s" % (curflair, ytdata["length"], ytdata["channel"]))
+                else:    
+                    submission.mod.flair(text = "%s | :youtube: %s" % (ytdata["length"], ytdata["channel"]))
+            else:
+                display("{ERROR} Didn't pin comment due to a missing slash", concerning = submission.permalink)
+
+            # Sticking to the bottom means making it the 'second' sticked post, the first is reserved for moderator stickies.
+            # It has the side effect of appearing in the 'Community highlights' section (which is what we want)
+            # That feature seems poorely documented, not sure how many items can appear in there? Hopefully they automatically
+            # slide off in a queue according to age
+            submission.mod.sticky(bottom = True)
+
+    update_users_flair(str(submission.author))
+    reply = submission.reply(text + COMMENT_TAIL)
+    reply.mod.distinguish(sticky = True)
+    reply.mod.approve()
+
+def handle_comment(comment):
     response = None
     if "!mylambda" in comment.body.lower() and str(comment.author) != "SmallYTChannelBot":
-        response = handle_mylambda(comment, reddit)
+        response = handle_mylambda(comment)
 
     if "!givelambda" in comment.body.lower() and str(comment.author) != "SmallYTChannelBot":
-        response = handle_givelambda(comment, reddit)        
+        response = handle_givelambda(comment)        
 
-    if comment.body.startswith("!setlambda") and str(comment.author) in get_mods(reddit):
-        response = handle_setlambda(comment, reddit)
+    if comment.body.startswith("!setlambda") and str(comment.author) in get_mods():
+        response = handle_setlambda(comment)
 
     if response is not None:
         reply = comment.reply(response + COMMENT_TAIL)
         reply.mod.distinguish(sticky = False)
 
-def stream(reddit):
-    subreddit = reddit.subreddit(CONFIG["subreddit"])
+def stream():
+    subreddit = REDDIT.subreddit(CONFIG["subreddit"])
     streams = [subreddit.stream.comments(pause_after=-1), subreddit.stream.submissions(pause_after=-1)]
     with database.Database() as db:
         while True:
@@ -284,22 +268,18 @@ def stream(reddit):
 
                     db.add_to_blacklist(item.id)
                     if str(type(item)) == "<class 'praw.models.reddit.comment.Comment'>":
-                        handle_comment(item, reddit)
+                        handle_comment(item)
 
                     elif str(type(item)) == "<class 'praw.models.reddit.submission.Submission'>":
                         display("There has been a new submission: '%s', with flair '%s'" % (item.title, item.link_flair_text), concerning=item.permalink)
 
-                        if str(item.author) not in get_mods(reddit):
-                            reply = item.reply(handle_submission(item, reddit) + COMMENT_TAIL)
-                            reply.mod.distinguish(sticky = True)
-                            reply.mod.approve()
+                        if str(item.author) not in get_mods():
+                            handle_submission(item)
             time.sleep(30)
 
 def main():
-    reddit = praw.Reddit(**CONFIG["redditapi"])
-    reddit.validate_on_submit = True
     try:
-        stream(reddit)
+        stream()
     except Exception as e:
         display("{ERROR} %s" % str(e))
         time.sleep(60)
@@ -309,3 +289,4 @@ def main():
 if __name__ == "__main__":
     main()
 
+
diff --git a/youtubeInfoComment.j2 b/youtubeInfoComment.j2
new file mode 100644
index 0000000..34d1647
--- /dev/null
+++ b/youtubeInfoComment.j2
@@ -0,0 +1,23 @@
+
+
+##Video data:
+
+**Field**|**Data**
+:-|:-
+Title|{{ title }}
+Thumbnail|[Link]({{ thumbnail }})
+Views|{{ views }}
+Length|{{ length }}
+Likes|{{ likes }}
+Comments|{{ comments }}
+Description|{{ description }}
+
+##Channel Data:
+
+**Field**|**Data**
+:-|:-
+Name|{{ channel }}
+Thumbnail|[Link]({{ channelThumb }})
+Subscribers|{{ subscribers }}
+Videos|{{ videos }}
+Views|{{ channelViews }}
\ No newline at end of file