diff options
| -rwxr-xr-x | .gitignore | 6 | ||||
| -rwxr-xr-x | Dockerfile | 6 | ||||
| -rwxr-xr-x | Dockerfile_cron | 11 | ||||
| -rw-r--r-- | cron_entrypoint.sh | 6 | ||||
| -rwxr-xr-x | docker-compose.yml | 26 | ||||
| -rw-r--r-- | edaweb/app.py | 4 | ||||
| -rw-r--r-- | edaweb/services.py | 12 | ||||
| -rw-r--r-- | edaweb/static/robots.txt | 4 | ||||
| -rw-r--r-- | entrypoint.sh | 6 | ||||
| -rwxr-xr-x | scripts/export.sh | 2 | ||||
| -rwxr-xr-x | scripts/update.sh | 2 |
11 files changed, 64 insertions, 21 deletions
@@ -2,10 +2,10 @@ *.pub homelab-wiki/wiki.env homelab-wiki/images/* -edaweb.conf +*edaweb.conf markdowns/ -static/images/random.jpg -static/zips/*.zip +*random.jpg +*.zip .nfs* static/images/Thumbs.db nitter/nitter.conf @@ -1,14 +1,12 @@ -FROM ubuntu:22.10 +FROM ubuntu:jammy-20250404 MAINTAINER Eden Attenborough "eden.attenborough@outlook.com" ENV TZ=Europe/London RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get update -y -RUN apt-get install -y python3-pip python3-dev build-essential clang libffi-dev libxml2-dev libxslt-dev libjpeg-dev zlib1g-dev tmux cron +RUN apt-get install -y python3-pip python3-dev build-essential clang libffi-dev libxml2-dev libxslt-dev libjpeg-dev zlib1g-dev COPY . /app WORKDIR /app RUN pip3 install -r requirements.txt -RUN echo "*/30 * * * * root python3 /app/edaweb/cache.py > /proc/1/fd/1 2>/proc/1/fd/2" > /etc/crontab - ENTRYPOINT ["bash"] CMD ["entrypoint.sh"] diff --git a/Dockerfile_cron b/Dockerfile_cron new file mode 100755 index 0000000..dea3208 --- /dev/null +++ b/Dockerfile_cron @@ -0,0 +1,11 @@ +FROM reg.reaweb.uk/edaweb +MAINTAINER Eden Attenborough "eden.attenborough@outlook.com" +ENV TZ=Europe/London +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN apt-get update -y +RUN apt-get install -y tmux cron + +RUN echo "*/30 * * * * root python3 /app/edaweb/cache.py > /proc/1/fd/1 2>/proc/1/fd/2" > /etc/crontab + +ENTRYPOINT ["bash"] +CMD ["cron_entrypoint.sh"] diff --git a/cron_entrypoint.sh b/cron_entrypoint.sh new file mode 100644 index 0000000..2d98169 --- /dev/null +++ b/cron_entrypoint.sh @@ -0,0 +1,6 @@ +rm -rvf /app/edaweb/edaweb.conf/ +ln -s /app/edaweb.conf /app/edaweb/edaweb.conf +printenv | grep -v "no_proxy" >> /etc/environment +# tmux new-session -d -s "cron" 'cron -f || bash && bash'; +cron -f + diff --git a/docker-compose.yml b/docker-compose.yml index 3d5c58e..57cd71e 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: edaweb: build: @@ -7,9 +6,9 @@ services: image: reg.reaweb.uk/edaweb volumes: - /tmp/:/media/ISOs/ - - ./static/:/app/static/ + - ./edaweb/static/:/app/edaweb/static/ - ./edaweb.conf:/app/edaweb.conf - - ./edaweb-docker.pem:/keys/docker-key.pem + - edaweb-tmp:/tmp/ ports: - "6969:6969" networks: @@ -19,6 +18,24 @@ services: - mariadb:mysql restart: unless-stopped + edaweb_cron: + build: + context: . + dockerfile: Dockerfile_cron + image: reg.reaweb.uk/edaweb_cron + volumes: + - /tmp/:/media/ISOs/ + - ./edaweb/static/:/app/edaweb/static/ + - ./edaweb.conf:/app/edaweb.conf + - ./edaweb-docker.pem:/keys/docker-key.pem + - edaweb-tmp:/tmp/ + networks: + - db-network + - edaweb-net + external_links: + - mariadb:mysql + restart: unless-stopped + homelab-wiki: build: context: ./homelab-wiki @@ -38,6 +55,9 @@ services: - mariadb:mysql restart: unless-stopped +volumes: + edaweb-tmp: + networks: edaweb-net: external: diff --git a/edaweb/app.py b/edaweb/app.py index 6902fe4..436df7a 100644 --- a/edaweb/app.py +++ b/edaweb/app.py @@ -17,7 +17,7 @@ import io app = flask.Flask(__name__) CONFIG = configparser.ConfigParser(interpolation = None) -CONFIG.read("edaweb.conf") +CONFIG.read(os.path.join(os.path.dirname(__file__), "edaweb.conf")) shown_images = set() shown_sidebar_images = set() @@ -224,7 +224,7 @@ def serve_random(): sbi = services.get_random_image(tags) req = urllib.request.Request(sbi.imurl) mediaContent = urllib.request.urlopen(req).read() - with open(os.path.join("static", "images", "random.jpg"), "wb") as f: + with open(os.path.join(os.path.dirname(__file__), "static", "images", "random.jpg"), "wb") as f: f.write(mediaContent) with database.Database() as db: diff --git a/edaweb/services.py b/edaweb/services.py index 87af050..eca2bde 100644 --- a/edaweb/services.py +++ b/edaweb/services.py @@ -247,7 +247,7 @@ def parse_tweet(tweet_url): return dt, replying_to, text, images -def scrape_whispa(whispa_url, since): +def scrape_whispa(whispa_url, since = None): tree = html.fromstring(requests.get(whispa_url).content.decode()) qnas = [] # we're not doing proper HTML scraping here really... since the site uses client side rendering @@ -257,6 +257,9 @@ def scrape_whispa(whispa_url, since): if "receivedFeedback" in js: # my god this is horrible... for j in json.loads(json.loads(js[19:-1])[1][2:])[0][3]["loadedUser"]["receivedFeedback"]: + if j["childFeedback"] == []: + continue + dt = datetime.datetime.fromisoformat(j["childFeedback"][0]["createdAt"][:-1]) qnas.append({ @@ -359,7 +362,8 @@ def get_recent_commits(db, max_per_repo = 3): return sorted(out, key = lambda a: a["datetime"], reverse = True) if __name__ == "__main__": - import database + print(scrape_whispa(CONFIG.get("qnas", "url"))) + # import database - with database.Database() as db: - print(json.dumps(get_recent_commits(db), indent=4)) + # with database.Database() as db: + # print(json.dumps(get_recent_commits(db), indent=4)) diff --git a/edaweb/static/robots.txt b/edaweb/static/robots.txt index c2aab7e..7aa9c7e 100644 --- a/edaweb/static/robots.txt +++ b/edaweb/static/robots.txt @@ -1,2 +1,4 @@ User-agent: *
-Disallow: /
\ No newline at end of file +Allow: /
+User-agent: Googlebot-Image
+Disallow: *
diff --git a/entrypoint.sh b/entrypoint.sh index 335ad66..f8664f0 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,3 +1,5 @@ +rm -rvf /app/edaweb/edaweb.conf/ +ln -s /app/edaweb.conf /app/edaweb/edaweb.conf printenv | grep -v "no_proxy" >> /etc/environment -tmux new-session -d -s "cron" 'cron -f || bash && bash'; -python3 /app/edaweb/app.py --production
\ No newline at end of file +#tmux new-session -d -s "cron" 'cron -f || bash && bash'; +python3 /app/edaweb/app.py --production diff --git a/scripts/export.sh b/scripts/export.sh index c65a6e9..1d17cf1 100755 --- a/scripts/export.sh +++ b/scripts/export.sh @@ -9,4 +9,4 @@ read export_name echo "Exporting blog post " $id " to " $export_name touch $export_name -sudo docker run -it --entrypoint python3 -v "$(pwd)/edaweb.conf":/app/edaweb.conf -v "$(pwd)/$export_name":/app/$export_name --network mariadb --rm reg.reaweb.uk/edaweb /app/parser.py export -i $id -u root -o $export_name +sudo docker run -it --entrypoint python3 -v "$(pwd)/edaweb.conf":/app/edaweb/edaweb.conf -v "$(pwd)/$export_name":/app/$export_name --network mariadb --rm reg.reaweb.uk/edaweb /app/edaweb/parser.py export -i $id -u root -o $export_name diff --git a/scripts/update.sh b/scripts/update.sh index b082f18..fe225cb 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -3,4 +3,4 @@ echo -n "Input blog post ID to update: " read id -sudo docker run -it --entrypoint python3 -v "$(pwd)/edaweb.conf":/app/edaweb.conf -v "$(pwd)/$1":/app/$1 --network mariadb --rm reg.reaweb.uk/edaweb /app/parser.py update -i $id -u root -m $1 +sudo docker run -it --entrypoint python3 -v "$(pwd)/edaweb.conf":/app/edaweb/edaweb.conf -v "$(pwd)/$1":/app/$1 --network mariadb --rm reg.reaweb.uk/edaweb /app/edaweb/parser.py update -i $id -u root -m $1 |