From f2f734194c03dfff2024cf417c502515ddb7a855 Mon Sep 17 00:00:00 2001 From: jwansek Date: Sat, 21 May 2022 22:38:52 +0100 Subject: Added running as an API --- .gitignore | 3 +- API/Dockerfile | 7 +++ API/api.conf | 13 +++++ API/app.py | 109 +++++++++++++++++++++++++++++++++++++++ API/docker-compose.yml | 23 +++++++++ API/helpers.py | 40 ++++++++++++++ API/requirements.txt | 5 ++ Dockerfile | 1 + ExampleAssessments/CMP-4009B.yml | 71 ------------------------- ExampleAssessments/example.yml | 3 +- Smarker/database.py | 17 ++++-- Smarker/requirements.txt | 2 - Smarker/smarker.conf | 24 +++++++++ docs/source/api.rst | 26 ++++++++++ docs/source/conf.py | 1 + docs/source/index.rst | 27 +++++++--- 16 files changed, 284 insertions(+), 88 deletions(-) create mode 100644 API/Dockerfile create mode 100644 API/api.conf create mode 100644 API/app.py create mode 100644 API/docker-compose.yml create mode 100644 API/helpers.py create mode 100644 API/requirements.txt delete mode 100644 ExampleAssessments/CMP-4009B.yml create mode 100644 Smarker/smarker.conf create mode 100644 docs/source/api.rst diff --git a/.gitignore b/.gitignore index db2ce2b..dade9b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ +API/.uploads/*.* out/ *_report.* *.zip -smarker.conf +#smarker.conf *.aux *.pickle diff --git a/API/Dockerfile b/API/Dockerfile new file mode 100644 index 0000000..5d482c7 --- /dev/null +++ b/API/Dockerfile @@ -0,0 +1,7 @@ +FROM smarker +MAINTAINER Eden Attenborough "gae19jtu@uea.ac.uk" +COPY . /API +WORKDIR /API +RUN pip3 install -r requirements.txt +ENTRYPOINT ["python3"] +CMD ["app.py", "--production"] \ No newline at end of file diff --git a/API/api.conf b/API/api.conf new file mode 100644 index 0000000..b8143f3 --- /dev/null +++ b/API/api.conf @@ -0,0 +1,13 @@ +[production] +port = 6970 +host = 0.0.0.0 + +[testing] +port = 5002 +host = 0.0.0.0 + +[mysql] +host = vps.eda.gay +port = 3307 +user = root +passwd = ********** \ No newline at end of file diff --git a/API/app.py b/API/app.py new file mode 100644 index 0000000..e4cb769 --- /dev/null +++ b/API/app.py @@ -0,0 +1,109 @@ +from paste.translogger import TransLogger +from waitress import serve +import configparser +import werkzeug +import helpers +import flask +import sys +import os + +# os.environ["UPLOADS_DIR"] = "/media/veracrypt1/Edencloud/UniStuff/3.0 - CMP 3rd Year Project/Smarker/API/.uploads" + +sys.path.insert(1, os.path.join("..", "Smarker")) +import database + +app = flask.Flask(__name__) +app.config['UPLOAD_FOLDER'] = ".uploads" +API_CONF = configparser.ConfigParser() +API_CONF.read("api.conf") + + +@app.route("/api/helloworld") +def helloworld(): + """GET ``/api/helloworld`` + + Returns a friendly message to check the server is working + """ + return flask.jsonify({"hello": "world!"}) + +@app.route("/api/mark", methods = ["post"]) +def mark(): + """POST ``/api/mark`` + + Expects to be a POST request of ``Content-Type: multipart/form-data``. + + * Expects a valid API key under the key ``key`` + + * Expects an assessment name under the key ``assessment`` + + * Expects a submission zip file under the key ``zip`` + + * File dependences can be added with keys prefixed with ``filedep``, but a corrisponding key must also be present with the location of this file in the sandboxed environment: e.g. ``-F "filedep1=@../../aDependency.txt" -F "aDependency.txt=/aDependency.txt"`` + + Returns a report in the JSON format. + """ + # try: + assessment_name = flask.request.form.get('assessment') + api_key = flask.request.form.get('key') + files = [] + + with database.SmarkerDatabase( + host = API_CONF.get("mysql", "host"), + user = API_CONF.get("mysql", "user"), + passwd = API_CONF.get("mysql", "passwd"), + db = "Smarker", + port = API_CONF.getint("mysql", "port")) as db: + + if db.valid_apikey(api_key): + f = flask.request.files["zip"] + zip_name = f.filename + f.save(os.path.join(".uploads/", f.filename)) + # even though this is inside docker, we are accessing the HOST docker daemon + # so we have to pass through the HOST location for volumes... very annoying I know + # so we set this environment variable + # https://serverfault.com/questions/819369/mounting-a-volume-with-docker-in-docker + files.append("%s:/tmp/%s" % (os.path.join(os.environ["UPLOADS_DIR"], zip_name), zip_name)) + + for file_dep in flask.request.files.keys(): + if file_dep.startswith("filedep"): + f = flask.request.files[file_dep] + f.save(os.path.join(".uploads/", f.filename)) + dep_name = os.path.split(f.filename)[-1] + client_loc = flask.request.form.get(dep_name) + if client_loc is None: + return flask.abort(flask.Response("You need to specify a location to put file dependency '%s' e.g. '%s=/%s'" % (dep_name, dep_name, dep_name), status=500)) + + files.append("%s:%s" % (os.path.join(os.environ["UPLOADS_DIR"], dep_name), client_loc)) + + + try: + return flask.jsonify(helpers.run_smarker_simple(db, zip_name, assessment_name, files)) + except Exception as e: + flask.abort(flask.Response(str(e), status=500)) + else: + flask.abort(403) + # except (KeyError, TypeError, ValueError): + # flask.abort(400) + + +if __name__ == "__main__": + try: + if sys.argv[1] == "--production": + serve( + TransLogger(app), + host = API_CONF.get("production", "host"), + port = API_CONF.getint("production", "port"), + threads = 32 + ) + else: + app.run( + host = API_CONF.get("testing", "host"), + port = API_CONF.getint("testing", "port"), + debug = True + ) + except IndexError: + app.run( + host = API_CONF.get("testing", "host"), + port = API_CONF.getint("testing", "port"), + debug = True + ) \ No newline at end of file diff --git a/API/docker-compose.yml b/API/docker-compose.yml new file mode 100644 index 0000000..19d9d3c --- /dev/null +++ b/API/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3' + +services: + smarker: + build: + context: .. + dockerfile: Dockerfile + image: smarker + smarker-api: + build: + context: . + dockerfile: Dockerfile + image: smarker-api + ports: + - "6970:6970" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./.uploads/:/API/.uploads/ + - /tmp/:/tmp/ + environment: + - UPLOADS_DIR= + depends_on: + - smarker \ No newline at end of file diff --git a/API/helpers.py b/API/helpers.py new file mode 100644 index 0000000..692e566 --- /dev/null +++ b/API/helpers.py @@ -0,0 +1,40 @@ +import tempfile +import docker +import json +import os + +CLIENT = docker.from_env() + +def run_smarker_simple(db, zip_name, assessment_name, volumes): + with tempfile.TemporaryDirectory() as td: # remember to passthru /tmp/ as a volume + env = [ # probably need to find a better way tbh + "submission=/tmp/%s" % zip_name, + "assessment=%s" % assessment_name, + "format=json", + "output=/out/report.json" + ] + outjson = os.path.join(td, "report.json") + volumes.append("%s:/out/report.json" % (outjson)) + # print("file_deps:", volumes) + + try: + pypideps = db.get_assessment_yaml(assessment_name)["dependencies"]["libraries"] + env.append("SMARKERDEPS=" + ",".join(pypideps)) + except KeyError: + pass + # print("env: ", env) + + open(outjson, 'a').close() # make a blank file so docker doesnt make it as a directory + log = CLIENT.containers.run( + "smarker", + remove = True, + volumes = volumes, + environment = env + ) + print("log: ", log) + + for f in os.listdir(".uploads"): + os.remove(os.path.join(".uploads", f)) + + with open(outjson, "r") as f: + return json.load(f) diff --git a/API/requirements.txt b/API/requirements.txt new file mode 100644 index 0000000..32e26e6 --- /dev/null +++ b/API/requirements.txt @@ -0,0 +1,5 @@ +flask +PasteScript==3.2.0 +waitress +docker +werkzeug diff --git a/Dockerfile b/Dockerfile index 8f69849..529c75f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get update -y RUN apt-get install -y python3-pip python3-dev build-essential texlive-latex-base texlive-pictures texlive-science texlive-latex-extra wkhtmltopdf RUN mkdir /out +RUN mkdir /.uploads COPY ./Smarker /Smarker WORKDIR /Smarker RUN pip3 install -r requirements.txt diff --git a/ExampleAssessments/CMP-4009B.yml b/ExampleAssessments/CMP-4009B.yml deleted file mode 100644 index c945907..0000000 --- a/ExampleAssessments/CMP-4009B.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: CMP-4009B-2020-A2 -files: - - pjtool.py: - classes: - - Date: - methods: - - __init__(4) - - __str__(1) - - __eq__(2) - - __lt__(2) - - numYears(2) - - numMonths(2) - tests: - - | - d1 = pjtool.Date(2001, 8, 12) - d2 = pjtool.Date(2001, 8, 12) - assert d1 == d2 - - | - d1 = pjtool.Date(2001, 8, 12) - d2 = pjtool.Date(1999, 4, 5) - assert d1 != d2 - - | - d1 = pjtool.Date(2001, 8, 12) - d2 = pjtool.Date(1999, 4, 5) - assert d1 > d2 - - | - import random - d1 = pjtool.Date(random.randint(2000, 2050), 8, 12) - d2 = pjtool.Date(random.randint(1900, 2050), 4, 5) - assert d1.numYears(d2) == abs(d1.year - d2.year) - - | - d1 = pjtool.Date(2020, 5, 1) - d2 = pjtool.Date(2020, 6, 1) - assert d1.numMonths(d2) == 0 - - | - d1 = pjtool.Date(2020, 5, 1) - d2 = pjtool.Date(2020, 8, 1) - assert d1.numMonths(d2) == 2 - - tester.py: - functions: - - dateTester(2) - run: - - python tester.py: - regexes: - - True\nFalse - - turbine.py: - classes: - - Turbine: - methods: - - __init__(5) - - __str__(1) - - numYearsInst(2) - - serviceDue(2) - - serviceAt(2) - - powerAt(2) - - AdvTurbine: - methods: - - __init__(5) - - __str__(1) - - numYearsInst(2) - - serviceDue(2) - - serviceAt(2) - - powerAt(2) -produced_files: - - pdLine.png - - pdResult.txt -dependencies: - libraries: - - matplotlib - files: - - ../wsData.txt diff --git a/ExampleAssessments/example.yml b/ExampleAssessments/example.yml index 154c733..f998048 100644 --- a/ExampleAssessments/example.yml +++ b/ExampleAssessments/example.yml @@ -57,4 +57,5 @@ dependencies: files: - ../wsData.txt libraries: - - matplotlib \ No newline at end of file + - matplotlib + - opencv-python-headless \ No newline at end of file diff --git a/Smarker/database.py b/Smarker/database.py index 37a44db..a3f77af 100644 --- a/Smarker/database.py +++ b/Smarker/database.py @@ -1,5 +1,6 @@ from dataclasses import dataclass import pymysql +import secrets import yaml @dataclass @@ -54,7 +55,8 @@ class SmarkerDatabase: CREATE TABLE students( student_no VARCHAR(10) PRIMARY KEY NOT NULL, name TEXT NOT NULL, - email VARCHAR(50) NOT NULL + email VARCHAR(50) NOT NULL, + apikey VARCHAR(64) NOT NULL ); """) cursor.execute(""" @@ -175,8 +177,8 @@ class SmarkerDatabase: email (str): Student's email """ with self.__connection.cursor() as cursor: - cursor.execute("INSERT INTO students VALUES (%s, %s, %s);", - (student_id, name, email)) + cursor.execute("INSERT INTO students VALUES (%s, %s, %s, %s);", + (student_id, name, email, secrets.token_urlsafe(32))) self.__connection.commit() def add_submission(self, student_id, assessment_name, report_yaml, files): @@ -198,9 +200,9 @@ class SmarkerDatabase: cursor.execute(""" INSERT INTO submitted_files (submission_id, file_id, file_text) - VALUES (%s, (SELECT file_id FROM assessment_file WHERE file_name = %s), %s); + VALUES (%s, (SELECT file_id FROM assessment_file WHERE file_name = %s AND assessment_name = %s), %s); """, ( - submission_id, file_name, file_contents + submission_id, file_name, assessment_name, file_contents )) self.__connection.commit() @@ -240,6 +242,11 @@ class SmarkerDatabase: cursor.execute("SELECT file_name FROM assessment_file WHERE assessment_name = %s;", (assessment_name, )) return [i[0] for i in cursor.fetchall()] + def valid_apikey(self, key): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT apikey FROM students WHERE apikey = %s", (key, )) + return key in [i[0] for i in cursor.fetchall()] + if __name__ == "__main__": with SmarkerDatabase(host = "vps.eda.gay", user="root", passwd=input("Input password: "), db="Smarker", port=3307) as db: # print(db.get_assessments_required_files("example")) diff --git a/Smarker/requirements.txt b/Smarker/requirements.txt index 3be9c36..af89c27 100644 --- a/Smarker/requirements.txt +++ b/Smarker/requirements.txt @@ -1,5 +1,3 @@ -# sudo apt-get install wkhtmltopdf -# https://github.com/olivierverdier/python-latex-highlighting Jinja2==3.0.3 misaka==2.1.1 Pygments==2.10.0 diff --git a/Smarker/smarker.conf b/Smarker/smarker.conf new file mode 100644 index 0000000..3416564 --- /dev/null +++ b/Smarker/smarker.conf @@ -0,0 +1,24 @@ +[mysql] +host = vps.eda.gay +port = 3307 +user = root +passwd = ************ + +[tex] +columns = 1 +show_full_docs = True +show_source = True +show_all_regex_occurrences = True +show_all_run_output = True + +[md] +show_full_docs = False +show_source = False +show_all_regex_occurrences = True +show_all_run_output = False + +[txt] +show_full_docs = True +show_source = True +show_all_regex_occurrences = True +show_all_run_output = True \ No newline at end of file diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 0000000..61469f9 --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,26 @@ +.. _api: + +Running as an API +================= + +*Smarker* can be hosted on a server and accessed through an API. A valid docker-compose +file is in the ``API/`` directory. Since the API docker container accesses the host docker +daemon, you must pass set the host location of the ``.uploads/`` directory as the ``$UPLOADS_DIR`` +environment variable. + +.. autofunction:: app.helloworld + +.. autofunction:: app.mark + +An example CURL request could be: + +.. code-block:: bash + + curl -X POST -H "Content-Type: multipart/form-data" \ + -F "zip=@../100301654.zip" \ + -F "key=2g_yU7n1SqTODGQmpuViIAwbdbownmVDpjUl9NKkRqz" \ + -F "assessment=example" \ + -F "filedep1=@../../dependency.txt" \ + -F "dependency.txt=/dependency.txt" \ + "localhost:6970/api/mark" + diff --git a/docs/source/conf.py b/docs/source/conf.py index 5dd644b..a168856 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,6 +13,7 @@ import os import sys sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "Smarker"))) +sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "API"))) sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "ExampleAssessments"))) # print(os.listdir(os.path.abspath(os.path.join("..", "..", "Smarker")))) diff --git a/docs/source/index.rst b/docs/source/index.rst index f2d7426..80fefaf 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,6 +11,9 @@ Setting up * Add an assessment yaml- :ref:`assessmentyaml`. * Enroll students: ``python3 Smarker/assessments.py -s 123456789,Eden,Attenborough,E.Attenborough@uea.ac.uk`` +*Smarker* can be used inside docker, see :ref:`docker` (recommended for sandboxing client code) +and through an API- see :ref:`api`. + ``smarker.py`` usage ******************** @@ -37,20 +40,28 @@ Also see :ref:`assessments` .. toctree:: :maxdepth: 2 + :caption: Setting up: + + quickstart.rst + configfile.rst + +.. toctree:: + :maxdepth: 3 + :caption: Docker: + + docker.rst + assessmentyaml.rst + api.rst + +.. toctree:: + :maxdepth: 3 :caption: Modules: reflect.rst database.rst assessments.rst + api.rst -.. toctree:: - :maxdepth: 2 - :caption: Other Pages: - - quickstart.rst - configfile.rst - docker.rst - assessmentyaml.rst Indices and tables ================== -- cgit v1.2.3