From f2f734194c03dfff2024cf417c502515ddb7a855 Mon Sep 17 00:00:00 2001 From: jwansek Date: Sat, 21 May 2022 22:38:52 +0100 Subject: Added running as an API --- API/Dockerfile | 7 ++++ API/api.conf | 13 ++++++ API/app.py | 109 +++++++++++++++++++++++++++++++++++++++++++++++++ API/docker-compose.yml | 23 +++++++++++ API/helpers.py | 40 ++++++++++++++++++ API/requirements.txt | 5 +++ 6 files changed, 197 insertions(+) create mode 100644 API/Dockerfile create mode 100644 API/api.conf create mode 100644 API/app.py create mode 100644 API/docker-compose.yml create mode 100644 API/helpers.py create mode 100644 API/requirements.txt (limited to 'API') diff --git a/API/Dockerfile b/API/Dockerfile new file mode 100644 index 0000000..5d482c7 --- /dev/null +++ b/API/Dockerfile @@ -0,0 +1,7 @@ +FROM smarker +MAINTAINER Eden Attenborough "gae19jtu@uea.ac.uk" +COPY . /API +WORKDIR /API +RUN pip3 install -r requirements.txt +ENTRYPOINT ["python3"] +CMD ["app.py", "--production"] \ No newline at end of file diff --git a/API/api.conf b/API/api.conf new file mode 100644 index 0000000..b8143f3 --- /dev/null +++ b/API/api.conf @@ -0,0 +1,13 @@ +[production] +port = 6970 +host = 0.0.0.0 + +[testing] +port = 5002 +host = 0.0.0.0 + +[mysql] +host = vps.eda.gay +port = 3307 +user = root +passwd = ********** \ No newline at end of file diff --git a/API/app.py b/API/app.py new file mode 100644 index 0000000..e4cb769 --- /dev/null +++ b/API/app.py @@ -0,0 +1,109 @@ +from paste.translogger import TransLogger +from waitress import serve +import configparser +import werkzeug +import helpers +import flask +import sys +import os + +# os.environ["UPLOADS_DIR"] = "/media/veracrypt1/Edencloud/UniStuff/3.0 - CMP 3rd Year Project/Smarker/API/.uploads" + +sys.path.insert(1, os.path.join("..", "Smarker")) +import database + +app = flask.Flask(__name__) +app.config['UPLOAD_FOLDER'] = ".uploads" +API_CONF = configparser.ConfigParser() +API_CONF.read("api.conf") + + +@app.route("/api/helloworld") +def helloworld(): + """GET ``/api/helloworld`` + + Returns a friendly message to check the server is working + """ + return flask.jsonify({"hello": "world!"}) + +@app.route("/api/mark", methods = ["post"]) +def mark(): + """POST ``/api/mark`` + + Expects to be a POST request of ``Content-Type: multipart/form-data``. + + * Expects a valid API key under the key ``key`` + + * Expects an assessment name under the key ``assessment`` + + * Expects a submission zip file under the key ``zip`` + + * File dependences can be added with keys prefixed with ``filedep``, but a corrisponding key must also be present with the location of this file in the sandboxed environment: e.g. ``-F "filedep1=@../../aDependency.txt" -F "aDependency.txt=/aDependency.txt"`` + + Returns a report in the JSON format. + """ + # try: + assessment_name = flask.request.form.get('assessment') + api_key = flask.request.form.get('key') + files = [] + + with database.SmarkerDatabase( + host = API_CONF.get("mysql", "host"), + user = API_CONF.get("mysql", "user"), + passwd = API_CONF.get("mysql", "passwd"), + db = "Smarker", + port = API_CONF.getint("mysql", "port")) as db: + + if db.valid_apikey(api_key): + f = flask.request.files["zip"] + zip_name = f.filename + f.save(os.path.join(".uploads/", f.filename)) + # even though this is inside docker, we are accessing the HOST docker daemon + # so we have to pass through the HOST location for volumes... very annoying I know + # so we set this environment variable + # https://serverfault.com/questions/819369/mounting-a-volume-with-docker-in-docker + files.append("%s:/tmp/%s" % (os.path.join(os.environ["UPLOADS_DIR"], zip_name), zip_name)) + + for file_dep in flask.request.files.keys(): + if file_dep.startswith("filedep"): + f = flask.request.files[file_dep] + f.save(os.path.join(".uploads/", f.filename)) + dep_name = os.path.split(f.filename)[-1] + client_loc = flask.request.form.get(dep_name) + if client_loc is None: + return flask.abort(flask.Response("You need to specify a location to put file dependency '%s' e.g. '%s=/%s'" % (dep_name, dep_name, dep_name), status=500)) + + files.append("%s:%s" % (os.path.join(os.environ["UPLOADS_DIR"], dep_name), client_loc)) + + + try: + return flask.jsonify(helpers.run_smarker_simple(db, zip_name, assessment_name, files)) + except Exception as e: + flask.abort(flask.Response(str(e), status=500)) + else: + flask.abort(403) + # except (KeyError, TypeError, ValueError): + # flask.abort(400) + + +if __name__ == "__main__": + try: + if sys.argv[1] == "--production": + serve( + TransLogger(app), + host = API_CONF.get("production", "host"), + port = API_CONF.getint("production", "port"), + threads = 32 + ) + else: + app.run( + host = API_CONF.get("testing", "host"), + port = API_CONF.getint("testing", "port"), + debug = True + ) + except IndexError: + app.run( + host = API_CONF.get("testing", "host"), + port = API_CONF.getint("testing", "port"), + debug = True + ) \ No newline at end of file diff --git a/API/docker-compose.yml b/API/docker-compose.yml new file mode 100644 index 0000000..19d9d3c --- /dev/null +++ b/API/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3' + +services: + smarker: + build: + context: .. + dockerfile: Dockerfile + image: smarker + smarker-api: + build: + context: . + dockerfile: Dockerfile + image: smarker-api + ports: + - "6970:6970" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./.uploads/:/API/.uploads/ + - /tmp/:/tmp/ + environment: + - UPLOADS_DIR= + depends_on: + - smarker \ No newline at end of file diff --git a/API/helpers.py b/API/helpers.py new file mode 100644 index 0000000..692e566 --- /dev/null +++ b/API/helpers.py @@ -0,0 +1,40 @@ +import tempfile +import docker +import json +import os + +CLIENT = docker.from_env() + +def run_smarker_simple(db, zip_name, assessment_name, volumes): + with tempfile.TemporaryDirectory() as td: # remember to passthru /tmp/ as a volume + env = [ # probably need to find a better way tbh + "submission=/tmp/%s" % zip_name, + "assessment=%s" % assessment_name, + "format=json", + "output=/out/report.json" + ] + outjson = os.path.join(td, "report.json") + volumes.append("%s:/out/report.json" % (outjson)) + # print("file_deps:", volumes) + + try: + pypideps = db.get_assessment_yaml(assessment_name)["dependencies"]["libraries"] + env.append("SMARKERDEPS=" + ",".join(pypideps)) + except KeyError: + pass + # print("env: ", env) + + open(outjson, 'a').close() # make a blank file so docker doesnt make it as a directory + log = CLIENT.containers.run( + "smarker", + remove = True, + volumes = volumes, + environment = env + ) + print("log: ", log) + + for f in os.listdir(".uploads"): + os.remove(os.path.join(".uploads", f)) + + with open(outjson, "r") as f: + return json.load(f) diff --git a/API/requirements.txt b/API/requirements.txt new file mode 100644 index 0000000..32e26e6 --- /dev/null +++ b/API/requirements.txt @@ -0,0 +1,5 @@ +flask +PasteScript==3.2.0 +waitress +docker +werkzeug -- cgit v1.2.3