From 029ac27251769deeb17dac3d1b26cff5ff071e51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 2 Sep 2021 09:46:06 +0200 Subject: [PATCH] feat: support for user-submitted files Previously, user-uploaded files were stored in the /openedx/xqueue/openedx folder (considered by django to be a media subfolder). This folder was not being mounted on the host, thus causing files to be lost on container restart. Also, files were simply not reported by the `tutor submissions show` utility. We now bind-mount the media folder from the host. Media assets are served by uwsgi, which replaced gunicorn. When object storage is in use, we will have to point xqueue to the remote storage solution. This means that we need to add patches to the tutor-minio plugin. Close #2. --- README.rst | 18 ++++++++++++++++-- .../patches/local-docker-compose-services | 3 +-- tutorxqueue/plugin.py | 7 +++++++ .../templates/xqueue/apps/settings/tutor.py | 8 ++++++-- .../templates/xqueue/build/xqueue/Dockerfile | 12 ++++++++++-- 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index a9db174..c16cb97 100644 --- a/README.rst +++ b/README.rst @@ -35,13 +35,26 @@ In the Open edX studio, edit a course and add a new "Advanced blank problem" ("P print("hello world") - {"output": "hello world", "max_length": 2} + {"name": "hello world"} -Note that the queue name must be "openedx". +For a problem that includes a file submission, write instead:: + + + + + + + {"name": "file submission"} + + + + + +Note that in all cases, the queue name must be "openedx". Save and publish the created unit. Then, access the unit from the LMS and attempt to answer the problem. The answer is sent to the Xqueue service. If you know how to use the Xqueue API, you can access it at http(s)://xqueue.LMS_HOST (in production) or http://xqueue.local.overhang.io (in development). However, the Xqueue API is a bit awkward to use. Tutor provides a simple command-line interface to interact with the Xqueue service. @@ -69,6 +82,7 @@ Show the first submission that should be graded:: "grader_payload": "\n {\"output\": \"hello world\", \"max_length\": 2}\n ", "student_response": " # students write your program here\r\n print \"42\"\r\n " }, + "files": {}, "return_code": 0 } diff --git a/tutorxqueue/patches/local-docker-compose-services b/tutorxqueue/patches/local-docker-compose-services index 3645d91..49ec223 100644 --- a/tutorxqueue/patches/local-docker-compose-services +++ b/tutorxqueue/patches/local-docker-compose-services @@ -3,7 +3,7 @@ xqueue: image: {{ XQUEUE_DOCKER_IMAGE }} volumes: - ../plugins/xqueue/apps/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py:ro - - ../../data/xqueue:/openedx/data + - ../../data/xqueue/media:/openedx/data/media environment: DJANGO_SETTINGS_MODULE: xqueue.tutor restart: unless-stopped @@ -13,7 +13,6 @@ xqueue-consumer: image: {{ XQUEUE_DOCKER_IMAGE }} volumes: - ../plugins/xqueue/apps/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py:ro - - ../../data/xqueue:/openedx/data environment: DJANGO_SETTINGS_MODULE: xqueue.tutor restart: unless-stopped diff --git a/tutorxqueue/plugin.py b/tutorxqueue/plugin.py index f8270ea..bf507f3 100644 --- a/tutorxqueue/plugin.py +++ b/tutorxqueue/plugin.py @@ -153,10 +153,17 @@ class Client: submission_body = json.loads(data["xqueue_body"]) submission_id = header["submission_id"] submission_key = header["submission_key"] + submission_files = {} + for filename, path in json.loads(data["xqueue_files"]).items(): + if not path.startswith("http"): + # Relative path: prepend with server url + path = self.base_url + "/" + path + submission_files[filename] = path return { "id": submission_id, "key": submission_key, "body": submission_body, + "files": submission_files, "return_code": response["return_code"], } diff --git a/tutorxqueue/templates/xqueue/apps/settings/tutor.py b/tutorxqueue/templates/xqueue/apps/settings/tutor.py index b0c8754..02722a2 100644 --- a/tutorxqueue/templates/xqueue/apps/settings/tutor.py +++ b/tutorxqueue/templates/xqueue/apps/settings/tutor.py @@ -17,8 +17,12 @@ DATABASES = { } } -LOGGING = get_logger_config(log_dir="/openedx/data/", logging_env="tutor", dev_env=True) -LOGGING["loggers"][""]["handlers"].append("console") +# User-uploaded assets will be stored in this media folder +MEDIA_ROOT = "/openedx/data/media" +MEDIA_URL = "media/" + +LOGGING["handlers"].pop("local") +LOGGING["loggers"][""]["handlers"] = ["console"] LOGGING["loggers"]["submission_queue.management.commands.run_consumer"] = { "level": "WARN", "handlers": ["console"] diff --git a/tutorxqueue/templates/xqueue/build/xqueue/Dockerfile b/tutorxqueue/templates/xqueue/build/xqueue/Dockerfile index 716fba3..65dd7f7 100644 --- a/tutorxqueue/templates/xqueue/build/xqueue/Dockerfile +++ b/tutorxqueue/templates/xqueue/build/xqueue/Dockerfile @@ -6,11 +6,19 @@ RUN apt update && \ apt install -y language-pack-en git python3 python3-pip libmysqlclient-dev RUN ln -s /usr/bin/python3 /usr/bin/python -RUN mkdir /openedx /openedx/data +RUN mkdir /openedx /openedx/data /openedx/data/media RUN git clone https://github.com/edx/xqueue --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 /openedx/xqueue WORKDIR /openedx/xqueue RUN pip install -r requirements.txt +RUN pip install uwsgi==2.0.19.1 EXPOSE 8000 -CMD gunicorn --workers=2 --name xqueue --bind=0.0.0.0:8000 --max-requests=1000 xqueue.wsgi:application +CMD uwsgi \ + --static-map /media=/openedx/media/ \ + --http 0.0.0.0:8000 \ + --thunder-lock \ + --single-interpreter \ + --enable-threads \ + --processes=2 \ + --wsgi-file xqueue/wsgi.py