feat: support for user-submitted files

Previously, user-uploaded files were stored in the /openedx/xqueue/openedx
folder (considered by django to be a media subfolder). This folder was not
being mounted on the host, thus causing files to be lost on container restart.
Also, files were simply not reported by the `tutor submissions show` utility.

We now bind-mount the media folder from the host. Media assets are served by
uwsgi, which replaced gunicorn.

When object storage is in use, we will have to point xqueue to the remote
storage solution. This means that we need to add patches to the tutor-minio
plugin.

Close #2.

README.rst

@@ -35,13 +35,26 @@ In the Open edX studio, edit a course and add a new "Advanced blank problem" ("P
             print("hello world")
           </answer_display>
           <grader_payload>
-            {"output": "hello world", "max_length": 2}
+            {"name": "hello world"}
           </grader_payload>
         </codeparam>
       </coderesponse>
     </problem>
 
-Note that the queue name must be "openedx".
+For a problem that includes a file submission, write instead::
+
+  <problem>
+    <coderesponse queuename="openedx">
+      <filesubmission/>
+      <codeparam>
+        <grader_payload>
+          {"name": "file submission"}
+        </grader_payload>
+      </codeparam>
+    </coderesponse>
+  </problem>
+
+Note that in all cases, the queue name must be "openedx".
 
 Save and publish the created unit. Then, access the unit from the LMS and attempt to answer the problem. The answer is sent to the Xqueue service. If you know how to use the Xqueue API, you can access it at http(s)://xqueue.LMS_HOST (in production) or http://xqueue.local.overhang.io (in development). However, the Xqueue API is a bit awkward to use. Tutor provides a simple command-line interface to interact with the Xqueue service.
 
@@ -69,6 +82,7 @@ Show the first submission that should be graded::
         "grader_payload": "\n        {\"output\": \"hello world\", \"max_length\": 2}\n      ",
         "student_response": "        # students write your program here\r\n        print \"42\"\r\n      "
       },
+      "files": {},
       "return_code": 0
     }
 

tutorxqueue/patches/local-docker-compose-services

@@ -3,7 +3,7 @@ xqueue:
   image: {{ XQUEUE_DOCKER_IMAGE }}
   volumes:
     - ../plugins/xqueue/apps/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py:ro
-    - ../../data/xqueue:/openedx/data
+    - ../../data/xqueue/media:/openedx/data/media
   environment:
     DJANGO_SETTINGS_MODULE: xqueue.tutor
   restart: unless-stopped
@@ -13,7 +13,6 @@ xqueue-consumer:
   image: {{ XQUEUE_DOCKER_IMAGE }}
   volumes:
     - ../plugins/xqueue/apps/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py:ro
-    - ../../data/xqueue:/openedx/data
   environment:
     DJANGO_SETTINGS_MODULE: xqueue.tutor
   restart: unless-stopped

tutorxqueue/plugin.py

@@ -153,10 +153,17 @@ class Client:
         submission_body = json.loads(data["xqueue_body"])
         submission_id = header["submission_id"]
         submission_key = header["submission_key"]
+        submission_files = {}
+        for filename, path in json.loads(data["xqueue_files"]).items():
+            if not path.startswith("http"):
+                # Relative path: prepend with server url
+                path = self.base_url + "/" + path
+            submission_files[filename] = path
         return {
             "id": submission_id,
             "key": submission_key,
             "body": submission_body,
+            "files": submission_files,
             "return_code": response["return_code"],
         }
 

tutorxqueue/templates/xqueue/apps/settings/tutor.py

@@ -17,8 +17,12 @@ DATABASES = {
     }
 }
 
-LOGGING = get_logger_config(log_dir="/openedx/data/", logging_env="tutor", dev_env=True)
-LOGGING["loggers"][""]["handlers"].append("console")
+# User-uploaded assets will be stored in this media folder
+MEDIA_ROOT = "/openedx/data/media"
+MEDIA_URL = "media/"
+
+LOGGING["handlers"].pop("local")
+LOGGING["loggers"][""]["handlers"] = ["console"]
 LOGGING["loggers"]["submission_queue.management.commands.run_consumer"] = {
     "level": "WARN",
     "handlers": ["console"]

tutorxqueue/templates/xqueue/build/xqueue/Dockerfile

@@ -6,11 +6,19 @@ RUN apt update && \
   apt install -y language-pack-en git python3 python3-pip libmysqlclient-dev
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
-RUN mkdir /openedx /openedx/data
+RUN mkdir /openedx /openedx/data /openedx/data/media
 RUN git clone https://github.com/edx/xqueue --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 /openedx/xqueue
 WORKDIR /openedx/xqueue
 
 RUN pip install -r requirements.txt
+RUN pip install uwsgi==2.0.19.1
 
 EXPOSE 8000
-CMD gunicorn --workers=2 --name xqueue --bind=0.0.0.0:8000 --max-requests=1000 xqueue.wsgi:application
+CMD uwsgi \
+    --static-map /media=/openedx/media/ \
+    --http 0.0.0.0:8000 \
+    --thunder-lock \
+    --single-interpreter \
+    --enable-threads \
+    --processes=2 \
+    --wsgi-file xqueue/wsgi.py