feat: Dockerfile optimizations

.gitlab-ci.yml

@@ -2,7 +2,7 @@ variables:
     TUTOR_PLUGIN: discovery
     TUTOR_IMAGES: discovery
     TUTOR_PYPI_PACKAGE: tutor-discovery
-    OPENEDX_RELEASE: olive
+    OPENEDX_RELEASE: palm
     GITHUB_REPO: overhangio/tutor-discovery
 
 include:

setup.py

@@ -34,8 +34,8 @@ setup(
     long_description_content_type="text/x-rst",
     packages=find_packages(exclude=["tests*"]),
     include_package_data=True,
-    install_requires=["tutor>=15.0.0,<16.0.0"],
-    python_requires=">=3.7",
+    install_requires=["tutor>=16.0.0,<17.0.0"],
+    python_requires=">=3.8",
     entry_points={"tutor.plugin.v1": ["discovery = tutordiscovery.plugin"]},
     classifiers=[
         "Development Status :: 5 - Production/Stable",
@@ -43,9 +43,9 @@ setup(
         "License :: OSI Approved :: GNU Affero General Public License v3",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
 )

tutordiscovery/__about__.py

@@ -1,2 +1 @@
-__version__ = "15.0.0"
-
+__version__ = "16.0.0"

tutordiscovery/patches/local-docker-compose-jobs-services

@@ -4,4 +4,6 @@ discovery-job:
     DEFAULT_PARTNER_CODE: openedx
   volumes:
     - ../plugins/discovery/apps/settings/tutor:/openedx/discovery/course_discovery/settings/tutor:ro
-  depends_on: {{ [("lms", RUN_LMS), ("mysql", RUN_MYSQL)]|list_if }}
+  depends_on: 
+    - lms
+    {% if RUN_MYSQL %}- mysql{% endif %}

tutordiscovery/patches/local-docker-compose-services

@@ -5,4 +5,7 @@ discovery:
   restart: unless-stopped
   volumes:
     - ../plugins/discovery/apps/settings/tutor:/openedx/discovery/course_discovery/settings/tutor:ro
-  depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("lms", RUN_LMS), ("mysql", RUN_MYSQL)]|list_if }}
+  depends_on: 
+    - lms
+    {% if RUN_MYSQL %}- mysql{% endif %}
+    {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %}

tutordiscovery/plugin.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+
 from glob import glob
 import os
 import pkg_resources
+import typing as t
 
 from tutor import hooks as tutor_hooks
 
@@ -74,6 +77,7 @@ tutor_hooks.Filters.IMAGES_PUSH.add_item(
     )
 )
 
+
 # Automount /openedx/discovery folder from the container
 @tutor_hooks.Filters.COMPOSE_MOUNTS.add()
 def _mount_course_discovery(mounts, name):
@@ -115,3 +119,17 @@ tutor_hooks.Filters.CONFIG_UNIQUE.add_items(
 tutor_hooks.Filters.CONFIG_OVERRIDES.add_items(
     list(config.get("overrides", {}).items())
 )
+
+########################################
+# Credentials Public Host
+########################################
+
+
+@tutor_hooks.Filters.APP_PUBLIC_HOSTS.add()
+def _discovery_public_hosts(hosts: list[str], context_name: t.Literal["local", "dev"]) -> list[str]:
+    if context_name == "dev":
+        # todo: will may change the below dev port when i try this plugin in dev mode
+        hosts += ["discovery.{{ LMS_HOST }}:8000"]
+    else:
+        hosts += ["discovery.{{ LMS_HOST }}"]
+    return hosts

tutordiscovery/templates/discovery/build/discovery/Dockerfile

@@ -1,19 +1,39 @@
-FROM docker.io/ubuntu:20.04
+{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %}
+###### Minimal image with base system requirements for most stages
+FROM docker.io/ubuntu:20.04 as minimal
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update && \
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \ 
+    apt update && \
     apt install -y curl git-core language-pack-en python3 python3-pip python3-venv \
     build-essential libcairo2 libffi-dev libmysqlclient-dev libxml2-dev libxslt-dev libjpeg-dev libssl-dev
 ENV LC_ALL en_US.UTF-8
 
+###### Git-clone course-discovery repo
+FROM minimal as code
+ARG DISCOVERY_REPOSITORY=https://github.com/openedx/course-discovery.git
+ARG DISCOVERY_VERSION='{{ OPENEDX_COMMON_VERSION }}'
+RUN mkdir -p /openedx/discovery && \
+    git clone $DISCOVERY_REPOSITORY --branch $DISCOVERY_VERSION --depth 1 /openedx/discovery
+
+##### Empty layer with just the repo at the root.
+# This is useful when overriding the build context with a host repo:
+# docker build --build-context course-discovery=/path/to/course-discovery
+FROM scratch as discovery
+COPY --from=code /openedx/discovery /
+
+##### Production layer
+FROM minimal as production
+
+# Create app user
 ARG APP_USER_ID=1000
+RUN if [ "$APP_USER_ID" = 0 ]; then echo "app user may not be root" && false; fi
 RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER_ID} app
 USER ${APP_USER_ID}
 
-ARG DISCOVERY_REPOSITORY=https://github.com/edx/course-discovery.git
-ARG DISCOVERY_VERSION={{ OPENEDX_COMMON_VERSION }}
-RUN mkdir -p /openedx/discovery && \
-    git clone $DISCOVERY_REPOSITORY --branch $DISCOVERY_VERSION --depth 1 /openedx/discovery
+# Copy repo
+COPY --chown=app:app --from=discovery / /openedx/discovery
 WORKDIR /openedx/discovery
 
 # Setup empty yml config file, which is required by production settings
@@ -23,35 +43,37 @@ ENV DISCOVERY_CFG /openedx/config.yml
 # Install python venv
 RUN python3 -m venv ../venv/
 ENV PATH "/openedx/venv/bin:$PATH"
+
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
     # https://pypi.org/project/setuptools/
     # https://pypi.org/project/pip/
     # https://pypi.org/project/wheel/
-RUN pip install setuptools==65.5.1 pip==22.3.1 wheel==0.38.4
+    setuptools==67.7.2 pip==23.1.2. wheel==0.40.0
 
 # Install a recent version of nodejs
-RUN pip install nodeenv
+RUN pip install nodeenv==1.7.0
 # nodejs version picked from https://github.com/openedx/course-discovery/blob/master/Dockerfile
 RUN nodeenv /openedx/nodeenv --node=16.14.2 --prebuilt
 ENV PATH /openedx/nodeenv/bin:${PATH}
+# Install npm/bower requirements
+ARG NPM_REGISTRY='{{ NPM_REGISTRY }}'
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.npm/,sharing=shared,uid=${APP_USER_ID} {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY --production
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/bower/,sharing=shared,uid=${APP_USER_ID} {% endif %}./node_modules/.bin/bower install --allow-root --production
 
 # Install python and nodejs requirements
 # This is identical to "make production-requirements" but it was split in multiple
 # instructions to benefit from docker image caching
-RUN pip install -r requirements.txt
-{% for extra_requirements in DISCOVERY_EXTRA_PIP_REQUIREMENTS %}RUN pip install '{{ extra_requirements }}'
+# Install base requirements
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements.txt
+{% for extra_requirement in DISCOVERY_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}'
 {% endfor %}
 
-ARG NPM_REGISTRY={{ NPM_REGISTRY }}
-RUN npm install --verbose --registry=$NPM_REGISTRY --production
-RUN ./node_modules/.bin/bower install --allow-root --production
-
-# Install django-redis for using redis as a django cache
-# https://pypi.org/project/django-redis/
-RUN pip install django-redis==5.2.0
-
-# Install uwsgi
-# https://pypi.org/project/uWSGI/
-RUN pip install uwsgi==2.0.21
+# Install extra requirements
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+    # Use redis as a django cache https://pypi.org/project/django-redis/
+    django-redis==5.2.0 \
+    # uwsgi server https://pypi.org/project/uWSGI/
+    uwsgi==2.0.21
 
 # Collect static assets
 COPY --chown=app:app assets.py ./course_discovery/settings/assets.py