feat: upgrade to palm

.gitlab-ci.yml

@@ -2,7 +2,7 @@ variables:
     TUTOR_PLUGIN: discovery
     TUTOR_IMAGES: discovery
     TUTOR_PYPI_PACKAGE: tutor-discovery
-    OPENEDX_RELEASE: olive
+    OPENEDX_RELEASE: palm
     GITHUB_REPO: overhangio/tutor-discovery
 
 include:

README.rst

@@ -61,16 +61,21 @@ Re-indexing courses
 Caching programs
 ~~~~~~~~~~~~~~~~
 
-In order to cache programs in the LMS, you will need to manually create a catalog integration. This step should be performed just once::
+In order to cache programs in the LMS, you will need to manually create a catalog integration. Make sure you use staff user for the below command. If ``lms_catalog_service_user`` is not a staff user, then make it a staff user in your LMS User model. This step should be performed just once::
 
     tutor local run lms ./manage.py lms create_catalog_integrations --enabled \
         --internal_api_url="" \
         --service_username=lms_catalog_service_user
 
-Then::
+Then run the below command, this command will cause errors every time as it tries to cache programs from all sites that are added to your LMS sites model::
 
     tutor local run lms ./manage.py lms cache_programs
 
+If you don't want the errors, then make use of an extra argumet to the command .i.e. ``--domain``. This argument will be equal to ``local.overhang.io`` if you are running tutor local and ``local.overhang.io:8000`` if you are running tutor dev::
+    tutor local run lms ./manage.py lms cache_programs --domain="local.overhang.io"
+    or
+    tutor dev run lms ./manage.py lms cache_programs --domain="local.overhang.io:8000"
+
 This last step should be performed every time you create new or make changes to existing programs.
 
 Install extra requirements

setup.py

@@ -34,8 +34,8 @@ setup(
     long_description_content_type="text/x-rst",
     packages=find_packages(exclude=["tests*"]),
     include_package_data=True,
-    install_requires=["tutor>=15.0.0,<16.0.0"],
-    python_requires=">=3.7",
+    install_requires=["tutor>=16.0.0,<17.0.0"],
+    python_requires=">=3.8",
     entry_points={"tutor.plugin.v1": ["discovery = tutordiscovery.plugin"]},
     classifiers=[
         "Development Status :: 5 - Production/Stable",
@@ -43,9 +43,9 @@ setup(
         "License :: OSI Approved :: GNU Affero General Public License v3",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
 )

tutordiscovery/__about__.py

@@ -1,4 +1,4 @@
-__version__ = "15.0.0"
+__version__ = "16.0.0"
 
 # Handle version suffix for nightly, just like tutor core.
 __version_suffix__ = ""

tutordiscovery/patches/local-docker-compose-jobs-services

@@ -4,4 +4,6 @@ discovery-job:
     DEFAULT_PARTNER_CODE: openedx
   volumes:
     - ../plugins/discovery/apps/settings/tutor:/openedx/discovery/course_discovery/settings/tutor:ro
-  depends_on: {{ [("lms", RUN_LMS), ("mysql", RUN_MYSQL)]|list_if }}
+  depends_on: 
+    - lms
+    {% if RUN_MYSQL %}- mysql{% endif %}

tutordiscovery/patches/local-docker-compose-services

@@ -5,4 +5,7 @@ discovery:
   restart: unless-stopped
   volumes:
     - ../plugins/discovery/apps/settings/tutor:/openedx/discovery/course_discovery/settings/tutor:ro
-  depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("lms", RUN_LMS), ("mysql", RUN_MYSQL)]|list_if }}
+  depends_on: 
+    - lms
+    {% if RUN_MYSQL %}- mysql{% endif %}
+    {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %}

tutordiscovery/plugin.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+
 from glob import glob
 import os
 import pkg_resources
+import typing as t
 
 from tutor import hooks as tutor_hooks
 
@@ -74,11 +77,26 @@ tutor_hooks.Filters.IMAGES_PUSH.add_item(
     )
 )
 
+
+REPO_NAME = "course-discovery"
+APP_NAME = "discovery"
+
+
 # Automount /openedx/discovery folder from the container
 @tutor_hooks.Filters.COMPOSE_MOUNTS.add()
 def _mount_course_discovery(mounts, name):
-    if name == "course-discovery":
-        mounts.append(("discovery", "/openedx/discovery"))
+    if name == REPO_NAME:
+        mounts.append((APP_NAME, "/openedx/discovery"))
+    return mounts
+
+
+# Bind-mount repo at build-time, both for prod and dev images
+@tutor_hooks.Filters.IMAGES_BUILD_MOUNTS.add()
+def _mount_course_discovery_on_build(mounts: list[tuple[str, str]], host_path: str) -> list[tuple[str, str]]:
+    path_basename = os.path.basename(host_path)
+    if path_basename == REPO_NAME:
+        mounts.append((APP_NAME, f"{APP_NAME}-src"))
+        mounts.append((f"{APP_NAME}-dev", f"{APP_NAME}-src"))
     return mounts
 
 
@@ -115,3 +133,12 @@ tutor_hooks.Filters.CONFIG_UNIQUE.add_items(
 tutor_hooks.Filters.CONFIG_OVERRIDES.add_items(
     list(config.get("overrides", {}).items())
 )
+
+
+@tutor_hooks.Filters.APP_PUBLIC_HOSTS.add()
+def _print_discovery_public_hosts(hosts: list[str], context_name: t.Literal["local", "dev"]) -> list[str]:
+    if context_name == "dev":
+        hosts += ["{{ DISCOVERY_HOST }}:8381"]
+    else:
+        hosts += ["{{ DISCOVERY_HOST }}"]
+    return hosts

tutordiscovery/templates/discovery/build/discovery/Dockerfile

@@ -1,17 +1,23 @@
-FROM docker.io/ubuntu:20.04
+{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %}
+###### Minimal image with base system requirements for most stages
+FROM docker.io/ubuntu:20.04 as minimal
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update && \
-  apt install -y curl git-core language-pack-en python3 python3-pip python3-venv \
-  build-essential libcairo2 libffi-dev libmysqlclient-dev libxml2-dev libxslt-dev libjpeg-dev libssl-dev
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \ 
+    apt update && \
+    apt install -y curl git-core language-pack-en python3 python3-pip python3-venv \
+    build-essential libcairo2 libffi-dev libmysqlclient-dev libxml2-dev libxslt-dev libjpeg-dev libssl-dev
 ENV LC_ALL en_US.UTF-8
 
 ARG APP_USER_ID=1000
+RUN if [ "$APP_USER_ID" = 0 ]; then echo "app user may not be root" && false; fi
 RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER_ID} app
 USER ${APP_USER_ID}
 
+###### Git-clone course-discovery repo
 ARG DISCOVERY_REPOSITORY=https://github.com/edx/course-discovery.git
-ARG DISCOVERY_VERSION={{ OPENEDX_COMMON_VERSION }}
+ARG DISCOVERY_VERSION='{{ OPENEDX_COMMON_VERSION }}'
 RUN mkdir -p /openedx/discovery && \
     git clone $DISCOVERY_REPOSITORY --branch $DISCOVERY_VERSION --depth 1 /openedx/discovery
 WORKDIR /openedx/discovery
@@ -23,13 +29,15 @@ ENV DISCOVERY_CFG /openedx/config.yml
 # Install python venv
 RUN python3 -m venv ../venv/
 ENV PATH "/openedx/venv/bin:$PATH"
-# https://pypi.org/project/setuptools/
-# https://pypi.org/project/pip/
-# https://pypi.org/project/wheel/
-RUN pip install setuptools==65.5.1 pip==22.3.1 wheel==0.38.4
+
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+    # https://pypi.org/project/setuptools/
+    # https://pypi.org/project/pip/
+    # https://pypi.org/project/wheel/
+    setuptools==67.7.2 pip==23.1.2. wheel==0.40.0
 
 # Install a recent version of nodejs
-RUN pip install nodeenv
+RUN pip install nodeenv==1.7.0
 # nodejs version picked from https://github.com/openedx/course-discovery/blob/master/Dockerfile
 RUN nodeenv /openedx/nodeenv --node=16.14.2 --prebuilt
 ENV PATH /openedx/nodeenv/bin:${PATH}
@@ -37,21 +45,22 @@ ENV PATH /openedx/nodeenv/bin:${PATH}
 # Install python and nodejs requirements
 # This is identical to "make production-requirements" but it was split in multiple
 # instructions to benefit from docker image caching
-RUN pip install -r requirements.txt
-{% for extra_requirements in DISCOVERY_EXTRA_PIP_REQUIREMENTS %}RUN pip install '{{ extra_requirements }}'
+# Install base requirements
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements.txt
+{% for extra_requirement in DISCOVERY_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}'
 {% endfor %}
 
-ARG NPM_REGISTRY={{ NPM_REGISTRY }}
-RUN npm install --verbose --registry=$NPM_REGISTRY --production
-RUN ./node_modules/.bin/bower install --allow-root --production
+# Install npm, bower requirements
+ARG NPM_REGISTRY='{{ NPM_REGISTRY }}'
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.npm/,sharing=shared,uid=${APP_USER_ID} {% endif %}npm clean-install --verbose --no-audit --registry=$NPM_REGISTRY --production
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/bower,sharing=shared,uid=${APP_USER_ID} {% endif %}./node_modules/.bin/bower install --allow-root --production
 
-# Install django-redis for using redis as a django cache
-# https://pypi.org/project/django-redis/
-RUN pip install django-redis==5.2.0
-
-# Install uwsgi
-# https://pypi.org/project/uWSGI/
-RUN pip install uwsgi==2.0.21
+# Install extra requirements
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+    # Use redis as a django cache https://pypi.org/project/django-redis/
+    django-redis==5.2.0 \
+    # uwsgi server https://pypi.org/project/uWSGI/
+    uwsgi==2.0.21
 
 # Collect static assets
 COPY --chown=app:app assets.py ./course_discovery/settings/assets.py