remove hard-coded uid 1000

Linked Epic: https://github.com/overhangio/tutor/issues/1251

Some new rules that have been added are:

I: sort imports
N: check for pep8-naming standards
W292: check for missing extra line at EOF

Some of our lines, particularly in config, break the 88 character line length limit. For that, we add a # noqa: E501 to let ruff know to ignore the rule for that line

.hatch_build.py

@@ -18,5 +18,5 @@ def load_about() -> dict[str, str]:
     with open(
         os.path.join(HERE, "tutorcairn", "__about__.py"), "rt", encoding="utf-8"
     ) as f:
-        exec(f.read(), about)  # pylint: disable=exec-used
+        exec(f.read(), about)
     return about

CHANGELOG.md

@@ -19,6 +19,27 @@ instructions, because git commits are used to generate release notes:
 
 <!-- scriv-insert-here -->
 
+<a id='changelog-20.0.0'></a>
+## v20.0.0 (2025-06-05)
+
+- [Improvement] Test python package distribution build when running make test. (by @Danyal-Faheem)
+
+- 💥[Feature] Upgrade to teak. (by @Danyal-Faheem)
+
+<a id='changelog-19.0.4'></a>
+## v19.0.4 (2025-04-10)
+
+- [Bugfix] Remove deprecated filter_scopes from import_dashboard script. (by @Danyal-Faheem)
+
+<a id='changelog-19.0.3'></a>
+## v19.0.3 (2025-04-03)
+
+- [Bugfix] Migrate to native filters instead of filter box charts on default course overview dashboard which have been completely deprecated as of superset 4.0.0 release. (by @Danyal-Faheem)
+
+- [Improvement] Enable Alerts and Reports in superset. (by @Danyal-Faheem)
+
+- [Improvement] Make superset ports consistent on local and dev by running it on 2247 in both environments. (by @Danyal-Faheem)
+
 <a id='changelog-19.0.2'></a>
 ## v19.0.2 (2025-03-14)
 

Makefile

@@ -1,25 +1,30 @@
 .DEFAULT_GOAL := help
 .PHONY: docs
 SRC_DIRS = ./tutorcairn
-BLACK_OPTS = --exclude templates ${SRC_DIRS}
 
 # Warning: These checks are not necessarily run on every PR.
-test: test-lint test-types test-format  # Run some static checks.
+test: test-lint test-format test-types test-pythonpackage  # Run some static checks.
 
 test-format: ## Run code formatting tests
-	black --check --diff $(BLACK_OPTS)
+	ruff format --check --diff ${SRC_DIRS}
 
 test-lint: ## Run code linting tests
-	pylint --errors-only --enable=unused-import,unused-argument --ignore=templates --ignore=docs/_ext ${SRC_DIRS}
+	ruff check ${SRC_DIRS}
 
 test-types: ## Run type checks.
 	mypy --exclude=templates --ignore-missing-imports --implicit-reexport --strict ${SRC_DIRS}
 
-format: ## Format code automatically
-	black $(BLACK_OPTS)
+build-pythonpackage: ## Build the "tutor-cairn" python package for upload to pypi
+	python -m build --sdist
 
-isort: ##  Sort imports. This target is not mandatory because the output may be incompatible with black formatting. Provided for convenience purposes.
-	isort --skip=templates ${SRC_DIRS}
+test-pythonpackage: build-pythonpackage ## Test that package can be uploaded to pypi
+	twine check dist/tutor_cairn-$(shell make version).tar.gz
+
+format: ## Format code 
+	ruff format ${SRC_DIRS}
+
+fix-lint: ## Fix lint errors automatically
+	ruff check --fix ${SRC_DIRS}
 
 changelog-entry: ## Create a new changelog entry.
 	scriv create
@@ -27,6 +32,9 @@ changelog-entry: ## Create a new changelog entry.
 changelog: ## Collect changelog entries in the CHANGELOG.md file.
 	scriv collect
 
+version: ## Print the current tutor-cairn version
+	@python -c 'import io, os; about = {}; exec(io.open(os.path.join("tutorcairn", "__about__.py"), "rt", encoding="utf-8").read(), about); print(about["__version__"])'
+
 ESCAPE = 
 help: ## Print this help
 	@grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \

README.rst

@@ -20,15 +20,15 @@ We created a tool to help you answer all these questions. Cairn is a Tutor plugi
 Cairn vs alternatives
 ---------------------
 
-========================================== =====  ===================================================================================  ===================================================
-List of features                           Cairn  `Open edX Insights <https://edx.readthedocs.io/projects/edx-insights/en/latest/>`__  `Figures <https://github.com/appsembler/figures>`__
-========================================== =====  ===================================================================================  ===================================================
-Event aggregation                            ✅      ✅                                                                                    ❌
-Real-time data                               ✅      ❌                                                                                    ✅
-Easy to install                              ✅      ❌                                                                                    ✅
-Custom queries and dashboards                ✅      ❌                                                                                    ❌
-Works with the latest Open edX versions      ✅      ✅                                                                                    ❌
-========================================== =====  ===================================================================================  ===================================================
+========================================== ===== ====================================================================================  ===================================================================================================================  ===================================================
+List of features                           Cairn `Aspects <https://docs.openedx.org/projects/openedx-aspects/en/latest/index.html>`__  `Open edX Insights <https://docs.openedx.org/en/latest/site_ops/install_configure_run_guide/insights/index.html>`__  `Figures <https://github.com/appsembler/figures>`__
+========================================== ===== ====================================================================================  ===================================================================================================================  ===================================================
+Event aggregation                            ✅     ✅                                                                                    ✅                                                                                                                    ❌
+Real-time data                               ✅     ✅                                                                                    ❌                                                                                                                    ✅
+Easy to install                              ✅     ✅                                                                                    ❌                                                                                                                    ✅
+Custom queries and dashboards                ✅     ✅                                                                                    ❌                                                                                                                    ❌
+Works with the latest Open edX versions      ✅     ✅                                                                                    ✅                                                                                                                    ❌
+========================================== ===== ====================================================================================  ===================================================================================================================  ===================================================
 
 
 How does Cairn work?
@@ -262,7 +262,7 @@ Then apply changes with::
 Troubleshooting
 ---------------
 
-This Tutor plugin is maintained by Danyal Faheem from `Edly <https://edly.io>`__. Community support is available from the official `Open edX forum <https://discuss.openedx.org>`__. Do you need help with this plugin? See the `troubleshooting <https://docs.tutor.edly.io/troubleshooting.html>`__ section from the Tutor documentation.
+This Tutor plugin is maintained by Eemaan Amir from `Edly <https://edly.io>`__. Community support is available from the official `Open edX forum <https://discuss.openedx.org>`__. Do you need help with this plugin? See the `troubleshooting <https://docs.tutor.edly.io/troubleshooting.html>`__ section from the Tutor documentation.
 
 License
 -------

changelog.d/20250723_121728_danyal.faheem_migrate_ruff.md

@@ -0,0 +1 @@
+- [Improvement] Migrate from pylint and black to ruff. (by @Danyal-Faheem)

pyproject.toml

@@ -8,7 +8,7 @@ authors = [
   {name = "Edly"}, {email = "hello@edly.io"},
 ]
 maintainers = [
-  {name = "Danyal Faheem"}, {email = "danyal.faheem@arbisoft.com"}
+  {name = "Eemaan Amir"}, {email = "eemaan.amir@arbisoft.com"}
 ]
 description = "Scalable, real-time analytics for Open edX"
 readme = {file = "README.rst", content-type = "text/x-rst"}
@@ -25,16 +25,15 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-  "tutor>=19.0.0,<20.0.0",
+  "tutor>=20.0.0,<21.0.0",
 ]
 # these fields will be set by hatch_build.py
 dynamic = ["version"]
 
 [project.optional-dependencies]
 dev = [
-  "tutor[dev]>=19.0.0,<20.0.0", 
-  "pylint", 
-  "black"
+  "tutor[dev]>=20.0.0,<21.0.0",
+  "ruff",
 ]
 
 [project.entry-points."tutor.plugin.v1"]
@@ -65,3 +64,19 @@ exclude = ["tests*"]
 
 [tool.hatch.build.targets.wheel]
 packages = ["tutorcairn"]
+
+[tool.ruff]
+exclude = ["templates", "docs/_ext"]
+
+[tool.ruff.lint]
+# E: pycodestyle errors
+# I: isort
+# N: pep8-naming
+select = ["E", "I", "N"]
+
+# F401: unused-import
+# F841: unused-variable
+# W292: missing-newline-at-end-of-file
+extend-select = ["F401", "F841", "W292"]
+
+[tool.ruff.format]

tutorcairn/__about__.py

@@ -1 +1 @@
-__version__ = "19.0.2"
+__version__ = "20.0.0"

tutorcairn/patches/caddyfile

@@ -1,4 +1,4 @@
 # Cairn
 {{ CAIRN_HOST }}{$default_site_port} {
-    reverse_proxy cairn-superset:8000
+    reverse_proxy cairn-superset:2247
 }

tutorcairn/patches/k8s-deployments

@@ -130,9 +130,9 @@ spec:
         app.kubernetes.io/name: cairn-clickhouse
     spec:
       securityContext:
-        runAsUser: 1000
-        runAsGroup: 1000
-        fsGroup: 1000
+        runAsUser: {{ APP_USER_ID }}
+        runAsGroup: {{ APP_USER_ID }}
+        fsGroup: {{ APP_USER_ID }}
         fsGroupChangePolicy: "OnRootMismatch"
       containers:
         - name: cairn-clickhouse
@@ -183,8 +183,8 @@ spec:
         app.kubernetes.io/name: cairn-superset
     spec:
       securityContext:
-        runAsUser: 1000
-        runAsGroup: 1000
+        runAsUser: {{ APP_USER_ID }}
+        runAsGroup: {{ APP_USER_ID }}
       containers:
         - name: cairn-superset
           image: {{ CAIRN_SUPERSET_DOCKER_IMAGE }}
@@ -227,8 +227,8 @@ spec:
         app.kubernetes.io/name: cairn-superset-worker
     spec:
       securityContext:
-        runAsUser: 1000
-        runAsGroup: 1000
+        runAsUser: {{ APP_USER_ID }}
+        runAsGroup: {{ APP_USER_ID }}
       containers:
         - name: cairn-superset-worker
           image: {{ CAIRN_SUPERSET_DOCKER_IMAGE }}
@@ -261,8 +261,8 @@ spec:
         app.kubernetes.io/name: cairn-superset-worker-beat
     spec:
       securityContext:
-        runAsUser: 1000
-        runAsGroup: 1000
+        runAsUser: {{ APP_USER_ID }}
+        runAsGroup: {{ APP_USER_ID }}
       containers:
         - name: cairn-superset-worker-beat
           image: {{ CAIRN_SUPERSET_DOCKER_IMAGE }}

tutorcairn/patches/k8s-jobs

@@ -10,8 +10,8 @@ spec:
     spec:
       restartPolicy: Never
       securityContext:
-        runAsUser: 1000
-        runAsGroup: 1000
+        runAsUser: {{ APP_USER_ID }}
+        runAsGroup: {{ APP_USER_ID }}
       containers:
         - name: cairn-clickhouse
           image: {{ CAIRN_CLICKHOUSE_DOCKER_IMAGE }}

tutorcairn/patches/k8s-services

@@ -39,7 +39,7 @@ metadata:
 spec:
   type: NodePort
   ports:
-    - port: 8000
+    - port: 2247
       protocol: TCP
   selector:
     app.kubernetes.io/name: cairn-superset

tutorcairn/patches/local-docker-compose-jobs-services

@@ -1,6 +1,6 @@
 cairn-clickhouse-job:
     image: {{ CAIRN_CLICKHOUSE_DOCKER_IMAGE }}
-    user: "1000:1000"
+    user: "{{ APP_USER_ID }}:{{ APP_USER_ID }}"
     depends_on: {{ [("cairn-clickhouse", CAIRN_RUN_CLICKHOUSE)]|list_if }}
     volumes:
         - ../plugins/cairn/apps/clickhouse/auth.json:/scripts/clickhouse-auth.json:ro

tutorcairn/patches/local-docker-compose-permissions-command

@@ -1,2 +1,2 @@
-{% if CAIRN_RUN_CLICKHOUSE %}setowner 1000 /data/cairn-clickhouse{% endif %}
+{% if CAIRN_RUN_CLICKHOUSE %}setowner {{ APP_USER_ID }} /data/cairn-clickhouse{% endif %}
 {% if CAIRN_RUN_POSTGRESQL %}setowner 70 /data/cairn-postgresql{% endif %}

tutorcairn/patches/local-docker-compose-services

@@ -19,7 +19,7 @@ cairn-clickhouse:
         - ../../data/cairn/clickhouse:/var/lib/clickhouse
         - ../plugins/cairn/apps/clickhouse/users.d/cairn.xml:/etc/clickhouse-server/users.d/cairn.xml:ro
         - ../plugins/cairn/apps/clickhouse/auth.json:/scripts/clickhouse-auth.json:ro
-    user: "1000:1000"
+    user: "{{ APP_USER_ID }}:{{ APP_USER_ID }}"
     environment:
         CLICKHOUSE_DO_NOT_CHOWN: "1"
     ulimits:

tutorcairn/plugin.py

@@ -25,7 +25,7 @@ config: t.Dict[str, t.Dict[str, t.Any]] = {
         "HOST": "data.{{ LMS_HOST }}",
         # Clickhouse
         "RUN_CLICKHOUSE": True,
-        "CLICKHOUSE_DOCKER_IMAGE": "{{ DOCKER_REGISTRY }}overhangio/cairn-clickhouse:{{ CAIRN_VERSION }}",
+        "CLICKHOUSE_DOCKER_IMAGE": "{{ DOCKER_REGISTRY }}overhangio/cairn-clickhouse:{{ CAIRN_VERSION }}",  # noqa: E501
         "CLICKHOUSE_HOST": "cairn-clickhouse",
         "CLICKHOUSE_HTTP_PORT": 8123,
         "CLICKHOUSE_HTTP_SCHEME": "http",
@@ -38,7 +38,7 @@ config: t.Dict[str, t.Dict[str, t.Any]] = {
         "POSTGRESQL_PORT": "5432",
         "POSTGRESQL_DATABASE": "superset",
         "POSTGRESQL_USERNAME": "superset",
-        "SUPERSET_DOCKER_IMAGE": "{{ DOCKER_REGISTRY }}overhangio/cairn-superset:{{ CAIRN_VERSION }}",
+        "SUPERSET_DOCKER_IMAGE": "{{ DOCKER_REGISTRY }}overhangio/cairn-superset:{{ CAIRN_VERSION }}",  # noqa: E501
         "SUPERSET_LANGUAGE_CODE": "{{ LANGUAGE_CODE[:2] }}",
         # SSO
         "ENABLE_SSO": True,
@@ -142,7 +142,7 @@ def _print_superset_host(
 @click.option(
     "-p",
     "--password",
-    help="Specify password from the command line. If undefined, no password will be set. (Ignored with SSO)",
+    help="Specify password from the command line. If undefined, no password will be set. (Ignored with SSO)",  # noqa: E501
     hide_input=True,
 )
 @click.option(
@@ -181,7 +181,7 @@ def create_user_command(
     if bootstrap_dashboards:
         yield (
             "cairn-superset",
-            f"python ./superset/cairn/ctl.py bootstrap-dashboards {username} /app/bootstrap/courseoverview.json",
+            f"python ./superset/cairn/ctl.py bootstrap-dashboards {username} /app/bootstrap/courseoverview.json",  # noqa: E501
         )
 
 

tutorcairn/templates/cairn/apps/openedx/scripts/importcoursedata.py

@@ -8,9 +8,9 @@ import requests
 # https://mysqlclient.readthedocs.io/user_guide.html#mysql-c-api-function-mapping
 from MySQLdb._mysql import escape_string as sql_escape_string
 
-import lms.startup
+import django
 
-lms.startup.run()
+django.setup()
 
 from lms.djangoapps.courseware.courses import get_course
 from xmodule.modulestore.django import modulestore

tutorcairn/templates/cairn/apps/superset/superset_config.py

@@ -126,9 +126,9 @@ AUTH_ROLES_SYNC_AT_LOGIN = {{ CAIRN_AUTH_ROLES_SYNC_AT_LOGIN }}
 AUTH_USER_REGISTRATION = True
 {% endif %}
 
-class CeleryConfig:  # pylint: disable=too-few-public-methods
+class CeleryConfig:
     BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
-    CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks")
+    CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks","superset.tasks.thumbnails",)
     CELERYD_LOG_LEVEL = "DEBUG"
     CELERYD_PREFETCH_MULTIPLIER = 1
     CELERY_ACKS_LATE = False
@@ -142,10 +142,6 @@ class CeleryConfig:  # pylint: disable=too-few-public-methods
         },
     }
     CELERYBEAT_SCHEDULE = {
-        "email_reports.schedule_hourly": {
-            "task": "email_reports.schedule_hourly",
-            "schedule": crontab(minute=1, hour="*"),
-        },
         "reports.scheduler": {
             "task": "reports.scheduler",
             "schedule": crontab(minute="*", hour="*"),
@@ -159,13 +155,32 @@ class CeleryConfig:  # pylint: disable=too-few-public-methods
 
 CELERY_CONFIG = CeleryConfig
 
+# Email configuration
+SMTP_HOST = "{{ SMTP_HOST }}"
+SMTP_PORT = {{ SMTP_PORT }}
+SMTP_STARTTLS = {{ SMTP_USE_TLS }}
+SMTP_SSL = {{ SMTP_USE_SSL }}
+SMTP_USER = "{{ SMTP_USERNAME }}" # use the empty string "" if using an unauthenticated SMTP server
+SMTP_PASSWORD = "{{ SMTP_PASSWORD }}" # use the empty string "" if using an unauthenticated SMTP server
+SMTP_MAIL_FROM = "{{ CONTACT_EMAIL }}"
+EMAIL_REPORTS_SUBJECT_PREFIX = "[{{ PLATFORM_NAME }}] "
+
+ALERT_REPORTS_NOTIFICATION_DRY_RUN = False
+WEBDRIVER_BASEURL = "http://cairn-superset:2247/"
+# The base URL for the email report hyperlinks.
+WEBDRIVER_BASEURL_USER_FRIENDLY = "{{ CAIRN_HOST }}"
+
 # Avoid duplicate logging because of propagation to root logger
 logging.getLogger("superset").propagate = False
 
 # https://github.com/apache/superset/blob/master/RESOURCES/FEATURE_FLAGS.md
 FEATURE_FLAGS = {
     # Enable dashboard embedding
-    "EMBEDDED_SUPERSET": True
+    "EMBEDDED_SUPERSET": True,
+    # This feature is stable but known to have bugs from time to time
+    # Alerts and Reports also do not work on arm64 base images
+    "ALERT_REPORTS": True,
+    
 }
 
 ENABLE_CORS=True

tutorcairn/templates/cairn/build/cairn-superset/Dockerfile

@@ -7,16 +7,34 @@ FROM docker.io/apache/superset:4.0.0
 
 USER root
 
+# https://github.com/apache/superset/blob/4.0.0/Dockerfile#L109-L110
+# Extracted from https://superset.apache.org/docs/installation/alerts-reports/#custom-dockerfile
+ARG GECKODRIVER_VERSION=v0.33.0 \
+    FIREFOX_VERSION=117.0.1
+
+RUN apt-get update -qq \
+    && apt-get install -yqq --no-install-recommends \
+        libnss3 \
+        libdbus-glib-1-2 \
+        libgtk-3-0 \
+        libx11-xcb1 \
+        libasound2 \
+        libxtst6 \
+        wget \
+        pkg-config \
+    # Install GeckoDriver WebDriver
+    && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
+    # Install Firefox
+    && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
+    && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
+    && apt-get autoremove -yqq --purge wget && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/*
+
 # https://pypi.org/project/clickhouse-driver/
 # https://pypi.org/project/mysqlclient/
 # https://pypi.org/project/clickhouse-connect/
 # https://pypi.org/project/clickhouse-sqlalchemy/
 # https://pypi.org/project/Authlib/
 # We preserve the clickhouse-sqlalchemy package to keep backward compatibility with existing dashboards
-RUN apt-get update \
-    && apt-get install -y \
-        pkg-config
-
 RUN --mount=type=cache,target=/root/.cache/pip,sharing=shared pip install \
     clickhouse-driver==0.2.7 \
     mysqlclient==2.2.4 \
@@ -30,11 +48,11 @@ USER superset
 COPY --chown=superset:superset ./cairn /app/superset/cairn
 
 # This is required to have a proper healthcheck
-ENV SUPERSET_PORT=8000
+ENV SUPERSET_PORT=2247
 
 ENTRYPOINT []
 CMD gunicorn \
-    --bind  "0.0.0.0:8000" \
+    --bind  "0.0.0.0:2247" \
     --access-logfile '-' \
     --error-logfile '-' \
     --workers 2 \

tutorcairn/templates/cairn/build/cairn-superset/cairn/ctl.py

@@ -229,20 +229,6 @@ def import_dashboard(data, user, database):
 
         # Update filter mapping
         metadata = json.loads(dashboard.json_metadata)
-        new_filter_scopes = {}
-        for old_slice_id, filter_scope in metadata["filter_scopes"].items():
-            new_filter_scope = {}
-            for filter_name, properties in filter_scope.items():
-                properties["immune"] = [
-                    old_to_new_slice_id_map[old_immune_id]
-                    for old_immune_id in properties["immune"]
-                ]
-                new_filter_scope[filter_name] = properties
-            # Note that filter scope keys are str, not int
-            new_filter_scopes[
-                str(old_to_new_slice_id_map[int(old_slice_id)])
-            ] = new_filter_scope
-        metadata["filter_scopes"] = new_filter_scopes
         dashboard.json_metadata = json.dumps(metadata)
 
         # Load dashboard