🌅

.gitignore

@@ -0,0 +1,7 @@
+.*.swp
+!.gitignore
+TODO
+__pycache__
+*.egg-info/
+/build/
+/dist/

MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include tutorvision/patches *
+recursive-include tutorvision/templates *

README.rst

@@ -0,0 +1,52 @@
+vision plugin for `Tutor <https://docs.tutor.overhang.io>`__
+===================================================================================
+
+TODO:
+- Collect data with Vector
+    - Collect tracking logs
+    - Collect nginx logs
+    - Send logs to clickhouse
+    - Make it optional to mount /var/run/docker.sock
+    - adjust vector verbosity
+    - log everything to file instead of console? -> tmp volume
+- Provision clickhouse
+    - make database name a tutor config
+    - make clickhouse host a tutor config
+    - specify TTL for tables?
+- Expose data with redash
+- Provision dashboards
+- Utility tools for authentication
+- Kubernetes compatibility
+- Sweet readme
+
+Installation
+------------
+
+::
+
+    pip install git+https://github.com/overhangio/tutor-vision
+
+Usage
+-----
+
+::
+
+    tutor plugins enable vision
+    tutor local quickstart
+
+To access the analytics frontend, open http(s)://vision.<YOUR_LMS_HOST> in your browser. When running locally, this will be http://vision.local.overhang.io. The email address and password required for logging in are::
+
+    tutor config printvalue VISION_REDASH_ROOT_EMAIL
+    tutor config printvalue VISION_REDASH_ROOT_PASSWORD
+
+Development
+-----------
+
+To reload Vector configuration after changes to vector.toml, run::
+
+    tutor config save && tutor local exec vision-vector sh kill -s HUP
+
+License
+-------
+
+This software is licensed under the terms of the AGPLv3.

setup.py

@@ -0,0 +1,52 @@
+import io
+import os
+from setuptools import setup, find_packages
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+with io.open(os.path.join(here, "README.rst"), "rt", encoding="utf8") as f:
+    readme = f.read()
+
+about = {}
+with io.open(
+    os.path.join(here, "tutorvision", "__about__.py"),
+    "rt",
+    encoding="utf-8",
+) as f:
+    exec(f.read(), about)
+
+setup(
+    name="tutor-vision",
+    version=about["__version__"],
+    url="https://github.com/overhangio/tutor-vision",
+    project_urls={
+        "Code": "https://github.com/overhangio/tutor-vision",
+        "Issue tracker": "https://github.com/overhangio/tutor-vision/issues",
+    },
+    license="AGPLv3",
+    author="Overhang.IO",
+    description="vision plugin for Tutor",
+    long_description=readme,
+    packages=find_packages(exclude=["tests*"]),
+    include_package_data=True,
+    python_requires=">=3.5",
+    install_requires=["tutor-openedx"],
+    entry_points={
+        "tutor.plugin.v0": [
+            "vision = tutorvision.plugin"
+        ]
+    },
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: GNU Affero General Public License v3",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+    ],
+)

tutorvision/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

tutorvision/patches/caddyfile

@@ -0,0 +1,4 @@
+# Vision
+{{ VISION_REDASH_HOST }}{% if not ENABLE_HTTPS %}:80{% endif %} {
+    reverse_proxy nginx:80
+}

tutorvision/patches/local-docker-compose-jobs-services

@@ -0,0 +1,10 @@
+vision-clickhouse-job:
+    image: {{ VISION_CLICKHOUSE_DOCKER_IMAGE }}
+    depends_on: {{ [("vision-clickhouse", VISION_RUN_CLICKHOUSE)]|list_if }}
+vision-redash-job:
+    image: {{ VISION_REDASH_DOCKER_IMAGE }}
+    command: create_db
+    env_file: ../plugins/vision/apps/redash/env
+    depends_on:
+        - vision-postgres
+        - vision-redis

tutorvision/patches/local-docker-compose-services

@@ -0,0 +1,81 @@
+####### vision plugin
+
+# log collection
+vision-vector:
+    image: docker.io/timberio/vector:0.11.X-alpine
+    volumes:
+        - ../plugins/vision/apps/vector/vector.toml:/etc/vector/vector.toml:ro
+        - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+        - DOCKER_HOST=/var/run/docker.sock
+    restart: unless-stopped
+
+{% if VISION_RUN_CLICKHOUSE %}
+# log storage
+vision-clickhouse:
+    image: {{ VISION_CLICKHOUSE_DOCKER_IMAGE }}
+    volumes:
+        - ../../data/vision/clickhouse:/var/lib/clickhouse
+    ulimits:
+        nofile:
+            soft: 262144
+            hard: 262144
+    restart: unless-stopped
+{% endif %}
+
+# frontend
+vision-redash-server:
+    image: {{ VISION_REDASH_DOCKER_IMAGE }}
+    command: server
+    env_file: ../plugins/vision/apps/redash/env
+    environment:
+        REDASH_WEB_WORKERS: 4
+    restart: unless-stopped
+    depends_on:
+        - vision-postgres
+        - vision-redis
+vision-redash-scheduler:
+    image: {{ VISION_REDASH_DOCKER_IMAGE }}
+    command: scheduler
+    env_file: ../plugins/vision/apps/redash/env
+    environment:
+        QUEUES: "celery"
+        WORKERS_COUNT: 1
+    restart: unless-stopped
+    depends_on:
+        - vision-postgres
+        - vision-redis
+vision-redash-scheduled-worker:
+    image: {{ VISION_REDASH_DOCKER_IMAGE }}
+    command: worker
+    env_file: ../plugins/vision/apps/redash/env
+    environment:
+        QUEUES: "scheduled_queries,schemas"
+        WORKERS_COUNT: 1
+    restart: unless-stopped
+    depends_on:
+        - vision-postgres
+        - vision-redis
+vision-redash-adhoc-worker:
+    image: {{ VISION_REDASH_DOCKER_IMAGE }}
+    command: worker
+    env_file: ../plugins/vision/apps/redash/env
+    environment:
+        QUEUES: "queries"
+        WORKERS_COUNT: 2
+    restart: unless-stopped
+    depends_on:
+        - vision-postgres
+        - vision-redis
+vision-redis:
+    image: docker.io/redis:5.0-alpine
+    restart: unless-stopped
+vision-postgres:
+    image: docker.io/postgres:9.6-alpine
+    environment:
+        POSTGRES_USER: "{{ VISION_REDASH_POSTGRESQL_USER }}"
+        POSTGRES_PASSWORD: "{{ VISION_REDASH_POSTGRESQL_PASSWORD }}"
+        POSTGRES_DB: "{{ VISION_REDASH_POSTGRESQL_DB }}"
+    volumes:
+        - ../../data/vision/redash/postgres:/var/lib/postgresql/data
+    restart: unless-stopped

tutorvision/patches/nginx-extra

@@ -0,0 +1,19 @@
+# Vision
+upstream vision-backend {
+    server vision-redash-server:5000 fail_timeout=0;
+}
+server {
+  listen 80;
+  server_name {{ VISION_REDASH_HOST }};
+
+  # Disables server version feedback on pages and in headers
+  server_tokens off;
+
+  client_max_body_size 10m;
+
+  location / {
+    proxy_set_header Host $http_host;
+    proxy_redirect off;
+    proxy_pass http://vision-backend;
+  }
+}

tutorvision/plugin.py

@@ -0,0 +1,43 @@
+from glob import glob
+import os
+
+from .__about__ import __version__
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+
+templates = os.path.join(HERE, "templates")
+
+config = {
+    "add": {
+        "REDASH_POSTGRESQL_PASSWORD": "{{ 20|random_string }}",
+        "REDASH_COOKIE_SECRET": "{{ 20|random_string }}",
+        "REDASH_ROOT_PASSWORD": "{{ 20|random_string }}",
+        "REDASH_SECRET_KEY": "{{ 20|random_string }}",
+    },
+    "defaults": {
+        "CLICKHOUSE_DOCKER_IMAGE": "docker.io/yandex/clickhouse-server:20.8.14.4",
+        "RUN_CLICKHOUSE": True,
+        "CLICKHOUSE_HOST": "vision-clickhouse",
+        "CLICKHOUSE_HTTP_PORT": 8123,
+        "CLICKHOUSE_PORT": 9000,
+        "CLICKHOUSE_DATABASE": "openedx_{{ ID }}",
+        "REDASH_DOCKER_IMAGE": "docker.io/redash/redash:8.0.0.b32245",
+        "REDASH_POSTGRESQL_USER": "redash",
+        "REDASH_POSTGRESQL_DB": "redash",
+        "REDASH_HOST": "vision.{{ LMS_HOST }}",
+        "REDASH_ROOT_USERNAME": "admin",
+        "REDASH_ROOT_EMAIL": "{{ CONTACT_EMAIL }}",
+    },
+}
+
+hooks = {"init": ["vision-clickhouse", "vision-redash"]}
+
+
+def patches():
+    all_patches = {}
+    for path in glob(os.path.join(HERE, "patches", "*")):
+        with open(path) as patch_file:
+            name = os.path.basename(path)
+            content = patch_file.read()
+            all_patches[name] = content
+    return all_patches

tutorvision/templates/vision/apps/redash/env

@@ -0,0 +1,14 @@
+PYTHONUNBUFFERED=0
+REDASH_LOG_LEVEL=INFO
+REDASH_REDIS_URL=redis://vision-redis:6379/0
+REDASH_COOKIE_SECRET="{{ VISION_REDASH_COOKIE_SECRET }}"
+REDASH_SECRET_KEY="{{ VISION_REDASH_SECRET_KEY }}"
+REDASH_DATABASE_URL="postgresql://{{ VISION_REDASH_POSTGRESQL_USER }}:{{ VISION_REDASH_POSTGRESQL_PASSWORD }}@vision-postgres/{{ VISION_REDASH_POSTGRESQL_DB }}"
+REDASH_MAIL_SERVER="{{ SMTP_HOST }}"
+REDASH_MAIL_PORT="{{ SMTP_PORT }}"
+REDASH_MAIL_USE_TLS="{{ SMTP_USE_TLS }}"
+REDASH_MAIL_USE_SSL="{{ SMTP_USE_SSL }}"
+REDASH_MAIL_USERNAME="{{ SMTP_USERNAME }}"
+REDASH_MAIL_PASSWORD="{{ SMTP_PASSWORD }}"
+REDASH_MAIL_DEFAULT_SENDER="{{ CONTACT_EMAIL }}"
+REDASH_HOST="{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ VISION_REDASH_HOST }}"

tutorvision/templates/vision/apps/vector/vector.toml

@@ -0,0 +1,65 @@
+# Vector's API for introspection
+[api]
+enabled = true
+address = "127.0.0.1:8686"
+
+### Sources
+
+# Capture logs from all containers
+[sources.containers]
+type = "docker_logs"
+
+### Transforms
+[transforms.openedx_containers]
+type = "filter"
+inputs = ["containers"]
+condition.type = "check_fields"
+condition."label.com.docker.compose.regex" = "(lms|cms)"
+
+[transforms.nginx_containers]
+type = "filter"
+inputs = ["containers"]
+condition.type = "check_fields"
+condition."label.com.docker.compose.eq" = "nginx"
+
+[transforms.tracking_raw]
+type = "regex_parser"
+inputs = ["openedx_containers"]
+drop_failed = true
+drop_field = false
+field = "message"
+# 2021-03-09 13:08:41,292 INFO 21 [tracking] [user 3] [ip 172.18.0.1] logger.py:42 - {...}
+patterns = ["^.* \\[tracking\\] [^{}]* (?P<tracking_message>\\{.*\\})$"]
+
+[transforms.tracking]
+type = "remap"
+inputs = ["tracking_raw"]
+# Time formats: https://docs.rs/chrono/0.4.19/chrono/format/strftime/index.html#specifiers
+source = '''
+.message = .tracking_message
+.tracking_message = parse_json(.tracking_message)
+.time = parse_timestamp(.tracking_message.time, format="%+")
+del(.tracking_message)
+'''
+
+### Sinks
+
+# Log all events to stdout, for debugging
+[sinks.out]
+type = "console"
+inputs = ["tracking"]
+encoding.codec = "json"
+
+# Send logs to clickhouse
+[sinks.clickhouse]
+type = "clickhouse"
+encoding.only_fields = ["time", "message"]
+# Required: https://github.com/timberio/vector/issues/5797
+encoding.timestamp_format = "unix"
+inputs = ["tracking"]
+endpoint = "http://{{ VISION_CLICKHOUSE_HOST }}:{{ VISION_CLICKHOUSE_HTTP_PORT }}"
+database = "{{ VISION_CLICKHOUSE_DATABASE }}"
+table = "tracking"
+healthcheck = true
+
+{{ patch("vision-vector-toml") }}

tutorvision/templates/vision/hooks/vision-clickhouse/init

@@ -0,0 +1,21 @@
+clickhouse-client --host {{ VISION_CLICKHOUSE_HOST }} --port {{ VISION_CLICKHOUSE_PORT }} \
+    --query "CREATE DATABASE IF NOT EXISTS {{ VISION_CLICKHOUSE_DATABASE }}"
+
+# TODO add PARTITION BY?
+clickhouse-client --host {{ VISION_CLICKHOUSE_HOST }} --port {{ VISION_CLICKHOUSE_PORT}} --database {{ VISION_CLICKHOUSE_DATABASE }} \
+    --query 'CREATE TABLE IF NOT EXISTS tracking (
+    `time` DateTime,
+    `message` String
+) ENGINE MergeTree ORDER BY time'
+
+# TODO add materialized view https://youtu.be/pZkKsfr8n3M?t=1731
+clickhouse-client --host {{ VISION_CLICKHOUSE_HOST }} --port {{ VISION_CLICKHOUSE_PORT}} --database {{ VISION_CLICKHOUSE_DATABASE }} \
+--query 'CREATE VIEW IF NOT EXISTS events AS
+    SELECT
+        time,
+        JSONExtractString(message, 'name') as name,
+        JSONExtract(message, 'context', 'course_id', 'String') as course_id,
+        JSONExtract(message, 'context', 'user_id', 'Int64') as user_id
+    FROM tracking
+    WHERE JSONExtractString(message, 'event_source')='browser'
+    ORDER BY time'

tutorvision/templates/vision/hooks/vision-redash/init

@@ -0,0 +1,6 @@
+./manage.py database create_tables
+./manage.py users create_root --password={{ VISION_REDASH_ROOT_PASSWORD }} {{ VISION_REDASH_ROOT_EMAIL }} {{ VISION_REDASH_ROOT_USERNAME }} || echo "Skipping admin user creation"
+
+(./manage.py ds list | grep datalake && echo "datalake data source already exists") || \
+    (echo "creating datalake data source..." && \
+    ./manage.py ds new --type=clickhouse --options='{"url":"http://{{ VISION_CLICKHOUSE_HOST }}:{{ VISION_CLICKHOUSE_HTTP_PORT }}", "dbname": "{{ VISION_CLICKHOUSE_DATABASE }}"}' datalake)