diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d7533f3..92b7766 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,9 @@ variables: TUTOR_PLUGIN: vision + TUTOR_IMAGES: vision-clickhouse vision-superset TUTOR_PYPI_PRIVATE_PACKAGE: tutor-vision OPENEDX_RELEASE: koa include: - project: 'community/tutor-ci' file: 'plugin-gitlab-ci.yml' - diff --git a/README.rst b/README.rst index 5aa26ad..7b65f1d 100644 --- a/README.rst +++ b/README.rst @@ -3,15 +3,16 @@ Tutor Vision: scalable, real-time analytics for Open edX TODO: -- Expose data with redash - - Provision dashboards - - Expose grades +- Expose data with superset + - Expose grades and certificates - Reproduce dashboards from https://edx.readthedocs.io/projects/edx-insights/en/latest/Overview.html - Reproduce dashboards from https://datastudio.google.com/embed/u/0/reporting/1gd-YXUtHFzHm3qddPTO8r272kyRD-uDG/page/4f5xB - frontend user creation: - generate random frontend user password in "tutor vision frontend createuser" + - try out alerts - Kubernetes compatibility - Sweet readme +- Rename to ocean? Installation ------------ @@ -28,34 +29,25 @@ Usage tutor plugins enable vision tutor local quickstart -Create a root user to access the frontend:: +Create an admin user to access the frontend:: - # You will be prompted for your password - tutor vision frontend createuser --root admin admin@youremail.com - -Grant this user access to all data:: - - tutor vision datalake createuser admin - tutor vision datalake setpermissions admin - -You can then access the frontend with the user credentials you just created. Open http(s)://vision. in your browser. When running locally, this will be http://vision.local.overhang.io. + # You will be prompted for a new password + tutor local run vision-superset superset fab create-admin --username yourusername --email user@example.com +You can then access the frontend with the user credentials you just created. Open http(s)://vision. in your browser. When running locally, this will be http://vision.local.overhang.io. The admin user will automatically be granted access to the "openedx" database in Superset and will be able to query all tables. Management ---------- -To add a new, non-admin user:: +Most of your users should probably not have access to all data from all courses. To restrict a given user to one or more courses or organizations, select the course IDs and/or organization IDS to which the user should have access and create a user with limited access to the datalake:: - # Create a datalake user - tutor vision datalake createuser yourusername - # Remember to restrict access, otherwise the new user will have access to everything - tutor vision datalake setpermissions --course-id 'course-v1:edX+DemoX+Demo_Course' yourusername - # Create a corresponding user on the frontend - tutor vision frontend createuser yourusername yourusername@youremail.com + tutor local run vision-clickhouse vision createuser --course-id='course-v1:edX+DemoX+Demo_Course' --org-id='edX' yourusername -Note that you may grant a user access to the data of an organization instead of just a course. To do so, run:: +Then, create the corresponding user on the frontend:: - tutor vision datalake setpermissions --org-id yourorg yourusername + tutor local run vision-superset vision createuser yourusername yourusername@youremail.com + +Your frontend user will automatically be associated to the datalake database you created, provided they share the same name. Development ----------- @@ -67,27 +59,12 @@ To reload Vector configuration after changes to vector.toml, run:: To explore the clickhouse database as root, run:: - tutor local run vision-clickhouse clickhouse-client --host vision-clickhouse \ - --database $(tutor config printvalue VISION_CLICKHOUSE_DATABASE) \ - --user $(tutor config printvalue VISION_CLICKHOUSE_USERNAME) \ - --password $(tutor config printvalue VISION_CLICKHOUSE_PASSWORD) + tutor local run vision-clickhouse vision client -To launch a Python shell in Redash, run:: +To launch a Python shell in Superset, run:: - tutor local run vision-redash ./manage.py shell + tutor local run vision-superset superset shell -To backup an existing dashboard, with all its related queries and widgets, first find the dashboard slug from its url. Create a world-writable destination folder:: - - mkdir ./dashboards - chmod a+rwx dashboards - -Then run:: - - tutor local run -v $(pwd)/dashboards:/tmp/dashboards vision-redash python /redash/scripts/serialize.py dump --output /tmp/dashboards/dashboard.json > ./dashboard.json - -You can then re-import this dashboard, for instance to create the same dashboard in another user account:: - - tutor local run -v $(pwd)/dashboards:/tmp/dashboards vision-redash python /redash/scripts/serialize.py load /tmp/dashboards/dashboard.json License ------- diff --git a/tutorvision/cli.py b/tutorvision/cli.py index 9ba260e..f91ee5b 100644 --- a/tutorvision/cli.py +++ b/tutorvision/cli.py @@ -1,138 +1,135 @@ -import click - -from tutor import config as tutor_config -from tutor.commands.compose import ComposeJobRunner -from tutor.commands.local import docker_compose as local_docker_compose - - -@click.group(help="Manage your Vision platform") -def vision_command(): - pass - - -@click.group(help="Manage datalake access") -def datalake(): - pass - - -@click.command(name="createuser", help="Create new user or update existing one") -@click.argument("username") -@click.pass_obj -def datalake_createuser(context, username): - run_datalake_query(context.root, f"CREATE USER OR REPLACE {username}") - - -@click.command(name="setpermissions", help="Restrict user access") -@click.argument("username") -@click.option( - "-c", - "--course-id", - "course_ids", - multiple=True, - help=( - "Grant access to a course data. This option may be used multiple times to grant " - "access to multiple courses." - ), -) -@click.option( - "-o", - "--org-id", - "org_ids", - multiple=True, - help=( - "Grant access to the course data of an organization. This option may be used multiple times to grant " - "access to multiple organizations." - ), -) -@click.pass_obj -def datalake_setpermissions(context, username, course_ids, org_ids): - conditions = [] - for course_id in course_ids: - conditions.append(f"course_id = '{course_id}'") - for org_id in org_ids: - conditions.append(f"course_id LIKE 'course-v1:{org_id}+%'") - condition = "1" - if conditions: - condition = " OR ".join(conditions) - - # TODO rename courseenrollments to course_enrollments (and other tables as well) - # Note that the "CREATE TEMPORARY TABLE" grant is required to make use of "numbers()" functions. - query = f""" -GRANT CREATE TEMPORARY TABLE ON *.* TO {username}; - -GRANT SELECT ON events TO {username}; -CREATE ROW POLICY OR REPLACE {username} ON events AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; - -GRANT SELECT ON coursegrades TO {username}; -CREATE ROW POLICY OR REPLACE {username} ON coursegrades AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; - -GRANT SELECT ON courseenrollments TO {username}; -CREATE ROW POLICY OR REPLACE {username} ON courseenrollments AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; - -GRANT SELECT ON video_events TO {username}; -CREATE ROW POLICY OR REPLACE {username} ON video_events AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; - -GRANT SELECT ON video_view_segments TO {username}; -CREATE ROW POLICY OR REPLACE {username} ON video_view_segments AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; -""" - run_datalake_query(context.root, query) - - -def run_datalake_query(root, query): - config = tutor_config.load(root) - command_secure_opt = "--secure" if config["VISION_CLICKHOUSE_SCHEME"] == "https" else "" - command = f"""clickhouse client \ - {command_secure_opt} --host={config["VISION_CLICKHOUSE_HOST"]} --port={config["VISION_CLICKHOUSE_PORT"]} \ - --user={config["VISION_CLICKHOUSE_USERNAME"]} \ - --password={config["VISION_CLICKHOUSE_PASSWORD"]} \ - --database={config["VISION_CLICKHOUSE_DATABASE"]} \ - --multiline --multiquery \ - --query "{query}" - """ - runner = ComposeJobRunner(root, config, local_docker_compose) - runner.run_job("vision-clickhouse", command) - - -@click.group(name="frontend", help="Manage the frontend access") -def frontend_command(): - pass - - -@click.command(name="createuser", help="Create a new user to access the frontend") -@click.option( - "-p", - "--password", - default="", - prompt="User password", - hide_input=True, - confirmation_prompt=True, - help="User password: if undefined you will be prompted to input a password", -) -@click.option( - "-r", - "--root", - "is_root", - is_flag=True, - default=False, - help="Grant root/administration privileges on the frontend to this user", -) -@click.argument("username") -@click.argument("email") -@click.pass_obj -def frontend_createuser(context, password, is_root, username, email): - config = tutor_config.load(context.root) - command = "python /redash/scripts/createuser.py {is_root} --password={password} {username} {email}".format( - is_root="--root" if is_root else "", - password=password, - username=username, - email=email, - ) - runner = ComposeJobRunner(context.root, config, local_docker_compose) - runner.run_job("vision-redash", command) - - -datalake.add_command(datalake_createuser) -datalake.add_command(datalake_setpermissions) -vision_command.add_command(datalake) -frontend_command.add_command(frontend_createuser) -vision_command.add_command(frontend_command) +# TODO delete this module +# import click +# +# from tutor import config as tutor_config +# from tutor.commands.compose import ComposeJobRunner +# from tutor.commands.local import docker_compose as local_docker_compose +# +# +# @click.group(help="Manage your Vision platform") +# def vision_command(): +# pass +# +# +# @click.group(help="Print convenient commands that can be sent to the right containers", name="print") +# def print_command(): +# pass +# +# +# @click.command(name="datalake-create-user", help="Print user creation query") +# @click.argument("username") +# @click.pass_obj +# def print_datalakecreateuser(context, username): +# print(f"CREATE USER IF NOT EXISTS {username}") +# +# +# # @click.command(name="setpermissions", help="Restrict user access") +# # @click.argument("username") +# # @click.option( +# # "-c", +# # "--course-id", +# # "course_ids", +# # multiple=True, +# # help=( +# # "Grant access to a course data. This option may be used multiple times to grant " +# # "access to multiple courses." +# # ), +# # ) +# # @click.option( +# # "-o", +# # "--org-id", +# # "org_ids", +# # multiple=True, +# # help=( +# # "Grant access to the course data of an organization. This option may be used multiple times to grant " +# # "access to multiple organizations." +# # ), +# # ) +# # @click.pass_obj +# # def datalake_setpermissions(context, username, course_ids, org_ids): +# # conditions = [] +# # for course_id in course_ids: +# # conditions.append(f"course_id = '{course_id}'") +# # for org_id in org_ids: +# # conditions.append(f"course_id LIKE 'course-v1:{org_id}+%'") +# # condition = "1" +# # if conditions: +# # condition = " OR ".join(conditions) +# # +# # # Note that the "CREATE TEMPORARY TABLE" grant is required to make use of "numbers()" functions. +# # query = f""" +# # GRANT CREATE TEMPORARY TABLE ON *.* TO {username}; +# # +# # GRANT SELECT ON events TO {username}; +# # CREATE ROW POLICY OR REPLACE {username} ON events AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; +# # +# # GRANT SELECT ON course_grades TO {username}; +# # CREATE ROW POLICY OR REPLACE {username} ON course_grades AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; +# # +# # GRANT SELECT ON course_enrollments TO {username}; +# # CREATE ROW POLICY OR REPLACE {username} ON course_enrollments AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; +# # +# # GRANT SELECT ON video_events TO {username}; +# # CREATE ROW POLICY OR REPLACE {username} ON video_events AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; +# # +# # GRANT SELECT ON video_view_segments TO {username}; +# # CREATE ROW POLICY OR REPLACE {username} ON video_view_segments AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; +# # """ +# # run_datalake_query(context.root, query) +# # +# # +# # def run_datalake_query(root, query): +# # config = tutor_config.load(root) +# # command_secure_opt = "--secure" if config["VISION_CLICKHOUSE_SCHEME"] == "https" else "" +# # command = f"""clickhouse client \ +# # {command_secure_opt} --host={config["VISION_CLICKHOUSE_HOST"]} --port={config["VISION_CLICKHOUSE_PORT"]} \ +# # --user={config["VISION_CLICKHOUSE_USERNAME"]} \ +# # --password={config["VISION_CLICKHOUSE_PASSWORD"]} \ +# # --database={config["VISION_CLICKHOUSE_DATABASE"]} \ +# # --multiline --multiquery \ +# # --query "{query}" +# # """ +# # runner = ComposeJobRunner(root, config, local_docker_compose) +# # runner.run_job("vision-clickhouse", command) +# # +# # +# # @click.group(name="frontend", help="Manage the frontend access") +# # def frontend_command(): +# # pass +# # +# # +# # @click.command(name="createuser", help="Create a new user to access the frontend") +# # @click.option( +# # "-p", +# # "--password", +# # default="", +# # prompt="User password", +# # hide_input=True, +# # confirmation_prompt=True, +# # help="User password: if undefined you will be prompted to input a password", +# # ) +# # @click.option( +# # "-r", +# # "--admin", +# # "is_admin", +# # is_flag=True, +# # default=False, +# # help="Grant root/administration privileges on the frontend to this user", +# # ) +# # @click.argument("username") +# # @click.argument("email") +# # @click.pass_obj +# # def frontend_createuser(context, password, is_admin, username, email): +# # config = tutor_config.load(context.root) +# # # TODO in case of non-admin, we must define a --role +# # fab_cmd = "create-admin" if is_admin else "create-user" +# # command = f"superset fab {fab_cmd} --username {username} --email {email} --password {password}" +# # runner = ComposeJobRunner(context.root, config, local_docker_compose) +# # runner.run_job("vision-superset", command) +# +# +# print_command.add_command(print_datalakecreateuser) +# # datalake.add_command(datalake_setpermissions) +# vision_command.add_command(print_command) +# # frontend_command.add_command(frontend_createuser) +# # vision_command.add_command(frontend_command) diff --git a/tutorvision/patches/caddyfile b/tutorvision/patches/caddyfile index 59764cf..a9b7e87 100644 --- a/tutorvision/patches/caddyfile +++ b/tutorvision/patches/caddyfile @@ -1,4 +1,4 @@ # Vision -{{ VISION_REDASH_HOST }}{% if not ENABLE_HTTPS %}:80{% endif %} { +{{ VISION_SUPERSET_HOST }}{% if not ENABLE_HTTPS %}:80{% endif %} { reverse_proxy nginx:80 } \ No newline at end of file diff --git a/tutorvision/patches/local-docker-compose-jobs-services b/tutorvision/patches/local-docker-compose-jobs-services index ac88c29..11629d8 100644 --- a/tutorvision/patches/local-docker-compose-jobs-services +++ b/tutorvision/patches/local-docker-compose-jobs-services @@ -3,12 +3,10 @@ vision-clickhouse-job: depends_on: {{ [("vision-clickhouse", VISION_RUN_CLICKHOUSE)]|list_if }} volumes: - ../plugins/vision/apps/clickhouse/migrations.d/:/etc/clickhouse-server/migrations.d/:ro -vision-redash-job: - image: {{ VISION_REDASH_DOCKER_IMAGE }} - command: create_db - env_file: ../plugins/vision/apps/redash/env +vision-superset-job: + image: {{ VISION_SUPERSET_DOCKER_IMAGE }} volumes: - - ../plugins/vision/apps/redash/scripts/:/redash/scripts/:ro + - ../plugins/vision/apps/superset/superset_config.py:/app/superset_config.py:ro depends_on: - vision-postgresql - vision-redis diff --git a/tutorvision/patches/local-docker-compose-services b/tutorvision/patches/local-docker-compose-services index bef5773..e2ded30 100644 --- a/tutorvision/patches/local-docker-compose-services +++ b/tutorvision/patches/local-docker-compose-services @@ -17,6 +17,7 @@ vision-clickhouse: volumes: - ../../data/vision/clickhouse:/var/lib/clickhouse - ../plugins/vision/apps/clickhouse/users.d/vision.xml:/etc/clickhouse-server/users.d/vision.xml:ro + env_file: ../plugins/vision/apps/env ulimits: nofile: soft: 262144 @@ -24,64 +25,34 @@ vision-clickhouse: restart: unless-stopped {% endif %} -# redash config docs: -# https://github.com/getredash/setup/blob/master/data/docker-compose.yml -# https://github.com/getredash/redash/blob/master/CHANGELOG.md - -# frontend -vision-redash: - image: {{ VISION_REDASH_DOCKER_IMAGE }} - command: gunicorn -b 0.0.0.0:5000 --name redash --workers=2 --max-requests=1000 --max-requests-jitter=100 --timeout=120 redash.wsgi:app - env_file: ../plugins/vision/apps/redash/env - environment: - REDASH_WEB_WORKERS: 4 +vision-superset: + image: {{ VISION_SUPERSET_DOCKER_IMAGE }} volumes: - - ../plugins/vision/apps/redash/scripts/:/redash/scripts/:ro + - ../plugins/vision/apps/superset/superset_config.py:/app/superset_config.py:ro + - ../plugins/vision/apps/superset/bootstrap:/app/bootstrap + env_file: ../plugins/vision/apps/env restart: unless-stopped depends_on: - - vision-postgresql - vision-redis -vision-redash-scheduler: - image: {{ VISION_REDASH_DOCKER_IMAGE }} - command: scheduler - env_file: ../plugins/vision/apps/redash/env + - vision-postgresql +vision-superset-worker: + image: {{ VISION_SUPERSET_DOCKER_IMAGE }} + volumes: + - ../plugins/vision/apps/superset/superset_config.py:/app/superset_config.py:ro + command: celery worker --app=superset.tasks.celery_app:app -Ofair -l INFO restart: unless-stopped depends_on: - - vision-postgresql - vision-redis -vision-redash-worker-scheduled: - image: {{ VISION_REDASH_DOCKER_IMAGE }} - command: worker - env_file: ../plugins/vision/apps/redash/env - environment: - QUEUES: "scheduled_queries schemas" - WORKERS_COUNT: 1 + - vision-postgresql +vision-superset-worker-beat: + image: {{ VISION_SUPERSET_DOCKER_IMAGE }} + volumes: + - ../plugins/vision/apps/superset/superset_config.py:/app/superset_config.py:ro + command: celery beat --app=superset.tasks.celery_app:app --pidfile /tmp/celerybeat.pid -l INFO --schedule=/tmp/celerybeat-schedule restart: unless-stopped depends_on: - - vision-postgresql - vision-redis -vision-redash-worker-queries: - image: {{ VISION_REDASH_DOCKER_IMAGE }} - command: worker - env_file: ../plugins/vision/apps/redash/env - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - restart: unless-stopped - depends_on: - vision-postgresql - - vision-redis -vision-redash-worker-default: - image: {{ VISION_REDASH_DOCKER_IMAGE }} - command: worker - env_file: ../plugins/vision/apps/redash/env - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 1 - restart: unless-stopped - depends_on: - - vision-postgresql - - vision-redis vision-redis: image: docker.io/redis:5.0-alpine restart: unless-stopped diff --git a/tutorvision/patches/nginx-extra b/tutorvision/patches/nginx-extra index 6fbc94e..dafd0e7 100644 --- a/tutorvision/patches/nginx-extra +++ b/tutorvision/patches/nginx-extra @@ -1,10 +1,10 @@ # Vision upstream vision-backend { - server vision-redash:5000 fail_timeout=0; + server vision-superset:8000 fail_timeout=0; } server { listen 80; - server_name {{ VISION_REDASH_HOST }}; + server_name {{ VISION_SUPERSET_HOST }}; # Disables server version feedback on pages and in headers server_tokens off; diff --git a/tutorvision/plugin.py b/tutorvision/plugin.py index a70cfb3..2aa6034 100644 --- a/tutorvision/plugin.py +++ b/tutorvision/plugin.py @@ -2,7 +2,7 @@ from glob import glob import os from .__about__ import __version__ -from .cli import vision_command +# from .cli import vision_command # TODO remove this HERE = os.path.abspath(os.path.dirname(__file__)) @@ -12,12 +12,11 @@ config = { "add": { "CLICKHOUSE_PASSWORD": "{{ 20|random_string }}", "POSTGRESQL_PASSWORD": "{{ 20|random_string }}", - "REDASH_COOKIE_SECRET": "{{ 20|random_string }}", - "REDASH_PASSWORD": "{{ 20|random_string }}", - "REDASH_SECRET_KEY": "{{ 20|random_string }}", + "SUPERSET_SECRET_KEY": "{{ 20|random_string }}", }, "defaults": { - "CLICKHOUSE_DOCKER_IMAGE": "docker.io/yandex/clickhouse-server:21.2.7.11", + "VERSION": __version__, + "CLICKHOUSE_DOCKER_IMAGE": "{{ DOCKER_REGISTRY }}overhangio/clickhouse:{{ VISION_VERSION }}", "RUN_CLICKHOUSE": True, "CLICKHOUSE_SCHEME": "http", "CLICKHOUSE_HOST": "vision-clickhouse", @@ -26,17 +25,24 @@ config = { "CLICKHOUSE_DATABASE": "openedx", "CLICKHOUSE_USERNAME": "openedx", "DOCKER_HOST": "/var/run/docker.sock", - "POSTGRESQL_USER": "redash", - "POSTGRESQL_DB": "redash", - "REDASH_DOCKER_IMAGE": "docker.io/redash/redash:9.0.0-beta.b42121", - "REDASH_HOST": "vision.{{ LMS_HOST }}", - "REDASH_USERNAME": "admin", - "REDASH_EMAIL": "{{ CONTACT_EMAIL }}", + # TODO move data to mysql? + "POSTGRESQL_USER": "superset", + "POSTGRESQL_DB": "superset", + "RUN_CLICKHOUSE": True, + "SUPERSET_DOCKER_IMAGE": "{{ DOCKER_REGISTRY }}overhangio/superset:{{ VISION_VERSION }}", + "SUPERSET_HOST": "vision.{{ LMS_HOST }}", + "SUPERSET_DATABASE": "openedx", }, } -hooks = {"init": ["vision-clickhouse", "vision-redash"]} -command = vision_command +hooks = { + "build-image": { + "vision-clickhouse": "{{ VISION_CLICKHOUSE_DOCKER_IMAGE }}", + "vision-superset": "{{ VISION_SUPERSET_DOCKER_IMAGE }}" + }, + "init": ["vision-clickhouse", "vision-superset"], +} +# command = vision_command # TODO remove this def patches(): diff --git a/tutorvision/templates/vision/apps/clickhouse/migrations.d/0002_grades.sql b/tutorvision/templates/vision/apps/clickhouse/migrations.d/0002_grades.sql index d511ce6..df33ea8 100644 --- a/tutorvision/templates/vision/apps/clickhouse/migrations.d/0002_grades.sql +++ b/tutorvision/templates/vision/apps/clickhouse/migrations.d/0002_grades.sql @@ -1,4 +1,4 @@ -CREATE TABLE coursegrades +CREATE TABLE course_grades ( `percent_grade` Double, `passed_timestamp` DateTime NULL, @@ -8,4 +8,4 @@ CREATE TABLE coursegrades ENGINE = MySQL('{{ MYSQL_HOST }}:{{ MYSQL_PORT }}', '{{ OPENEDX_MYSQL_DATABASE }}', 'grades_persistentcoursegrade', '{{ OPENEDX_MYSQL_USERNAME }}', '{{ OPENEDX_MYSQL_PASSWORD }}'); -- Grant everyone access to the view -CREATE ROW POLICY common ON coursegrades FOR SELECT USING 1 TO ALL; +CREATE ROW POLICY common ON course_grades FOR SELECT USING 1 TO ALL; diff --git a/tutorvision/templates/vision/apps/clickhouse/migrations.d/0003_course_enrollments.sql b/tutorvision/templates/vision/apps/clickhouse/migrations.d/0003_course_enrollments.sql new file mode 100644 index 0000000..6b4816a --- /dev/null +++ b/tutorvision/templates/vision/apps/clickhouse/migrations.d/0003_course_enrollments.sql @@ -0,0 +1,43 @@ +CREATE TABLE openedx_course_enrollments +( + `created` DateTime NULL, + `user_id` UInt64, + `course_id` String, + `is_active` UInt8, + `mode` String +) +ENGINE = MySQL('{{ MYSQL_HOST }}:{{ MYSQL_PORT }}', '{{ OPENEDX_MYSQL_DATABASE }}', 'student_courseenrollment', '{{ OPENEDX_MYSQL_USERNAME }}', '{{ OPENEDX_MYSQL_PASSWORD }}'); + +CREATE TABLE openedx_user_profiles +( + `user_id` UInt64, + `year_of_birth` UInt32, + `gender` String, + `level_of_education` String, + `city` String, + `state` String, + `country` String +) +ENGINE = MySQL('{{ MYSQL_HOST }}:{{ MYSQL_PORT }}', '{{ OPENEDX_MYSQL_DATABASE }}', 'auth_userprofile', '{{ OPENEDX_MYSQL_USERNAME }}', '{{ OPENEDX_MYSQL_PASSWORD }}'); + +-- enable live views +set allow_experimental_live_view = 1; + +CREATE LIVE VIEW course_enrollments WITH PERIODIC REFRESH 30 AS +SELECT + openedx_course_enrollments.course_id AS course_id, + openedx_course_enrollments.created AS enrollment_created, + openedx_course_enrollments.is_active AS enrollment_is_active, + openedx_course_enrollments.mode AS enrollment_mode, + openedx_course_enrollments.user_id AS user_id, + openedx_user_profiles.year_of_birth AS user_year_of_birth, + openedx_user_profiles.gender AS user_gender, + openedx_user_profiles.level_of_education AS user_level_of_education, + openedx_user_profiles.city AS user_city, + openedx_user_profiles.state AS user_state, + openedx_user_profiles.country AS user_country +FROM openedx_course_enrollments +INNER JOIN openedx_user_profiles ON openedx_course_enrollments.user_id = openedx_user_profiles.user_id; + +-- Grant everyone access to the view +CREATE ROW POLICY common ON course_enrollments FOR SELECT USING 1 TO ALL; diff --git a/tutorvision/templates/vision/apps/clickhouse/migrations.d/0003_create_courseenrollments_view.sql b/tutorvision/templates/vision/apps/clickhouse/migrations.d/0003_create_courseenrollments_view.sql deleted file mode 100644 index 3b85b50..0000000 --- a/tutorvision/templates/vision/apps/clickhouse/migrations.d/0003_create_courseenrollments_view.sql +++ /dev/null @@ -1,43 +0,0 @@ -CREATE TABLE openedx_courseenrollments -( - `created` DateTime NULL, - `user_id` UInt64, - `course_id` String, - `is_active` UInt8, - `mode` String -) -ENGINE = MySQL('{{ MYSQL_HOST }}:{{ MYSQL_PORT }}', '{{ OPENEDX_MYSQL_DATABASE }}', 'student_courseenrollment', '{{ OPENEDX_MYSQL_USERNAME }}', '{{ OPENEDX_MYSQL_PASSWORD }}'); - -CREATE TABLE openedx_userprofiles -( - `user_id` UInt64, - `year_of_birth` UInt32, - `gender` String, - `level_of_education` String, - `city` String, - `state` String, - `country` String -) -ENGINE = MySQL('{{ MYSQL_HOST }}:{{ MYSQL_PORT }}', '{{ OPENEDX_MYSQL_DATABASE }}', 'auth_userprofile', '{{ OPENEDX_MYSQL_USERNAME }}', '{{ OPENEDX_MYSQL_PASSWORD }}'); - --- enable live views -set allow_experimental_live_view = 1; - -CREATE LIVE VIEW courseenrollments WITH PERIODIC REFRESH 30 AS -SELECT - openedx_courseenrollments.course_id AS course_id, - openedx_courseenrollments.created AS enrollment_created, - openedx_courseenrollments.is_active AS enrollment_is_active, - openedx_courseenrollments.mode AS enrollment_mode, - openedx_courseenrollments.user_id AS user_id, - openedx_userprofiles.year_of_birth AS user_year_of_birth, - openedx_userprofiles.gender AS user_gender, - openedx_userprofiles.level_of_education AS user_level_of_education, - openedx_userprofiles.city AS user_city, - openedx_userprofiles.state AS user_state, - openedx_userprofiles.country AS user_country -FROM openedx_courseenrollments -INNER JOIN openedx_userprofiles ON openedx_courseenrollments.user_id = openedx_userprofiles.user_id; - --- Grant everyone access to the view -CREATE ROW POLICY common ON courseenrollments FOR SELECT USING 1 TO ALL; diff --git a/tutorvision/templates/vision/apps/clickhouse/migrations.d/0005_course_blocks.sql b/tutorvision/templates/vision/apps/clickhouse/migrations.d/0005_course_blocks.sql new file mode 100644 index 0000000..4844594 --- /dev/null +++ b/tutorvision/templates/vision/apps/clickhouse/migrations.d/0005_course_blocks.sql @@ -0,0 +1,14 @@ +CREATE TABLE course_blocks +( + `course_id` String, + `block_key` String, + `block_id` String, + `position` Integer, + `display_name` String, + `full_name` String +) +ENGINE MergeTree +ORDER BY (course_id, position, block_id); + +-- Grant everyone access to the table +CREATE ROW POLICY common ON course_blocks FOR SELECT USING 1 TO ALL; diff --git a/tutorvision/templates/vision/apps/clickhouse/migrations.d/0006_course_block_completion.sql b/tutorvision/templates/vision/apps/clickhouse/migrations.d/0006_course_block_completion.sql new file mode 100644 index 0000000..15ffbcd --- /dev/null +++ b/tutorvision/templates/vision/apps/clickhouse/migrations.d/0006_course_block_completion.sql @@ -0,0 +1,28 @@ +CREATE TABLE openedx_block_completion +( + `modified` DateTime NULL, + `course_key` String, + `block_key` String, + `block_type` String, + `user_id` UInt64, + `completion` Float32 +) +ENGINE = MySQL('{{ MYSQL_HOST }}:{{ MYSQL_PORT }}', '{{ OPENEDX_MYSQL_DATABASE }}', 'completion_blockcompletion', '{{ OPENEDX_MYSQL_USERNAME }}', '{{ OPENEDX_MYSQL_PASSWORD }}'); + +-- enable live views +set allow_experimental_live_view = 1; + +CREATE LIVE VIEW course_block_completion WITH PERIODIC REFRESH 30 AS +SELECT + openedx_block_completion.course_key AS course_id, + openedx_block_completion.block_key AS block_key, + openedx_block_completion.user_id AS user_id, + openedx_block_completion.completion AS completion, + course_blocks.position as position, + course_blocks.display_name as display_name, + course_blocks.full_name as full_name +FROM openedx_block_completion +INNER JOIN course_blocks ON openedx_block_completion.block_key = course_blocks.block_key; + +-- Grant everyone access to the view +CREATE ROW POLICY common ON course_block_completion FOR SELECT USING 1 TO ALL; diff --git a/tutorvision/templates/vision/apps/env b/tutorvision/templates/vision/apps/env new file mode 100644 index 0000000..0b2f7b8 --- /dev/null +++ b/tutorvision/templates/vision/apps/env @@ -0,0 +1,5 @@ +VISION_CLICKHOUSE_HOST={{ VISION_CLICKHOUSE_HOST }} +VISION_CLICKHOUSE_PORT={{ VISION_CLICKHOUSE_PORT }} +VISION_CLICKHOUSE_USERNAME={{ VISION_CLICKHOUSE_USERNAME }} +VISION_CLICKHOUSE_PASSWORD={{ VISION_CLICKHOUSE_PASSWORD }} +VISION_CLICKHOUSE_DATABASE={{ VISION_CLICKHOUSE_DATABASE }} diff --git a/tutorvision/templates/vision/apps/openedx/importcoursedata.py b/tutorvision/templates/vision/apps/openedx/importcoursedata.py new file mode 100644 index 0000000..a9039b4 --- /dev/null +++ b/tutorvision/templates/vision/apps/openedx/importcoursedata.py @@ -0,0 +1,81 @@ +import urllib.request +import urllib.parse + +import lms.startup + +lms.startup.run() + +from courseware.courses import get_course +from MySQLdb import escape_string as sql_escape_string +from opaque_keys.edx.keys import CourseKey +from xmodule.modulestore.django import modulestore + +# TODO actually run this script during init by mounting inside the lms container +def main(): + module_store = modulestore() + for course in module_store.get_courses(): + import_course(course.id) + + +def import_course(course_key): + course_id = str(course_key) + # Reload course to fetch all children items + course = get_course(course_key, depth=None) + print("======================", course_id, course.display_name) + values = [ + sql_query( + "('{}', '{}', '{}', '{}', '{}', '{}')", + course_id, + str(child.location), + child.location.block_id, + str(position), + child.display_name, + full_name, + ) + for position, (child, full_name) in enumerate(iter_course_blocks(course)) + ] + if values: + print( + f"Inserting {len(values)} items in course_blocks for course '{course_id}'..." + ) + make_query( + sql_query( + "ALTER TABLE course_blocks DELETE WHERE course_id = '{}';", + course_id, + ) + ) + insert_query = sql_query( + "INSERT INTO course_blocks (course_id, block_key, block_id, position, display_name, full_name) VALUES " + ) + insert_query += ", ".join(values) + make_query(insert_query) + + +def iter_course_blocks(item, prefix=""): + prefix += item.display_name or "N/A" + yield item, prefix + prefix += " > " + for child in item.get_children(): + yield from iter_course_blocks(child, prefix=prefix) + + +def sql_query(template, *args, **kwargs): + args = [sql_escape_string(arg).decode() for arg in args] + kwargs = {key: sql_escape_string(value).decode() for key, value in kwargs.items()} + return template.format(*args, **kwargs) + + +def make_query(query): + # TODO pass connection strings + url = "http://vision-clickhouse:8123/?%s" % urllib.parse.urlencode( + {"database": "openedx"} + ) + try: + urllib.request.urlopen(url, data=query.encode()) + except urllib.request.HTTPError as e: + print(e.read().decode()) + raise + + +if __name__ == "__main__": + main() diff --git a/tutorvision/templates/vision/apps/redash/env b/tutorvision/templates/vision/apps/redash/env deleted file mode 100644 index 13c756b..0000000 --- a/tutorvision/templates/vision/apps/redash/env +++ /dev/null @@ -1,23 +0,0 @@ -PYTHONUNBUFFERED=0 -PYTHONPATH=/app - -# Clickhouse -CLICKHOUSE_SCHEME={{ VISION_CLICKHOUSE_SCHEME }} -CLICKHOUSE_HOST={{ VISION_CLICKHOUSE_HOST }} -CLICKHOUSE_PORT={{ VISION_CLICKHOUSE_HTTP_PORT }} -CLICKHOUSE_DATABASE={{ VISION_CLICKHOUSE_DATABASE }} - -# Redash configuration -REDASH_LOG_LEVEL=INFO -REDASH_REDIS_URL=redis://vision-redis:6379/0 -REDASH_COOKIE_SECRET={{ VISION_REDASH_COOKIE_SECRET }} -REDASH_SECRET_KEY={{ VISION_REDASH_SECRET_KEY }} -REDASH_DATABASE_URL=postgresql://{{ VISION_POSTGRESQL_USER }}:{{ VISION_POSTGRESQL_PASSWORD }}@vision-postgresql/{{ VISION_POSTGRESQL_DB }} -REDASH_MAIL_SERVER={{ SMTP_HOST }} -REDASH_MAIL_PORT={{ SMTP_PORT }} -REDASH_MAIL_USE_TLS={{ SMTP_USE_TLS }} -REDASH_MAIL_USE_SSL={{ SMTP_USE_SSL }} -REDASH_MAIL_USERNAME={{ SMTP_USERNAME }} -REDASH_MAIL_PASSWORD={{ SMTP_PASSWORD }} -REDASH_MAIL_DEFAULT_SENDER={{ CONTACT_EMAIL }} -REDASH_HOST={% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ VISION_REDASH_HOST }} diff --git a/tutorvision/templates/vision/apps/redash/scripts/createuser.py b/tutorvision/templates/vision/apps/redash/scripts/createuser.py deleted file mode 100755 index 5319ed0..0000000 --- a/tutorvision/templates/vision/apps/redash/scripts/createuser.py +++ /dev/null @@ -1,150 +0,0 @@ -#! /usr/bin/env python3 - -import argparse -from getpass import getpass -import os - -from redash import create_app -from redash import models -from redash.query_runner.clickhouse import ClickHouse -from redash.utils.configuration import ConfigurationContainer - - -def main(): - parser = argparse.ArgumentParser( - description="Create a Redash user with the corresponding data source pointing to Clickhouse" - ) - parser.add_argument("-r", "--root", action="store_true", help="Make a root user") - parser.add_argument( - "-p", "--password", help="If undefined, you will be prompted for a password" - ) - parser.add_argument("username") - parser.add_argument("email") - args = parser.parse_args() - - app = create_app() - app.app_context().push() - - password = args.password - while not password: - password = getpass(prompt="User password: ") - - org = get_default_org() - group = get_group(org, args.username, is_root=args.root) - user = get_user(org, group, args.username, args.email, password, is_root=args.root) - get_datasource(org, group, user.name) - - -def get_default_org(): - org_slug = "default" - org = models.Organization.get_by_slug(org_slug) - if org: - print("Org already exists") - else: - print("Creating org...") - org = models.Organization( - name=org_slug, slug=org_slug, settings={"beacon_consent": False} - ) - models.db.session.add(org) - models.db.session.commit() - - # Get org admin group - if org.admin_group: - print("Org admin group already exists") - else: - print("Creating org admin group...") - admin_group = models.Group( - name="admin", - permissions=["admin", "super_admin"], - org=org, - type=models.Group.BUILTIN_GROUP, - ) - models.db.session.add_all([org, admin_group]) - models.db.session.commit() - - return org - - -def get_group(org, username, is_root=False): - group = models.Group.query.filter( - models.Group.name == username, models.Group.org == org - ).first() - if group: - print("Group '{}' already exists".format(username)) - else: - excluded_permissions = [] if is_root else ["list_users"] - permissions = [ - permission - for permission in models.Group.DEFAULT_PERMISSIONS - if permission not in excluded_permissions - ] - group = models.Group(name=username, org=org, permissions=permissions) - models.db.session.add(group) - models.db.session.commit() - print("Created group '{}'".format(group.name)) - if is_root: - for permission in ["admin", "super_admin"]: - if permission not in group.permissions: - print("Adding permission '{}' to group".format(permission)) - group.permissions.append(permission) - models.db.session.add(group) - models.db.session.commit() - return group - - -def get_user(org, group, username, email, password, is_root=False): - user = models.User.query.filter(models.User.email == email).first() - if user: - print("User already exists") - else: - user = models.User(org=org, email=email, name=username, group_ids=[group.id]) - print("Created user '{}/{}'".format(user.email, user.name)) - user.hash_password(password) - models.db.session.add(user) - models.db.session.commit() - if is_root: - if org.admin_group.id in user.group_ids: - print("User is already in admin group") - else: - print("Adding user to admin group...") - user.group_ids = user.group_ids + [org.admin_group.id] - models.db.session.add(user) - models.db.session.commit() - return user - - -def get_datasource(org, group, username): - # Get or create datasource - options = ConfigurationContainer( - { - "url": "{}://{}:{}".format( - os.environ["CLICKHOUSE_SCHEME"], - os.environ["CLICKHOUSE_HOST"], - os.environ["CLICKHOUSE_PORT"], - ), - "user": username, - "password": "", - "dbname": os.environ["CLICKHOUSE_DATABASE"], - }, - ClickHouse.configuration_schema(), - ) - data_source = models.DataSource.query.filter( - models.DataSource.name == username - ).first() - if data_source: - print("Data source already exists") - else: - data_source = models.DataSource( - name=username, - type="clickhouse", - options=options, - org=org, - ) - data_source_group = models.DataSourceGroup(data_source=data_source, group=group) - models.db.session.add_all([data_source, data_source_group]) - models.db.session.commit() - print("Created datasource '{}'".format(data_source.name)) - - -if __name__ == "__main__": - main() diff --git a/tutorvision/templates/vision/apps/redash/scripts/importcoursedata.py b/tutorvision/templates/vision/apps/redash/scripts/importcoursedata.py deleted file mode 100644 index 26d6bc7..0000000 --- a/tutorvision/templates/vision/apps/redash/scripts/importcoursedata.py +++ /dev/null @@ -1,27 +0,0 @@ -import lms.startup - -lms.startup.run() - -from xmodule.modulestore.django import modulestore -from opaque_keys.edx.keys import CourseKey -from courseware.courses import get_course - - -def main(): - module_store = modulestore() - for course in module_store.get_courses(): - course_id = course.id.html_id() - course_key = CourseKey.from_string(course_id) - course = get_course(course_key, depth=None) - for position, child in enumerate(iter_children(course)): - print(course.display_name, position, child.location.block_id, child.display_name) - - -def iter_children(item): - yield item - for child in item.get_children(): - yield from iter_children(child) - - -if __name__ == "__main__": - main() diff --git a/tutorvision/templates/vision/apps/redash/scripts/serialize.py b/tutorvision/templates/vision/apps/redash/scripts/serialize.py deleted file mode 100644 index dd1d23a..0000000 --- a/tutorvision/templates/vision/apps/redash/scripts/serialize.py +++ /dev/null @@ -1,295 +0,0 @@ -import argparse -from datetime import datetime -import json - -from redash import create_app -from redash import models -from redash.models.users import Group, User -from redash import serializers - - -def main(): - parser = argparse.ArgumentParser( - description="Dump a dashboard to JSON, along with all associated widgets, queries and visualizations" - ) - subparsers = parser.add_subparsers() - - # Dump command - dump_command = subparsers.add_parser("dump", help="Dump objects to JSON") - dump_command.set_defaults(func=dump) - dump_command.add_argument( - "-o", - "--output", - default="-", - help="Destination file. Leave to '-' to print in stdout", - ) - dump_command.add_argument("username") - dump_command.add_argument("slug") - - # Load command - load_command = subparsers.add_parser("load", help="Load objects from JSON") - load_command.set_defaults(func=load) - load_command.add_argument( - "-n", - "--dry-run", - action="store_true", - help="Do not actually commit changes (good for debugging)", - ) - load_command.add_argument( - "-s", - "--skip-existing", - action="store_true", - help="Do not actually create objects when an object with identical name already exists", - ) - load_command.add_argument( - "-d", - "--datasource", - help="Assign queries to the specified datasource. If undefined, queries will be associated to the first datasource of the first user group.", - ) - load_command.add_argument( - "path", - help="JSON-formatted file to load data from", - ) - load_command.add_argument("username", help="Add dashboard to this user account") - - app = create_app() - app.app_context().push() - - args = parser.parse_args() - args.func(args) - - -def dump(args): - user = get_user(args.username) - dashboard = get_dashboard(user, args.slug) - data = serialize(dashboard) - data = clean_data(data) - serialized = json.dumps(data, sort_keys=True, indent=2) - save_to(serialized, args.output) - - -def load(args): - with open(args.path) as f: - serialized = json.load(f) - - deserializer = Deserializer( - args.username, - data_source_id=args.datasource, - dry_run=args.dry_run, - skip_existing=args.skip_existing, - ) - deserializer.deserialize_dashboard(serialized) - - -class Deserializer: - def __init__( - self, username, data_source_id=None, dry_run=False, skip_existing=False - ): - self.user = get_user(username) - self.data_source_id = data_source_id - if not self.data_source_id: - group = Group.query.get(self.user.group_ids[0]) - self.data_source_id = group.data_sources[0].data_source.id - print( - "Objects will be associated to datasource #{}".format(self.data_source_id) - ) - - self.dry_run = dry_run - self.skip_existing = skip_existing - self.widget_id_map = {} - self.visualization_id_map = {} - self.query_id_map = {} - - def deserialize_dashboard(self, params): - print("--- Creating dashboard '{}'...".format(params["name"])) - dashboard = models.Dashboard( - user=self.user, - org=self.user.org, - name=params["name"], - layout=[], - is_draft=False, - ) - models.db.session.add(dashboard) - - for widget in params["widgets"]: - visualization = widget["visualization"] - self.deserialize_query(visualization["query"]) - self.deserialize_visualization(visualization) - self.deserialize_widget(dashboard.id, widget) - - if self.dry_run: - print("Dry-run mode: changes are discarded.") - else: - models.db.session.commit() - - def deserialize_query(self, params): - old_id = params["id"] - if old_id in self.query_id_map: - return - - if self.skip_existing: - existing_query = models.Query.query.filter( - models.Query.user == self.user, models.Query.name == params["name"] - ).first() - if existing_query: - print( - "--- Skipping creation of query '{}' which already exists".format( - params["name"] - ) - ) - self.query_id_map[old_id] = existing_query.id - return - - print("--- Creating query '{}'...".format(params["name"])) - # If there are parameter queries, create these first - for parameter in params["options"]["parameters"]: - if "query" in parameter: - parameter_query_params = parameter.pop("query") - self.deserialize_query(parameter_query_params) - parameter["queryId"] = self.query_id_map[parameter_query_params["id"]] - - query = models.Query( - user=self.user, - org=self.user.org, - data_source_id=self.data_source_id, - **project_all_but(params, ["id"]) - ) - models.db.session.add(query) - self.query_id_map[old_id] = query.id - - def deserialize_visualization(self, params): - old_id = params["id"] - if old_id in self.visualization_id_map: - return - - query_id = self.query_id_map[params["query"]["id"]] - if self.skip_existing: - existing_visualization = ( - models.Visualization.query.join(models.Query) - .filter( - models.Visualization.query_id == query_id, - models.Query.user == self.user, - models.Visualization.name == params["name"], - ) - .first() - ) - if existing_visualization: - print( - "--- Skipping creation of visualization '{}' which already exists".format( - params["name"] - ) - ) - self.visualization_id_map[old_id] = existing_visualization.id - return - - print("--- Creating visualization '{}'...".format(params["name"])) - options = json.dumps(params["options"]) - visualization = models.Visualization( - query_id=query_id, - options=options, - **project_all_but(params, ["id", "options", "query"]) - ) - models.db.session.add(visualization) - self.visualization_id_map[old_id] = visualization.id - - def deserialize_widget(self, dashboard_id, params): - old_id = params["id"] - if old_id in self.widget_id_map: - return - print("--- Creating widget...") - - widget = models.Widget( - visualization_id=self.visualization_id_map[params["visualization"]["id"]], - dashboard_id=dashboard_id, - options=json.dumps(params["options"]), - **project_all_but(params, ["id", "options", "visualization"]) - ) - models.db.session.add(widget) - self.widget_id_map[old_id] = widget.id - - -def get_user(username): - org = models.Organization.get_by_slug("default") - return org.users.filter(User.name == username).one() - - -def get_dashboard(user, slug): - return models.Dashboard.query.filter( - models.Dashboard.user == user, models.Dashboard.slug == slug - ).one() - - -def serialize(dashboard): - return { - "id": dashboard.id, - "name": dashboard.name, - "widgets": [serialize_widget(w) for w in dashboard.widgets], - } - - -def serialize_widget(widget): - """ - TODO don't include draft or achived queries. - """ - widget_fields = set(["id", "width", "options", "text", "visualization"]) - visualization_fields = set( - ["id", "type", "name", "description", "options", "query"] - ) - query_fields = set( - ["id", "name", "description", "query_text", "schedule", "options"] - ) - serialized = serializers.serialize_widget(widget) - serialized = project(serialized, widget_fields) - if "visualization" in serialized: - serialized["visualization"] = project( - serialized["visualization"], visualization_fields - ) - # Convert 'query.query' field to 'query.query_text' - serialized["visualization"]["query"]["query_text"] = serialized[ - "visualization" - ]["query"]["query"] - # If query depends on some other parameter, save this other query, too - for parameter in serialized["visualization"]["query"]["options"]["parameters"]: - if "queryId" in parameter: - parameter_query = models.Query.query.get(parameter.pop("queryId")) - parameter["query"] = serializers.serialize_query(parameter_query) - # don't forget to convert query to query_text - parameter["query"]["query_text"] = parameter["query"]["query"] - parameter["query"] = project(parameter["query"], query_fields) - serialized["visualization"]["query"] = project( - serialized["visualization"]["query"], query_fields - ) - return serialized - - -def project(data, allowed_keys): - return {key: value for key, value in data.items() if key in allowed_keys} - - -def project_all_but(data, excluded_keys): - return {key: value for key, value in data.items() if key not in excluded_keys} - - -def clean_data(data): - """ - Convert all datetime objects to str, for serialization (recursively). - """ - if isinstance(data, datetime): - return str(data) - if isinstance(data, dict): - return {key: clean_data(value) for key, value in data.items()} - if isinstance(data, list): - return [clean_data(d) for d in data] - return data - - -def save_to(content, path): - if path == "-": - print(content) - return - with open(path, "w") as of: - of.write(content) - - -if __name__ == "__main__": - main() diff --git a/tutorvision/templates/vision/apps/superset/bootstrap/studentdemographics.json b/tutorvision/templates/vision/apps/superset/bootstrap/studentdemographics.json new file mode 100644 index 0000000..2274b28 --- /dev/null +++ b/tutorvision/templates/vision/apps/superset/bootstrap/studentdemographics.json @@ -0,0 +1,446 @@ +{ + "dashboards": [ + { + "__Dashboard__": { + "css": "", + "dashboard_title": "Student Demographics", + "description": null, + "json_metadata": "{\"timed_refresh_immune_slices\": [], \"expanded_slices\": {}, \"refresh_frequency\": 0, \"default_filters\": \"{\\\"7\\\": {\\\"__time_range\\\": \\\"No filter\\\"}}\", \"color_scheme\": null, \"filter_scopes\": {\"7\": {\"course_id\": {\"scope\": [\"ROOT_ID\"], \"immune\": []}, \"__time_range\": {\"scope\": [\"ROOT_ID\"], \"immune\": []}}}, \"remote_id\": 3}", + "position_json": "{\"CHART-CzBvnn-6dh\":{\"children\":[],\"id\":\"CHART-CzBvnn-6dh\",\"meta\":{\"chartId\":9,\"height\":50,\"sliceName\":\"Enrolled learners level of education\",\"uuid\":\"7ecc94c7-576e-48ca-bbd8-0e2831117b35\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-EMm5Btin1\"],\"type\":\"CHART\"},\"CHART-ITeQKYqMxt\":{\"children\":[],\"id\":\"CHART-ITeQKYqMxt\",\"meta\":{\"chartId\":10,\"height\":50,\"sliceName\":\"Enrolled learners gender\",\"uuid\":\"74763712-0193-4c3b-84ef-1a34b2aa17d8\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-EMm5Btin1\"],\"type\":\"CHART\"},\"CHART-JDUlfaXdNk\":{\"children\":[],\"id\":\"CHART-JDUlfaXdNk\",\"meta\":{\"chartId\":7,\"height\":50,\"sliceName\":\"Select course ID\",\"uuid\":\"b156195f-4dfd-42b7-aa51-68e23a759914\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-EMm5Btin1\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-EMm5Btin1\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"Student Demographics\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-EMm5Btin1\":{\"children\":[\"CHART-JDUlfaXdNk\",\"CHART-CzBvnn-6dh\",\"CHART-ITeQKYqMxt\"],\"id\":\"ROW-EMm5Btin1\",\"meta\":{\"0\":\"ROOT_ID\",\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", + "slices": [ + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.Course enrollments", + "datasource_type": "table", + "id": 9, + "params": "{\"adhoc_filters\": [{\"clause\": \"WHERE\", \"comparator\": \"1\", \"expressionType\": \"SIMPLE\", \"filterOptionName\": \"filter_tx1k2g8b0zg_muir0gyx6u8\", \"isExtra\": false, \"isNew\": false, \"operator\": \"==\", \"sqlExpression\": null, \"subject\": \"enrollment_is_active\"}], \"color_scheme\": \"supersetColors\", \"datasource\": \"10__table\", \"date_format\": \"smart_date\", \"donut\": true, \"extra_form_data\": {}, \"groupby\": [\"level_of_education\"], \"innerRadius\": 43, \"label_line\": true, \"label_type\": \"key\", \"labels_outside\": true, \"legendOrientation\": \"top\", \"legendType\": \"scroll\", \"metric\": \"count\", \"number_format\": \"SMART_NUMBER\", \"outerRadius\": 70, \"row_limit\": 1000, \"show_labels\": true, \"show_labels_threshold\": 1, \"show_legend\": true, \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"viz_type\": \"pie\", \"remote_id\": 9, \"datasource_name\": \"Course enrollments\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Enrolled learners level of education", + "viz_type": "pie" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.Course enrollments", + "datasource_type": "table", + "id": 10, + "params": "{\"adhoc_filters\": [{\"clause\": \"WHERE\", \"comparator\": \"1\", \"expressionType\": \"SIMPLE\", \"filterOptionName\": \"filter_tx1k2g8b0zg_muir0gyx6u8\", \"isExtra\": false, \"isNew\": false, \"operator\": \"==\", \"sqlExpression\": null, \"subject\": \"enrollment_is_active\"}], \"color_scheme\": \"bnbColors\", \"datasource\": \"10__table\", \"date_format\": \"smart_date\", \"donut\": true, \"extra_form_data\": {}, \"groupby\": [\"gender\"], \"innerRadius\": 43, \"label_line\": true, \"label_type\": \"key\", \"labels_outside\": true, \"legendOrientation\": \"top\", \"legendType\": \"scroll\", \"metric\": \"count\", \"number_format\": \"SMART_NUMBER\", \"outerRadius\": 70, \"row_limit\": 10, \"show_labels\": true, \"show_labels_threshold\": 1, \"show_legend\": true, \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"pie\", \"remote_id\": 10, \"datasource_name\": \"Course enrollments\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Enrolled learners gender", + "viz_type": "pie" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.Course enrollments", + "datasource_type": "table", + "id": 7, + "params": "{\"adhoc_filters\": [], \"datasource\": \"10__table\", \"date_filter\": false, \"extra_form_data\": {}, \"filter_configs\": [{\"asc\": true, \"clearable\": true, \"column\": \"course_id\", \"key\": \"CQE2v7Ajx\", \"label\": \"Course ID\", \"multiple\": true, \"searchAllOptions\": false}], \"slice_id\": 7, \"time_grain_sqla\": \"P1D\", \"time_range\": \"Last week\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"filter_box\", \"remote_id\": 7, \"datasource_name\": \"Course enrollments\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Select course ID", + "viz_type": "filter_box" + } + } + ], + "slug": null + } + } + ], + "datasources": [ + { + "__SqlaTable__": { + "cache_timeout": null, + "columns": [ + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "course_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 51, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "f30d1ace-61e7-417d-bb42-cbe2202ba77d", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "enrollment_created", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 52, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "NULLABLE(DATETIME)", + "uuid": "214e5525-f504-4967-98fe-594246784958", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "enrollment_is_active", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 53, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "UINT8", + "uuid": "adda2e42-b890-4614-ad32-70b29df97a50", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "enrollment_mode", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 54, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "95b86e16-a423-46dd-8572-aa59de477a49", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 55, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "UINT64", + "uuid": "349b4226-71f9-4e36-919a-f3166905519f", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_year_of_birth", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 56, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "UINT32", + "uuid": "a97a8573-d7f6-4a2f-a36a-ad6605649785", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_gender", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 57, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "7a78fd5a-7408-4feb-9981-b8a1519305ec", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_level_of_education", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 58, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "14b87739-fd01-460f-a3cc-cd1a222be651", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_city", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 59, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "120e3998-09bd-4c4b-9c60-40bf6406c8ce", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_state", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 60, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "2656d45a-c083-4966-9aef-1370fefda078", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_country", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 61, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "06f431fd-e83d-4551-8fca-808bc2f39854", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "level_of_education", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 62, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "NULLABLE(STRING)", + "uuid": "0862dff7-c61f-4e72-be59-af77825af9ff", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "level_of_education_order", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 63, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "NULLABLE(UINT8)", + "uuid": "94f69a29-a2fb-4dee-b9ba-e612fed44a73", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "gender", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 64, + "is_active": true, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "NULLABLE(STRING)", + "uuid": "8b474ea7-40d3-4280-96d9-9a9e15c3997f", + "verbose_name": null + } + } + ], + "database_id": 1, + "default_endpoint": null, + "description": null, + "extra": null, + "fetch_values_predicate": null, + "filter_select_enabled": false, + "main_dttm_col": null, + "metrics": [ + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "d3format": null, + "description": null, + "expression": "count(*)", + "extra": "{\"warning_markdown\":null}", + "id": 16, + "metric_name": "count", + "metric_type": null, + "table_id": 10, + "uuid": "0967c3ab-96e0-4d06-9861-573e5beac200", + "verbose_name": null, + "warning_text": null + } + } + ], + "offset": 0, + "params": "{\"remote_id\": 10, \"database_name\": \"admin\", \"import_time\": 1622022621}", + "schema": "openedx", + "sql": "SELECT\r\n *,\r\n CASE\r\n WHEN user_gender = 'f' THEN 'Female'\r\n WHEN user_gender = 'm' THEN 'Male'\r\n WHEN user_gender = 'o' THEN 'Other'\r\n END AS gender,\r\n CASE\r\n WHEN user_level_of_education = 'none' THEN 'No formal education'\r\n WHEN user_level_of_education = 'b' THEN 'Bachelor''s degree'\r\n WHEN user_level_of_education = 'a' THEN 'Associate degree'\r\n WHEN user_level_of_education = 'hs' THEN 'Secondary/high school'\r\n WHEN user_level_of_education = 'jhs' THEN 'Junior secondary/junior high/middle school'\r\n WHEN user_level_of_education = 'el' THEN 'Elementary/primary school'\r\n WHEN user_level_of_education = 'm' THEN 'Master''s or professional degree'\r\n WHEN user_level_of_education = 'p' THEN 'Doctorate'\r\n WHEN user_level_of_education = 'other' THEN 'Other education'\r\n END AS level_of_education,\r\n CASE\r\n WHEN user_level_of_education = 'none' THEN 1\r\n WHEN user_level_of_education = 'b' THEN 2\r\n WHEN user_level_of_education = 'a' THEN 3\r\n WHEN user_level_of_education = 'hs' THEN 4\r\n WHEN user_level_of_education = 'jhs' THEN 5\r\n WHEN user_level_of_education = 'el' THEN 6\r\n WHEN user_level_of_education = 'm' THEN 7\r\n WHEN user_level_of_education = 'p' THEN 8\r\n WHEN user_level_of_education = 'other' THEN 9\r\n END AS level_of_education_order\r\nFROM openedx.course_enrollments", + "table_name": "Course enrollments", + "template_params": null + } + } + ] +} diff --git a/tutorvision/templates/vision/apps/superset/bootstrap/studentengagement.json b/tutorvision/templates/vision/apps/superset/bootstrap/studentengagement.json new file mode 100644 index 0000000..0523d24 --- /dev/null +++ b/tutorvision/templates/vision/apps/superset/bootstrap/studentengagement.json @@ -0,0 +1,602 @@ +{ + "dashboards": [ + { + "__Dashboard__": { + "css": "", + "dashboard_title": "Student Engagement", + "description": null, + "json_metadata": "{\"timed_refresh_immune_slices\": [], \"filter_scopes\": {\"17\": {\"course_id\": {\"scope\": [\"ROOT_ID\"], \"immune\": []}, \"__time_range\": {\"scope\": [\"ROOT_ID\"], \"immune\": []}}}, \"expanded_slices\": {}, \"refresh_frequency\": 0, \"default_filters\": \"{\\\"17\\\": {\\\"__time_range\\\": \\\"Last week\\\"}}\", \"color_scheme\": null, \"remote_id\": 2}", + "position_json": "{\"CHART-_XdNUl5YJ9\":{\"children\":[],\"id\":\"CHART-_XdNUl5YJ9\",\"meta\":{\"chartId\":17,\"height\":35,\"sliceName\":\"Select course ID and time range\",\"uuid\":\"80ca2797-395e-45cb-a14f-c6a98cf0d9d1\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-yPuXNZUCnv\"],\"type\":\"CHART\"},\"CHART-p4ta63zmN2\":{\"children\":[],\"id\":\"CHART-p4ta63zmN2\",\"meta\":{\"chartId\":6,\"height\":35,\"sliceName\":\"Watched a video\",\"uuid\":\"fdc5ce1f-412f-434f-8a7c-d4ef3d2ede7c\",\"width\":2},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-yPuXNZUCnv\"],\"type\":\"CHART\"},\"CHART-t7KLpPYQxw\":{\"children\":[],\"id\":\"CHART-t7KLpPYQxw\",\"meta\":{\"chartId\":8,\"height\":35,\"sliceName\":\"Tried a problem\",\"uuid\":\"dfd0088c-74dd-4dfb-a221-4c1633d17072\",\"width\":2},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-yPuXNZUCnv\"],\"type\":\"CHART\"},\"CHART-xXCRFE4mZa\":{\"children\":[],\"id\":\"CHART-xXCRFE4mZa\",\"meta\":{\"chartId\":5,\"height\":35,\"sliceName\":\"Active students\",\"uuid\":\"b46a1e93-2bf6-4330-b9b4-67ae57d45a4e\",\"width\":2},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-yPuXNZUCnv\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-yPuXNZUCnv\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"Student Engagement\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-yPuXNZUCnv\":{\"children\":[\"CHART-_XdNUl5YJ9\",\"CHART-xXCRFE4mZa\",\"CHART-p4ta63zmN2\",\"CHART-t7KLpPYQxw\"],\"id\":\"ROW-yPuXNZUCnv\",\"meta\":{\"0\":\"ROOT_ID\",\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", + "slices": [ + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.User events", + "datasource_type": "table", + "id": 6, + "params": "{\"adhoc_filters\": [{\"clause\": \"WHERE\", \"comparator\": \"'play_video'\", \"expressionType\": \"SIMPLE\", \"filterOptionName\": \"filter_7autufejah_v1qfuj79p2a\", \"isExtra\": false, \"isNew\": false, \"operator\": \"==\", \"sqlExpression\": null, \"subject\": \"name\"}], \"datasource\": \"9__table\", \"extra_form_data\": {}, \"granularity_sqla\": \"time\", \"header_font_size\": 0.4, \"metric\": \"Distinct user IDs\", \"slice_id\": 6, \"subheader_font_size\": 0.15, \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"big_number_total\", \"y_axis_format\": \"SMART_NUMBER\", \"remote_id\": 6, \"datasource_name\": \"User events\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Watched a video", + "viz_type": "big_number_total" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.User events", + "datasource_type": "table", + "id": 5, + "params": "{\"adhoc_filters\": [], \"datasource\": \"9__table\", \"extra_form_data\": {}, \"granularity_sqla\": \"time\", \"header_font_size\": 0.4, \"metric\": \"Distinct user IDs\", \"slice_id\": 5, \"subheader\": \"\", \"subheader_font_size\": 0.15, \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"big_number_total\", \"y_axis_format\": \"SMART_NUMBER\", \"remote_id\": 5, \"datasource_name\": \"User events\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Active students", + "viz_type": "big_number_total" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.User events", + "datasource_type": "table", + "id": 8, + "params": "{\"adhoc_filters\": [{\"clause\": \"WHERE\", \"comparator\": \"problem_check\", \"expressionType\": \"SIMPLE\", \"filterOptionName\": \"filter_7autufejah_v1qfuj79p2a\", \"isExtra\": false, \"isNew\": false, \"operator\": \"==\", \"sqlExpression\": null, \"subject\": \"name\"}], \"datasource\": \"9__table\", \"extra_form_data\": {}, \"granularity_sqla\": \"time\", \"header_font_size\": 0.4, \"metric\": \"Distinct user IDs\", \"subheader_font_size\": 0.15, \"time_range\": \"DATEADD(DATETIME(\\\"now\\\"), -7, day) : now\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"big_number_total\", \"y_axis_format\": \"SMART_NUMBER\", \"remote_id\": 8, \"datasource_name\": \"User events\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Tried a problem", + "viz_type": "big_number_total" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.Course enrollments", + "datasource_type": "table", + "id": 17, + "params": "{\"adhoc_filters\": [], \"datasource\": \"10__table\", \"date_filter\": true, \"extra_form_data\": {}, \"filter_configs\": [{\"asc\": true, \"clearable\": true, \"column\": \"course_id\", \"key\": \"CQE2v7Ajx\", \"label\": \"Course ID\", \"multiple\": true, \"searchAllOptions\": false}], \"slice_id\": 17, \"time_grain_sqla\": \"PT1M\", \"time_range\": \"Last week\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"filter_box\", \"remote_id\": 17, \"datasource_name\": \"Course enrollments\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Select course ID and time range", + "viz_type": "filter_box" + } + } + ], + "slug": null + } + } + ], + "datasources": [ + { + "__SqlaTable__": { + "cache_timeout": null, + "columns": [ + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:15:53" + }, + "column_name": "time", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:15:53" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 47, + "is_active": true, + "is_dttm": true, + "python_date_format": null, + "table_id": 9, + "type": "DATETIME", + "uuid": "33179b70-8d58-4f4e-b4b8-be67177ad571", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:15:53" + }, + "column_name": "course_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:15:53" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 48, + "is_active": true, + "is_dttm": false, + "python_date_format": null, + "table_id": 9, + "type": "STRING", + "uuid": "c6e20ed6-d503-44db-8d2c-276759bd3b55", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:15:53" + }, + "column_name": "name", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:15:53" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 49, + "is_active": true, + "is_dttm": false, + "python_date_format": null, + "table_id": 9, + "type": "STRING", + "uuid": "c5b4ef75-3c49-44a2-b266-ce8be95d8db1", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:15:53" + }, + "column_name": "user_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:15:53" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 50, + "is_active": true, + "is_dttm": false, + "python_date_format": null, + "table_id": 9, + "type": "INT64", + "uuid": "e7eb1920-3a1f-4807-a8f6-9009e59ffa74", + "verbose_name": null + } + } + ], + "database_id": 1, + "default_endpoint": null, + "description": null, + "extra": null, + "fetch_values_predicate": null, + "filter_select_enabled": false, + "main_dttm_col": null, + "metrics": [ + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:18:54" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:18:54" + }, + "d3format": null, + "description": null, + "expression": "COUNT(DISTINCT(user_id))", + "extra": "{}", + "id": 15, + "metric_name": "Distinct user IDs", + "metric_type": null, + "table_id": 9, + "uuid": "69932270-ec14-4c56-abf9-8d117ffcd056", + "verbose_name": "", + "warning_text": null + } + } + ], + "offset": 0, + "params": "{\"remote_id\": 9, \"database_name\": \"admin\", \"import_time\": 1621942851}", + "schema": "openedx", + "sql": "SELECT time,\r\n course_id, name, user_id\r\nFROM openedx.events\r\nWHERE event_source = 'browser'", + "table_name": "User events", + "template_params": null + } + }, + { + "__SqlaTable__": { + "cache_timeout": null, + "columns": [ + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "course_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 51, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "f30d1ace-61e7-417d-bb42-cbe2202ba77d", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "enrollment_created", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 52, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "NULLABLE(DATETIME)", + "uuid": "214e5525-f504-4967-98fe-594246784958", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "enrollment_is_active", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 53, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "UINT8", + "uuid": "adda2e42-b890-4614-ad32-70b29df97a50", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "enrollment_mode", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 54, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "95b86e16-a423-46dd-8572-aa59de477a49", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 55, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "UINT64", + "uuid": "349b4226-71f9-4e36-919a-f3166905519f", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_year_of_birth", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 56, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "UINT32", + "uuid": "a97a8573-d7f6-4a2f-a36a-ad6605649785", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_gender", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 57, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "7a78fd5a-7408-4feb-9981-b8a1519305ec", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_level_of_education", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 58, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "14b87739-fd01-460f-a3cc-cd1a222be651", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_city", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 59, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "120e3998-09bd-4c4b-9c60-40bf6406c8ce", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_state", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 60, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "2656d45a-c083-4966-9aef-1370fefda078", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "user_country", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 61, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "STRING", + "uuid": "06f431fd-e83d-4551-8fca-808bc2f39854", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "level_of_education", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 62, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "NULLABLE(STRING)", + "uuid": "0862dff7-c61f-4e72-be59-af77825af9ff", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "level_of_education_order", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 63, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "NULLABLE(UINT8)", + "uuid": "94f69a29-a2fb-4dee-b9ba-e612fed44a73", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "column_name": "gender", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 64, + "is_active": true, + "is_dttm": false, + "python_date_format": null, + "table_id": 10, + "type": "NULLABLE(STRING)", + "uuid": "8b474ea7-40d3-4280-96d9-9a9e15c3997f", + "verbose_name": null + } + } + ], + "database_id": 1, + "default_endpoint": null, + "description": null, + "extra": null, + "fetch_values_predicate": null, + "filter_select_enabled": false, + "main_dttm_col": null, + "metrics": [ + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T16:46:34" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T16:41:16" + }, + "d3format": null, + "description": null, + "expression": "count(*)", + "extra": "{\"warning_markdown\":null}", + "id": 16, + "metric_name": "count", + "metric_type": null, + "table_id": 10, + "uuid": "0967c3ab-96e0-4d06-9861-573e5beac200", + "verbose_name": null, + "warning_text": null + } + } + ], + "offset": 0, + "params": "{\"remote_id\": 10, \"database_name\": \"admin\", \"import_time\": 1622022621}", + "schema": "openedx", + "sql": "SELECT\r\n *,\r\n CASE\r\n WHEN user_gender = 'f' THEN 'Female'\r\n WHEN user_gender = 'm' THEN 'Male'\r\n WHEN user_gender = 'o' THEN 'Other'\r\n END AS gender,\r\n CASE\r\n WHEN user_level_of_education = 'none' THEN 'No formal education'\r\n WHEN user_level_of_education = 'b' THEN 'Bachelor''s degree'\r\n WHEN user_level_of_education = 'a' THEN 'Associate degree'\r\n WHEN user_level_of_education = 'hs' THEN 'Secondary/high school'\r\n WHEN user_level_of_education = 'jhs' THEN 'Junior secondary/junior high/middle school'\r\n WHEN user_level_of_education = 'el' THEN 'Elementary/primary school'\r\n WHEN user_level_of_education = 'm' THEN 'Master''s or professional degree'\r\n WHEN user_level_of_education = 'p' THEN 'Doctorate'\r\n WHEN user_level_of_education = 'other' THEN 'Other education'\r\n END AS level_of_education,\r\n CASE\r\n WHEN user_level_of_education = 'none' THEN 1\r\n WHEN user_level_of_education = 'b' THEN 2\r\n WHEN user_level_of_education = 'a' THEN 3\r\n WHEN user_level_of_education = 'hs' THEN 4\r\n WHEN user_level_of_education = 'jhs' THEN 5\r\n WHEN user_level_of_education = 'el' THEN 6\r\n WHEN user_level_of_education = 'm' THEN 7\r\n WHEN user_level_of_education = 'p' THEN 8\r\n WHEN user_level_of_education = 'other' THEN 9\r\n END AS level_of_education_order\r\nFROM openedx.course_enrollments", + "table_name": "Course enrollments", + "template_params": null + } + } + ] +} \ No newline at end of file diff --git a/tutorvision/templates/vision/apps/superset/bootstrap/videoinsights.json b/tutorvision/templates/vision/apps/superset/bootstrap/videoinsights.json new file mode 100644 index 0000000..1e3f2ee --- /dev/null +++ b/tutorvision/templates/vision/apps/superset/bootstrap/videoinsights.json @@ -0,0 +1,893 @@ +{ + "dashboards": [ + { + "__Dashboard__": { + "css": "", + "dashboard_title": "Video insights", + "description": null, + "json_metadata": "{\"timed_refresh_immune_slices\": [], \"filter_scopes\": {\"2\": {\"video_id\": {\"scope\": [\"ROOT_ID\"], \"immune\": []}}, \"15\": {\"__time_range\": {\"scope\": [\"ROOT_ID\"], \"immune\": []}}, \"18\": {\"course_id\": {\"scope\": [\"ROOT_ID\"], \"immune\": []}}}, \"expanded_slices\": {}, \"refresh_frequency\": 0, \"default_filters\": \"{\\\"15\\\": {\\\"__time_range\\\": \\\"Last week\\\"}}\", \"color_scheme\": null, \"remote_id\": 8}", + "position_json": "{\"CHART-4D6dMxMeMT\":{\"children\":[],\"id\":\"CHART-4D6dMxMeMT\",\"meta\":{\"chartId\":16,\"height\":31,\"sliceName\":\"Average learner watch time\",\"uuid\":\"bfbf6b44-928c-4141-a9c8-ae99cc5a87c4\",\"width\":2},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-Y2wbakJtSE\"],\"type\":\"CHART\"},\"CHART-VhAIYfUByf\":{\"children\":[],\"id\":\"CHART-VhAIYfUByf\",\"meta\":{\"chartId\":18,\"height\":22,\"sliceName\":\"Select video course ID\",\"uuid\":\"1ee17d19-94d5-4b95-8b05-abca22775d41\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-o8xUPCrbfL\"],\"type\":\"CHART\"},\"CHART-WHW9xRSdwN\":{\"children\":[],\"id\":\"CHART-WHW9xRSdwN\",\"meta\":{\"chartId\":4,\"height\":50,\"sliceName\":\"Video X-Ray\",\"uuid\":\"34e9ed3f-c83a-46d2-8e24-e05170104376\",\"width\":6},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-Y2wbakJtSE\"],\"type\":\"CHART\"},\"CHART-cYIhXnlqIn\":{\"children\":[],\"id\":\"CHART-cYIhXnlqIn\",\"meta\":{\"chartId\":14,\"height\":31,\"sliceName\":\"Unique viewers\",\"uuid\":\"e2bc6e71-eefa-457b-b7d8-8bee1ae8bae9\",\"width\":2},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-Y2wbakJtSE\"],\"type\":\"CHART\"},\"CHART-kjX2iwL1Bv\":{\"children\":[],\"id\":\"CHART-kjX2iwL1Bv\",\"meta\":{\"chartId\":15,\"height\":22,\"sliceName\":\"Video event time range\",\"sliceNameOverride\":\"Select time range\",\"uuid\":\"2aca5461-3b26-499d-b6d6-a1edb7f45def\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-o8xUPCrbfL\"],\"type\":\"CHART\"},\"CHART-ozRb_PBpqs\":{\"children\":[],\"id\":\"CHART-ozRb_PBpqs\",\"meta\":{\"chartId\":2,\"height\":22,\"sliceName\":\"Select video ID\",\"uuid\":\"270b8fd9-561b-4b9c-8e4e-9688b43ef283\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-o8xUPCrbfL\"],\"type\":\"CHART\"},\"CHART-qiBNxe25XS\":{\"children\":[],\"id\":\"CHART-qiBNxe25XS\",\"meta\":{\"chartId\":13,\"height\":31,\"sliceName\":\"Total watch time\",\"uuid\":\"9688f829-fa3d-4b0d-a715-cb2beedffe2d\",\"width\":2},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-Y2wbakJtSE\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-o8xUPCrbfL\",\"ROW-Y2wbakJtSE\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"Video insights\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-Y2wbakJtSE\":{\"children\":[\"CHART-cYIhXnlqIn\",\"CHART-qiBNxe25XS\",\"CHART-4D6dMxMeMT\",\"CHART-WHW9xRSdwN\"],\"id\":\"ROW-Y2wbakJtSE\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"},\"ROW-o8xUPCrbfL\":{\"children\":[\"CHART-VhAIYfUByf\",\"CHART-ozRb_PBpqs\",\"CHART-kjX2iwL1Bv\"],\"id\":\"ROW-o8xUPCrbfL\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", + "slices": [ + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.Video views", + "datasource_type": "table", + "id": 4, + "params": "{\"adhoc_filters\": [], \"bar_stacked\": true, \"bottom_margin\": \"auto\", \"color_scheme\": \"supersetColors\", \"columns\": [], \"datasource\": \"5__table\", \"extra_form_data\": {}, \"granularity_sqla\": \"start_time\", \"groupby\": [\"bin\"], \"label_colors\": {}, \"metrics\": [\"Unique viewers\", \"Replay views\"], \"order_bars\": true, \"order_desc\": false, \"row_limit\": 10000, \"show_controls\": true, \"slice_id\": 4, \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"timeseries_limit_metric\": {\"aggregate\": null, \"column\": null, \"expressionType\": \"SQL\", \"hasCustomLabel\": false, \"isNew\": false, \"label\": \"bin\", \"optionName\": \"metric_87t090k54u_lu4sqilcuz\", \"sqlExpression\": \"bin\"}, \"url_params\": {}, \"viz_type\": \"dist_bar\", \"x_ticks_layout\": \"auto\", \"y_axis_bounds\": [null, null], \"y_axis_format\": \"SMART_NUMBER\", \"remote_id\": 4, \"datasource_name\": \"Video views\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Video X-Ray", + "viz_type": "dist_bar" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.Video views", + "datasource_type": "table", + "id": 14, + "params": "{\"adhoc_filters\": [], \"datasource\": \"5__table\", \"extra_form_data\": {}, \"granularity_sqla\": \"start_time\", \"header_font_size\": 0.4, \"metric\": \"Unique viewers\", \"subheader_font_size\": 0.15, \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"big_number_total\", \"y_axis_format\": \"SMART_NUMBER\", \"remote_id\": 14, \"datasource_name\": \"Video views\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Unique viewers", + "viz_type": "big_number_total" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.video_view_segments", + "datasource_type": "table", + "id": 13, + "params": "{\"adhoc_filters\": [], \"datasource\": \"12__table\", \"extra_form_data\": {}, \"granularity_sqla\": \"start_time\", \"header_font_size\": 0.4, \"metric\": \"Watch time (ms)\", \"slice_id\": 13, \"subheader\": \"\", \"subheader_font_size\": 0.15, \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"big_number_total\", \"y_axis_format\": \"DURATION\", \"remote_id\": 13, \"datasource_name\": \"video_view_segments\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Total watch time", + "viz_type": "big_number_total" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.video_view_segments", + "datasource_type": "table", + "id": 16, + "params": "{\"adhoc_filters\": [], \"datasource\": \"12__table\", \"extra_form_data\": {}, \"granularity_sqla\": \"start_time\", \"header_font_size\": 0.4, \"metric\": \"Average watch time (ms)\", \"subheader\": \"\", \"subheader_font_size\": 0.15, \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"big_number_total\", \"y_axis_format\": \"DURATION\", \"remote_id\": 16, \"datasource_name\": \"video_view_segments\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Average learner watch time", + "viz_type": "big_number_total" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.video_events", + "datasource_type": "table", + "id": 2, + "params": "{\"adhoc_filters\": [], \"datasource\": \"6__table\", \"date_filter\": false, \"extra_form_data\": {}, \"filter_configs\": [{\"asc\": true, \"clearable\": false, \"column\": \"video_id\", \"key\": \"9k_yQUvjp\", \"label\": \"Video ID\", \"multiple\": false, \"searchAllOptions\": false}], \"granularity_sqla\": \"time\", \"slice_id\": 2, \"time_grain_sqla\": null, \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"filter_box\", \"remote_id\": 2, \"datasource_name\": \"video_events\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Select video ID", + "viz_type": "filter_box" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.video_events", + "datasource_type": "table", + "id": 18, + "params": "{\"adhoc_filters\": [], \"datasource\": \"6__table\", \"date_filter\": false, \"extra_form_data\": {}, \"filter_configs\": [{\"asc\": true, \"clearable\": true, \"column\": \"course_id\", \"key\": \"CQE2v7Ajx\", \"label\": \"Course ID\", \"multiple\": true, \"searchAllOptions\": false}], \"granularity_sqla\": \"time\", \"time_grain_sqla\": \"P1D\", \"time_range\": \"Last week\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"filter_box\", \"remote_id\": 18, \"datasource_name\": \"video_events\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Select video course ID", + "viz_type": "filter_box" + } + }, + { + "__Slice__": { + "cache_timeout": null, + "datasource_name": "openedx.video_events", + "datasource_type": "table", + "id": 15, + "params": "{\"adhoc_filters\": [], \"datasource\": \"6__table\", \"date_filter\": true, \"extra_form_data\": {}, \"filter_configs\": [], \"granularity_sqla\": \"time\", \"slice_id\": 15, \"time_grain_sqla\": null, \"time_range\": \"Last week\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"url_params\": {}, \"viz_type\": \"filter_box\", \"remote_id\": 15, \"datasource_name\": \"video_events\", \"schema\": \"openedx\", \"database_name\": \"admin\"}", + "slice_name": "Video event time range", + "viz_type": "filter_box" + } + } + ], + "slug": null + } + } + ], + "datasources": [ + { + "__SqlaTable__": { + "cache_timeout": null, + "columns": [ + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "course_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 69, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 12, + "type": "STRING", + "uuid": "c0f70f90-5f3d-4aca-b69f-56512daff155", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "video_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 70, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 12, + "type": "STRING", + "uuid": "83683155-54ab-4921-b16b-9e5d9abe541b", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "user_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 71, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 12, + "type": "INT64", + "uuid": "5ef88039-a246-4df0-a255-8a895005f740", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "start_time", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 72, + "is_active": null, + "is_dttm": true, + "python_date_format": null, + "table_id": 12, + "type": "DATETIME", + "uuid": "a844854f-59d6-4811-8b8b-573b06a91006", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "start_position", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 73, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 12, + "type": "FLOAT", + "uuid": "124ccd6b-6dd6-46c5-b268-b7bf66fc7595", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "start_event", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 74, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 12, + "type": "STRING", + "uuid": "dec56e1c-cf31-4566-804d-4d8efd741037", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "end_time", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 75, + "is_active": null, + "is_dttm": true, + "python_date_format": null, + "table_id": 12, + "type": "DATETIME", + "uuid": "65c2d28c-50dd-4862-951f-baa527c4fa67", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "end_position", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 76, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 12, + "type": "FLOAT", + "uuid": "9881998d-ecfb-4cac-a74e-b6b457059aa8", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "end_event", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 77, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 12, + "type": "STRING", + "uuid": "ae4b8ee8-7f9d-4682-8da3-a61d685d90fe", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "column_name": "duration", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 78, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 12, + "type": "FLOAT", + "uuid": "bf49c6fe-26ad-476d-ab49-234a053ceb74", + "verbose_name": null + } + } + ], + "database_id": 1, + "default_endpoint": null, + "description": null, + "extra": null, + "fetch_values_predicate": null, + "filter_select_enabled": false, + "main_dttm_col": "start_time", + "metrics": [ + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:12:39" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:12:00" + }, + "d3format": null, + "description": null, + "expression": "COUNT(*)", + "extra": "{\"warning_markdown\":null}", + "id": 20, + "metric_name": "count", + "metric_type": null, + "table_id": 12, + "uuid": "79c9721e-ac8d-41a0-b36e-06ff33f16a6a", + "verbose_name": "COUNT(*)", + "warning_text": null + } + }, + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T15:43:59" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:19:14" + }, + "d3format": null, + "description": null, + "expression": "SUM(end_position - start_position) * 1000", + "extra": "{\"warning_markdown\":null}", + "id": 22, + "metric_name": "Watch time (ms)", + "metric_type": null, + "table_id": 12, + "uuid": "70ceb0e6-7397-43bb-8565-e5b97ab9f536", + "verbose_name": "", + "warning_text": null + } + }, + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T16:04:29" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T15:43:59" + }, + "d3format": null, + "description": null, + "expression": "MAX(end_position) - MIN(start_position)", + "extra": "{\"warning_markdown\":null}", + "id": 23, + "metric_name": "Portion viewed per user", + "metric_type": null, + "table_id": 12, + "uuid": "bb800160-8987-4f93-8715-7d50e7a060d9", + "verbose_name": "", + "warning_text": null + } + }, + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T16:05:52" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T16:04:29" + }, + "d3format": null, + "description": null, + "expression": "SUM(end_position - start_position) * 1000 / COUNT(DISTINCT(user_id))", + "extra": "{\"warning_markdown\":null}", + "id": 24, + "metric_name": "Average watch time (ms)", + "metric_type": null, + "table_id": 12, + "uuid": "42b1b78b-275c-4181-88ff-99d6cf158c8e", + "verbose_name": "", + "warning_text": null + } + } + ], + "offset": 0, + "params": "{\"remote_id\": 12, \"database_name\": \"admin\", \"import_time\": 1621942851}", + "schema": "openedx", + "sql": "", + "table_name": "video_view_segments", + "template_params": null + } + }, + { + "__SqlaTable__": { + "cache_timeout": null, + "columns": [ + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T11:20:55" + }, + "column_name": "bin", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T10:59:59" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 24, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 5, + "type": "UINT64", + "uuid": "c05a60c8-f85a-42e6-92a4-d7c7743fe517", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T11:33:27" + }, + "column_name": "video_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:23:48" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 34, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 5, + "type": "STRING", + "uuid": "482c34a7-ab7b-431f-96e2-a85a4e11ca7c", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T11:35:16" + }, + "column_name": "start_time", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:33:38" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 36, + "is_active": null, + "is_dttm": true, + "python_date_format": null, + "table_id": 5, + "type": "DATETIME", + "uuid": "25d3ef61-62d2-429c-9601-b2644e3b7b40", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T15:52:44" + }, + "column_name": "user_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T14:43:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 37, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 5, + "type": "INT64", + "uuid": "883b1869-5061-416b-a87a-5d4f58e65935", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T15:52:44" + }, + "column_name": "course_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T14:43:16" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 38, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 5, + "type": "STRING", + "uuid": "73186ac4-bef3-418d-a87e-d12d0797dc77", + "verbose_name": null + } + } + ], + "database_id": 1, + "default_endpoint": null, + "description": null, + "extra": null, + "fetch_values_predicate": null, + "filter_select_enabled": false, + "main_dttm_col": null, + "metrics": [ + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T11:33:27" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:25:39" + }, + "d3format": null, + "description": null, + "expression": "count(*)", + "extra": "{\"warning_markdown\":null}", + "id": 9, + "metric_name": "Total views", + "metric_type": null, + "table_id": 5, + "uuid": "b72919ce-a9df-4b3d-b827-a1a6a1c8a7d2", + "verbose_name": "", + "warning_text": null + } + }, + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T11:33:27" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:29:01" + }, + "d3format": null, + "description": null, + "expression": "count(distinct(user_id))", + "extra": "{\"warning_markdown\":null}", + "id": 10, + "metric_name": "Unique viewers", + "metric_type": null, + "table_id": 5, + "uuid": "1a29c35b-1dbf-4145-9ef0-403b9b0562e5", + "verbose_name": "", + "warning_text": null + } + }, + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-20T11:33:27" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:29:01" + }, + "d3format": null, + "description": null, + "expression": "count(*) - count(distinct(user_id))", + "extra": "{\"warning_markdown\":null}", + "id": 11, + "metric_name": "Replay views", + "metric_type": null, + "table_id": 5, + "uuid": "2d9e87a5-1cc4-44ec-8b6a-4d8c48e6d2d0", + "verbose_name": "", + "warning_text": null + } + } + ], + "offset": 0, + "params": "{\"remote_id\": 5, \"database_name\": \"admin\", \"import_time\": 1621942851}", + "schema": "openedx", + "sql": "SELECT arrayJoin(range(toUInt64(floor(start_position)), toUInt64(ceil(end_position)))) AS bin,\r\n start_time,\r\n user_id,\r\n video_id,\r\n course_id\r\nFROM video_view_segments\r\nORDER BY bin", + "table_name": "Video views", + "template_params": "" + } + }, + { + "__SqlaTable__": { + "cache_timeout": null, + "columns": [ + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:07:01" + }, + "column_name": "course_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:11:29" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 28, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 6, + "type": "STRING", + "uuid": "cc99f7d6-a643-4989-a1bb-07b2dfaed0c8", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:07:01" + }, + "column_name": "video_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:11:29" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 29, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 6, + "type": "STRING", + "uuid": "77f01447-3b85-42b2-a00d-23accf89f0a4", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:07:01" + }, + "column_name": "user_id", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:11:29" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 30, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 6, + "type": "INT64", + "uuid": "37e2fff6-1c54-4569-aee3-160ada330b15", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:07:01" + }, + "column_name": "name", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:11:29" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 31, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 6, + "type": "STRING", + "uuid": "522466fd-fef8-405e-b5e5-dc800348f821", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:07:01" + }, + "column_name": "time", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:11:29" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 32, + "is_active": null, + "is_dttm": true, + "python_date_format": null, + "table_id": 6, + "type": "DATETIME", + "uuid": "b4581512-5a65-4178-b8e3-fbfd7b30d2a3", + "verbose_name": null + } + }, + { + "__TableColumn__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:07:01" + }, + "column_name": "position", + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:11:29" + }, + "description": null, + "expression": null, + "filterable": true, + "groupby": true, + "id": 33, + "is_active": null, + "is_dttm": false, + "python_date_format": null, + "table_id": 6, + "type": "FLOAT", + "uuid": "9057df8b-e555-4a85-ad6a-50dcdb2e778f", + "verbose_name": null + } + } + ], + "database_id": 1, + "default_endpoint": null, + "description": null, + "extra": null, + "fetch_values_predicate": null, + "filter_select_enabled": false, + "main_dttm_col": "time", + "metrics": [ + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:07:01" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-20T11:11:29" + }, + "d3format": null, + "description": null, + "expression": "COUNT(*)", + "extra": "{\"warning_markdown\":null}", + "id": 8, + "metric_name": "count", + "metric_type": null, + "table_id": 6, + "uuid": "8df32dd0-fa09-4805-bf56-b720bfbbd1d1", + "verbose_name": "COUNT(*)", + "warning_text": null + } + }, + { + "__SqlMetric__": { + "changed_by_fk": 1, + "changed_on": { + "__datetime__": "2021-05-24T10:07:01" + }, + "created_by_fk": 1, + "created_on": { + "__datetime__": "2021-05-24T10:07:01" + }, + "d3format": null, + "description": null, + "expression": "sum(end_position - start_position)", + "extra": "{}", + "id": 19, + "metric_name": "View time", + "metric_type": null, + "table_id": 6, + "uuid": "cf034920-8bba-43d6-95da-8bb6ec8ef1d5", + "verbose_name": "", + "warning_text": null + } + } + ], + "offset": 0, + "params": "{\"remote_id\": 6, \"database_name\": \"admin\", \"import_time\": 1621942851}", + "schema": "openedx", + "sql": "", + "table_name": "video_events", + "template_params": null + } + } + ] +} \ No newline at end of file diff --git a/tutorvision/templates/vision/apps/superset/superset_config.py b/tutorvision/templates/vision/apps/superset/superset_config.py new file mode 100644 index 0000000..736bcfb --- /dev/null +++ b/tutorvision/templates/vision/apps/superset/superset_config.py @@ -0,0 +1,67 @@ +import logging + +from cachelib.redis import RedisCache +from celery.schedules import crontab + +# https://superset.apache.org/docs/installation/configuring-superset +SECRET_KEY = "{{ VISION_SUPERSET_SECRET_KEY }}" +SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://{{ VISION_POSTGRESQL_USER }}:{{ VISION_POSTGRESQL_PASSWORD }}@vision-postgresql/{{ VISION_POSTGRESQL_DB }}" + +DATA_CACHE_CONFIG = { + "CACHE_TYPE": "redis", + "CACHE_DEFAULT_TIMEOUT": 60 * 60 * 24, # 1 day default (in secs) + "CACHE_KEY_PREFIX": "superset_results", + "CACHE_REDIS_URL": "redis://redis:6379/0", +} +CACHE_CONFIG = DATA_CACHE_CONFIG + +# Borrowed from superset/docker/pythonpath_dev/superset_config.py +REDIS_HOST = "redis" +REDIS_PORT = "6379" +REDIS_CELERY_DB = 0 +REDIS_RESULTS_DB = 1 +RESULTS_BACKEND = RedisCache(host="redis", port=6379, key_prefix="superset_results") + +class CeleryConfig: # pylint: disable=too-few-public-methods + BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}" + CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks") + CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}" + CELERYD_LOG_LEVEL = "DEBUG" + CELERYD_PREFETCH_MULTIPLIER = 1 + CELERY_ACKS_LATE = False + CELERY_ANNOTATIONS = { + "sql_lab.get_sql_results": {"rate_limit": "100/s"}, + "email_reports.send": { + "rate_limit": "1/s", + "time_limit": 120, + "soft_time_limit": 150, + "ignore_result": True, + }, + } + CELERYBEAT_SCHEDULE = { + "email_reports.schedule_hourly": { + "task": "email_reports.schedule_hourly", + "schedule": crontab(minute=1, hour="*"), + }, + "reports.scheduler": { + "task": "reports.scheduler", + "schedule": crontab(minute="*", hour="*"), + }, + "reports.prune_log": { + "task": "reports.prune_log", + "schedule": crontab(minute=0, hour=0), + }, + } + + +CELERY_CONFIG = CeleryConfig + +# Avoid duplicate logging because of propagation to root logger +logging.getLogger("superset").propagate = False + +# Enable some custom feature flags +# Do this once native filters are fully functional https://github.com/apache/superset/projects/15+ +# def get_vision_feature_flags(flags): +# flags["DASHBOARD_NATIVE_FILTERS"] = True +# return flags +# GET_FEATURE_FLAGS_FUNC = get_vision_feature_flags diff --git a/tutorvision/templates/vision/build/vision-clickhouse/Dockerfile b/tutorvision/templates/vision/build/vision-clickhouse/Dockerfile new file mode 100644 index 0000000..5e8fe43 --- /dev/null +++ b/tutorvision/templates/vision/build/vision-clickhouse/Dockerfile @@ -0,0 +1,6 @@ +FROM docker.io/yandex/clickhouse-server:21.2.7.11 + +RUN apt update && apt install -y python3 +COPY ./scripts /scripts +RUN chmod a+x /scripts/* +ENV PATH /scripts:${PATH} diff --git a/tutorvision/templates/vision/build/vision-clickhouse/scripts/vision b/tutorvision/templates/vision/build/vision-clickhouse/scripts/vision new file mode 100644 index 0000000..c63f798 --- /dev/null +++ b/tutorvision/templates/vision/build/vision-clickhouse/scripts/vision @@ -0,0 +1,105 @@ +#! /usr/bin/env python3 +import argparse +import os +import subprocess + + +def main(): + parser = argparse.ArgumentParser("Manage your Clickhouse instance") + subparsers = parser.add_subparsers() + + # Run a clickhouse client + parser_client = subparsers.add_parser("client") + parser_client.set_defaults(func=command_client) + + # Create user + parser_createuser = subparsers.add_parser("createuser") + parser_createuser.add_argument( + "-c", + "--course-id", + action="append", + help="Restrict user to access data only from these courses.", + ) + parser_createuser.add_argument( + "-o", + "--org-id", + action="append", + help="Restrict user to access data only from these organizations.", + ) + parser_createuser.add_argument("username") + parser_createuser.set_defaults(func=command_create_user) + + args = parser.parse_args() + args.func(args) + + +def command_client(args): + run_query() + + +def command_create_user(args): + conditions = [] + course_ids = args.course_id or [] + org_ids = args.org_id or [] + for course_id in course_ids: + conditions.append(f"course_id = '{course_id}'") + for org_id in org_ids: + conditions.append(f"course_id LIKE 'course-v1:{org_id}+%'") + condition = " OR ".join(conditions) if conditions else "1" + username = args.username + # Note that the "CREATE TEMPORARY TABLE" grant is required to make use of "numbers()" functions. + # TODO how to automatically list the tables to which users should be granted access? + query = f""" +CREATE USER IF NOT EXISTS {username}; +GRANT CREATE TEMPORARY TABLE ON *.* TO {username}; + +GRANT SELECT ON events TO {username}; +CREATE ROW POLICY OR REPLACE {username} ON events AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; + +GRANT SELECT ON course_grades TO {username}; +CREATE ROW POLICY OR REPLACE {username} ON course_grades AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; + +GRANT SELECT ON course_enrollments TO {username}; +CREATE ROW POLICY OR REPLACE {username} ON course_enrollments AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; + +GRANT SELECT ON video_events TO {username}; +CREATE ROW POLICY OR REPLACE {username} ON video_events AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; + +GRANT SELECT ON video_view_segments TO {username}; +CREATE ROW POLICY OR REPLACE {username} ON video_view_segments AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; + +GRANT SELECT ON course_block_completion TO {username}; +CREATE ROW POLICY OR REPLACE {username} ON course_block_completion AS RESTRICTIVE FOR SELECT USING {condition} TO {username}; +""" + run_query(query) + + +def run_query(query=None): + args = [] + if os.environ.get("VISION_CLICKHOUSE_SCHEME") == "https": + args.append("--secure") + if query: + args += ["--query", query] + command = [ + "clickhouse", + "client", + "--host", + os.environ["VISION_CLICKHOUSE_HOST"], + "--port", + os.environ["VISION_CLICKHOUSE_PORT"], + "--user", + os.environ["VISION_CLICKHOUSE_USERNAME"], + "--password", + os.environ["VISION_CLICKHOUSE_PASSWORD"], + "--database", + os.environ["VISION_CLICKHOUSE_DATABASE"], + "--multiline", + "--multiquery", + *args, + ] + print(" ".join(command)) + subprocess.check_call(command) + + +if __name__ == "__main__": + main() diff --git a/tutorvision/templates/vision/build/vision-superset/Dockerfile b/tutorvision/templates/vision/build/vision-superset/Dockerfile new file mode 100644 index 0000000..7770789 --- /dev/null +++ b/tutorvision/templates/vision/build/vision-superset/Dockerfile @@ -0,0 +1,29 @@ +# Superset image with additional database drivers +# https://hub.docker.com/r/apache/superset +# https://superset.apache.org/docs/databases/installing-database-drivers +FROM docker.io/apache/superset:a9d888ad402ebb35da45df446997c426d6abee9d + +USER root +# https://pypi.org/project/clickhouse-driver/ +# https://pypi.org/project/clickhouse-sqlalchemy/ +# https://pypi.org/project/mysqlclient/ +RUN pip install clickhouse-driver==0.2.0 clickhouse-sqlalchemy==0.1.6 mysqlclient==2.0.3 + +COPY ./scripts /scripts +RUN chmod a+x /scripts/* +ENV PATH /scripts:${PATH} + +USER superset + +ENTRYPOINT [] +CMD gunicorn \ + --bind "0.0.0.0:8000" \ + --access-logfile '-' \ + --error-logfile '-' \ + --workers 2 \ + --worker-class gthread \ + --threads 20 \ + --timeout 60 \ + --limit-request-line 0 \ + --limit-request-field_size 0 \ + "${FLASK_APP}" diff --git a/tutorvision/templates/vision/build/vision-superset/scripts/vision b/tutorvision/templates/vision/build/vision-superset/scripts/vision new file mode 100644 index 0000000..7abde1b --- /dev/null +++ b/tutorvision/templates/vision/build/vision-superset/scripts/vision @@ -0,0 +1,249 @@ +#! /usr/bin/env python3 + +import argparse +from getpass import getpass +import json +import os +from time import time + +from superset.app import create_app + +app = create_app() +app.app_context().push() + +from superset.connectors.sqla.models import SqlaTable +from superset.models.core import Database +from superset.models.slice import Slice +from superset.extensions import db, security_manager +import superset.dashboards.commands.importers.v0 as importers +from superset.utils.core import get_or_create_db +from werkzeug.security import generate_password_hash + + +now = time() + + +def main(): + parser = argparse.ArgumentParser( + description="Bootstrap user creation and dashboards" + ) + subparsers = parser.add_subparsers() + + # Create user + parser_user = subparsers.add_parser("createuser", help="Create or update user") + parser_user.add_argument( + "-d", + "--db", + "--database", + help=( + "Name of the Superset database to which the user should be granted access." + " Defaults to the username." + ), + ) + parser_user.add_argument( + "-r", + "--role", + help=( + "Name of the role to which the user should be assigned." + " Defaults to the username." + ), + ) + parser_user.add_argument( + "-p", + "--password", + help="User password. If undefined, you will be prompted for one.", + ) + parser_user.add_argument( + "--firstname", default="", help="User first name (optional)." + ) + parser_user.add_argument( + "--lastname", default="", help="User last name (optional)." + ) + parser_user.add_argument("username") + parser_user.add_argument("email") + parser_user.set_defaults(func=bootstrap_user) + + # Bootstrap dashboards + parser_dashboards = subparsers.add_parser( + "bootstrap-dashboards", + help="Bootstrap datasets and dashboards for a given user", + ) + parser_dashboards.add_argument( + "-d", + "--db", + "--database", + help=( + "Name of the Superset database to which the objects should be linked." + " By default, this will be the same as the username." + ), + ) + parser_dashboards.add_argument("username") + parser_dashboards.add_argument("path", nargs="+") + parser_dashboards.set_defaults(func=bootstrap_dashboards) + + args = parser.parse_args() + args.func(args) + + +def bootstrap_user(args): + # Bootstrap database + database_name = args.db or args.username + + # Get or create user + user = security_manager.find_user(args.username) + if user: + print(f"User '{args.username}' already exists. Skipping creation.") + if args.password: + print("Setting user password...") + user.password = generate_password_hash(args.password) + db.session.add(user) + db.session.commit() + else: + print(f"Creating user '{args.username}'...") + password = args.password + while not password: + password = getpass() + security_manager.add_user( + args.username, + args.firstname, + args.lastname, + args.email, + security_manager.find_role("Gamma"), + password=password, + ) + user = security_manager.find_user(args.username) + + # Associate role with the same name to user, if it exists + role_name = args.role or args.username + + def check_permission(permission_view): + permission_name = str(permission_view) + if permission_name in [ + "can save on Datasource", + "can sqllab on Superset", + "can sql json on Superset", + "menu access on Datasets", + "menu access on SQL Lab", + ]: + return True + if permission_name.startswith(f"database access on [{database_name}]"): + return True + if permission_name.startswith(f"schema access on [{database_name}]"): + return True + return False + + security_manager.set_role(role_name, check_permission) + role = security_manager.find_role(role_name) + if role in user.roles: + print(f"Role '{role_name}' is already associated to user.") + else: + print(f"Associating role '{role_name}' to user...") + user.roles.append(role) + db.session.add(user) + db.session.commit() + print("Done.") + + +def bootstrap_database(database_name): + host = os.environ["VISION_CLICKHOUSE_HOST"] + port = os.environ["VISION_CLICKHOUSE_PORT"] + database = os.environ["VISION_CLICKHOUSE_DATABASE"] + uri = f"clickhouse+native://{username}:@{host}:{port}/{database}" + get_or_create_db(database_name, uri, always_create=True) + + +def bootstrap_dashboards(args): + database_name = args.db or args.username + database = load_database(database_name) + user = load_user(args.username) + + for path in args.path: + print( + "importing dashboard {} for user='{}' db='{}'... ".format( + path, user.username, database.database_name + ) + ) + dashboard = load_dashboard_file(path) + import_dashboard(dashboard, user, database) + + +def load_database(database_name): + return ( + db.session.query(Database).filter(Database.database_name == database_name).one() + ) + + +def load_user(username): + return ( + db.session.query(security_manager.user_model) + .filter(security_manager.user_model.username == username) + .one() + ) + + +def load_dashboard_file(path): + with open(path) as f: + return json.load(f, object_hook=importers.decode_dashboards) + + +def import_dashboard(data, user, database): + # Load datasets + for dataset in data["datasources"]: + dataset.params = "{}" + # This should overwrite the existing dataset, if any with the same name + new_dataset_id = importers.import_dataset(dataset, database.id, now) + new_dataset = db.session.query(SqlaTable).get(new_dataset_id) + # Make current user the new owner + new_dataset.owners.append(user) + db.session.add(new_dataset) + # db.session.commit() + + for dashboard in data["dashboards"]: + # Load slices + new_slices = [] + slices = dashboard.slices[:] + old_to_new_slice_id_map = {} + + for slice in slices: + # Make sure the new slice does not point to the old slice + params_dict = slice.params_dict + params_dict["database_name"] = database.name + params_dict.pop("remote_id") + slice.params = json.dumps(params_dict) + old_slice_id = slice.id + slice.id = None + # Create new slice + new_slice_id = importers.import_chart(slice, None, now) + new_slice = db.session.query(Slice).get(new_slice_id) + old_to_new_slice_id_map[old_slice_id] = new_slice_id + # Make current user the new owner + new_slice.owners.append(user) + new_slices.append(new_slice) + db.session.add(new_slice) + # db.session.commit() + + # Add slices to dashboard + dashboard.slices = new_slices + # Set dashboard owner + dashboard.owners.append(user) + # Update position JSON + position = dashboard.position.copy() + for chart in position.values(): + if ( + "meta" in chart + and isinstance(chart["meta"], dict) + and "chartId" in chart["meta"] + ): + chart["meta"]["chartId"] = old_to_new_slice_id_map[ + chart["meta"]["chartId"] + ] + dashboard.position_json = json.dumps(position) + # Load dashboard + db.session.add(dashboard) + + # Commit changes + db.session.commit() + + +if __name__ == "__main__": + main() diff --git a/tutorvision/templates/vision/hooks/vision-redash/init b/tutorvision/templates/vision/hooks/vision-redash/init deleted file mode 100644 index c4fdceb..0000000 --- a/tutorvision/templates/vision/hooks/vision-redash/init +++ /dev/null @@ -1 +0,0 @@ -./manage.py database create_tables diff --git a/tutorvision/templates/vision/hooks/vision-superset/init b/tutorvision/templates/vision/hooks/vision-superset/init new file mode 100644 index 0000000..da8f592 --- /dev/null +++ b/tutorvision/templates/vision/hooks/vision-superset/init @@ -0,0 +1,8 @@ +# Apply migrations +superset db upgrade + +# Create default roles and permissions +superset init + +# Create/Update database with full access +superset set-database-uri --database-name={{ VISION_SUPERSET_DATABASE }} --uri='clickhouse+native://{{ VISION_CLICKHOUSE_USERNAME }}:{{ VISION_CLICKHOUSE_PASSWORD }}@{{ VISION_CLICKHOUSE_HOST }}:{{ VISION_CLICKHOUSE_PORT }}/{{ VISION_CLICKHOUSE_DATABASE }}'