Get rid of a few TODOs
This commit is contained in:
Régis Behmo
parent
2d1313f472
commit
c2096efa72
17
README.rst
17
README.rst
@ -3,13 +3,6 @@ Tutor Vision: scalable, real-time analytics for Open edX
|
||||
|
||||
TODO:
|
||||
|
||||
- Expose data with superset
|
||||
- Expose grades and certificates
|
||||
- Reproduce dashboards from https://edx.readthedocs.io/projects/edx-insights/en/latest/Overview.html
|
||||
- Reproduce dashboards from https://datastudio.google.com/embed/u/0/reporting/1gd-YXUtHFzHm3qddPTO8r272kyRD-uDG/page/4f5xB
|
||||
- frontend user creation:
|
||||
- generate random frontend user password in "tutor vision frontend createuser"
|
||||
- try out alerts
|
||||
- Kubernetes compatibility
|
||||
- Sweet readme
|
||||
- Rename to ocean?
|
||||
@ -49,6 +42,16 @@ Then, create the corresponding user on the frontend::
|
||||
|
||||
Your frontend user will automatically be associated to the datalake database you created, provided they share the same name.
|
||||
|
||||
Course block IDs and names are loaded from the Open edX modulestore into the datalake. After making changes to your course, you might want to refresh the course structure stored in the datalake. To do so, run::
|
||||
|
||||
tutor local init --limit=vision
|
||||
|
||||
Or, if you want to avoid running the full plugin initialization::
|
||||
|
||||
tutor local run -v $(tutor config printroot)/env/plugins/vision/apps/openedx/scripts/:/openedx/scripts lms \
|
||||
python /openedx/scripts/importcoursedata.py \
|
||||
"http://$(tutor config printvalue VISION_CLICKHOUSE_USERNAME):$(tutor config printvalue VISION_CLICKHOUSE_PASSWORD)@$(tutor config printvalue VISION_CLICKHOUSE_HOST):$(tutor config printvalue VISION_CLICKHOUSE_HTTP_PORT)/?database=$(tutor config printvalue VISION_CLICKHOUSE_DATABASE)"
|
||||
|
||||
Development
|
||||
-----------
|
||||
|
||||
|
||||
@ -1,135 +0,0 @@
|
||||
# TODO delete this module
|
||||
# import click
|
||||
#
|
||||
# from tutor import config as tutor_config
|
||||
# from tutor.commands.compose import ComposeJobRunner
|
||||
# from tutor.commands.local import docker_compose as local_docker_compose
|
||||
#
|
||||
#
|
||||
# @click.group(help="Manage your Vision platform")
|
||||
# def vision_command():
|
||||
# pass
|
||||
#
|
||||
#
|
||||
# @click.group(help="Print convenient commands that can be sent to the right containers", name="print")
|
||||
# def print_command():
|
||||
# pass
|
||||
#
|
||||
#
|
||||
# @click.command(name="datalake-create-user", help="Print user creation query")
|
||||
# @click.argument("username")
|
||||
# @click.pass_obj
|
||||
# def print_datalakecreateuser(context, username):
|
||||
# print(f"CREATE USER IF NOT EXISTS {username}")
|
||||
#
|
||||
#
|
||||
# # @click.command(name="setpermissions", help="Restrict user access")
|
||||
# # @click.argument("username")
|
||||
# # @click.option(
|
||||
# # "-c",
|
||||
# # "--course-id",
|
||||
# # "course_ids",
|
||||
# # multiple=True,
|
||||
# # help=(
|
||||
# # "Grant access to a course data. This option may be used multiple times to grant "
|
||||
# # "access to multiple courses."
|
||||
# # ),
|
||||
# # )
|
||||
# # @click.option(
|
||||
# # "-o",
|
||||
# # "--org-id",
|
||||
# # "org_ids",
|
||||
# # multiple=True,
|
||||
# # help=(
|
||||
# # "Grant access to the course data of an organization. This option may be used multiple times to grant "
|
||||
# # "access to multiple organizations."
|
||||
# # ),
|
||||
# # )
|
||||
# # @click.pass_obj
|
||||
# # def datalake_setpermissions(context, username, course_ids, org_ids):
|
||||
# # conditions = []
|
||||
# # for course_id in course_ids:
|
||||
# # conditions.append(f"course_id = '{course_id}'")
|
||||
# # for org_id in org_ids:
|
||||
# # conditions.append(f"course_id LIKE 'course-v1:{org_id}+%'")
|
||||
# # condition = "1"
|
||||
# # if conditions:
|
||||
# # condition = " OR ".join(conditions)
|
||||
# #
|
||||
# # # Note that the "CREATE TEMPORARY TABLE" grant is required to make use of "numbers()" functions.
|
||||
# # query = f"""
|
||||
# # GRANT CREATE TEMPORARY TABLE ON *.* TO {username};
|
||||
# #
|
||||
# # GRANT SELECT ON events TO {username};
|
||||
# # CREATE ROW POLICY OR REPLACE {username} ON events AS RESTRICTIVE FOR SELECT USING {condition} TO {username};
|
||||
# #
|
||||
# # GRANT SELECT ON course_grades TO {username};
|
||||
# # CREATE ROW POLICY OR REPLACE {username} ON course_grades AS RESTRICTIVE FOR SELECT USING {condition} TO {username};
|
||||
# #
|
||||
# # GRANT SELECT ON course_enrollments TO {username};
|
||||
# # CREATE ROW POLICY OR REPLACE {username} ON course_enrollments AS RESTRICTIVE FOR SELECT USING {condition} TO {username};
|
||||
# #
|
||||
# # GRANT SELECT ON video_events TO {username};
|
||||
# # CREATE ROW POLICY OR REPLACE {username} ON video_events AS RESTRICTIVE FOR SELECT USING {condition} TO {username};
|
||||
# #
|
||||
# # GRANT SELECT ON video_view_segments TO {username};
|
||||
# # CREATE ROW POLICY OR REPLACE {username} ON video_view_segments AS RESTRICTIVE FOR SELECT USING {condition} TO {username};
|
||||
# # """
|
||||
# # run_datalake_query(context.root, query)
|
||||
# #
|
||||
# #
|
||||
# # def run_datalake_query(root, query):
|
||||
# # config = tutor_config.load(root)
|
||||
# # command_secure_opt = "--secure" if config["VISION_CLICKHOUSE_SCHEME"] == "https" else ""
|
||||
# # command = f"""clickhouse client \
|
||||
# # {command_secure_opt} --host={config["VISION_CLICKHOUSE_HOST"]} --port={config["VISION_CLICKHOUSE_PORT"]} \
|
||||
# # --user={config["VISION_CLICKHOUSE_USERNAME"]} \
|
||||
# # --password={config["VISION_CLICKHOUSE_PASSWORD"]} \
|
||||
# # --database={config["VISION_CLICKHOUSE_DATABASE"]} \
|
||||
# # --multiline --multiquery \
|
||||
# # --query "{query}"
|
||||
# # """
|
||||
# # runner = ComposeJobRunner(root, config, local_docker_compose)
|
||||
# # runner.run_job("vision-clickhouse", command)
|
||||
# #
|
||||
# #
|
||||
# # @click.group(name="frontend", help="Manage the frontend access")
|
||||
# # def frontend_command():
|
||||
# # pass
|
||||
# #
|
||||
# #
|
||||
# # @click.command(name="createuser", help="Create a new user to access the frontend")
|
||||
# # @click.option(
|
||||
# # "-p",
|
||||
# # "--password",
|
||||
# # default="",
|
||||
# # prompt="User password",
|
||||
# # hide_input=True,
|
||||
# # confirmation_prompt=True,
|
||||
# # help="User password: if undefined you will be prompted to input a password",
|
||||
# # )
|
||||
# # @click.option(
|
||||
# # "-r",
|
||||
# # "--admin",
|
||||
# # "is_admin",
|
||||
# # is_flag=True,
|
||||
# # default=False,
|
||||
# # help="Grant root/administration privileges on the frontend to this user",
|
||||
# # )
|
||||
# # @click.argument("username")
|
||||
# # @click.argument("email")
|
||||
# # @click.pass_obj
|
||||
# # def frontend_createuser(context, password, is_admin, username, email):
|
||||
# # config = tutor_config.load(context.root)
|
||||
# # # TODO in case of non-admin, we must define a --role
|
||||
# # fab_cmd = "create-admin" if is_admin else "create-user"
|
||||
# # command = f"superset fab {fab_cmd} --username {username} --email {email} --password {password}"
|
||||
# # runner = ComposeJobRunner(context.root, config, local_docker_compose)
|
||||
# # runner.run_job("vision-superset", command)
|
||||
#
|
||||
#
|
||||
# print_command.add_command(print_datalakecreateuser)
|
||||
# # datalake.add_command(datalake_setpermissions)
|
||||
# vision_command.add_command(print_command)
|
||||
# # frontend_command.add_command(frontend_createuser)
|
||||
# # vision_command.add_command(frontend_command)
|
||||
@ -10,3 +10,14 @@ vision-superset-job:
|
||||
depends_on:
|
||||
- vision-postgresql
|
||||
- vision-redis
|
||||
vision-openedx-job:
|
||||
image: {{ DOCKER_IMAGE_OPENEDX }}
|
||||
environment:
|
||||
SERVICE_VARIANT: lms
|
||||
SETTINGS: ${TUTOR_EDX_PLATFORM_SETTINGS:-tutor.production}
|
||||
volumes:
|
||||
- ../apps/openedx/settings/lms/:/openedx/edx-platform/lms/envs/tutor/:ro
|
||||
- ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/:ro
|
||||
- ../apps/openedx/config/:/openedx/config/:ro
|
||||
- ../plugins/vision/apps/openedx/scripts/:/openedx/scripts/:ro
|
||||
depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB)]|list_if }}
|
||||
@ -2,7 +2,6 @@ from glob import glob
|
||||
import os
|
||||
|
||||
from .__about__ import __version__
|
||||
# from .cli import vision_command # TODO remove this
|
||||
|
||||
HERE = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
@ -40,9 +39,8 @@ hooks = {
|
||||
"vision-clickhouse": "{{ VISION_CLICKHOUSE_DOCKER_IMAGE }}",
|
||||
"vision-superset": "{{ VISION_SUPERSET_DOCKER_IMAGE }}"
|
||||
},
|
||||
"init": ["vision-clickhouse", "vision-superset"],
|
||||
"init": ["vision-clickhouse", "vision-superset", "vision-openedx"],
|
||||
}
|
||||
# command = vision_command # TODO remove this
|
||||
|
||||
|
||||
def patches():
|
||||
|
||||
@ -1,23 +1,33 @@
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import argparse
|
||||
|
||||
import requests
|
||||
from MySQLdb import escape_string as sql_escape_string
|
||||
|
||||
import lms.startup
|
||||
|
||||
lms.startup.run()
|
||||
|
||||
from courseware.courses import get_course
|
||||
from MySQLdb import escape_string as sql_escape_string
|
||||
from opaque_keys.edx.keys import CourseKey
|
||||
from xmodule.modulestore.django import modulestore
|
||||
|
||||
# TODO actually run this script during init by mounting inside the lms container
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Import course block information into the datalake"
|
||||
)
|
||||
parser.add_argument("-c", "--course-id", action="append", help="Limit import to these courses")
|
||||
parser.add_argument("uri", help="Clickhouse URI")
|
||||
args = parser.parse_args()
|
||||
|
||||
module_store = modulestore()
|
||||
course_ids = args.course_id or []
|
||||
for course in module_store.get_courses():
|
||||
import_course(course.id)
|
||||
if str(course.id) in course_ids or not course_ids:
|
||||
import_course(course.id, args.uri)
|
||||
|
||||
|
||||
def import_course(course_key):
|
||||
def import_course(course_key, clickhouse_uri):
|
||||
course_id = str(course_key)
|
||||
# Reload course to fetch all children items
|
||||
course = get_course(course_key, depth=None)
|
||||
@ -42,13 +52,14 @@ def import_course(course_key):
|
||||
sql_query(
|
||||
"ALTER TABLE course_blocks DELETE WHERE course_id = '{}';",
|
||||
course_id,
|
||||
)
|
||||
),
|
||||
clickhouse_uri,
|
||||
)
|
||||
insert_query = sql_query(
|
||||
"INSERT INTO course_blocks (course_id, block_key, block_id, position, display_name, full_name) VALUES "
|
||||
)
|
||||
insert_query += ", ".join(values)
|
||||
make_query(insert_query)
|
||||
make_query(insert_query, clickhouse_uri)
|
||||
|
||||
|
||||
def iter_course_blocks(item, prefix=""):
|
||||
@ -65,16 +76,11 @@ def sql_query(template, *args, **kwargs):
|
||||
return template.format(*args, **kwargs)
|
||||
|
||||
|
||||
def make_query(query):
|
||||
# TODO pass connection strings
|
||||
url = "http://vision-clickhouse:8123/?%s" % urllib.parse.urlencode(
|
||||
{"database": "openedx"}
|
||||
)
|
||||
try:
|
||||
urllib.request.urlopen(url, data=query.encode())
|
||||
except urllib.request.HTTPError as e:
|
||||
print(e.read().decode())
|
||||
raise
|
||||
def make_query(query, url):
|
||||
response = requests.post(url, data=query)
|
||||
if response.status_code != 200:
|
||||
print(response.content.decode())
|
||||
raise ValueError("An error occurred while attempting to post a query")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
1
tutorvision/templates/vision/hooks/vision-openedx/init
Normal file
1
tutorvision/templates/vision/hooks/vision-openedx/init
Normal file
@ -0,0 +1 @@
|
||||
python /openedx/scripts/importcoursedata.py http://{{ VISION_CLICKHOUSE_USERNAME }}:{{ VISION_CLICKHOUSE_PASSWORD }}@{{ VISION_CLICKHOUSE_HOST }}:{{ VISION_CLICKHOUSE_HTTP_PORT }}/?database={{ VISION_CLICKHOUSE_DATABASE }}
|
||||
Loading…
x
Reference in New Issue
Block a user
