tutor/tutor-cairn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: import/export script

Browse Source
This commit is contained in:
Régis Behmo 2021-04-27 14:56:17 +02:00
parent a4e3a28328
commit a1812988f8
10 changed files with 496 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
README.rst
View File

@ -88,6 +88,19 @@ To launch a Python shell in Redash, run::
tutor local run vision-redash ./manage.py shell
To backup an existing dashboard, with all its related queries and widgets, first find the dashboard slug from its url. Create a world-writable destination folder::
mkdir ./dashboards
chmod a+rwx dashboards
Then run::
tutor local run -v $(pwd)/dashboards:/tmp/dashboards vision-redash python /redash/scripts/serialize.py dump --output /tmp/dashboards/dashboard.json <your username> <dashboard slug> > ./dashboard.json
You can then re-import this dashboard, for instance to create the same dashboard in another user account::
tutor local run -v $(pwd)/dashboards:/tmp/dashboards vision-redash python /redash/scripts/serialize.py load /tmp/dashboards/dashboard.json <your username>
License
-------

39
setup.py
View File

@ -2,22 +2,31 @@ import io
import os
from setuptools import setup, find_packages
here = os.path.abspath(os.path.dirname(__file__))
HERE = os.path.abspath(os.path.dirname(__file__))
with io.open(os.path.join(here, "README.rst"), "rt", encoding="utf8") as f:
readme = f.read()
about = {}
with io.open(
os.path.join(here, "tutorvision", "__about__.py"),
"rt",
encoding="utf-8",
) as f:
exec(f.read(), about)
def load_readme():
with io.open(os.path.join(HERE, "README.rst"), "rt", encoding="utf8") as f:
return f.read()
def load_about():
about = {}
with io.open(
os.path.join(HERE, "tutorvision", "__about__.py"),
"rt",
encoding="utf-8",
) as f:
exec(f.read(), about) # pylint: disable=exec-used
return about
ABOUT = load_about()
setup(
name="tutor-vision",
version=about["__version__"],
version=ABOUT["__version__"],
url="https://github.com/overhangio/tutor-vision",
project_urls={
"Code": "https://github.com/overhangio/tutor-vision",
@ -26,16 +35,12 @@ setup(
license="AGPLv3",
author="Overhang.IO",
description="vision plugin for Tutor",
long_description=readme,
long_description=load_readme(),
packages=find_packages(exclude=["tests*"]),
include_package_data=True,
python_requires=">=3.5",
install_requires=["tutor-openedx"],
entry_points={
"tutor.plugin.v0": [
"vision = tutorvision.plugin"
]
},
entry_points={"tutor.plugin.v0": ["vision = tutorvision.plugin"]},
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",

16
tutorvision/cli.py
View File

@ -110,18 +110,14 @@ def frontend_command():
@click.pass_obj
def frontend_createuser(context, password, is_root, username, email):
config = tutor_config.load(context.root)
config.update(
{
"password": password,
"is_root": is_root,
"username": username,
"email": email,
}
command = "python /redash/scripts/createuser.py {is_root} --password={password} {username} {email}".format(
is_root="--root" if is_root else "",
password=password,
username=username,
email=email,
)
runner = ComposeJobRunner(context.root, config, local_docker_compose)
runner.run_job_from_template(
"vision-redash", "vision", "hooks", "vision-redash", "createuser"
)
runner.run_job("vision-redash", command)
datalake.add_command(datalake_createuser)

4
tutorvision/patches/local-docker-compose-jobs-services
View File

@ -7,6 +7,8 @@ vision-redash-job:
image: {{ VISION_REDASH_DOCKER_IMAGE }}
command: create_db
env_file: ../plugins/vision/apps/redash/env
volumes:
- ../plugins/vision/apps/redash/scripts/:/redash/scripts/:ro
depends_on:
- vision-postgresql
- vision-redis
- vision-redis

2
tutorvision/patches/local-docker-compose-services
View File

@ -35,6 +35,8 @@ vision-redash:
env_file: ../plugins/vision/apps/redash/env
environment:
REDASH_WEB_WORKERS: 4
volumes:
- ../plugins/vision/apps/redash/scripts/:/redash/scripts/:ro
restart: unless-stopped
depends_on:
- vision-postgresql

12
tutorvision/templates/vision/apps/clickhouse/migrations.d/0004_video_views.sql
View File

@ -63,15 +63,3 @@ WHERE end_event IN ('pause_video', 'stop_video', 'seek_video');
CREATE ROW POLICY common ON video_events FOR SELECT USING 1 TO ALL;
CREATE ROW POLICY common ON video_view_segments FOR SELECT USING 1 TO ALL;
-- TODO remove this
-- SELECT arrayJoin(range(toUInt64(floor(start_position)), toUInt64(ceil(end_position)))) AS bin,
-- count(*) AS total_views,
-- count(distinct(user_id)) AS unique_views,
-- total_views - unique_views AS replay_views,
-- video_id
-- FROM video_view_segments
-- WHERE video_id = 'DEFINE_ME_video_id'
-- GROUP BY bin,
-- video_id
-- ORDER BY bin

10
tutorvision/templates/vision/apps/redash/env
View File

@ -1,4 +1,12 @@
PYTHONUNBUFFERED=0
PYTHONPATH=/app
# Clickhouse
CLICKHOUSE_HOST="{{ VISION_CLICKHOUSE_HOST }}"
CLICKHOUSE_PORT="{{ VISION_CLICKHOUSE_HTTP_PORT }}"
CLICKHOUSE_DATABASE="{{ VISION_CLICKHOUSE_DATABASE }}"
# Redash configuration
REDASH_LOG_LEVEL=INFO
REDASH_REDIS_URL=redis://vision-redis:6379/0
REDASH_COOKIE_SECRET="{{ VISION_REDASH_COOKIE_SECRET }}"
@ -11,4 +19,4 @@ REDASH_MAIL_USE_SSL="{{ SMTP_USE_SSL }}"
REDASH_MAIL_USERNAME="{{ SMTP_USERNAME }}"
REDASH_MAIL_PASSWORD="{{ SMTP_PASSWORD }}"
REDASH_MAIL_DEFAULT_SENDER="{{ CONTACT_EMAIL }}"
REDASH_HOST="{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ VISION_REDASH_HOST }}"
REDASH_HOST="{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ VISION_REDASH_HOST }}"

146
tutorvision/templates/vision/apps/redash/scripts/createuser.py Executable file
View File

@ -0,0 +1,146 @@
#! /usr/bin/env python3
import argparse
from getpass import getpass
import os
from redash import create_app
from redash import models
from redash.query_runner.clickhouse import ClickHouse
from redash.utils.configuration import ConfigurationContainer
def main():
parser = argparse.ArgumentParser(
description="Create a Redash user with the corresponding data source pointing to Clickhouse"
)
parser.add_argument("-r", "--root", action="store_true", help="Make a root user")
parser.add_argument(
"-p", "--password", help="If undefined, you will be prompted for a password"
)
parser.add_argument("username")
parser.add_argument("email")
args = parser.parse_args()
app = create_app()
app.app_context().push()
password = args.password
while not password:
password = getpass(prompt="User password: ")
org = get_default_org()
group = get_group(org, args.username, is_root=args.root)
user = get_user(org, group, args.username, args.email, password, is_root=args.root)
get_datasource(org, group, user.name)
def get_default_org():
org_slug = "default"
org = models.Organization.get_by_slug(org_slug)
if org:
print("Org already exists")
else:
print("Creating org...")
org = models.Organization(
name=org_slug, slug=org_slug, settings={"beacon_consent": False}
)
models.db.session.add(org)
models.db.session.commit()
# Get org admin group
if org.admin_group:
print("Org admin group already exists")
else:
print("Creating org admin group...")
admin_group = models.Group(
name="admin",
permissions=["admin", "super_admin"],
org=org,
type=models.Group.BUILTIN_GROUP,
)
models.db.session.add_all([org, admin_group])
models.db.session.commit()
return org
def get_group(org, username, is_root=False):
group = models.Group.query.filter(
models.Group.name == username, models.Group.org == org
).first()
if group:
print("Group '{}' already exists".format(username))
else:
excluded_permissions = [] if is_root else ["list_users"]
permissions = filter(
lambda p: p not in excluded_permissions, models.Group.DEFAULT_PERMISSIONS
)
group = models.Group(name=username, org=org, permissions=permissions)
models.db.session.add(group)
models.db.session.commit()
print("Created group '{}'".format(group.name))
if is_root:
for permission in ["admin", "super_admin"]:
if permission not in group.permissions:
print("Adding permission '{}' to group".format(permission))
group.permissions.append(permission)
models.db.session.add(group)
models.db.session.commit()
return group
def get_user(org, group, username, email, password, is_root=False):
user = models.User.query.filter(models.User.email == email).first()
if user:
print("User already exists")
else:
user = models.User(org=org, email=email, name=username, group_ids=[group.id])
print("Created user '{}/{}'".format(user.email, user.name))
user.hash_password(password)
models.db.session.add(user)
models.db.session.commit()
if is_root:
if org.admin_group.id in user.group_ids:
print("User is already in admin group")
else:
print("Adding user to admin group...")
user.group_ids = user.group_ids + [org.admin_group.id]
models.db.session.add(user)
models.db.session.commit()
return user
def get_datasource(org, group, username):
# Get or create datasource
options = ConfigurationContainer(
{
"url": "http://{}:{}".format(
os.environ["CLICKHOUSE_HOST"], os.environ["CLICKHOUSE_PORT"]
),
"user": username,
"password": "",
"dbname": os.environ["CLICKHOUSE_DATABASE"],
},
ClickHouse.configuration_schema(),
)
data_source = models.DataSource.query.filter(
models.DataSource.name == username
).first()
if data_source:
print("Data source already exists")
else:
data_source = models.DataSource(
name=username,
type="clickhouse",
options=options,
org=org,
)
data_source_group = models.DataSourceGroup(data_source=data_source, group=group)
models.db.session.add_all([data_source, data_source_group])
models.db.session.commit()
print("Created datasource '{}'".format(data_source.name))
if __name__ == "__main__":
main()

295
tutorvision/templates/vision/apps/redash/scripts/serialize.py Normal file
View File

@ -0,0 +1,295 @@
import argparse
from datetime import datetime
import json
from redash import create_app
from redash import models
from redash.models.users import Group, User
from redash import serializers
def main():
parser = argparse.ArgumentParser(
description="Dump a dashboard to JSON, along with all associated widgets, queries and visualizations"
)
subparsers = parser.add_subparsers()
# Dump command
dump_command = subparsers.add_parser("dump", help="Dump objects to JSON")
dump_command.set_defaults(func=dump)
dump_command.add_argument(
"-o",
"--output",
default="-",
help="Destination file. Leave to '-' to print in stdout",
)
dump_command.add_argument("username")
dump_command.add_argument("slug")
# Load command
load_command = subparsers.add_parser("load", help="Load objects from JSON")
load_command.set_defaults(func=load)
load_command.add_argument(
"-n",
"--dry-run",
action="store_true",
help="Do not actually commit changes (good for debugging)",
)
load_command.add_argument(
"-s",
"--skip-existing",
action="store_true",
help="Do not actually create objects when an object with identical name already exists",
)
load_command.add_argument(
"-d",
"--datasource",
help="Assign queries to the specified datasource. If undefined, queries will be associated to the first datasource of the first user group.",
)
load_command.add_argument(
"path",
help="JSON-formatted file to load data from",
)
load_command.add_argument("username", help="Add dashboard to this user account")
app = create_app()
app.app_context().push()
args = parser.parse_args()
args.func(args)
def dump(args):
user = get_user(args.username)
dashboard = get_dashboard(user, args.slug)
data = serialize(dashboard)
data = clean_data(data)
serialized = json.dumps(data, sort_keys=True, indent=2)
save_to(serialized, args.output)
def load(args):
with open(args.path) as f:
serialized = json.load(f)
deserializer = Deserializer(
args.username,
data_source_id=args.datasource,
dry_run=args.dry_run,
skip_existing=args.skip_existing,
)
deserializer.deserialize_dashboard(serialized)
class Deserializer:
def __init__(
self, username, data_source_id=None, dry_run=False, skip_existing=False
):
self.user = get_user(username)
self.data_source_id = data_source_id
if not self.data_source_id:
group = Group.query.get(self.user.group_ids[0])
self.data_source_id = group.data_sources[0].data_source.id
print(
"Objects will be associated to datasource #{}".format(self.data_source_id)
)
self.dry_run = dry_run
self.skip_existing = skip_existing
self.widget_id_map = {}
self.visualization_id_map = {}
self.query_id_map = {}
def deserialize_dashboard(self, params):
print("--- Creating dashboard '{}'...".format(params["name"]))
dashboard = models.Dashboard(
user=self.user,
org=self.user.org,
name=params["name"],
layout=[],
is_draft=False,
)
models.db.session.add(dashboard)
for widget in params["widgets"]:
visualization = widget["visualization"]
self.deserialize_query(visualization["query"])
self.deserialize_visualization(visualization)
self.deserialize_widget(dashboard.id, widget)
if self.dry_run:
print("Dry-run mode: changes are discarded.")
else:
models.db.session.commit()
def deserialize_query(self, params):
old_id = params["id"]
if old_id in self.query_id_map:
return
if self.skip_existing:
existing_query = models.Query.query.filter(
models.Query.user == self.user, models.Query.name == params["name"]
).first()
if existing_query:
print(
"--- Skipping creation of query '{}' which already exists".format(
params["name"]
)
)
self.query_id_map[old_id] = existing_query.id
return
print("--- Creating query '{}'...".format(params["name"]))
# If there are parameter queries, create these first
for parameter in params["options"]["parameters"]:
if "query" in parameter:
parameter_query_params = parameter.pop("query")
self.deserialize_query(parameter_query_params)
parameter["queryId"] = self.query_id_map[parameter_query_params["id"]]
query = models.Query(
user=self.user,
org=self.user.org,
data_source_id=self.data_source_id,
**project_all_but(params, ["id"])
)
models.db.session.add(query)
self.query_id_map[old_id] = query.id
def deserialize_visualization(self, params):
old_id = params["id"]
if old_id in self.visualization_id_map:
return
query_id = self.query_id_map[params["query"]["id"]]
if self.skip_existing:
existing_visualization = (
models.Visualization.query.join(models.Query)
.filter(
models.Visualization.query_id == query_id,
models.Query.user == self.user,
models.Visualization.name == params["name"],
)
.first()
)
if existing_visualization:
print(
"--- Skipping creation of visualization '{}' which already exists".format(
params["name"]
)
)
self.visualization_id_map[old_id] = existing_visualization.id
return
print("--- Creating visualization '{}'...".format(params["name"]))
options = json.dumps(params["options"])
visualization = models.Visualization(
query_id=query_id,
options=options,
**project_all_but(params, ["id", "options", "query"])
)
models.db.session.add(visualization)
self.visualization_id_map[old_id] = visualization.id
def deserialize_widget(self, dashboard_id, params):
old_id = params["id"]
if old_id in self.widget_id_map:
return
print("--- Creating widget...")
widget = models.Widget(
visualization_id=self.visualization_id_map[params["visualization"]["id"]],
dashboard_id=dashboard_id,
options=json.dumps(params["options"]),
**project_all_but(params, ["id", "options", "visualization"])
)
models.db.session.add(widget)
self.widget_id_map[old_id] = widget.id
def get_user(username):
org = models.Organization.get_by_slug("default")
return org.users.filter(User.name == username).one()
def get_dashboard(user, slug):
return models.Dashboard.query.filter(
models.Dashboard.user == user, models.Dashboard.slug == slug
).one()
def serialize(dashboard):
return {
"id": dashboard.id,
"name": dashboard.name,
"widgets": [serialize_widget(w) for w in dashboard.widgets],
}
def serialize_widget(widget):
"""
TODO don't include draft or achived queries.
"""
widget_fields = set(["id", "width", "options", "text", "visualization"])
visualization_fields = set(
["id", "type", "name", "description", "options", "query"]
)
query_fields = set(
["id", "name", "description", "query_text", "schedule", "options"]
)
serialized = serializers.serialize_widget(widget)
serialized = project(serialized, widget_fields)
if "visualization" in serialized:
serialized["visualization"] = project(
serialized["visualization"], visualization_fields
)
# Convert 'query.query' field to 'query.query_text'
serialized["visualization"]["query"]["query_text"] = serialized[
"visualization"
]["query"]["query"]
# If query depends on some other parameter, save this other query, too
for parameter in serialized["visualization"]["query"]["options"]["parameters"]:
if "queryId" in parameter:
parameter_query = models.Query.query.get(parameter.pop("queryId"))
parameter["query"] = serializers.serialize_query(parameter_query)
# don't forget to convert query to query_text
parameter["query"]["query_text"] = parameter["query"]["query"]
parameter["query"] = project(parameter["query"], query_fields)
serialized["visualization"]["query"] = project(
serialized["visualization"]["query"], query_fields
)
return serialized
def project(data, allowed_keys):
return {key: value for key, value in data.items() if key in allowed_keys}
def project_all_but(data, excluded_keys):
return {key: value for key, value in data.items() if key not in excluded_keys}
def clean_data(data):
"""
Convert all datetime objects to str, for serialization (recursively).
"""
if isinstance(data, datetime):
return str(data)
if isinstance(data, dict):
return {key: clean_data(value) for key, value in data.items()}
if isinstance(data, list):
return [clean_data(d) for d in data]
return data
def save_to(content, path):
if path == "-":
print(content)
return
with open(path, "w") as of:
of.write(content)
if __name__ == "__main__":
main()

99
tutorvision/templates/vision/hooks/vision-redash/createuser
View File

@ -1,99 +0,0 @@
cat << EOF | python
from redash import create_app
from redash import models
from redash.query_runner.clickhouse import ClickHouse
from redash.utils.configuration import ConfigurationContainer
app = create_app()
app.app_context().push()
# Get organization
org_slug = "default"
org = models.Organization.get_by_slug(org_slug)
if org:
print("Org already exists")
else:
print("Creating org...")
org = models.Organization(name=org_slug, slug=org_slug, settings={"beacon_consent": False})
models.db.session.add(org)
models.db.session.commit()
# Get org admin group
if org.admin_group:
print("Org admin group already exists")
else:
print("Creating org admin group...")
admin_group = models.Group(
name="admin",
permissions=["admin", "super_admin"],
org=org,
type=models.Group.BUILTIN_GROUP,
)
models.db.session.add_all([org, admin_group])
models.db.session.commit()
# Get or create group
group = models.Group.query.filter(models.Group.name == "{{ username }}", models.Group.org == org).first()
if group:
print("Group '{{ username }}' already exists")
else:
excluded_permissions = {% if is_root %}[]{% else %}["list_users"]{% endif %}
permissions = filter(lambda p: p not in excluded_permissions, models.Group.DEFAULT_PERMISSIONS)
group = models.Group(name="{{ username }}", org=org, permissions=permissions)
models.db.session.add(group)
models.db.session.commit()
print("Created group '{}'".format(group.name))
{% if is_root %}
for permission in ["admin", "super_admin"]:
if permission not in group.permissions:
print("Adding permission '{}' to group".format(permission))
group.permissions.append(permission)
models.db.session.add(group)
models.db.session.commit()
{% endif %}
# Get or create user
user = models.User.query.filter(models.User.email == "{{ email }}").first()
if user:
print("User already exists")
else:
user = models.User(org=org, email="{{ email }}", name="{{ username }}", group_ids=[group.id])
print("Created user '{}/{}'".format(user.email, user.name))
user.hash_password("""{{ password }}""")
models.db.session.add(user)
models.db.session.commit()
{% if is_root %}
if org.admin_group.id in user.group_ids:
print("User is already in admin group")
else:
print("Adding user to admin group...")
user.group_ids = user.group_ids + [org.admin_group.id]
models.db.session.add(user)
models.db.session.commit()
{% endif %}
# Get or create datasource
options = ConfigurationContainer(
{
"url": "http://{{ VISION_CLICKHOUSE_HOST }}:{{ VISION_CLICKHOUSE_HTTP_PORT }}",
"user": "{{ username }}",
"password": "",
"dbname": "{{ VISION_CLICKHOUSE_DATABASE }}",
},
ClickHouse.configuration_schema()
)
data_source = models.DataSource.query.filter(models.DataSource.name == "{{ username }}").first()
if data_source:
print("Data source already exists")
else:
data_source = models.DataSource(
name="{{ username }}",
type="clickhouse",
options=options,
org=org,
)
data_source_group = models.DataSourceGroup(data_source=data_source, group=group)
models.db.session.add_all([data_source, data_source_group])
models.db.session.commit()
print("Created datasource '{}'".format(data_source.name))
EOF