diff --git a/tutordash/server/app.py b/tutordash/server/app.py index d2dfcc5..1d35fd1 100644 --- a/tutordash/server/app.py +++ b/tutordash/server/app.py @@ -1,13 +1,8 @@ import asyncio -import contextlib import json -import shlex -import subprocess -import tempfile -import threading import typing as t -import aiofiles + from quart import ( Quart, make_response, @@ -19,262 +14,8 @@ from quart import ( from quart.helpers import WerkzeugResponse from quart.typing import ResponseTypes -import tutor.env -from tutor.exceptions import TutorError -from tutor import fmt, hooks -from tutor.types import Config -import tutor.utils -from tutor.commands.cli import cli - -SHORT_SLEEP_SECONDS = 0.1 - - -class TutorProject: - """ - Provide access to the current Tutor project root and configuration. - """ - - CONFIG: dict[str, t.Any] = {} # TODO this attribute is unused. Delete it? - ROOT: str = "" - - @classmethod - def connect(cls, root: str) -> None: - """ - Call whenever we are ready to connect to the Tutor hooks API. - """ - if not cls.ROOT: - # Hook up TutorProject with Tutor hooks -- just once - hooks.Actions.PROJECT_ROOT_READY.add()(cls._dash_on_project_root_ready) - hooks.Actions.CONFIG_LOADED.add()(cls._dash_on_config_loaded) - hooks.Actions.CORE_READY.do() # discover plugins - hooks.Actions.PROJECT_ROOT_READY.do(root) - - @classmethod - def _dash_on_project_root_ready(cls, root: str) -> None: - cls.ROOT = root - - @classmethod - def _dash_on_config_loaded(cls, config: Config) -> None: - cls.CONFIG = config - - -class TutorCli: - """ - Run Tutor commands and capture the output in a file. - - Output must be a file because subprocess.Popen requires stdout.fileno() to be - available. We store logs in temporary files. - - Tutor commands are not meant to be run in parallel. Thus, there must be only one - instance running at any time: calling functions are responsible for calling - TutorCliPool instead of this class. - """ - - def __init__(self, args: list[str]) -> None: - """ - Each instance can be interrupted from other threads via the stop flag. - """ - self.args = args - self.log_file = tempfile.NamedTemporaryFile( - "ab", prefix="tutor-dash-", suffix=".log" - ) - self._stop_flag = threading.Event() - - @property - def log_path(self) -> str: - """ - Path to the log file - """ - return self.log_file.name - - @property - def command(self) -> str: - """ - Tutor command executed by this runner. - """ - return shlex.join(["tutor"] + self.args) - - def run(self) -> None: - """ - Execute some arbitrary tutor command. - - Output will be captured in the log file. - """ - app.logger.info( - "Running command: tutor %s (logs: %s)", self.command, self.log_path - ) - - # Override execute function - with self.patch_objects(): - try: - # Call tutor command - cli(self.args) # pylint: disable=no-value-for-parameter - except TutorError as e: - # This happens for incorrect commands - self.log_file.write(e.args[0].encode()) - except SystemExit: - pass - self.log_file.flush() - - def stop(self) -> None: - """ - Sets the stop flag, whic is monitored by all subprocess.Popen commands. - """ - app.logger.info("Stopping Tutor command: %s...", self.command) - self._stop_flag.set() - - async def iter_logs(self) -> t.AsyncGenerator[str, None]: - """ - Async stream content from file. Output is prefixed by the running command. - - This will handle gracefully file deletion. Note however that if the file is - truncated, all contents added to the beginning until the current position will be - missed. - """ - yield f"$ {self.command}\n" - async with aiofiles.open(self.log_path, "rb") as f: - # Note that file reading needs to happen from the file path, because it maye - # be done from a separate thread, where the file object is not available. - while True: - content = await f.read() - if content: - yield content.decode() - else: - await asyncio.sleep(SHORT_SLEEP_SECONDS) - - # Mocking functions to override tutor functions that write to stdout - @contextlib.contextmanager - def patch_objects(self) -> t.Iterator[None]: - refs = [ - (tutor.utils, "execute", self._mock_execute), - (fmt.click, "echo", self._mock_click_echo), - (fmt.click, "style", self._mock_click_style), - ] - old_objects = [] - for module, object_name, new_object in refs: - # backup old object - old_objects.append((module, object_name, getattr(module, object_name))) - # override object - setattr(module, object_name, new_object) - try: - yield None - finally: - # restore old objects - for module, object_name, old_object in old_objects: - setattr(module, object_name, old_object) - - def _mock_click_echo(self, text: str, **_kwargs: t.Any) -> None: - """ - Mock click.echo to write to log file - """ - self.log_file.write(text.encode()) - self.log_file.write(b"\n") - - def _mock_click_style(self, text: str, **_kwargs: t.Any) -> str: - """ - Mock click.style to strip ANSI colors - - TODO convert to HTML color codes? - """ - return text - - def _mock_execute(self, *command: str) -> int: - """ - Mock tutor.utils.execute. - """ - command_string = shlex.join(command) - with subprocess.Popen( - command, stdout=self.log_file, stderr=self.log_file - ) as popen: - while popen.returncode is None: - try: - popen.wait(timeout=0.5) - except subprocess.TimeoutExpired as e: - # Check every now and then whether we should stop - if self._stop_flag.is_set(): - popen.kill() - popen.wait() - raise TutorError( - f"Stopping child command: {command_string}" - ) from e - except Exception as e: - popen.kill() - popen.wait() - raise TutorError(f"Command failed: {command_string}") from e - - if popen.returncode > 0: - raise TutorError( - f"Command failed with status {popen.returncode}: {command_string}" - ) - return popen.returncode - - -class TutorCliPool: - INSTANCE: t.Optional["TutorCli"] = None - THREAD: t.Optional[threading.Thread] = None - - @classmethod - def run_parallel(cls, args: list[str]) -> None: - """ - Run a command in a separate thread. This command automatically stops any running - command. - """ - # Stop any running command - cls.stop() - - # Start thread - cls.INSTANCE = TutorCli(args) - cls.THREAD = threading.Thread(target=cls.INSTANCE.run) - cls.THREAD.start() - - # Watch for exit - app.add_background_task(cls.stop_on_exit, cls.INSTANCE, cls.THREAD) - - @classmethod - def stop(cls) -> None: - """ - Stop running instance. - - This is a no-op when there is no running thread, so it's safe to call any time. - """ - if cls.INSTANCE and cls.THREAD: - cls.stop_runner_thread(cls.INSTANCE, cls.THREAD) - - @staticmethod - def stop_runner_thread( - tutor_cli_runner: TutorCli, thread: threading.Thread - ) -> None: - """ - Set runner stop flag and wait for thread to complete. - """ - if thread.is_alive(): - tutor_cli_runner.stop() - thread.join() - - @classmethod - async def stop_on_exit( - cls, tutor_cli_runner: TutorCli, thread: threading.Thread - ) -> None: - """ - This background task will stop the runner whenever the Quart app is - requested to stop/exit/shutdown. This happens for instance on dev reload. - """ - try: - while thread.is_alive(): - await asyncio.sleep(SHORT_SLEEP_SECONDS) - finally: - cls.stop_runner_thread(tutor_cli_runner, thread) - - @classmethod - async def iter_logs(cls) -> t.AsyncGenerator[str, None]: - """ - Iterate indefinitely from any running instance. When an existing instance is - replaced by another one, previous logs are not deleted. New ones are simply - appended. - """ - while cls.INSTANCE: - async for log in cls.INSTANCE.iter_logs(): - yield log +from . import constants +from . import tutorclient app = Quart( @@ -288,7 +29,7 @@ def run(root: str, **app_kwargs: t.Any) -> None: """ Bootstrap the Quart app and run it. """ - TutorProject.connect(root) + tutorclient.Project.connect(root) # TODO app.run() should be called only in development app.run(**app_kwargs) @@ -302,11 +43,13 @@ async def home() -> str: @app.get("/plugin/") async def plugin(name: str) -> str: # TODO check that plugin exists - is_enabled = name in hooks.Filters.PLUGINS_LOADED.iterate() + is_enabled = name in tutorclient.Client.enabled_plugins() return await render_template( "plugin.html", plugin_name=name, is_enabled=is_enabled, + config_unique=tutorclient.Client.plugin_config_unique(name), + config_defaults=tutorclient.Client.plugin_config_defaults(name), **shared_template_context(), ) @@ -316,25 +59,22 @@ async def toggle_plugin(name: str) -> WerkzeugResponse: # TODO check plugin exists form = await request.form enable_plugin = form.get("enabled") == "on" - return tutor_cli(["plugins", "enable" if enable_plugin else "disable", name]) + command = ["plugins", "enable" if enable_plugin else "disable", name] + tutorclient.CliPool.run_sequential(command) + # TODO error management + return redirect(url_for("plugin", name=name)) -def tutor_cli(command: list[str]) -> WerkzeugResponse: - # Run command asynchronously - # if TutorCli.is_thread_alive(): - # TODO return 400 if thread is active - # TODO parse command from JSON request body - TutorCliPool.run_parallel(command) - return redirect(url_for("tutor_cli_logs")) +# def tutor_cli(command: list[str]) -> WerkzeugResponse: +# # Run command asynchronously +# # if TutorCli.is_thread_alive(): +# # TODO return 400 if thread is active +# # TODO parse command from JSON request body +# tutorclient.CliPool.run_parallel(app, command) +# return redirect(url_for("tutor_cli_logs")) -@app.post("/tutor/cli/stop") -async def tutor_cli_stop() -> WerkzeugResponse: - TutorCliPool.stop() - return redirect(url_for("tutor_cli_logs")) - - -@app.get("/tutor/logs") +@app.get("/tutor/cli/logs") async def tutor_cli_logs() -> str: return await render_template("tutor_cli_logs.html", **shared_template_context()) @@ -363,11 +103,11 @@ async def tutor_cli_logs_stream() -> ResponseTypes: async def send_events() -> t.AsyncIterator[bytes]: while True: # TODO this is again causing the stream to never stop... - async for data in TutorCliPool.iter_logs(): + async for data in tutorclient.CliPool.iter_logs(): json_data = json.dumps(data) event = f"data: {json_data}\nevent: logs\n\n" yield event.encode() - await asyncio.sleep(SHORT_SLEEP_SECONDS) + await asyncio.sleep(constants.SHORT_SLEEP_SECONDS) response = await make_response( send_events(), @@ -381,6 +121,12 @@ async def tutor_cli_logs_stream() -> ResponseTypes: return response +@app.post("/tutor/cli/stop") +async def tutor_cli_stop() -> WerkzeugResponse: + tutorclient.CliPool.stop() + return redirect(url_for("tutor_cli_logs")) + + def shared_template_context() -> dict[str, t.Any]: """ Common context shared between all views that make use of the base template. @@ -388,6 +134,6 @@ def shared_template_context() -> dict[str, t.Any]: TODO isn't there a better way to achieve that? Either template variable or Quart feature. """ return { - "installed_plugins": sorted(set(hooks.Filters.PLUGINS_INSTALLED.iterate())), - "enabled_plugins": sorted(set(hooks.Filters.PLUGINS_LOADED.iterate())), + "installed_plugins": tutorclient.Client.installed_plugins(), + "enabled_plugins": tutorclient.Client.enabled_plugins(), } diff --git a/tutordash/server/constants.py b/tutordash/server/constants.py new file mode 100644 index 0000000..f617517 --- /dev/null +++ b/tutordash/server/constants.py @@ -0,0 +1,2 @@ + +SHORT_SLEEP_SECONDS = 0.1 diff --git a/tutordash/server/static/scss/dash.scss b/tutordash/server/static/scss/dash.scss index 9f6cd77..d84b142 100644 --- a/tutordash/server/static/scss/dash.scss +++ b/tutordash/server/static/scss/dash.scss @@ -70,7 +70,7 @@ body { overflow: hidden; padding-bottom: 20px; // necessary to display the bottom scrollbar - pre { + pre#tutor-logs { overflow: scroll; height: calc(100% - 20px); // must match .content padding-bottom to display scrollbar } diff --git a/tutordash/server/templates/plugin.html b/tutordash/server/templates/plugin.html index 381d4b4..5c81130 100644 --- a/tutordash/server/templates/plugin.html +++ b/tutordash/server/templates/plugin.html @@ -6,4 +6,23 @@ Enabled:
+ +{% if is_enabled %} +

Configuration

+ +{% if config_unique %} +

Unique settings

+{% for key, value in config_unique.items() %} +

{{ key }}: 

{{ value }}

+{% endfor %} +{% endif %} + +{% if config_defaults %} +

Default settings

+{% for key, value in config_defaults.items() %} +

{{ key }}: 

{{ value }}

+{% endfor %} +{% endif %} + +{% endif %} {% endblock %} diff --git a/tutordash/server/tutorclient.py b/tutordash/server/tutorclient.py new file mode 100644 index 0000000..b386c09 --- /dev/null +++ b/tutordash/server/tutorclient.py @@ -0,0 +1,305 @@ +import asyncio +import contextlib +import logging +import shlex +import subprocess +import tempfile +import threading +import typing as t + +import aiofiles +from quart import Quart + +import tutor.env +from tutor.exceptions import TutorError +from tutor import fmt, hooks +from tutor.types import Config +import tutor.commands.cli +import tutor.utils + +from . import constants + +logger = logging.getLogger(__name__) + + +class Project: + """ + Provide access to the current Tutor project root and configuration. + """ + + CONFIG: dict[str, t.Any] = {} # TODO this attribute is unused. Delete it? + ROOT: str = "" + + @classmethod + def connect(cls, root: str) -> None: + """ + Call whenever we are ready to connect to the Tutor hooks API. + """ + if not cls.ROOT: + # Hook up TutorProject with Tutor hooks -- just once + hooks.Actions.PROJECT_ROOT_READY.add()(cls._dash_on_project_root_ready) + hooks.Actions.CONFIG_LOADED.add()(cls._dash_on_config_loaded) + hooks.Actions.CORE_READY.do() # discover plugins + hooks.Actions.PROJECT_ROOT_READY.do(root) + + @classmethod + def _dash_on_project_root_ready(cls, root: str) -> None: + cls.ROOT = root + + @classmethod + def _dash_on_config_loaded(cls, config: Config) -> None: + cls.CONFIG = config + + +class Cli: + """ + Run Tutor commands and capture the output in a file. + + Output must be a file because subprocess.Popen requires stdout.fileno() to be + available. We store logs in temporary files. + + Tutor commands are not meant to be run in parallel. Thus, there must be only one + instance running at any time: calling functions are responsible for calling + CliPool instead of this class. + """ + + def __init__(self, args: list[str]) -> None: + """ + Each instance can be interrupted from other threads via the stop flag. + """ + self.args = args + self.log_file = tempfile.NamedTemporaryFile( + "ab", prefix="tutor-dash-", suffix=".log" + ) + self._stop_flag = threading.Event() + + @property + def log_path(self) -> str: + """ + Path to the log file + """ + return self.log_file.name + + @property + def command(self) -> str: + """ + Tutor command executed by this runner. + """ + return shlex.join(["tutor"] + self.args) + + def run(self) -> None: + """ + Execute some arbitrary tutor command. + + Output will be captured in the log file. + """ + logger.info("Running command: tutor %s (logs: %s)", self.command, self.log_path) + + # Override execute function + with self.patch_objects(): + try: + # Call tutor command + # pylint: disable=no-value-for-parameter + tutor.commands.cli.cli(self.args) + except TutorError as e: + # This happens for incorrect commands + self.log_file.write(e.args[0].encode()) + except SystemExit: + pass + self.log_file.flush() + + def stop(self) -> None: + """ + Sets the stop flag, whic is monitored by all subprocess.Popen commands. + """ + logger.info("Stopping Tutor command: %s...", self.command) + self._stop_flag.set() + + async def iter_logs(self) -> t.AsyncGenerator[str, None]: + """ + Async stream content from file. Output is prefixed by the running command. + + This will handle gracefully file deletion. Note however that if the file is + truncated, all contents added to the beginning until the current position will be + missed. + """ + yield f"$ {self.command}\n" + async with aiofiles.open(self.log_path, "rb") as f: + # Note that file reading needs to happen from the file path, because it maye + # be done from a separate thread, where the file object is not available. + while True: + content = await f.read() + if content: + yield content.decode() + else: + await asyncio.sleep(constants.SHORT_SLEEP_SECONDS) + + # Mocking functions to override tutor functions that write to stdout + @contextlib.contextmanager + def patch_objects(self) -> t.Iterator[None]: + refs = [ + (tutor.utils, "execute", self._mock_execute), + (fmt.click, "echo", self._mock_click_echo), + (fmt.click, "style", self._mock_click_style), + ] + old_objects = [] + for module, object_name, new_object in refs: + # backup old object + old_objects.append((module, object_name, getattr(module, object_name))) + # override object + setattr(module, object_name, new_object) + try: + yield None + finally: + # restore old objects + for module, object_name, old_object in old_objects: + setattr(module, object_name, old_object) + + def _mock_click_echo(self, text: str, **_kwargs: t.Any) -> None: + """ + Mock click.echo to write to log file + """ + self.log_file.write(text.encode()) + self.log_file.write(b"\n") + + def _mock_click_style(self, text: str, **_kwargs: t.Any) -> str: + """ + Mock click.style to strip ANSI colors + + TODO convert to HTML color codes? + """ + return text + + def _mock_execute(self, *command: str) -> int: + """ + Mock tutor.utils.execute. + """ + command_string = shlex.join(command) + with subprocess.Popen( + command, stdout=self.log_file, stderr=self.log_file + ) as popen: + while popen.returncode is None: + try: + popen.wait(timeout=0.5) + except subprocess.TimeoutExpired as e: + # Check every now and then whether we should stop + if self._stop_flag.is_set(): + popen.kill() + popen.wait() + raise TutorError( + f"Stopping child command: {command_string}" + ) from e + except Exception as e: + popen.kill() + popen.wait() + raise TutorError(f"Command failed: {command_string}") from e + + if popen.returncode > 0: + raise TutorError( + f"Command failed with status {popen.returncode}: {command_string}" + ) + return popen.returncode + + +class CliPool: + CLI_INSTANCE: t.Optional[Cli] = None + THREAD: t.Optional[threading.Thread] = None + + @classmethod + def run_sequential(cls, args: list[str]) -> None: + cls.stop() + cls.CLI_INSTANCE = Cli(args) + cls.CLI_INSTANCE.run() + + @classmethod + def run_parallel(cls, app: Quart, args: list[str]) -> None: + """ + Run a command in a separate thread. This command automatically stops any running + command. + """ + # Stop any running command + cls.stop() + + # Start thread + cls.CLI_INSTANCE = Cli(args) + cls.THREAD = threading.Thread(target=cls.CLI_INSTANCE.run) + cls.THREAD.start() + + # Watch for exit + app.add_background_task(cls.stop_on_exit, cls.CLI_INSTANCE, cls.THREAD) + + @classmethod + def stop(cls) -> None: + """ + Stop running instance. + + This is a no-op when there is no running thread, so it's safe to call any time. + """ + if cls.CLI_INSTANCE and cls.THREAD: + cls.stop_runner_thread(cls.CLI_INSTANCE, cls.THREAD) + + @staticmethod + def stop_runner_thread(tutor_cli_runner: Cli, thread: threading.Thread) -> None: + """ + Set runner stop flag and wait for thread to complete. + """ + if thread.is_alive(): + tutor_cli_runner.stop() + thread.join() + + @classmethod + async def stop_on_exit( + cls, tutor_cli_runner: Cli, thread: threading.Thread + ) -> None: + """ + This background task will stop the runner whenever the Quart app is + requested to stop/exit/shutdown. This happens for instance on dev reload. + """ + try: + while thread.is_alive(): + await asyncio.sleep(constants.SHORT_SLEEP_SECONDS) + finally: + cls.stop_runner_thread(tutor_cli_runner, thread) + + @classmethod + async def iter_logs(cls) -> t.AsyncGenerator[str, None]: + """ + Iterate indefinitely from any running instance. When an existing instance is + replaced by another one, previous logs are not deleted. New ones are simply + appended. + """ + while cls.CLI_INSTANCE: + async for log in cls.CLI_INSTANCE.iter_logs(): + yield log + + +class Client: + + @classmethod + def installed_plugins(cls) -> list[str]: + return sorted(set(hooks.Filters.PLUGINS_INSTALLED.iterate())) + + @classmethod + def enabled_plugins(cls) -> list[str]: + return list(hooks.Filters.PLUGINS_LOADED.iterate()) + + @classmethod + def plugin_config_unique(cls, name: str) -> Config: + plugin_config = hooks.Filters.CONFIG_UNIQUE.iterate_from_context( + hooks.Contexts.app(name).name + ) + # TODO IMPORTANT enable/disable/enable does not work because of the Python + # module import cache. After a plugin module is imported, the plugin is + # disabled, and when we try to enable it again the module is not imported, + # because of the import cache. + user_config = { + key: Project.CONFIG.get(key, value) for key, value in plugin_config + } + return user_config + + @classmethod + def plugin_config_defaults(cls, name: str) -> Config: + config = hooks.Filters.CONFIG_DEFAULTS.iterate_from_context( + hooks.Contexts.app(name).name + ) + return dict(config)