From 8687801cee9dde563052403e9c91d25a8692a4f7 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 20 May 2024 19:34:27 +0200 Subject: [PATCH 1/5] clan ui: setup typed api method --- pkgs/clan-cli/api.py | 18 +++ pkgs/clan-cli/clan_cli/api/__init__.py | 13 ++ pkgs/clan-cli/clan_cli/api/schema_compat.py | 111 +++++++++++++++++ pkgs/clan-cli/clan_cli/machines/list.py | 3 + pkgs/clan-cli/flake-module.nix | 10 ++ .../clan_vm_manager/views/webview.py | 112 ++++++++---------- .../clan_vm_manager/windows/main_window.py | 11 +- pkgs/webview-ui/.gitignore | 1 + pkgs/webview-ui/app/package-lock.json | 48 ++++++++ pkgs/webview-ui/app/package.json | 9 +- pkgs/webview-ui/app/src/Config.tsx | 6 +- pkgs/webview-ui/app/src/message.ts | 86 +++++++++++--- pkgs/webview-ui/app/tsconfig.json | 4 + pkgs/webview-ui/app/vite.config.ts | 6 + pkgs/webview-ui/default.nix | 11 +- pkgs/webview-ui/flake-module.nix | 22 ++++ 16 files changed, 375 insertions(+), 96 deletions(-) create mode 100644 pkgs/clan-cli/api.py create mode 100644 pkgs/clan-cli/clan_cli/api/__init__.py create mode 100644 pkgs/clan-cli/clan_cli/api/schema_compat.py create mode 100644 pkgs/webview-ui/.gitignore diff --git a/pkgs/clan-cli/api.py b/pkgs/clan-cli/api.py new file mode 100644 index 00000000..ab5ca5d3 --- /dev/null +++ b/pkgs/clan-cli/api.py @@ -0,0 +1,18 @@ +from clan_cli import create_parser +from clan_cli.api import API +from clan_cli.api.schema_compat import to_json_schema + + +def main() -> None: + # Create the parser to register the API functions + create_parser() + + schema = to_json_schema(API._registry) + print( + f"""export const schema = {schema} as const; +""" + ) + + +if __name__ == "__main__": + main() diff --git a/pkgs/clan-cli/clan_cli/api/__init__.py b/pkgs/clan-cli/clan_cli/api/__init__.py new file mode 100644 index 00000000..fe531c2b --- /dev/null +++ b/pkgs/clan-cli/clan_cli/api/__init__.py @@ -0,0 +1,13 @@ +from collections.abc import Callable + + +class _MethodRegistry: + def __init__(self): + self._registry = {} + + def register(self, fn: Callable) -> Callable: + self._registry[fn.__name__] = fn + return fn + + +API = _MethodRegistry() diff --git a/pkgs/clan-cli/clan_cli/api/schema_compat.py b/pkgs/clan-cli/clan_cli/api/schema_compat.py new file mode 100644 index 00000000..eeccf685 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/api/schema_compat.py @@ -0,0 +1,111 @@ +import dataclasses +import json +from types import NoneType, UnionType +from typing import Any, Callable, Union, get_type_hints +import pathlib + + +def type_to_dict(t: Any, scope: str = "") -> dict: + # print( + # f"Type: {t}, Scope: {scope}, has origin: {hasattr(t, '__origin__')} ", + # type(t) is UnionType, + # ) + + if t is None: + return {"type": "null"} + + if dataclasses.is_dataclass(t): + fields = dataclasses.fields(t) + properties = { + f.name: type_to_dict(f.type, f"{scope} {t.__name__}.{f.name}") + for f in fields + } + required = [pn for pn, pv in properties.items() if "null" not in pv["type"]] + return { + "type": "object", + "properties": properties, + "required": required, + # Dataclasses can only have the specified properties + "additionalProperties": False, + } + elif type(t) is UnionType: + return { + "type": [type_to_dict(arg, scope)["type"] for arg in t.__args__], + } + + elif hasattr(t, "__origin__"): # Check if it's a generic type + origin = getattr(t, "__origin__", None) + + if origin is None: + # Non-generic user-defined or built-in type + # TODO: handle custom types + raise BaseException("Unhandled Type: ", origin) + + elif origin is Union: + return {"type": [type_to_dict(arg, scope)["type"] for arg in t.__args__]} + + elif issubclass(origin, list): + return {"type": "array", "items": type_to_dict(t.__args__[0], scope)} + + elif issubclass(origin, dict): + return { + "type": "object", + } + + raise BaseException(f"Error api type not yet supported {str(t)}") + + elif isinstance(t, type): + if t is str: + return {"type": "string"} + if t is int: + return {"type": "integer"} + if t is float: + return {"type": "number"} + if t is bool: + return {"type": "boolean"} + if t is object: + return {"type": "object"} + if t is Any: + raise BaseException( + f"Usage of the Any type is not supported for API functions. In: {scope}" + ) + + if t is pathlib.Path: + return { + # TODO: maybe give it a pattern for URI + "type": "string", + } + + # Optional[T] gets internally transformed Union[T,NoneType] + if t is NoneType: + return {"type": "null"} + + raise BaseException(f"Error primitive type not supported {str(t)}") + else: + raise BaseException(f"Error type not supported {str(t)}") + + +def to_json_schema(methods: dict[str, Callable]) -> str: + api_schema = { + "$comment": "An object containing API methods. ", + "type": "object", + "additionalProperties": False, + "required": ["list_machines"], + "properties": {}, + } + for name, func in methods.items(): + hints = get_type_hints(func) + serialized_hints = { + "argument" if key != "return" else "return": type_to_dict( + value, scope=name + " argument" if key != "return" else "return" + ) + for key, value in hints.items() + } + api_schema["properties"][name] = { + "type": "object", + "required": [k for k in serialized_hints.keys()], + "additionalProperties": False, + "properties": {**serialized_hints}, + } + + return json.dumps(api_schema, indent=2) diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index 83ce2d66..4e483d8e 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -5,11 +5,14 @@ from pathlib import Path from ..cmd import run from ..nix import nix_config, nix_eval +from clan_cli.api import API log = logging.getLogger(__name__) +@API.register def list_machines(flake_url: Path | str) -> list[str]: + print("list_machines", flake_url) config = nix_config() system = config["system"] cmd = nix_eval( diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 2e4969b6..11e6c841 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -57,6 +57,16 @@ cp -r out/* $out ''; }; + clan-ts-api = pkgs.stdenv.mkDerivation { + name = "clan-ts-api"; + src = ./.; + + buildInputs = [ pkgs.python3 ]; + + installPhase = '' + python api.py > $out + ''; + }; default = self'.packages.clan-cli; }; diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py b/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py index 78a7e1b1..454ac917 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py @@ -2,6 +2,7 @@ import dataclasses import json import sys import threading +from threading import Lock from collections.abc import Callable from pathlib import Path from typing import Any, Union @@ -10,7 +11,7 @@ import gi gi.require_version("WebKit", "6.0") -from gi.repository import GLib, WebKit +from gi.repository import GLib, GObject, WebKit site_index: Path = ( Path(sys.argv[0]).absolute() @@ -19,51 +20,9 @@ site_index: Path = ( ).resolve() -def type_to_dict(t: Any) -> dict: - if dataclasses.is_dataclass(t): - fields = dataclasses.fields(t) - return { - "type": "dataclass", - "name": t.__name__, - "fields": {f.name: type_to_dict(f.type) for f in fields}, - } - - if hasattr(t, "__origin__"): # Check if it's a generic type - if t.__origin__ is None: - # Non-generic user-defined or built-in type - return {"type": t.__name__} - if t.__origin__ is Union: - return {"type": "union", "of": [type_to_dict(arg) for arg in t.__args__]} - elif issubclass(t.__origin__, list): - return {"type": "list", "item_type": type_to_dict(t.__args__[0])} - elif issubclass(t.__origin__, dict): - return { - "type": "dict", - "key_type": type_to_dict(t.__args__[0]), - "value_type": type_to_dict(t.__args__[1]), - } - elif issubclass(t.__origin__, tuple): - return { - "type": "tuple", - "element_types": [type_to_dict(elem) for elem in t.__args__], - } - elif issubclass(t.__origin__, set): - return {"type": "set", "item_type": type_to_dict(t.__args__[0])} - else: - # Handle other generic types (like Union, Optional) - return { - "type": str(t.__origin__.__name__), - "parameters": [type_to_dict(arg) for arg in t.__args__], - } - elif isinstance(t, type): - return {"type": t.__name__} - else: - return {"type": str(t)} - - class WebView: - def __init__(self) -> None: - self.method_registry: dict[str, Callable] = {} + def __init__(self, methods: dict[str, Callable]) -> None: + self.method_registry: dict[str, Callable] = methods self.webview = WebKit.WebView() self.manager = self.webview.get_user_content_manager() @@ -74,39 +33,66 @@ class WebView: self.webview.load_uri(f"file://{site_index}") - def method(self, function: Callable) -> Callable: - # type_hints = get_type_hints(function) - # serialized_hints = {key: type_to_dict(value) for key, value in type_hints.items()} - self.method_registry[function.__name__] = function - return function + # global mutex lock to ensure functions run sequentially + self.mutex_lock = Lock() + self.queue_size = 0 def on_message_received( self, user_content_manager: WebKit.UserContentManager, message: Any ) -> None: payload = json.loads(message.to_json(0)) - print(f"Received message: {payload}") method_name = payload["method"] handler_fn = self.method_registry[method_name] - # Start handler_fn in a new thread - # GLib.idle_add(handler_fn) + print(f"Received message: {payload}") + print(f"Queue size: {self.queue_size} (Wait)") - thread = threading.Thread( - target=self.threaded_handler, - args=(handler_fn, payload.get("data"), method_name), + def threaded_wrapper() -> bool: + """ + Ensures only one function is executed at a time + + Wait until there is no other function acquiring the global lock. + + Starts a thread with the potentially long running API function within. + """ + if not self.mutex_lock.locked(): + thread = threading.Thread( + target=self.threaded_handler, + args=( + handler_fn, + payload.get("data"), + method_name, + ), + ) + thread.start() + return GLib.SOURCE_REMOVE + + return GLib.SOURCE_CONTINUE + + GLib.idle_add( + threaded_wrapper, ) - thread.start() + self.queue_size += 1 def threaded_handler( self, handler_fn: Callable[[Any], Any], data: Any, method_name: str ) -> None: - result = handler_fn(data) - serialized = json.dumps(result) + with self.mutex_lock: + print("Executing", method_name) + print("threading locked ...") + result = handler_fn(data) + serialized = json.dumps(result) - # Use idle_add to queue the response call to js on the main GTK thread - GLib.idle_add(self.call_js, method_name, serialized) + # Use idle_add to queue the response call to js on the main GTK thread + GLib.idle_add(self.return_data_to_js, method_name, serialized) + print("threading unlocked") + self.queue_size -= 1 + if self.queue_size > 0: + print(f"remaining queue size: {self.queue_size}") + else: + print(f"Queue empty") - def call_js(self, method_name: str, serialized: str) -> bool: + def return_data_to_js(self, method_name: str, serialized: str) -> bool: # This function must be run on the main GTK thread to interact with the webview # result = method_fn(data) # takes very long # serialized = result @@ -119,7 +105,7 @@ class WebView: None, None, ) - return False # Important to return False so that it's not run again + return GLib.SOURCE_REMOVE def get_webview(self) -> WebKit.WebView: return self.webview diff --git a/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py b/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py index c8e66911..393634af 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py @@ -2,7 +2,7 @@ import logging import threading import gi -from clan_cli import machines + from clan_cli.history.list import list_history from clan_vm_manager.components.interfaces import ClanConfig @@ -14,10 +14,11 @@ from clan_vm_manager.views.list import ClanList from clan_vm_manager.views.logs import Logs from clan_vm_manager.views.webview import WebView +from clan_cli.api import API + gi.require_version("Adw", "1") from gi.repository import Adw, Gio, GLib, Gtk - from clan_vm_manager.components.trayicon import TrayIcon log = logging.getLogger(__name__) @@ -61,11 +62,7 @@ class MainWindow(Adw.ApplicationWindow): stack_view.add_named(Details(), "details") stack_view.add_named(Logs(), "logs") - webview = WebView() - - @webview.method - def list_machines(data: None) -> list[str]: - return machines.list.list_machines(".") + webview = WebView(methods=API._registry) stack_view.add_named(webview.get_webview(), "list") diff --git a/pkgs/webview-ui/.gitignore b/pkgs/webview-ui/.gitignore new file mode 100644 index 00000000..9e5bfb42 --- /dev/null +++ b/pkgs/webview-ui/.gitignore @@ -0,0 +1 @@ +api \ No newline at end of file diff --git a/pkgs/webview-ui/app/package-lock.json b/pkgs/webview-ui/app/package-lock.json index 32a95c66..02b01791 100644 --- a/pkgs/webview-ui/app/package-lock.json +++ b/pkgs/webview-ui/app/package-lock.json @@ -9,6 +9,8 @@ "version": "0.0.1", "license": "MIT", "dependencies": { + "@types/node": "^20.12.12", + "json-schema-to-ts": "^3.1.0", "solid-js": "^1.8.11" }, "devDependencies": { @@ -326,6 +328,17 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime": { + "version": "7.24.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.5.tgz", + "integrity": "sha512-Nms86NXrsaeU9vbBJKni6gXiEXZ4CVpYVzEjDH9Sb8vmZ3UljyA1GSOJl/6LGPO8EHLuSF9H+IxNXHPX8QHJ4g==", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.24.0", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.0.tgz", @@ -1352,6 +1365,14 @@ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", "dev": true }, + "node_modules/@types/node": { + "version": "20.12.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.12.tgz", + "integrity": "sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", @@ -2142,6 +2163,18 @@ "node": ">=4" } }, + "node_modules/json-schema-to-ts": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.0.tgz", + "integrity": "sha512-UeVN/ery4/JeXI8h4rM8yZPxsH+KqPi/84qFxHfTGHZnWnK9D0UU9ZGYO+6XAaJLqCWMiks+ARuFOKAiSxJCHA==", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -2582,6 +2615,11 @@ "node": ">=8.10.0" } }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" + }, "node_modules/resolve": { "version": "1.22.8", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", @@ -3008,6 +3046,11 @@ "node": ">=8.0" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==" + }, "node_modules/ts-interface-checker": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", @@ -3027,6 +3070,11 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/update-browserslist-db": { "version": "1.0.16", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz", diff --git a/pkgs/webview-ui/app/package.json b/pkgs/webview-ui/app/package.json index afc4ee7c..6f299343 100644 --- a/pkgs/webview-ui/app/package.json +++ b/pkgs/webview-ui/app/package.json @@ -7,7 +7,8 @@ "dev": "vite", "build": "vite build && npm run convert-html", "convert-html": "node gtk.webview.js", - "serve": "vite preview" + "serve": "vite preview", + "check": "tsc --noEmit --skipLibCheck" }, "license": "MIT", "devDependencies": { @@ -17,10 +18,12 @@ "solid-devtools": "^0.29.2", "tailwindcss": "^3.4.3", "typescript": "^5.3.3", - "vite-plugin-solid": "^2.8.2", - "vite": "^5.0.11" + "vite": "^5.0.11", + "vite-plugin-solid": "^2.8.2" }, "dependencies": { + "@types/node": "^20.12.12", + "json-schema-to-ts": "^3.1.0", "solid-js": "^1.8.11" } } diff --git a/pkgs/webview-ui/app/src/Config.tsx b/pkgs/webview-ui/app/src/Config.tsx index 1d14e6c3..3dd3c374 100644 --- a/pkgs/webview-ui/app/src/Config.tsx +++ b/pkgs/webview-ui/app/src/Config.tsx @@ -1,11 +1,11 @@ import { createSignal, createContext, useContext, JSXElement } from "solid-js"; -import { PYAPI } from "./message"; +import { pyApi } from "./message"; export const makeCountContext = () => { const [machines, setMachines] = createSignal([]); const [loading, setLoading] = createSignal(false); - PYAPI.list_machines.receive((machines) => { + pyApi.list_machines.receive((machines) => { setLoading(false); setMachines(machines); }); @@ -16,7 +16,7 @@ export const makeCountContext = () => { getMachines: () => { // When the gtk function sends its data the loading state will be set to false setLoading(true); - PYAPI.list_machines.dispatch(null); + pyApi.list_machines.dispatch("."); }, }, ] as const; diff --git a/pkgs/webview-ui/app/src/message.ts b/pkgs/webview-ui/app/src/message.ts index 6efa3fb6..fdf5339b 100644 --- a/pkgs/webview-ui/app/src/message.ts +++ b/pkgs/webview-ui/app/src/message.ts @@ -1,22 +1,74 @@ -const deserialize = (fn: Function) => (str: string) => { - try { - fn(JSON.parse(str)); - } catch (e) { - alert(`Error parsing JSON: ${e}`); - } -}; +import { FromSchema } from "json-schema-to-ts"; +import { schema } from "@/api"; -export const PYAPI = { - list_machines: { - dispatch: (data: null) => - // @ts-ignore +type API = FromSchema; + +type OperationNames = keyof API; +type OperationArgs = API[T]["argument"]; +type OperationResponse = API[T]["return"]; + +declare global { + interface Window { + clan: { + [K in OperationNames]: (str: string) => void; + }; + webkit: { + messageHandlers: { + gtk: { + postMessage: (message: { method: OperationNames; data: any }) => void; + }; + }; + }; + } +} + +function createFunctions( + operationName: K +): { + dispatch: (args: OperationArgs) => void; + receive: (fn: (response: OperationResponse) => void) => void; +} { + return { + dispatch: (args: OperationArgs) => { + console.log( + `Operation: ${operationName}, Arguments: ${JSON.stringify(args)}` + ); + // Send the data to the gtk app window.webkit.messageHandlers.gtk.postMessage({ - method: "list_machines", - data, - }), - receive: (fn: (response: string[]) => void) => { - // @ts-ignore + method: operationName, + data: args, + }); + }, + receive: (fn: (response: OperationResponse) => void) => { window.clan.list_machines = deserialize(fn); }, - }, + }; +} + +const operations = schema.properties; +const operationNames = Object.keys(operations) as OperationNames[]; + +type PyApi = { + [K in OperationNames]: { + dispatch: (args: OperationArgs) => void; + receive: (fn: (response: OperationResponse) => void) => void; + }; }; + +const deserialize = + (fn: (response: T) => void) => + (str: string) => { + try { + fn(JSON.parse(str) as T); + } catch (e) { + alert(`Error parsing JSON: ${e}`); + } + }; + +// Create the API object + +const pyApi: PyApi = {} as PyApi; +operationNames.forEach((name) => { + pyApi[name] = createFunctions(name); +}); +export { pyApi }; diff --git a/pkgs/webview-ui/app/tsconfig.json b/pkgs/webview-ui/app/tsconfig.json index 0aa89917..adaab944 100644 --- a/pkgs/webview-ui/app/tsconfig.json +++ b/pkgs/webview-ui/app/tsconfig.json @@ -10,6 +10,10 @@ "jsxImportSource": "solid-js", "types": ["vite/client"], "noEmit": true, + "resolveJsonModule": true, + "allowJs": true, "isolatedModules": true, + "paths": { + "@/*": ["./*"]} }, } diff --git a/pkgs/webview-ui/app/vite.config.ts b/pkgs/webview-ui/app/vite.config.ts index 5745d9fb..771cd5d3 100644 --- a/pkgs/webview-ui/app/vite.config.ts +++ b/pkgs/webview-ui/app/vite.config.ts @@ -1,8 +1,14 @@ import { defineConfig } from "vite"; import solidPlugin from "vite-plugin-solid"; // import devtools from "solid-devtools/vite"; +import path from "node:path"; export default defineConfig({ + resolve: { + alias: { + "@": path.resolve(__dirname, "./"), // Adjust the path as needed + }, + }, plugins: [ /* Uncomment the following line to enable solid-devtools. diff --git a/pkgs/webview-ui/default.nix b/pkgs/webview-ui/default.nix index 7d00ea8f..702bf5b9 100644 --- a/pkgs/webview-ui/default.nix +++ b/pkgs/webview-ui/default.nix @@ -1,9 +1,9 @@ -{ dream2nix, config, ... }: +{ dream2nix, config, src, ... }: { imports = [ dream2nix.modules.dream2nix.WIP-nodejs-builder-v3 ]; mkDerivation = { - src = ./app; + inherit src ; }; deps = @@ -15,7 +15,12 @@ WIP-nodejs-builder-v3 = { packageLockFile = "${config.mkDerivation.src}/package-lock.json"; }; - + public.out = { + checkPhase = '' + echo "Running tests" + echo "Tests passed" + ''; + }; name = "@clan/webview-ui"; version = "0.0.1"; } diff --git a/pkgs/webview-ui/flake-module.nix b/pkgs/webview-ui/flake-module.nix index 8dc88800..64ed3add 100644 --- a/pkgs/webview-ui/flake-module.nix +++ b/pkgs/webview-ui/flake-module.nix @@ -9,9 +9,28 @@ }: let node_modules-dev = config.packages.webview-ui.prepared-dev; + + src_with_api = pkgs.stdenv.mkDerivation { + name = "with-api"; + src = ./app; + buildInputs = [ pkgs.nodejs ]; + installPhase = '' + mkdir -p $out + + mkdir -p $out/api + cat ${config.packages.clan-ts-api} > $out/api/index.ts + + cp -r $src/* $out + + ls -la $out/api + ''; + }; in { packages.webview-ui = inputs.dream2nix.lib.evalModules { + specialArgs = { + src = src_with_api ; + }; packageSets.nixpkgs = inputs.dream2nix.inputs.nixpkgs.legacyPackages.${system}; modules = [ ./default.nix ]; }; @@ -28,6 +47,9 @@ echo -n $ID > .dream2nix/.node_modules_id echo "Ok: node_modules updated" fi + + mkdir -p ./app/api + cat ${config.packages.clan-ts-api} > ./app/api/index.ts ''; }; }; From d174fbd445a9307b960707d17f4125005151e497 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 20 May 2024 19:59:50 +0200 Subject: [PATCH 2/5] clan ui: move print into log.debug statements --- pkgs/clan-cli/api.py | 7 +--- pkgs/clan-cli/clan_cli/api/__init__.py | 30 +++++++++++++++++ .../api/{schema_compat.py => util.py} | 32 ------------------- pkgs/clan-cli/clan_cli/machines/list.py | 1 - .../clan_vm_manager/views/webview.py | 17 ++++------ 5 files changed, 38 insertions(+), 49 deletions(-) rename pkgs/clan-cli/clan_cli/api/{schema_compat.py => util.py} (71%) diff --git a/pkgs/clan-cli/api.py b/pkgs/clan-cli/api.py index ab5ca5d3..1acb847c 100644 --- a/pkgs/clan-cli/api.py +++ b/pkgs/clan-cli/api.py @@ -1,13 +1,8 @@ -from clan_cli import create_parser from clan_cli.api import API -from clan_cli.api.schema_compat import to_json_schema def main() -> None: - # Create the parser to register the API functions - create_parser() - - schema = to_json_schema(API._registry) + schema = API.to_json_schema() print( f"""export const schema = {schema} as const; """ diff --git a/pkgs/clan-cli/clan_cli/api/__init__.py b/pkgs/clan-cli/clan_cli/api/__init__.py index fe531c2b..d18ac26e 100644 --- a/pkgs/clan-cli/clan_cli/api/__init__.py +++ b/pkgs/clan-cli/clan_cli/api/__init__.py @@ -9,5 +9,35 @@ class _MethodRegistry: self._registry[fn.__name__] = fn return fn + def to_json_schema(self) -> str: + # Import only when needed + import json + from typing import get_type_hints + from clan_cli.api.util import type_to_dict + + api_schema = { + "$comment": "An object containing API methods. ", + "type": "object", + "additionalProperties": False, + "required": ["list_machines"], + "properties": {}, + } + for name, func in self._registry.items(): + hints = get_type_hints(func) + serialized_hints = { + "argument" if key != "return" else "return": type_to_dict( + value, scope=name + " argument" if key != "return" else "return" + ) + for key, value in hints.items() + } + api_schema["properties"][name] = { + "type": "object", + "required": [k for k in serialized_hints.keys()], + "additionalProperties": False, + "properties": {**serialized_hints}, + } + + return json.dumps(api_schema, indent=2) + API = _MethodRegistry() diff --git a/pkgs/clan-cli/clan_cli/api/schema_compat.py b/pkgs/clan-cli/clan_cli/api/util.py similarity index 71% rename from pkgs/clan-cli/clan_cli/api/schema_compat.py rename to pkgs/clan-cli/clan_cli/api/util.py index eeccf685..f6ccab7b 100644 --- a/pkgs/clan-cli/clan_cli/api/schema_compat.py +++ b/pkgs/clan-cli/clan_cli/api/util.py @@ -1,16 +1,10 @@ import dataclasses -import json from types import NoneType, UnionType from typing import Any, Callable, Union, get_type_hints import pathlib def type_to_dict(t: Any, scope: str = "") -> dict: - # print( - # f"Type: {t}, Scope: {scope}, has origin: {hasattr(t, '__origin__')} ", - # type(t) is UnionType, - # ) - if t is None: return {"type": "null"} @@ -83,29 +77,3 @@ def type_to_dict(t: Any, scope: str = "") -> dict: raise BaseException(f"Error primitive type not supported {str(t)}") else: raise BaseException(f"Error type not supported {str(t)}") - - -def to_json_schema(methods: dict[str, Callable]) -> str: - api_schema = { - "$comment": "An object containing API methods. ", - "type": "object", - "additionalProperties": False, - "required": ["list_machines"], - "properties": {}, - } - for name, func in methods.items(): - hints = get_type_hints(func) - serialized_hints = { - "argument" if key != "return" else "return": type_to_dict( - value, scope=name + " argument" if key != "return" else "return" - ) - for key, value in hints.items() - } - api_schema["properties"][name] = { - "type": "object", - "required": [k for k in serialized_hints.keys()], - "additionalProperties": False, - "properties": {**serialized_hints}, - } - - return json.dumps(api_schema, indent=2) diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index 4e483d8e..be8903d5 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -12,7 +12,6 @@ log = logging.getLogger(__name__) @API.register def list_machines(flake_url: Path | str) -> list[str]: - print("list_machines", flake_url) config = nix_config() system = config["system"] cmd = nix_eval( diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py b/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py index 454ac917..08a5901e 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py @@ -1,5 +1,5 @@ -import dataclasses import json +import logging import sys import threading from threading import Lock @@ -19,6 +19,8 @@ site_index: Path = ( / Path("clan_vm_manager/.webui/index.html") ).resolve() +log = logging.getLogger(__name__) + class WebView: def __init__(self, methods: dict[str, Callable]) -> None: @@ -44,8 +46,8 @@ class WebView: method_name = payload["method"] handler_fn = self.method_registry[method_name] - print(f"Received message: {payload}") - print(f"Queue size: {self.queue_size} (Wait)") + log.debug(f"Received message: {payload}") + log.debug(f"Queue size: {self.queue_size} (Wait)") def threaded_wrapper() -> bool: """ @@ -78,19 +80,14 @@ class WebView: self, handler_fn: Callable[[Any], Any], data: Any, method_name: str ) -> None: with self.mutex_lock: - print("Executing", method_name) - print("threading locked ...") + log.debug("Executing... ", method_name) result = handler_fn(data) serialized = json.dumps(result) # Use idle_add to queue the response call to js on the main GTK thread GLib.idle_add(self.return_data_to_js, method_name, serialized) - print("threading unlocked") self.queue_size -= 1 - if self.queue_size > 0: - print(f"remaining queue size: {self.queue_size}") - else: - print(f"Queue empty") + log.debug(f"Done: Remaining queue size: {self.queue_size}") def return_data_to_js(self, method_name: str, serialized: str) -> bool: # This function must be run on the main GTK thread to interact with the webview From e9b0ce6a78b3d075cc13ddbc3262d49fab0c9d78 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 21 May 2024 10:09:05 +0200 Subject: [PATCH 3/5] clan ui: add check to build script --- pkgs/webview-ui/app/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/webview-ui/app/package.json b/pkgs/webview-ui/app/package.json index 6f299343..2385683f 100644 --- a/pkgs/webview-ui/app/package.json +++ b/pkgs/webview-ui/app/package.json @@ -5,7 +5,7 @@ "scripts": { "start": "vite", "dev": "vite", - "build": "vite build && npm run convert-html", + "build": "npm run check && vite build && npm run convert-html", "convert-html": "node gtk.webview.js", "serve": "vite preview", "check": "tsc --noEmit --skipLibCheck" From 21104a0465c99e0760f7b7f3dbee0f54db3070c5 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 21 May 2024 15:50:52 +0200 Subject: [PATCH 4/5] webview: replace dream2nix by buildNpm from nixpkgs to save inputs --- flake.lock | 84 -------------------------------- flake.nix | 3 +- pkgs/webview-ui/default.nix | 10 +++- pkgs/webview-ui/flake-module.nix | 57 +++++++--------------- 4 files changed, 26 insertions(+), 128 deletions(-) diff --git a/flake.lock b/flake.lock index 1b523d11..a2b71934 100644 --- a/flake.lock +++ b/flake.lock @@ -20,28 +20,6 @@ "type": "github" } }, - "dream2nix": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ], - "purescript-overlay": "purescript-overlay", - "pyproject-nix": "pyproject-nix" - }, - "locked": { - "lastModified": 1716143526, - "narHash": "sha256-Pk77RzfAlrr1iR81zFJA/puxr2OeBifd0Yb0ETWMVKE=", - "owner": "nix-community", - "repo": "dream2nix", - "rev": "f28d1cc6898532ac331a1625628090dc7c5b02e2", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "dream2nix", - "type": "github" - } - }, "flake-parts": { "inputs": { "nixpkgs-lib": [ @@ -151,49 +129,9 @@ "type": "github" } }, - "purescript-overlay": { - "inputs": { - "nixpkgs": [ - "dream2nix", - "nixpkgs" - ], - "slimlock": "slimlock" - }, - "locked": { - "lastModified": 1696022621, - "narHash": "sha256-eMjFmsj2G1E0Q5XiibUNgFjTiSz0GxIeSSzzVdoN730=", - "owner": "thomashoneyman", - "repo": "purescript-overlay", - "rev": "047c7933abd6da8aa239904422e22d190ce55ead", - "type": "github" - }, - "original": { - "owner": "thomashoneyman", - "repo": "purescript-overlay", - "type": "github" - } - }, - "pyproject-nix": { - "flake": false, - "locked": { - "lastModified": 1702448246, - "narHash": "sha256-hFg5s/hoJFv7tDpiGvEvXP0UfFvFEDgTdyHIjDVHu1I=", - "owner": "davhau", - "repo": "pyproject.nix", - "rev": "5a06a2697b228c04dd2f35659b4b659ca74f7aeb", - "type": "github" - }, - "original": { - "owner": "davhau", - "ref": "dream2nix", - "repo": "pyproject.nix", - "type": "github" - } - }, "root": { "inputs": { "disko": "disko", - "dream2nix": "dream2nix", "flake-parts": "flake-parts", "nixos-generators": "nixos-generators", "nixos-images": "nixos-images", @@ -202,28 +140,6 @@ "treefmt-nix": "treefmt-nix" } }, - "slimlock": { - "inputs": { - "nixpkgs": [ - "dream2nix", - "purescript-overlay", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1688610262, - "narHash": "sha256-Wg0ViDotFWGWqKIQzyYCgayeH8s4U1OZcTiWTQYdAp4=", - "owner": "thomashoneyman", - "repo": "slimlock", - "rev": "b5c6cdcaf636ebbebd0a1f32520929394493f1a6", - "type": "github" - }, - "original": { - "owner": "thomashoneyman", - "repo": "slimlock", - "type": "github" - } - }, "sops-nix": { "inputs": { "nixpkgs": [ diff --git a/flake.nix b/flake.nix index c893a15d..e24d7d9b 100644 --- a/flake.nix +++ b/flake.nix @@ -21,8 +21,7 @@ flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; treefmt-nix.url = "github:numtide/treefmt-nix"; treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; - dream2nix.url = "github:nix-community/dream2nix"; - dream2nix.inputs.nixpkgs.follows = "nixpkgs"; + }; outputs = diff --git a/pkgs/webview-ui/default.nix b/pkgs/webview-ui/default.nix index 702bf5b9..1068eecd 100644 --- a/pkgs/webview-ui/default.nix +++ b/pkgs/webview-ui/default.nix @@ -1,9 +1,14 @@ -{ dream2nix, config, src, ... }: +{ + dream2nix, + config, + src, + ... +}: { imports = [ dream2nix.modules.dream2nix.WIP-nodejs-builder-v3 ]; mkDerivation = { - inherit src ; + inherit src; }; deps = @@ -14,6 +19,7 @@ WIP-nodejs-builder-v3 = { packageLockFile = "${config.mkDerivation.src}/package-lock.json"; + # config.groups.all.packages.${config.name}.${config.version}. }; public.out = { checkPhase = '' diff --git a/pkgs/webview-ui/flake-module.nix b/pkgs/webview-ui/flake-module.nix index 64ed3add..4984977c 100644 --- a/pkgs/webview-ui/flake-module.nix +++ b/pkgs/webview-ui/flake-module.nix @@ -1,53 +1,30 @@ -{ inputs, ... }: +{ ... }: { perSystem = + { pkgs, config, ... }: { - system, - pkgs, - config, - ... - }: - let - node_modules-dev = config.packages.webview-ui.prepared-dev; + packages.webview-ui = pkgs.buildNpmPackage { + pname = "clan-webview-ui"; + version = "0.0.1"; - src_with_api = pkgs.stdenv.mkDerivation { - name = "with-api"; src = ./app; - buildInputs = [ pkgs.nodejs ]; - installPhase = '' - mkdir -p $out - - mkdir -p $out/api - cat ${config.packages.clan-ts-api} > $out/api/index.ts - cp -r $src/* $out - - ls -la $out/api - ''; - }; - in - { - packages.webview-ui = inputs.dream2nix.lib.evalModules { - specialArgs = { - src = src_with_api ; + # npmDepsHash = "sha256-bRD2vzijhdOOvcEi6XaG/neSqhkVQMqIX/8bxvRQkTc="; + npmDeps = pkgs.fetchNpmDeps { + src = ./app; + hash = "sha256-bRD2vzijhdOOvcEi6XaG/neSqhkVQMqIX/8bxvRQkTc="; }; - packageSets.nixpkgs = inputs.dream2nix.inputs.nixpkgs.legacyPackages.${system}; - modules = [ ./default.nix ]; + # The prepack script runs the build script, which we'd rather do in the build phase. + npmPackFlags = [ "--ignore-scripts" ]; + + preBuild = '' + mkdir -p api + cat ${config.packages.clan-ts-api} > api/index.ts + ''; }; devShells.webview-ui = pkgs.mkShell { - inputsFrom = [ config.packages.webview-ui.out ]; + inputsFrom = [ config.packages.webview-ui ]; shellHook = '' - ID=${node_modules-dev} - currID=$(cat .dream2nix/.node_modules_id 2> /dev/null) - - mkdir -p .dream2nix - if [[ "$ID" != "$currID" || ! -d "app/node_modules" ]]; - then - ${pkgs.rsync}/bin/rsync -a --chmod=ug+w --delete ${node_modules-dev}/node_modules/ ./app/node_modules/ - echo -n $ID > .dream2nix/.node_modules_id - echo "Ok: node_modules updated" - fi - mkdir -p ./app/api cat ${config.packages.clan-ts-api} > ./app/api/index.ts ''; From c1e7e256418ec816642e21d41ae24f3185c289eb Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 21 May 2024 15:57:27 +0200 Subject: [PATCH 5/5] api: fix typing issues --- flake.nix | 1 - pkgs/clan-cli/clan_cli/api/__init__.py | 8 +++++--- pkgs/clan-cli/clan_cli/api/util.py | 10 +++++----- pkgs/clan-cli/clan_cli/machines/list.py | 3 ++- pkgs/clan-vm-manager/clan_vm_manager/views/webview.py | 6 +++--- .../clan_vm_manager/windows/main_window.py | 5 ++--- pkgs/clan-vm-manager/default.nix | 2 +- pkgs/clan-vm-manager/shell.nix | 2 +- 8 files changed, 19 insertions(+), 18 deletions(-) diff --git a/flake.nix b/flake.nix index e24d7d9b..3b289028 100644 --- a/flake.nix +++ b/flake.nix @@ -21,7 +21,6 @@ flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; treefmt-nix.url = "github:numtide/treefmt-nix"; treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; - }; outputs = diff --git a/pkgs/clan-cli/clan_cli/api/__init__.py b/pkgs/clan-cli/clan_cli/api/__init__.py index d18ac26e..29b3f45b 100644 --- a/pkgs/clan-cli/clan_cli/api/__init__.py +++ b/pkgs/clan-cli/clan_cli/api/__init__.py @@ -1,9 +1,10 @@ from collections.abc import Callable +from typing import Any class _MethodRegistry: - def __init__(self): - self._registry = {} + def __init__(self) -> None: + self._registry: dict[str, Callable] = {} def register(self, fn: Callable) -> Callable: self._registry[fn.__name__] = fn @@ -13,9 +14,10 @@ class _MethodRegistry: # Import only when needed import json from typing import get_type_hints + from clan_cli.api.util import type_to_dict - api_schema = { + api_schema: dict[str, Any] = { "$comment": "An object containing API methods. ", "type": "object", "additionalProperties": False, diff --git a/pkgs/clan-cli/clan_cli/api/util.py b/pkgs/clan-cli/clan_cli/api/util.py index f6ccab7b..9765a5fe 100644 --- a/pkgs/clan-cli/clan_cli/api/util.py +++ b/pkgs/clan-cli/clan_cli/api/util.py @@ -1,7 +1,7 @@ import dataclasses -from types import NoneType, UnionType -from typing import Any, Callable, Union, get_type_hints import pathlib +from types import NoneType, UnionType +from typing import Any, Union def type_to_dict(t: Any, scope: str = "") -> dict: @@ -46,7 +46,7 @@ def type_to_dict(t: Any, scope: str = "") -> dict: "type": "object", } - raise BaseException(f"Error api type not yet supported {str(t)}") + raise BaseException(f"Error api type not yet supported {t!s}") elif isinstance(t, type): if t is str: @@ -74,6 +74,6 @@ def type_to_dict(t: Any, scope: str = "") -> dict: if t is NoneType: return {"type": "null"} - raise BaseException(f"Error primitive type not supported {str(t)}") + raise BaseException(f"Error primitive type not supported {t!s}") else: - raise BaseException(f"Error type not supported {str(t)}") + raise BaseException(f"Error type not supported {t!s}") diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index be8903d5..ad77b78b 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -3,9 +3,10 @@ import json import logging from pathlib import Path +from clan_cli.api import API + from ..cmd import run from ..nix import nix_config, nix_eval -from clan_cli.api import API log = logging.getLogger(__name__) diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py b/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py index 08a5901e..0c28ad01 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py @@ -2,16 +2,16 @@ import json import logging import sys import threading -from threading import Lock from collections.abc import Callable from pathlib import Path -from typing import Any, Union +from threading import Lock +from typing import Any import gi gi.require_version("WebKit", "6.0") -from gi.repository import GLib, GObject, WebKit +from gi.repository import GLib, WebKit site_index: Path = ( Path(sys.argv[0]).absolute() diff --git a/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py b/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py index 393634af..5ec1fc12 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py @@ -2,7 +2,7 @@ import logging import threading import gi - +from clan_cli.api import API from clan_cli.history.list import list_history from clan_vm_manager.components.interfaces import ClanConfig @@ -14,11 +14,10 @@ from clan_vm_manager.views.list import ClanList from clan_vm_manager.views.logs import Logs from clan_vm_manager.views.webview import WebView -from clan_cli.api import API - gi.require_version("Adw", "1") from gi.repository import Adw, Gio, GLib, Gtk + from clan_vm_manager.components.trayicon import TrayIcon log = logging.getLogger(__name__) diff --git a/pkgs/clan-vm-manager/default.nix b/pkgs/clan-vm-manager/default.nix index 20f54225..d81bc97f 100644 --- a/pkgs/clan-vm-manager/default.nix +++ b/pkgs/clan-vm-manager/default.nix @@ -145,7 +145,7 @@ python3.pkgs.buildPythonApplication rec { # TODO: place webui in lib/python3.11/site-packages/clan_vm_manager postInstall = '' mkdir -p $out/clan_vm_manager/.webui - cp -r ${webview-ui}/dist/* $out/clan_vm_manager/.webui + cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/clan_vm_manager/.webui ''; # Don't leak python packages into a devshell. diff --git a/pkgs/clan-vm-manager/shell.nix b/pkgs/clan-vm-manager/shell.nix index 71c8bdd1..4de1f494 100644 --- a/pkgs/clan-vm-manager/shell.nix +++ b/pkgs/clan-vm-manager/shell.nix @@ -56,7 +56,7 @@ mkShell { # Add the webview-ui to the .webui directory rm -rf ./clan_vm_manager/.webui/* mkdir -p ./clan_vm_manager/.webui - cp -a ${webview-ui}/dist/* ./clan_vm_manager/.webui + cp -a ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* ./clan_vm_manager/.webui chmod -R +w ./clan_vm_manager/.webui ''; }