diff --git a/pkgs/clan-cli/api.py b/pkgs/clan-cli/api.py new file mode 100644 index 00000000..ab5ca5d3 --- /dev/null +++ b/pkgs/clan-cli/api.py @@ -0,0 +1,18 @@ +from clan_cli import create_parser +from clan_cli.api import API +from clan_cli.api.schema_compat import to_json_schema + + +def main() -> None: + # Create the parser to register the API functions + create_parser() + + schema = to_json_schema(API._registry) + print( + f"""export const schema = {schema} as const; +""" + ) + + +if __name__ == "__main__": + main() diff --git a/pkgs/clan-cli/clan_cli/api/__init__.py b/pkgs/clan-cli/clan_cli/api/__init__.py new file mode 100644 index 00000000..fe531c2b --- /dev/null +++ b/pkgs/clan-cli/clan_cli/api/__init__.py @@ -0,0 +1,13 @@ +from collections.abc import Callable + + +class _MethodRegistry: + def __init__(self): + self._registry = {} + + def register(self, fn: Callable) -> Callable: + self._registry[fn.__name__] = fn + return fn + + +API = _MethodRegistry() diff --git a/pkgs/clan-cli/clan_cli/api/schema_compat.py b/pkgs/clan-cli/clan_cli/api/schema_compat.py new file mode 100644 index 00000000..eeccf685 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/api/schema_compat.py @@ -0,0 +1,111 @@ +import dataclasses +import json +from types import NoneType, UnionType +from typing import Any, Callable, Union, get_type_hints +import pathlib + + +def type_to_dict(t: Any, scope: str = "") -> dict: + # print( + # f"Type: {t}, Scope: {scope}, has origin: {hasattr(t, '__origin__')} ", + # type(t) is UnionType, + # ) + + if t is None: + return {"type": "null"} + + if dataclasses.is_dataclass(t): + fields = dataclasses.fields(t) + properties = { + f.name: type_to_dict(f.type, f"{scope} {t.__name__}.{f.name}") + for f in fields + } + required = [pn for pn, pv in properties.items() if "null" not in pv["type"]] + return { + "type": "object", + "properties": properties, + "required": required, + # Dataclasses can only have the specified properties + "additionalProperties": False, + } + elif type(t) is UnionType: + return { + "type": [type_to_dict(arg, scope)["type"] for arg in t.__args__], + } + + elif hasattr(t, "__origin__"): # Check if it's a generic type + origin = getattr(t, "__origin__", None) + + if origin is None: + # Non-generic user-defined or built-in type + # TODO: handle custom types + raise BaseException("Unhandled Type: ", origin) + + elif origin is Union: + return {"type": [type_to_dict(arg, scope)["type"] for arg in t.__args__]} + + elif issubclass(origin, list): + return {"type": "array", "items": type_to_dict(t.__args__[0], scope)} + + elif issubclass(origin, dict): + return { + "type": "object", + } + + raise BaseException(f"Error api type not yet supported {str(t)}") + + elif isinstance(t, type): + if t is str: + return {"type": "string"} + if t is int: + return {"type": "integer"} + if t is float: + return {"type": "number"} + if t is bool: + return {"type": "boolean"} + if t is object: + return {"type": "object"} + if t is Any: + raise BaseException( + f"Usage of the Any type is not supported for API functions. In: {scope}" + ) + + if t is pathlib.Path: + return { + # TODO: maybe give it a pattern for URI + "type": "string", + } + + # Optional[T] gets internally transformed Union[T,NoneType] + if t is NoneType: + return {"type": "null"} + + raise BaseException(f"Error primitive type not supported {str(t)}") + else: + raise BaseException(f"Error type not supported {str(t)}") + + +def to_json_schema(methods: dict[str, Callable]) -> str: + api_schema = { + "$comment": "An object containing API methods. ", + "type": "object", + "additionalProperties": False, + "required": ["list_machines"], + "properties": {}, + } + for name, func in methods.items(): + hints = get_type_hints(func) + serialized_hints = { + "argument" if key != "return" else "return": type_to_dict( + value, scope=name + " argument" if key != "return" else "return" + ) + for key, value in hints.items() + } + api_schema["properties"][name] = { + "type": "object", + "required": [k for k in serialized_hints.keys()], + "additionalProperties": False, + "properties": {**serialized_hints}, + } + + return json.dumps(api_schema, indent=2) diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index 83ce2d66..4e483d8e 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -5,11 +5,14 @@ from pathlib import Path from ..cmd import run from ..nix import nix_config, nix_eval +from clan_cli.api import API log = logging.getLogger(__name__) +@API.register def list_machines(flake_url: Path | str) -> list[str]: + print("list_machines", flake_url) config = nix_config() system = config["system"] cmd = nix_eval( diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 2e4969b6..11e6c841 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -57,6 +57,16 @@ cp -r out/* $out ''; }; + clan-ts-api = pkgs.stdenv.mkDerivation { + name = "clan-ts-api"; + src = ./.; + + buildInputs = [ pkgs.python3 ]; + + installPhase = '' + python api.py > $out + ''; + }; default = self'.packages.clan-cli; }; diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py b/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py index 78a7e1b1..454ac917 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/webview.py @@ -2,6 +2,7 @@ import dataclasses import json import sys import threading +from threading import Lock from collections.abc import Callable from pathlib import Path from typing import Any, Union @@ -10,7 +11,7 @@ import gi gi.require_version("WebKit", "6.0") -from gi.repository import GLib, WebKit +from gi.repository import GLib, GObject, WebKit site_index: Path = ( Path(sys.argv[0]).absolute() @@ -19,51 +20,9 @@ site_index: Path = ( ).resolve() -def type_to_dict(t: Any) -> dict: - if dataclasses.is_dataclass(t): - fields = dataclasses.fields(t) - return { - "type": "dataclass", - "name": t.__name__, - "fields": {f.name: type_to_dict(f.type) for f in fields}, - } - - if hasattr(t, "__origin__"): # Check if it's a generic type - if t.__origin__ is None: - # Non-generic user-defined or built-in type - return {"type": t.__name__} - if t.__origin__ is Union: - return {"type": "union", "of": [type_to_dict(arg) for arg in t.__args__]} - elif issubclass(t.__origin__, list): - return {"type": "list", "item_type": type_to_dict(t.__args__[0])} - elif issubclass(t.__origin__, dict): - return { - "type": "dict", - "key_type": type_to_dict(t.__args__[0]), - "value_type": type_to_dict(t.__args__[1]), - } - elif issubclass(t.__origin__, tuple): - return { - "type": "tuple", - "element_types": [type_to_dict(elem) for elem in t.__args__], - } - elif issubclass(t.__origin__, set): - return {"type": "set", "item_type": type_to_dict(t.__args__[0])} - else: - # Handle other generic types (like Union, Optional) - return { - "type": str(t.__origin__.__name__), - "parameters": [type_to_dict(arg) for arg in t.__args__], - } - elif isinstance(t, type): - return {"type": t.__name__} - else: - return {"type": str(t)} - - class WebView: - def __init__(self) -> None: - self.method_registry: dict[str, Callable] = {} + def __init__(self, methods: dict[str, Callable]) -> None: + self.method_registry: dict[str, Callable] = methods self.webview = WebKit.WebView() self.manager = self.webview.get_user_content_manager() @@ -74,39 +33,66 @@ class WebView: self.webview.load_uri(f"file://{site_index}") - def method(self, function: Callable) -> Callable: - # type_hints = get_type_hints(function) - # serialized_hints = {key: type_to_dict(value) for key, value in type_hints.items()} - self.method_registry[function.__name__] = function - return function + # global mutex lock to ensure functions run sequentially + self.mutex_lock = Lock() + self.queue_size = 0 def on_message_received( self, user_content_manager: WebKit.UserContentManager, message: Any ) -> None: payload = json.loads(message.to_json(0)) - print(f"Received message: {payload}") method_name = payload["method"] handler_fn = self.method_registry[method_name] - # Start handler_fn in a new thread - # GLib.idle_add(handler_fn) + print(f"Received message: {payload}") + print(f"Queue size: {self.queue_size} (Wait)") - thread = threading.Thread( - target=self.threaded_handler, - args=(handler_fn, payload.get("data"), method_name), + def threaded_wrapper() -> bool: + """ + Ensures only one function is executed at a time + + Wait until there is no other function acquiring the global lock. + + Starts a thread with the potentially long running API function within. + """ + if not self.mutex_lock.locked(): + thread = threading.Thread( + target=self.threaded_handler, + args=( + handler_fn, + payload.get("data"), + method_name, + ), + ) + thread.start() + return GLib.SOURCE_REMOVE + + return GLib.SOURCE_CONTINUE + + GLib.idle_add( + threaded_wrapper, ) - thread.start() + self.queue_size += 1 def threaded_handler( self, handler_fn: Callable[[Any], Any], data: Any, method_name: str ) -> None: - result = handler_fn(data) - serialized = json.dumps(result) + with self.mutex_lock: + print("Executing", method_name) + print("threading locked ...") + result = handler_fn(data) + serialized = json.dumps(result) - # Use idle_add to queue the response call to js on the main GTK thread - GLib.idle_add(self.call_js, method_name, serialized) + # Use idle_add to queue the response call to js on the main GTK thread + GLib.idle_add(self.return_data_to_js, method_name, serialized) + print("threading unlocked") + self.queue_size -= 1 + if self.queue_size > 0: + print(f"remaining queue size: {self.queue_size}") + else: + print(f"Queue empty") - def call_js(self, method_name: str, serialized: str) -> bool: + def return_data_to_js(self, method_name: str, serialized: str) -> bool: # This function must be run on the main GTK thread to interact with the webview # result = method_fn(data) # takes very long # serialized = result @@ -119,7 +105,7 @@ class WebView: None, None, ) - return False # Important to return False so that it's not run again + return GLib.SOURCE_REMOVE def get_webview(self) -> WebKit.WebView: return self.webview diff --git a/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py b/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py index c8e66911..393634af 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py @@ -2,7 +2,7 @@ import logging import threading import gi -from clan_cli import machines + from clan_cli.history.list import list_history from clan_vm_manager.components.interfaces import ClanConfig @@ -14,10 +14,11 @@ from clan_vm_manager.views.list import ClanList from clan_vm_manager.views.logs import Logs from clan_vm_manager.views.webview import WebView +from clan_cli.api import API + gi.require_version("Adw", "1") from gi.repository import Adw, Gio, GLib, Gtk - from clan_vm_manager.components.trayicon import TrayIcon log = logging.getLogger(__name__) @@ -61,11 +62,7 @@ class MainWindow(Adw.ApplicationWindow): stack_view.add_named(Details(), "details") stack_view.add_named(Logs(), "logs") - webview = WebView() - - @webview.method - def list_machines(data: None) -> list[str]: - return machines.list.list_machines(".") + webview = WebView(methods=API._registry) stack_view.add_named(webview.get_webview(), "list") diff --git a/pkgs/webview-ui/.gitignore b/pkgs/webview-ui/.gitignore new file mode 100644 index 00000000..9e5bfb42 --- /dev/null +++ b/pkgs/webview-ui/.gitignore @@ -0,0 +1 @@ +api \ No newline at end of file diff --git a/pkgs/webview-ui/app/package-lock.json b/pkgs/webview-ui/app/package-lock.json index 32a95c66..02b01791 100644 --- a/pkgs/webview-ui/app/package-lock.json +++ b/pkgs/webview-ui/app/package-lock.json @@ -9,6 +9,8 @@ "version": "0.0.1", "license": "MIT", "dependencies": { + "@types/node": "^20.12.12", + "json-schema-to-ts": "^3.1.0", "solid-js": "^1.8.11" }, "devDependencies": { @@ -326,6 +328,17 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime": { + "version": "7.24.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.5.tgz", + "integrity": "sha512-Nms86NXrsaeU9vbBJKni6gXiEXZ4CVpYVzEjDH9Sb8vmZ3UljyA1GSOJl/6LGPO8EHLuSF9H+IxNXHPX8QHJ4g==", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.24.0", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.0.tgz", @@ -1352,6 +1365,14 @@ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", "dev": true }, + "node_modules/@types/node": { + "version": "20.12.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.12.tgz", + "integrity": "sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", @@ -2142,6 +2163,18 @@ "node": ">=4" } }, + "node_modules/json-schema-to-ts": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.0.tgz", + "integrity": "sha512-UeVN/ery4/JeXI8h4rM8yZPxsH+KqPi/84qFxHfTGHZnWnK9D0UU9ZGYO+6XAaJLqCWMiks+ARuFOKAiSxJCHA==", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -2582,6 +2615,11 @@ "node": ">=8.10.0" } }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" + }, "node_modules/resolve": { "version": "1.22.8", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", @@ -3008,6 +3046,11 @@ "node": ">=8.0" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==" + }, "node_modules/ts-interface-checker": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", @@ -3027,6 +3070,11 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/update-browserslist-db": { "version": "1.0.16", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz", diff --git a/pkgs/webview-ui/app/package.json b/pkgs/webview-ui/app/package.json index afc4ee7c..6f299343 100644 --- a/pkgs/webview-ui/app/package.json +++ b/pkgs/webview-ui/app/package.json @@ -7,7 +7,8 @@ "dev": "vite", "build": "vite build && npm run convert-html", "convert-html": "node gtk.webview.js", - "serve": "vite preview" + "serve": "vite preview", + "check": "tsc --noEmit --skipLibCheck" }, "license": "MIT", "devDependencies": { @@ -17,10 +18,12 @@ "solid-devtools": "^0.29.2", "tailwindcss": "^3.4.3", "typescript": "^5.3.3", - "vite-plugin-solid": "^2.8.2", - "vite": "^5.0.11" + "vite": "^5.0.11", + "vite-plugin-solid": "^2.8.2" }, "dependencies": { + "@types/node": "^20.12.12", + "json-schema-to-ts": "^3.1.0", "solid-js": "^1.8.11" } } diff --git a/pkgs/webview-ui/app/src/Config.tsx b/pkgs/webview-ui/app/src/Config.tsx index 1d14e6c3..3dd3c374 100644 --- a/pkgs/webview-ui/app/src/Config.tsx +++ b/pkgs/webview-ui/app/src/Config.tsx @@ -1,11 +1,11 @@ import { createSignal, createContext, useContext, JSXElement } from "solid-js"; -import { PYAPI } from "./message"; +import { pyApi } from "./message"; export const makeCountContext = () => { const [machines, setMachines] = createSignal([]); const [loading, setLoading] = createSignal(false); - PYAPI.list_machines.receive((machines) => { + pyApi.list_machines.receive((machines) => { setLoading(false); setMachines(machines); }); @@ -16,7 +16,7 @@ export const makeCountContext = () => { getMachines: () => { // When the gtk function sends its data the loading state will be set to false setLoading(true); - PYAPI.list_machines.dispatch(null); + pyApi.list_machines.dispatch("."); }, }, ] as const; diff --git a/pkgs/webview-ui/app/src/message.ts b/pkgs/webview-ui/app/src/message.ts index 6efa3fb6..fdf5339b 100644 --- a/pkgs/webview-ui/app/src/message.ts +++ b/pkgs/webview-ui/app/src/message.ts @@ -1,22 +1,74 @@ -const deserialize = (fn: Function) => (str: string) => { - try { - fn(JSON.parse(str)); - } catch (e) { - alert(`Error parsing JSON: ${e}`); - } -}; +import { FromSchema } from "json-schema-to-ts"; +import { schema } from "@/api"; -export const PYAPI = { - list_machines: { - dispatch: (data: null) => - // @ts-ignore +type API = FromSchema; + +type OperationNames = keyof API; +type OperationArgs = API[T]["argument"]; +type OperationResponse = API[T]["return"]; + +declare global { + interface Window { + clan: { + [K in OperationNames]: (str: string) => void; + }; + webkit: { + messageHandlers: { + gtk: { + postMessage: (message: { method: OperationNames; data: any }) => void; + }; + }; + }; + } +} + +function createFunctions( + operationName: K +): { + dispatch: (args: OperationArgs) => void; + receive: (fn: (response: OperationResponse) => void) => void; +} { + return { + dispatch: (args: OperationArgs) => { + console.log( + `Operation: ${operationName}, Arguments: ${JSON.stringify(args)}` + ); + // Send the data to the gtk app window.webkit.messageHandlers.gtk.postMessage({ - method: "list_machines", - data, - }), - receive: (fn: (response: string[]) => void) => { - // @ts-ignore + method: operationName, + data: args, + }); + }, + receive: (fn: (response: OperationResponse) => void) => { window.clan.list_machines = deserialize(fn); }, - }, + }; +} + +const operations = schema.properties; +const operationNames = Object.keys(operations) as OperationNames[]; + +type PyApi = { + [K in OperationNames]: { + dispatch: (args: OperationArgs) => void; + receive: (fn: (response: OperationResponse) => void) => void; + }; }; + +const deserialize = + (fn: (response: T) => void) => + (str: string) => { + try { + fn(JSON.parse(str) as T); + } catch (e) { + alert(`Error parsing JSON: ${e}`); + } + }; + +// Create the API object + +const pyApi: PyApi = {} as PyApi; +operationNames.forEach((name) => { + pyApi[name] = createFunctions(name); +}); +export { pyApi }; diff --git a/pkgs/webview-ui/app/tsconfig.json b/pkgs/webview-ui/app/tsconfig.json index 0aa89917..adaab944 100644 --- a/pkgs/webview-ui/app/tsconfig.json +++ b/pkgs/webview-ui/app/tsconfig.json @@ -10,6 +10,10 @@ "jsxImportSource": "solid-js", "types": ["vite/client"], "noEmit": true, + "resolveJsonModule": true, + "allowJs": true, "isolatedModules": true, + "paths": { + "@/*": ["./*"]} }, } diff --git a/pkgs/webview-ui/app/vite.config.ts b/pkgs/webview-ui/app/vite.config.ts index 5745d9fb..771cd5d3 100644 --- a/pkgs/webview-ui/app/vite.config.ts +++ b/pkgs/webview-ui/app/vite.config.ts @@ -1,8 +1,14 @@ import { defineConfig } from "vite"; import solidPlugin from "vite-plugin-solid"; // import devtools from "solid-devtools/vite"; +import path from "node:path"; export default defineConfig({ + resolve: { + alias: { + "@": path.resolve(__dirname, "./"), // Adjust the path as needed + }, + }, plugins: [ /* Uncomment the following line to enable solid-devtools. diff --git a/pkgs/webview-ui/default.nix b/pkgs/webview-ui/default.nix index 7d00ea8f..702bf5b9 100644 --- a/pkgs/webview-ui/default.nix +++ b/pkgs/webview-ui/default.nix @@ -1,9 +1,9 @@ -{ dream2nix, config, ... }: +{ dream2nix, config, src, ... }: { imports = [ dream2nix.modules.dream2nix.WIP-nodejs-builder-v3 ]; mkDerivation = { - src = ./app; + inherit src ; }; deps = @@ -15,7 +15,12 @@ WIP-nodejs-builder-v3 = { packageLockFile = "${config.mkDerivation.src}/package-lock.json"; }; - + public.out = { + checkPhase = '' + echo "Running tests" + echo "Tests passed" + ''; + }; name = "@clan/webview-ui"; version = "0.0.1"; } diff --git a/pkgs/webview-ui/flake-module.nix b/pkgs/webview-ui/flake-module.nix index 8dc88800..64ed3add 100644 --- a/pkgs/webview-ui/flake-module.nix +++ b/pkgs/webview-ui/flake-module.nix @@ -9,9 +9,28 @@ }: let node_modules-dev = config.packages.webview-ui.prepared-dev; + + src_with_api = pkgs.stdenv.mkDerivation { + name = "with-api"; + src = ./app; + buildInputs = [ pkgs.nodejs ]; + installPhase = '' + mkdir -p $out + + mkdir -p $out/api + cat ${config.packages.clan-ts-api} > $out/api/index.ts + + cp -r $src/* $out + + ls -la $out/api + ''; + }; in { packages.webview-ui = inputs.dream2nix.lib.evalModules { + specialArgs = { + src = src_with_api ; + }; packageSets.nixpkgs = inputs.dream2nix.inputs.nixpkgs.legacyPackages.${system}; modules = [ ./default.nix ]; }; @@ -28,6 +47,9 @@ echo -n $ID > .dream2nix/.node_modules_id echo "Ok: node_modules updated" fi + + mkdir -p ./app/api + cat ${config.packages.clan-ts-api} > ./app/api/index.ts ''; }; };