add webview threaded api

.gitignore

@@ -13,6 +13,9 @@ nixos.qcow2
 **/*.glade~
 /docs/out
 
+# dream2nix
+.dream2nix
+
 # python
 __pycache__
 .coverage
@@ -28,3 +31,8 @@ build
 build-dir
 repo
 .env
+
+# node
+node_modules
+dist
+.webui

flake.lock

@@ -20,6 +20,28 @@
         "type": "github"
       }
     },
+    "dream2nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "purescript-overlay": "purescript-overlay",
+        "pyproject-nix": "pyproject-nix"
+      },
+      "locked": {
+        "lastModified": 1715711628,
+        "narHash": "sha256-MwkdhFpFBABp6IZWy/A2IwDe5Y1z0qZXInTO6AtvGZY=",
+        "owner": "nix-community",
+        "repo": "dream2nix",
+        "rev": "995e831dac8c2c843f1289d15dfec526cb84afdd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "dream2nix",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
@@ -129,9 +151,49 @@
         "type": "github"
       }
     },
+    "purescript-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "dream2nix",
+          "nixpkgs"
+        ],
+        "slimlock": "slimlock"
+      },
+      "locked": {
+        "lastModified": 1696022621,
+        "narHash": "sha256-eMjFmsj2G1E0Q5XiibUNgFjTiSz0GxIeSSzzVdoN730=",
+        "owner": "thomashoneyman",
+        "repo": "purescript-overlay",
+        "rev": "047c7933abd6da8aa239904422e22d190ce55ead",
+        "type": "github"
+      },
+      "original": {
+        "owner": "thomashoneyman",
+        "repo": "purescript-overlay",
+        "type": "github"
+      }
+    },
+    "pyproject-nix": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1702448246,
+        "narHash": "sha256-hFg5s/hoJFv7tDpiGvEvXP0UfFvFEDgTdyHIjDVHu1I=",
+        "owner": "davhau",
+        "repo": "pyproject.nix",
+        "rev": "5a06a2697b228c04dd2f35659b4b659ca74f7aeb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "davhau",
+        "ref": "dream2nix",
+        "repo": "pyproject.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "disko": "disko",
+        "dream2nix": "dream2nix",
         "flake-parts": "flake-parts",
         "nixos-generators": "nixos-generators",
         "nixos-images": "nixos-images",
@@ -140,6 +202,28 @@
         "treefmt-nix": "treefmt-nix"
       }
     },
+    "slimlock": {
+      "inputs": {
+        "nixpkgs": [
+          "dream2nix",
+          "purescript-overlay",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1688610262,
+        "narHash": "sha256-Wg0ViDotFWGWqKIQzyYCgayeH8s4U1OZcTiWTQYdAp4=",
+        "owner": "thomashoneyman",
+        "repo": "slimlock",
+        "rev": "b5c6cdcaf636ebbebd0a1f32520929394493f1a6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "thomashoneyman",
+        "repo": "slimlock",
+        "type": "github"
+      }
+    },
     "sops-nix": {
       "inputs": {
         "nixpkgs": [

flake.nix

@@ -21,6 +21,8 @@
     flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
     treefmt-nix.url = "github:numtide/treefmt-nix";
     treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
+    dream2nix.url = "github:nix-community/dream2nix";
+    dream2nix.inputs.nixpkgs.follows = "nixpkgs";
   };
 
   outputs =

pkgs/clan-vm-manager/clan_vm_manager/views/webview.py

@@ -1,21 +1,61 @@
+from dataclasses import dataclass
+import dataclasses 
+
+import json
 import sys
 from pathlib import Path
-from typing import Any, List
-from clan_cli import machines
-import time
+import threading
+from typing import Any, Callable, Union, get_type_hints
+
 import gi
-import json
+from clan_cli import machines
 
 gi.require_version("WebKit", "6.0")
 
-from gi.repository import WebKit
+from gi.repository import WebKit, GLib
 
-site_index: Path = (Path(sys.argv[0]).absolute() / Path("../..") / Path("web/app/dist/index.html") ).resolve()
+site_index: Path = (
+    Path(sys.argv[0]).absolute() / Path("../..") / Path("clan_vm_manager/.webui/index.html")
+).resolve()
 
+def type_to_dict(t: Any) -> dict:
+    if dataclasses.is_dataclass(t):
+        fields = dataclasses.fields(t)
+        return {
+            'type': 'dataclass',
+            'name': t.__name__,
+            'fields': {f.name: type_to_dict(f.type) for f in fields}
+        }
+        
+    if hasattr(t, '__origin__'):  # Check if it's a generic type
+        if t.__origin__ is None:
+            # Non-generic user-defined or built-in type
+            return {'type': t.__name__}
+        if t.__origin__ is Union:
+            return {
+                'type': 'union',
+                'of': [type_to_dict(arg) for arg in t.__args__]
+            }
+        elif issubclass(t.__origin__, list):
+            return {'type': 'list', 'item_type': type_to_dict(t.__args__[0])}
+        elif issubclass(t.__origin__, dict):
+            return {'type': 'dict', 'key_type': type_to_dict(t.__args__[0]), 'value_type': type_to_dict(t.__args__[1])}
+        elif issubclass(t.__origin__, tuple):
+            return {'type': 'tuple', 'element_types': [type_to_dict(elem) for elem in t.__args__]}
+        elif issubclass(t.__origin__, set):
+            return {'type': 'set', 'item_type': type_to_dict(t.__args__[0])}
+        else:
+            # Handle other generic types (like Union, Optional)
+            return {'type': str(t.__origin__.__name__), 'parameters': [type_to_dict(arg) for arg in t.__args__]}
+    elif isinstance(t, type):
+        return {'type': t.__name__}
+    else:
+        return {'type': str(t)}
+
+class WebView:
+    method_registry: dict[str,Callable] = {}
 
-class WebView():
     def __init__(self) -> None:
-
         self.webview = WebKit.WebView()
 
         self.manager = self.webview.get_user_content_manager()
@@ -26,26 +66,52 @@ class WebView():
 
         self.webview.load_uri(f"file://{site_index}")
 
-    def on_message_received(
-        self, user_content_manager: WebKit.UserContentManager, message: Any
-    ) -> None:
-        # payload = json.loads(message.to_json(0))
-        # TODO: 
-        # Dynamically call functions in the js context
-        # I.e. the result function should have the same name as the target method in the gtk context
-        # Example: 
-        # request -> { method: "list_machines", data: None }
-        # internally call list_machines and serialize the result
-        # result -> window.clan.list_machines(`{serialized}`)
 
-        list_of_machines = machines.list.list_machines(".")
-        serialized = json.dumps(list_of_machines)
-        # Important: use ` backticks to avoid escaping issues with conflicting quotes in js and json
-        self.webview.evaluate_javascript(f"""
-            setTimeout(() => {{ 
-                window.clan.setMachines(`{serialized}`);
-            }},2000);
-        """, -1, None, None, None)
+    def method(self, function: Callable) -> Callable:
+        # type_hints = get_type_hints(function)
+        # serialized_hints = {key: type_to_dict(value) for key, value in type_hints.items()}
+        self.method_registry[function.__name__] = function
+        return function
+
+
+    def on_message_received(self, user_content_manager: WebKit.UserContentManager, message: Any) -> None:
+        payload = json.loads(message.to_json(0))
+        print(f"Received message: {payload}")
+        method_name = payload["method"]
+        handler_fn = self.method_registry[method_name]
+        
+        # Start handler_fn in a new thread
+        thread = threading.Thread(target=self.threaded_handler, args=(handler_fn, payload.get("data"), method_name))
+        thread.start()
+
+    def threaded_handler(self, handler_fn: Callable[[Any],Any], data: Any, method_name: str) -> None:
+        result = handler_fn(data)
+        serialized = json.dumps(result)
+        thread_id = threading.get_ident()
+
+        # Use idle_add to queue the response call to js on the main GTK thread
+        GLib.idle_add(self.call_js, method_name, serialized)
+        
+
+    def call_js(self, method_name: str, serialized: str) -> bool:
+        # This function must be run on the main GTK thread to interact with the webview
+        self.webview.evaluate_javascript(
+            f"""
+            window.clan.{method_name}(`{serialized}`);
+            """,
+            -1,
+            None,
+            None,
+            None,
+        )
+        return False  # Important to return False so that it's not run again
 
     def get_webview(self) -> WebKit.WebView:
         return self.webview
+
+
+webview = WebView()
+
+@webview.method
+def list_machines(data: None) -> list[str]:
+    return machines.list.list_machines(".")

pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py

@@ -11,7 +11,7 @@ from clan_vm_manager.singletons.use_vms import ClanStore
 from clan_vm_manager.views.details import Details
 from clan_vm_manager.views.list import ClanList
 from clan_vm_manager.views.logs import Logs
-from clan_vm_manager.views.webview import WebView
+from clan_vm_manager.views.webview import webview
 
 gi.require_version("Adw", "1")
 
@@ -59,7 +59,7 @@ class MainWindow(Adw.ApplicationWindow):
         stack_view.add_named(ClanList(config), "list")
         stack_view.add_named(Details(), "details")
         stack_view.add_named(Logs(), "logs")
-        stack_view.add_named(WebView().get_webview(), "webview")
+        stack_view.add_named(webview.get_webview(), "webview")
 
         stack_view.set_visible_child_name(config.initial_view)
 

pkgs/clan-vm-manager/flake-module.nix

@@ -4,7 +4,7 @@
     { config, pkgs, ... }:
     {
       devShells.clan-vm-manager = pkgs.callPackage ./shell.nix {
-        inherit (config.packages) clan-vm-manager;
+        inherit (config.packages) clan-vm-manager webview-ui;
       };
       packages.clan-vm-manager = pkgs.python3.pkgs.callPackage ./default.nix {
         inherit (config.packages) clan-cli;

pkgs/clan-vm-manager/shell.nix

@@ -10,7 +10,7 @@
   python3,
   gtk4,
   libadwaita,
-  nodejs_latest
+  nodejs_latest,
 }:
 
 let

pkgs/clan-vm-manager/web/app/.gitignore

@@ -1,2 +0,0 @@
-node_modules
-dist

pkgs/flake-module.nix

@@ -6,6 +6,7 @@
     ./clan-vm-manager/flake-module.nix
     ./installer/flake-module.nix
     ./schemas/flake-module.nix
+    ./webview-ui/flake-module.nix
   ];
 
   perSystem =

pkgs/webview-ui/.envrc

@@ -0,0 +1,6 @@
+source_up
+
+watch_file flake-module.nix default.nix
+
+# Because we depend on nixpkgs sources, uploading to builders takes a long time
+use flake .#webview-ui --builders ''

pkgs/webview-ui/app/gtk.webview.js

@@ -0,0 +1,57 @@
+const fs = require("fs");
+const path = require("path");
+
+const distPath = path.resolve(__dirname, "dist");
+const manifestPath = path.join(distPath, ".vite/manifest.json");
+const outputPath = path.join(distPath, "index.html");
+
+fs.readFile(manifestPath, { encoding: "utf8" }, (err, data) => {
+  if (err) {
+    return console.error("Failed to read manifest:", err);
+  }
+
+  const manifest = JSON.parse(data);
+  /** @type {{    file: string;    name: string;    src: string;    isEntry: bool;    css: string[];  } []} */
+  const assets = Object.values(manifest);
+
+  console.log(`Generate custom index.html from ${manifestPath} ...`);
+  // Start with a basic HTML structure
+  let htmlContent = `<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Webview UI</title>`;
+
+  // Add linked stylesheets
+  assets.forEach((asset) => {
+    asset.css.forEach((cssEntry) => {
+      htmlContent += `\n    <link rel="stylesheet" href="${cssEntry}">`;
+    });
+  });
+
+  htmlContent += `
+</head>
+<body>
+    <div id="app"></div>
+`;
+  // Add scripts
+  assets.forEach((asset) => {
+    if (asset.file.endsWith(".js")) {
+      htmlContent += `\n    <script src="${asset.file}"></script>`;
+    }
+  });
+
+  htmlContent += `
+</body>
+</html>`;
+
+  // Write the HTML file
+  fs.writeFile(outputPath, htmlContent, (err) => {
+    if (err) {
+      console.error("Failed to write custom index.html:", err);
+    } else {
+      console.log("Custom index.html generated successfully!");
+    }
+  });
+});

pkgs/webview-ui/app/package-lock.json

@@ -1,12 +1,12 @@
 {
-  "name": "vite-template-solid",
-  "version": "0.0.0",
+  "name": "@clan/webview-ui",
+  "version": "0.0.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "vite-template-solid",
-      "version": "0.0.0",
+      "name": "@clan/webview-ui",
+      "version": "0.0.1",
       "license": "MIT",
       "dependencies": {
         "solid-js": "^1.8.11"

pkgs/webview-ui/app/package.json

@@ -1,23 +1,24 @@
 {
-  "name": "vite-template-solid",
-  "version": "0.0.0",
+  "name": "@clan/webview-ui",
+  "version": "0.0.1",
   "description": "",
   "scripts": {
     "start": "vite",
     "dev": "vite",
-    "build": "vite build",
+    "build": "vite build && npm run convert-html",
+    "convert-html": "node gtk.webview.js",
     "serve": "vite preview"
   },
   "license": "MIT",
   "devDependencies": {
-    "solid-devtools": "^0.29.2",
-    "typescript": "^5.3.3",
-    "vite": "^5.0.11",
-    "vite-plugin-solid": "^2.8.2",
     "autoprefixer": "^10.4.19",
     "daisyui": "^4.11.1",
     "postcss": "^8.4.38",
-    "tailwindcss": "^3.4.3"
+    "solid-devtools": "^0.29.2",
+    "tailwindcss": "^3.4.3",
+    "typescript": "^5.3.3",
+    "vite-plugin-solid": "^2.8.2",
+    "vite": "^5.0.11"
   },
   "dependencies": {
     "solid-js": "^1.8.11"

pkgs/webview-ui/app/src/App.tsx

@@ -10,6 +10,7 @@ const App: Component = () => {
     <CountProvider>
       <div class="w-full flex items-center flex-col gap-2 my-2">
         <div>Clan</div>
+        <p>Current route: {route()}</p>
 
         <div class="flex items-center">
           <button
@@ -18,18 +19,15 @@ const App: Component = () => {
           >
             Navigate to {route() === "home" ? "graph" : "home"}
           </button>
-          <Switch fallback={<p>{route()} not found</p>}>
-            <Match when={route() == "home"}>
-              <p>Current route: {route()}</p>
-            </Match>
-            <Match when={route() == "graph"}>
-              <p>Current route: {route()}</p>
-            </Match>
-          </Switch>
-        </div>
-        <div class="flex items-center">
-          <Nested />
         </div>
+        <Switch fallback={<p>{route()} not found</p>}>
+          <Match when={route() == "home"}>
+            <Nested />
+          </Match>
+          <Match when={route() == "graph"}>
+            <p></p>
+          </Match>
+        </Switch>
       </div>
     </CountProvider>
   );

pkgs/webview-ui/app/src/Config.tsx

@@ -1,35 +1,22 @@
 import { createSignal, createContext, useContext, JSXElement } from "solid-js";
+import { PYAPI } from "./message";
 
-const initialValue = 0 as const;
 export const makeCountContext = () => {
-  const [count, setCount] = createSignal(0);
   const [machines, setMachines] = createSignal<string[]>([]);
   const [loading, setLoading] = createSignal(false);
 
-  // Add this callback to global window so we can test it from gtk
-  // @ts-ignore
-  window.clan.setMachines = (data: str) => {
-    try {
-      setMachines(JSON.parse(data));
-    } catch (e) {
-      alert(`Error parsing JSON: ${e}`);
-    } finally {
-      setLoading(false);
-    }
-  };
+  PYAPI.list_machines.receive((machines) => {
+    setLoading(false);
+    setMachines(machines);
+  });
 
   return [
-    { count, loading, machines },
+    { loading, machines },
     {
-      setCount,
-      setLoading,
-      setMachines,
       getMachines: () => {
         // When the gtk function sends its data the loading state will be set to false
         setLoading(true);
-        // Example of how to dispatch a gtk function
-        // @ts-ignore
-        window.webkit.messageHandlers.gtk.postMessage(1);
+        PYAPI.list_machines.dispatch(null);
       },
     },
   ] as const;
@@ -38,11 +25,8 @@ export const makeCountContext = () => {
 type CountContextType = ReturnType<typeof makeCountContext>;
 
 export const CountContext = createContext<CountContextType>([
-  { count: () => initialValue, loading: () => false, machines: () => [] },
+  { loading: () => false, machines: () => [] },
   {
-    setCount: () => {},
-    setLoading: () => {},
-    setMachines: () => {},
     getMachines: () => {},
   },
 ]);

pkgs/webview-ui/app/src/message.ts

@@ -0,0 +1,22 @@
+const deserialize = (fn: Function) => (str: string) => {
+  try {
+    fn(JSON.parse(str));
+  } catch (e) {
+    alert(`Error parsing JSON: ${e}`);
+  }
+};
+
+export const PYAPI = {
+  list_machines: {
+    dispatch: (data: null) =>
+      // @ts-ignore
+      window.webkit.messageHandlers.gtk.postMessage({
+        method: "list_machines",
+        data,
+      }),
+    receive: (fn: (response: string[]) => void) => {
+      // @ts-ignore
+      window.clan.list_machines = deserialize(fn);
+    },
+  },
+};

pkgs/webview-ui/default.nix

@@ -0,0 +1,21 @@
+{ dream2nix, config, ... }:
+{
+  imports = [ dream2nix.modules.dream2nix.WIP-nodejs-builder-v3 ];
+
+  mkDerivation = {
+    src = ./app;
+  };
+
+  deps =
+    { nixpkgs, ... }:
+    {
+      inherit (nixpkgs) stdenv;
+    };
+
+  WIP-nodejs-builder-v3 = {
+    packageLockFile = "${config.mkDerivation.src}/package-lock.json";
+  };
+
+  name = "@clan/webview-ui";
+  version = "0.0.1";
+}

pkgs/webview-ui/flake-module.nix

@@ -0,0 +1,31 @@
+{ inputs, ... }:
+{
+  perSystem =
+    { system, pkgs, config, ... }:
+    let 
+      node_modules-dev = config.packages.webview-ui.prepared-dev;
+    in
+    {
+      packages.webview-ui = inputs.dream2nix.lib.evalModules {
+        packageSets.nixpkgs = inputs.dream2nix.inputs.nixpkgs.legacyPackages.${system};
+        modules = [
+          ./default.nix
+        ];
+      };
+      devShells.webview-ui = pkgs.mkShell {
+        inputsFrom = [ config.packages.webview-ui.out ];
+        shellHook = ''
+          ID=${node_modules-dev}
+          currID=$(cat .dream2nix/.node_modules_id 2> /dev/null)
+
+          mkdir -p .dream2nix
+          if [[ "$ID" != "$currID" || ! -d "app/node_modules"  ]];
+          then
+            ${pkgs.rsync}/bin/rsync -a --chmod=ug+w  --delete ${node_modules-dev}/node_modules/ ./app/node_modules/
+            echo -n $ID > .dream2nix/.node_modules_id
+            echo "Ok: node_modules updated"
+          fi
+        '';
+      };
+    };
+}