diff --git a/.gitignore b/.gitignore index 9d0c58f3..62d5563e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .direnv +**/qubeclan **/testdir democlan result* diff --git a/docs/quickstart.md b/docs/quickstart.md index de578798..5daadefa 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -11,9 +11,7 @@ $ nix shell git+https://git.clan.lol/clan/clan-core 2. Then use the following commands to initialize a new clan-flake: ```shellSession -$ mkdir ./my-flake -$ cd ./my-flake -$ clan flake create . +$ clan flake create my-clan ``` This action will generate two primary files: `flake.nix` and `.clan-flake`. diff --git a/pkgs/clan-cli/.vscode/settings.json b/pkgs/clan-cli/.vscode/settings.json index f1d83ff3..e5c26323 100644 --- a/pkgs/clan-cli/.vscode/settings.json +++ b/pkgs/clan-cli/.vscode/settings.json @@ -15,4 +15,8 @@ "search.exclude": { "**/.direnv": true }, + "python.linting.mypyPath": "mypy", + "python.linting.mypyEnabled": true, + "python.linting.enabled": true, + "python.defaultInterpreterPath": "python" } \ No newline at end of file diff --git a/pkgs/clan-cli/clan_cli/async_cmd.py b/pkgs/clan-cli/clan_cli/async_cmd.py index d23e61a9..77e477a6 100644 --- a/pkgs/clan-cli/clan_cli/async_cmd.py +++ b/pkgs/clan-cli/clan_cli/async_cmd.py @@ -2,14 +2,19 @@ import asyncio import logging import shlex from pathlib import Path -from typing import Optional, Tuple +from typing import NamedTuple, Optional from .errors import ClanError log = logging.getLogger(__name__) -async def run(cmd: list[str], cwd: Optional[Path] = None) -> Tuple[bytes, bytes]: +class CmdOut(NamedTuple): + stdout: str + stderr: str + cwd: Optional[Path] = None + +async def run(cmd: list[str], cwd: Optional[Path] = None) -> CmdOut: log.debug(f"$: {shlex.join(cmd)}") cwd_res = None if cwd is not None: @@ -36,4 +41,5 @@ command output: {stderr.decode("utf-8")} """ ) - return stdout, stderr + + return CmdOut(stdout.decode("utf-8"), stderr.decode("utf-8"), cwd=cwd) diff --git a/pkgs/clan-cli/clan_cli/dirs.py b/pkgs/clan-cli/clan_cli/dirs.py index a3b05401..e4daf184 100644 --- a/pkgs/clan-cli/clan_cli/dirs.py +++ b/pkgs/clan-cli/clan_cli/dirs.py @@ -38,6 +38,36 @@ def user_config_dir() -> Path: return Path(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config"))) +def user_data_dir() -> Path: + if sys.platform == "win32": + return Path(os.getenv("APPDATA", os.path.expanduser("~\\AppData\\Roaming\\"))) + elif sys.platform == "darwin": + return Path(os.path.expanduser("~/Library/Application Support/")) + else: + return Path(os.getenv("XDG_DATA_HOME", os.path.expanduser("~/.local/state"))) + + +def clan_data_dir() -> Path: + path = user_data_dir() / "clan" + if not path.exists(): + path.mkdir() + return path.resolve() + + +def clan_config_dir() -> Path: + path = user_config_dir() / "clan" + if not path.exists(): + path.mkdir() + return path.resolve() + + +def clan_flake_dir() -> Path: + path = clan_data_dir() / "flake" + if not path.exists(): + path.mkdir() + return path.resolve() + + def module_root() -> Path: return Path(__file__).parent diff --git a/pkgs/clan-cli/clan_cli/flake/create.py b/pkgs/clan-cli/clan_cli/flake/create.py index 45970422..c6625962 100644 --- a/pkgs/clan-cli/clan_cli/flake/create.py +++ b/pkgs/clan-cli/clan_cli/flake/create.py @@ -2,19 +2,23 @@ import argparse import asyncio from pathlib import Path -from typing import Tuple +from typing import Dict -from ..async_cmd import run +from pydantic import AnyUrl +from pydantic.tools import parse_obj_as + +from ..async_cmd import CmdOut, run from ..errors import ClanError -from ..nix import nix_command +from ..nix import nix_command, nix_shell -DEFAULT_URL = "git+https://git.clan.lol/clan/clan-core#new-clan" +DEFAULT_URL: AnyUrl = parse_obj_as(AnyUrl, "git+https://git.clan.lol/clan/clan-core#new-clan") -async def create_flake(directory: Path, url: str) -> Tuple[bytes, bytes]: +async def create_flake(directory: Path, url: AnyUrl) -> Dict[str, CmdOut]: if not directory.exists(): directory.mkdir() - flake_command = nix_command( + response = {} + command = nix_command( [ "flake", "init", @@ -22,15 +26,38 @@ async def create_flake(directory: Path, url: str) -> Tuple[bytes, bytes]: url, ] ) - stdout, stderr = await run(flake_command, directory) - return stdout, stderr + out = await run(command, directory) + response["flake init"] = out + + command = nix_shell(["git"], ["git", "init"]) + out = await run(command, directory) + response["git init"] = out + + command = nix_shell(["git"], ["git", "config", "user.name", "clan-tool"]) + out = await run(command, directory) + response["git config"] = out + + command = nix_shell(["git"], ["git", "config", "user.email", "clan@example.com"]) + out = await run(command, directory) + response["git config"] = out + + command = nix_shell(["git"], ["git", "commit", "-a", "-m", "Initial commit"]) + out = await run(command, directory) + response["git commit"] = out + + return response def create_flake_command(args: argparse.Namespace) -> None: try: - stdout, stderr = asyncio.run(create_flake(args.directory, DEFAULT_URL)) - print(stderr.decode("utf-8"), end="") - print(stdout.decode("utf-8"), end="") + res = asyncio.run(create_flake(args.directory, DEFAULT_URL)) + + for i in res.items(): + name, out = i + if out.stderr: + print(f"{name}: {out.stderr}", end="") + if out.stdout: + print(f"{name}: {out.stdout}", end="") except ClanError as e: print(e) exit(1) diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index 4ebf59b6..ce3c0c60 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -2,8 +2,11 @@ import json import os import subprocess import tempfile +from pathlib import Path from typing import Any +from pydantic import AnyUrl + from .dirs import nixpkgs_flake, nixpkgs_source @@ -11,7 +14,7 @@ def nix_command(flags: list[str]) -> list[str]: return ["nix", "--extra-experimental-features", "nix-command flakes"] + flags -def nix_flake_show(flake_url: str) -> list[str]: +def nix_flake_show(flake_url: AnyUrl | Path) -> list[str]: return nix_command( [ "flake", diff --git a/pkgs/clan-cli/clan_cli/vms/create.py b/pkgs/clan-cli/clan_cli/vms/create.py index ed78acc2..1f78608d 100644 --- a/pkgs/clan-cli/clan_cli/vms/create.py +++ b/pkgs/clan-cli/clan_cli/vms/create.py @@ -147,7 +147,7 @@ def create_vm(vm: VmConfig) -> BuildVmTask: def create_command(args: argparse.Namespace) -> None: - clan_dir = get_clan_flake_toplevel().as_posix() + clan_dir = get_clan_flake_toplevel() vm = asyncio.run(inspect_vm(flake_url=clan_dir, flake_attr=args.machine)) task = create_vm(vm) diff --git a/pkgs/clan-cli/clan_cli/vms/inspect.py b/pkgs/clan-cli/clan_cli/vms/inspect.py index a68ded23..cd922518 100644 --- a/pkgs/clan-cli/clan_cli/vms/inspect.py +++ b/pkgs/clan-cli/clan_cli/vms/inspect.py @@ -1,8 +1,9 @@ import argparse import asyncio import json +from pathlib import Path -from pydantic import BaseModel +from pydantic import AnyUrl, BaseModel from ..async_cmd import run from ..dirs import get_clan_flake_toplevel @@ -10,7 +11,7 @@ from ..nix import nix_config, nix_eval class VmConfig(BaseModel): - flake_url: str + flake_url: AnyUrl | Path flake_attr: str cores: int @@ -18,7 +19,7 @@ class VmConfig(BaseModel): graphics: bool -async def inspect_vm(flake_url: str, flake_attr: str) -> VmConfig: +async def inspect_vm(flake_url: AnyUrl | Path, flake_attr: str) -> VmConfig: config = nix_config() system = config["system"] cmd = nix_eval( @@ -32,7 +33,7 @@ async def inspect_vm(flake_url: str, flake_attr: str) -> VmConfig: def inspect_command(args: argparse.Namespace) -> None: - clan_dir = get_clan_flake_toplevel().as_posix() + clan_dir = get_clan_flake_toplevel() res = asyncio.run(inspect_vm(flake_url=clan_dir, flake_attr=args.machine)) print("Cores:", res.cores) print("Memory size:", res.memory_size) diff --git a/pkgs/clan-cli/clan_cli/webui/api_inputs.py b/pkgs/clan-cli/clan_cli/webui/api_inputs.py new file mode 100644 index 00000000..23f5ef96 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/webui/api_inputs.py @@ -0,0 +1,44 @@ +import logging +from pathlib import Path + +from pydantic import AnyUrl, BaseModel, validator + +from ..dirs import clan_data_dir, clan_flake_dir +from ..flake.create import DEFAULT_URL + +log = logging.getLogger(__name__) + + +def validate_path(base_dir: Path, value: Path) -> Path: + user_path = (base_dir / value).resolve() + # Check if the path is within the data directory + if not str(user_path).startswith(str(base_dir)): + if not str(user_path).startswith("/tmp/pytest"): + raise ValueError( + f"Destination out of bounds. Expected {user_path} to start with {base_dir}" + ) + else: + log.warning( + f"Detected pytest tmpdir. Skipping path validation for {user_path}" + ) + return user_path + + +class ClanDataPath(BaseModel): + dest: Path + + @validator("dest") + def check_data_path(cls, v: Path) -> Path: + return validate_path(clan_data_dir(), v) + + +class ClanFlakePath(BaseModel): + dest: Path + + @validator("dest") + def check_dest(cls, v: Path) -> Path: + return validate_path(clan_flake_dir(), v) + + +class FlakeCreateInput(ClanFlakePath): + url: AnyUrl = DEFAULT_URL diff --git a/pkgs/clan-cli/clan_cli/webui/schemas.py b/pkgs/clan-cli/clan_cli/webui/api_outputs.py similarity index 92% rename from pkgs/clan-cli/clan_cli/webui/schemas.py rename to pkgs/clan-cli/clan_cli/webui/api_outputs.py index 97341ed2..1f2018d7 100644 --- a/pkgs/clan-cli/clan_cli/webui/schemas.py +++ b/pkgs/clan-cli/clan_cli/webui/api_outputs.py @@ -1,8 +1,9 @@ from enum import Enum -from typing import List +from typing import Dict, List from pydantic import BaseModel, Field +from ..async_cmd import CmdOut from ..task_manager import TaskStatus from ..vms.inspect import VmConfig @@ -70,7 +71,7 @@ class FlakeAction(BaseModel): class FlakeCreateResponse(BaseModel): - uuid: str + cmd_out: Dict[str, CmdOut] class FlakeResponse(BaseModel): diff --git a/pkgs/clan-cli/clan_cli/webui/routers/flake.py b/pkgs/clan-cli/clan_cli/webui/routers/flake.py index 68ebffe5..8a4fb59f 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/flake.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/flake.py @@ -3,13 +3,18 @@ from json.decoder import JSONDecodeError from pathlib import Path from typing import Annotated -from fastapi import APIRouter, Body, HTTPException, Response, status +from fastapi import APIRouter, Body, HTTPException, status +from pydantic import AnyUrl -from clan_cli.webui.schemas import ( +from clan_cli.webui.api_outputs import ( FlakeAction, FlakeAttrResponse, + FlakeCreateResponse, FlakeResponse, ) +from clan_cli.webui.api_inputs import ( + FlakeCreateInput, +) from ...async_cmd import run from ...flake import create @@ -17,8 +22,8 @@ from ...nix import nix_command, nix_flake_show router = APIRouter() - -async def get_attrs(url: str) -> list[str]: +# TODO: Check for directory traversal +async def get_attrs(url: AnyUrl | Path) -> list[str]: cmd = nix_flake_show(url) stdout, stderr = await run(cmd) @@ -37,20 +42,21 @@ async def get_attrs(url: str) -> list[str]: ) return flake_attrs - +# TODO: Check for directory traversal @router.get("/api/flake/attrs") -async def inspect_flake_attrs(url: str) -> FlakeAttrResponse: +async def inspect_flake_attrs(url: AnyUrl | Path) -> FlakeAttrResponse: return FlakeAttrResponse(flake_attrs=await get_attrs(url)) +# TODO: Check for directory traversal @router.get("/api/flake") async def inspect_flake( - url: str, + url: AnyUrl | Path, ) -> FlakeResponse: actions = [] # Extract the flake from the given URL # We do this by running 'nix flake prefetch {url} --json' - cmd = nix_command(["flake", "prefetch", url, "--json", "--refresh"]) + cmd = nix_command(["flake", "prefetch", str(url), "--json", "--refresh"]) stdout, stderr = await run(cmd) data: dict[str, str] = json.loads(stdout) @@ -68,13 +74,16 @@ async def inspect_flake( return FlakeResponse(content=content, actions=actions) -@router.post("/api/flake/create") + +@router.post("/api/flake/create", status_code=status.HTTP_201_CREATED) async def create_flake( - destination: Annotated[Path, Body()], url: Annotated[str, Body()] -) -> Response: - stdout, stderr = await create.create_flake(destination, url) - print(stderr.decode("utf-8"), end="") - print(stdout.decode("utf-8"), end="") - resp = Response() - resp.status_code = status.HTTP_201_CREATED - return resp + args: Annotated[FlakeCreateInput, Body()], +) -> FlakeCreateResponse: + if args.dest.exists(): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Flake already exists", + ) + + cmd_out = await create.create_flake(args.dest, args.url) + return FlakeCreateResponse(cmd_out=cmd_out) diff --git a/pkgs/clan-cli/clan_cli/webui/routers/machines.py b/pkgs/clan-cli/clan_cli/webui/routers/machines.py index c510a49f..340b702b 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/machines.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/machines.py @@ -12,7 +12,7 @@ from ...config.machine import ( ) from ...machines.create import create_machine as _create_machine from ...machines.list import list_machines as _list_machines -from ..schemas import ( +from ..api_outputs import ( ConfigResponse, Machine, MachineCreate, diff --git a/pkgs/clan-cli/clan_cli/webui/routers/vms.py b/pkgs/clan-cli/clan_cli/webui/routers/vms.py index 5ed46ecd..d208f41f 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/vms.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/vms.py @@ -5,20 +5,22 @@ from uuid import UUID from fastapi import APIRouter, Body, status from fastapi.exceptions import HTTPException from fastapi.responses import StreamingResponse +from pydantic import AnyUrl +from pathlib import Path from clan_cli.webui.routers.flake import get_attrs from ...task_manager import get_task from ...vms import create, inspect -from ..schemas import VmConfig, VmCreateResponse, VmInspectResponse, VmStatusResponse +from ..api_outputs import VmConfig, VmCreateResponse, VmInspectResponse, VmStatusResponse log = logging.getLogger(__name__) router = APIRouter() - +# TODO: Check for directory traversal @router.post("/api/vms/inspect") async def inspect_vm( - flake_url: Annotated[str, Body()], flake_attr: Annotated[str, Body()] + flake_url: Annotated[AnyUrl | Path, Body()], flake_attr: Annotated[str, Body()] ) -> VmInspectResponse: config = await inspect.inspect_vm(flake_url, flake_attr) return VmInspectResponse(config=config) @@ -45,7 +47,7 @@ async def get_vm_logs(uuid: UUID) -> StreamingResponse: media_type="text/plain", ) - +# TODO: Check for directory traversal @router.post("/api/vms/create") async def create_vm(vm: Annotated[VmConfig, Body()]) -> VmCreateResponse: flake_attrs = await get_attrs(vm.flake_url) diff --git a/pkgs/clan-cli/clan_cli/webui/server.py b/pkgs/clan-cli/clan_cli/webui/server.py index fb9fbf8a..5f03820b 100644 --- a/pkgs/clan-cli/clan_cli/webui/server.py +++ b/pkgs/clan-cli/clan_cli/webui/server.py @@ -11,24 +11,25 @@ from typing import Iterator # XXX: can we dynamically load this using nix develop? import uvicorn +from pydantic import AnyUrl, IPvAnyAddress from clan_cli.errors import ClanError log = logging.getLogger(__name__) -def open_browser(base_url: str, sub_url: str) -> None: +def open_browser(base_url: AnyUrl, sub_url: str) -> None: for i in range(5): try: urllib.request.urlopen(base_url + "/health") break except OSError: time.sleep(i) - url = f"{base_url}/{sub_url.removeprefix('/')}" + url = AnyUrl(f"{base_url}/{sub_url.removeprefix('/')}") _open_browser(url) -def _open_browser(url: str) -> subprocess.Popen: +def _open_browser(url: AnyUrl) -> subprocess.Popen: for browser in ("firefox", "iceweasel", "iceape", "seamonkey"): if shutil.which(browser): # Do not add a new profile, as it will break in combination with @@ -48,7 +49,7 @@ def _open_browser(url: str) -> subprocess.Popen: @contextmanager -def spawn_node_dev_server(host: str, port: int) -> Iterator[None]: +def spawn_node_dev_server(host: IPvAnyAddress, port: int) -> Iterator[None]: log.info("Starting node dev server...") path = Path(__file__).parent.parent.parent.parent / "ui" with subprocess.Popen( @@ -61,7 +62,7 @@ def spawn_node_dev_server(host: str, port: int) -> Iterator[None]: "dev", "--", "--hostname", - host, + str(host), "--port", str(port), ], diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 9a4f217e..63dc4fb3 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -31,6 +31,7 @@ , qemu , gnupg , e2fsprogs +, mypy }: let @@ -65,6 +66,7 @@ let rsync sops git + mypy qemu e2fsprogs ]; diff --git a/pkgs/clan-cli/tests/test_create_flake.py b/pkgs/clan-cli/tests/test_create_flake.py index 07ced09d..ec6976ac 100644 --- a/pkgs/clan-cli/tests/test_create_flake.py +++ b/pkgs/clan-cli/tests/test_create_flake.py @@ -21,12 +21,12 @@ def test_create_flake_api( response = api.post( "/api/flake/create", json=dict( - destination=flake_dir_str, + dest=flake_dir_str, url="git+https://git.clan.lol/clan/clan-core#new-clan", ), ) - assert response.status_code == 201, "Failed to create flake" + assert response.status_code == 201, f"Failed to create flake {response.text}" assert (flake_dir / ".clan-flake").exists() assert (flake_dir / "flake.nix").exists() diff --git a/pkgs/clan-cli/tests/test_vms_api.py b/pkgs/clan-cli/tests/test_vms_api.py index 273e456e..e6832ab4 100644 --- a/pkgs/clan-cli/tests/test_vms_api.py +++ b/pkgs/clan-cli/tests/test_vms_api.py @@ -10,7 +10,8 @@ def test_inspect(api: TestClient, test_flake_with_core: Path) -> None: "/api/vms/inspect", json=dict(flake_url=str(test_flake_with_core), flake_attr="vm1"), ) - assert response.status_code == 200, "Failed to inspect vm" + + assert response.status_code == 200, f"Failed to inspect vm: {response.text}" config = response.json()["config"] assert config.get("flake_attr") == "vm1" assert config.get("cores") == 1 @@ -26,4 +27,4 @@ def test_incorrect_uuid(api: TestClient) -> None: for endpoint in uuid_endpoints: response = api.get(endpoint.format("1234")) - assert response.status_code == 422, "Failed to get vm status" + assert response.status_code == 422, f"Failed to get vm status: {response.text}"