Merge pull request 'contributing.md: Fixed missing direnv install step' (#457) from Qubasa-main into main
All checks were successful
checks / test (push) Successful in 45s
assets1 / test (push) Successful in 46s
checks-impure / test (push) Successful in 1m21s

This commit is contained in:
clan-bot 2023-11-03 21:44:44 +00:00
commit d4f73cb32f
24 changed files with 167 additions and 57 deletions

View File

@ -10,6 +10,11 @@ Welcome to our website template repository! This template is designed to help yo
**Dependency Management**: We use the [Nix package manager](https://nixos.org/) to manage dependencies and ensure reproducibility, making your development process more robust.
## Supported Operating Systems
- Linux
- macOS
# Getting Started with the Development Environment
Let's get your development environment up and running:
@ -28,11 +33,20 @@ Let's get your development environment up and running:
curl -sfL https://direnv.net/install.sh | bash
```
3. **Clone the Repository and Navigate**:
3. **Add direnv to your shell**:
- Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
You can do this by executing following command:
```bash
echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
```
4. **Clone the Repository and Navigate**:
- Clone this repository and navigate to it.
4. **Allow .envrc**:
5. **Allow .envrc**:
- When you enter the directory, you'll receive an error message like this:
```bash
@ -40,7 +54,7 @@ Let's get your development environment up and running:
```
- Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
5. **Build the Backend**:
6. **Build the Backend**:
- Go to the `pkgs/clan-cli` directory and execute:
```bash
@ -48,7 +62,7 @@ Let's get your development environment up and running:
```
- Wait for the backend to build.
6. **Start the Backend Server**:
7. **Start the Backend Server**:
- To start the backend server, execute:
```bash
@ -56,7 +70,7 @@ Let's get your development environment up and running:
```
- The server will automatically restart if any Python files change.
7. **Build the Frontend**:
8. **Build the Frontend**:
- In a different shell, navigate to the `pkgs/ui` directory and execute:
```bash
@ -64,7 +78,7 @@ Let's get your development environment up and running:
```
- Wait for the frontend to build.
8. **Start the Frontend**:
9. **Start the Frontend**:
- To start the frontend, execute:
```bash
npm run dev
@ -194,4 +208,4 @@ To make the most of this template:
- Set the option to "Delete pull request branch after merge by default."
- Also, set the default merge style to "Rebase then create merge commit."
With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!
With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!.

View File

@ -2,7 +2,7 @@
import argparse
from .create import register_create_parser
from .list import register_list_parser
from .list_flakes import register_list_parser
# takes a (sub)parser and configures it

View File

@ -1,11 +1,15 @@
import logging
from pathlib import Path
from typing import NewType
from typing import NewType, Union
from pydantic import AnyUrl
log = logging.getLogger(__name__)
FlakeName = NewType("FlakeName", str)
FlakeUrl = Union[AnyUrl, Path]
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()

View File

@ -11,23 +11,16 @@ from typing import Iterator
from uuid import UUID
from ..dirs import clan_flakes_dir, specific_flake_dir
from ..errors import ClanError
from ..nix import nix_build, nix_config, nix_eval, nix_shell
from ..task_manager import BaseTask, Command, create_task
from ..types import validate_path
from .inspect import VmConfig, inspect_vm
def is_path_or_url(s: str) -> str | None:
# check if s is a valid path
if os.path.exists(s):
return "path"
# check if s is a valid URL
elif re.match(r"^https?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s):
return "URL"
# otherwise, return None
else:
return None
def is_flake_url(s: str) -> bool:
if re.match(r"^http.?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s) is not None:
return True
return False
class BuildVmTask(BaseTask):
@ -95,12 +88,8 @@ class BuildVmTask(BaseTask):
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
res = is_path_or_url(str(self.vm.flake_url))
if res is None:
raise ClanError(
f"flake_url must be a valid path or URL, got {self.vm.flake_url}"
)
elif res == "path": # Only generate secrets for local clans
# Only generate secrets for local clans
if not is_flake_url(str(self.vm.flake_url)):
cmd = next(cmds)
if Path(self.vm.flake_url).is_dir():
cmd.run(
@ -203,8 +192,10 @@ def create_vm(vm: VmConfig, nix_options: list[str] = []) -> BuildVmTask:
def create_command(args: argparse.Namespace) -> None:
clan_dir = specific_flake_dir(args.flake)
vm = asyncio.run(inspect_vm(flake_url=clan_dir, flake_attr=args.machine))
flake_url = args.flake
if not is_flake_url(args.flake):
flake_url = specific_flake_dir(args.flake)
vm = asyncio.run(inspect_vm(flake_url=flake_url, flake_attr=args.machine))
task = create_vm(vm, args.option)
for line in task.log_lines():

View File

@ -70,6 +70,10 @@ class FlakeAction(BaseModel):
uri: str
class FlakeListResponse(BaseModel):
flakes: list[str]
class FlakeCreateResponse(BaseModel):
cmd_out: Dict[str, CmdOut]

View File

@ -9,6 +9,7 @@ from ..errors import ClanError
from .assets import asset_path
from .error_handlers import clan_error_handler
from .routers import clan_modules, flake, health, machines, root, vms
from .tags import tags_metadata
origins = [
"http://localhost:3000",
@ -39,6 +40,9 @@ def setup_app() -> FastAPI:
app.mount("/static", StaticFiles(directory=asset_path()), name="static")
# Add tag descriptions to the OpenAPI schema
app.openapi_tags = tags_metadata
for route in app.routes:
if isinstance(route, APIRoute):
route.operation_id = route.name # in this case, 'read_items'

View File

@ -9,12 +9,13 @@ from clan_cli.types import FlakeName
from ..api_outputs import (
ClanModulesResponse,
)
from ..tags import Tags
log = logging.getLogger(__name__)
router = APIRouter()
@router.get("/api/{flake_name}/clan_modules")
@router.get("/api/{flake_name}/clan_modules", tags=[Tags.modules])
async def list_clan_modules(flake_name: FlakeName) -> ClanModulesResponse:
module_names, error = get_clan_module_names(flake_name)
if error is not None:

View File

@ -13,12 +13,14 @@ from clan_cli.webui.api_outputs import (
FlakeAction,
FlakeAttrResponse,
FlakeCreateResponse,
FlakeListResponse,
FlakeResponse,
)
from ...async_cmd import run
from ...flakes import create
from ...flakes import create, list_flakes
from ...nix import nix_command, nix_flake_show
from ..tags import Tags
router = APIRouter()
@ -45,13 +47,13 @@ async def get_attrs(url: AnyUrl | Path) -> list[str]:
# TODO: Check for directory traversal
@router.get("/api/flake/attrs")
@router.get("/api/flake/attrs", tags=[Tags.flake])
async def inspect_flake_attrs(url: AnyUrl | Path) -> FlakeAttrResponse:
return FlakeAttrResponse(flake_attrs=await get_attrs(url))
# TODO: Check for directory traversal
@router.get("/api/flake")
@router.get("/api/flake/inspect", tags=[Tags.flake])
async def inspect_flake(
url: AnyUrl | Path,
) -> FlakeResponse:
@ -76,7 +78,15 @@ async def inspect_flake(
return FlakeResponse(content=content, actions=actions)
@router.post("/api/flake/create", status_code=status.HTTP_201_CREATED)
@router.get("/api/flake/list", tags=[Tags.flake])
async def list_all_flakes() -> FlakeListResponse:
flakes = list_flakes.list_flakes()
return FlakeListResponse(flakes=flakes)
@router.post(
"/api/flake/create", tags=[Tags.flake], status_code=status.HTTP_201_CREATED
)
async def create_flake(
args: Annotated[FlakeCreateInput, Body()],
) -> FlakeCreateResponse:

View File

@ -23,12 +23,13 @@ from ..api_outputs import (
Status,
VerifyMachineResponse,
)
from ..tags import Tags
log = logging.getLogger(__name__)
router = APIRouter()
@router.get("/api/{flake_name}/machines")
@router.get("/api/{flake_name}/machines", tags=[Tags.machine])
async def list_machines(flake_name: FlakeName) -> MachinesResponse:
machines = []
for m in _list_machines(flake_name):
@ -37,7 +38,7 @@ async def list_machines(flake_name: FlakeName) -> MachinesResponse:
return MachinesResponse(machines=machines)
@router.post("/api/{flake_name}/machines", status_code=201)
@router.post("/api/{flake_name}/machines", tags=[Tags.machine], status_code=201)
async def create_machine(
flake_name: FlakeName, machine: Annotated[MachineCreate, Body()]
) -> MachineResponse:
@ -45,19 +46,19 @@ async def create_machine(
return MachineResponse(machine=Machine(name=machine.name, status=Status.UNKNOWN))
@router.get("/api/{flake_name}/machines/{name}")
@router.get("/api/{flake_name}/machines/{name}", tags=[Tags.machine])
async def get_machine(flake_name: FlakeName, name: str) -> MachineResponse:
log.error("TODO")
return MachineResponse(machine=Machine(name=name, status=Status.UNKNOWN))
@router.get("/api/{flake_name}/machines/{name}/config")
@router.get("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
async def get_machine_config(flake_name: FlakeName, name: str) -> ConfigResponse:
config = config_for_machine(flake_name, name)
return ConfigResponse(config=config)
@router.put("/api/{flake_name}/machines/{name}/config")
@router.put("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
async def set_machine_config(
flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
) -> ConfigResponse:
@ -65,13 +66,13 @@ async def set_machine_config(
return ConfigResponse(config=config)
@router.get("/api/{flake_name}/machines/{name}/schema")
@router.get("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse:
schema = schema_for_machine(flake_name, name)
return SchemaResponse(schema=schema)
@router.put("/api/{flake_name}/machines/{name}/schema")
@router.put("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
async def set_machine_schema(
flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
) -> SchemaResponse:
@ -79,7 +80,7 @@ async def set_machine_schema(
return SchemaResponse(schema=schema)
@router.get("/api/{flake_name}/machines/{name}/verify")
@router.get("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
async def get_verify_machine_config(
flake_name: FlakeName, name: str
) -> VerifyMachineResponse:
@ -88,7 +89,7 @@ async def get_verify_machine_config(
return VerifyMachineResponse(success=success, error=error)
@router.put("/api/{flake_name}/machines/{name}/verify")
@router.put("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
async def put_verify_machine_config(
flake_name: FlakeName,
name: str,

View File

@ -6,13 +6,14 @@ from pathlib import Path
from fastapi import APIRouter, Response
from ..assets import asset_path
from ..tags import Tags
router = APIRouter()
log = logging.getLogger(__name__)
@router.get("/{path_name:path}")
@router.get("/{path_name:path}", tags=[Tags.root])
async def root(path_name: str) -> Response:
if path_name == "":
path_name = "index.html"

View File

@ -18,13 +18,14 @@ from ..api_outputs import (
VmInspectResponse,
VmStatusResponse,
)
from ..tags import Tags
log = logging.getLogger(__name__)
router = APIRouter()
# TODO: Check for directory traversal
@router.post("/api/vms/inspect")
@router.post("/api/vms/inspect", tags=[Tags.vm])
async def inspect_vm(
flake_url: Annotated[AnyUrl | Path, Body()], flake_attr: Annotated[str, Body()]
) -> VmInspectResponse:
@ -32,7 +33,7 @@ async def inspect_vm(
return VmInspectResponse(config=config)
@router.get("/api/vms/{uuid}/status")
@router.get("/api/vms/{uuid}/status", tags=[Tags.vm])
async def get_vm_status(uuid: UUID) -> VmStatusResponse:
task = get_task(uuid)
log.debug(msg=f"error: {task.error}, task.status: {task.status}")
@ -40,7 +41,7 @@ async def get_vm_status(uuid: UUID) -> VmStatusResponse:
return VmStatusResponse(status=task.status, error=error)
@router.get("/api/vms/{uuid}/logs")
@router.get("/api/vms/{uuid}/logs", tags=[Tags.vm])
async def get_vm_logs(uuid: UUID) -> StreamingResponse:
# Generator function that yields log lines as they are available
def stream_logs() -> Iterator[str]:
@ -55,7 +56,7 @@ async def get_vm_logs(uuid: UUID) -> StreamingResponse:
# TODO: Check for directory traversal
@router.post("/api/vms/create")
@router.post("/api/vms/create", tags=[Tags.vm])
async def create_vm(vm: Annotated[VmConfig, Body()]) -> VmCreateResponse:
flake_attrs = await get_attrs(vm.flake_url)
if vm.flake_attr not in flake_attrs:

View File

@ -0,0 +1,41 @@
from enum import Enum
from typing import Any, Dict, List
class Tags(Enum):
flake = "flake"
machine = "machine"
vm = "vm"
modules = "modules"
root = "root"
def __str__(self) -> str:
return self.value
tags_metadata: List[Dict[str, Any]] = [
{
"name": str(Tags.flake),
"description": "Operations on a flake.",
"externalDocs": {
"description": "What is a flake?",
"url": "https://www.tweag.io/blog/2020-05-25-flakes/",
},
},
{
"name": str(Tags.machine),
"description": "Manage physical machines. Instances of a flake",
},
{
"name": str(Tags.vm),
"description": "Manage virtual machines. Instances of a flake",
},
{
"name": str(Tags.modules),
"description": "Manage cLAN modules of a flake",
},
{
"name": str(Tags.root),
"description": "This serves as the frontend delivery",
},
]

View File

@ -8,6 +8,8 @@ from pathlib import Path
from typing import Iterator, NamedTuple
import pytest
from pydantic import AnyUrl
from pydantic.tools import parse_obj_as
from root import CLAN_CORE
from clan_cli.dirs import nixpkgs_source
@ -117,6 +119,16 @@ def test_flake_with_core(
)
@pytest.fixture
def test_democlan_url(
monkeypatch: pytest.MonkeyPatch, temporary_home: Path
) -> Iterator[AnyUrl]:
yield parse_obj_as(
AnyUrl,
"https://git.clan.lol/clan/democlan/archive/main.tar.gz",
)
@pytest.fixture
def test_flake_with_core_and_pass(
monkeypatch: pytest.MonkeyPatch, temporary_home: Path

View File

@ -8,6 +8,15 @@ from fixtures_flakes import FlakeForTest
log = logging.getLogger(__name__)
@pytest.mark.impure
def test_list_flakes(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
response = api.get("/api/flake/list")
assert response.status_code == 200, "Failed to list flakes"
data = response.json()
print("Data: ", data)
assert data.get("flakes") == ["test_flake_with_core"]
@pytest.mark.impure
def test_inspect_ok(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
params = {"url": str(test_flake_with_core.path)}
@ -38,7 +47,7 @@ def test_inspect_err(api: TestClient) -> None:
def test_inspect_flake(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
params = {"url": str(test_flake_with_core.path)}
response = api.get(
"/api/flake",
"/api/flake/inspect",
params=params,
)
assert response.status_code == 200, "Failed to inspect vm"

View File

@ -7,6 +7,7 @@ from api import TestClient
from cli import Cli
from fixtures_flakes import FlakeForTest, create_flake
from httpx import SyncByteStream
from pydantic import AnyUrl
from root import CLAN_CORE
from clan_cli.types import FlakeName
@ -42,7 +43,7 @@ def remote_flake_with_vm_without_secrets(
)
def generic_create_vm_test(api: TestClient, flake: Path, vm: str) -> None:
def generic_create_vm_test(api: TestClient, flake: Path | AnyUrl, vm: str) -> None:
print(f"flake_url: {flake} ")
response = api.post(
"/api/vms/create",
@ -113,3 +114,18 @@ def test_create_remote(
generic_create_vm_test(
api, remote_flake_with_vm_without_secrets.path, "vm_without_secrets"
)
# TODO: We need a test that creates the same VM twice, and checks that the second time it fails
# TODO: Democlan needs a machine called testVM, which is headless and gets executed by this test below
# pytest -n0 -s tests/test_vms_api_create.py::test_create_from_democlan
# @pytest.mark.skipif(not os.path.exists("/dev/kvm"), reason="Requires KVM")
# @pytest.mark.impure
# def test_create_from_democlan(
# api: TestClient,
# test_democlan_url: AnyUrl) -> None:
# generic_create_vm_test(
# api, test_democlan_url, "defaultVM"
# )

View File

@ -25,7 +25,7 @@ pkgs.mkShell {
# re-generate the api code
rm -rf api openapi.json
rm -rf src/api openapi.json
cp ${clanPkgs.clan-openapi}/openapi.json .
orval
'';

View File

@ -1,5 +1,5 @@
"use client";
import { useGetMachineSchema } from "@/api/default/default";
import { useGetMachineSchema } from "@/api/machine/machine";
import { Check, Error } from "@mui/icons-material";
import {
Box,

View File

@ -1,4 +1,4 @@
import { useListMachines } from "@/api/default/default";
import { useListMachines } from "@/api/machine/machine";
import { MachinesResponse } from "@/api/model";
import { AxiosError, AxiosResponse } from "axios";
import React, {

View File

@ -1,6 +1,6 @@
"use client";
import { useListMachines } from "@/api/default/default";
import { useListMachines } from "@/api/machine/machine";
import { Machine, MachinesResponse } from "@/api/model";
import { AxiosError, AxiosResponse } from "axios";
import React, {

View File

@ -1,4 +1,4 @@
import { inspectVm } from "@/api/default/default";
import { inspectVm } from "@/api/vm/vm";
import { HTTPValidationError, VmConfig } from "@/api/model";
import { AxiosError } from "axios";
import { useEffect, useState } from "react";

View File

@ -10,7 +10,8 @@ import {
} from "@mui/material";
import { Controller, SubmitHandler, UseFormReturn } from "react-hook-form";
import { FlakeBadge } from "../flakeBadge/flakeBadge";
import { createVm, useInspectFlakeAttrs } from "@/api/default/default";
import { createVm } from "@/api/vm/vm";
import { useInspectFlakeAttrs } from "@/api/flake/flake";
import { VmConfig } from "@/api/model";
import { Dispatch, SetStateAction, useEffect, useState } from "react";
import { toast } from "react-hot-toast";

View File

@ -7,7 +7,7 @@ import { Typography, Button } from "@mui/material";
import { ConfirmVM } from "./confirmVM";
import { Log } from "./log";
import GppMaybeIcon from "@mui/icons-material/GppMaybe";
import { useInspectFlake } from "@/api/default/default";
import { useInspectFlake } from "@/api/flake/flake";
interface ConfirmProps {
flakeUrl: string;

View File

@ -1,5 +1,5 @@
"use client";
import { useGetVmLogs } from "@/api/default/default";
import { useGetVmLogs } from "@/api/vm/vm";
import { Log } from "./log";
import { LoadingOverlay } from "./loadingOverlay";