feat: Add new API endpoints and HTML pages for ML model management

- Implemented HTML pages for datasets, models, training, testing, and results.
- Created API endpoints for managing repositories, results, tests, and training sessions.
- Added functionality for streaming training progress via Server-Sent Events (SSE).
- Introduced a Dockerfile for the ML runner with necessary dependencies.
- Developed an SDK for user code execution within the runner container.
- Enhanced CSS styles for improved UI layout and navigation.
- Established a layout template for consistent HTML structure across pages.
- Added JavaScript for dynamic interactions on the models page.
- Implemented WebSocket handling for real-time communication with kiosk devices and controllers.
- Implemented model registration and management API at /api/models
- Added Gitea proxy API for repository interactions at /api/repos
- Created results API for listing and comparing training results at /api/results
- Developed training management API for enqueueing and retrieving training jobs at /api/trainings
- Introduced SSE endpoint for live training progress updates
- Added HTML pages for models, datasets, and training management
- Created a Dockerfile for the ML runner with necessary dependencies
- Developed SDK for user code execution within the runner container
- Enhanced CSS styles for improved UI/UX
- Implemented WebSocket communication for real-time device and controller interactions in the kiosk system
This commit is contained in:
Giuseppe Raffa
2026-04-28 09:24:38 +02:00
parent ee478e52ef
commit 0ce879aa44
81 changed files with 7491 additions and 746 deletions

80
ml/runner/sdk.py Normal file
View File

@@ -0,0 +1,80 @@
"""meb_ml — SDK importabile dal codice utente dentro il container runner.
API:
from meb_ml import emit_metric, emit_series, emit_matrix, emit_log, save_artifact
emit_metric(iter=10, loss=0.23)
emit_series("roc_curve", x=fpr, y=tpr, kind="line")
emit_matrix("confusion", labels=[...], values=[[...],[...]])
emit_log("info", "epoch done")
Scrive righe JSON su stdout; il parent (ml-service) le inoltra su Redis/Influx.
Per risultati finali scrivere `out/metrics.json` con:
{"metrics": {...}, "plots": {"loss_curve": {"x": [...], "y": [...]}, ...}}
"""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
from typing import Any, Iterable, Sequence
def _print(obj: dict) -> None:
sys.stdout.write(json.dumps(obj, default=float) + "\n")
sys.stdout.flush()
def emit_metric(**fields: Any) -> None:
_print({"type": "metric", **fields})
def emit_series(name: str, x: Sequence, y: Sequence, kind: str = "line") -> None:
_print({
"type": "series",
"name": name,
"kind": kind,
"x": list(x),
"y": list(y),
})
def emit_matrix(name: str, labels: Sequence, values: Sequence[Sequence]) -> None:
_print({
"type": "matrix",
"name": name,
"labels": list(labels),
"values": [list(row) for row in values],
})
def emit_log(level: str, message: str) -> None:
_print({"type": "log", "level": level, "message": message})
def save_artifact(path: str) -> str:
"""Copia `path` nella cartella artefatti (MEB_ARTIFACTS_DIR). Ritorna la dest."""
dest_dir = Path(os.environ.get("MEB_ARTIFACTS_DIR", "/workdir/out"))
dest_dir.mkdir(parents=True, exist_ok=True)
src = Path(path)
dest = dest_dir / src.name
dest.write_bytes(src.read_bytes())
return str(dest)
def dataset_path() -> str:
return os.environ["MEB_DATASET_PATH"]
def artifacts_dir() -> str:
return os.environ.get("MEB_ARTIFACTS_DIR", "/workdir/out")
def read_test_input() -> dict:
"""Legge un singolo JSON da stdin (per script di test)."""
return json.loads(sys.stdin.readline())
def write_test_output(outputs: dict) -> None:
_print({"type": "result", "outputs": outputs})