feat: Add new API endpoints and HTML pages for ML model management

- Implemented HTML pages for datasets, models, training, testing, and results.
- Created API endpoints for managing repositories, results, tests, and training sessions.
- Added functionality for streaming training progress via Server-Sent Events (SSE).
- Introduced a Dockerfile for the ML runner with necessary dependencies.
- Developed an SDK for user code execution within the runner container.
- Enhanced CSS styles for improved UI layout and navigation.
- Established a layout template for consistent HTML structure across pages.
- Added JavaScript for dynamic interactions on the models page.
- Implemented WebSocket handling for real-time communication with kiosk devices and controllers.
- Implemented model registration and management API at /api/models
- Added Gitea proxy API for repository interactions at /api/repos
- Created results API for listing and comparing training results at /api/results
- Developed training management API for enqueueing and retrieving training jobs at /api/trainings
- Introduced SSE endpoint for live training progress updates
- Added HTML pages for models, datasets, and training management
- Created a Dockerfile for the ML runner with necessary dependencies
- Developed SDK for user code execution within the runner container
- Enhanced CSS styles for improved UI/UX
- Implemented WebSocket communication for real-time device and controller interactions in the kiosk system

copernicus/core/copernicus.py

@@ -2,6 +2,7 @@ import hashlib
 import io
 import logging
 import os
+import threading
 from datetime import datetime, timezone
 from typing import Callable, List, Optional
 
@@ -11,13 +12,20 @@ from core.cache import cache_get, cache_set
 
 logger = logging.getLogger(__name__)
 
-# ── Chiavi Redis e TTL ────────────────────────────────────────────────
+# Lock di "single-flight" per il fetch del catalogo Copernicus.
+# Senza questo, N richieste concorrenti con cache miss farebbero N chiamate
+# all'SDK (10-30s ciascuna, ~200MB di response). Con il lock, solo la prima
+# scarica e popola la cache; le altre attendono e leggono da cache.
+_catalog_fetch_lock = threading.Lock()
+
+# ── Chiavi cache e TTL ────────────────────────────────────────────────
 # Chiave per il catalogo completo Copernicus
 _CATALOG_KEY = "marine:catalog:full"
-# TTL del catalogo: 1 ora (il catalogo Copernicus cambia raramente)
-_CATALOG_TTL = 3600
-# TTL per i risultati di ricerca: 30 minuti
-_SEARCH_TTL = 1800
+# TTL L1 (Redis): 2 ore. L2 (disco) usa il default 30 giorni.
+# Il catalogo Copernicus cambia raramente, ha senso tenerlo a lungo su disco.
+_CATALOG_TTL = 2 * 3600
+# TTL L1 per le ricerche utente: 2 ore. Su disco 30 giorni.
+_SEARCH_TTL = 2 * 3600
 
 
 def _fmt_description(name: Optional[str]) -> Optional[str]:
@@ -44,10 +52,17 @@ def _get_raw_catalog() -> dict:
         logger.debug("[Catalogo] Servito da cache Redis")
         return cached
 
-    # Cache miss: interroga Copernicus SDK (operazione lenta, ~10-30s)
-    logger.info("[Catalogo] Cache miss, scaricamento da Copernicus SDK...")
-    import copernicusmarine
-    catalog = copernicusmarine.describe(disable_progress_bar=True)
+    # Single-flight: solo un thread alla volta scarica il catalogo. Gli altri
+    # attendono il lock e poi leggono il valore appena messo in cache.
+    with _catalog_fetch_lock:
+        cached = cache_get(_CATALOG_KEY)
+        if cached is not None:
+            return cached
+
+        # Cache miss: interroga Copernicus SDK (operazione lenta, ~10-30s)
+        logger.info("[Catalogo] Cache miss, scaricamento da Copernicus SDK...")
+        import copernicusmarine
+        catalog = copernicusmarine.describe(disable_progress_bar=True)
 
     # Serializza la risposta SDK in un dizionario standard
     if hasattr(catalog, "model_dump"):
@@ -57,11 +72,11 @@ def _get_raw_catalog() -> dict:
     else:
         result = catalog
 
-    # Salva in Redis per le prossime richieste (TTL 1 ora)
-    cache_set(_CATALOG_KEY, result, _CATALOG_TTL)
-    logger.info("[Catalogo] Salvato in cache Redis")
+        # Salva in Redis per le prossime richieste (TTL 1 ora)
+        cache_set(_CATALOG_KEY, result, _CATALOG_TTL)
+        logger.info("[Catalogo] Salvato in cache Redis")
 
-    return result
+        return result
 
 
 def _get_dataset_reqs(ds: dict) -> tuple: