From 0ce879aa4449ecfad3eacc84e11e292802ad8ac4 Mon Sep 17 00:00:00 2001 From: Giuseppe Raffa <77052701+sesee3@users.noreply.github.com> Date: Tue, 28 Apr 2026 09:24:38 +0200 Subject: [PATCH] feat: Add new API endpoints and HTML pages for ML model management - Implemented HTML pages for datasets, models, training, testing, and results. - Created API endpoints for managing repositories, results, tests, and training sessions. - Added functionality for streaming training progress via Server-Sent Events (SSE). - Introduced a Dockerfile for the ML runner with necessary dependencies. - Developed an SDK for user code execution within the runner container. - Enhanced CSS styles for improved UI layout and navigation. - Established a layout template for consistent HTML structure across pages. - Added JavaScript for dynamic interactions on the models page. - Implemented WebSocket handling for real-time communication with kiosk devices and controllers. - Implemented model registration and management API at /api/models - Added Gitea proxy API for repository interactions at /api/repos - Created results API for listing and comparing training results at /api/results - Developed training management API for enqueueing and retrieving training jobs at /api/trainings - Introduced SSE endpoint for live training progress updates - Added HTML pages for models, datasets, and training management - Created a Dockerfile for the ML runner with necessary dependencies - Developed SDK for user code execution within the runner container - Enhanced CSS styles for improved UI/UX - Implemented WebSocket communication for real-time device and controller interactions in the kiosk system --- api/package-lock.json | 193 ++++++ api/package.json | 1 + api/src/index.js | 27 + api/src/migrations/001_ml_datasets.sql | 71 ++ .../002_ml_datasets_bucket_default.sql | 5 + api/src/migrations/003_ml_models.sql | 35 + api/src/migrations/004_ml_trainings.sql | 26 + api/src/migrations/005_ml_tests.sql | 17 + api/src/migrations/006_jobs.sql | 27 + api/src/migrations/007_kiosktemplates.sql | 79 +++ api/src/migrations/008_rulesets.sql | 124 ++++ api/src/routes/docs.js | 80 +++ api/src/routes/jobs.js | 120 ++++ api/src/routes/kiosk.js | 443 +++++++++++++ api/src/routes/kiosk.public.js | 49 ++ api/src/routes/kiosk.sensor.js | 67 ++ api/src/routes/marine.datasets.js | 232 +++++++ api/src/routes/pageconnections.js | 100 +++ api/src/routes/queue.js | 35 + api/src/routes/rules.js | 609 ++++++++++++++++++ api/src/storage/influx.js | 4 +- api/src/storage/minio.js | 56 +- api/src/storage/postgres.js | 9 + console/src/index.js | 24 + console/src/pages/dashboard.html | 348 +++++----- console/src/pages/documentation.html | 230 +++++++ console/src/pages/forecasts.html | 300 +++++++++ console/src/pages/kioskedit.html | 21 - console/src/pages/kiosklive.html | 331 ++++++++++ console/src/pages/live.html | 22 - console/src/pages/marine.html | 301 +++++++++ console/src/pages/rulesets.html | 410 +++++++----- console/src/pages/sessions.html | 22 - console/src/static/styles/dashboard.css | 22 + console/src/static/styles/rulesets.css | 78 +++ console/src/static/styles/style.css | 57 +- console/src/static/theme-toggle.js | 85 --- copernicus/core/cache.py | 303 +++++++-- copernicus/core/copernicus.py | 41 +- copernicus/main.py | 12 +- copernicus/routers/jobs.py | 46 +- copernicus/schemas.py | 2 +- docker-compose.yml | 68 +- ml/.env.example | 45 ++ ml/Dockerfile | 3 + ml/core/api_client.py | 72 +++ ml/core/config.py | 64 ++ ml/core/db.py | 53 ++ ml/core/docker_runner.py | 439 +++++++++++++ ml/core/gitea.py | 57 ++ ml/core/influx_client.py | 75 +++ ml/core/minio_client.py | 118 ++++ ml/core/model_spec.py | 90 +++ ml/core/redis_client.py | 29 + ml/core/worker.py | 54 ++ ml/main.py | 95 ++- ml/requirements.txt | 14 +- ml/routers/datasets.py | 160 +++++ ml/routers/models.py | 131 ++++ ml/routers/pages.py | 75 +++ ml/routers/repos.py | 51 ++ ml/routers/results.py | 89 +++ ml/routers/tests.py | 109 ++++ ml/routers/trainings.py | 129 ++++ ml/routers/trainings_stream.py | 64 ++ ml/runner/Dockerfile | 18 + ml/runner/sdk.py | 80 +++ ml/static/styles/ml.css | 146 +++++ ml/templates/_layout.html | 33 + ml/templates/datasets.html | 39 ++ ml/templates/models.html | 57 ++ ml/templates/results.html | 116 +--- ml/templates/test.html | 33 + ml/templates/train.html | 35 + realtime/package-lock.json | 153 +++++ realtime/src/index.js | 4 + realtime/src/routes/kiosk.js | 33 +- realtime/src/routes/rules.js | 57 ++ realtime/src/store/influx.js | 60 +- realtime/src/ws/handler.js | 58 +- realtime/src/ws/kiosk.js | 167 +++++ 81 files changed, 7491 insertions(+), 746 deletions(-) create mode 100644 api/src/migrations/001_ml_datasets.sql create mode 100644 api/src/migrations/002_ml_datasets_bucket_default.sql create mode 100644 api/src/migrations/003_ml_models.sql create mode 100644 api/src/migrations/004_ml_trainings.sql create mode 100644 api/src/migrations/005_ml_tests.sql create mode 100644 api/src/migrations/006_jobs.sql create mode 100644 api/src/migrations/007_kiosktemplates.sql create mode 100644 api/src/migrations/008_rulesets.sql create mode 100644 api/src/routes/docs.js create mode 100644 api/src/routes/jobs.js create mode 100644 api/src/routes/kiosk.js create mode 100644 api/src/routes/kiosk.public.js create mode 100644 api/src/routes/kiosk.sensor.js create mode 100644 api/src/routes/marine.datasets.js create mode 100644 api/src/routes/pageconnections.js create mode 100644 api/src/routes/queue.js create mode 100644 api/src/routes/rules.js create mode 100644 console/src/pages/documentation.html create mode 100644 console/src/pages/forecasts.html create mode 100644 console/src/pages/kiosklive.html create mode 100644 console/src/pages/marine.html delete mode 100644 console/src/static/theme-toggle.js create mode 100644 ml/core/api_client.py create mode 100644 ml/core/config.py create mode 100644 ml/core/db.py create mode 100644 ml/core/docker_runner.py create mode 100644 ml/core/gitea.py create mode 100644 ml/core/influx_client.py create mode 100644 ml/core/minio_client.py create mode 100644 ml/core/model_spec.py create mode 100644 ml/core/redis_client.py create mode 100644 ml/core/worker.py create mode 100644 ml/routers/datasets.py create mode 100644 ml/routers/models.py create mode 100644 ml/routers/pages.py create mode 100644 ml/routers/repos.py create mode 100644 ml/routers/results.py create mode 100644 ml/routers/tests.py create mode 100644 ml/routers/trainings.py create mode 100644 ml/routers/trainings_stream.py create mode 100644 ml/runner/Dockerfile create mode 100644 ml/runner/sdk.py create mode 100644 ml/static/styles/ml.css create mode 100644 ml/templates/_layout.html create mode 100644 ml/templates/models.html create mode 100644 realtime/src/routes/rules.js create mode 100644 realtime/src/ws/kiosk.js diff --git a/api/package-lock.json b/api/package-lock.json index 496e131..e0513db 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -17,6 +17,7 @@ "ioredis": "^5.10.0", "jsonwebtoken": "^9.0.3", "minio": "^8.0.7", + "multer": "^1.4.5-lts.1", "pg": "^8.20.0" } }, @@ -63,6 +64,12 @@ "node": ">= 0.6" } }, + "node_modules/append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", + "license": "MIT" + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -123,6 +130,23 @@ "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -170,6 +194,51 @@ "node": ">=0.10.0" } }, + "node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/concat-stream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/concat-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/concat-stream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/content-disposition": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", @@ -229,6 +298,12 @@ "node": ">=6.6.0" } }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -663,6 +738,12 @@ "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", "license": "MIT" }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, "node_modules/jsonwebtoken": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", @@ -821,6 +902,15 @@ "url": "https://opencollective.com/express" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minio": { "version": "8.0.7", "resolved": "https://registry.npmjs.org/minio/-/minio-8.0.7.tgz", @@ -875,12 +965,86 @@ "node": ">= 0.6" } }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/multer": { + "version": "1.4.5-lts.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.2.tgz", + "integrity": "sha512-VzGiVigcG9zUAoCNU+xShztrlr1auZOlurXynNvO9GiWD1/mTBbUljOKY+qMeazBqXgRnjzeEgJI/wyjJUHg9A==", + "deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.0.0", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.4", + "object-assign": "^4.1.1", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/multer/node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/multer/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/multer/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/multer/node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/negotiator": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", @@ -890,6 +1054,15 @@ "node": ">= 0.6" } }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -1085,6 +1258,12 @@ "node": ">=0.10.0" } }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -1424,6 +1603,14 @@ "stream-chain": "^2.2.5" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/strict-uri-encode": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", @@ -1486,6 +1673,12 @@ "node": ">= 0.6" } }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "license": "MIT" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/api/package.json b/api/package.json index daca388..8ae1ab0 100644 --- a/api/package.json +++ b/api/package.json @@ -18,6 +18,7 @@ "ioredis": "^5.10.0", "jsonwebtoken": "^9.0.3", "minio": "^8.0.7", + "multer": "^1.4.5-lts.1", "pg": "^8.20.0" } } \ No newline at end of file diff --git a/api/src/index.js b/api/src/index.js index 21b218a..e53c8f2 100644 --- a/api/src/index.js +++ b/api/src/index.js @@ -57,6 +57,12 @@ app.get('/health', async (req, res) => { const paramsSensorRoutes = require('./routes/params.sensor'); app.use('/params/sensor', paramsSensorRoutes); +const kioskSensorRoutes = require('./routes/kiosk.sensor'); +app.use('/kiosk/sensor', kioskSensorRoutes); + +const kioskPublicRoutes = require('./routes/kiosk.public'); +app.use('/kiosk', kioskPublicRoutes); + // Middleware di autenticazione per tutte le API protette app.use(requireAuth); @@ -75,6 +81,27 @@ app.use('/settings', settingsRoutes) const sessionsRoutes = require('./routes/sessions') app.use('/sessions', sessionsRoutes) +const docsRoutes = require('./routes/docs') +app.use('/docs', docsRoutes) + +const marineDatasetsRoutes = require('./routes/marine.datasets') +app.use('/marine/datasets', marineDatasetsRoutes) + +const jobsRoutes = require('./routes/jobs') +app.use('/jobs', jobsRoutes) + +const queueRoutes = require('./routes/queue') +app.use('/queue', queueRoutes) + +const pageconnectionsRoutes = require('./routes/pageconnections') +app.use('/pageconnections', pageconnectionsRoutes) + +const kioskRoutes = require('./routes/kiosk') +app.use('/kiosk', kioskRoutes) + +const rulesRoutes = require('./routes/rules') +app.use('/rules', rulesRoutes) + app.listen(PORT, '0.0.0.0', () => { console.log(`Started on port ${PORT}`); }); diff --git a/api/src/migrations/001_ml_datasets.sql b/api/src/migrations/001_ml_datasets.sql new file mode 100644 index 0000000..50135bf --- /dev/null +++ b/api/src/migrations/001_ml_datasets.sql @@ -0,0 +1,71 @@ +-- -- Database: ml +-- -- Eseguire con: psql -U meb -d ml -f 001_ml_datasets.sql +-- -- +-- -- Tabella dei metadati dei dataset salvati su MinIO. +-- -- Ogni riga è associata ad un file nel bucket "ml.datasets" (o altri bucket future) +-- -- tramite minio_key (= nome oggetto in MinIO, che è anche il suo "id" nativo). + +-- CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +-- CREATE TABLE IF NOT EXISTS datasets ( +-- id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + +-- -- Storage MinIO +-- minio_key TEXT NOT NULL UNIQUE, -- es. "2026-04-22_currents_med.csv" +-- bucket TEXT NOT NULL DEFAULT 'ml.datasets', + +-- -- Identità dataset +-- nome TEXT NOT NULL, +-- description TEXT, +-- tags TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], +-- type TEXT NOT NULL DEFAULT 'copernicus', -- copernicus | custom | imported +-- format TEXT NOT NULL, -- csv | json | netcdf +-- notes TEXT, + +-- -- Provenienza / audit +-- created_by TEXT NOT NULL, -- username +-- created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +-- updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +-- last_used_at TIMESTAMPTZ, + +-- -- Misure del file +-- size_bytes BIGINT NOT NULL DEFAULT 0, +-- row_count BIGINT, -- numero righe (se noto) +-- columns TEXT[], -- nomi colonne finali (dopo rename) + +-- -- Specifico Copernicus (nullable per altri type) +-- copernicus_dataset_id TEXT, +-- variables TEXT[], -- variabili richieste +-- variable_renames JSONB, -- {original: custom} +-- bbox JSONB, -- [min_lon, min_lat, max_lon, max_lat] +-- start_date DATE, +-- end_date DATE, + +-- -- Estensibile per type futuri senza migration +-- params JSONB NOT NULL DEFAULT '{}'::JSONB, + +-- -- Versioning semplice +-- version INT NOT NULL DEFAULT 1, + +-- CONSTRAINT datasets_format_ok CHECK (format IN ('csv','json','netcdf')), +-- CONSTRAINT datasets_type_ok CHECK (type IN ('copernicus','custom','imported')) +-- ); + +-- CREATE INDEX IF NOT EXISTS idx_datasets_created_by ON datasets(created_by); +-- CREATE INDEX IF NOT EXISTS idx_datasets_type ON datasets(type); +-- CREATE INDEX IF NOT EXISTS idx_datasets_tags ON datasets USING gin(tags); +-- CREATE INDEX IF NOT EXISTS idx_datasets_created_at ON datasets(created_at DESC); +-- CREATE INDEX IF NOT EXISTS idx_datasets_minio_key ON datasets(minio_key); + +-- -- Trigger per aggiornare updated_at automaticamente +-- CREATE OR REPLACE FUNCTION set_updated_at() RETURNS TRIGGER AS $$ +-- BEGIN +-- NEW.updated_at = NOW(); +-- RETURN NEW; +-- END; +-- $$ LANGUAGE plpgsql; + +-- DROP TRIGGER IF EXISTS trg_datasets_updated_at ON datasets; +-- CREATE TRIGGER trg_datasets_updated_at +-- BEFORE UPDATE ON datasets +-- FOR EACH ROW EXECUTE FUNCTION set_updated_at(); diff --git a/api/src/migrations/002_ml_datasets_bucket_default.sql b/api/src/migrations/002_ml_datasets_bucket_default.sql new file mode 100644 index 0000000..bdbda76 --- /dev/null +++ b/api/src/migrations/002_ml_datasets_bucket_default.sql @@ -0,0 +1,5 @@ +-- Database: ml +-- DEPRECATED: la colonna `bucket` e' stata rimossa dalla tabella `datasets`. +-- Il bucket e' ora fisso a 'ml.datasets' lato applicazione (vedi +-- api/src/routes/marine.datasets.js e ml/routers/datasets.py). +-- Questo file e' lasciato vuoto per non rompere lo storico delle migration. diff --git a/api/src/migrations/003_ml_models.sql b/api/src/migrations/003_ml_models.sql new file mode 100644 index 0000000..64d0e66 --- /dev/null +++ b/api/src/migrations/003_ml_models.sql @@ -0,0 +1,35 @@ +-- Database: ml +-- Registro modelli ML: ogni riga punta a una repo Gitea con codice del modello. + +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +CREATE TABLE IF NOT EXISTS models ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + type TEXT NOT NULL, -- 'xgboost'|'lstm'|'sklearn'|... + gitea_repo TEXT NOT NULL, -- "owner/repo" + default_branch TEXT NOT NULL DEFAULT 'main', + spec JSONB, -- copia cached di model.yml @ tip del default_branch + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS idx_models_created_by ON models(created_by); +CREATE INDEX IF NOT EXISTS idx_models_type ON models(type); + +CREATE TABLE IF NOT EXISTS model_notes ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + model_id UUID NOT NULL REFERENCES models(id) ON DELETE CASCADE, + author TEXT NOT NULL, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS idx_model_notes_model ON model_notes(model_id, created_at DESC); + +DROP TRIGGER IF EXISTS trg_models_updated_at ON models; +CREATE TRIGGER trg_models_updated_at + BEFORE UPDATE ON models + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- La colonna `bucket` su datasets e' stata rimossa: il bucket e' fisso a +-- 'ml.datasets' (vedi codice). Lasciato come no-op per coerenza storica. \ No newline at end of file diff --git a/api/src/migrations/004_ml_trainings.sql b/api/src/migrations/004_ml_trainings.sql new file mode 100644 index 0000000..a47003c --- /dev/null +++ b/api/src/migrations/004_ml_trainings.sql @@ -0,0 +1,26 @@ +-- Database: ml +-- Storico training di modelli. Le time-series (cpu/mem/loss) vivono su InfluxDB; +-- qui salviamo solo anagrafica + risultati finali e riepilogo risorse. + +CREATE TABLE IF NOT EXISTS trainings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + model_id UUID NOT NULL REFERENCES models(id) ON DELETE CASCADE, + version TEXT NOT NULL, + patch TEXT NOT NULL, -- git commit sha (short o full) + dataset_id UUID NOT NULL, + started_by TEXT NOT NULL, + queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + started_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + duration_ms BIGINT, + status TEXT NOT NULL DEFAULT 'queued', -- queued|running|succeeded|failed|cancelled + artifacts_prefix TEXT, -- es. "models///" + results JSONB, -- final metrics + plots (arrays puri) + resource_summary JSONB, -- {cpu_peak,cpu_avg,mem_peak_mb,mem_avg_mb,samples} + error TEXT, + CONSTRAINT trainings_status_ok CHECK (status IN ('queued','running','succeeded','failed','cancelled')), + UNIQUE(model_id, version, patch) +); +CREATE INDEX IF NOT EXISTS idx_trainings_model ON trainings(model_id, started_at DESC); +CREATE INDEX IF NOT EXISTS idx_trainings_status ON trainings(status); +CREATE INDEX IF NOT EXISTS idx_trainings_user ON trainings(started_by); diff --git a/api/src/migrations/005_ml_tests.sql b/api/src/migrations/005_ml_tests.sql new file mode 100644 index 0000000..8c61549 --- /dev/null +++ b/api/src/migrations/005_ml_tests.sql @@ -0,0 +1,17 @@ +-- Database: ml +-- Sessioni di test: una sessione contiene 1..N run (set di input → output). + +CREATE TABLE IF NOT EXISTS tests ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + training_id UUID NOT NULL REFERENCES trainings(id) ON DELETE CASCADE, + user_id TEXT NOT NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + ended_at TIMESTAMPTZ, + runs JSONB NOT NULL DEFAULT '[]'::JSONB, + -- ogni elemento: {inputs, outputs, duration_ms, cpu_peak, mem_peak_mb, ts} + model_size_bytes BIGINT, + notes TEXT +); +CREATE INDEX IF NOT EXISTS idx_tests_training ON tests(training_id); +CREATE INDEX IF NOT EXISTS idx_tests_user ON tests(user_id); +CREATE INDEX IF NOT EXISTS idx_tests_started ON tests(started_at DESC); diff --git a/api/src/migrations/006_jobs.sql b/api/src/migrations/006_jobs.sql new file mode 100644 index 0000000..5a55738 --- /dev/null +++ b/api/src/migrations/006_jobs.sql @@ -0,0 +1,27 @@ +-- Database: ml +-- Tabella jobs: ciclo di vita di un lavoro asincrono (training oggi, domani altro). +-- L'api-service espone /jobs /queue /pageconnections per coordinare accessi e coda. + +CREATE TABLE IF NOT EXISTS jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + type TEXT NOT NULL, -- 'train' | 'test' | ... + status TEXT NOT NULL DEFAULT 'queued',-- queued|running|succeeded|failed|cancelled + payload JSONB NOT NULL DEFAULT '{}'::JSONB, + result JSONB, + error TEXT, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + started_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + CONSTRAINT jobs_status_ok CHECK (status IN ('queued','running','succeeded','failed','cancelled')) +); +CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status); +CREATE INDEX IF NOT EXISTS idx_jobs_type ON jobs(type); +CREATE INDEX IF NOT EXISTS idx_jobs_created_by ON jobs(created_by); +CREATE INDEX IF NOT EXISTS idx_jobs_created_at ON jobs(created_at DESC); + +DROP TRIGGER IF EXISTS trg_jobs_updated_at ON jobs; +CREATE TRIGGER trg_jobs_updated_at + BEFORE UPDATE ON jobs + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); diff --git a/api/src/migrations/007_kiosktemplates.sql b/api/src/migrations/007_kiosktemplates.sql new file mode 100644 index 0000000..ae713c9 --- /dev/null +++ b/api/src/migrations/007_kiosktemplates.sql @@ -0,0 +1,79 @@ +-- ============================================================ +-- Kiosk schema (DB: sensors) +-- ------------------------------------------------------------ +-- SOLO struttura: PK, FK, NOT NULL, sequence, indice. +-- Tutta la logica (id char(8), updated_at, defaults, CHECK, +-- "un solo active") e' gestita lato applicazione (Node/JS). +-- ============================================================ + +-- ───────────────────────────────────────────────────────────── +-- Templates +-- ───────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS kiosktemplates ( + id char(8) PRIMARY KEY, + name varchar(50) NOT NULL, + tags text[], + active boolean, + archived boolean, + created_at timestamp, + updated_at timestamp +); + +-- Idempotente: se la tabella esiste gia', garantisci PK + NOT NULL critici. +DO $$ BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'kiosktemplates'::regclass AND contype = 'p' + ) THEN + ALTER TABLE kiosktemplates ADD PRIMARY KEY (id); + END IF; +END $$; + +ALTER TABLE kiosktemplates ALTER COLUMN name SET NOT NULL; + +-- ───────────────────────────────────────────────────────────── +-- Elements (un template -> N elementi) +-- ───────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS kioskelements ( + id bigint PRIMARY KEY, + template_id char(8) NOT NULL, + font integer, + label varchar(100), + x integer, + y integer, + width integer, + height integer, + color varchar(20) +); + +DO $$ BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'kioskelements'::regclass AND contype = 'p' + ) THEN + ALTER TABLE kioskelements ADD PRIMARY KEY (id); + END IF; +END $$; + +ALTER TABLE kioskelements ALTER COLUMN template_id SET NOT NULL; + +-- Auto-increment per kioskelements.id (l'app fa INSERT senza specificare id). +CREATE SEQUENCE IF NOT EXISTS kioskelements_id_seq OWNED BY kioskelements.id; +SELECT setval( + 'kioskelements_id_seq', + COALESCE((SELECT MAX(id) FROM kioskelements), 0) + 1, + false +); +ALTER TABLE kioskelements ALTER COLUMN id SET DEFAULT nextval('kioskelements_id_seq'); + +-- Foreign Key con CASCADE. +ALTER TABLE kioskelements DROP CONSTRAINT IF EXISTS fk_kioskelements_template; +ALTER TABLE kioskelements + ADD CONSTRAINT fk_kioskelements_template + FOREIGN KEY (template_id) + REFERENCES kiosktemplates(id) + ON DELETE CASCADE + ON UPDATE CASCADE; + +CREATE INDEX IF NOT EXISTS kioskelements_template_idx + ON kioskelements (template_id); diff --git a/api/src/migrations/008_rulesets.sql b/api/src/migrations/008_rulesets.sql new file mode 100644 index 0000000..d3434a0 --- /dev/null +++ b/api/src/migrations/008_rulesets.sql @@ -0,0 +1,124 @@ +-- ============================================================ +-- Rulesets schema (DB: rules) +-- ------------------------------------------------------------ +-- Un ruleset e' una collezione versionata di "items" (paths +-- Signal K per i logs, codici openmeteo per i forecasts, ...). +-- Modello: +-- * 5 tipi fissi: logs | forecast_current | forecast_hourly +-- | marine_current | marine_hourly +-- * Un solo ruleset puo' essere "active" per ciascun tipo. +-- * Le versioni sono triple di interi 1..100 (major.build.patch). +-- * Gli items sono JSONB per massima flessibilita'. +-- * Ogni item ha un "ref" stabile scelto dall'utente: e' la +-- chiave logica che garantisce continuita' su InfluxDB anche +-- se il path del sensore cambia. +-- * Le deployments tracciano quale ruleset-version e' stato +-- pushato ad ogni sensore. +-- ============================================================ + +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +-- ───────────────────────────────────────────────────────────── +-- RULESETS +-- ───────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS rulesets ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + type TEXT NOT NULL + CHECK (type IN ('logs','forecast_current','forecast_hourly','marine_current','marine_hourly')), + version_major SMALLINT NOT NULL DEFAULT 1 CHECK (version_major BETWEEN 1 AND 100), + version_build SMALLINT NOT NULL DEFAULT 0 CHECK (version_build BETWEEN 0 AND 100), + version_patch SMALLINT NOT NULL DEFAULT 0 CHECK (version_patch BETWEEN 0 AND 100), + description TEXT NOT NULL DEFAULT '', + tags TEXT[] NOT NULL DEFAULT '{}', + -- items: [{ ref, path, enabled, meta: {...} }, ...] + -- ref: identificatore logico stabile (chiave su Influx) + -- path: SK path (logs) | codice openmeteo (forecast/marine) + -- meta: { name, unit, measurement, sk_path, group_name, category, ... } + items JSONB NOT NULL DEFAULT '[]'::jsonb, + active BOOLEAN NOT NULL DEFAULT false, + archived BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + UNIQUE (type, version_major, version_build, version_patch) +); + +-- solo UN ruleset active per tipo (archiviati esclusi) +CREATE UNIQUE INDEX IF NOT EXISTS rulesets_one_active_per_type + ON rulesets (type) + WHERE active = true AND archived = false; + +CREATE INDEX IF NOT EXISTS rulesets_type_idx ON rulesets (type); +CREATE INDEX IF NOT EXISTS rulesets_active_idx ON rulesets (type) WHERE active = true; +CREATE INDEX IF NOT EXISTS rulesets_archived_idx ON rulesets (archived); +CREATE INDEX IF NOT EXISTS rulesets_items_gin_idx ON rulesets USING GIN (items); + +-- Validazione items: array di oggetti con almeno ref+path +CREATE OR REPLACE FUNCTION rulesets_validate_items() RETURNS trigger AS $$ +DECLARE + refs TEXT[]; +BEGIN + IF jsonb_typeof(NEW.items) <> 'array' THEN + RAISE EXCEPTION 'items must be a JSON array'; + END IF; + + -- tutti gli item devono avere ref non vuoto e path (anche vuoto ammesso) + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements(NEW.items) it + WHERE jsonb_typeof(it) <> 'object' + OR NULLIF(it->>'ref','') IS NULL + ) THEN + RAISE EXCEPTION 'every item must be an object with a non-empty "ref"'; + END IF; + + -- unicita' ref all'interno dello stesso ruleset + SELECT array_agg(it->>'ref') INTO refs + FROM jsonb_array_elements(NEW.items) it; + IF (SELECT count(DISTINCT x) FROM unnest(refs) x) <> COALESCE(array_length(refs,1),0) THEN + RAISE EXCEPTION 'item refs must be unique within the ruleset'; + END IF; + + NEW.updated_at := NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS rulesets_validate_trigger ON rulesets; +CREATE TRIGGER rulesets_validate_trigger + BEFORE INSERT OR UPDATE ON rulesets + FOR EACH ROW EXECUTE FUNCTION rulesets_validate_items(); + + +-- ───────────────────────────────────────────────────────────── +-- DEPLOYMENTS +-- Traccia quale ruleset-version e' stato pushato ad ogni +-- sensore (per tipo). Un solo ruleset per (sensor,type). +-- ───────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS ruleset_deployments ( + sensor_name TEXT NOT NULL, + type TEXT NOT NULL, + ruleset_id UUID NOT NULL REFERENCES rulesets(id) ON DELETE CASCADE, + deployed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + acked_at TIMESTAMPTZ, + PRIMARY KEY (sensor_name, type) +); + +CREATE INDEX IF NOT EXISTS ruleset_deployments_ruleset_idx + ON ruleset_deployments (ruleset_id); + + +-- ───────────────────────────────────────────────────────────── +-- AUDIT LOG (opzionale ma utile) +-- ───────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS ruleset_changes ( + id BIGSERIAL PRIMARY KEY, + ruleset_id UUID, + type TEXT, + action TEXT NOT NULL, -- created | updated | activated | archived | deleted | deployed + user_id TEXT, + payload JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS ruleset_changes_ruleset_idx ON ruleset_changes (ruleset_id); +CREATE INDEX IF NOT EXISTS ruleset_changes_created_idx ON ruleset_changes (created_at DESC); diff --git a/api/src/routes/docs.js b/api/src/routes/docs.js new file mode 100644 index 0000000..c733017 --- /dev/null +++ b/api/src/routes/docs.js @@ -0,0 +1,80 @@ +/** + * Gestione file Markdown su MinIO nel bucket "documentation". + * + * GET /docs → lista file (name, size, lastModified) + * GET /docs/:name → contenuto markdown raw + * POST /docs → crea nuovo documento body {name, content} + * PUT /docs/:name → sovrascrive contenuto body {content} + * DELETE /docs/:name → elimina + */ + +const express = require('express'); +const { listObjects, readText, writeText, removeObject } = require('../storage/minio'); + +const router = express.Router(); +const BUCKET = 'documentation'; + +const sanitize = (name) => { + // Solo caratteri safe per oggetti MinIO; forza estensione .md + const clean = String(name || '').trim().replace(/[^a-zA-Z0-9 _\-./]/g, '').replace(/\.+/g, '.'); + if (!clean) return null; + return clean.endsWith('.md') ? clean : `${clean}.md`; +}; + +router.get('/', async (req, res) => { + try { + const files = await listObjects(BUCKET); + res.json(files.filter(f => f.name.endsWith('.md'))); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +router.get('/:name', async (req, res) => { + const name = sanitize(req.params.name); + if (!name) return res.status(400).json({ error: 'invalid name' }); + try { + const content = await readText(BUCKET, name); + res.type('text/markdown').send(content); + } catch (e) { + if (e.code === 'NoSuchKey') return res.status(404).json({ error: 'not found' }); + res.status(500).json({ error: e.message }); + } +}); + +router.post('/', async (req, res) => { + const name = sanitize(req.body?.name); + const content = req.body?.content ?? ''; + if (!name) return res.status(400).json({ error: 'name required' }); + try { + const r = await writeText(BUCKET, name, content); + res.status(201).json(r); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +router.put('/:name', async (req, res) => { + const name = sanitize(req.params.name); + if (!name) return res.status(400).json({ error: 'invalid name' }); + const content = req.body?.content ?? ''; + try { + const r = await writeText(BUCKET, name, content); + res.json(r); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +router.delete('/:name', async (req, res) => { + const name = sanitize(req.params.name); + if (!name) return res.status(400).json({ error: 'invalid name' }); + try { + await removeObject(BUCKET, name); + res.status(204).end(); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +module.exports = router; diff --git a/api/src/routes/jobs.js b/api/src/routes/jobs.js new file mode 100644 index 0000000..c229945 --- /dev/null +++ b/api/src/routes/jobs.js @@ -0,0 +1,120 @@ +/** + * /jobs — ciclo di vita dei job asincroni (es. training). + * + * Tabella: jobs (db "ml") — vedi migrations/006_jobs.sql + */ +const express = require('express'); +const crypto = require('crypto'); +const { query } = require('../storage/postgres'); + +const router = express.Router(); + +// In assenza dei trigger/funzioni DB (`gen_random_uuid` default, +// `set_updated_at` trigger, `jobs_status_ok` CHECK) gestiamo tutto qui. +const VALID_STATUSES = ['queued', 'running', 'succeeded', 'failed', 'cancelled']; +function genUUID() { return crypto.randomUUID(); } + +function rowToJob(r) { + return { + id: r.id, + type: r.type, + status: r.status, + payload: r.payload, + result: r.result, + error: r.error, + created_by: r.created_by, + created_at: r.created_at, + updated_at: r.updated_at, + started_at: r.started_at, + finished_at: r.finished_at, + }; +} + +router.post('/', async (req, res) => { + try { + const { type, created_by, payload } = req.body || {}; + if (!type) return res.status(400).json({ error: 'type required' }); + const newId = genUUID(); + const r = await query( + `INSERT INTO jobs (id, type, created_by, payload, status, created_at, updated_at) + VALUES ($1, $2, $3, $4::jsonb, 'queued', NOW(), NOW()) RETURNING *`, + [newId, type, created_by || req.user?.username || 'unknown', JSON.stringify(payload || {})], + 'ml' + ); + res.status(201).json(rowToJob(r.rows[0])); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +router.get('/', async (req, res) => { + try { + const filters = []; + const params = []; + if (req.query.type) { params.push(req.query.type); filters.push(`type = $${params.length}`); } + if (req.query.status) { params.push(req.query.status); filters.push(`status = $${params.length}`); } + if (req.query.user === 'me' && req.user?.username) { + params.push(req.user.username); + filters.push(`created_by = $${params.length}`); + } + const limit = Math.min(parseInt(req.query.limit || '100', 10), 500); + params.push(limit); + const where = filters.length ? `WHERE ${filters.join(' AND ')}` : ''; + const r = await query( + `SELECT * FROM jobs ${where} ORDER BY created_at DESC LIMIT $${params.length}`, + params, 'ml' + ); + res.json(r.rows.map(rowToJob)); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +router.get('/:id', async (req, res) => { + try { + const r = await query('SELECT * FROM jobs WHERE id = $1', [req.params.id], 'ml'); + if (!r.rows.length) return res.status(404).json({ error: 'not found' }); + res.json(rowToJob(r.rows[0])); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +router.patch('/:id', async (req, res) => { + try { + const allowed = ['status', 'payload', 'result', 'error', 'started_at', 'finished_at']; + const sets = []; + const params = []; + + // CHECK status sostituito da whitelist applicativa + if ('status' in req.body && !VALID_STATUSES.includes(req.body.status)) { + return res.status(400).json({ + error: `invalid status, must be one of: ${VALID_STATUSES.join(', ')}` + }); + } + + for (const k of allowed) { + if (k in req.body) { + params.push(k === 'payload' || k === 'result' ? JSON.stringify(req.body[k]) : req.body[k]); + const cast = (k === 'payload' || k === 'result') ? '::jsonb' : ''; + sets.push(`${k} = $${params.length}${cast}`); + } + } + if (!sets.length) return res.status(400).json({ error: 'no fields' }); + + // Trigger trg_jobs_updated_at non presente: lo facciamo manualmente. + sets.push('updated_at = NOW()'); + + params.push(req.params.id); + const r = await query( + `UPDATE jobs SET ${sets.join(', ')} WHERE id = $${params.length} RETURNING *`, + params, 'ml' + ); + if (!r.rows.length) return res.status(404).json({ error: 'not found' }); + res.json(rowToJob(r.rows[0])); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +module.exports = router; diff --git a/api/src/routes/kiosk.js b/api/src/routes/kiosk.js new file mode 100644 index 0000000..6a8e2ca --- /dev/null +++ b/api/src/routes/kiosk.js @@ -0,0 +1,443 @@ +/** + * Kiosk templates API + * Base: /kiosk + * + * Schema reale (DB `sensors`): + * kiosktemplates(id char(8) PK, name varchar(50) NOT NULL, + * tags text[], active bool, archived bool, + * created_at timestamp, updated_at timestamp) + * kioskelements(id bigint PK auto-seq, template_id char(8) NOT NULL FK CASCADE, + * font, label varchar(100), x, y, width, height, color varchar(20)) + * + * NOTA: il DB contiene SOLO i constraint strutturali (PK, FK, NOT NULL). + * Tutta la logica (id char(8), updated_at, "un solo active per tabella", + * validazione, CHECK su x/y/w/h, prevenzione active+archived) e' gestita qui. + * + * Le route restituiscono il template arricchito con `elements`: + * { id, name, tags, active, archived, created_at, updated_at, elements: [...] } + */ + +const router = require('express').Router(); +const crypto = require('crypto'); +const { query, getClient } = require('../storage/postgres'); + +const DB = 'kiosk'; // pool puntato al DB `sensors` (vedi postgres.js) + +// Sostituisce il default DB `gen_short_id8()` (funzione SQL non presente nel DB). +function genShortId8() { + return crypto.randomBytes(4).toString('hex'); // 8 char hex [0-9a-f] +} + +// ──────────────────────────────────────────────────────────── +// Helpers +// ──────────────────────────────────────────────────────────── + +const ELEMENT_FIELDS = ['font', 'label', 'x', 'y', 'width', 'height', 'color']; + +function sanitizeElement(e) { + const out = {}; + if (e.font !== undefined) out.font = parseInt(e.font, 10); + if (e.label !== undefined) out.label = String(e.label).slice(0, 100); + if (e.x !== undefined) out.x = parseInt(e.x, 10); + if (e.y !== undefined) out.y = parseInt(e.y, 10); + if (e.width !== undefined) out.width = parseInt(e.width, 10); + if (e.height !== undefined) out.height = parseInt(e.height, 10); + if (e.color !== undefined) out.color = String(e.color).slice(0, 20); + return out; +} + +function validateElementForInsert(e) { + const errs = []; + if (typeof e.x !== 'number' || e.x < 0) errs.push('x must be >= 0'); + if (typeof e.y !== 'number' || e.y < 0) errs.push('y must be >= 0'); + if (typeof e.width !== 'number' || e.width <= 0) errs.push('width must be > 0'); + if (typeof e.height !== 'number' || e.height <= 0) errs.push('height must be > 0'); + return errs.length ? errs.join(', ') : null; +} + +/** + * Inserisce N elementi per un template in batch. + * Usa il client passato (per transazione). + */ +async function insertElements(client, templateId, elements) { + if (!Array.isArray(elements) || !elements.length) return []; + const rows = []; + for (const raw of elements) { + const e = sanitizeElement(raw); + const err = validateElementForInsert({ x:0, y:0, width:1, height:1, ...e }); + if (err) throw new Error(`element invalid: ${err}`); + const r = await client.query( + `INSERT INTO kioskelements (template_id, font, label, x, y, width, height, color) + VALUES ($1, COALESCE($2,16), COALESCE($3,''), COALESCE($4,0), COALESCE($5,0), + COALESCE($6,1), COALESCE($7,1), COALESCE($8,'#1e293b')) + RETURNING *`, + [templateId, e.font, e.label, e.x, e.y, e.width, e.height, e.color] + ); + rows.push(r.rows[0]); + } + return rows; +} + +/** + * Ritorna un template con la lista elements aggregata. + */ +async function fetchTemplateWithElements(idOrCondition, value) { + const where = idOrCondition === 'active' + ? 't.active = true AND t.archived = false' + : 't.id = $1'; + const params = idOrCondition === 'active' ? [] : [value]; + + const sql = ` + SELECT t.*, + COALESCE( + (SELECT json_agg(e.* ORDER BY e.id) + FROM kioskelements e WHERE e.template_id = t.id), + '[]'::json + ) AS elements + FROM kiosktemplates t + WHERE ${where} + LIMIT 1`; + const r = await query(sql, params, DB); + return r.rows[0] || null; +} + +// ════════════════════════════════════════════════════════════ +// READS +// ════════════════════════════════════════════════════════════ + +// GET /kiosk/template/active → template attivo (con elements) +router.get('/template/active', async (req, res) => { + try { + const tpl = await fetchTemplateWithElements('active'); + if (!tpl) return res.status(404).json({ error: 'no active template' }); + res.json(tpl); + } catch (err) { + console.error('[KIOSK] active error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// GET /kiosk/templates → lista (senza elements) +router.get('/templates', async (req, res) => { + try { + const r = await query( + `SELECT t.id, t.name, t.tags, t.active, t.archived, + t.created_at, t.updated_at, + (SELECT COUNT(*)::int FROM kioskelements e WHERE e.template_id = t.id) AS elements_count + FROM kiosktemplates t + ORDER BY t.updated_at DESC`, + [], DB + ); + res.json(r.rows); + } catch (err) { + console.error('[KIOSK] list error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// GET /kiosk/templates/:id → dettaglio con elements +router.get('/templates/:id', async (req, res) => { + try { + const tpl = await fetchTemplateWithElements('id', req.params.id); + if (!tpl) return res.status(404).json({ error: 'not found' }); + res.json(tpl); + } catch (err) { + console.error('[KIOSK] get error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// ════════════════════════════════════════════════════════════ +// WRITES +// ════════════════════════════════════════════════════════════ + +// POST /kiosk/templates → crea template + elements (transazionale) +router.post('/templates', async (req, res) => { + const { name, tags, elements } = req.body || {}; + if (!name || typeof name !== 'string') { + return res.status(400).json({ error: 'name required' }); + } + const tagsArr = Array.isArray(tags) ? tags.map(String) : []; + const els = Array.isArray(elements) ? elements : []; + + const client = await getClient(DB); + try { + await client.query('BEGIN'); + + // Genera id in app (no default DB). Retry se collisione (estremamente rara). + let tpl = null; + for (let attempt = 0; attempt < 5 && !tpl; attempt++) { + const id = genShortId8(); + try { + const t = await client.query( + `INSERT INTO kiosktemplates (id, name, tags, active, archived, created_at, updated_at) + VALUES ($1, $2, $3, false, false, NOW(), NOW()) RETURNING *`, + [id, name.slice(0, 50), tagsArr] + ); + tpl = t.rows[0]; + } catch (e) { + if (e.code !== '23505') throw e; // PK conflict → retry + } + } + if (!tpl) throw new Error('id generation failed'); + + const insertedEls = await insertElements(client, tpl.id, els); + await client.query('COMMIT'); + res.status(201).json({ ...tpl, elements: insertedEls }); + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + console.error('[KIOSK] create error:', err.message); + res.status(400).json({ error: err.message || 'internal error' }); + } finally { + client.release(); + } +}); + +// PUT /kiosk/templates/:id → patch metadata. Se `elements` viene passato, +// sostituisce TUTTI gli elements (delete + insert in transazione). +router.put('/templates/:id', async (req, res) => { + const { name, tags, elements } = req.body || {}; + const fields = [], values = []; + let i = 1; + + if (name !== undefined) { fields.push(`name = $${i++}`); values.push(String(name).slice(0, 50)); } + if (tags !== undefined) { + if (!Array.isArray(tags)) return res.status(400).json({ error: 'tags must be array' }); + fields.push(`tags = $${i++}`); values.push(tags.map(String)); + } + + if (!fields.length && elements === undefined) { + return res.status(400).json({ error: 'no fields' }); + } + + const client = await getClient(DB); + try { + await client.query('BEGIN'); + + let tpl; + if (fields.length) { + // Trigger set_updated_at non presente: lo facciamo manualmente. + fields.push('updated_at = NOW()'); + values.push(req.params.id); + const r = await client.query( + `UPDATE kiosktemplates SET ${fields.join(', ')} + WHERE id = $${i} RETURNING *`, + values + ); + if (!r.rows[0]) { + await client.query('ROLLBACK'); + return res.status(404).json({ error: 'not found' }); + } + tpl = r.rows[0]; + } else { + const r = await client.query(`SELECT * FROM kiosktemplates WHERE id = $1`, [req.params.id]); + if (!r.rows[0]) { + await client.query('ROLLBACK'); + return res.status(404).json({ error: 'not found' }); + } + tpl = r.rows[0]; + } + + let elsRows; + if (Array.isArray(elements)) { + await client.query(`DELETE FROM kioskelements WHERE template_id = $1`, [tpl.id]); + elsRows = await insertElements(client, tpl.id, elements); + } else { + const r = await client.query( + `SELECT * FROM kioskelements WHERE template_id = $1 ORDER BY id`, + [tpl.id] + ); + elsRows = r.rows; + } + + await client.query('COMMIT'); + res.json({ ...tpl, elements: elsRows }); + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + console.error('[KIOSK] update error:', err.message); + res.status(400).json({ error: err.message || 'internal error' }); + } finally { + client.release(); + } +}); + +// POST /kiosk/templates/:id/activate → attiva (disattiva tutti gli altri) +router.post('/templates/:id/activate', async (req, res) => { + const client = await getClient(DB); + try { + await client.query('BEGIN'); + + // pre-check: archived non puo' diventare active + const cur = await client.query( + `SELECT archived FROM kiosktemplates WHERE id = $1`, + [req.params.id] + ); + if (!cur.rows[0]) { + await client.query('ROLLBACK'); + return res.status(404).json({ error: 'not found' }); + } + if (cur.rows[0].archived) { + await client.query('ROLLBACK'); + return res.status(409).json({ error: 'cannot activate archived template' }); + } + + await client.query( + `UPDATE kiosktemplates SET active = false, updated_at = NOW() + WHERE active = true AND id <> $1`, + [req.params.id] + ); + const r = await client.query( + `UPDATE kiosktemplates SET active = true, updated_at = NOW() + WHERE id = $1 RETURNING *`, + [req.params.id] + ); + await client.query('COMMIT'); + + const tpl = r.rows[0]; + + // notifica realtime (best-effort) + const RT = process.env.REALTIME_URL || 'http://meb-realtime:3000'; + const KEY = process.env.INTERNAL_API_KEY; + if (KEY) { + fetch(`${RT}/kiosk/notify-active`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': KEY }, + body: JSON.stringify({ template: tpl }) + }).catch(() => {}); + } + + res.json(tpl); + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + console.error('[KIOSK] activate error:', err.message); + res.status(500).json({ error: 'internal error' }); + } finally { + client.release(); + } +}); + +// PATCH /kiosk/templates/:id/archive → toggle archived (e disattiva) +router.patch('/templates/:id/archive', async (req, res) => { + try { + const cur = await query( + `SELECT archived FROM kiosktemplates WHERE id = $1`, + [req.params.id], DB + ); + if (!cur.rows[0]) return res.status(404).json({ error: 'not found' }); + const willArchive = !cur.rows[0].archived; + const r = await query( + `UPDATE kiosktemplates + SET archived = $1, + active = CASE WHEN $1 = true THEN false ELSE active END, + updated_at = NOW() + WHERE id = $2 RETURNING *`, + [willArchive, req.params.id], DB + ); + res.json(r.rows[0]); + } catch (err) { + console.error('[KIOSK] archive error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// DELETE /kiosk/templates/:id +// FK CASCADE non e' garantita lato DB: cancelliamo manualmente in transazione +// prima gli elements e poi il template. +router.delete('/templates/:id', async (req, res) => { + const client = await getClient(DB); + try { + await client.query('BEGIN'); + const check = await client.query( + `SELECT active FROM kiosktemplates WHERE id = $1`, + [req.params.id] + ); + if (!check.rows[0]) { + await client.query('ROLLBACK'); + return res.status(404).json({ error: 'not found' }); + } + if (check.rows[0].active) { + await client.query('ROLLBACK'); + return res.status(409).json({ error: 'cannot delete active template' }); + } + await client.query(`DELETE FROM kioskelements WHERE template_id = $1`, [req.params.id]); + await client.query(`DELETE FROM kiosktemplates WHERE id = $1`, [req.params.id]); + await client.query('COMMIT'); + res.json({ deleted: true }); + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + console.error('[KIOSK] delete error:', err.message); + res.status(500).json({ error: 'internal error' }); + } finally { + client.release(); + } +}); + +// ════════════════════════════════════════════════════════════ +// ELEMENTS — CRUD granulare (utile per editor live) +// ════════════════════════════════════════════════════════════ + +// POST /kiosk/templates/:id/elements → aggiunge un singolo element +router.post('/templates/:id/elements', async (req, res) => { + const client = await getClient(DB); + try { + const tpl = await client.query( + `SELECT id FROM kiosktemplates WHERE id = $1`, [req.params.id] + ); + if (!tpl.rows[0]) return res.status(404).json({ error: 'template not found' }); + + const [created] = await insertElements(client, req.params.id, [req.body || {}]); + // bumpa updated_at del template + await client.query(`UPDATE kiosktemplates SET updated_at = NOW() WHERE id = $1`, [req.params.id]); + res.status(201).json(created); + } catch (err) { + console.error('[KIOSK] add element error:', err.message); + res.status(400).json({ error: err.message || 'internal error' }); + } finally { + client.release(); + } +}); + +// PUT /kiosk/templates/:id/elements/:elementId → patch element +router.put('/templates/:id/elements/:elementId', async (req, res) => { + const e = sanitizeElement(req.body || {}); + const fields = [], values = []; + let i = 1; + for (const k of ELEMENT_FIELDS) { + if (e[k] !== undefined) { fields.push(`${k} = $${i++}`); values.push(e[k]); } + } + if (!fields.length) return res.status(400).json({ error: 'no fields' }); + + values.push(req.params.elementId, req.params.id); + try { + const r = await query( + `UPDATE kioskelements SET ${fields.join(', ')} + WHERE id = $${i++} AND template_id = $${i} RETURNING *`, + values, DB + ); + if (!r.rows[0]) return res.status(404).json({ error: 'not found' }); + await query(`UPDATE kiosktemplates SET updated_at = NOW() WHERE id = $1`, [req.params.id], DB); + res.json(r.rows[0]); + } catch (err) { + console.error('[KIOSK] update element error:', err.message); + res.status(400).json({ error: err.message || 'internal error' }); + } +}); + +// DELETE /kiosk/templates/:id/elements/:elementId +router.delete('/templates/:id/elements/:elementId', async (req, res) => { + try { + const r = await query( + `DELETE FROM kioskelements + WHERE id = $1 AND template_id = $2 RETURNING id`, + [req.params.elementId, req.params.id], DB + ); + if (!r.rows[0]) return res.status(404).json({ error: 'not found' }); + await query(`UPDATE kiosktemplates SET updated_at = NOW() WHERE id = $1`, [req.params.id], DB); + res.json({ deleted: true }); + } catch (err) { + console.error('[KIOSK] delete element error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +module.exports = router; diff --git a/api/src/routes/kiosk.public.js b/api/src/routes/kiosk.public.js new file mode 100644 index 0000000..0be47ab --- /dev/null +++ b/api/src/routes/kiosk.public.js @@ -0,0 +1,49 @@ +const router = require('express').Router(); +const { query } = require('../storage/postgres'); + +// Endpoint pubblici (usati dal plugin kiosk sulla barca). Solo read. +// Restituisce template + array `elements`. + +const DB = 'kiosk'; + +const SELECT_WITH_ELEMENTS = ` + SELECT t.*, + COALESCE( + (SELECT json_agg(e.* ORDER BY e.id) + FROM kioskelements e WHERE e.template_id = t.id), + '[]'::json + ) AS elements + FROM kiosktemplates t +`; + +router.get('/template/active', async (req, res) => { + try { + const r = await query( + `${SELECT_WITH_ELEMENTS} + WHERE t.active = true AND t.archived = false + LIMIT 1`, + [], DB + ); + if (!r.rows[0]) return res.status(404).json({ error: 'no active template' }); + res.json(r.rows[0]); + } catch (err) { + console.error('[KIOSK/PUB] active error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +router.get('/templates/:id', async (req, res) => { + try { + const r = await query( + `${SELECT_WITH_ELEMENTS} WHERE t.id = $1 LIMIT 1`, + [req.params.id], DB + ); + if (!r.rows[0]) return res.status(404).json({ error: 'not found' }); + res.json(r.rows[0]); + } catch (err) { + console.error('[KIOSK/PUB] get error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +module.exports = router; diff --git a/api/src/routes/kiosk.sensor.js b/api/src/routes/kiosk.sensor.js new file mode 100644 index 0000000..5b89839 --- /dev/null +++ b/api/src/routes/kiosk.sensor.js @@ -0,0 +1,67 @@ +const router = require('express').Router(); +const crypto = require('crypto'); +const { query } = require('../storage/postgres'); + +const DB = 'kiosk'; + +function hash(code) { return crypto.createHash('sha256').update(code).digest('hex'); } + +async function authSensor(req, res, next) { + try { + const r = await query( + 'SELECT id, active FROM sensors WHERE code_hash = $1', + [hash(req.params.sensorCode)], 'sensors' + ); + if (!r.rows[0]) return res.status(401).json({ error: 'invalid sensor code' }); + if (!r.rows[0].active) return res.status(403).json({ error: 'sensor inactive' }); + req.sensorId = r.rows[0].id; + next(); + } catch (err) { + console.error('[KIOSK/SENSOR] auth error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +} + +const SELECT_WITH_ELEMENTS = ` + SELECT t.*, + COALESCE( + (SELECT json_agg(e.* ORDER BY e.id) + FROM kioskelements e WHERE e.template_id = t.id), + '[]'::json + ) AS elements + FROM kiosktemplates t +`; + +// GET /kiosk/sensor/:sensorCode/template/active +router.get('/:sensorCode/template/active', authSensor, async (req, res) => { + try { + const r = await query( + `${SELECT_WITH_ELEMENTS} + WHERE t.active = true AND t.archived = false + LIMIT 1`, + [], DB + ); + if (!r.rows[0]) return res.status(404).json({ error: 'no active template' }); + res.json(r.rows[0]); + } catch (err) { + console.error('[KIOSK/SENSOR] active error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// GET /kiosk/sensor/:sensorCode/templates/:id +router.get('/:sensorCode/templates/:id', authSensor, async (req, res) => { + try { + const r = await query( + `${SELECT_WITH_ELEMENTS} WHERE t.id = $1 LIMIT 1`, + [req.params.id], DB + ); + if (!r.rows[0]) return res.status(404).json({ error: 'not found' }); + res.json(r.rows[0]); + } catch (err) { + console.error('[KIOSK/SENSOR] get error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +module.exports = router; diff --git a/api/src/routes/marine.datasets.js b/api/src/routes/marine.datasets.js new file mode 100644 index 0000000..820f11c --- /dev/null +++ b/api/src/routes/marine.datasets.js @@ -0,0 +1,232 @@ +/** + * /marine/datasets — CRUD sulla tabella `datasets` del database `ml`. + * + * Layout: + * POST /marine/datasets/upload (multipart: file + metadata JSON) + * GET /marine/datasets (query: ?tags=a,b&type=copernicus&mine=1) + * GET /marine/datasets/:id (metadata) + * GET /marine/datasets/:id/download (presigned URL 1h) + * GET /marine/datasets/:id/raw (stream diretto) + * PATCH /marine/datasets/:id (aggiorna nome/tags/notes) + * DELETE /marine/datasets/:id (rimuove da MinIO + DB) + * + * I file vivono SEMPRE nel bucket MinIO "ml.datasets". La colonna `file_key` salva + * il nome dell'oggetto (basta `${uuid}.${ext}`, senza prefissi). + */ + +const express = require('express'); +const multer = require('multer'); +const { randomUUID } = require('crypto'); + +const { query } = require('../storage/postgres'); +const { bucketExists, upload, download, removeObject, getFileStream } = require('../storage/minio'); + +const router = express.Router(); +const upload_mw = multer({ storage: multer.memoryStorage(), limits: { fileSize: 500 * 1024 * 1024 } }); + +// Bucket MinIO fisso per tutti i dataset. +const BUCKET = 'ml.datasets'; + +const parseTags = (s) => (s ? String(s).split(',').map(x => x.trim()).filter(Boolean) : []); + +function rowToDataset(r) { + return { + id: r.id, + file_key: r.file_key, + nome: r.nome, + description: r.description, + tags: r.tags, + type: r.type, + format: r.format, + notes: r.notes, + created_by: r.created_by, + created_at: r.created_at, + updated_at: r.updated_at, + size_bytes: Number(r.size_bytes), + row_count: r.row_count != null ? Number(r.row_count) : null, + columns: r.columns, + copernicus_id: r.copernicus_id, + variables: r.variables, + variable_renames: r.variable_renames, + bbox: r.bbox, + start_date: r.start_date, + end_date: r.end_date, + params: r.params, + version: r.version, + }; +} + +// ── LIST ───────────────────────────────────────────────────────────────── +router.get('/', async (req, res) => { + try { + const filters = []; + const params = []; + if (req.query.type) { + params.push(req.query.type); + filters.push(`type = $${params.length}`); + } + if (req.query.tags) { + const tags = parseTags(req.query.tags); + if (tags.length) { + params.push(tags); + filters.push(`tags && $${params.length}`); + } + } + if (req.query.mine === '1' && req.user?.username) { + params.push(req.user.username); + filters.push(`created_by = $${params.length}`); + } + if (req.query.search) { + params.push(`%${req.query.search}%`); + filters.push(`(nome ILIKE $${params.length} OR description ILIKE $${params.length})`); + } + const where = filters.length ? `WHERE ${filters.join(' AND ')}` : ''; + const r = await query( + `SELECT * FROM datasets ${where} ORDER BY created_at DESC LIMIT 500`, + params, 'ml' + ); + res.json({ count: r.rows.length, datasets: r.rows.map(rowToDataset) }); + } catch (e) { + console.error('[marine/datasets list]', e); + res.status(500).json({ error: e.message }); + } +}); + +// ── UPLOAD (usato dal servizio copernicus) ─────────────────────────────── +router.post('/upload', upload_mw.single('file'), async (req, res) => { + try { + if (!req.file) return res.status(400).json({ error: 'file required (multipart field "file")' }); + + let meta = {}; + try { meta = JSON.parse(req.body.metadata || '{}'); } catch { /* ignore */ } + + const fmt = (meta.type && ['csv', 'json', 'netcdf'].includes(meta.type)) ? meta.type : 'csv'; + const id = randomUUID(); + const ext = fmt === 'netcdf' ? 'nc' : fmt; + const fileKey = `${id}.${ext}`; + + await bucketExists(BUCKET); + await upload(BUCKET, fileKey, req.file.buffer, req.file.size, req.file.mimetype || 'application/octet-stream'); + + const createdBy = req.user?.username || meta.created_by || 'unknown'; + const insert = await query( + `INSERT INTO datasets ( + id, file_key, nome, description, tags, type, format, notes, + created_by, size_bytes, copernicus_id, variables, variable_renames, + bbox, start_date, end_date, params + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17) + RETURNING *`, + [ + id, + fileKey, + meta.nome || req.file.originalname || fileKey, + meta.description || null, + Array.isArray(meta.tags) ? meta.tags : [], + 'copernicus', + fmt, + meta.notes || null, + createdBy, + req.file.size, + meta.copernicus_id || meta.copernicus_dataset_id || null, + Array.isArray(meta.variables) ? meta.variables : null, + meta.variable_renames ? JSON.stringify(meta.variable_renames) : null, + meta.bbox ? JSON.stringify(meta.bbox) : null, + meta.start_date || null, + meta.end_date || null, + JSON.stringify(meta.params || {}), + ], + 'ml' + ); + res.status(201).json(rowToDataset(insert.rows[0])); + } catch (e) { + console.error('[marine/datasets upload]', e); + res.status(500).json({ error: e.message }); + } +}); + +// ── DETAIL ─────────────────────────────────────────────────────────────── +router.get('/:id', async (req, res) => { + try { + const r = await query(`SELECT * FROM datasets WHERE id = $1`, [req.params.id], 'ml'); + if (!r.rows.length) return res.status(404).json({ error: 'not found' }); + res.json(rowToDataset(r.rows[0])); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +// ── DOWNLOAD presigned ─────────────────────────────────────────────────── +router.get('/:id/download', async (req, res) => { + try { + const r = await query(`SELECT file_key FROM datasets WHERE id = $1`, [req.params.id], 'ml'); + if (!r.rows.length) return res.status(404).json({ error: 'not found' }); + const { file_key } = r.rows[0]; + const url = await download(BUCKET, file_key, 3600); + res.json({ url, expires_in: 3600 }); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +// ── STREAM raw (per download diretto dal browser) ──────────────────────── +router.get('/:id/raw', async (req, res) => { + try { + const r = await query(`SELECT file_key, nome, format FROM datasets WHERE id = $1`, [req.params.id], 'ml'); + if (!r.rows.length) return res.status(404).json({ error: 'not found' }); + const { file_key, nome, format } = r.rows[0]; + const mime = format === 'json' ? 'application/json' : format === 'csv' ? 'text/csv' : 'application/octet-stream'; + const ext = format === 'netcdf' ? 'nc' : format; + res.setHeader('Content-Type', mime); + res.setHeader('Content-Disposition', `attachment; filename="${encodeURIComponent(nome)}.${ext}"`); + const stream = await getFileStream(BUCKET, file_key); + stream.on('error', (err) => { console.error(err); res.end(); }); + stream.pipe(res); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +// ── PATCH metadata ─────────────────────────────────────────────────────── +router.patch('/:id', async (req, res) => { + try { + const allowed = ['nome', 'description', 'tags', 'notes']; + const sets = []; + const params = []; + for (const k of allowed) { + if (k in req.body) { + params.push(req.body[k]); + sets.push(`${k} = $${params.length}`); + } + } + if (!sets.length) return res.status(400).json({ error: 'no fields to update' }); + + // Trigger updated_at non presente nel DB: lo aggiorniamo manualmente. + sets.push('updated_at = NOW()'); + + params.push(req.params.id); + const r = await query( + `UPDATE datasets SET ${sets.join(', ')} WHERE id = $${params.length} RETURNING *`, + params, 'ml' + ); + if (!r.rows.length) return res.status(404).json({ error: 'not found' }); + res.json(rowToDataset(r.rows[0])); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +// ── DELETE ─────────────────────────────────────────────────────────────── +router.delete('/:id', async (req, res) => { + try { + const r = await query(`SELECT file_key FROM datasets WHERE id = $1`, [req.params.id], 'ml'); + if (!r.rows.length) return res.status(404).json({ error: 'not found' }); + const { file_key } = r.rows[0]; + try { await removeObject(BUCKET, file_key); } catch (e) { console.warn('[minio remove]', e.message); } + await query(`DELETE FROM datasets WHERE id = $1`, [req.params.id], 'ml'); + res.status(204).end(); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +module.exports = router; diff --git a/api/src/routes/pageconnections.js b/api/src/routes/pageconnections.js new file mode 100644 index 0000000..7ba6686 --- /dev/null +++ b/api/src/routes/pageconnections.js @@ -0,0 +1,100 @@ +/** + * /pageconnections — registro sessioni di pagina attive, con heartbeat. + * + * Storage Redis: + * pageconn:{page} ZSET score=lastPing, member=session_id + * pageconn:meta:{session_id} HASH {page, user_id, created_at} + * + * Limiti: + * page = "test" → max 2 session_id attive (entro TTL heartbeat). Altrimenti 429. + */ + +const express = require('express'); +const Redis = require('ioredis'); + +const router = express.Router(); + +const redis = new Redis({ + host: process.env.REDIS_HOST || 'meb-redis', + port: parseInt(process.env.REDIS_PORT || '6379', 10), +}); + +const HEARTBEAT_TTL_SEC = 30; +const LIMITS = { test: 2 }; + +function nowSec() { return Math.floor(Date.now() / 1000); } + +async function activeMembers(page) { + const min = nowSec() - HEARTBEAT_TTL_SEC; + // rimuovi stale + await redis.zremrangebyscore(`pageconn:${page}`, '-inf', `(${min}`); + return redis.zrange(`pageconn:${page}`, 0, -1); +} + +router.post('/', async (req, res) => { + try { + const { page, session_id, user_id } = req.body || {}; + if (!page || !session_id) return res.status(400).json({ error: 'page and session_id required' }); + + const active = await activeMembers(page); + const limit = LIMITS[page]; + if (limit && !active.includes(session_id) && active.length >= limit) { + return res.status(429).json({ error: 'slot full', active: active.length, limit }); + } + const ts = nowSec(); + await redis.zadd(`pageconn:${page}`, ts, session_id); + await redis.hset(`pageconn:meta:${session_id}`, { + page, + user_id: user_id || req.user?.username || 'unknown', + created_at: String(ts), + }); + await redis.expire(`pageconn:meta:${session_id}`, HEARTBEAT_TTL_SEC * 4); + res.status(201).json({ session_id, page, active: (await activeMembers(page)).length, limit: limit || null }); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +router.post('/:sid/ping', async (req, res) => { + try { + const sid = req.params.sid; + const meta = await redis.hgetall(`pageconn:meta:${sid}`); + if (!meta || !meta.page) return res.status(404).json({ error: 'session not found' }); + const ts = nowSec(); + await redis.zadd(`pageconn:${meta.page}`, ts, sid); + await redis.expire(`pageconn:meta:${sid}`, HEARTBEAT_TTL_SEC * 4); + res.json({ ok: true, ts }); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +router.delete('/:sid', async (req, res) => { + try { + const sid = req.params.sid; + const meta = await redis.hgetall(`pageconn:meta:${sid}`); + if (meta && meta.page) { + await redis.zrem(`pageconn:${meta.page}`, sid); + } + await redis.del(`pageconn:meta:${sid}`); + res.status(204).end(); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +router.get('/:page', async (req, res) => { + try { + const members = await activeMembers(req.params.page); + res.json({ + page: req.params.page, + active: members.length, + limit: LIMITS[req.params.page] || null, + sessions: members, + }); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +module.exports = router; diff --git a/api/src/routes/queue.js b/api/src/routes/queue.js new file mode 100644 index 0000000..1d5480c --- /dev/null +++ b/api/src/routes/queue.js @@ -0,0 +1,35 @@ +/** + * /queue — stato delle code (solo read). La coda vera è gestita in Redis + * dai servizi esecutori (ml-service per `train`). Qui aggreghiamo lo stato + * leggendo la tabella `jobs`. + */ +const express = require('express'); +const { query } = require('../storage/postgres'); + +const router = express.Router(); + +router.get('/', async (req, res) => { + try { + const type = req.query.type || 'train'; + const r = await query( + `SELECT id, type, status, created_by, created_at, started_at + FROM jobs + WHERE type = $1 AND status IN ('queued','running') + ORDER BY created_at ASC`, + [type], 'ml' + ); + const queued = r.rows.filter(x => x.status === 'queued'); + const running = r.rows.filter(x => x.status === 'running'); + res.json({ + type, + queued_count: queued.length, + running_count: running.length, + queued, + running, + }); + } catch (e) { + res.status(500).json({ error: e.message }); + } +}); + +module.exports = router; diff --git a/api/src/routes/rules.js b/api/src/routes/rules.js new file mode 100644 index 0000000..434a736 --- /dev/null +++ b/api/src/routes/rules.js @@ -0,0 +1,609 @@ +/** + * Rulesets API + * Base: /rules + * + * Tipi supportati: logs | forecast_current | forecast_hourly | marine_current | marine_hourly + * + * Un ruleset ha: + * - version {major, build, patch} (interi 1..100, unici per tipo) + * - description, tags[] + * - items JSONB: array di { ref, path, enabled, meta } + * ref: identificatore STABILE scelto dall'utente (chiave logica, usata come tag Influx) + * path: SK path (logs) o codice openmeteo (forecast/marine) + * meta: libero (unit, measurement, sk_path, name, group_name, category, ...) + * - active (un solo attivo per tipo), archived + * + * Deploy: POST /rules/:type/:id/deploy { sensors: [name,...] } + * -> salva in ruleset_deployments + * -> notifica il servizio realtime (HTTP interno) che fara' il push WS al plugin + */ + +const router = require('express').Router(); +const crypto = require('crypto'); +const { query, getClient } = require('../storage/postgres'); + +const DB = 'rules'; +const VALID_TYPES = ['logs', 'forecast_current', 'forecast_hourly', 'marine_current', 'marine_hourly']; + +// Sostituisce il default DB `gen_random_uuid()` (estensione pgcrypto non presente). +function genUUID() { + return crypto.randomUUID(); +} + +function isValidType(t) { return VALID_TYPES.includes(t); } + +function parseVersion(body) { + // accetta sia { version_major, version_build, version_patch } che { version: "1.0.0" } + let M = body?.version_major, B = body?.version_build, P = body?.version_patch; + if (M === undefined && typeof body?.version === 'string') { + const m = body.version.match(/^(\d+)\.(\d+)\.(\d+)$/); + if (m) { M = +m[1]; B = +m[2]; P = +m[3]; } + } + const toInt = v => (v === undefined || v === null || v === '') ? null : parseInt(v, 10); + M = toInt(M); B = toInt(B); P = toInt(P); + return { M, B, P }; +} + +function validVersionPart(n, min = 0) { + return Number.isInteger(n) && n >= min && n <= 100; +} + +function validateItems(items) { + if (!Array.isArray(items)) return 'items must be an array'; + const refs = new Set(); + for (const it of items) { + if (!it || typeof it !== 'object') return 'each item must be an object'; + if (!it.ref || typeof it.ref !== 'string') return 'each item needs a non-empty "ref"'; + if (refs.has(it.ref)) return `duplicate ref "${it.ref}"`; + refs.add(it.ref); + if (it.path !== undefined && typeof it.path !== 'string') return `item "${it.ref}" has invalid path`; + if (it.enabled !== undefined && typeof it.enabled !== 'boolean') return `item "${it.ref}" enabled must be boolean`; + if (it.meta !== undefined && (typeof it.meta !== 'object' || it.meta === null || Array.isArray(it.meta))) { + return `item "${it.ref}" meta must be an object`; + } + } + return null; +} + +function normalizeItems(items) { + return (items || []).map(it => ({ + ref: String(it.ref), + path: it.path != null ? String(it.path) : '', + enabled: it.enabled === undefined ? true : !!it.enabled, + meta: it.meta && typeof it.meta === 'object' ? it.meta : {} + })); +} + +function rowToRuleset(row) { + if (!row) return null; + return { + id: row.id, + type: row.type, + version: { + major: row.version_major, + build: row.version_build, + patch: row.version_patch, + str: `${row.version_major}.${row.version_build}.${row.version_patch}` + }, + description: row.description, + tags: row.tags || [], + items: row.items || [], + active: row.active, + archived: row.archived, + created_at: row.created_at, + updated_at: row.updated_at + }; +} + +async function logChange(rulesetId, type, action, userId, payload) { + try { + await query( + `INSERT INTO ruleset_changes (ruleset_id, type, action, user_id, payload) + VALUES ($1, $2, $3, $4, $5)`, + [rulesetId, type, action, userId || null, payload ? JSON.stringify(payload) : null], + DB + ); + } catch (e) { + console.error('[RULES] audit log error:', e.message); + } +} + +/* ══════════════════════════════════════════════════════════════ + SENSORS (deve stare PRIMA delle route /:type/:id per via del routing Express) + ══════════════════════════════════════════════════════════════ */ + +// GET /rules/-/sensors → lista sensori disponibili (dal DB sensors) +router.get('/-/sensors', async (req, res) => { + try { + const r = await query(`SELECT name, created_at FROM sensors ORDER BY name`, [], 'sensors'); + res.json(r.rows); + } catch (err) { + console.error('[RULES] sensors list error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +/* ══════════════════════════════════════════════════════════════ + READS + ══════════════════════════════════════════════════════════════ */ + +// GET /rules → { logs:[], forecast_current:[], ... } (lista completa, senza items per leggerezza) +router.get('/', async (req, res) => { + try { + const r = await query( + `SELECT id, type, version_major, version_build, version_patch, + description, tags, active, archived, created_at, updated_at, + jsonb_array_length(items) AS items_count + FROM rulesets + ORDER BY type, version_major DESC, version_build DESC, version_patch DESC`, + [], DB + ); + const grouped = Object.fromEntries(VALID_TYPES.map(t => [t, []])); + for (const row of r.rows) { + grouped[row.type].push({ + ...rowToRuleset(row), + items: undefined, + items_count: row.items_count + }); + } + res.json(grouped); + } catch (err) { + console.error('[RULES] list error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// GET /rules/:type → lista versioni del tipo +router.get('/:type', async (req, res) => { + const { type } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + try { + const r = await query( + `SELECT * FROM rulesets WHERE type = $1 + ORDER BY version_major DESC, version_build DESC, version_patch DESC`, + [type], DB + ); + res.json(r.rows.map(rowToRuleset)); + } catch (err) { + console.error('[RULES] list type error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// GET /rules/:type/:id +router.get('/:type/:id', async (req, res) => { + const { type, id } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + try { + const r = await query(`SELECT * FROM rulesets WHERE id = $1 AND type = $2`, [id, type], DB); + if (!r.rows[0]) return res.status(404).json({ error: 'not found' }); + res.json(rowToRuleset(r.rows[0])); + } catch (err) { + console.error('[RULES] get error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// GET /rules/:type/active → ruleset attivo +router.get('/:type/-/active', async (req, res) => { + const { type } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + try { + const r = await query( + `SELECT * FROM rulesets WHERE type = $1 AND active = true AND archived = false LIMIT 1`, + [type], DB + ); + if (!r.rows[0]) return res.status(404).json({ error: 'no active ruleset' }); + res.json(rowToRuleset(r.rows[0])); + } catch (err) { + console.error('[RULES] active error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +/* ══════════════════════════════════════════════════════════════ + WRITES + ══════════════════════════════════════════════════════════════ */ + +// POST /rules/:type +router.post('/:type', async (req, res) => { + const { type } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + + let { M, B, P } = parseVersion(req.body); + if (M === null) M = 1; + if (B === null) B = 0; + if (P === null) P = 0; + if (!validVersionPart(M, 1) || !validVersionPart(B) || !validVersionPart(P)) { + return res.status(400).json({ error: 'version parts must be integers (major 1..100, build/patch 0..100)' }); + } + + const description = typeof req.body?.description === 'string' ? req.body.description : ''; + const tags = Array.isArray(req.body?.tags) ? req.body.tags.map(String) : []; + const items = normalizeItems(req.body?.items); + const itemsErr = validateItems(items); + if (itemsErr) return res.status(400).json({ error: itemsErr }); + + try { + const newId = genUUID(); + const r = await query( + `INSERT INTO rulesets (id, type, version_major, version_build, version_patch, + description, tags, items, active, archived, + created_at, updated_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,false,false,NOW(),NOW()) RETURNING *`, + [newId, type, M, B, P, description, tags, JSON.stringify(items)], + DB + ); + const rs = rowToRuleset(r.rows[0]); + logChange(rs.id, type, 'created', req.user?.user_id, { version: rs.version.str }); + res.status(201).json(rs); + } catch (err) { + if (err.code === '23505') { + return res.status(409).json({ error: 'version already exists for this type' }); + } + console.error('[RULES] create error:', err.message); + res.status(500).json({ error: err.message || 'internal error' }); + } +}); + +// PUT /rules/:type/:id → update campi (version, description, tags, items) +router.put('/:type/:id', async (req, res) => { + const { type, id } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + + const fields = []; + const values = []; + let i = 1; + + if (req.body?.description !== undefined) { + fields.push(`description = $${i++}`); values.push(String(req.body.description)); + } + if (req.body?.tags !== undefined) { + if (!Array.isArray(req.body.tags)) return res.status(400).json({ error: 'tags must be array' }); + fields.push(`tags = $${i++}`); values.push(req.body.tags.map(String)); + } + if (req.body?.items !== undefined) { + const items = normalizeItems(req.body.items); + const itemsErr = validateItems(items); + if (itemsErr) return res.status(400).json({ error: itemsErr }); + fields.push(`items = $${i++}`); values.push(JSON.stringify(items)); + } + if (req.body?.version_major !== undefined || req.body?.version_build !== undefined || + req.body?.version_patch !== undefined || req.body?.version !== undefined) { + const { M, B, P } = parseVersion(req.body); + if (!validVersionPart(M, 1) || !validVersionPart(B) || !validVersionPart(P)) { + return res.status(400).json({ error: 'version parts must be integers (major 1..100, build/patch 0..100)' }); + } + fields.push(`version_major = $${i++}`); values.push(M); + fields.push(`version_build = $${i++}`); values.push(B); + fields.push(`version_patch = $${i++}`); values.push(P); + } + + if (!fields.length) return res.status(400).json({ error: 'no fields to update' }); + + // Trigger set_updated_at non presente: lo facciamo manualmente. + fields.push('updated_at = NOW()'); + values.push(id, type); + try { + const r = await query( + `UPDATE rulesets SET ${fields.join(', ')} + WHERE id = $${i++} AND type = $${i} + RETURNING *`, + values, DB + ); + if (!r.rows[0]) return res.status(404).json({ error: 'not found' }); + const rs = rowToRuleset(r.rows[0]); + logChange(rs.id, type, 'updated', req.user?.user_id, null); + res.json(rs); + } catch (err) { + if (err.code === '23505') { + return res.status(409).json({ error: 'version already exists for this type' }); + } + console.error('[RULES] update error:', err.message); + res.status(500).json({ error: err.message || 'internal error' }); + } +}); + +// PATCH /rules/:type/:id/active → toggle (deattiva le altre dello stesso tipo) +router.patch('/:type/:id/active', async (req, res) => { + const { type, id } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + + const client = await getClient(DB); + try { + await client.query('BEGIN'); + const cur = await client.query( + `SELECT active, archived FROM rulesets WHERE id = $1 AND type = $2`, + [id, type] + ); + if (!cur.rows[0]) { await client.query('ROLLBACK'); return res.status(404).json({ error: 'not found' }); } + if (cur.rows[0].archived && !cur.rows[0].active) { + await client.query('ROLLBACK'); + return res.status(409).json({ error: 'cannot activate an archived ruleset' }); + } + const willActivate = !cur.rows[0].active; + if (willActivate) { + await client.query( + `UPDATE rulesets SET active = false, updated_at = NOW() + WHERE type = $1 AND active = true`, + [type] + ); + } + const r = await client.query( + `UPDATE rulesets SET active = $1, updated_at = NOW() + WHERE id = $2 RETURNING *`, + [willActivate, id] + ); + await client.query('COMMIT'); + logChange(id, type, willActivate ? 'activated' : 'deactivated', req.user?.user_id, null); + res.json({ active: r.rows[0].active, ruleset: rowToRuleset(r.rows[0]) }); + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + console.error('[RULES] active error:', err.message); + res.status(500).json({ error: 'internal error' }); + } finally { + client.release(); + } +}); + +// PATCH /rules/:type/:id/archive → toggle +router.patch('/:type/:id/archive', async (req, res) => { + const { type, id } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + try { + const cur = await query(`SELECT archived, active FROM rulesets WHERE id = $1 AND type = $2`, [id, type], DB); + if (!cur.rows[0]) return res.status(404).json({ error: 'not found' }); + const willArchive = !cur.rows[0].archived; + // archiviare implica disattivare + const r = await query( + `UPDATE rulesets SET archived = $1, + active = CASE WHEN $1 = true THEN false ELSE active END, + updated_at = NOW() + WHERE id = $2 RETURNING *`, + [willArchive, id], DB + ); + logChange(id, type, willArchive ? 'archived' : 'unarchived', req.user?.user_id, null); + res.json({ archived: r.rows[0].archived, ruleset: rowToRuleset(r.rows[0]) }); + } catch (err) { + console.error('[RULES] archive error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// DELETE /rules/:type/:id +// FK CASCADE non e' garantita lato DB: cancelliamo manualmente in transazione +// prima i deployments e poi il ruleset. ruleset_changes (audit) viene preservato +// volutamente — annulliamo solo ruleset_id se necessario. +router.delete('/:type/:id', async (req, res) => { + const { type, id } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + const client = await getClient(DB); + try { + await client.query('BEGIN'); + const cur = await client.query( + `SELECT active FROM rulesets WHERE id = $1 AND type = $2`, + [id, type] + ); + if (!cur.rows[0]) { + await client.query('ROLLBACK'); + return res.status(404).json({ error: 'not found' }); + } + if (cur.rows[0].active) { + await client.query('ROLLBACK'); + return res.status(409).json({ error: 'cannot delete active ruleset' }); + } + await client.query(`DELETE FROM ruleset_deployments WHERE ruleset_id = $1`, [id]); + await client.query(`DELETE FROM rulesets WHERE id = $1`, [id]); + await client.query('COMMIT'); + logChange(id, type, 'deleted', req.user?.user_id, null); + res.json({ deleted: true }); + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + console.error('[RULES] delete error:', err.message); + res.status(500).json({ error: 'internal error' }); + } finally { + client.release(); + } +}); + +/* ══════════════════════════════════════════════════════════════ + ITEMS: helper endpoints (comodi per la UI) + ══════════════════════════════════════════════════════════════ */ + +// POST /rules/:type/:id/items → aggiungi item +router.post('/:type/:id/items', async (req, res) => { + const { type, id } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + try { + const cur = await query(`SELECT items FROM rulesets WHERE id = $1 AND type = $2`, [id, type], DB); + if (!cur.rows[0]) return res.status(404).json({ error: 'not found' }); + const items = cur.rows[0].items || []; + const newItem = normalizeItems([req.body || {}])[0]; + if (!newItem.ref) return res.status(400).json({ error: 'ref required' }); + if (items.some(it => it.ref === newItem.ref)) { + return res.status(409).json({ error: `ref "${newItem.ref}" already exists` }); + } + items.push(newItem); + await query(`UPDATE rulesets SET items = $1, updated_at = NOW() WHERE id = $2`, [JSON.stringify(items), id], DB); + logChange(id, type, 'item_added', req.user?.user_id, { ref: newItem.ref }); + res.status(201).json(newItem); + } catch (err) { + console.error('[RULES] add item error:', err.message); + res.status(500).json({ error: err.message || 'internal error' }); + } +}); + +// PUT /rules/:type/:id/items/:ref → patch item (per ref) +router.put('/:type/:id/items/:ref', async (req, res) => { + const { type, id, ref } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + try { + const cur = await query(`SELECT items FROM rulesets WHERE id = $1 AND type = $2`, [id, type], DB); + if (!cur.rows[0]) return res.status(404).json({ error: 'not found' }); + const items = cur.rows[0].items || []; + const idx = items.findIndex(it => it.ref === ref); + if (idx < 0) return res.status(404).json({ error: 'item not found' }); + + const body = req.body || {}; + const newRef = body.ref !== undefined ? String(body.ref) : items[idx].ref; + if (newRef !== ref && items.some(it => it.ref === newRef)) { + return res.status(409).json({ error: `ref "${newRef}" already exists` }); + } + items[idx] = { + ref: newRef, + path: body.path !== undefined ? String(body.path) : items[idx].path, + enabled: body.enabled !== undefined ? !!body.enabled : items[idx].enabled, + meta: body.meta !== undefined + ? (body.meta && typeof body.meta === 'object' && !Array.isArray(body.meta) ? body.meta : {}) + : { ...(items[idx].meta || {}), ...Object.fromEntries( + Object.entries(body).filter(([k]) => !['ref','path','enabled','meta'].includes(k)) + ) } + }; + await query(`UPDATE rulesets SET items = $1, updated_at = NOW() WHERE id = $2`, [JSON.stringify(items), id], DB); + logChange(id, type, 'item_updated', req.user?.user_id, { ref: newRef }); + res.json(items[idx]); + } catch (err) { + console.error('[RULES] update item error:', err.message); + res.status(500).json({ error: err.message || 'internal error' }); + } +}); + +// PATCH /rules/:type/:id/items/:ref/toggle +router.patch('/:type/:id/items/:ref/toggle', async (req, res) => { + const { type, id, ref } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + try { + const cur = await query(`SELECT items FROM rulesets WHERE id = $1 AND type = $2`, [id, type], DB); + if (!cur.rows[0]) return res.status(404).json({ error: 'not found' }); + const items = cur.rows[0].items || []; + const idx = items.findIndex(it => it.ref === ref); + if (idx < 0) return res.status(404).json({ error: 'item not found' }); + items[idx].enabled = !items[idx].enabled; + await query(`UPDATE rulesets SET items = $1, updated_at = NOW() WHERE id = $2`, [JSON.stringify(items), id], DB); + res.json({ enabled: items[idx].enabled }); + } catch (err) { + console.error('[RULES] toggle item error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// DELETE /rules/:type/:id/items/:ref +router.delete('/:type/:id/items/:ref', async (req, res) => { + const { type, id, ref } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + try { + const cur = await query(`SELECT items FROM rulesets WHERE id = $1 AND type = $2`, [id, type], DB); + if (!cur.rows[0]) return res.status(404).json({ error: 'not found' }); + const items = (cur.rows[0].items || []).filter(it => it.ref !== ref); + await query(`UPDATE rulesets SET items = $1, updated_at = NOW() WHERE id = $2`, [JSON.stringify(items), id], DB); + res.json({ deleted: true }); + } catch (err) { + console.error('[RULES] delete item error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +/* ══════════════════════════════════════════════════════════════ + SENSORS & DEPLOYMENT + ══════════════════════════════════════════════════════════════ */ + +// GET /rules/:type/:id/deployments → sensori su cui e' deployato +router.get('/:type/:id/deployments', async (req, res) => { + const { type, id } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + try { + const r = await query( + `SELECT sensor_name, deployed_at, acked_at + FROM ruleset_deployments + WHERE ruleset_id = $1 AND type = $2 + ORDER BY deployed_at DESC`, + [id, type], DB + ); + res.json(r.rows); + } catch (err) { + console.error('[RULES] deployments error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +// POST /rules/:type/:id/deploy { sensors: [name, ...] } +// Registra il deploy e notifica il servizio realtime, che fara' il push WS al plugin. +router.post('/:type/:id/deploy', async (req, res) => { + const { type, id } = req.params; + if (!isValidType(type)) return res.status(400).json({ error: 'invalid type' }); + const sensors = Array.isArray(req.body?.sensors) ? req.body.sensors.map(String).filter(Boolean) : []; + if (!sensors.length) return res.status(400).json({ error: 'sensors array required' }); + + try { + const rs = await query(`SELECT * FROM rulesets WHERE id = $1 AND type = $2`, [id, type], DB); + if (!rs.rows[0]) return res.status(404).json({ error: 'not found' }); + if (rs.rows[0].archived) return res.status(409).json({ error: 'cannot deploy archived ruleset' }); + + const ruleset = rowToRuleset(rs.rows[0]); + + // upsert deployments + for (const name of sensors) { + await query( + `INSERT INTO ruleset_deployments (sensor_name, type, ruleset_id, deployed_at, acked_at) + VALUES ($1, $2, $3, NOW(), NULL) + ON CONFLICT (sensor_name, type) DO UPDATE + SET ruleset_id = EXCLUDED.ruleset_id, + deployed_at = NOW(), + acked_at = NULL`, + [name, type, id], DB + ); + } + + // notifica realtime (best-effort) + const RT = process.env.REALTIME_URL || 'http://meb-realtime:3000'; + const KEY = process.env.INTERNAL_API_KEY; + const results = { pushed: [], offline: [], errors: [] }; + if (KEY) { + try { + const r = await fetch(`${RT}/rules/push`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': KEY }, + body: JSON.stringify({ sensors, type, ruleset }) + }); + if (r.ok) { + const j = await r.json().catch(() => ({})); + Object.assign(results, j); + } else { + results.errors.push(`realtime HTTP ${r.status}`); + } + } catch (e) { + results.errors.push(`realtime unreachable: ${e.message}`); + } + } else { + results.errors.push('INTERNAL_API_KEY missing'); + } + + logChange(id, type, 'deployed', req.user?.user_id, { sensors, results }); + res.json({ deployed: sensors, ...results }); + } catch (err) { + console.error('[RULES] deploy error:', err.message); + res.status(500).json({ error: err.message || 'internal error' }); + } +}); + +// POST /rules/:type/:id/ack { sensor } → chiamato dal servizio realtime quando il plugin conferma +router.post('/:type/:id/ack', async (req, res) => { + if (!req.internal) return res.status(403).json({ error: 'forbidden' }); + const { type, id } = req.params; + const sensor = req.body?.sensor; + if (!isValidType(type) || !sensor) return res.status(400).json({ error: 'bad request' }); + try { + await query( + `UPDATE ruleset_deployments + SET acked_at = NOW() + WHERE sensor_name = $1 AND type = $2 AND ruleset_id = $3`, + [String(sensor), type, id], DB + ); + res.json({ ok: true }); + } catch (err) { + console.error('[RULES] ack error:', err.message); + res.status(500).json({ error: 'internal error' }); + } +}); + +module.exports = router; diff --git a/api/src/storage/influx.js b/api/src/storage/influx.js index eb31f57..e57c006 100644 --- a/api/src/storage/influx.js +++ b/api/src/storage/influx.js @@ -60,7 +60,9 @@ async function query(bucket, relativeTime, measurement, sensor, field) { } -const sessionBucket = 'boat'; +// Sorgente di verità per i logs di sessione: stesso bucket usato da +// realtime/store/influx.js. Sovrascrivibile via env per ambiente. +const sessionBucket = process.env.INFLX_BUCKET_LOGS || process.env.INFLX_BUCKET || 'logs'; /** * Query storica per una sessione di registrazione. diff --git a/api/src/storage/minio.js b/api/src/storage/minio.js index fc016b8..e4c3299 100644 --- a/api/src/storage/minio.js +++ b/api/src/storage/minio.js @@ -8,6 +8,13 @@ const client = new Minio.Client({ secretKey: process.env.MINIO_SECRET_KEY }) +// Unified ML bucket: tutti gli oggetti del dominio ML vivono nel bucket +// indicato da MINIO_BUCKET (default "ml"), con prefissi logici: +// datasets/. +// models////... +// trainings//logs.jsonl +const ML_BUCKET = process.env.MINIO_BUCKET || 'ml'; + // Buckets @@ -132,7 +139,51 @@ async function checkMinio() { } } +/** + * Legge un oggetto e restituisce il suo contenuto come stringa UTF-8 (comodo per file testo/markdown). + */ +async function readText(bucket, objectName) { + const stream = await client.getObject(bucket, objectName); + return await new Promise((resolve, reject) => { + const chunks = []; + stream.on('data', (c) => chunks.push(c)); + stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + stream.on('error', reject); + }); +} + +/** + * Scrive una stringa come oggetto (auto-create del bucket). + */ +async function writeText(bucket, objectName, content, contentType = 'text/markdown; charset=utf-8') { + await bucketExists(bucket); + const buf = Buffer.from(content, 'utf8'); + await client.putObject(bucket, objectName, buf, buf.length, { 'Content-Type': contentType }); + return { bucket, objectName, size: buf.length }; +} + +/** + * Elenca oggetti di un bucket in forma compatta (name, size, lastModified). + */ +async function listObjects(bucket) { + await bucketExists(bucket); + return new Promise((resolve, reject) => { + const out = []; + const stream = client.listObjects(bucket, '', true); + stream.on('data', (o) => out.push({ + name: o.name, + size: o.size, + lastModified: o.lastModified, + etag: o.etag, + })); + stream.on('error', reject); + stream.on('end', () => resolve(out)); + }); +} + module.exports = { + client, + ML_BUCKET, bucketExists, getBuckets, getBucket, @@ -142,5 +193,8 @@ module.exports = { upload, download, getFileStream, - checkMinio + checkMinio, + readText, + writeText, + listObjects, } \ No newline at end of file diff --git a/api/src/storage/postgres.js b/api/src/storage/postgres.js index 55b4124..4114016 100644 --- a/api/src/storage/postgres.js +++ b/api/src/storage/postgres.js @@ -15,6 +15,11 @@ const pools = { users: new Pool({ ...config, database: process.env.USERS_DB }), sensors: new Pool({ ...config, database: process.env.SENSOR_DB || 'users' }), rules: new Pool({ ...config, database: process.env.RULES_DB || 'rules' }), + ml: new Pool({ ...config, database: process.env.ML_DB || 'ml' }), + references: new Pool({ ...config, database: process.env.REFERENCES_DB || 'references' }), + // Le tabelle kiosktemplates / kioskelements vivono nel DB `sensors`. + // KIOSK_DB resta override-abile per environment legacy. + kiosk: new Pool({ ...config, database: process.env.KIOSK_DB || 'sensors' }), } Object.entries(pools).forEach(([name, pool]) => { @@ -73,6 +78,10 @@ async function remove(table, condition, params, type = 'users') { return await query(sql, params, type); } +// initKioskSchema rimosso: lo schema kiosktemplates/kioskelements vive nel DB +// `sensors` ed e' gestito dalla migration 007_kiosktemplates.sql. L'auto-create +// qui creava un schema legacy divergente (UUID + JSONB content) sul DB sbagliato. + async function checkPostgres() { const status = {}; console.log("Checking PostgreSQL connections with config:", config); diff --git a/console/src/index.js b/console/src/index.js index ace74f0..d96e6b8 100644 --- a/console/src/index.js +++ b/console/src/index.js @@ -66,6 +66,30 @@ app.get('/sessions', renderPage('sessions', { mapboxToken: process.env.MAPBOX_TOKEN || '' })); +app.get('/kioskedit', renderPage('kioskedit')); + +app.get('/kiosklive', renderPage('kiosklive', { + apiUrl: process.env.API_URL || 'http://localhost:3003', + realtimeWsUrl: process.env.REALTIME_WS_URL || 'ws://localhost:3002' +})); + +app.get('/forecasts', renderPage('forecasts', { + apiUrl: process.env.API_URL || 'http://localhost:3003', + realtimeWsUrl: process.env.REALTIME_WS_URL || 'ws://localhost:3002' +})); + +app.get('/documentation', renderPage('documentation', { + apiUrl: process.env.API_URL || 'http://localhost:3003' +})); +// retro-compatibilità: il link della dashboard punta ancora a /documentations +app.get('/documentations', (req, res) => res.redirect(301, '/documentation')); + +app.get('/marine', renderPage('marine', { + apiUrl: process.env.API_URL || 'http://localhost:3003', + marineUrl: process.env.MARINE_URL || (process.env.API_URL || 'http://localhost:3003') + '/marine' +})); + + app.listen(PORT, '0.0.0.0', () => { console.log(`Started on port ${PORT}`); }); diff --git a/console/src/pages/dashboard.html b/console/src/pages/dashboard.html index 665f546..f8d4097 100644 --- a/console/src/pages/dashboard.html +++ b/console/src/pages/dashboard.html @@ -1,178 +1,188 @@ - - - + + -
-
-

+
+
+

-
- -

username

- Impostazioni +
+

username

+ Impostazioni +
+ +
+
🚤
+

Benvenuto nella MEB Console

+

Usa uno dei tool a disposizione per visualizzare i dati dalla barca in tempo reale, creare nuovi dataset + condivisi e

+
+ + + + +
+

Trasmissione e Analisi dati

+ +
+ + +
+

Live

+

Visualizza i dati dei sensori sulla barca in tempo reale.

+
+ + + +
+

Analisi Dati

+

Analizza i dati raccolti sulle performance e sulla navigazione.

+
+ + + +
+

Previsioni

+

Visualizza le condizioni meteo attuali e le previsioni future.

+
+ + + +
+

In sviluppoKiosk

+

Modifica cosa vede il pilota a bordo in tempo reale.

+
+ + + + +
+ + + + + + + + + +
+ +
+

Grafana

+

Spostati sulla Dashboard di Grafana per visualizzare grafici e dati.

+
+ + +
+ +
+ + + +
+

Dati & Risorse

+ +
+ + +
+ +

In sviluppoRulesets

+

Gestisci i template di configurazione per weather, data e logs.

+
+ + + + +
+

Documentazione

+

Consulta i file Markdown di documentazione sul server e risorse aggiuntive sui modelli ML. +

+
+ + + +
+

In sviluppoDiario di Bordo

+

Tieni traccia delle modifiche, dei progressi e dei cambiamenti del progetto.

+
+ + + + +
+ + + + + + + + + +
+ +
+

Repositories

+

Controlla le repositories di codice di tutte le componenti del server, del plugin e dei modelli ML +

+
+ + + +
+

Copernicus

+

Accedi alle risorse sui dati di Copernicus e crea dataset personalizzati.

+
+ + +
+ +
+ + +
+

ML

+ +
+ +
+

Datasets

+

Gestisci i dataset disponibili

+
+ + + +
+ +

In sviluppoTest del Timone

+

Scegli un modello ML e simula in tempo reale i suggerimenti di sterzate in base alle condizioni di navigazione

+
+ + + +
+

Console ML

+

Accedi alla console ML per effettuare test, training, agire sul codice e confrontare i risultati.

+
+ + +
+ +
+ +
- -
-
🚤
-

Benvenuto nella MEB Console

-

Usa uno dei tool a disposizione per visualizzare i dati dalla barca in tempo reale, creare nuovi dataset condivisi e

-
- -
- -
-

Trasmissione e Analisi dati

-
- - -
-

Live

-

Visualizza i dati dei sensori sulla barca in tempo reale.

-
- - - -
-

Analisi Dati

-

Analizza i dati raccolti sulle performance e sulla navigazione.

-
- - - -
-

Previsioni

-

Visualizza le condizioni meteo attuali e le previsioni future.

-
- - - -
-

Kiosk

-

Modifica cosa vede il pilota a bordo in tempo reale.

-
- - - - - -
-

Dati & Risorse

-
- - - -
-

Rulesets

-

Gestisci i template di configurazione per weather, data e logs.

-
- - - -
-

Copernicus

-

Accedi alle risorse sui dati di Copernicus e crea dataset personalizzati.

-
- - - -
- - - - - - - - - - -
-
-

Grafana

-

Spostati sulla Dashboard di Grafana per visualizzare grafici e dati.

-
- - -
-

ML

-
- - - -
-

Dataset

-

Gestisci i dataset disponibili

-
- - - -
-

Documentazione

-

Consulta i file Markdown di documentazione sul server e risorse aggiuntive sui modelli ML.

-
- - - -
-

Diario di Bordo

-

Tieni traccia delle modifiche, dei progressi e dei cambiamenti del progetto.

-
- - - -
- - - - - - - - - - -
-
-

Repositories

-

Controlla le repositories di codice di tutte le componenti del server, del plugin e - dei modelli ML -

-
- - -
- -
- - - - \ No newline at end of file + \ No newline at end of file diff --git a/console/src/pages/documentation.html b/console/src/pages/documentation.html new file mode 100644 index 0000000..7975444 --- /dev/null +++ b/console/src/pages/documentation.html @@ -0,0 +1,230 @@ + + + + + Documentazione — MEB Console + + + + + + + +
+
+

Documentazione

+
+ ← Dashboard +
+
+ +
+ + +
+
+
Nessun documento selezionato
+
+
+ + +
+ +
+
+
+
+
+
📄
+
Seleziona un documento dalla sidebar, o creane uno nuovo.
+
+
+ +
+
+
+ +
+
+ + + + diff --git a/console/src/pages/forecasts.html b/console/src/pages/forecasts.html new file mode 100644 index 0000000..0dabbf9 --- /dev/null +++ b/console/src/pages/forecasts.html @@ -0,0 +1,300 @@ + + + + + Previsioni — MEB Console + + + + + +
+
+

Previsioni meteo-marine

+
+ In attesa… + Dashboard +
+
+ +
+
+

Temperatura

+
--°C
+
Umidità: --%
+
+
+

Vento

+
--m/s
+
Raffiche: -- m/s · Dir: --°
+
+
+

Pressione

+
--hPa
+
Nuvole: --% · Prob. pioggia: --%
+
+
+

Onde

+
--m
+
Periodo: --s · Dir: --°
+
+
+ +
+ Intervallo storico: + + + + +
+ +
+
+

Temperatura & Umidità

+ +
+
+

Vento (velocità + raffiche)

+ +
+
+

Pressione & Copertura

+ +
+
+

Precipitazioni

+ +
+
+

Onde — altezza & periodo

+ +
+
+

Direzione (corrente)

+
+ NS + EW + + + + + + + + + + + + + +
+
+ ■ Vento   ■ Onde +
+
+
+ +

+ Fonte: plugin SignalK → Open-Meteo (current ogni 5 min, hourly ogni 60 min) + dati marini. + Live via WebSocket; storico da InfluxDB. +

+
+ + + + diff --git a/console/src/pages/kioskedit.html b/console/src/pages/kioskedit.html index 8cbd012..4daf192 100644 --- a/console/src/pages/kioskedit.html +++ b/console/src/pages/kioskedit.html @@ -9,16 +9,6 @@ - @@ -48,7 +38,6 @@

0 cards

- @@ -567,15 +556,5 @@ ws.onclose = () => { - - diff --git a/console/src/pages/kiosklive.html b/console/src/pages/kiosklive.html new file mode 100644 index 0000000..92cbe1a --- /dev/null +++ b/console/src/pages/kiosklive.html @@ -0,0 +1,331 @@ + + + + +Kiosk Live + + + + + +
+
connecting…
+ + +
+ + + +
+ +
+
+
+

No box selected

+ +
Click a box to edit it. Changes apply live to the kiosk.
+
+
+ + + +
+ + + + diff --git a/console/src/pages/live.html b/console/src/pages/live.html index 36c68d2..0dc610f 100644 --- a/console/src/pages/live.html +++ b/console/src/pages/live.html @@ -11,16 +11,6 @@ .expanded-chart-container { display: none; } .comparison-sidebar { display: none; } - @@ -60,7 +50,6 @@

In attesa di dati...

-

Sensore

@@ -788,14 +777,3 @@ document.getElementById('saveSessionLabelBtn').onclick = async () => { }; - - - diff --git a/console/src/pages/marine.html b/console/src/pages/marine.html new file mode 100644 index 0000000..ed6d2fa --- /dev/null +++ b/console/src/pages/marine.html @@ -0,0 +1,301 @@ + + + + + Copernicus Marine — MEB Console + + + + +
+
+

Copernicus Marine

+
← Dashboard
+
+ +
+ + +
+ + + + + + +
+
+

Scarica dataset

+
+
+ + + +
+
+
+
+
+
+
+
+
+ + + + + + + +
+ + +
+ +
+
+
+
+ + + + diff --git a/console/src/pages/rulesets.html b/console/src/pages/rulesets.html index be5c19e..cae8447 100644 --- a/console/src/pages/rulesets.html +++ b/console/src/pages/rulesets.html @@ -4,16 +4,6 @@ - @@ -27,14 +17,14 @@

Rulesets

- - - - + + + + +
- Salvato
@@ -51,7 +41,7 @@
- +
@@ -78,9 +68,15 @@
Versione - +
+ + . + + . + +
- + + +
+
Deploy ai sensori
+
+
+
Caricamento sensori...
+
+
+ + +
+
+
+
@@ -133,47 +144,89 @@ const API_URL = '{{ apiUrl }}'; // --- State --- -let currentType = 'weather'; +const TYPES = ['logs','forecast_current','forecast_hourly','marine_current','marine_hourly']; +let currentType = 'logs'; let currentFilter = 'all'; let currentSort = 'created_at'; let allRules = []; -let openRule = null; // rule attualmente aperta nel popup +let openRule = null; let saveTimers = {}; +let sensorsCache = []; +let deploymentsForOpen = []; // deployments relativi alla rule aperta // --- Item field definitions per tipo --- +// Per ogni tipo, definiamo gli input visibili. Tutti finiscono nei campi +// { ref, path, enabled, meta: {...} } del JSONB dell'item. +// +// - logs: ref, path (SK path), meta.measurement, meta.unit +// - forecast_*: ref, path (codice openmeteo), meta.sk_path, meta.unit +// - marine_*: ref, path (codice openmeteo), meta.sk_path, meta.unit const ITEM_SCHEMA = { - weather: [ - { key: 'group_name', label: 'Gruppo', cls: 'medium' }, - { key: 'ref', label: 'Ref', cls: 'medium' }, - { key: 'name', label: 'Nome', cls: 'wide' }, - { key: 'unit', label: 'Unita', cls: 'narrow' }, - ], - laterforecasts: [ - { key: 'group_name', label: 'Gruppo', cls: 'medium' }, - { key: 'ref', label: 'Ref', cls: 'medium' }, - { key: 'name', label: 'Nome', cls: 'wide' }, - { key: 'unit', label: 'Unita', cls: 'narrow' }, - ], - data: [ - { key: 'category', label: 'Categoria', cls: 'medium' }, - { key: 'path', label: 'Path', cls: 'wide' }, - { key: 'unit', label: 'Unita', cls: 'narrow' }, - ], logs: [ - { key: 'path', label: 'Path', cls: 'wide' }, - { key: 'ref', label: 'Ref', cls: 'narrow' }, - { key: 'unit', label: 'Unita', cls: 'narrow' }, - { key: 'measurement', label: 'Measurement', cls: 'medium' }, + { key: 'ref', label: 'Ref', cls: 'medium' }, + { key: 'path', label: 'SK Path', cls: 'wide' }, + { key: 'meta.measurement', label: 'Measurement', cls: 'medium' }, + { key: 'meta.unit', label: 'Unita', cls: 'narrow' }, + ], + forecast_current: [ + { key: 'ref', label: 'Ref', cls: 'medium' }, + { key: 'path', label: 'OpenMeteo', cls: 'medium' }, + { key: 'meta.sk_path', label: 'SK Path', cls: 'wide' }, + { key: 'meta.unit', label: 'Unita', cls: 'narrow' }, + ], + forecast_hourly: [ + { key: 'ref', label: 'Ref', cls: 'medium' }, + { key: 'path', label: 'OpenMeteo', cls: 'medium' }, + { key: 'meta.sk_path', label: 'SK Path', cls: 'wide' }, + { key: 'meta.unit', label: 'Unita', cls: 'narrow' }, + ], + marine_current: [ + { key: 'ref', label: 'Ref', cls: 'medium' }, + { key: 'path', label: 'OpenMeteo', cls: 'medium' }, + { key: 'meta.sk_path', label: 'SK Path', cls: 'wide' }, + { key: 'meta.unit', label: 'Unita', cls: 'narrow' }, + ], + marine_hourly: [ + { key: 'ref', label: 'Ref', cls: 'medium' }, + { key: 'path', label: 'OpenMeteo', cls: 'medium' }, + { key: 'meta.sk_path', label: 'SK Path', cls: 'wide' }, + { key: 'meta.unit', label: 'Unita', cls: 'narrow' }, ], }; -const HAS_DESC = { weather: true, laterforecasts: true, data: false, logs: true }; +// ========== Helpers ========== + +function esc(str) { + if (str === null || str === undefined) return ''; + const d = document.createElement('div'); + d.textContent = String(str); + return d.innerHTML; +} + +function getField(obj, dottedKey) { + if (!dottedKey.includes('.')) return obj?.[dottedKey]; + const [a, b] = dottedKey.split('.'); + return obj?.[a]?.[b]; +} + +function setField(obj, dottedKey, value) { + if (!dottedKey.includes('.')) { obj[dottedKey] = value; return; } + const [a, b] = dottedKey.split('.'); + if (!obj[a] || typeof obj[a] !== 'object') obj[a] = {}; + obj[a][b] = value; +} + +function versionStr(v) { + if (!v) return ''; + if (v.str) return v.str; + return `${v.major ?? 0}.${v.build ?? 0}.${v.patch ?? 0}`; +} // ========== API helpers ========== async function api(method, path, body) { const opts = { method, headers: {}, credentials: 'include' }; - if (body) { + if (body !== undefined) { opts.headers['Content-Type'] = 'application/json'; opts.body = JSON.stringify(body); } @@ -198,13 +251,27 @@ async function loadRules() { } } +async function loadSensors() { + if (sensorsCache.length) return sensorsCache; + try { + sensorsCache = await api('GET', '/rules/-/sensors'); + } catch (err) { + console.error('Error loading sensors:', err); + sensorsCache = []; + } + return sensorsCache; +} + function filterAndSort(rules) { let filtered = rules; if (currentFilter === 'active') filtered = rules.filter(r => r.active && !r.archived); else if (currentFilter === 'archived') filtered = rules.filter(r => r.archived); filtered.sort((a, b) => { - if (currentSort === 'version') return (b.version || '').localeCompare(a.version || '', undefined, { numeric: true }); + if (currentSort === 'version') { + const va = versionStr(a.version), vb = versionStr(b.version); + return vb.localeCompare(va, undefined, { numeric: true }); + } return new Date(b.created_at) - new Date(a.created_at); }); return filtered; @@ -215,7 +282,7 @@ function renderGrid() { const rules = filterAndSort(allRules); if (rules.length === 0) { - grid.innerHTML = '
Nessuna rule trovata
'; + grid.innerHTML = '
Nessuna versione trovata
'; return; } @@ -228,13 +295,14 @@ function renderGrid() { const tags = (r.tags || []).map(t => `${esc(t)}`).join(''); const desc = r.description ? `
${esc(r.description)}
` : ''; const date = r.created_at ? new Date(r.created_at).toLocaleDateString('it-IT', { day: '2-digit', month: 'short', year: 'numeric' }) : ''; + const itemsCount = (r.items_count !== undefined ? r.items_count : (r.items?.length || 0)); return ` -
+
-
v${esc(r.version)}
- ${esc(r.id)} +
v${esc(versionStr(r.version))}
+ ${esc(String(r.id).slice(0,8))}…
${badges.join('')}
@@ -242,18 +310,12 @@ function renderGrid() { ${tags ? `
${tags}
` : ''}
${date} + ${itemsCount} items
`; }).join(''); } -function esc(str) { - if (!str) return ''; - const d = document.createElement('div'); - d.textContent = str; - return d.innerHTML; -} - // ========== Type Picker ========== document.querySelectorAll('#typePicker button').forEach(btn => { @@ -285,10 +347,20 @@ document.getElementById('sortSelect').onchange = (e) => { document.getElementById('newRuleBtn').onclick = async () => { try { + // Calcola una versione libera: prendi la maggiore esistente e incrementa patch + let M = 1, B = 0, P = 0; + if (allRules.length) { + const sorted = [...allRules].sort((a,b) => { + const va = a.version, vb = b.version; + return (vb.major - va.major) || (vb.build - va.build) || (vb.patch - va.patch); + }); + const top = sorted[0].version; + M = top.major; B = top.build; P = Math.min(100, top.patch + 1); + if (P === 100 && top.patch === 100) { P = 0; B = Math.min(100, B + 1); } + } const rule = await api('POST', `/rules/${currentType}`, { - version: '1.0.0', - tags: [], - description: HAS_DESC[currentType] ? '' : undefined + version_major: M, version_build: B, version_patch: P, + tags: [], description: '', items: [] }); allRules.unshift(rule); renderGrid(); @@ -296,6 +368,7 @@ document.getElementById('newRuleBtn').onclick = async () => { flash('Salvato'); } catch (err) { console.error('Error creating rule:', err); + alert(`Errore: ${err.message}`); } }; @@ -305,17 +378,23 @@ async function openRuleDetail(ruleId) { try { const data = await api('GET', `/rules/${currentType}/${ruleId}`); openRule = data; + deploymentsForOpen = []; + try { + deploymentsForOpen = await api('GET', `/rules/${currentType}/${ruleId}/deployments`); + } catch {} + await loadSensors(); renderPopup(); document.getElementById('ruleOverlay').style.display = 'flex'; } catch (err) { console.error('Error loading rule detail:', err); + alert(`Errore: ${err.message}`); } } function closePopup() { document.getElementById('ruleOverlay').style.display = 'none'; openRule = null; - loadRules(); // refresh grid + loadRules(); } document.getElementById('popupClose').onclick = closePopup; @@ -325,31 +404,24 @@ document.getElementById('ruleOverlay').onclick = (e) => { function renderPopup() { const r = openRule; - document.getElementById('popupId').textContent = r.id; - document.getElementById('popupVersion').value = r.version || ''; + document.getElementById('popupId').textContent = `${currentType} · ${String(r.id).slice(0,8)}`; + document.getElementById('popupVMajor').value = r.version?.major ?? 1; + document.getElementById('popupVBuild').value = r.version?.build ?? 0; + document.getElementById('popupVPatch').value = r.version?.patch ?? 0; + document.getElementById('popupDesc').value = r.description || ''; - // Description - const descRow = document.getElementById('popupDescRow'); - if (HAS_DESC[currentType]) { - descRow.style.display = 'flex'; - document.getElementById('popupDesc').value = r.description || ''; - } else { - descRow.style.display = 'none'; - } - - // Tags renderTags(); - - // Action buttons state updateActionButtons(); - - // Items renderItems(); + renderDeploySensors(); + document.getElementById('deployResult').textContent = ''; } // --- Auto-save fields --- -document.getElementById('popupVersion').oninput = () => debounceFieldSave('version'); +['popupVMajor','popupVBuild','popupVPatch'].forEach(id => { + document.getElementById(id).oninput = () => debounceFieldSave('version'); +}); document.getElementById('popupDesc').oninput = () => debounceFieldSave('description'); function debounceFieldSave(field) { @@ -360,18 +432,22 @@ function debounceFieldSave(field) { async function saveRuleField(field) { if (!openRule) return; const body = {}; - if (field === 'version') body.version = document.getElementById('popupVersion').value.trim(); - if (field === 'description') body.description = document.getElementById('popupDesc').value.trim(); + if (field === 'version') { + body.version_major = parseInt(document.getElementById('popupVMajor').value, 10) || 1; + body.version_build = parseInt(document.getElementById('popupVBuild').value, 10) || 0; + body.version_patch = parseInt(document.getElementById('popupVPatch').value, 10) || 0; + } + if (field === 'description') body.description = document.getElementById('popupDesc').value; try { const updated = await api('PUT', `/rules/${currentType}/${openRule.id}`, body); Object.assign(openRule, updated); - // Update in allRules too const idx = allRules.findIndex(r => r.id === openRule.id); if (idx >= 0) Object.assign(allRules[idx], updated); flash('Salvato', 'popupSaving'); } catch (err) { console.error('Error saving field:', err); + flash('Errore: ' + err.message, 'popupSaving'); } } @@ -380,9 +456,7 @@ async function saveRuleField(field) { function renderTags() { const wrap = document.getElementById('popupTags'); const input = document.getElementById('tagInput'); - // Remove old chips wrap.querySelectorAll('.rs-tag-chip').forEach(c => c.remove()); - // Re-add chips before input (openRule.tags || []).forEach((tag, i) => { const chip = document.createElement('span'); chip.className = 'rs-tag-chip'; @@ -404,9 +478,7 @@ document.getElementById('tagInput').onkeydown = async (e) => { openRule.tags = updated.tags; renderTags(); flash('Salvato', 'popupSaving'); - } catch (err) { - console.error('Error adding tag:', err); - } + } catch (err) { console.error(err); } } }; @@ -419,9 +491,7 @@ async function removeTag(idx) { openRule.tags = updated.tags; renderTags(); flash('Salvato', 'popupSaving'); - } catch (err) { - console.error('Error removing tag:', err); - } + } catch (err) { console.error(err); } } // --- Action Buttons --- @@ -430,10 +500,8 @@ function updateActionButtons() { const r = openRule; const activeBtn = document.getElementById('toggleActiveBtn'); const archiveBtn = document.getElementById('toggleArchiveBtn'); - activeBtn.textContent = r.active ? 'Disattiva' : 'Attiva'; activeBtn.className = `rs-action-btn active-toggle${r.active ? '' : ' off'}`; - archiveBtn.textContent = r.archived ? 'Dearchivia' : 'Archivia'; archiveBtn.className = `rs-action-btn archive-toggle${r.archived ? ' on' : ''}`; } @@ -443,10 +511,12 @@ document.getElementById('toggleActiveBtn').onclick = async () => { try { const res = await api('PATCH', `/rules/${currentType}/${openRule.id}/active`); openRule.active = res.active; + if (res.ruleset) Object.assign(openRule, res.ruleset); updateActionButtons(); flash('Salvato', 'popupSaving'); } catch (err) { - console.error('Error toggling active:', err); + console.error(err); + alert(`Errore: ${err.message}`); } }; @@ -455,23 +525,20 @@ document.getElementById('toggleArchiveBtn').onclick = async () => { try { const res = await api('PATCH', `/rules/${currentType}/${openRule.id}/archive`); openRule.archived = res.archived; + if (res.ruleset) Object.assign(openRule, res.ruleset); updateActionButtons(); flash('Salvato', 'popupSaving'); - } catch (err) { - console.error('Error toggling archive:', err); - } + } catch (err) { console.error(err); alert(`Errore: ${err.message}`); } }; document.getElementById('deleteRuleBtn').onclick = () => { if (!openRule) return; - showConfirm('Elimina Rule', `Vuoi eliminare la rule ${openRule.id}? Questa azione e irreversibile.`, async () => { + showConfirm('Elimina Versione', `Eliminare la versione v${versionStr(openRule.version)}? Questa azione e irreversibile.`, async () => { try { await api('DELETE', `/rules/${currentType}/${openRule.id}`); allRules = allRules.filter(r => r.id !== openRule.id); closePopup(); - } catch (err) { - console.error('Error deleting rule:', err); - } + } catch (err) { console.error(err); alert(`Errore: ${err.message}`); } }); }; @@ -481,95 +548,152 @@ function renderItems() { const schema = ITEM_SCHEMA[currentType]; const items = openRule.items || []; - // Labels row const labelsRow = document.getElementById('itemLabelsRow'); labelsRow.innerHTML = schema.map(f => `${f.label}`).join('') + 'On'; - // Items list const list = document.getElementById('itemsList'); if (items.length === 0) { - list.innerHTML = '
Nessun item. Clicca "+ Aggiungi" per crearne uno.
'; + list.innerHTML = '
Nessun item. Clicca "+ Aggiungi".
'; return; } list.innerHTML = items.map(item => { const fields = schema.map(f => - `` + `` ).join(''); - const toggleCls = item.enabled ? 'on' : ''; - return `
+ const toggleCls = item.enabled !== false ? 'on' : ''; + return `
${fields} -
- +
+
`; }).join(''); } document.getElementById('addItemBtn').onclick = async () => { if (!openRule) return; - const schema = ITEM_SCHEMA[currentType]; - const body = {}; - // Fill with empty/default values - schema.forEach(f => { body[f.key] = ''; }); - - // Need at least non-empty values — open with placeholders - // For now, create with placeholder values - schema.forEach(f => { body[f.key] = f.key === 'enabled' ? true : '-'; }); - + const ref = prompt('Ref dell\'item (identificatore stabile, es. "batt_traction_soc"):'); + if (!ref) return; try { - const item = await api('POST', `/rules/${currentType}/${openRule.id}/items`, body); + const item = await api('POST', `/rules/${currentType}/${openRule.id}/items`, { + ref: ref.trim(), path: '', enabled: true, meta: {} + }); if (!openRule.items) openRule.items = []; openRule.items.push(item); renderItems(); flash('Salvato', 'popupSaving'); } catch (err) { - console.error('Error adding item:', err); + console.error(err); + alert(`Errore: ${err.message}`); } }; async function saveItemField(input) { if (!openRule) return; - const itemId = input.dataset.itemId; + const ref = input.dataset.ref; const field = input.dataset.field; - const value = input.value.trim(); + const value = input.value; + + const item = (openRule.items || []).find(i => i.ref === ref); + if (!item) return; + + // costruisci body rispettando ref/path/enabled oppure meta. + const body = {}; + if (field === 'ref') body.ref = value.trim(); + else if (field === 'path') body.path = value; + else if (field === 'enabled') body.enabled = !!value; + else if (field.startsWith('meta.')) { + const metaKey = field.slice(5); + body.meta = { ...(item.meta || {}), [metaKey]: value }; + } + try { - await api('PUT', `/rules/${currentType}/${openRule.id}/items/${itemId}`, { [field]: value }); - // Update local - const item = openRule.items.find(i => String(i.id) === String(itemId)); - if (item) item[field] = value; + const updated = await api('PUT', `/rules/${currentType}/${openRule.id}/items/${encodeURIComponent(ref)}`, body); + // replace item in place + const idx = openRule.items.findIndex(i => i.ref === ref); + if (idx >= 0) openRule.items[idx] = updated; flash('Salvato', 'popupSaving'); } catch (err) { - console.error('Error saving item field:', err); + console.error(err); + flash('Errore', 'popupSaving'); } } -async function toggleItem(itemId) { +async function toggleItem(ref) { if (!openRule) return; try { - const res = await api('PATCH', `/rules/${currentType}/${openRule.id}/items/${itemId}/toggle`); - const item = openRule.items.find(i => i.id === itemId); + const res = await api('PATCH', `/rules/${currentType}/${openRule.id}/items/${encodeURIComponent(ref)}/toggle`); + const item = openRule.items.find(i => i.ref === ref); if (item) item.enabled = res.enabled; renderItems(); flash('Salvato', 'popupSaving'); - } catch (err) { - console.error('Error toggling item:', err); - } + } catch (err) { console.error(err); } } -async function deleteItem(itemId) { +async function deleteItem(ref) { if (!openRule) return; try { - await api('DELETE', `/rules/${currentType}/${openRule.id}/items/${itemId}`); - openRule.items = openRule.items.filter(i => i.id !== itemId); + await api('DELETE', `/rules/${currentType}/${openRule.id}/items/${encodeURIComponent(ref)}`); + openRule.items = openRule.items.filter(i => i.ref !== ref); renderItems(); flash('Salvato', 'popupSaving'); - } catch (err) { - console.error('Error deleting item:', err); - } + } catch (err) { console.error(err); } } +// --- Deploy --- + +function renderDeploySensors() { + const wrap = document.getElementById('deploySensorsList'); + if (!sensorsCache.length) { + wrap.innerHTML = '
Nessun sensore registrato
'; + return; + } + const byName = Object.fromEntries(deploymentsForOpen.map(d => [d.sensor_name, d])); + wrap.innerHTML = sensorsCache.map(s => { + const d = byName[s.name]; + let status = ''; + if (d) { + status = d.acked_at + ? `ACK ${new Date(d.acked_at).toLocaleString('it-IT')}` + : `In attesa…`; + } + return ``; + }).join(''); +} + +document.getElementById('deployBtn').onclick = async () => { + if (!openRule) return; + if (openRule.archived) { alert('Non puoi deployare una versione archiviata'); return; } + const checks = [...document.querySelectorAll('.rs-deploy-check:checked')]; + const sensors = checks.map(c => c.value); + if (!sensors.length) { alert('Seleziona almeno un sensore'); return; } + const resultEl = document.getElementById('deployResult'); + resultEl.textContent = 'Invio...'; + try { + const res = await api('POST', `/rules/${currentType}/${openRule.id}/deploy`, { sensors }); + const parts = []; + if (res.pushed?.length) parts.push(`${res.pushed.length} online`); + if (res.offline?.length) parts.push(`${res.offline.length} offline`); + if (res.errors?.length) parts.push(`${res.errors.length} errori`); + resultEl.textContent = parts.join(' · ') || 'OK'; + // refresh deployments + try { deploymentsForOpen = await api('GET', `/rules/${currentType}/${openRule.id}/deployments`); renderDeploySensors(); } catch {} + } catch (err) { + console.error(err); + resultEl.textContent = `Errore: ${err.message}`; + } +}; + // ========== Confirm Dialog ========== let confirmCallback = null; @@ -592,7 +716,7 @@ document.getElementById('confirmOk').onclick = async () => { confirmCallback = null; }; -// ========== Flash "Salvato" indicator ========== +// ========== Flash ========== function flash(text, elId = 'savingIndicator') { const el = document.getElementById(elId); @@ -603,18 +727,10 @@ function flash(text, elId = 'savingIndicator') { // ========== Init ========== -document.addEventListener('DOMContentLoaded', () => loadRules()); - - - - - diff --git a/console/src/pages/sessions.html b/console/src/pages/sessions.html index 9d46e46..0992c78 100644 --- a/console/src/pages/sessions.html +++ b/console/src/pages/sessions.html @@ -244,16 +244,6 @@ .map-bar .filter button { color: #cbd5e1; } .map-bar .filter button.active { background: #3b82f6; color: #fff; } - @@ -289,7 +279,6 @@

Sessioni

-
@@ -929,16 +918,5 @@ document.getElementById('changeSessionBtn').onclick = () => { // --- Init --- document.addEventListener('DOMContentLoaded', loadSessionsList); - - - diff --git a/console/src/static/styles/dashboard.css b/console/src/static/styles/dashboard.css index 4e0230a..609afd3 100644 --- a/console/src/static/styles/dashboard.css +++ b/console/src/static/styles/dashboard.css @@ -23,4 +23,26 @@ .card[title="Live"]:hover::before { opacity: 0.2; +} + +.category { + padding-bottom: 8%; +} + +.category h2 { + display: block; + text-align: center; + padding: 20px 20px; + margin: 0 0 0.75rem; + font-size: 1rem; + opacity: 0.7; + margin-bottom: 10px; + font-weight:900 +} + +.category section { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 1.5rem; + margin-inline: 30px; } \ No newline at end of file diff --git a/console/src/static/styles/rulesets.css b/console/src/static/styles/rulesets.css index 0b3b55a..6bd3639 100644 --- a/console/src/static/styles/rulesets.css +++ b/console/src/static/styles/rulesets.css @@ -769,3 +769,81 @@ .rs-back:hover { color: var(--accent-color); } + +/* ── Version number inputs (major.build.patch) ── */ +.rs-version-inputs { + display: inline-flex; + align-items: center; + gap: 4px; +} +.rs-version-num { + width: 48px; + padding: 6px 8px; + border: 1px solid var(--border-color, #e2e8f0); + border-radius: 6px; + font-size: 0.85rem; + text-align: center; + background: var(--input-bg, #fff); + color: var(--text-primary); + -moz-appearance: textfield; +} +.rs-version-num::-webkit-outer-spin-button, +.rs-version-num::-webkit-inner-spin-button { -webkit-appearance: none; margin: 0; } +.rs-version-dot { + color: var(--text-tertiary, #94a3b8); + font-weight: 600; +} + +.rs-card-items { + font-size: 0.75rem; + color: var(--text-tertiary, #94a3b8); +} + +/* ── Deploy section ── */ +.rs-deploy-wrap { + display: flex; + flex-direction: column; + gap: 12px; +} +.rs-deploy-sensors { + display: flex; + flex-direction: column; + gap: 6px; + max-height: 240px; + overflow-y: auto; + padding: 8px; + border: 1px solid var(--border-color, #e2e8f0); + border-radius: 8px; + background: var(--input-bg, #fafafa); +} +.rs-deploy-item { + display: flex; + align-items: center; + gap: 10px; + padding: 6px 8px; + border-radius: 6px; + font-size: 0.85rem; + cursor: pointer; + transition: background 0.15s ease; +} +.rs-deploy-item:hover { background: rgba(0,0,0,0.03); } +.rs-deploy-name { flex: 1; font-weight: 500; color: var(--text-primary); } +.rs-deploy-check { width: 16px; height: 16px; cursor: pointer; } +.rs-deploy-status { + font-size: 0.72rem; + padding: 2px 8px; + border-radius: 4px; + font-weight: 500; +} +.rs-deploy-status.ok { background: #dcfce7; color: #166534; } +.rs-deploy-status.pending { background: #fef3c7; color: #92400e; } + +.rs-deploy-actions { + display: flex; + align-items: center; + gap: 12px; +} +.rs-deploy-result { + font-size: 0.8rem; + color: var(--text-secondary); +} diff --git a/console/src/static/styles/style.css b/console/src/static/styles/style.css index 0b3b772..e4cb703 100644 --- a/console/src/static/styles/style.css +++ b/console/src/static/styles/style.css @@ -1,17 +1,16 @@ :root { --accent-color: #2563eb; --accent-hover: #1d4ed8; - --accent-light: #eff6ff; + --accent-light: #dce6f3; --accent-border: #bfdbfe; - --text-primary: #0f172a; + --text-primary: #000000; --text-secondary: #4755698f; --text-tertiary: #94a3b8c0; - --surface: #f8fafc; + --surface: #ffffff; --header-bg: rgba(255, 255, 255, 0.85); - /* For Glassmorphism */ --header-border: #e2e8f0; --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.05); @@ -32,7 +31,7 @@ --text-secondary: #cbd5e1; --text-tertiary: #94a3b8; - --surface: #0f172a; + --surface: #000000; --header-bg: rgba(15, 23, 42, 0.85); --header-border: #334155; @@ -42,26 +41,6 @@ } } -/* Manual dark mode toggle */ -body.dark-mode { - --accent-color: #3b82f6; - --accent-hover: #2563eb; - --accent-light: #1e3a8a; - --accent-border: #1e40af; - - --text-primary: #f1f5f9; - --text-secondary: #cbd5e1; - --text-tertiary: #94a3b8; - - --surface: #0f172a; - - --header-bg: rgba(15, 23, 42, 0.85); - --header-border: #334155; - - --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.3); - --shadow-md: 0 4px 6px -1px rgba(0, 0, 0, 0.3), 0 2px 4px -1px rgba(0, 0, 0, 0.2); -} - /* Smooth transition for dark mode */ body { transition: background-color 0.3s ease, color 0.3s ease; @@ -192,7 +171,6 @@ button.prominent:active { .card p { margin: 0.25rem 0 0; color: var(--text-secondary); - opacity: 0.4; font-size: 0.8rem; text-align: left; } @@ -206,7 +184,34 @@ button.prominent:active { grid-column: 1 / -1; } +.card.disabled { + pointer-events: none; + cursor: default; + /* No global opacity on the container to allow badge to remain fully visible */ +} +/* Dim specific internals to 50% while keeping the badge fully opaque */ +.card.disabled h3, +.card.disabled p, +.card.disabled .page-icon { + opacity: 0.5; +} + +.card.disabled .badge { + opacity: 1 !important; +} + +.card .badge { + background-color: #ef4444; + color: #fff; + font-size: 0.6rem; + padding: 2px 10px; + font-weight: 700; + margin-bottom: 2px; + margin-right: 6px; + vertical-align: middle; + display: inline-block; +} diff --git a/console/src/static/theme-toggle.js b/console/src/static/theme-toggle.js deleted file mode 100644 index 9309ff6..0000000 --- a/console/src/static/theme-toggle.js +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Dark Mode Theme Toggle - * Manages light/dark theme with localStorage persistence - */ - -const THEME_STORAGE_KEY = 'meb-console-theme'; -const DARK_MODE_CLASS = 'dark-mode'; - -/** - * Initialize theme on page load - */ -function initializeTheme() { - const savedTheme = localStorage.getItem(THEME_STORAGE_KEY); - const prefersDarkMode = window.matchMedia('(prefers-color-scheme: dark)').matches; - - // Use saved preference, or fallback to system preference - const shouldBeDark = savedTheme ? savedTheme === 'dark' : prefersDarkMode; - - if (shouldBeDark) { - enableDarkMode(); - } else { - disableDarkMode(); - } -} - -/** - * Enable dark mode - */ -function enableDarkMode() { - document.documentElement.classList.add(DARK_MODE_CLASS); - document.body.classList.add(DARK_MODE_CLASS); - localStorage.setItem(THEME_STORAGE_KEY, 'dark'); - updateThemeToggleButton(); -} - -/** - * Disable dark mode (light mode) - */ -function disableDarkMode() { - document.documentElement.classList.remove(DARK_MODE_CLASS); - document.body.classList.remove(DARK_MODE_CLASS); - localStorage.setItem(THEME_STORAGE_KEY, 'light'); - updateThemeToggleButton(); -} - -/** - * Toggle dark mode - */ -function toggleDarkMode() { - const isDarkMode = document.body.classList.contains(DARK_MODE_CLASS); - - if (isDarkMode) { - disableDarkMode(); - } else { - enableDarkMode(); - } -} - -/** - * Update theme toggle button appearance - */ -function updateThemeToggleButton() { - const themeBtn = document.getElementById('theme-toggle-btn'); - if (themeBtn) { - const isDarkMode = document.body.classList.contains(DARK_MODE_CLASS); - themeBtn.textContent = isDarkMode ? '☀️' : '🌙'; - themeBtn.title = isDarkMode ? 'Light Mode' : 'Dark Mode'; - } -} - -/** - * Listen for system theme preference changes - */ -function listenToSystemThemeChanges() { - window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', (e) => { - if (!localStorage.getItem(THEME_STORAGE_KEY)) { - // If no manual preference set, follow system preference - e.matches ? enableDarkMode() : disableDarkMode(); - } - }); -} - -// Initialize on load -document.addEventListener('DOMContentLoaded', initializeTheme); -listenToSystemThemeChanges(); diff --git a/copernicus/core/cache.py b/copernicus/core/cache.py index 09881f9..2a546da 100644 --- a/copernicus/core/cache.py +++ b/copernicus/core/cache.py @@ -1,125 +1,292 @@ """ -Redis Keys: -- marine:catalog:full → lista dei dataset completo (TTL 1h) -- marine:catalog:search:{hash} → risultati ricerca (TTL 30min) -- marine:job:{session_id} → stato job download (TTL 48h) +Cache two-tier per il servizio Marine. + +L1 = Redis (RAM): scadenza 2 ore, velocissima, condivisa tra processi. +L2 = SQLite+disco: persistente (200GB), fallback quando Redis non c'è + o quando L1 è scaduta. Scadenza configurabile (default 30 giorni). + +Flusso lettura: + 1. Prova L1 (Redis). Se hit → ritorna. + 2. Prova L2 (SQLite). Se hit non scaduta → ritorna E ripopola L1 (re-warm). + 3. Miss totale → None. + +Flusso scrittura: + Scrive in entrambi i tier contemporaneamente. + +Chiavi standard: +- marine:catalog:full → lista completa dataset Copernicus +- marine:catalog:search:{hash} → risultati ricerca utente +- marine:job:{session_id} → stato job download (solo Redis, ephemeri) """ +import gzip import json -import os import logging +import os +import sqlite3 +import threading +import time +from pathlib import Path from typing import Any, Optional import redis logger = logging.getLogger(__name__) -# Configurazione Redis da variabili ambiente +# ── Config ─────────────────────────────────────────────────────────────── REDIS_HOST = os.getenv("REDIS_HOST", "meb-redis") REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) -# Pool di connessioni condiviso (thread-safe, riutilizzabile) +# Il volume persistente è montato dal container, default /app/cache +CACHE_DIR = Path(os.getenv("CACHE_DIR", "/app/cache")) +CACHE_DB = CACHE_DIR / "catalog.sqlite" +BLOB_DIR = CACHE_DIR / "blobs" + +# TTL default +DEFAULT_REDIS_TTL = 2 * 3600 # 2 ore (L1) +DEFAULT_DISK_TTL = 30 * 24 * 3600 # 30 giorni (L2) + +# Soglia sopra la quale il valore va in un file su disco invece che in sqlite +BLOB_THRESHOLD_BYTES = 64 * 1024 # 64 KB + +# ── Stato globale ──────────────────────────────────────────────────────── _pool: Optional[redis.ConnectionPool] = None _client: Optional[redis.Redis] = None +_redis_disabled = False + +_sqlite_lock = threading.Lock() +_sqlite_initialized = False -def _get_client() -> Optional[redis.Redis]: - """Restituisce il client Redis singleton con connection pool. - Ritorna None se Redis non è raggiungibile.""" - global _pool, _client - +# ── Redis (L1) ─────────────────────────────────────────────────────────── +def _get_redis() -> Optional[redis.Redis]: + global _pool, _client, _redis_disabled + if _redis_disabled: + return None if _client is not None: return _client - try: _pool = redis.ConnectionPool( host=REDIS_HOST, port=REDIS_PORT, - # Decodifica automatica delle risposte in stringhe UTF-8 - decode_responses=True, - # Massimo 5 connessioni nel pool (VPS 1-core, non serve di più) + decode_responses=False, # tratto blob binari (gzip) max_connections=5, - # Timeout connessione e socket per evitare blocchi socket_connect_timeout=3, socket_timeout=3, - # Riprova automaticamente se la connessione viene interrotta retry_on_timeout=True, ) _client = redis.Redis(connection_pool=_pool) - # Test connessione _client.ping() - logger.info("[Redis] Connessione stabilita per il servizio Marine") + logger.info("[Cache] Redis L1 connesso") return _client except Exception as e: - logger.warning(f"[Redis] Non disponibile, la cache è disabilitata: {e}") + logger.warning(f"[Cache] Redis non disponibile, uso solo disco: {e}") + _redis_disabled = True _client = None return None -def cache_get(key: str) -> Optional[Any]: - """Legge un valore dalla cache Redis. +# ── SQLite (L2) ────────────────────────────────────────────────────────── +def _ensure_sqlite() -> sqlite3.Connection: + """Apre/crea il db SQLite su disco. Crea anche la dir blob.""" + global _sqlite_initialized + CACHE_DIR.mkdir(parents=True, exist_ok=True) + BLOB_DIR.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(CACHE_DB), timeout=5.0, isolation_level=None) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + if not _sqlite_initialized: + conn.execute(""" + CREATE TABLE IF NOT EXISTS cache ( + key TEXT PRIMARY KEY, + expires_at INTEGER NOT NULL, + is_blob INTEGER NOT NULL DEFAULT 0, + value BLOB, + blob_path TEXT, + size_bytes INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_cache_expires ON cache(expires_at)") + _sqlite_initialized = True + return conn - Args: - key: Chiave Redis (es. 'marine:catalog:full') - Returns: - Il valore deserializzato da JSON, oppure None se non trovato o errore - """ +def _blob_path(key: str) -> Path: + # Nome file safe: solo caratteri alfanumerici + hash per unicità + safe = "".join(c if c.isalnum() or c in ("-", "_") else "_" for c in key) + return BLOB_DIR / f"{safe}.json.gz" + + +def _disk_get(key: str) -> Optional[Any]: try: - client = _get_client() - if client is None: + with _sqlite_lock: + conn = _ensure_sqlite() + row = conn.execute( + "SELECT expires_at, is_blob, value, blob_path FROM cache WHERE key = ?", + (key,) + ).fetchone() + if row is None: return None - - data = client.get(key) - if data is None: + expires_at, is_blob, value, blob_path = row + if expires_at < int(time.time()): + # Scaduta: la elimino in lazy + _disk_delete(key) return None - - return json.loads(data) + if is_blob: + data = Path(blob_path).read_bytes() + else: + data = value + return json.loads(gzip.decompress(data).decode("utf-8")) except Exception as e: - logger.warning(f"[Redis] Errore lettura chiave '{key}': {e}") + logger.warning(f"[Cache] Errore lettura disco '{key}': {e}") return None -def cache_set(key: str, value: Any, ttl: int = 3600) -> bool: - """Scrive un valore nella cache Redis con TTL. - - Args: - key: Chiave Redis - value: Valore da serializzare in JSON - ttl: Tempo di vita in secondi (default: 1 ora) - - Returns: - True se scritto con successo, False altrimenti - """ +def _disk_set(key: str, raw_gz: bytes, ttl: int) -> None: try: - client = _get_client() - if client is None: - return False - - serialized = json.dumps(value) - client.setex(key, ttl, serialized) - return True + expires_at = int(time.time()) + ttl + updated_at = int(time.time()) + size = len(raw_gz) + if size > BLOB_THRESHOLD_BYTES: + path = _blob_path(key) + path.write_bytes(raw_gz) + with _sqlite_lock: + conn = _ensure_sqlite() + conn.execute( + "INSERT OR REPLACE INTO cache(key, expires_at, is_blob, value, blob_path, size_bytes, updated_at) " + "VALUES(?,?,?,?,?,?,?)", + (key, expires_at, 1, None, str(path), size, updated_at) + ) + else: + with _sqlite_lock: + conn = _ensure_sqlite() + conn.execute( + "INSERT OR REPLACE INTO cache(key, expires_at, is_blob, value, blob_path, size_bytes, updated_at) " + "VALUES(?,?,?,?,?,?,?)", + (key, expires_at, 0, raw_gz, None, size, updated_at) + ) except Exception as e: - logger.warning(f"[Redis] Errore scrittura chiave '{key}': {e}") + logger.warning(f"[Cache] Errore scrittura disco '{key}': {e}") + + +def _disk_delete(key: str) -> None: + try: + with _sqlite_lock: + conn = _ensure_sqlite() + row = conn.execute("SELECT blob_path FROM cache WHERE key = ?", (key,)).fetchone() + conn.execute("DELETE FROM cache WHERE key = ?", (key,)) + if row and row[0]: + try: + Path(row[0]).unlink(missing_ok=True) + except Exception: + pass + except Exception as e: + logger.warning(f"[Cache] Errore delete disco '{key}': {e}") + + +# ── API pubblica ───────────────────────────────────────────────────────── +def cache_get(key: str) -> Optional[Any]: + """Legge L1 → L2. Se L2 hit, ripopola L1 (re-warm).""" + # L1 + client = _get_redis() + if client is not None: + try: + raw = client.get(key) + if raw is not None: + return json.loads(gzip.decompress(raw).decode("utf-8")) + except Exception as e: + logger.warning(f"[Cache] Errore Redis '{key}': {e}") + + # L2 + value = _disk_get(key) + if value is not None and client is not None: + # Re-warm L1 con TTL standard + try: + raw_gz = gzip.compress(json.dumps(value).encode("utf-8")) + client.setex(key, DEFAULT_REDIS_TTL, raw_gz) + except Exception: + pass + return value + + +def cache_set(key: str, value: Any, ttl: int = DEFAULT_REDIS_TTL, disk_ttl: Optional[int] = None) -> bool: + """Scrive in L1 (ttl) e L2 (disk_ttl, default 30 giorni). + Per chiavi ephemere (es. job state) passa disk_ttl=0 per saltare il disco.""" + if disk_ttl is None: + disk_ttl = DEFAULT_DISK_TTL + try: + serialized = json.dumps(value).encode("utf-8") + raw_gz = gzip.compress(serialized) + except Exception as e: + logger.warning(f"[Cache] Errore serializzazione '{key}': {e}") return False + ok = False + # L1 + client = _get_redis() + if client is not None: + try: + client.setex(key, ttl, raw_gz) + ok = True + except Exception as e: + logger.warning(f"[Cache] Errore scrittura Redis '{key}': {e}") + + # L2 + if disk_ttl > 0: + _disk_set(key, raw_gz, disk_ttl) + ok = True + + return ok + def cache_delete(key: str) -> bool: - """Elimina una chiave dalla cache Redis. + client = _get_redis() + if client is not None: + try: + client.delete(key) + except Exception: + pass + _disk_delete(key) + return True - Args: - key: Chiave Redis da eliminare - Returns: - True se eliminata, False altrimenti - """ +def cache_stats() -> dict: + """Ritorna statistiche della cache: utile per /health e debug.""" + stats = {"redis": False, "disk": {"entries": 0, "bytes": 0, "blobs": 0}} + if _get_redis() is not None: + stats["redis"] = True try: - client = _get_client() - if client is None: - return False + with _sqlite_lock: + conn = _ensure_sqlite() + row = conn.execute( + "SELECT COUNT(*), COALESCE(SUM(size_bytes),0), COALESCE(SUM(is_blob),0) FROM cache" + ).fetchone() + stats["disk"]["entries"] = row[0] + stats["disk"]["bytes"] = row[1] + stats["disk"]["blobs"] = row[2] + except Exception: + pass + return stats - client.delete(key) - return True + +def cache_sweep() -> int: + """Rimuove voci scadute su disco (da chiamare periodicamente). Ritorna numero eliminate.""" + try: + now = int(time.time()) + with _sqlite_lock: + conn = _ensure_sqlite() + rows = conn.execute( + "SELECT key, blob_path FROM cache WHERE expires_at < ?", (now,) + ).fetchall() + conn.execute("DELETE FROM cache WHERE expires_at < ?", (now,)) + for _, path in rows: + if path: + try: + Path(path).unlink(missing_ok=True) + except Exception: + pass + return len(rows) except Exception as e: - logger.warning(f"[Redis] Errore eliminazione chiave '{key}': {e}") - return False + logger.warning(f"[Cache] Errore sweep: {e}") + return 0 diff --git a/copernicus/core/copernicus.py b/copernicus/core/copernicus.py index 1ab20d6..7ae624f 100644 --- a/copernicus/core/copernicus.py +++ b/copernicus/core/copernicus.py @@ -2,6 +2,7 @@ import hashlib import io import logging import os +import threading from datetime import datetime, timezone from typing import Callable, List, Optional @@ -11,13 +12,20 @@ from core.cache import cache_get, cache_set logger = logging.getLogger(__name__) -# ── Chiavi Redis e TTL ──────────────────────────────────────────────── +# Lock di "single-flight" per il fetch del catalogo Copernicus. +# Senza questo, N richieste concorrenti con cache miss farebbero N chiamate +# all'SDK (10-30s ciascuna, ~200MB di response). Con il lock, solo la prima +# scarica e popola la cache; le altre attendono e leggono da cache. +_catalog_fetch_lock = threading.Lock() + +# ── Chiavi cache e TTL ──────────────────────────────────────────────── # Chiave per il catalogo completo Copernicus _CATALOG_KEY = "marine:catalog:full" -# TTL del catalogo: 1 ora (il catalogo Copernicus cambia raramente) -_CATALOG_TTL = 3600 -# TTL per i risultati di ricerca: 30 minuti -_SEARCH_TTL = 1800 +# TTL L1 (Redis): 2 ore. L2 (disco) usa il default 30 giorni. +# Il catalogo Copernicus cambia raramente, ha senso tenerlo a lungo su disco. +_CATALOG_TTL = 2 * 3600 +# TTL L1 per le ricerche utente: 2 ore. Su disco 30 giorni. +_SEARCH_TTL = 2 * 3600 def _fmt_description(name: Optional[str]) -> Optional[str]: @@ -44,10 +52,17 @@ def _get_raw_catalog() -> dict: logger.debug("[Catalogo] Servito da cache Redis") return cached - # Cache miss: interroga Copernicus SDK (operazione lenta, ~10-30s) - logger.info("[Catalogo] Cache miss, scaricamento da Copernicus SDK...") - import copernicusmarine - catalog = copernicusmarine.describe(disable_progress_bar=True) + # Single-flight: solo un thread alla volta scarica il catalogo. Gli altri + # attendono il lock e poi leggono il valore appena messo in cache. + with _catalog_fetch_lock: + cached = cache_get(_CATALOG_KEY) + if cached is not None: + return cached + + # Cache miss: interroga Copernicus SDK (operazione lenta, ~10-30s) + logger.info("[Catalogo] Cache miss, scaricamento da Copernicus SDK...") + import copernicusmarine + catalog = copernicusmarine.describe(disable_progress_bar=True) # Serializza la risposta SDK in un dizionario standard if hasattr(catalog, "model_dump"): @@ -57,11 +72,11 @@ def _get_raw_catalog() -> dict: else: result = catalog - # Salva in Redis per le prossime richieste (TTL 1 ora) - cache_set(_CATALOG_KEY, result, _CATALOG_TTL) - logger.info("[Catalogo] Salvato in cache Redis") + # Salva in Redis per le prossime richieste (TTL 1 ora) + cache_set(_CATALOG_KEY, result, _CATALOG_TTL) + logger.info("[Catalogo] Salvato in cache Redis") - return result + return result def _get_dataset_reqs(ds: dict) -> tuple: diff --git a/copernicus/main.py b/copernicus/main.py index 3987b31..4dece26 100644 --- a/copernicus/main.py +++ b/copernicus/main.py @@ -12,11 +12,16 @@ from fastapi.middleware.cors import CORSMiddleware load_dotenv() from routers import catalog, datasets, jobs +from core.cache import cache_stats, cache_sweep @asynccontextmanager async def lifespan(app: FastAPI): api_url = os.getenv("API_SERVICE_URL", "http://api:3003") + # Pulizia voci scadute della cache su disco all'avvio + removed = cache_sweep() + if removed: + print(f"[Cache] Rimosse {removed} voci scadute dal disco") yield @@ -50,4 +55,9 @@ async def root(): @app.get("/health", tags=["health"]) async def health(): - return {"status": "healthy"} + return {"status": "healthy", "cache": cache_stats()} + + +@app.post("/cache/sweep", tags=["health"]) +async def sweep(): + return {"removed": cache_sweep()} diff --git a/copernicus/routers/jobs.py b/copernicus/routers/jobs.py index dbb7d66..c4d130d 100644 --- a/copernicus/routers/jobs.py +++ b/copernicus/routers/jobs.py @@ -7,6 +7,7 @@ Flusso: import json import os +import threading import uuid from typing import Any, Dict @@ -24,6 +25,13 @@ API_URL = os.getenv("API_SERVICE_URL", "http://api:3003") # TTL per lo stato dei job: 48 ore (i job completati vengono puliti automaticamente) _JOB_TTL = 48 * 3600 +# Limite di download Copernicus concorrenti. Le subset() dell'SDK sono +# CPU + memoria intensive (xarray + netCDF + pandas conversion) e sul server +# le risorse sono limitate. Senza semaforo, N utenti che cliccano insieme +# saturano la RAM e fanno OOM-kill del processo. +_DOWNLOAD_CONCURRENCY = int(os.getenv("MARINE_DOWNLOAD_CONCURRENCY", "2")) +_download_semaphore = threading.BoundedSemaphore(_DOWNLOAD_CONCURRENCY) + def _job_key(session_id: str) -> str: """Genera la chiave Redis per un job.""" @@ -42,7 +50,7 @@ def _set_job(session_id: str, **kwargs): if job is None: return job.update(kwargs) - cache_set(_job_key(session_id), job, _JOB_TTL) + cache_set(_job_key(session_id), job, _JOB_TTL, disk_ttl=0) def _run_download(session_id: str, req: DownloadJobRequest, username: str, user_token: str): @@ -55,20 +63,26 @@ def _run_download(session_id: str, req: DownloadJobRequest, username: str, user_ _set_job(session_id, progress=pct, message=msg) try: - _set_job(session_id, status="downloading", progress=5, message="Scarico da Copernicus Marine...") + _set_job(session_id, status="queued", progress=2, message="In coda (max concorrenti raggiunto)...") - # Scarica dati dal catalogo Copernicus - df = copernicus.download_dataset( - dataset_id=req.dataset_id, - variables=req.variables, - min_longitude=req.min_longitude, - max_longitude=req.max_longitude, - min_latitude=req.min_latitude, - max_latitude=req.max_latitude, - start_datetime=req.start_date, - end_datetime=req.end_date, - progress_callback=progress, - ) + # Acquisisce uno slot di download (blocca se già al limite). Garantisce + # che il numero di chiamate Copernicus simultanee non superi + # MARINE_DOWNLOAD_CONCURRENCY, proteggendo CPU/RAM del server. + with _download_semaphore: + _set_job(session_id, status="downloading", progress=5, message="Scarico da Copernicus Marine...") + + # Scarica dati dal catalogo Copernicus + df = copernicus.download_dataset( + dataset_id=req.dataset_id, + variables=req.variables, + min_longitude=req.min_longitude, + max_longitude=req.max_longitude, + min_latitude=req.min_latitude, + max_latitude=req.max_latitude, + start_datetime=req.start_date, + end_datetime=req.end_date, + progress_callback=progress, + ) _set_job(session_id, status="converting", progress=80, message="Creo il file...") @@ -85,7 +99,7 @@ def _run_download(session_id: str, req: DownloadJobRequest, username: str, user_ "created_by": username, "type": req.format, "notes": req.notes, - "copernicus_dataset_id": req.dataset_id, + "copernicus_id": req.dataset_id, "variables": req.variables, "variable_renames": req.variable_renames, "bbox": [req.min_longitude, req.min_latitude, req.max_longitude, req.max_latitude], @@ -129,7 +143,7 @@ async def new_download_session( "message": "In coda", "dataset_id": None, } - cache_set(_job_key(session_id), initial_state, _JOB_TTL) + cache_set(_job_key(session_id), initial_state, _JOB_TTL, disk_ttl=0) # Avvia il download in background background_tasks.add_task(_run_download, session_id, req, user["username"], user["token"]) diff --git a/copernicus/schemas.py b/copernicus/schemas.py index 2a97b0b..d1c8d5f 100644 --- a/copernicus/schemas.py +++ b/copernicus/schemas.py @@ -65,7 +65,7 @@ class DatasetMeta(BaseModel): notes: str = "" version: int = 1 filename: str - copernicus_dataset_id: str + copernicus_id: str variables: List[str] = [] bbox: List[float] = [] # [min_lon, min_lat, max_lon, max_lat] start_date: str diff --git a/docker-compose.yml b/docker-compose.yml index 4006b75..30e2cc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,8 +106,12 @@ services: context: ./ml dockerfile: Dockerfile restart: unless-stopped + command: uvicorn main:app --host 0.0.0.0 --port 3007 --reload volumes: - ./ml:/app + - /var/run/docker.sock:/var/run/docker.sock + - ml_tmp:/var/ml/tmp + - ml_gitcache:/var/ml/gitcache env_file: - ./ml/.env networks: @@ -117,34 +121,43 @@ services: - "traefik.enable=true" - "traefik.http.routers.ml.rule=Host(`ml.${DOMAIN:-mebboat.it}`)" - "traefik.http.routers.ml.entrypoints=websecure" - - "traefik.http.services.ml.loadbalancer.server.port=8000" + - "traefik.http.services.ml.loadbalancer.server.port=3007" - "traefik.http.routers.ml.tls.certresolver=letsencrypt" - "traefik.docker.network=meb-public" -# marine: -# container_name: marine-service -# build: -# context: ./marine -# dockerfile: Dockerfile -# restart: unless-stopped -# volumes: -# - ./marine:/app -# env_file: -# - ./marine/.env -# environment: -# - REDIS_HOST=meb-redis -# - REDIS_PORT=6379 -# networks: -# - meb-proxy-net -# - meb-internal -# labels: -# - "traefik.enable=true" -# - "traefik.http.routers.marine.rule=Host(`api.${DOMAIN:-mebboat.it}`) && PathPrefix(`/marine`)" -# - "traefik.http.routers.marine.entrypoints=web" -# - "traefik.http.services.marine.loadbalancer.server.port=8001" -# - "traefik.docker.network=meb-proxy-net" -# - "traefik.http.middlewares.marine-strip.stripprefix.prefixes=/marine" -# - "traefik.http.routers.marine.middlewares=marine-strip" + copernicus: + container_name: copernicus-service + build: + context: ./copernicus + dockerfile: Dockerfile + restart: unless-stopped + volumes: + - ./copernicus:/app + - copernicus_cache:/app/cache + env_file: + - ./copernicus/.env + environment: + - REDIS_HOST=meb-redis + - REDIS_PORT=6379 + - API_SERVICE_URL=http://api:3003 + - CACHE_DIR=/app/cache + - MINIO_ENDPOINT=minio + - MINIO_PORT=9000 + networks: + - meb-public + - meb-private + labels: + - "traefik.enable=true" + # Esponi sotto api.mebboat.it/marine/* (Traefik strippa "/marine") + - "traefik.http.routers.copernicus.rule=Host(`api.${DOMAIN:-mebboat.it}`) && PathPrefix(`/marine`)" + - "traefik.http.routers.copernicus.entrypoints=websecure" + - "traefik.http.routers.copernicus.tls.certresolver=letsencrypt" + - "traefik.http.services.copernicus.loadbalancer.server.port=8000" + - "traefik.docker.network=meb-public" + - "traefik.http.middlewares.copernicus-strip.stripprefix.prefixes=/marine" + - "traefik.http.routers.copernicus.middlewares=copernicus-strip" + # Priorità alta: la regola col PathPrefix deve vincere su quella generica api. + - "traefik.http.routers.copernicus.priority=100" # circuits: # container_name: meb-circuits @@ -184,3 +197,8 @@ networks: external: true meb-private: external: true + +volumes: + copernicus_cache: + ml_tmp: + ml_gitcache: diff --git a/ml/.env.example b/ml/.env.example index e69de29..6360b49 100644 --- a/ml/.env.example +++ b/ml/.env.example @@ -0,0 +1,45 @@ +PORT=3007 + +# Auth condiviso +JWT_SECRET=change-me +INTERNAL_API_KEY=change-me +AUTH_LOGIN_URL=https://auth.mebboat.it/login + +# Postgres (db ml) +PG_HOST=meb-postgres +PG_PORT=5432 +DB_USER=meb +DB_PASSWORD=meb +ML_DB=ml + +# Redis +REDIS_HOST=meb-redis +REDIS_PORT=6379 + +# MinIO (bucket unico) +MINIO_ENDPOINT=minio +MINIO_PORT=9000 +MINIO_USE_SSL=false +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +MINIO_BUCKET=ml + +# InfluxDB +INFLUX_URL=http://meb-influx:8086 +INFLUX_TOKEN= +INFLUX_ORG=meb +INFLUX_BUCKET=ml_metrics + +# Gitea (self-hosted esterno) +GITEA_URL=https://git.mebboat.it +GITEA_TOKEN= + +# API service +API_URL=http://api:3003 + +# Training runtime +ML_TRAIN_CONCURRENCY=1 +ML_RUNNER_IMAGE=meb-ml-runner:latest +ML_RUNNER_TMP=/var/ml/tmp +ML_GITCACHE_DIR=/var/ml/gitcache +ML_MAX_UPLOAD_MB=500 diff --git a/ml/Dockerfile b/ml/Dockerfile index cc0f8cd..96f65e7 100644 --- a/ml/Dockerfile +++ b/ml/Dockerfile @@ -3,6 +3,9 @@ FROM python:3.11-slim WORKDIR /app ENV PYTHONUNBUFFERED=1 +RUN apt-get update && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + COPY ./requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/ml/core/api_client.py b/ml/core/api_client.py new file mode 100644 index 0000000..2361a84 --- /dev/null +++ b/ml/core/api_client.py @@ -0,0 +1,72 @@ +"""Client HTTP verso l'api-service (service-to-service via x-api-key). + +Espone accesso a: + /jobs ciclo di vita job + /queue stato coda + /pageconnections registro sessioni di pagina (enforcement /test max 2) +""" +from __future__ import annotations + +from typing import Any, Optional + +import httpx + +from core.config import settings + + +def _headers() -> dict: + return {"x-api-key": settings.internal_api_key, "Content-Type": "application/json"} + + +async def _req(method: str, path: str, json: Optional[dict] = None, params: Optional[dict] = None) -> Any: + url = f"{settings.api_url}{path}" + async with httpx.AsyncClient(timeout=10.0) as c: + r = await c.request(method, url, json=json, params=params, headers=_headers()) + r.raise_for_status() + if r.status_code == 204 or not r.content: + return None + return r.json() + + +# ── jobs ──────────────────────────────────────────────────────────────────── +async def create_job(type_: str, created_by: str, payload: dict) -> dict: + return await _req("POST", "/jobs", json={"type": type_, "created_by": created_by, "payload": payload}) + + +async def update_job(job_id: str, **fields) -> dict: + return await _req("PATCH", f"/jobs/{job_id}", json=fields) + + +async def get_job(job_id: str) -> dict: + return await _req("GET", f"/jobs/{job_id}") + + +async def list_jobs(type_: Optional[str] = None, status: Optional[str] = None, limit: int = 50) -> list: + params = {"limit": str(limit)} + if type_: + params["type"] = type_ + if status: + params["status"] = status + return await _req("GET", "/jobs", params=params) or [] + + +# ── queue ─────────────────────────────────────────────────────────────────── +async def queue_status(type_: str = "train") -> dict: + return await _req("GET", "/queue", params={"type": type_}) + + +# ── page connections ─────────────────────────────────────────────────────── +async def page_connect(page: str, user_id: str, session_id: str) -> dict: + return await _req("POST", "/pageconnections", json={"page": page, "user_id": user_id, "session_id": session_id}) + + +async def page_ping(session_id: str) -> dict: + return await _req("POST", f"/pageconnections/{session_id}/ping") + + +async def page_disconnect(session_id: str) -> None: + await _req("DELETE", f"/pageconnections/{session_id}") + + +async def page_count(page: str) -> dict: + return await _req("GET", f"/pageconnections/{page}") diff --git a/ml/core/config.py b/ml/core/config.py new file mode 100644 index 0000000..8529c17 --- /dev/null +++ b/ml/core/config.py @@ -0,0 +1,64 @@ +"""Configurazione centralizzata del servizio ML, letta da env.""" +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _b(name: str, default: bool = False) -> bool: + return os.environ.get(name, str(default)).lower() in ("1", "true", "yes", "on") + + +@dataclass(frozen=True) +class Settings: + # Postgres (db "ml") + pg_host: str = os.environ.get("PG_HOST", "meb-postgres") + pg_port: int = int(os.environ.get("PG_PORT", "5432")) + pg_user: str = os.environ.get("DB_USER", "meb") + pg_password: str = os.environ.get("DB_PASSWORD", "meb") + pg_db: str = os.environ.get("ML_DB", "ml") + + # Redis + redis_host: str = os.environ.get("REDIS_HOST", "meb-redis") + redis_port: int = int(os.environ.get("REDIS_PORT", "6379")) + + # MinIO (bucket unico) + minio_endpoint: str = os.environ.get("MINIO_ENDPOINT", "minio") + minio_port: int = int(os.environ.get("MINIO_PORT", "9000")) + minio_use_ssl: bool = _b("MINIO_USE_SSL", False) + minio_access_key: str = os.environ.get("MINIO_ACCESS_KEY", "") + minio_secret_key: str = os.environ.get("MINIO_SECRET_KEY", "") + minio_bucket: str = os.environ.get("MINIO_BUCKET", "ml") + + # InfluxDB — accetta sia INFLUX_* che INFLX_* per allinearsi alle var già + # usate dagli altri servizi (realtime, api) senza dover duplicare la config. + influx_url: str = os.environ.get("INFLUX_URL") or os.environ.get("INFLX_URL", "http://meb-influx:8086") + influx_token: str = os.environ.get("INFLUX_TOKEN") or os.environ.get("INFLX_TOKEN", "") + influx_org: str = os.environ.get("INFLUX_ORG") or os.environ.get("INFLX_ORG", "meb") + # Bucket dedicato alle metriche di training/test ML, separato dai logs e + # dai dati meteo. Sovrascrivibile via INFLUX_BUCKET o ML_INFLUX_BUCKET. + influx_bucket: str = os.environ.get("ML_INFLUX_BUCKET") or os.environ.get("INFLUX_BUCKET", "ml_metrics") + + # Gitea (installato esternamente) + gitea_url: str = os.environ.get("GITEA_URL", "") + gitea_token: str = os.environ.get("GITEA_TOKEN", "") + + # API service (per jobs/queue/pageconnections) + api_url: str = os.environ.get("API_URL", "http://api:3003") + internal_api_key: str = os.environ.get("INTERNAL_API_KEY", "") + + # Auth (condiviso) + jwt_secret: str = os.environ.get("JWT_SECRET", "") + auth_login_url: str = os.environ.get("AUTH_LOGIN_URL", "https://auth.mebboat.it/login") + + # Esecuzione training + train_concurrency: int = int(os.environ.get("ML_TRAIN_CONCURRENCY", "1")) + runner_image: str = os.environ.get("ML_RUNNER_IMAGE", "meb-ml-runner:latest") + runner_tmp_dir: str = os.environ.get("ML_RUNNER_TMP", "/var/ml/tmp") + gitcache_dir: str = os.environ.get("ML_GITCACHE_DIR", "/var/ml/gitcache") + + # Limiti runtime + max_upload_mb: int = int(os.environ.get("ML_MAX_UPLOAD_MB", "500")) + + +settings = Settings() diff --git a/ml/core/db.py b/ml/core/db.py new file mode 100644 index 0000000..575fc77 --- /dev/null +++ b/ml/core/db.py @@ -0,0 +1,53 @@ +"""Connessione asyncpg al database ml. Pool singleton.""" +from __future__ import annotations + +import asyncpg +from typing import Optional + +from core.config import settings + +_pool: Optional[asyncpg.Pool] = None + + +async def init_pool() -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool( + host=settings.pg_host, + port=settings.pg_port, + user=settings.pg_user, + password=settings.pg_password, + database=settings.pg_db, + min_size=1, + max_size=10, + command_timeout=30, + ) + return _pool + + +async def close_pool() -> None: + global _pool + if _pool is not None: + await _pool.close() + _pool = None + + +def pool() -> asyncpg.Pool: + if _pool is None: + raise RuntimeError("DB pool not initialized — call init_pool() at startup") + return _pool + + +async def fetch(sql: str, *args): + async with pool().acquire() as c: + return await c.fetch(sql, *args) + + +async def fetchrow(sql: str, *args): + async with pool().acquire() as c: + return await c.fetchrow(sql, *args) + + +async def execute(sql: str, *args): + async with pool().acquire() as c: + return await c.execute(sql, *args) diff --git a/ml/core/docker_runner.py b/ml/core/docker_runner.py new file mode 100644 index 0000000..9ff9eca --- /dev/null +++ b/ml/core/docker_runner.py @@ -0,0 +1,439 @@ +"""Runner Docker per train e test. + +train: + - clone repo Gitea @ sha + - prepara workdir /var/ml/tmp/{training_id} + - scarica dataset da MinIO in workdir/data. + - docker run meb-ml-runner con mount tmp, env, limits da model.yml + - legge stdout JSON → Redis stream + Influx; docker stats ogni 5s + - a fine: collect outputs, upload su MinIO prefix artifacts_prefix + - UPDATE trainings + +test: + - analogo ma sincrono, stdin JSON → stdout JSON +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import shutil +import subprocess +import time +import uuid +from pathlib import Path +from typing import Any, Optional + +import docker +from influxdb_client import Point + +from core import db, gitea, influx_client, minio_client, redis_client +from core.config import settings +from core.model_spec import fetch_and_parse_spec + +log = logging.getLogger(__name__) + +_docker = None + + +def _docker_client(): + global _docker + if _docker is None: + _docker = docker.from_env() + return _docker + + +async def _emit(stream_key: str, payload: dict) -> None: + try: + await redis_client.client().xadd(stream_key, {"payload": json.dumps(payload)}, maxlen=10_000) + except Exception as e: + log.warning("xadd failed: %s", e) + + +async def _clone_repo(owner_repo: str, sha: str, dest: Path) -> None: + dest.mkdir(parents=True, exist_ok=True) + url = gitea.clone_url(owner_repo) + # clone shallow del branch/sha specifico + # per evitare leak del token nei log, logghiamo solo host + proc = await asyncio.create_subprocess_exec( + "git", "clone", "--depth", "50", url, str(dest), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + _, err = await proc.communicate() + if proc.returncode != 0: + raise RuntimeError(f"git clone failed: {err.decode(errors='replace')[:400]}") + # checkout sha + proc = await asyncio.create_subprocess_exec( + "git", "-C", str(dest), "checkout", sha, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + _, err = await proc.communicate() + if proc.returncode != 0: + raise RuntimeError(f"git checkout failed: {err.decode(errors='replace')[:400]}") + + +async def _download_dataset(dataset_id: str, dest: Path) -> str: + row = await db.fetchrow( + "SELECT file_key, format FROM datasets WHERE id = $1", uuid.UUID(dataset_id) + ) + if not row: + raise RuntimeError("dataset not found") + data = minio_client.get_bytes(row["file_key"], bucket="ml.datasets") + ext = {"csv": "csv", "json": "json", "netcdf": "nc"}.get(row["format"], "bin") + out = dest / f"data.{ext}" + out.write_bytes(data) + return str(out) + + +def _stats_loop_sync(container, training_id: str, model_id: str, samples: list, stop_evt: asyncio.Event, loop: asyncio.AbstractEventLoop): + """Sincrono, eseguito in thread. Ogni 5s legge docker stats → Influx + samples.""" + while not stop_evt.is_set(): + try: + stats = container.stats(stream=False) + # CPU% + cpu_delta = stats["cpu_stats"]["cpu_usage"]["total_usage"] - stats["precpu_stats"]["cpu_usage"]["total_usage"] + sys_delta = stats["cpu_stats"].get("system_cpu_usage", 0) - stats["precpu_stats"].get("system_cpu_usage", 0) + online = stats["cpu_stats"].get("online_cpus") or len(stats["cpu_stats"]["cpu_usage"].get("percpu_usage") or [1]) + cpu_pct = (cpu_delta / sys_delta) * online * 100.0 if sys_delta > 0 else 0.0 + mem_mb = (stats["memory_stats"].get("usage") or 0) / (1024 * 1024) + + samples.append((cpu_pct, mem_mb)) + point = ( + Point("ml_training") + .tag("training_id", training_id) + .tag("model_id", model_id) + .field("cpu_pct", float(cpu_pct)) + .field("mem_mb", float(mem_mb)) + ) + asyncio.run_coroutine_threadsafe(influx_client.write_points([point]), loop) + except Exception as e: + log.warning("stats loop error: %s", e) + time.sleep(5) + + +async def _stream_container_logs(container, training_id: str, model_id: str, stream_key: str): + """Legge stdout del container, pubblica righe JSON su Redis stream e Influx.""" + def _iter(): + return container.logs(stream=True, follow=True, stdout=True, stderr=True) + + loop = asyncio.get_event_loop() + it = await loop.run_in_executor(None, _iter) + + while True: + line = await loop.run_in_executor(None, next, it, None) + if line is None: + break + try: + text = line.decode("utf-8", errors="replace").rstrip("\n") + except Exception: + continue + if not text: + continue + # righe non-JSON → log + payload: dict + if text.startswith("{") and text.endswith("}"): + try: + payload = json.loads(text) + except json.JSONDecodeError: + payload = {"type": "log", "level": "info", "message": text} + else: + payload = {"type": "log", "level": "info", "message": text} + + await _emit(stream_key, payload) + + if payload.get("type") == "metric": + p = Point("ml_training").tag("training_id", training_id).tag("model_id", model_id) + for k, v in payload.items(): + if k == "type": + continue + if isinstance(v, (int, float)): + p = p.field(k, float(v)) + try: + await influx_client.write_points([p]) + except Exception as e: + log.warning("influx write metric failed: %s", e) + + +async def run_training_job(training_id: str) -> None: + """Esegue un job di training end-to-end. Aggiorna Postgres e Redis state.""" + r = redis_client.client() + state_key = f"ml:train:{training_id}" + stream_key = f"ml:train:{training_id}:events" + + tr = await db.fetchrow("SELECT * FROM trainings WHERE id = $1", uuid.UUID(training_id)) + if not tr: + log.error("training %s not found", training_id) + return + model = await db.fetchrow("SELECT * FROM models WHERE id = $1", tr["model_id"]) + if not model: + await db.execute( + "UPDATE trainings SET status='failed', error=$2 WHERE id=$1", + uuid.UUID(training_id), "model not found", + ) + return + + await db.execute( + "UPDATE trainings SET status='running', started_at=NOW() WHERE id=$1", + uuid.UUID(training_id), + ) + await r.hset(state_key, mapping={"status": "running", "progress": "0", "message": "starting"}) + + workdir = Path(settings.runner_tmp_dir) / training_id + artifacts_prefix = f"models/{tr['model_id']}/{tr['version']}/{tr['patch']}" + error: Optional[str] = None + samples: list[tuple[float, float]] = [] + try: + workdir.mkdir(parents=True, exist_ok=True) + await _emit(stream_key, {"type": "log", "level": "info", "message": "cloning repo"}) + await _clone_repo(model["gitea_repo"], tr["patch"], workdir / "repo") + + await _emit(stream_key, {"type": "log", "level": "info", "message": "parsing model.yml"}) + spec = await fetch_and_parse_spec(model["gitea_repo"], tr["patch"]) or {} + train_spec = spec.get("train", {}) + entrypoint = train_spec.get("entrypoint") or "python -m src.train" + resources = spec.get("resources", {}) or {} + + await _emit(stream_key, {"type": "log", "level": "info", "message": "downloading dataset"}) + dataset_path = await _download_dataset(str(tr["dataset_id"]), workdir) + + out_dir = workdir / "out" + out_dir.mkdir(exist_ok=True) + + # docker run + dc = _docker_client() + await _emit(stream_key, {"type": "log", "level": "info", "message": "starting container"}) + container = dc.containers.run( + settings.runner_image, + command=["sh", "-c", f"cd /workdir/repo && pip install -q -r requirements.txt 2>&1 || true && {entrypoint}"], + detach=True, + working_dir="/workdir/repo", + environment={ + "MEB_DATASET_PATH": f"/workdir/{Path(dataset_path).name}", + "MEB_ARTIFACTS_DIR": "/workdir/out", + "MEB_TRAINING_ID": training_id, + }, + volumes={str(workdir): {"bind": "/workdir", "mode": "rw"}}, + network_mode="none", + mem_limit=f"{int(resources.get('mem_mb', 2048))}m", + nano_cpus=int(float(resources.get("cpu", 1)) * 1e9), + read_only=False, + tty=False, + detach_mode=None, + ) + + loop = asyncio.get_event_loop() + stop_evt = asyncio.Event() + stats_task = loop.run_in_executor( + None, _stats_loop_sync, container, training_id, str(tr["model_id"]), samples, stop_evt, loop + ) + log_task = asyncio.create_task( + _stream_container_logs(container, training_id, str(tr["model_id"]), stream_key) + ) + + # attendi exit + exit_code = await loop.run_in_executor(None, lambda: container.wait()["StatusCode"]) + stop_evt.set() + await log_task + try: + stats_task.cancel() + except Exception: + pass + + if exit_code != 0: + error = f"container exited with code {exit_code}" + + # raccogli outputs + results: dict = {} + final_metrics_path = out_dir / "metrics.json" + if final_metrics_path.exists(): + try: + results = json.loads(final_metrics_path.read_text()) + except Exception: + results = {"raw": final_metrics_path.read_text()[:10000]} + + # upload artefatti (tutta la cartella out/) + for p in out_dir.rglob("*"): + if p.is_file(): + rel = p.relative_to(out_dir).as_posix() + key = f"{artifacts_prefix}/{rel}" + minio_client.put_bytes(key, p.read_bytes()) + + # upload logs jsonl dallo stream redis (copia su minio per persistenza) + try: + entries = await r.xrange(stream_key, min="-", max="+") + lines = "\n".join(json.dumps({"id": i, **({"payload": json.loads(f.get("payload", "{}"))} if "payload" in f else f)}) for i, f in entries) + minio_client.put_bytes(f"trainings/{training_id}/logs.jsonl", lines.encode("utf-8"), "application/x-ndjson") + except Exception as e: + log.warning("log archive failed: %s", e) + + cpu_avg = sum(s[0] for s in samples) / len(samples) if samples else 0.0 + cpu_peak = max((s[0] for s in samples), default=0.0) + mem_avg = sum(s[1] for s in samples) / len(samples) if samples else 0.0 + mem_peak = max((s[1] for s in samples), default=0.0) + resource_summary = { + "cpu_avg": round(cpu_avg, 2), + "cpu_peak": round(cpu_peak, 2), + "mem_avg_mb": round(mem_avg, 2), + "mem_peak_mb": round(mem_peak, 2), + "samples": len(samples), + } + + status = "failed" if error else "succeeded" + await db.execute( + """ + UPDATE trainings SET + status=$2, + finished_at=NOW(), + duration_ms=EXTRACT(EPOCH FROM (NOW() - started_at))*1000, + artifacts_prefix=$3, + results=$4::jsonb, + resource_summary=$5::jsonb, + error=$6 + WHERE id=$1 + """, + uuid.UUID(training_id), + status, + artifacts_prefix, + json.dumps(results), + json.dumps(resource_summary), + error, + ) + await r.hset(state_key, mapping={"status": status, "progress": "100", "message": error or "done"}) + await _emit(stream_key, {"type": "end", "status": status, "error": error}) + + # Flush dei punti Influx accumulati durante il training (batched). + await influx_client.flush() + + try: + container.remove(force=True) + except Exception: + pass + + except Exception as e: + log.exception("training %s failed: %s", training_id, e) + await db.execute( + "UPDATE trainings SET status='failed', finished_at=NOW(), error=$2 WHERE id=$1", + uuid.UUID(training_id), str(e)[:1000], + ) + await r.hset(state_key, mapping={"status": "failed", "message": str(e)[:200]}) + await _emit(stream_key, {"type": "end", "status": "failed", "error": str(e)[:400]}) + finally: + # cleanup workdir + try: + shutil.rmtree(workdir, ignore_errors=True) + except Exception: + pass + + +async def run_test_once(training_id: str, inputs: dict) -> dict: + """Esegue una singola predizione via container spawn.""" + tr = await db.fetchrow( + "SELECT t.*, m.gitea_repo FROM trainings t JOIN models m ON t.model_id = m.id WHERE t.id=$1", + uuid.UUID(training_id), + ) + if not tr: + raise RuntimeError("training not found") + + spec = await fetch_and_parse_spec(tr["gitea_repo"], tr["patch"]) or {} + test_spec = spec.get("test") or {} + entrypoint = test_spec.get("entrypoint") or "python -m src.predict" + + workdir = Path(settings.runner_tmp_dir) / f"test-{uuid.uuid4()}" + workdir.mkdir(parents=True, exist_ok=True) + try: + await _clone_repo(tr["gitea_repo"], tr["patch"], workdir / "repo") + + # scarica artefatti + if tr["artifacts_prefix"]: + art_dir = workdir / "artifacts" + art_dir.mkdir(exist_ok=True) + for obj in minio_client.list_prefix(tr["artifacts_prefix"] + "/"): + rel = obj["name"][len(tr["artifacts_prefix"]) + 1:] + out_path = art_dir / rel + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(minio_client.get_bytes(obj["name"])) + + dc = _docker_client() + payload = json.dumps({"inputs": inputs}).encode() + container = dc.containers.run( + settings.runner_image, + command=["sh", "-c", f"cd /workdir/repo && pip install -q -r requirements.txt 2>&1 >/dev/null || true && {entrypoint}"], + detach=True, + working_dir="/workdir/repo", + environment={ + "MEB_ARTIFACTS_DIR": "/workdir/artifacts", + "MEB_TRAINING_ID": training_id, + }, + volumes={str(workdir): {"bind": "/workdir", "mode": "ro"}}, + network_mode="none", + mem_limit="2048m", + nano_cpus=int(1e9), + stdin_open=True, + tty=False, + ) + + # scrivi input su stdin via attach socket + sock = container.attach_socket(params={"stdin": 1, "stream": 1}) + try: + sock._sock.sendall(payload + b"\n") + except Exception: + pass + try: + sock.close() + except Exception: + pass + + loop = asyncio.get_event_loop() + # stats peak + peak_cpu = 0.0 + peak_mem = 0.0 + stop = False + + def _stats(): + nonlocal peak_cpu, peak_mem, stop + for st in container.stats(stream=True, decode=True): + if stop: + return + try: + cpu_delta = st["cpu_stats"]["cpu_usage"]["total_usage"] - st["precpu_stats"]["cpu_usage"]["total_usage"] + sys_delta = st["cpu_stats"].get("system_cpu_usage", 0) - st["precpu_stats"].get("system_cpu_usage", 0) + online = st["cpu_stats"].get("online_cpus") or 1 + cpu_pct = (cpu_delta / sys_delta) * online * 100 if sys_delta > 0 else 0 + mem_mb = (st["memory_stats"].get("usage") or 0) / (1024 * 1024) + peak_cpu = max(peak_cpu, cpu_pct) + peak_mem = max(peak_mem, mem_mb) + except Exception: + pass + + stats_fut = loop.run_in_executor(None, _stats) + + exit_info = await loop.run_in_executor(None, container.wait) + stop = True + logs = container.logs(stdout=True, stderr=False).decode("utf-8", errors="replace") + try: + container.remove(force=True) + except Exception: + pass + + outputs: dict = {} + for line in logs.strip().splitlines(): + line = line.strip() + if line.startswith("{") and line.endswith("}"): + try: + obj = json.loads(line) + if "outputs" in obj: + outputs = obj["outputs"] + break + except json.JSONDecodeError: + continue + + return { + "outputs": outputs, + "exit_code": exit_info.get("StatusCode"), + "cpu_peak": round(peak_cpu, 2), + "mem_peak_mb": round(peak_mem, 2), + "raw_log": logs[-2000:], + } + finally: + shutil.rmtree(workdir, ignore_errors=True) diff --git a/ml/core/gitea.py b/ml/core/gitea.py new file mode 100644 index 0000000..b81d2cb --- /dev/null +++ b/ml/core/gitea.py @@ -0,0 +1,57 @@ +"""Client Gitea: browse repo, branches, commits, file raw, clone URL autenticato.""" +from __future__ import annotations + +from typing import Optional + +import httpx + +from core.config import settings + + +def _headers() -> dict: + h = {"Accept": "application/json"} + if settings.gitea_token: + h["Authorization"] = f"token {settings.gitea_token}" + return h + + +def clone_url(owner_repo: str) -> str: + """URL https://oauth2:TOKEN@/owner/repo.git — usato SOLO lato server.""" + if not settings.gitea_url: + raise RuntimeError("GITEA_URL not configured") + base = settings.gitea_url.rstrip("/") + if settings.gitea_token: + base = base.replace("https://", f"https://oauth2:{settings.gitea_token}@").replace( + "http://", f"http://oauth2:{settings.gitea_token}@" + ) + return f"{base}/{owner_repo}.git" + + +async def _get(path: str, params: Optional[dict] = None) -> list | dict: + url = f"{settings.gitea_url.rstrip('/')}/api/v1{path}" + async with httpx.AsyncClient(timeout=15.0) as c: + r = await c.get(url, params=params, headers=_headers()) + r.raise_for_status() + return r.json() + + +async def list_repos(limit: int = 50) -> list[dict]: + data = await _get("/repos/search", params={"limit": str(limit)}) + return data.get("data", []) if isinstance(data, dict) else [] + + +async def list_branches(owner_repo: str) -> list[dict]: + return await _get(f"/repos/{owner_repo}/branches") + + +async def list_commits(owner_repo: str, branch: str = "main", limit: int = 50) -> list[dict]: + return await _get(f"/repos/{owner_repo}/commits", params={"sha": branch, "limit": str(limit)}) + + +async def get_file_raw(owner_repo: str, ref: str, path: str) -> bytes: + """Scarica il file raw alla revisione indicata.""" + url = f"{settings.gitea_url.rstrip('/')}/api/v1/repos/{owner_repo}/raw/{path}" + async with httpx.AsyncClient(timeout=15.0) as c: + r = await c.get(url, params={"ref": ref}, headers=_headers()) + r.raise_for_status() + return r.content diff --git a/ml/core/influx_client.py b/ml/core/influx_client.py new file mode 100644 index 0000000..9438d44 --- /dev/null +++ b/ml/core/influx_client.py @@ -0,0 +1,75 @@ +"""Client InfluxDB (influxdb-client sync wrapper in thread-pool per async). + +Le scritture usano il batching async dell'SDK invece di SYNCHRONOUS. +Le metriche di training arrivano in burst (logs container, stats loop ogni 5s): +con SYNCHRONOUS ogni write era una HTTP request bloccante. Con WriteOptions +batched, l'SDK accumula i Point e fa flush periodico in background, senza +perdere durabilità (flush forzato a fine training). +""" +from __future__ import annotations + +import asyncio +from typing import Iterable, Optional + +from influxdb_client import InfluxDBClient, Point, WriteOptions + +from core.config import settings + +_client: Optional[InfluxDBClient] = None +_write_api = None + + +def client() -> InfluxDBClient: + global _client, _write_api + if _client is None: + _client = InfluxDBClient( + url=settings.influx_url, token=settings.influx_token, org=settings.influx_org + ) + _write_api = _client.write_api(write_options=WriteOptions( + batch_size=200, + flush_interval=2_000, + jitter_interval=200, + retry_interval=2_000, + max_retries=3, + )) + return _client + + +def _wa(): + client() + return _write_api + + +async def write_points(points: Iterable[Point]) -> None: + wa = _wa() + pts = list(points) + await asyncio.to_thread(wa.write, settings.influx_bucket, settings.influx_org, pts) + + +async def flush() -> None: + """Forza il flush del buffer batched. Da chiamare a fine training per + garantire che tutte le metriche raccolte siano persistite.""" + if _write_api is None: + return + try: + await asyncio.to_thread(_write_api.flush) + except Exception: + pass + + +async def query_flux(flux: str) -> list[dict]: + c = client() + def _q(): + tables = c.query_api().query(flux, org=settings.influx_org) + out = [] + for table in tables: + for r in table.records: + out.append({ + "time": r.get_time().isoformat() if r.get_time() else None, + "measurement": r.get_measurement(), + "field": r.get_field(), + "value": r.get_value(), + "tags": {k: v for k, v in r.values.items() if k.startswith("_") is False and k not in ("result", "table")}, + }) + return out + return await asyncio.to_thread(_q) diff --git a/ml/core/minio_client.py b/ml/core/minio_client.py new file mode 100644 index 0000000..7d1d5b9 --- /dev/null +++ b/ml/core/minio_client.py @@ -0,0 +1,118 @@ +"""Wrapper MinIO: bucket unico (settings.minio_bucket) con prefissi logici. + +Prefissi usati: + datasets/. + models//spec.yml + models////... (artefatti training) + trainings//logs.jsonl +""" +from __future__ import annotations + +import io +from datetime import timedelta +from typing import Iterable, Optional + +from minio import Minio +from minio.error import S3Error + +from core.config import settings + + +_client: Optional[Minio] = None + + +def client() -> Minio: + global _client + if _client is None: + _client = Minio( + f"{settings.minio_endpoint}:{settings.minio_port}", + access_key=settings.minio_access_key, + secret_key=settings.minio_secret_key, + secure=settings.minio_use_ssl, + ) + return _client + + +def _bucket(b: Optional[str] = None) -> str: + return b or settings.minio_bucket + + +def ensure_bucket(bucket: Optional[str] = None) -> None: + name = _bucket(bucket) + c = client() + if not c.bucket_exists(name): + c.make_bucket(name) + + +def put_bytes(key: str, data: bytes, content_type: str = "application/octet-stream", + bucket: Optional[str] = None) -> None: + ensure_bucket(bucket) + client().put_object( + _bucket(bucket), + key, + io.BytesIO(data), + length=len(data), + content_type=content_type, + ) + + +def put_stream(key: str, stream, length: int, content_type: str = "application/octet-stream", + bucket: Optional[str] = None) -> None: + ensure_bucket(bucket) + client().put_object( + _bucket(bucket), key, stream, length=length, content_type=content_type + ) + + +def get_bytes(key: str, bucket: Optional[str] = None) -> bytes: + r = client().get_object(_bucket(bucket), key) + try: + return r.read() + finally: + r.close() + r.release_conn() + + +def remove(key: str, bucket: Optional[str] = None) -> None: + try: + client().remove_object(_bucket(bucket), key) + except S3Error: + pass + + +def remove_prefix(prefix: str, bucket: Optional[str] = None) -> int: + name = _bucket(bucket) + n = 0 + for obj in client().list_objects(name, prefix=prefix, recursive=True): + try: + client().remove_object(name, obj.object_name) + n += 1 + except S3Error: + pass + return n + + +def presigned_get(key: str, expires_seconds: int = 3600, bucket: Optional[str] = None) -> str: + return client().presigned_get_object( + _bucket(bucket), key, expires=timedelta(seconds=expires_seconds) + ) + + +def list_prefix(prefix: str, bucket: Optional[str] = None) -> list[dict]: + out = [] + for obj in client().list_objects(_bucket(bucket), prefix=prefix, recursive=True): + out.append({ + "name": obj.object_name, + "size": obj.size, + "last_modified": obj.last_modified.isoformat() if obj.last_modified else None, + "etag": obj.etag, + }) + return out + + +def check() -> bool: + try: + client().list_buckets() + return True + except Exception: + return False diff --git a/ml/core/model_spec.py b/ml/core/model_spec.py new file mode 100644 index 0000000..3448378 --- /dev/null +++ b/ml/core/model_spec.py @@ -0,0 +1,90 @@ +"""Parse e validazione del contratto `model.yml` nelle repo utente. + +Schema sintetico (vedi piano): + name, type, version, python + train: {entrypoint, inputs, outputs, metrics} + test: {entrypoint, io, input_schema[], output_schema[]} + resources: {cpu, mem_mb, gpu} +""" +from __future__ import annotations + +from typing import Any, Optional + +import yaml +from pydantic import BaseModel, ValidationError + +from core import gitea, redis_client + + +class _FieldSpec(BaseModel): + name: str + dtype: str + min: Optional[float] = None + max: Optional[float] = None + unit: Optional[str] = None + + +class _Train(BaseModel): + entrypoint: str + inputs: dict = {} + outputs: dict = {} + metrics: dict = {} + + +class _Test(BaseModel): + entrypoint: str + io: str = "stdio_json" + input_schema: list[_FieldSpec] = [] + output_schema: list[_FieldSpec] = [] + + +class ModelSpec(BaseModel): + name: str + type: str + version: str = "0.1.0" + python: str = "3.11" + train: _Train + test: Optional[_Test] = None + resources: dict = {} + + +def parse_yaml(content: bytes | str) -> dict: + """Parsa stringa YAML → dict validato. Solleva ValueError su errore.""" + if isinstance(content, bytes): + content = content.decode("utf-8") + try: + raw = yaml.safe_load(content) or {} + spec = ModelSpec(**raw) + return spec.model_dump() + except (yaml.YAMLError, ValidationError) as e: + raise ValueError(f"invalid model.yml: {e}") from e + + +async def fetch_and_parse_spec(owner_repo: str, ref: str) -> Optional[dict]: + """Recupera model.yml dalla repo alla revisione e lo parsa. + Cache Redis `ml:modelspec:{repo}:{ref}` TTL 1h. + """ + cache_key = f"ml:modelspec:{owner_repo}:{ref}" + try: + cached = await redis_client.client().get(cache_key) + if cached: + import json + return json.loads(cached) + except Exception: + pass + + try: + raw = await gitea.get_file_raw(owner_repo, ref, "model.yml") + except Exception: + try: + raw = await gitea.get_file_raw(owner_repo, ref, "model.yaml") + except Exception: + return None + spec = parse_yaml(raw) + + try: + import json + await redis_client.client().set(cache_key, json.dumps(spec), ex=3600) + except Exception: + pass + return spec diff --git a/ml/core/redis_client.py b/ml/core/redis_client.py new file mode 100644 index 0000000..7f759ef --- /dev/null +++ b/ml/core/redis_client.py @@ -0,0 +1,29 @@ +"""Client Redis asincrono (redis-py asyncio). Singleton semplice.""" +from __future__ import annotations + +from typing import Optional + +import redis.asyncio as redis + +from core.config import settings + +_client: Optional[redis.Redis] = None + + +def client() -> redis.Redis: + global _client + if _client is None: + _client = redis.Redis( + host=settings.redis_host, + port=settings.redis_port, + decode_responses=True, + health_check_interval=30, + ) + return _client + + +async def close() -> None: + global _client + if _client is not None: + await _client.aclose() + _client = None diff --git a/ml/core/worker.py b/ml/core/worker.py new file mode 100644 index 0000000..f84f4e3 --- /dev/null +++ b/ml/core/worker.py @@ -0,0 +1,54 @@ +"""Worker loop: BRPOP da ml:queue:train e dispatch al docker_runner. + +Parte N task asincroni concorrenti (settings.train_concurrency). +""" +from __future__ import annotations + +import asyncio +import logging + +from core import redis_client +from core.config import settings +from core.docker_runner import run_training_job + +log = logging.getLogger(__name__) + +_tasks: list[asyncio.Task] = [] + + +async def _worker_loop(idx: int): + r = redis_client.client() + log.info("ml worker[%d] started", idx) + while True: + try: + res = await r.brpop("ml:queue:train", timeout=10) + except Exception as e: + log.warning("brpop error: %s", e) + await asyncio.sleep(2) + continue + if res is None: + continue + _, training_id = res + log.info("worker[%d] picked training %s", idx, training_id) + try: + await run_training_job(training_id) + except Exception: + log.exception("worker[%d] training %s crashed", idx, training_id) + + +def start_workers() -> None: + global _tasks + n = max(1, settings.train_concurrency) + for i in range(n): + _tasks.append(asyncio.create_task(_worker_loop(i))) + + +async def stop_workers() -> None: + for t in _tasks: + t.cancel() + for t in _tasks: + try: + await t + except Exception: + pass + _tasks.clear() diff --git a/ml/main.py b/ml/main.py index 51046b2..15f2a9e 100644 --- a/ml/main.py +++ b/ml/main.py @@ -1,19 +1,90 @@ -from fastapi import FastAPI, Request, Response, Header -from fastapi.responses import HTMLResponse, JSONResponse -import time +"""ml-service — FastAPI entrypoint. + +Monta: + / → RedirectResponse + /datasets /models /train /test /results → pagine Jinja + /api/datasets /api/models /api/repos /api/trainings /api/tests /api/results → JSON + /api/trainings/{id}/events → SSE + /health → check + /static/* → file statici +""" +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles + +from core import db, minio_client, redis_client, worker + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger(__name__) + +STATIC_DIR = Path(__file__).resolve().parent / "static" + + +@asynccontextmanager +async def lifespan(app: FastAPI): + log.info("ml-service starting") + await db.init_pool() + try: + minio_client.ensure_bucket() + except Exception as e: + log.warning("minio bucket ensure failed: %s", e) + worker.start_workers() + yield + log.info("ml-service stopping") + await worker.stop_workers() + await db.close_pool() + await redis_client.close() + + +app = FastAPI(title="MEB ML Service", lifespan=lifespan) + +# static +app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") -app = FastAPI() @app.get("/health") -def health(): +async def health(): + pg_ok = True + try: + await db.fetchrow("SELECT 1") + except Exception: + pg_ok = False + redis_ok = True + try: + await redis_client.client().ping() + except Exception: + redis_ok = False return { - "status": "ok", + "status": "ok" if (pg_ok and redis_ok) else "degraded", "service": "ml", - "version": "1.0.0", - "build_number": "1", - "version_state": "dev" + "postgres": "connected" if pg_ok else "disconnected", + "redis": "connected" if redis_ok else "disconnected", + "minio": "connected" if minio_client.check() else "disconnected", + "version": "2.0.0", } -@app.get("/") -def root(): - return {"message": "ML Service"} \ No newline at end of file + +from routers import ( # noqa: E402 + datasets, + models, + pages, + repos, + results, + tests, + trainings, + trainings_stream, +) + +app.include_router(pages.router) +app.include_router(datasets.router) +app.include_router(models.router) +app.include_router(repos.router) +app.include_router(trainings.router) +app.include_router(trainings_stream.router) +app.include_router(tests.router) +app.include_router(results.router) diff --git a/ml/requirements.txt b/ml/requirements.txt index 80345f5..405f802 100644 --- a/ml/requirements.txt +++ b/ml/requirements.txt @@ -1,3 +1,15 @@ fastapi -uvicorn +uvicorn[standard] PyJWT +asyncpg +redis>=5 +minio +influxdb-client +docker +PyYAML +pydantic>=2 +python-multipart +jinja2 +aiofiles +httpx +sse-starlette diff --git a/ml/routers/datasets.py b/ml/routers/datasets.py new file mode 100644 index 0000000..c9b61a4 --- /dev/null +++ b/ml/routers/datasets.py @@ -0,0 +1,160 @@ +"""API datasets (ml.mebboat.it/api/datasets). + +Upload/list/get/download/delete. Storage: + MinIO bucket "ml" con key "datasets/." + Postgres db "ml" tabella "datasets" +""" +from __future__ import annotations + +import json +import uuid +from typing import Optional + +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile + +from core import db, minio_client +from core.auth import require_auth + +router = APIRouter(prefix="/api/datasets", tags=["datasets"]) + +# Bucket MinIO fisso per tutti i dataset (no prefix nelle key). +BUCKET = "ml.datasets" +_EXT = {"csv": "csv", "json": "json", "netcdf": "nc"} + + +def _row(r) -> dict: + if r is None: + return None + d = dict(r) + # asyncpg ritorna JSONB come dict già; date/time come datetime + for k in ("created_at", "updated_at", "start_date", "end_date"): + if d.get(k) is not None and hasattr(d[k], "isoformat"): + d[k] = d[k].isoformat() + return d + + +@router.get("") +async def list_datasets( + type: Optional[str] = Query(None), + tags: Optional[str] = Query(None), + mine: Optional[int] = Query(None), + search: Optional[str] = Query(None), + user=Depends(require_auth), +): + where = [] + args: list = [] + if type: + args.append(type) + where.append(f"type = ${len(args)}") + if tags: + tag_arr = [t.strip() for t in tags.split(",") if t.strip()] + if tag_arr: + args.append(tag_arr) + where.append(f"tags && ${len(args)}") + if mine and user.get("username"): + args.append(user["username"]) + where.append(f"created_by = ${len(args)}") + if search: + args.append(f"%{search}%") + where.append(f"(nome ILIKE ${len(args)} OR description ILIKE ${len(args)})") + sql = "SELECT * FROM datasets" + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY created_at DESC LIMIT 500" + rows = await db.fetch(sql, *args) + return {"count": len(rows), "datasets": [_row(r) for r in rows]} + + +@router.post("", status_code=201) +async def upload_dataset( + file: UploadFile = File(...), + metadata: str = Form("{}"), + user=Depends(require_auth), +): + try: + meta = json.loads(metadata or "{}") + except json.JSONDecodeError: + raise HTTPException(400, "metadata must be valid JSON") + + fmt = meta.get("format") or meta.get("type") or "csv" + if fmt not in ("csv", "json", "netcdf"): + fmt = "csv" + ext = _EXT[fmt] + ds_id = str(uuid.uuid4()) + file_key = f"{ds_id}.{ext}" + + data = await file.read() + minio_client.put_bytes(file_key, data, content_type=file.content_type or "application/octet-stream", bucket=BUCKET) + + created_by = user.get("username") or meta.get("created_by") or "unknown" + row = await db.fetchrow( + """ + INSERT INTO datasets ( + id, file_key, nome, description, tags, type, format, notes, + created_by, size_bytes, copernicus_id + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) + RETURNING * + """, + uuid.UUID(ds_id), + file_key, + meta.get("nome") or file.filename or file_key, + meta.get("description"), + meta.get("tags") or [], + meta.get("dataset_type") or "custom", + fmt, + meta.get("notes"), + created_by, + len(data), + meta.get("copernicus_id") or meta.get("copernicus_dataset_id"), + ) + return _row(row) + + +@router.get("/{dataset_id}") +async def get_dataset(dataset_id: str, user=Depends(require_auth)): + row = await db.fetchrow("SELECT * FROM datasets WHERE id = $1", uuid.UUID(dataset_id)) + if not row: + raise HTTPException(404, "not found") + return _row(row) + + +@router.get("/{dataset_id}/download") +async def download_dataset(dataset_id: str, user=Depends(require_auth)): + row = await db.fetchrow("SELECT file_key FROM datasets WHERE id = $1", uuid.UUID(dataset_id)) + if not row: + raise HTTPException(404, "not found") + url = minio_client.presigned_get(row["file_key"], 3600, bucket=BUCKET) + return {"url": url, "expires_in": 3600} + + +@router.patch("/{dataset_id}") +async def patch_dataset(dataset_id: str, body: dict, user=Depends(require_auth)): + allowed = {"nome", "description", "tags", "notes"} + sets = [] + args: list = [] + for k, v in body.items(): + if k in allowed: + args.append(v) + sets.append(f"{k} = ${len(args)}") + if not sets: + raise HTTPException(400, "no fields to update") + # Trigger updated_at non presente nel DB: lo aggiorniamo manualmente. + sets.append("updated_at = NOW()") + args.append(uuid.UUID(dataset_id)) + row = await db.fetchrow( + f"UPDATE datasets SET {', '.join(sets)} WHERE id = ${len(args)} RETURNING *", + *args, + ) + if not row: + raise HTTPException(404, "not found") + return _row(row) + + +@router.delete("/{dataset_id}", status_code=204) +async def delete_dataset(dataset_id: str, user=Depends(require_auth)): + row = await db.fetchrow("SELECT file_key FROM datasets WHERE id = $1", uuid.UUID(dataset_id)) + if not row: + raise HTTPException(404, "not found") + minio_client.remove(row["file_key"], bucket=BUCKET) + await db.execute("DELETE FROM datasets WHERE id = $1", uuid.UUID(dataset_id)) + return None diff --git a/ml/routers/models.py b/ml/routers/models.py new file mode 100644 index 0000000..5d3409c --- /dev/null +++ b/ml/routers/models.py @@ -0,0 +1,131 @@ +"""API /api/models — registro modelli (repo Gitea + metadata).""" +from __future__ import annotations + +import uuid +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException + +from core import db +from core.auth import require_auth +from core.model_spec import fetch_and_parse_spec + +router = APIRouter(prefix="/api/models", tags=["models"]) + + +def _row(r) -> Optional[dict]: + if r is None: + return None + d = dict(r) + for k in ("created_at", "updated_at"): + if d.get(k) is not None and hasattr(d[k], "isoformat"): + d[k] = d[k].isoformat() + return d + + +@router.get("") +async def list_models(user=Depends(require_auth)): + rows = await db.fetch("SELECT * FROM models ORDER BY created_at DESC LIMIT 500") + return {"count": len(rows), "models": [_row(r) for r in rows]} + + +@router.post("", status_code=201) +async def create_model(body: dict, user=Depends(require_auth)): + required = ("name", "type", "gitea_repo") + for k in required: + if not body.get(k): + raise HTTPException(400, f"missing field: {k}") + + # prova a pre-caricare model.yml dal default branch (non fatale) + spec = None + try: + spec = await fetch_and_parse_spec(body["gitea_repo"], body.get("default_branch") or "main") + except Exception: + spec = None + + row = await db.fetchrow( + """ + INSERT INTO models (name, type, gitea_repo, default_branch, spec, created_by) + VALUES ($1,$2,$3,$4,$5,$6) + RETURNING * + """, + body["name"], + body["type"], + body["gitea_repo"], + body.get("default_branch") or "main", + spec, + user.get("username") or "unknown", + ) + return _row(row) + + +@router.get("/{model_id}") +async def get_model(model_id: str, user=Depends(require_auth)): + row = await db.fetchrow("SELECT * FROM models WHERE id = $1", uuid.UUID(model_id)) + if not row: + raise HTTPException(404, "not found") + return _row(row) + + +@router.patch("/{model_id}") +async def patch_model(model_id: str, body: dict, user=Depends(require_auth)): + allowed = {"name", "type", "default_branch"} + sets = [] + args: list = [] + for k, v in body.items(): + if k in allowed: + args.append(v) + sets.append(f"{k} = ${len(args)}") + if not sets: + raise HTTPException(400, "no fields to update") + args.append(uuid.UUID(model_id)) + row = await db.fetchrow( + f"UPDATE models SET {', '.join(sets)} WHERE id = ${len(args)} RETURNING *", + *args, + ) + if not row: + raise HTTPException(404, "not found") + return _row(row) + + +@router.delete("/{model_id}", status_code=204) +async def delete_model(model_id: str, user=Depends(require_auth)): + await db.execute("DELETE FROM models WHERE id = $1", uuid.UUID(model_id)) + return None + + +# ── Notes ────────────────────────────────────────────────────────────────── +@router.get("/{model_id}/notes") +async def list_notes(model_id: str, user=Depends(require_auth)): + rows = await db.fetch( + "SELECT id, author, text, created_at FROM model_notes WHERE model_id = $1 ORDER BY created_at DESC", + uuid.UUID(model_id), + ) + return [ + { + "id": str(r["id"]), + "author": r["author"], + "text": r["text"], + "created_at": r["created_at"].isoformat(), + } + for r in rows + ] + + +@router.post("/{model_id}/notes", status_code=201) +async def add_note(model_id: str, body: dict, user=Depends(require_auth)): + text = (body.get("text") or "").strip() + if not text: + raise HTTPException(400, "text required") + row = await db.fetchrow( + "INSERT INTO model_notes (model_id, author, text) VALUES ($1, $2, $3) RETURNING *", + uuid.UUID(model_id), + user.get("username") or "unknown", + text, + ) + return { + "id": str(row["id"]), + "author": row["author"], + "text": row["text"], + "created_at": row["created_at"].isoformat(), + } diff --git a/ml/routers/pages.py b/ml/routers/pages.py new file mode 100644 index 0000000..00a60b0 --- /dev/null +++ b/ml/routers/pages.py @@ -0,0 +1,75 @@ +"""Pagine HTML servite direttamente da ml.mebboat.it. + +Layout: + / redirect a /datasets (o landing console) + /datasets lista/upload dataset + /models registro modelli + /train avvia training + /test esegue test su modello trainato + /results storico e confronto risultati +""" +from __future__ import annotations + +from pathlib import Path + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.templating import Jinja2Templates + +from core.auth import _verify +from core.config import settings + +router = APIRouter(tags=["pages"]) + +TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "templates" +templates = Jinja2Templates(directory=str(TEMPLATES_DIR)) + + +def _user_or_redirect(request: Request): + """Per le pagine, se non autenticato redirect al login. Ritorna user dict o RedirectResponse.""" + token = request.cookies.get("auth_token") + auth = request.headers.get("authorization") + if not token and auth and auth.startswith("Bearer "): + token = auth[7:] + user = _verify(token) + if not user: + target = str(request.url) + return RedirectResponse(url=f"{settings.auth_login_url}?redirect={target}", status_code=302) + return user + + +def _render(request: Request, template: str, **ctx): + user = _user_or_redirect(request) + if isinstance(user, RedirectResponse): + return user + return templates.TemplateResponse(template, {"request": request, "user": user, **ctx}) + + +@router.get("/", response_class=HTMLResponse) +async def home(request: Request): + return RedirectResponse(url="/datasets") + + +@router.get("/datasets", response_class=HTMLResponse) +async def page_datasets(request: Request): + return _render(request, "datasets.html", page="datasets") + + +@router.get("/models", response_class=HTMLResponse) +async def page_models(request: Request): + return _render(request, "models.html", page="models") + + +@router.get("/train", response_class=HTMLResponse) +async def page_train(request: Request): + return _render(request, "train.html", page="train") + + +@router.get("/test", response_class=HTMLResponse) +async def page_test(request: Request): + return _render(request, "test.html", page="test") + + +@router.get("/results", response_class=HTMLResponse) +async def page_results(request: Request): + return _render(request, "results.html", page="results") diff --git a/ml/routers/repos.py b/ml/routers/repos.py new file mode 100644 index 0000000..0948f2d --- /dev/null +++ b/ml/routers/repos.py @@ -0,0 +1,51 @@ +"""API /api/repos — proxy autenticato verso Gitea.""" +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, Query + +from core import gitea +from core.auth import require_auth +from core.model_spec import fetch_and_parse_spec + +router = APIRouter(prefix="/api/repos", tags=["repos"]) + + +@router.get("") +async def list_repos(user=Depends(require_auth)): + try: + return await gitea.list_repos() + except Exception as e: + raise HTTPException(502, f"gitea: {e}") + + +@router.get("/{owner}/{repo}/branches") +async def branches(owner: str, repo: str, user=Depends(require_auth)): + try: + return await gitea.list_branches(f"{owner}/{repo}") + except Exception as e: + raise HTTPException(502, f"gitea: {e}") + + +@router.get("/{owner}/{repo}/commits") +async def commits(owner: str, repo: str, branch: str = Query("main"), user=Depends(require_auth)): + try: + return await gitea.list_commits(f"{owner}/{repo}", branch) + except Exception as e: + raise HTTPException(502, f"gitea: {e}") + + +@router.get("/{owner}/{repo}/file") +async def file_raw(owner: str, repo: str, ref: str, path: str, user=Depends(require_auth)): + try: + raw = await gitea.get_file_raw(f"{owner}/{repo}", ref, path) + return {"content": raw.decode("utf-8", errors="replace"), "size": len(raw)} + except Exception as e: + raise HTTPException(404, f"file not found: {e}") + + +@router.get("/{owner}/{repo}/spec") +async def spec(owner: str, repo: str, ref: str = Query("main"), user=Depends(require_auth)): + s = await fetch_and_parse_spec(f"{owner}/{repo}", ref) + if s is None: + raise HTTPException(404, "model.yml not found at ref") + return s diff --git a/ml/routers/results.py b/ml/routers/results.py new file mode 100644 index 0000000..aad0925 --- /dev/null +++ b/ml/routers/results.py @@ -0,0 +1,89 @@ +"""API /api/results — lista trainings/tests + compare multi-training.""" +from __future__ import annotations + +import uuid +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Query + +from core import db, influx_client +from core.auth import require_auth +from core.config import settings + +router = APIRouter(prefix="/api/results", tags=["results"]) + + +def _row(r): + if r is None: + return None + d = dict(r) + for k in ("queued_at", "started_at", "finished_at", "started_at", "ended_at"): + if d.get(k) is not None and hasattr(d[k], "isoformat"): + d[k] = d[k].isoformat() + return d + + +@router.get("") +async def list_results( + model_id: Optional[str] = Query(None), + user=Depends(require_auth), +): + where = [] + args: list = [] + if model_id: + args.append(uuid.UUID(model_id)) + where.append(f"model_id = ${len(args)}") + sql = "SELECT * FROM trainings" + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY finished_at DESC NULLS LAST, queued_at DESC LIMIT 200" + rows = await db.fetch(sql, *args) + return {"count": len(rows), "trainings": [_row(r) for r in rows]} + + +@router.get("/{training_id}") +async def get_result(training_id: str, user=Depends(require_auth)): + row = await db.fetchrow("SELECT * FROM trainings WHERE id = $1", uuid.UUID(training_id)) + if not row: + raise HTTPException(404, "not found") + # timeseries via Influx: loss per iter + cpu/mem + flux = ( + f'from(bucket:"{settings.influx_bucket}") ' + f'|> range(start:-90d) ' + f'|> filter(fn: (r) => r._measurement == "ml_training" and r.training_id == "{training_id}")' + ) + try: + ts = await influx_client.query_flux(flux) + except Exception: + ts = [] + return {"training": _row(row), "timeseries": ts} + + +@router.get("/compare") +async def compare( + trainings: str = Query(..., description="comma-separated training IDs"), + user=Depends(require_auth), +): + ids = [s.strip() for s in trainings.split(",") if s.strip()] + if len(ids) < 2: + raise HTTPException(400, "at least 2 training IDs required") + out = [] + for tid in ids: + try: + tid_uuid = uuid.UUID(tid) + except ValueError: + continue + row = await db.fetchrow("SELECT * FROM trainings WHERE id = $1", tid_uuid) + if not row: + continue + flux = ( + f'from(bucket:"{settings.influx_bucket}") ' + f'|> range(start:-90d) ' + f'|> filter(fn: (r) => r._measurement == "ml_training" and r.training_id == "{tid}")' + ) + try: + ts = await influx_client.query_flux(flux) + except Exception: + ts = [] + out.append({"training": _row(row), "timeseries": ts}) + return {"results": out} diff --git a/ml/routers/tests.py b/ml/routers/tests.py new file mode 100644 index 0000000..3e919e3 --- /dev/null +++ b/ml/routers/tests.py @@ -0,0 +1,109 @@ +"""API /api/tests — sessioni di test su training esistente (max 2 utenti simultanei).""" +from __future__ import annotations + +import json +import time +import uuid +from typing import Optional + +import httpx +from fastapi import APIRouter, Depends, HTTPException + +from core import api_client, db, minio_client +from core.auth import require_auth +from core.docker_runner import run_test_once + +router = APIRouter(prefix="/api/tests", tags=["tests"]) + + +def _row(r): + if r is None: + return None + d = dict(r) + for k in ("started_at", "ended_at"): + if d.get(k) is not None and hasattr(d[k], "isoformat"): + d[k] = d[k].isoformat() + return d + + +@router.post("/sessions", status_code=201) +async def start_session(body: dict, user=Depends(require_auth)): + training_id = body.get("training_id") + if not training_id: + raise HTTPException(400, "training_id required") + + tr = await db.fetchrow( + "SELECT id, status FROM trainings WHERE id = $1", uuid.UUID(training_id) + ) + if not tr: + raise HTTPException(404, "training not found") + if tr["status"] != "succeeded": + raise HTTPException(409, "training not completed") + + sid = str(uuid.uuid4()) + try: + await api_client.page_connect("test", user.get("username") or "unknown", sid) + except httpx.HTTPStatusError as e: + if e.response.status_code == 429: + raise HTTPException(429, "test slots full (max 2 users)") + raise HTTPException(502, f"api: {e}") + + row = await db.fetchrow( + "INSERT INTO tests (id, training_id, user_id) VALUES ($1,$2,$3) RETURNING *", + uuid.UUID(sid), + uuid.UUID(training_id), + user.get("username") or "unknown", + ) + return _row(row) + + +@router.post("/sessions/{session_id}/ping") +async def ping_session(session_id: str, user=Depends(require_auth)): + try: + await api_client.page_ping(session_id) + except httpx.HTTPStatusError as e: + raise HTTPException(e.response.status_code, e.response.text) + return {"ok": True} + + +@router.post("/sessions/{session_id}/runs", status_code=201) +async def run_test(session_id: str, body: dict, user=Depends(require_auth)): + row = await db.fetchrow("SELECT * FROM tests WHERE id = $1", uuid.UUID(session_id)) + if not row: + raise HTTPException(404, "session not found") + + inputs = body.get("inputs") or {} + t0 = time.monotonic() + try: + result = await run_test_once(str(row["training_id"]), inputs) + except Exception as e: + raise HTTPException(500, f"test run failed: {e}") + dt_ms = int((time.monotonic() - t0) * 1000) + + run = { + "inputs": inputs, + "outputs": result.get("outputs", {}), + "duration_ms": dt_ms, + "cpu_peak": result.get("cpu_peak"), + "mem_peak_mb": result.get("mem_peak_mb"), + "ts": time.time(), + } + await db.execute( + "UPDATE tests SET runs = runs || $1::jsonb WHERE id = $2", + json.dumps([run]), + uuid.UUID(session_id), + ) + return run + + +@router.delete("/sessions/{session_id}", status_code=204) +async def end_session(session_id: str, user=Depends(require_auth)): + await db.execute( + "UPDATE tests SET ended_at = NOW() WHERE id = $1 AND ended_at IS NULL", + uuid.UUID(session_id), + ) + try: + await api_client.page_disconnect(session_id) + except Exception: + pass + return None diff --git a/ml/routers/trainings.py b/ml/routers/trainings.py new file mode 100644 index 0000000..5833149 --- /dev/null +++ b/ml/routers/trainings.py @@ -0,0 +1,129 @@ +"""API /api/trainings — enqueue, list, get, artifacts.""" +from __future__ import annotations + +import json +import uuid +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Query + +from core import db, minio_client, redis_client, api_client +from core.auth import require_auth + +router = APIRouter(prefix="/api/trainings", tags=["trainings"]) + + +def _row(r) -> Optional[dict]: + if r is None: + return None + d = dict(r) + for k in ("queued_at", "started_at", "finished_at"): + if d.get(k) is not None and hasattr(d[k], "isoformat"): + d[k] = d[k].isoformat() + return d + + +@router.get("") +async def list_trainings( + model_id: Optional[str] = Query(None), + status: Optional[str] = Query(None), + limit: int = Query(100, le=500), + user=Depends(require_auth), +): + where = [] + args: list = [] + if model_id: + args.append(uuid.UUID(model_id)) + where.append(f"model_id = ${len(args)}") + if status: + args.append(status) + where.append(f"status = ${len(args)}") + sql = "SELECT * FROM trainings" + if where: + sql += " WHERE " + " AND ".join(where) + args.append(limit) + sql += f" ORDER BY queued_at DESC LIMIT ${len(args)}" + rows = await db.fetch(sql, *args) + return {"count": len(rows), "trainings": [_row(r) for r in rows]} + + +@router.post("", status_code=202) +async def enqueue_training(body: dict, user=Depends(require_auth)): + for k in ("model_id", "version", "patch", "dataset_id"): + if not body.get(k): + raise HTTPException(400, f"missing field: {k}") + + model_row = await db.fetchrow("SELECT * FROM models WHERE id = $1", uuid.UUID(body["model_id"])) + if not model_row: + raise HTTPException(404, "model not found") + + ds_row = await db.fetchrow("SELECT id FROM datasets WHERE id = $1", uuid.UUID(body["dataset_id"])) + if not ds_row: + raise HTTPException(404, "dataset not found") + + try: + training_row = await db.fetchrow( + """ + INSERT INTO trainings (model_id, version, patch, dataset_id, started_by, status) + VALUES ($1,$2,$3,$4,$5,'queued') + RETURNING * + """, + uuid.UUID(body["model_id"]), + body["version"], + body["patch"], + uuid.UUID(body["dataset_id"]), + user.get("username") or "unknown", + ) + except Exception as e: + raise HTTPException(409, f"training already exists or invalid: {e}") + + training_id = str(training_row["id"]) + + # crea job lato api-service (cross-service registry) + try: + await api_client.create_job( + "train", + created_by=user.get("username") or "unknown", + payload={ + "training_id": training_id, + "model_id": body["model_id"], + "version": body["version"], + "patch": body["patch"], + "dataset_id": body["dataset_id"], + }, + ) + except Exception as e: + # non-fatale: il worker locale può comunque procedere; logghiamo e continuiamo + import logging + logging.warning("create_job failed: %s", e) + + # enqueue in Redis (il worker locale lo raccoglie) + await redis_client.client().lpush("ml:queue:train", training_id) + await redis_client.client().hset( + f"ml:train:{training_id}", + mapping={"status": "queued", "progress": "0", "message": "queued"}, + ) + await redis_client.client().expire(f"ml:train:{training_id}", 48 * 3600) + + return _row(training_row) + + +@router.get("/{training_id}") +async def get_training(training_id: str, user=Depends(require_auth)): + row = await db.fetchrow("SELECT * FROM trainings WHERE id = $1", uuid.UUID(training_id)) + if not row: + raise HTTPException(404, "not found") + return _row(row) + + +@router.get("/{training_id}/artifacts") +async def list_artifacts(training_id: str, user=Depends(require_auth)): + row = await db.fetchrow( + "SELECT artifacts_prefix FROM trainings WHERE id = $1", uuid.UUID(training_id) + ) + if not row or not row["artifacts_prefix"]: + raise HTTPException(404, "no artifacts") + objs = minio_client.list_prefix(row["artifacts_prefix"] + "/") + for o in objs: + o["url"] = minio_client.presigned_get(o["name"], 3600) + return objs diff --git a/ml/routers/trainings_stream.py b/ml/routers/trainings_stream.py new file mode 100644 index 0000000..92d4f2b --- /dev/null +++ b/ml/routers/trainings_stream.py @@ -0,0 +1,64 @@ +"""SSE endpoint per live progress del training. + +GET /api/trainings/{id}/events + Streamma eventi dal Redis stream `ml:train:{id}:events` via Server-Sent Events. + Termina quando lo stato del training è terminale (succeeded/failed/cancelled). +""" +from __future__ import annotations + +import asyncio +import json +import uuid + +from fastapi import APIRouter, Depends, HTTPException +from sse_starlette.sse import EventSourceResponse + +from core import db, redis_client +from core.auth import require_auth + +router = APIRouter(prefix="/api/trainings", tags=["trainings-sse"]) + +_TERMINAL = {"succeeded", "failed", "cancelled"} + + +@router.get("/{training_id}/events") +async def training_events(training_id: str, user=Depends(require_auth)): + # verifica esistenza + row = await db.fetchrow("SELECT status FROM trainings WHERE id = $1", uuid.UUID(training_id)) + if not row: + raise HTTPException(404, "not found") + + stream_key = f"ml:train:{training_id}:events" + status_key = f"ml:train:{training_id}" + + async def gen(): + last_id = "0-0" + r = redis_client.client() + while True: + try: + # XREAD block 5s per non tenere la connessione idle troppo a lungo + resp = await r.xread({stream_key: last_id}, count=50, block=5000) + except Exception as e: + yield {"event": "error", "data": json.dumps({"error": str(e)})} + await asyncio.sleep(1) + continue + + if resp: + for _stream, entries in resp: + for entry_id, fields in entries: + last_id = entry_id + yield {"event": "message", "id": entry_id, "data": json.dumps(fields)} + + # controlla stato terminale + state = await r.hget(status_key, "status") + if not state: + # fallback su db se redis scaduto + db_row = await db.fetchrow( + "SELECT status FROM trainings WHERE id = $1", uuid.UUID(training_id) + ) + state = db_row["status"] if db_row else "unknown" + if state in _TERMINAL: + yield {"event": "end", "data": json.dumps({"status": state})} + return + + return EventSourceResponse(gen()) diff --git a/ml/runner/Dockerfile b/ml/runner/Dockerfile new file mode 100644 index 0000000..8b7d4ac --- /dev/null +++ b/ml/runner/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir \ + numpy pandas scikit-learn \ + xgboost \ + matplotlib \ + pyyaml + +COPY sdk.py /opt/meb/meb_ml.py +ENV PYTHONPATH=/opt/meb + +WORKDIR /workdir +CMD ["bash"] diff --git a/ml/runner/sdk.py b/ml/runner/sdk.py new file mode 100644 index 0000000..c421090 --- /dev/null +++ b/ml/runner/sdk.py @@ -0,0 +1,80 @@ +"""meb_ml — SDK importabile dal codice utente dentro il container runner. + +API: + from meb_ml import emit_metric, emit_series, emit_matrix, emit_log, save_artifact + + emit_metric(iter=10, loss=0.23) + emit_series("roc_curve", x=fpr, y=tpr, kind="line") + emit_matrix("confusion", labels=[...], values=[[...],[...]]) + emit_log("info", "epoch done") + +Scrive righe JSON su stdout; il parent (ml-service) le inoltra su Redis/Influx. +Per risultati finali scrivere `out/metrics.json` con: + {"metrics": {...}, "plots": {"loss_curve": {"x": [...], "y": [...]}, ...}} +""" +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any, Iterable, Sequence + + +def _print(obj: dict) -> None: + sys.stdout.write(json.dumps(obj, default=float) + "\n") + sys.stdout.flush() + + +def emit_metric(**fields: Any) -> None: + _print({"type": "metric", **fields}) + + +def emit_series(name: str, x: Sequence, y: Sequence, kind: str = "line") -> None: + _print({ + "type": "series", + "name": name, + "kind": kind, + "x": list(x), + "y": list(y), + }) + + +def emit_matrix(name: str, labels: Sequence, values: Sequence[Sequence]) -> None: + _print({ + "type": "matrix", + "name": name, + "labels": list(labels), + "values": [list(row) for row in values], + }) + + +def emit_log(level: str, message: str) -> None: + _print({"type": "log", "level": level, "message": message}) + + +def save_artifact(path: str) -> str: + """Copia `path` nella cartella artefatti (MEB_ARTIFACTS_DIR). Ritorna la dest.""" + dest_dir = Path(os.environ.get("MEB_ARTIFACTS_DIR", "/workdir/out")) + dest_dir.mkdir(parents=True, exist_ok=True) + src = Path(path) + dest = dest_dir / src.name + dest.write_bytes(src.read_bytes()) + return str(dest) + + +def dataset_path() -> str: + return os.environ["MEB_DATASET_PATH"] + + +def artifacts_dir() -> str: + return os.environ.get("MEB_ARTIFACTS_DIR", "/workdir/out") + + +def read_test_input() -> dict: + """Legge un singolo JSON da stdin (per script di test).""" + return json.loads(sys.stdin.readline()) + + +def write_test_output(outputs: dict) -> None: + _print({"type": "result", "outputs": outputs}) diff --git a/ml/static/styles/ml.css b/ml/static/styles/ml.css new file mode 100644 index 0000000..b08286a --- /dev/null +++ b/ml/static/styles/ml.css @@ -0,0 +1,146 @@ +.ml-nav { + display: flex; + gap: 16px; + align-items: center; +} +.ml-nav a { + text-decoration: none; + color: var(--text-secondary); + font-weight: 600; + padding: 8px 12px; + border-radius: var(--radius-md); + transition: all 0.2s ease; +} +.ml-nav a:hover { background: var(--accent-light); color: var(--accent-color); } +.ml-nav a.active { background: var(--accent-light); color: var(--accent-color); } + +.container { + max-width: 1200px; + margin: 24px auto; + padding: 0 24px; +} + +.page-head { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 20px; +} +.page-head h2 { font-size: 1.5rem; } + +.list { + display: flex; + flex-direction: column; + gap: 8px; +} +.list .item { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border: 1px solid var(--header-border); + border-radius: var(--radius-lg); + background: #fff; + transition: box-shadow 0.12s ease; +} +.list .item:hover { box-shadow: var(--shadow-md); } +.list .meta { color: var(--text-secondary); font-size: 0.85rem; } + +.form-row { + display: flex; + flex-wrap: wrap; + gap: 12px; + align-items: flex-end; + margin-bottom: 20px; +} +.form-row label { + display: flex; + flex-direction: column; + gap: 4px; + font-size: 0.85rem; + color: var(--text-secondary); +} +.form-row input, .form-row select, .form-row textarea { + padding: 8px 12px; + border: 1px solid var(--header-border); + border-radius: var(--radius-md); + font-family: inherit; +} + +.hidden { display: none !important; } + +.queue-info { + font-size: 0.9rem; + color: var(--text-secondary); + padding: 6px 12px; + background: var(--accent-light); + border-radius: var(--radius-md); +} + +.charts { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 16px; + margin: 16px 0; +} + +.logs { + background: #0f172a; + color: #cbd5e1; + padding: 12px; + border-radius: var(--radius-md); + font-family: ui-monospace, monospace; + font-size: 0.8rem; + max-height: 320px; + overflow: auto; + white-space: pre-wrap; +} + +.detail { + border: 1px solid var(--header-border); + border-radius: var(--radius-lg); + padding: 16px; + margin-top: 16px; + background: #fff; + position: relative; +} +.detail #btn-close-detail { + position: absolute; + top: 8px; + right: 8px; + padding: 4px 10px; +} + +dialog { + border: 1px solid var(--header-border); + border-radius: var(--radius-lg); + padding: 24px; + width: min(500px, 90vw); +} +dialog form { display: flex; flex-direction: column; gap: 12px; } +dialog label { display: flex; flex-direction: column; gap: 4px; font-size: 0.85rem; } +dialog menu { display: flex; justify-content: flex-end; gap: 8px; margin-top: 16px; padding: 0; } + +table { + width: 100%; + border-collapse: collapse; + margin-top: 12px; +} +th, td { padding: 8px 12px; border-bottom: 1px solid var(--header-border); text-align: left; font-size: 0.9rem; } + +code { + font-family: ui-monospace, monospace; + background: #f1f5f9; + padding: 2px 6px; + border-radius: 4px; + font-size: 0.85em; +} + +pre { + background: #f8fafc; + padding: 12px; + border-radius: var(--radius-md); + overflow: auto; + font-family: ui-monospace, monospace; + font-size: 0.8rem; +} diff --git a/ml/templates/_layout.html b/ml/templates/_layout.html new file mode 100644 index 0000000..e8b3806 --- /dev/null +++ b/ml/templates/_layout.html @@ -0,0 +1,33 @@ + + + + + + ML — {% block title %}{{ page|capitalize }}{% endblock %} + + + + +
+

Modelli ML

+ +
+

{{ user.username }}

+ +
+
+ +
+ {% block content %}{% endblock %} +
+ + + {% block scripts %}{% endblock %} + + diff --git a/ml/templates/datasets.html b/ml/templates/datasets.html index e69de29..e2b8eff 100644 --- a/ml/templates/datasets.html +++ b/ml/templates/datasets.html @@ -0,0 +1,39 @@ +{% extends "_layout.html" %} +{% block title %}Datasets{% endblock %} +{% block content %} +
+

Datasets

+ +
+ +
+ + +
+

Carica dataset

+ + + + + + + + + + +
+ +{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/ml/templates/models.html b/ml/templates/models.html new file mode 100644 index 0000000..0e90342 --- /dev/null +++ b/ml/templates/models.html @@ -0,0 +1,57 @@ +{% extends "_layout.html" %} +{% block title %}Modelli{% endblock %} +{% block content %} +
+

Modelli

+ +
+ +
+ + + + +
+

Nuovo modello

+ + + + + + + + +
+ +{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/ml/templates/results.html b/ml/templates/results.html index 6c220f0..cf3495c 100644 --- a/ml/templates/results.html +++ b/ml/templates/results.html @@ -1,89 +1,33 @@ - +{% extends "_layout.html" %} +{% block title %}Risultati{% endblock %} +{% block content %} +
+

Risultati training

+ +
- - - Risultati - +
- - - - - -
-

Risultati

-
-

Utente

- -
-
- -
- -
- -
-

- Seleziona -

- -

- una sessione di training eseguita per visualizzarne i risultati -

-
- -
- -
-

sessione 1

-
-

24/03/2026

-

12:00

-

dataset: d-1

-
- -
- -
-

sessione 2

-

24/03/2026

- -
- -
- -
- -
- - - - - + +{% endblock %} +{% block scripts %} + + +{% endblock %} diff --git a/ml/templates/test.html b/ml/templates/test.html index e69de29..bce545e 100644 --- a/ml/templates/test.html +++ b/ml/templates/test.html @@ -0,0 +1,33 @@ +{% extends "_layout.html" %} +{% block title %}Test{% endblock %} +{% block content %} +
+

Test modello

+
Slot: –/2
+
+ + + +
+ + + +
+ + +{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/ml/templates/train.html b/ml/templates/train.html index e69de29..4d42ff3 100644 --- a/ml/templates/train.html +++ b/ml/templates/train.html @@ -0,0 +1,35 @@ +{% extends "_layout.html" %} +{% block title %}Train{% endblock %} +{% block content %} +
+

Avvia training

+
Coda: –
+
+ +
+ + + + + + +
+ + + +
+

Recenti

+
+
+{% endblock %} +{% block scripts %} + + +{% endblock %} diff --git a/realtime/package-lock.json b/realtime/package-lock.json index ef635c9..bbc052f 100644 --- a/realtime/package-lock.json +++ b/realtime/package-lock.json @@ -11,8 +11,10 @@ "@influxdata/influxdb-client": "^1.35.0", "@influxdata/influxdb-client-apis": "^1.35.0", "@msgpack/msgpack": "^3.1.3", + "cookie-parser": "^1.4.7", "express": "^5.2.1", "ioredis": "^5.10.0", + "jsonwebtoken": "^9.0.3", "pg": "^8.20.0", "ws": "^8.19.0" } @@ -84,6 +86,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -162,6 +170,25 @@ "node": ">= 0.6" } }, + "node_modules/cookie-parser": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.7.tgz", + "integrity": "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw==", + "license": "MIT", + "dependencies": { + "cookie": "0.7.2", + "cookie-signature": "1.0.6" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/cookie-parser/node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "license": "MIT" + }, "node_modules/cookie-signature": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", @@ -220,6 +247,15 @@ "node": ">= 0.4" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -525,18 +561,103 @@ "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", "license": "MIT" }, + "node_modules/jsonwebtoken": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", + "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", + "license": "MIT", + "dependencies": { + "jws": "^4.0.1", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", "license": "MIT" }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "license": "MIT" + }, "node_modules/lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", "license": "MIT" }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "license": "MIT" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "license": "MIT" + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "license": "MIT" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -876,12 +997,44 @@ "node": ">= 18" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/send": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", diff --git a/realtime/src/index.js b/realtime/src/index.js index b778497..ad79b3b 100644 --- a/realtime/src/index.js +++ b/realtime/src/index.js @@ -45,9 +45,13 @@ app.get('/health', (req, res) => { app.use('/connect', require('./routes/connect')); app.use('/sensors', require('./routes/sensors')); app.use('/sessions', require('./routes/sessions')); +app.use('/rules', require('./routes/rules')); const server = app.listen(3000, '0.0.0.0', () => { console.log(`Realtime started`); }); wsHandler.setup(server); + +// deve essere caricato DOPO setup per avere kioskRelay pronto +app.use('/kiosk', require('./routes/kiosk')); diff --git a/realtime/src/routes/kiosk.js b/realtime/src/routes/kiosk.js index 1b8ac67..d457179 100644 --- a/realtime/src/routes/kiosk.js +++ b/realtime/src/routes/kiosk.js @@ -1,12 +1,29 @@ const router = require('express').Router(); -const db = require('../store/db'); +const { kioskRelay } = require('../ws/handler'); -// Endpoint per ricevere dati dal kiosk -router.post('/data', async (req, res) => { - const { session_id, sensor_code, value, timestamp } = req.body; - if (!session_id || !sensor_code || value === undefined) { - return res.status(400).json({ error: 'Missing required fields' }); - } +const INTERNAL_KEY = process.env.INTERNAL_API_KEY; + +function requireInternal(req, res, next) { + if (!INTERNAL_KEY || req.headers['x-api-key'] !== INTERNAL_KEY) + return res.status(403).json({ error: 'forbidden' }); + next(); +} + +// Chiamato dall'API quando cambia il template attivo +router.post('/notify-active', requireInternal, (req, res) => { + const { template } = req.body || {}; + if (!template || !template.id) return res.status(400).json({ error: 'template.id required' }); + kioskRelay.notifyActiveTemplateChange(template); + res.json({ ok: true }); }); -module.exports = router; \ No newline at end of file +// Stato dispositivi connessi (diagnostica) +router.get('/status', requireInternal, (req, res) => { + const list = []; + for (const [name, ws] of kioskRelay.devices) { + list.push({ sensor: name, templateId: ws.templateId || null, lastSeen: ws.lastSeen || null }); + } + res.json({ devices: list }); +}); + +module.exports = router; diff --git a/realtime/src/routes/rules.js b/realtime/src/routes/rules.js new file mode 100644 index 0000000..db4de26 --- /dev/null +++ b/realtime/src/routes/rules.js @@ -0,0 +1,57 @@ +/** + * Relay HTTP → WS per il push dei rulesets ai sensori. + * Chiamato SOLO dal servizio api (internal, x-api-key). + * + * POST /rules/push + * Body: { sensors: [name, ...], type, ruleset } + * -> invia msgpack { _t: 'ruleset_update', type, ruleset } ad ogni sensore + * online tramite la connessione WS gia' stabilita. + */ + +const router = require('express').Router(); +const { encode } = require('@msgpack/msgpack'); +const { connectedSensors } = require('../ws/handler'); + +const INTERNAL_KEY = process.env.INTERNAL_API_KEY; + +function requireInternal(req, res, next) { + const k = req.headers['x-api-key']; + if (!INTERNAL_KEY || !k || k !== INTERNAL_KEY) { + return res.status(403).json({ error: 'forbidden' }); + } + next(); +} + +router.post('/push', requireInternal, (req, res) => { + const { sensors, type, ruleset } = req.body || {}; + if (!Array.isArray(sensors) || !sensors.length) return res.status(400).json({ error: 'sensors array required' }); + if (!type || !ruleset) return res.status(400).json({ error: 'type and ruleset required' }); + + const payload = { _t: 'ruleset_update', type, ruleset }; + let encoded; + try { + encoded = encode(payload); + } catch (err) { + return res.status(500).json({ error: `encode error: ${err.message}` }); + } + + const pushed = [], offline = [], errors = []; + for (const name of sensors) { + const ws = connectedSensors.get(name); + if (!ws || ws.readyState !== ws.OPEN) { + offline.push(name); + continue; + } + try { + ws.send(encoded); + pushed.push(name); + } catch (err) { + errors.push({ sensor: name, error: err.message }); + } + } + + console.log(`[RULES] push type=${type} v=${ruleset?.version?.str || '?'} → pushed=${pushed.length} offline=${offline.length} err=${errors.length}`); + res.json({ pushed, offline, errors }); +}); + +module.exports = router; diff --git a/realtime/src/store/influx.js b/realtime/src/store/influx.js index dc1b56b..8d1a212 100644 --- a/realtime/src/store/influx.js +++ b/realtime/src/store/influx.js @@ -5,16 +5,40 @@ const client = new InfluxDB({ token: process.env.INFLX_TOKEN, }); -const bucket = process.env.INFLX_BUCKET || 'logs'; const org = process.env.INFLX_ORG; -const writeApi = client.getWriteApi(org, bucket, 'ms', { - flushInterval: 100, - batchSize: 50, -}); +// Bucket dedicati per dominio. Il default per i logs viene mantenuto su +// INFLX_BUCKET per retro-compatibilità con la configurazione esistente. +// Per i dati meteo current e forecast usiamo bucket separati: sono dati +// indipendenti dai logs (frequenze e retention diverse) e tenerli separati +// permette policy di retention più aggressive per i forecast (timestamp +// futuri sovrascritti spesso) senza toccare il volume dei logs. +const BUCKETS = { + logs: process.env.INFLX_BUCKET_LOGS || process.env.INFLX_BUCKET || 'logs', + weather: process.env.INFLX_BUCKET_WEATHER || 'weather_current', + weather_forecast: process.env.INFLX_BUCKET_FORECAST || 'weather_forecast', +}; + +const writeApis = {}; +function getWriteApi(bucket) { + if (!writeApis[bucket]) { + writeApis[bucket] = client.getWriteApi(org, bucket, 'ms', { + flushInterval: 1000, + batchSize: 200, + maxRetries: 3, + }); + } + return writeApis[bucket]; +} + +function bucketFor(measurement) { + if (measurement === 'weather') return BUCKETS.weather; + if (measurement === 'weather_forecast') return BUCKETS.weather_forecast; + return BUCKETS.logs; +} /** - * Scrive dati generici su InfluxDB senza mapping. + * Scrive dati generici su InfluxDB nel bucket appropriato per il measurement. * @param {string} measurement - nome della measurement (es. 'logs', 'weather') * @param {Object} fields - campi { key: value } * @param {string} sensor - nome del sensore @@ -36,11 +60,12 @@ function writeGenericData(measurement, fields, sensor, session, timestamp) { } } - writeApi.writePoint(point); + getWriteApi(bucketFor(measurement)).writePoint(point); } /** * Scrive un batch di punti forecast (previsioni orarie). + * Usa il bucket weather_forecast (non i logs). * @param {Array} points - array di [timestamp_ms, { key: value, ... }] * @param {string} sensor - nome del sensore * @param {string} session - id sessione @@ -52,14 +77,14 @@ function writeForecastBatch(points, sensor, session) { } /** - * Forza il flush del buffer di scrittura. + * Forza il flush dei buffer di scrittura su tutti i bucket. */ async function flush() { - try { - await writeApi.flush(); - } catch (err) { - console.error('[INFLUX] Flush error:', err.message); - } + await Promise.all(Object.values(writeApis).map(async (wa) => { + try { await wa.flush(); } catch (err) { + console.error('[INFLUX] Flush error:', err.message); + } + })); } /** @@ -72,7 +97,7 @@ async function flush() { async function queryHistory(sensor, session, since) { const queryApi = client.getQueryApi(org); const fluxQuery = ` - from(bucket: "${bucket}") + from(bucket: "${BUCKETS.logs}") |> range(start: ${since}) |> filter(fn: (r) => r._measurement == "logs") |> filter(fn: (r) => r.sensor == "${sensor}") @@ -95,10 +120,6 @@ async function queryHistory(sensor, session, since) { /** * Esporta tutti i dati di una sessione come CSV. - * @param {string} sensor - nome sensore - * @param {string} session - session_id - * @param {string} since - ISO timestamp inizio (opzionale, default -30d) - * @returns {string} CSV content */ async function exportSessionCSV(sensor, session, since) { const start = since || '-30d'; @@ -106,7 +127,6 @@ async function exportSessionCSV(sensor, session, since) { if (rows.length === 0) return ''; - // Raccogli tutti i field names (esclusi meta InfluxDB) const metaKeys = new Set(['result', 'table', '_start', '_stop', '_measurement', 'sensor', 'session', '']); const fieldNames = new Set(); for (const row of rows) { @@ -133,4 +153,4 @@ async function exportSessionCSV(sensor, session, since) { return header + '\n' + csvRows.join('\n') + '\n'; } -module.exports = { writeGenericData, writeForecastBatch, flush, queryHistory, exportSessionCSV }; +module.exports = { writeGenericData, writeForecastBatch, flush, queryHistory, exportSessionCSV, BUCKETS }; diff --git a/realtime/src/ws/handler.js b/realtime/src/ws/handler.js index 5a60b23..0f48e14 100644 --- a/realtime/src/ws/handler.js +++ b/realtime/src/ws/handler.js @@ -3,6 +3,7 @@ const { decode } = require('@msgpack/msgpack'); const { consumeConnectionToken, appendAsConnection, query, hset, del } = require('../store/redis'); const { writeGenericData, writeForecastBatch } = require('../store/influx'); const db = require('../store/db'); +const kioskRelay = require('./kiosk'); // In-memory registries const sensorWatchers = new Map(); // sensorName → Set (watchers) @@ -42,6 +43,9 @@ function setup(server) { handleSensorConnection(ws); }); + } else if (path === '/kiosk') { + await kioskRelay.handleUpgrade(wss, req, socket, head, url); + } else if (path === '/live') { wss.handleUpgrade(req, socket, head, (ws) => { handleWatcherConnection(ws); @@ -95,6 +99,58 @@ async function handleSensorConnection(ws) { return; } + // Reset sessione richiesto dal plugin (es. dopo un nuovo ruleset + // di logs/meteo). La connessione WS persiste: cambiamo solo il + // sessionId, marchiamo la vecchia come disconnessa e creiamo la + // nuova in sessiondataref. I dati successivi useranno il nuovo tag. + if (packet._t === 'session_reset') { + const prev = ws.sessionId; + const next = generateSessionId(); + ws.sessionId = next; + console.log(`[${sensorName}] session_reset ${prev} → ${next} (reason: ${packet.reason || 'n/a'})`); + try { + await db.query('sensors', + `UPDATE sessiondataref SET disconnected_at = NOW() WHERE session_id = $1 AND disconnected_at IS NULL`, + [prev] + ); + await db.query('sensors', + `INSERT INTO sessiondataref (session_id, sensor_name, name, created_at) + VALUES ($1, $2, $3, NOW()) + ON CONFLICT (session_id) DO NOTHING`, + [next, sensorName, next] + ); + } catch (err) { + console.error(`[${sensorName}] session_reset DB error:`, err.message); + } + hset(`sensors:${sensorName}`, 'session', next); + try { + const { encode } = require('@msgpack/msgpack'); + ws.send(encode({ _t: 'session_id', sessionId: next, prev })); + } catch (err) { + console.error(`[${sensorName}] session_reset reply error:`, err.message); + } + return; + } + + // ACK di un ruleset ricevuto e applicato: il plugin ci dice + // che la versione X del tipo Y e' ora attiva sul device. + if (packet._t === 'ruleset_ack') { + const { type, ruleset_id } = packet; + if (type && ruleset_id) { + const API = process.env.API_URL || 'http://meb-api:3000'; + const KEY = process.env.INTERNAL_API_KEY; + if (KEY) { + fetch(`${API}/rules/${type}/${ruleset_id}/ack`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': KEY }, + body: JSON.stringify({ sensor: sensorName }) + }).catch(err => console.error(`[${sensorName}] ruleset_ack forward error:`, err.message)); + } + console.log(`[${sensorName}] ruleset_ack type=${type} id=${ruleset_id}`); + } + return; + } + const { ts, _m, ...fields } = packet; // InfluxDB: usa SEMPRE sessionId come tag (non cambia mai) @@ -203,4 +259,4 @@ function handleWatcherConnection(ws) { }); } -module.exports = { setup, connectedSensors }; +module.exports = { setup, connectedSensors, kioskRelay }; diff --git a/realtime/src/ws/kiosk.js b/realtime/src/ws/kiosk.js new file mode 100644 index 0000000..31cd1e4 --- /dev/null +++ b/realtime/src/ws/kiosk.js @@ -0,0 +1,167 @@ +/** + * Kiosk realtime relay. + * Due ruoli: + * - device: il plugin kiosk sulla barca (uno per sensorName) + * - controller: la pagina kiosklive.html (N per sensorName) + * Messaggi JSON (no msgpack, canale leggero). + */ +const jwt = require('jsonwebtoken'); +const { consumeConnectionToken } = require('../store/redis'); + +const devices = new Map(); // sensorName → ws +const controllers = new Map(); // sensorName → Set + +const JWT_SECRET = process.env.JWT_SECRET; + +function verifyJwt(token) { + if (!token || !JWT_SECRET) return null; + try { return jwt.verify(token, JWT_SECRET, { algorithms: ['HS256'] }); } catch { return null; } +} + +function extractCookie(req, name) { + const raw = req.headers.cookie || ''; + const m = raw.split(';').map(s => s.trim()).find(s => s.startsWith(name + '=')); + return m ? decodeURIComponent(m.slice(name.length + 1)) : null; +} + +function send(ws, obj) { + if (ws && ws.readyState === ws.OPEN) ws.send(JSON.stringify(obj)); +} + +function broadcastControllers(sensorName, obj) { + const set = controllers.get(sensorName); + if (!set) return; + const msg = JSON.stringify(obj); + for (const c of set) if (c.readyState === c.OPEN) c.send(msg); +} + +function deviceStatus(sensorName) { + const d = devices.get(sensorName); + return { + t: 'kiosk_status', + online: !!d, + templateId: d?.templateId || null, + lastSeen: d?.lastSeen || null + }; +} + +/** + * Gestisce l'upgrade per /kiosk?role=device|controller&sensor= + * @returns true se gestito + */ +async function handleUpgrade(wss, req, socket, head, url) { + const role = url.searchParams.get('role'); + const sensorName = url.searchParams.get('sensor'); + + if (!role || !sensorName) { + socket.write('HTTP/1.1 400 Bad Request\r\n\r\n'); socket.destroy(); return true; + } + + if (role === 'device') { + const token = url.searchParams.get('token'); + const sensor = token ? await consumeConnectionToken(token) : null; + if (!sensor || sensor !== sensorName) { + socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n'); socket.destroy(); return true; + } + wss.handleUpgrade(req, socket, head, (ws) => { + ws.sensorName = sensorName; + ws.role = 'device'; + attachDevice(ws); + }); + return true; + } + + if (role === 'controller') { + const token = extractCookie(req, 'auth_token') || url.searchParams.get('token'); + const payload = verifyJwt(token); + if (!payload) { + socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n'); socket.destroy(); return true; + } + wss.handleUpgrade(req, socket, head, (ws) => { + ws.sensorName = sensorName; + ws.role = 'controller'; + ws.user = payload.sub; + attachController(ws); + }); + return true; + } + + socket.write('HTTP/1.1 400 Bad Request\r\n\r\n'); socket.destroy(); return true; +} + +function attachDevice(ws) { + const name = ws.sensorName; + const prev = devices.get(name); + if (prev && prev.readyState === prev.OPEN) prev.close(4000, 'replaced'); + devices.set(name, ws); + ws.lastSeen = Date.now(); + broadcastControllers(name, deviceStatus(name)); + console.log(`[kiosk] device online: ${name}`); + + ws.on('message', (raw) => { + let m; try { m = JSON.parse(raw.toString()); } catch { return; } + ws.lastSeen = Date.now(); + switch (m.t) { + case 'hello': + ws.templateId = m.templateId || null; + broadcastControllers(name, deviceStatus(name)); + break; + case 'ack': + broadcastControllers(name, m); + break; + case 'heartbeat': + break; + default: + // echo diagnostici opzionali + break; + } + }); + + const hb = setInterval(() => { if (ws.readyState === ws.OPEN) ws.ping(); }, 25000); + + ws.on('close', () => { + clearInterval(hb); + if (devices.get(name) === ws) devices.delete(name); + broadcastControllers(name, deviceStatus(name)); + console.log(`[kiosk] device offline: ${name}`); + }); + ws.on('error', () => {}); +} + +function attachController(ws) { + const name = ws.sensorName; + if (!controllers.has(name)) controllers.set(name, new Set()); + controllers.get(name).add(ws); + send(ws, deviceStatus(name)); + console.log(`[kiosk] controller connected on ${name} (total=${controllers.get(name).size})`); + + ws.on('message', (raw) => { + let m; try { m = JSON.parse(raw.toString()); } catch { return; } + const allowed = ['patch_box','add_box','remove_box','load_template','apply_inline','persist','reload']; + if (!allowed.includes(m.t)) return; + const device = devices.get(name); + if (!device || device.readyState !== device.OPEN) { + send(ws, { t: 'ack', cmdId: m.cmdId, ok: false, err: 'device offline' }); + return; + } + send(device, m); + }); + + ws.on('close', () => { + const set = controllers.get(name); + if (set) { set.delete(ws); if (!set.size) controllers.delete(name); } + }); + ws.on('error', () => {}); +} + +/** HTTP notify usato dall'API quando cambia template attivo */ +function notifyActiveTemplateChange(template) { + for (const [name, ws] of devices) { + send(ws, { t: 'load_template', templateId: template.id }); + } + for (const name of controllers.keys()) { + broadcastControllers(name, { t: 'active_template_changed', templateId: template.id }); + } +} + +module.exports = { handleUpgrade, notifyActiveTemplateChange, devices };