[Feature/simultaneous_queues] Improved version for scenario management (#29)

* feature: multiple queues working with different users using the default scenario configuration

* fix network prefix 192.168.1 changed to 192.168.2

* Fix different queue for each user

* feature: show scenarios in the dashboard depending on the role of the user

* refactor: changed location of utils.py

* fix: nebula-frontend port

* fix: launching scenarios with same user in different frontend instance

* fix: launching scenarios with same user in different frontend instance

* fix: stopping only instance

* fix: stopping only instance

* fix: remove instance network

* fix: killing other instance docker when starting new scenario

* fix: stop bugs

* feature: display queues for each user

* Fix persistence on session and dbs

* Fix persistence on session

* fix: scenario database, database dashboard

* Minor changes in user management

* Improve update docker images

* remove reconnections (more debug needed)

* fix: scenario title setted to empty when theres no scenario title

* fix: redirect to dashboard.py

* fix: waf containers on production mode working

* fix: change user in config files automatically

* fix: remove unnecesary variable user

* remove unused functions

* fix: blockchain network endpoint

* fix: deploying different scenarios with process

* define NEBULA Controller API, update makefile

* enable controller endpoint from the frontend

* fix: connection between nodes in process deployment

* refactor: fronted.log in run_frontend for frontend.log

* feature: halt scenarios when RAM or GPU usage is maxed out

* fix: obtain list of gpus available based in user

* fix: gpus list

* fix: docker ips and gpus lists

* Fix generate statistics

---------

Co-authored-by: FerTV <fernando.torres.vega@gmail.com>

Author: enriquetomasmb

Makefile

@@ -3,13 +3,6 @@ PYTHON_VERSION := 3.11
 UV_INSTALL_SCRIPT := https://astral.sh/uv/install.sh
 PATH := $(HOME)/.local/bin:$(PATH)
 
-command_exists = $(shell command -v $(1) >/dev/null 2>&1 && echo true || echo false)
-
-define install_uv
-	@echo "📦 uv is not installed. Installing uv..."
-	@curl -LsSf $(UV_INSTALL_SCRIPT) | sh
-endef
-
 .PHONY: check-uv
 check-uv:		## Check and install uv if necessary
 	@if command -v $(UV) >/dev/null 2>&1; then \
@@ -33,33 +26,26 @@ install-python: check-uv	## Install Python with uv
 .PHONY: install
 install: install-python		## Install core dependencies
 	@echo "📦 Installing core dependencies with uv"
-	@$(UV) sync --group core
+	@$(UV) sync --group controller --group core
 	@echo "🔧 Installing pre-commit hooks"
 	@$(UV) run pre-commit install
 	@echo ""
-	@echo "🐳 Building nebula-frontend docker image. Do you want to continue (overrides existing image)? (y/n)"
-	@read ans; if [ "$${ans:-N}" = y ]; then \
-		docker build -t nebula-frontend -f nebula/frontend/Dockerfile .; \
-	else \
-		echo "Skipping nebula-frontend docker build."; \
-	fi
-	@echo ""
-	@echo "🐳 Building nebula-core docker image. Do you want to continue? (overrides existing image)? (y/n)"
-	@read ans; if [ "$${ans:-N}" = y ]; then \
-		docker build -t nebula-core .; \
-	else \
-		echo "Skipping nebula-core docker build."; \
-	fi
+	@$(MAKE) update
 	@echo ""
 	@$(MAKE) shell
 
-.PHONY: full-install
-full-install: install-python	## Install all dependencies (core, docs)
-	@echo "📦 Installing all dependencies with uv"
-	@$(UV) sync --group core --group docs
-	@echo "🔧 Installing pre-commit hooks"
-	@$(UV) run pre-commit install
-	@$(MAKE) shell
+.PHONY: install-production
+install-production: install	## Install production dependencies
+	@echo "🐳 Updating production docker images..."
+	@echo "🐳 Building nebula-waf"
+	@docker build -t nebula-waf -f nebula/addons/waf/Dockerfile-waf --build-arg USER=$(USER) nebula/addons/waf
+	@echo "🐳 Building nebula-loki"
+	@docker build -t nebula-waf-loki -f nebula/addons/waf/Dockerfile-loki nebula/addons/waf
+	@echo "🐳 Building nebula-promtail"
+	@docker build -t nebula-waf-promtail -f nebula/addons/waf/Dockerfile-promtail --build-arg USER=$(USER) nebula/addons/waf
+	@echo "🐳 Building nebula-grafana"
+	@docker build -t nebula-waf-grafana -f nebula/addons/waf/Dockerfile-grafana --build-arg USER=$(USER) nebula/addons/waf
+	echo "🐳 Docker images updated."
 
 .PHONY: shell
 shell:				## Start a shell in the uv environment
@@ -79,20 +65,31 @@ shell:				## Start a shell in the uv environment
 		echo "🚀 Created by \033[1;34mEnrique Tomás Martínez Beltrán\033[0m <\033[1;34menriquetomas@um.es\033[0m>"; \
 	fi
 
+.PHONY: update
+update:				## Update docker images
+	@echo "🐳 Updating docker images..."
+	@echo "🐳 Building nebula-frontend docker image. Do you want to continue (overrides existing image)? (y/n)"
+	@read ans; if [ "$${ans:-N}" = y ]; then \
+		docker build -t nebula-frontend -f nebula/frontend/Dockerfile .; \
+	else \
+		echo "Skipping nebula-frontend docker build."; \
+	fi
+	@echo ""
+	@echo "🐳 Building nebula-core docker image. Do you want to continue? (overrides existing image)? (y/n)"
+	@read ans; if [ "$${ans:-N}" = y ]; then \
+		docker build -t nebula-core .; \
+	else \
+		echo "Skipping nebula-core docker build."; \
+	fi
+	echo "🐳 Docker images updated."
+
 .PHONY: lock
 lock:				## Update the lock file
 	@echo "🔒 This will update the lock file. Do you want to continue? (y/n)"
 	@read ans && [ $${ans:-N} = y ] || { echo "Lock cancelled."; exit 1; }
 	@echo "🔒 Locking dependencies..."
 	@$(UV) lock
 
-.PHONY: update-libs
-update-libs:			## Update libraries to the latest version
-	@echo "🔧 This will override the versions of current libraries. Do you want to continue? (y/n)"
-	@read ans && [ $${ans:-N} = y ] || { echo "Update cancelled."; exit 1; }
-	@echo "📦 Updating libraries..."
-	@$(UV) update
-
 .PHONY: check
 check:				## Run code quality tools
 	@echo "🛠️ Running code quality checks"
@@ -127,6 +124,12 @@ publish:			## Publish a release to PyPI
 .PHONY: build-and-publish
 build-and-publish: build publish	## Build and publish the package
 
+.PHONY: doc-install
+full-install: install-python	## Install dependencies for documentation
+	@echo "📦 Installing doc dependencies with uv"
+	@$(UV) sync --group core --group docs
+	@$(MAKE) shell
+
 .PHONY: doc-test
 doc-test:			## Test if documentation can be built without errors
 	@$(UV) run mkdocs build -f docs/mkdocs.yml -d _build -s
@@ -139,12 +142,6 @@ doc-build:			## Build the documentation
 doc-serve:			## Serve the documentation locally
 	@$(UV) run mkdocs serve -f docs/mkdocs.yml
 
-.PHONY: format
-format:				## Format code with black and isort
-	@echo "🎨 Formatting code"
-	@$(UV) run black .
-	@$(UV) run isort .
-
 .PHONY: clean
 clean: clean-build		## Clean up build artifacts and caches
 	@echo "🧹 Cleaning up build artifacts and caches"

app/main.py

@@ -9,6 +9,14 @@
 
 argparser = argparse.ArgumentParser(description="Controller of NEBULA platform", add_help=False)
 
+argparser.add_argument(
+    "-cp",
+    "--controllerport",
+    dest="controllerport",
+    default=5000,
+    help="Controller port (default: 5000)",
+)
+
 argparser.add_argument(
     "--grafanaport",
     dest="grafanaport",
@@ -38,6 +46,14 @@
     help="Frontend port (default: 6060)",
 )
 
+argparser.add_argument(
+    "-sp",
+    "--statsport",
+    dest="statsport",
+    default=8080,
+    help="Statistics port (default: 8080)",
+)
+
 argparser.add_argument("-t", "--test", dest="test", action="store_true", default=False, help="Run tests")
 
 argparser.add_argument(
@@ -49,42 +65,41 @@
     default=None,
     help="Stop NEBULA platform or nodes only (use '--stop nodes' to stop only the nodes)",
 )
-argparser.add_argument(
-    "-sp",
-    "--statsport",
-    dest="statsport",
-    default=8080,
-    help="Statistics port (default: 8080)",
-)
+
 argparser.add_argument("-s", "--simulation", action="store_false", dest="simulation", help="Run simulation")
+
 argparser.add_argument(
     "-c",
     "--config",
     dest="config",
     default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "config"),
     help="Config directory path",
 )
+
 argparser.add_argument(
     "-l",
     "--logs",
     dest="logs",
     default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "logs"),
     help="Logs directory path",
 )
+
 argparser.add_argument(
     "-ce",
     "--certs",
     dest="certs",
     default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "certs"),
     help="Certs directory path",
 )
+
 argparser.add_argument(
     "-e",
     "--env",
     dest="env",
     default=os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env"),
     help=".env file path",
 )
+
 argparser.add_argument(
     "-p",
     "--production",
@@ -93,6 +108,7 @@
     default=False,
     help="Production mode",
 )
+
 argparser.add_argument(
     "-ad",
     "--advanced",
@@ -101,20 +117,23 @@
     default=False,
     help="Advanced analytics",
 )
+
 argparser.add_argument(
     "-v",
     "--version",
     action="version",
     version="%(prog)s " + nebula.__version__,
     help="Show version",
 )
+
 argparser.add_argument(
     "-a",
     "--about",
     action="version",
     version="Created by Enrique Tomás Martínez Beltrán",
     help="Show author",
 )
+
 argparser.add_argument("-h", "--help", action="help", default=argparse.SUPPRESS, help="Show help")
 
 args = argparser.parse_args()

docs/_prebuilt/installation.md

@@ -125,7 +125,7 @@ frontend, controller, and nodes.
 
 ## Possible issues during the installation or execution
 
-If frontend is not working, check the logs in app/logs/server.log
+If frontend is not working, check the logs in app/logs/frontend.log
 
 If any of the following errors appear, take a look at the docker logs of
 the nebula-frontend container:

nebula/addons/waf/Dockerfile-grafana

@@ -1,5 +1,10 @@
 FROM grafana/grafana:latest
 
+ARG USER
+ENV USER=${USER}
+
 COPY ./grafana/dashboard_config.yml /etc/grafana/provisioning/dashboards/local.yml
 COPY ./grafana/automatic.yml /etc/grafana/provisioning/datasources/automatic.yml
 COPY ./grafana/dashboard.json /var/lib/grafana/dashboards/dashboard.json
+
+RUN sed -i "s|http://nebula|http://$USER|g" /etc/grafana/provisioning/datasources/automatic.yml

nebula/addons/waf/Dockerfile-loki

@@ -1 +1,3 @@
 FROM grafana/loki:latest
+
+COPY loki-config.yml /mnt/config/loki-config.yml

nebula/addons/waf/Dockerfile-promtail

@@ -1 +1,8 @@
 FROM grafana/promtail:latest
+
+ARG USER
+ENV USER=${USER}
+
+COPY promtail-config.yml /etc/promtail/config.yml
+
+RUN sed -i "s|http://nebula|http://$USER|g" /etc/promtail/config.yml

nebula/addons/waf/Dockerfile-waf

@@ -3,6 +3,9 @@ FROM owasp/modsecurity-crs:3.3.5-nginx-202310170110
 
 ARG NGINX_VERSION=1.24.0
 
+ARG USER
+ENV USER=${USER}
+
 # Installed necessary packages
 RUN apt-get update && apt-get install -y libmaxminddb0 libmaxminddb-dev mmdb-bin git wget
 RUN apt install -y build-essential libpcre3 libpcre3-dev zlib1g zlib1g-dev libssl-dev
@@ -37,6 +40,9 @@ RUN wget https://git.io/GeoLite2-Country.mmdb -P /usr/share/GeoIP/
 
 # nginx configuration files
 COPY default.conf /etc/nginx/templates/conf.d/default.conf.template
+
+RUN sed -i "s|http://nebula|http://${USER}|g" /etc/nginx/templates/conf.d/default.conf.template
+
 COPY nginx.conf /etc/nginx/templates/nginx.conf.template
 
 # owasp crs

nebula/addons/waf/default.conf

@@ -12,7 +12,7 @@ server {
     listen 80 default_server;
 
     server_name localhost;
-    set $upstream http://nebula-frontend; # Change this
+    set $upstream http://nebula-nebula-frontend; # Change this
     set $always_redirect off;
     modsecurity on;
     location /nebula {

nebula/addons/waf/grafana/automatic.yml

@@ -1,6 +1,6 @@
 datasources:
 - name: Loki
   type: loki
-  url: http://loki:3100
+  url: http://nebula-nebula-waf-loki:3100
   isDefault: true
   editable: true

nebula/addons/waf/promtail-config.yml

@@ -6,7 +6,7 @@ positions:
   filename: /tmp/positions.yaml
 
 clients:
-  - url: http://loki:3100/loki/api/v1/push
+  - url: http://nebula-nebula-waf-loki:3100/loki/api/v1/push
 
 scrape_configs:
   - job_name: nginx