Skip to content

Commit 70ad813

Browse files
enriquetomasmbenriquetomasmb
committed
fix process deployment
1 parent be8c2d8 commit 70ad813

File tree

8 files changed

+34
-65
lines changed

8 files changed

+34
-65
lines changed

‎app/deployer.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ def run_observer():
219219

220220
class Deployer:
221221
def __init__(self, args):
222-
self.controller_port = int(args.controllerport) if hasattr(args, "controllerport") else 5000
222+
self.controller_port = int(args.controllerport) if hasattr(args, "controllerport") else 5050
223223
self.waf_port = int(args.wafport) if hasattr(args, "wafport") else 6000
224224
self.frontend_port = int(args.webport) if hasattr(args, "webport") else 6060
225225
self.grafana_port = int(args.grafanaport) if hasattr(args, "grafanaport") else 6040
@@ -338,13 +338,13 @@ def start(self):
338338

339339
# Check ports available
340340
if not SocketUtils.is_port_open(self.controller_port):
341-
self.controller_port = SocketUtils.find_free_port()
341+
self.controller_port = SocketUtils.find_free_port(start_port=self.controller_port)
342342

343343
if not SocketUtils.is_port_open(self.frontend_port):
344-
self.frontend_port = SocketUtils.find_free_port(self.controller_port + 1)
344+
self.frontend_port = SocketUtils.find_free_port(start_port=self.frontend_port)
345345

346346
if not SocketUtils.is_port_open(self.statistics_port):
347-
self.statistics_port = SocketUtils.find_free_port(self.frontend_port + 1)
347+
self.statistics_port = SocketUtils.find_free_port(start_port=self.statistics_port)
348348

349349
self.run_controller()
350350
logging.info("NEBULA Controller is running")
@@ -440,7 +440,7 @@ def run_frontend(self):
440440
"NEBULA_HOST_PLATFORM": self.host_platform,
441441
"NEBULA_DEFAULT_USER": "admin",
442442
"NEBULA_DEFAULT_PASSWORD": "admin",
443-
"NEBULA_CONTROLLER_PORT": 5000,
443+
"NEBULA_CONTROLLER_PORT": self.controller_port,
444444
"NEBULA_CONTROLLER_HOST": self.controller_host,
445445
}
446446

@@ -520,14 +520,14 @@ def run_controller(self):
520520
"NEBULA_LOGS_DIR": "/nebula/app/logs/",
521521
"NEBULA_CERTS_DIR": "/nebula/app/certs/",
522522
"NEBULA_HOST_PLATFORM": self.host_platform,
523-
"NEBULA_CONTROLLER_PORT": 5000,
523+
"NEBULA_CONTROLLER_PORT": self.controller_port,
524524
"NEBULA_CONTROLLER_HOST": self.controller_host,
525525
"NEBULA_FRONTEND_PORT": self.frontend_port,
526526
}
527527

528528
volumes = ["/nebula", "/var/run/docker.sock"]
529529

530-
ports = [5000]
530+
ports = [self.controller_port]
531531

532532
host_config = client.api.create_host_config(
533533
binds=[
@@ -536,7 +536,7 @@ def run_controller(self):
536536
f"{self.databases_dir}:/nebula/app/databases"
537537
],
538538
extra_hosts={"host.docker.internal": "host-gateway"},
539-
port_bindings={5000: self.controller_port},
539+
port_bindings={self.controller_port: self.controller_port},
540540
)
541541

542542
networking_config = client.api.create_networking_config({

‎app/main.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@
1313
"-cp",
1414
"--controllerport",
1515
dest="controllerport",
16-
default=5000,
17-
help="Controller port (default: 5000)",
16+
default=5050,
17+
help="Controller port (default: 5050)",
1818
)
1919

2020
argparser.add_argument(

‎nebula/config/config.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ def __setup_logging(self, log_filename):
9090
log_console_format = f"{CYAN}%(asctime)s - {self.participant['device_args']['name']} - [%(filename)s:%(lineno)d]{RESET}\n%(message)s"
9191

9292
console_handler = logging.StreamHandler()
93-
console_handler.setLevel(logging.CRITICAL)
93+
console_handler.setLevel(logging.INFO if self.participant["device_args"]["logging"] else logging.CRITICAL)
9494
console_handler.setFormatter(Formatter(log_console_format))
9595

9696
file_handler = FileHandler(f"{log_filename}.log", mode="w", encoding="utf-8")
@@ -126,7 +126,7 @@ def __set_training_logging(self):
126126
level = logging.DEBUG if self.participant["device_args"]["logging"] else logging.CRITICAL
127127

128128
console_handler = logging.StreamHandler()
129-
console_handler.setLevel(logging.CRITICAL)
129+
console_handler.setLevel(logging.INFO if self.participant["device_args"]["logging"] else logging.CRITICAL)
130130
console_handler.setFormatter(Formatter(log_console_format))
131131

132132
file_handler = FileHandler(f"{training_log_filename}.log", mode="w", encoding="utf-8")

‎nebula/controller/controller.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -875,7 +875,7 @@ async def verify_user_controller(user: str = Body(...), password: str = Body(...
875875
if __name__ == "__main__":
876876
# Parse args from command line
877877
parser = argparse.ArgumentParser()
878-
parser.add_argument("--port", type=int, default=5000, help="Port to run the controller on.")
878+
parser.add_argument("--port", type=int, default=5050, help="Port to run the controller on.")
879879
args = parser.parse_args()
880880
logging.info(f"Starting frontend on port {args.port}")
881881
import uvicorn

‎nebula/controller/scenarios.py

Lines changed: 14 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1051,39 +1051,13 @@ def start_nodes_process(self):
10511051

10521052
# Include additional config to the participants
10531053
for idx, node in enumerate(self.config.participants):
1054-
node["tracking_args"]["log_dir"] = os.path.join(
1055-
self.root_path, "..", "nebula", "controller", "..", "..", "app", "logs"
1056-
)
1057-
node["tracking_args"]["config_dir"] = os.path.join(
1058-
self.root_path, "..", "nebula", "controller", "..", "..", "app", "config", self.scenario_name
1059-
)
1054+
node["tracking_args"]["log_dir"] = os.path.join(self.root_path, "app", "logs")
1055+
node["tracking_args"]["config_dir"] = os.path.join(self.root_path, "app", "config", self.scenario_name)
10601056
node["scenario_args"]["controller"] = self.controller
10611057
node["scenario_args"]["deployment"] = self.scenario.deployment
1062-
node["security_args"]["certfile"] = os.path.join(
1063-
self.root_path,
1064-
"..",
1065-
"nebula",
1066-
"controller",
1067-
"..",
1068-
"..",
1069-
"app",
1070-
"certs",
1071-
f"participant_{node['device_args']['idx']}_cert.pem",
1072-
)
1073-
node["security_args"]["keyfile"] = os.path.join(
1074-
self.root_path,
1075-
"..",
1076-
"nebula",
1077-
"controller",
1078-
"..",
1079-
"..",
1080-
"app",
1081-
"certs",
1082-
f"participant_{node['device_args']['idx']}_key.pem",
1083-
)
1084-
node["security_args"]["cafile"] = os.path.join(
1085-
self.root_path, "..", "nebula", "controller", "..", "..", "app", "certs", "ca_cert.pem"
1086-
)
1058+
node["security_args"]["certfile"] = os.path.join(self.root_path, "app", "certs", f"participant_{node['device_args']['idx']}_cert.pem")
1059+
node["security_args"]["keyfile"] = os.path.join(self.root_path, "app", "certs", f"participant_{node['device_args']['idx']}_key.pem")
1060+
node["security_args"]["cafile"] = os.path.join(self.root_path, "app", "certs", "ca_cert.pem")
10871061

10881062
# Write the config file in config directory
10891063
with open(f"{self.config_dir}/participant_{node['device_args']['idx']}.json", "w") as f:
@@ -1109,19 +1083,19 @@ def start_nodes_process(self):
11091083
commands += "Start-Sleep -Seconds 2\n"
11101084

11111085
commands += f'Write-Host "Running node {node["device_args"]["idx"]}..."\n'
1112-
commands += f'$OUT_FILE = "{self.processes_root_path}\\app\\logs\\{self.scenario_name}\\participant_{node["device_args"]["idx"]}.out"\n'
1113-
commands += f'$ERROR_FILE = "{self.processes_root_path}\\app\\logs\\{self.scenario_name}\\participant_{node["device_args"]["idx"]}.err"\n'
1086+
commands += f'$OUT_FILE = "{self.root_path}\\app\\logs\\{self.scenario_name}\\participant_{node["device_args"]["idx"]}.out"\n'
1087+
commands += f'$ERROR_FILE = "{self.root_path}\\app\\logs\\{self.scenario_name}\\participant_{node["device_args"]["idx"]}.err"\n'
11141088

11151089
# Use Start-Process for executing Python in background and capture PID
1116-
commands += f"""$process = Start-Process -FilePath "python" -ArgumentList "{self.processes_root_path}\\nebula\\core\\node.py {self.processes_root_path}\\app\\config\\{self.scenario_name}\\participant_{node["device_args"]["idx"]}.json" -PassThru -NoNewWindow -RedirectStandardOutput $OUT_FILE -RedirectStandardError $ERROR_FILE
1090+
commands += f"""$process = Start-Process -FilePath "python" -ArgumentList "{self.root_path}\\nebula\\core\\node.py {self.root_path}\\app\\config\\{self.scenario_name}\\participant_{node["device_args"]["idx"]}.json" -PassThru -NoNewWindow -RedirectStandardOutput $OUT_FILE -RedirectStandardError $ERROR_FILE
11171091
Add-Content -Path $PID_FILE -Value $process.Id
11181092
"""
11191093

11201094
commands += 'Write-Host "All nodes started. PIDs stored in $PID_FILE"\n'
11211095

1122-
with open(f"/nebula/app/config/{self.scenario_name}/current_scenario_commands.ps1", "w") as f:
1096+
with open(f"{self.config_dir}/current_scenario_commands.ps1", "w") as f:
11231097
f.write(commands)
1124-
os.chmod(f"/nebula/app/config/{self.scenario_name}/current_scenario_commands.ps1", 0o755)
1098+
os.chmod(f"{self.config_dir}/current_scenario_commands.ps1", 0o755)
11251099
else:
11261100
commands = '#!/bin/bash\n\nPID_FILE="$(dirname "$0")/current_scenario_pids.txt"\n\n> $PID_FILE\n\n'
11271101
sorted_participants = sorted(
@@ -1135,22 +1109,15 @@ def start_nodes_process(self):
11351109
else:
11361110
commands += "sleep 2\n"
11371111
commands += f'echo "Running node {node["device_args"]["idx"]}..."\n'
1138-
# commands += f"OUT_FILE={self.processes_root_path}/app/logs/{self.scenario_name}/participant_{node['device_args']['idx']}.out\n"
1139-
commands += f"python nebula/core/node.py app/config/{self.scenario_name}/participant_{node['device_args']['idx']}.json &\n"
1112+
commands += f"OUT_FILE={self.root_path}/app/logs/{self.scenario_name}/participant_{node['device_args']['idx']}.out\n"
1113+
commands += f"python {self.root_path}/nebula/core/node.py {self.root_path}/app/config/{self.scenario_name}/participant_{node['device_args']['idx']}.json &\n"
11401114
commands += "echo $! >> $PID_FILE\n\n"
11411115

11421116
commands += 'echo "All nodes started. PIDs stored in $PID_FILE"\n'
11431117

1144-
logging.info(f"[FER] root_path {self.root_path} processes_root_path {self.processes_root_path}")
1145-
logging.info(f"OUT_FILE=app/logs/{self.scenario_name}/participant_{node['device_args']['idx']}.out\n")
1146-
1147-
with open(
1148-
f"{self.processes_root_path}/app/config/{self.scenario_name}/current_scenario_commands.sh", "w"
1149-
) as f:
1118+
with open(f"{self.config_dir}/current_scenario_commands.sh", "w") as f:
11501119
f.write(commands)
1151-
os.chmod(
1152-
f"{self.processes_root_path}/app/config/{self.scenario_name}/current_scenario_commands.sh", 0o755
1153-
)
1120+
os.chmod(f"{self.config_dir}/current_scenario_commands.sh", 0o755)
11541121

11551122
except Exception as e:
11561123
raise Exception(f"Error starting nodes as processes: {e}")

‎nebula/controller/start_services.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ NEBULA_SOCK=nebula.sock
1414
echo "NEBULA_PRODUCTION: $NEBULA_PRODUCTION"
1515
if [ "$NEBULA_PRODUCTION" = "False" ]; then
1616
echo "Starting Gunicorn in dev mode..."
17-
uvicorn nebula.controller.controller:app --host 0.0.0.0 --port 5000 --log-level debug --proxy-headers --forwarded-allow-ips "*" &
17+
uvicorn nebula.controller.controller:app --host 0.0.0.0 --port $NEBULA_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" &
1818
else
1919
echo "Starting Gunicorn in production mode..."
20-
uvicorn nebula.controller.controller:app --host 0.0.0.0 --port 5000 --log-level info --proxy-headers --forwarded-allow-ips "*" &
20+
uvicorn nebula.controller.controller:app --host 0.0.0.0 --port $NEBULA_CONTROLLER_PORT --log-level info --proxy-headers --forwarded-allow-ips "*" &
2121
fi
2222

2323
tail -f /dev/null

‎nebula/core/datasets/nebuladataset.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from abc import ABC, abstractmethod
55
from types import SimpleNamespace
66
from typing import Any
7+
import time
78

89
import h5py
910
import matplotlib
@@ -29,6 +30,7 @@ def wait_for_file(file_path):
2930
"""Wait until the given file exists, polling every 'interval' seconds."""
3031
while not os.path.exists(file_path):
3132
logging_training.info(f"Waiting for file: {file_path}")
33+
time.sleep(1)
3234
return
3335

3436

‎nebula/frontend/app.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ class Settings:
2727
2828
Attributes:
2929
controller_host (str): Hostname or IP address of the Nebula controller service.
30-
controller_port (int): Port on which the Nebula controller listens (default: 5000).
30+
controller_port (int): Port on which the Nebula controller listens (default: 5050).
3131
resources_threshold (float): Threshold for resource usage alerts (default: 0.0).
3232
port (int): Port for the Nebula frontend service (default: 6060).
3333
production (bool): Whether the application is running in production mode.
@@ -47,7 +47,7 @@ class Settings:
4747
"""
4848

4949
controller_host: str = os.environ.get("NEBULA_CONTROLLER_HOST")
50-
controller_port: int = os.environ.get("NEBULA_CONTROLLER_PORT", 5000)
50+
controller_port: int = os.environ.get("NEBULA_CONTROLLER_PORT", 5050)
5151
resources_threshold: float = 0.0
5252
port: int = os.environ.get("NEBULA_FRONTEND_PORT", 6060)
5353
production: bool = os.environ.get("NEBULA_PRODUCTION", "False") == "True"
@@ -2206,7 +2206,7 @@ async def nebula_dashboard_deployment_run(
22062206
if __name__ == "__main__":
22072207
# Parse args from command line
22082208
parser = argparse.ArgumentParser()
2209-
parser.add_argument("--port", type=int, default=5000, help="Port to run the frontend on.")
2209+
parser.add_argument("--port", type=int, default=6060, help="Port to run the frontend on.")
22102210
args = parser.parse_args()
22112211
logging.info(f"Starting frontend on port {args.port}")
22122212
import uvicorn

0 commit comments

Comments
 (0)