Feature/SK-1575 | Test local updates on Dev Studio

.ci/tests/examples/api_test.py

@@ -1,5 +1,6 @@
 import fire
 import yaml
+from server_functions import ServerFunctions
 
 from fedn import APIClient
 
@@ -32,6 +33,15 @@ def test_api_get_methods():
     assert clients_count
     print("Clients count: ", clients_count, flush=True)
 
+    client_id = clients["result"][0]["client_id"]
+    client_obj = client.get_client(client_id)
+    assert client_obj
+    assert client_id == client_obj["client_id"]
+    print("Client: ", client_obj, flush=True)
+
+    assert clients_count == len(clients["result"])
+    assert clients_count == clients["count"]
+
     # --- Combiners --- #
 
     combiners = client.get_combiners()
@@ -46,6 +56,9 @@ def test_api_get_methods():
     assert combiner
     print("Combiner: ", combiner, flush=True)
 
+    assert combiners_count == len(combiners["result"])
+    assert combiners_count == combiners["count"]
+
     # --- Controllers --- #
 
     status = client.get_controller_status()
@@ -64,12 +77,16 @@ def test_api_get_methods():
 
     models_from_trail = client.get_model_trail()
     assert models_from_trail
-    print("Models: ", models_from_trail, flush=True)
+    assert len(models_from_trail) == models_count
+    print("Models (model trail): ", models_from_trail, flush=True)
 
     active_model = client.get_active_model()
     assert active_model
     print("Active model: ", active_model, flush=True)
 
+    assert models_count == len(models["result"])
+    assert models_count == models["count"]
+
     # --- Packages --- #
 
     packages = client.get_packages()
@@ -88,6 +105,9 @@ def test_api_get_methods():
     assert checksum
     print("Checksum: ", checksum, flush=True)
 
+    assert packages_count == len(packages["result"])
+    assert packages_count == packages["count"]
+
     # --- Rounds --- #
 
     rounds = client.get_rounds()
@@ -98,6 +118,9 @@ def test_api_get_methods():
     assert rounds_count
     print("Rounds count: ", rounds_count, flush=True)
 
+    assert rounds_count == len(rounds["result"])
+    assert rounds_count == rounds["count"]
+
     # --- Sessions --- #
 
     sessions = client.get_sessions()
@@ -108,6 +131,14 @@ def test_api_get_methods():
     assert sessions_count
     print("Sessions count: ", sessions_count, flush=True)
 
+    session = client.get_session(id=sessions["result"][0]["session_id"])
+    assert session
+    assert session["session_id"] == sessions["result"][0]["session_id"]
+    print("Session: ", session, flush=True)
+
+    assert sessions_count == len(sessions["result"])
+    assert sessions_count == sessions["count"]
+
     # --- Statuses --- #
 
     statuses = client.get_statuses()
@@ -118,6 +149,9 @@ def test_api_get_methods():
     assert statuses_count
     print("Statuses count: ", statuses_count, flush=True)
 
+    assert statuses_count == len(statuses["result"])
+    assert statuses_count == statuses["count"]
+
     # --- Validations --- #
 
     validations = client.get_validations()
@@ -128,6 +162,13 @@ def test_api_get_methods():
     assert validations_count
     print("Validations count: ", validations_count, flush=True)
 
+    assert validations_count == len(validations["result"])
+    assert validations_count == validations["count"]
+
+
+def start_sf_session(name, rounds, helper):
+    client = APIClient(host="localhost", port=8092)
+    client.start_session(name=name, rounds=rounds, helper=helper, server_functions=ServerFunctions)
 
 if __name__ == '__main__':
 
@@ -136,6 +177,7 @@ def test_api_get_methods():
         'set_seed': client.set_active_model,
         'set_package': client.set_active_package,
         'start_session': client.start_session,
+        'start_sf_session': start_sf_session,
         'get_client_config': _download_config,
         'test_api_get_methods': test_api_get_methods,
     })

.ci/tests/examples/run.sh

@@ -31,6 +31,9 @@ else
         up -d --build combiner api-server mongo minio client1   
 fi
 
+# add server functions to python path to import server functions code
+export PYTHONPATH="$PYTHONPATH:../server-functions"
+
 >&2 echo "Wait for reducer to start"
 python ../../.ci/tests/examples/wait_for.py reducer
 
@@ -46,8 +49,14 @@ python ../../.ci/tests/examples/wait_for.py clients
 >&2 echo "Upload seed"
 python ../../.ci/tests/examples/api_test.py set_seed --path seed.npz
 
->&2 echo "Start session"
-python ../../.ci/tests/examples/api_test.py start_session --name "session" --rounds 3 --helper "$helper"
+if [ "$example" == "server-functions" ]; then
+    >&2 echo "Start serverfunctions session"
+    python ../../.ci/tests/examples/api_test.py start_sf_session --name "session" --rounds 3 --helper "$helper"
+else
+    >&2 echo "Start session"
+    python ../../.ci/tests/examples/api_test.py start_session --name "session" --rounds 3 --helper "$helper"
+fi
+
 
 >&2 echo "Checking rounds success"
 python ../../.ci/tests/examples/wait_for.py rounds

.ci/tests/studio/studio.sh

@@ -32,7 +32,7 @@ fi
 fedn studio login -u $STUDIO_USER -P $STUDIO_PASSWORD -H $STUDIO_HOST
 fedn project create -n citest -H $STUDIO_HOST --no-interactive
 sleep 5
-FEDN_PROJECT=$(fedn project list -H $STUDIO_HOST | awk 'NR>=1 {print $1; exit}')
+FEDN_PROJECT=$(fedn project list -H $STUDIO_HOST --no-header | awk 'NR>=1 {print $3; exit}')
 fedn project set-context -id $FEDN_PROJECT -H $STUDIO_HOST
 pushd examples/$FEDN_EXAMPLE
 fedn client get-config -n test -g $FEDN_NR_CLIENTS -H $STUDIO_HOST
@@ -43,4 +43,11 @@ for i in $(seq 0 $(($FEDN_NR_CLIENTS - 1))); do
 done
 popd
 sleep 5
-pytest .ci/tests/studio/tests.py
+# add server functions so we can import it in start_session
+export PYTHONPATH="$PYTHONPATH:$(pwd)/examples/server-functions"
+pytest .ci/tests/studio/tests.py -x
+sleep 5
+# run with server functions
+export FEDN_SERVER_FUNCTIONS="1"
+export SESSION_NUMBER="2"
+pytest .ci/tests/studio/tests.py -x

.ci/tests/studio/tests.py

@@ -1,14 +1,15 @@
-import os
+import os, sys
 import time
 import pytest
 from fedn import APIClient
 from fedn.cli.shared import get_token, get_project_url
+from server_functions import ServerFunctions
+from fedn.common.log_config import logger
 
 @pytest.fixture(scope="module")
 def fedn_client():
     token = get_token(token=None, usr_token=False)
     host = get_project_url("", "", None, False)
-    print(f"Connecting to {host}")
     client = APIClient(host=host, token=token, secure=True, verify=True)
     return client
 
@@ -19,11 +20,13 @@ def fedn_env():
         "FEDN_ROUND_TIMEOUT": int(os.environ.get("FEDN_ROUND_TIMEOUT", 180)),
         "FEDN_BUFFER_SIZE": int(os.environ.get("FEDN_BUFFER_SIZE", -1)),
         "FEDN_NR_CLIENTS": int(os.environ.get("FEDN_NR_CLIENTS", 2)),
-        "FEDN_CLIENT_TIMEOUT": int(os.environ.get("FEDN_CLIENT_TIMEOUT", 60)),
+        "FEDN_CLIENT_TIMEOUT": int(os.environ.get("FEDN_CLIENT_TIMEOUT", 600)),
         "FEDN_FL_ALG": os.environ.get("FEDN_FL_ALG", "fedavg"),
         "FEDN_NR_EXPECTED_AGG": int(os.environ.get("FEDN_NR_EXPECTED_AGG", 2)), # Number of expected aggregated models per combiner
         "FEDN_SESSION_TIMEOUT": int(os.environ.get("FEDN_SESSION_TIMEOUT", 300)), # Session timeout in seconds, all rounds must be finished within this time
-        "FEDN_SESSION_NAME": os.environ.get("FEDN_SESSION_NAME", "test")
+        "FEDN_SESSION_NAME": os.environ.get("FEDN_SESSION_NAME", "test"),
+        "FEDN_SERVER_FUNCTIONS": os.environ.get("FEDN_SERVER_FUNCTIONS", 0),
+        "SESSION_NUMBER": os.environ.get("SESSION_NUMBER", 1)
     }
 
 @pytest.mark.order(1)
@@ -49,14 +52,16 @@ def test_start_session(self, fedn_client, fedn_env):
             rounds=fedn_env["FEDN_NR_ROUNDS"], 
             round_buffer_size=fedn_env["FEDN_BUFFER_SIZE"],
             min_clients=fedn_env["FEDN_NR_CLIENTS"], 
-            requested_clients=fedn_env["FEDN_NR_CLIENTS"]
+            requested_clients=fedn_env["FEDN_NR_CLIENTS"],
+            server_functions=ServerFunctions if fedn_env["FEDN_SERVER_FUNCTIONS"] else None
         )
         assert result["message"] == "Session started", f"Expected status 'Session started', got {result['message']}"
 
     @pytest.mark.order(3)
     def test_session_completion(self, fedn_client, fedn_env):
         session_obj = fedn_client.get_sessions()
-        assert session_obj["count"] == 1, f"Expected 1 session, got {session_obj['count']}"
+        session_number = int(fedn_env["SESSION_NUMBER"])
+        assert session_obj["count"] == session_number, f"Expected {session_number} session/s, got {session_obj['count']}"
         session_result = session_obj["result"][0]
 
         start_time = time.time()
@@ -77,13 +82,14 @@ def test_session_completion(self, fedn_client, fedn_env):
     @pytest.mark.order(4)
     def test_rounds_completion(self, fedn_client, fedn_env):
         start_time = time.time()
+        session_number = int(fedn_env["SESSION_NUMBER"])
         while time.time() - start_time < fedn_env["FEDN_SESSION_TIMEOUT"]:
             rounds_obj = fedn_client.get_rounds()
-            if rounds_obj["count"] == fedn_env["FEDN_NR_ROUNDS"]:
+            if rounds_obj["count"] == session_number * fedn_env["FEDN_NR_ROUNDS"]:
                 break
             time.sleep(5)
         else:
-            raise TimeoutError(f"Expected {fedn_env['FEDN_NR_ROUNDS']} rounds, but got {rounds_obj['count']} within {fedn_env['FEDN_SESSION_TIMEOUT']} seconds")
+            raise TimeoutError(f"Expected {session_number * fedn_env['FEDN_NR_ROUNDS']} rounds, but got {rounds_obj['count']} within {fedn_env['FEDN_SESSION_TIMEOUT']} seconds")
 
         rounds_result = rounds_obj["result"]
         for round in rounds_result:
@@ -96,14 +102,15 @@ def test_rounds_completion(self, fedn_client, fedn_env):
     @pytest.mark.order(5)
     def test_validations(self, fedn_client, fedn_env):
         start_time = time.time()
+        session_number = int(fedn_env["SESSION_NUMBER"])
         while time.time() - start_time < fedn_env["FEDN_SESSION_TIMEOUT"]:
             validation_obj = fedn_client.get_validations()
-            if validation_obj["count"] == fedn_env["FEDN_NR_ROUNDS"] * fedn_env["FEDN_NR_CLIENTS"]:
+            if validation_obj["count"] == session_number * fedn_env["FEDN_NR_ROUNDS"] * fedn_env["FEDN_NR_CLIENTS"]:
                 break
             time.sleep(5)
         else:
-            raise TimeoutError(f"Expected {fedn_env['FEDN_NR_ROUNDS'] * fedn_env['FEDN_NR_CLIENTS']} validations, but got {validation_obj['count']} within {fedn_env['FEDN_SESSION_TIMEOUT']} seconds")
+            raise TimeoutError(f"Expected {session_number * fedn_env['FEDN_NR_ROUNDS'] * fedn_env['FEDN_NR_CLIENTS']} validations, but got {validation_obj['count']} within {fedn_env['FEDN_SESSION_TIMEOUT']} seconds")
 
         # We could assert or test model convergence here
 
-        print("All tests passed!", flush=True)
+        logger.info("All tests passed!")

.github/workflows/integration-tests.yaml

@@ -23,6 +23,7 @@ jobs:
         to_test:
           - "mnist-keras numpyhelper"
           - "mnist-pytorch numpyhelper"
+          - "server-functions numpyhelper"
         python_version: ["3.9", "3.10", "3.11", "3.12"]
         os:
           - ubuntu-24.04

.github/workflows/test-api-endpoints.yaml

@@ -27,14 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install .
-
-    - name: Start mongo and minio
-      run: |
-        docker compose up -d mongo minio
-
-    - name: Start FEDn controller
-      run: |
-        fedn controller start &
+
     - name: Run tests
       run: |
         python3 -m unittest fedn.network.api.tests

.vscode/launch.json

@@ -6,7 +6,7 @@
             "type": "python",
             "request": "launch",
             "module": "pytest",
-            "justMyCode": true
+            "justMyCode": false
         },
         {
             "args": [
@@ -17,7 +17,7 @@
             "type": "python",
             "request": "launch",
             "module": "pytest",
-            "justMyCode": true
+            "justMyCode": false
         },
         {
             "name": "Run current file",