Merge branch 'CiscoDevNet:develop' into bgp_fabric

Author: mkilar123

.github/workflows/main.yml

@@ -21,15 +21,15 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ansible: [2.15.12, 2.16.7]
+        ansible: [2.15.13, 2.16.14, 2.17.12, 2.18.6]
     steps:
     - name: Check out code
       uses: actions/checkout@v2
 
-    - name: Set up Python "3.10"
+    - name: Set up Python "3.11"
       uses: actions/setup-python@v1
       with:
-        python-version: "3.10"
+        python-version: "3.11"
 
     - name: Install ansible-base (v${{ matrix.ansible }})
       run: pip install https://github.com/ansible/ansible/archive/v${{ matrix.ansible }}.tar.gz --disable-pip-version-check
@@ -52,11 +52,17 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ansible: [2.15.12, 2.16.7]
+        ansible: [2.15.13, 2.16.14, 2.17.12, 2.18.6]
         python: ["3.9", "3.10", "3.11"]
         exclude:
-          - ansible: 2.16.7
+          - ansible: 2.16.14
             python: "3.9"
+          - ansible: 2.17.12
+            python: "3.9"
+          - ansible: 2.18.6
+            python: "3.9"
+          - ansible: 2.18.6
+            python: "3.10"
     steps:
     - name: Set up Python (v${{ matrix.python }})
       uses: actions/setup-python@v1
@@ -87,15 +93,21 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ansible: [2.15.12, 2.16.7]
+        ansible: [2.15.13, 2.16.14, 2.17.12, 2.18.6]
     steps:
-    - name: Set up Python "3.10"
+    - name: Set up Python "3.11"
       uses: actions/setup-python@v1
       with:
-        python-version: "3.10"
+        python-version: "3.11"
 
     - name: Install ansible-base (v${{ matrix.ansible }})
       run: pip install https://github.com/ansible/ansible/archive/v${{ matrix.ansible }}.tar.gz --disable-pip-version-check
+    
+    - name: Install Pydantic (v2)
+      run: pip install pydantic==2.11.4
+    
+    - name: Install DeepDiff (v8.5.0)
+      run: pip install deepdiff==8.5.0
 
     - name: Install coverage (v7.3.4)
       run: pip install coverage==7.3.4
@@ -120,4 +132,4 @@ jobs:
 
     - name: Generate coverage report
       run: coverage report
-      working-directory: /home/runner/.ansible/collections/ansible_collections/cisco/dcnm
+      working-directory: /home/runner/.ansible/collections/ansible_collections/cisco/dcnm

.gitignore

@@ -79,4 +79,5 @@ venv.bak/
 .mypy_cache/
 
 # Ignore Integration Tests Files Directories
-tests/integration/targets/ndfc_interface/files
+tests/integration/targets/ndfc_interface/files
+tests/integration/targets/dcnm_network/files

playbooks/roles/dcnm_network/dcnm_tests.yaml

@@ -7,29 +7,71 @@
 # Modify the vars section with details for your testing setup.
 #
 # NOTES:
-# 1. Ensure that the switches defined by ansible_switch1 and ansible_switch2 are
+# 1. Ensure that the switches defined by switch1 and switch2 are
 #    not vPC Pairs.
 - hosts: dcnm
   gather_facts: no
   connection: ansible.netcommon.httpapi
 
   vars:
+
+    #----------------------------------
     # Uncomment testcase to run a specific test
     # testcase: replaced_net_all
-    test_fabric: nac-ndfc1
-    ansible_switch1: 192.168.1.1
-    ansible_switch2: 192.168.1.2
-    ansible_sw1_int1: Ethernet1/15
-    ansible_sw1_int2: Ethernet1/16
-    ansible_sw1_int3: Ethernet1/17
-    ansible_sw1_int4: Ethernet1/18
-    #---
-    ansible_sw2_int1: Ethernet1/15
-    ansible_sw2_int2: Ethernet1/16
-    ansible_sw2_int3: Ethernet1/17
-    ansible_sw2_int4: Ethernet1/18
-    ansible_sw2_int5: Ethernet1/19
-    ansible_sw2_int6: Ethernet1/20
-
+    test_data_common:
+      #---
+      fabric: fabric-stage
+      deploy: false
+      #---
+      # Resources
+      #---
+      sw1: 192.168.1.1
+      sw2: 192.168.1.2
+      #---
+      # Common VRF setup
+      #---
+      vrf_1: ansible-vrf-int1
+      vrf_1_id: 9008011
+      vrf_1_vlan_id: 500
+      #---
+      vrf_2: Tenant-1
+      vrf_2_id: 9008012
+      vrf_2_vlan_id: 501
+      #---
+      vrf_3: Tenant-2
+      vrf_3_id: 9008013
+      vrf_3_vlan_id: 502
+      #---
+      # Interfaces
+      #---
+      sw1_int1: Ethernet1/15
+      sw1_int2: Ethernet1/16
+      sw1_int3: Ethernet1/17
+      sw1_int4: Ethernet1/18
+      #---
+      sw2_int1: Ethernet1/15
+      sw2_int2: Ethernet1/16
+      sw2_int3: Ethernet1/17
+      sw2_int4: Ethernet1/18
+      sw2_int5: Ethernet1/19
+      sw2_int6: Ethernet1/20
+      #---
+      net1: ansible-net13
+      net1_net_id: 7005
+      net1_default_net_template: Default_Network_Universal
+      net1_net_extension_template: Default_Network_Extension_Universal
+      net1_vlan_id: 1500
+      net1_gw_ip_subnet: '192.168.30.1/24'
+      #---
+      net2: ansible-net12
+      net2_net_id: 7002
+      net2_default_net_template: Default_Network_Universal
+      net2_net_extension_template: Default_Network_Extension_Universal
+      net2_vlan_id: 151
+      net2_gw_ip_subnet: '192.168.40.1/24'
+      #---
+      net1_vrf: Tenant-1
+      net2_vrf: Tenant-2
+  
   roles:
     - dcnm_network

plugins/action/tests/__init__.py

@@ -0,0 +1,140 @@
+from __future__ import absolute_import, division, print_function
+
+
+__metaclass__ = type
+
+from pprint import pprint
+import json
+from ansible.utils.display import Display
+from ansible.plugins.action import ActionBase
+from ansible.module_utils.six import raise_from
+from ansible.errors import AnsibleError
+from ansible.module_utils.common.text.converters import to_native
+from ..plugin_utils.tools import load_yaml_file, process_deepdiff
+from ..plugin_utils.pydantic_schemas.dcnm_network.schemas import DcnmNetworkQuerySchema
+
+try:
+    from deepdiff import DeepDiff
+except ImportError as imp_exc:
+    DEEPDIFF_IMPORT_ERROR = imp_exc
+else:
+    DEEPDIFF_IMPORT_ERROR = None
+
+if DEEPDIFF_IMPORT_ERROR:
+    raise_from(
+        AnsibleError('DeepDiff must be installed to use this plugin. Use pip or install test-requirements.'),
+        DEEPDIFF_IMPORT_ERROR)
+
+display = Display()
+
+
+class ActionModule(ActionBase):
+
+    def verify_deleted(self, results, check_deleted, expected_data, ndfc_data, config_path):
+        if not check_deleted:
+            return None
+        existing_networks = set()
+        for network in ndfc_data["response"]:
+            existing_networks.add(network["parent"]["networkName"])
+
+        if config_path == "":
+            # check for full delete
+            if not ndfc_data["failed"] and len(existing_networks) == 0:
+                results['msg'] = 'All networks are deleted'
+            else:
+                print("Networks still existing: ")
+                print(existing_networks)
+                results['failed'] = True
+                results['msg'] = 'Error: Expected full delete as config_path is empty but networks still exist.'
+                if ndfc_data["failed"]:
+                    results['msg'] += '\n\nError: ' + ndfc_data["error"]
+                return results
+            return results
+        # checks for a partial delete
+        deleted_networks = set()
+        for network in expected_data["response"]:
+            deleted_networks.add(network["parent"]["networkName"])
+
+        remaining_networks = existing_networks.intersection(deleted_networks)
+        if len(remaining_networks) > 0:
+            results['failed'] = True
+            print("Expected networks to be deleted: ")
+            print(deleted_networks)
+            print("\nNetworks present in NDFC: ")
+            print(existing_networks)
+            print("\nNetworks still not deleted: ")
+            print(remaining_networks)
+            results['msg'] = 'All networks are not deleted'
+            return results
+
+        print("Expected networks to be deleted: ")
+        print(deleted_networks)
+        print("\n\nNetworks present in NDFC: ")
+        print(existing_networks)
+        print("Networks still not deleted: ")
+        print(remaining_networks)
+        results['failed'] = False
+        results['msg'] = 'Provided networks are deleted'
+        return results
+
+    def run(self, tmp=None, task_vars=None):
+        results = super(ActionModule, self).run(tmp, task_vars)
+        results['failed'] = False
+
+        ndfc_data = self._task.args.get('ndfc_data', None)
+        test_data = self._task.args.get('test_data', None)
+        config_path = self._task.args.get('config_path', None)
+        check_deleted = self._task.args.get('check_deleted', False)
+        ignore_fields = list(self._task.args.get('ignore_fields', []))
+        for input_item in [ndfc_data, test_data, config_path]:
+            if input_item is None:
+                results['failed'] = True
+                results['msg'] = f"Required input parameter not found: '{input_item}'"
+                return results
+
+        # removes ansible embeddings and converts to native python types
+        native_ndfc_data = json.loads(json.dumps(ndfc_data, default=to_native))
+
+        test_fabric = test_data['fabric']
+
+        expected_data_parsed = None
+        if config_path != "":
+            # only parse if config file exists
+            expected_config_data = load_yaml_file(config_path)
+            expected_data = DcnmNetworkQuerySchema.yaml_config_to_dict(expected_config_data, test_fabric)
+
+            expected_data_parsed = DcnmNetworkQuerySchema.model_validate(expected_data).model_dump(exclude_none=True)
+
+        ndfc_data_parsed = DcnmNetworkQuerySchema.model_validate(native_ndfc_data).model_dump(exclude_none=True)
+
+        if deleted_results := self.verify_deleted(results, check_deleted, expected_data_parsed, ndfc_data_parsed, config_path):
+            return deleted_results
+
+        validity = DeepDiff(
+            expected_data_parsed,
+            ndfc_data_parsed,
+            ignore_order=True,
+            cutoff_distance_for_pairs=0,
+            cutoff_intersection_for_pairs=0,
+            report_repetition=True
+        )
+        # Process the output of deepdiff to make it easier to read
+        # Effects the iterable_item_added and iterable_item_removed to remove unneeded fields
+        # ignore_extra_fields=True will ignore dictionary_item_added changes
+        # This is useful when the the actual data has more fields than the expected data
+        # keys_to_ignore is a list of fields to ignore, useful for auto provisioned fields which are not known
+        processed_validity = process_deepdiff(validity, keys_to_ignore=ignore_fields, ignore_extra_fields=True)
+        if processed_validity == {}:
+            results['failed'] = False
+            results['msg'] = f'Data is valid. \n\n Expected data: \n\n{expected_data}\n\nActual data: \n\n{ndfc_data_parsed}'
+        else:
+            results['failed'] = True
+            print("\n\nExpected: ")
+            pprint(expected_data_parsed)
+            print("\n\nActual: ")
+            pprint(ndfc_data_parsed)
+            print("\n\nDifferences: ")
+            pprint(processed_validity)
+            results['msg'] = 'Data is not valid.'
+
+        return results