refactored merged and replaced. Added ignore_fields to process_deepdiff. moved templates into separate folders

Author: neiljohn-gh

plugins/action/tests/integration/ndfc_network_validate.py

@@ -85,7 +85,7 @@ def run(self, tmp=None, task_vars=None):
         test_data = self._task.args.get('test_data', None)
         config_path = self._task.args.get('config_path', None)
         check_deleted = self._task.args.get('check_deleted', False)
-
+        ignore_fields = list(self._task.args.get('ignore_fields', []))
         for input_item in [ndfc_data, test_data, config_path]:
             if input_item is None:
                 results['failed'] = True
@@ -108,7 +108,6 @@ def run(self, tmp=None, task_vars=None):
         if deleted_results := self.verify_deleted(results, check_deleted, expected_data_parsed, ndfc_data_parsed, config_path):
             return deleted_results
 
-        # normal checking when check_delete is false
         validity = DeepDiff(
             expected_data_parsed,
             ndfc_data_parsed,
@@ -121,7 +120,8 @@ def run(self, tmp=None, task_vars=None):
         # Effects the iterable_item_added and iterable_item_removed to remove unneeded fields
         # ignore_extra_fields=True will ignore dictionary_item_added changes
         # This is useful when the the actual data has more fields than the expected data
-        processed_validity = process_deepdiff(validity, ignore_extra_fields=True)
+        # keys_to_ignore is a list of fields to ignore, useful for auto provisioned fields which are not known
+        processed_validity = process_deepdiff(validity, keys_to_ignore=ignore_fields, ignore_extra_fields=True)
         if processed_validity == {}:
             results['failed'] = False
             results['msg'] = f'Data is valid. \n\n Expected data: \n\n{expected_data}\n\nActual data: \n\n{ndfc_data_parsed}'

plugins/action/tests/plugin_utils/pydantic_schemas/dcnm_network/schemas.py

@@ -70,6 +70,7 @@ class NetworkTemplateConfig(BaseModel):
         mtu: Optional[CoercedStr] = ""
         vlanName: Optional[str] = None
         intfDescription: Optional[str] = None
+        tag: Optional[CoercedStr] = None
 
     class Parent(BaseModel):
         fabric: Optional[str] = None
@@ -158,6 +159,8 @@ def yaml_config_to_dict(cls, expected_config_data, test_fabric):
                     network_dict["parent"]["networkTemplateConfig"]["vlanName"] = network['vlan_name']
                 if 'int_desc' in network:
                     network_dict["parent"]["networkTemplateConfig"]["intfDescription"] = network['int_desc']
+                if 'routing_tag' in network:
+                    network_dict["parent"]["networkTemplateConfig"]["tag"] = network['routing_tag']
             if 'vrf_name' in network:
                 network_dict["parent"]["vrf"] = network['vrf_name']
             # if deploy:

plugins/action/tests/plugin_utils/tools.py

@@ -18,7 +18,7 @@ def load_yaml_file(file_path):
             raise ValueError(f"Error parsing YAML file '{file_path}': {e}") from e
 
 
-def process_deepdiff(deepdiff_output, ignore_extra_fields=False):
+def process_deepdiff(deepdiff_output, keys_to_ignore, ignore_extra_fields=False):
     """
     Process deepdiff output to extract paths ignoring indices and find differences.
     Returns a dictionary with the same structure as deepdiff output but with
@@ -135,6 +135,12 @@ def extract_values(data, current_path="", values=None):
         for path in paths_to_remove:
             del diff_paths['added_values'][path]
 
+        diff_paths_keys = list(diff_paths['added_values'].keys())
+        for path in diff_paths_keys:
+            for field in keys_to_ignore:
+                if field in path:
+                    del diff_paths['added_values'][path]
+
         # Remove added_values if it becomes empty after filtering
         if not diff_paths['added_values']:
             del diff_paths['added_values']

tests/integration/targets/dcnm_network/templates/merged/dcnm_network_merged_double_net_auto_vlan_conf.j2

@@ -0,0 +1,34 @@
+---
+# This NDFC test data structure is auto-generated
+# DO NOT EDIT MANUALLY
+#
+
+
+# ------------------------------
+# Network 1
+# ------------------------------
+- net_name: "{{ test_data_common.net1 }}"
+  vrf_name: "{{ test_data_common.net1_vrf }}"
+  net_id: "{{ test_data_common.net1_net_id }}"
+  net_template: "{{ test_data_common.net1_default_net_template }}"
+  net_extension_template: "{{ test_data_common.net1_net_extension_template }}"
+  gw_ip_subnet: "{{ test_data_common.net1_gw_ip_subnet }}"
+  attach:
+  - ip_address: "{{ test_data_common.sw1 }}"
+    ports: ["{{ test_data_common.sw1_int1 }}", "{{ test_data_common.sw1_int2 }}"]
+  deploy: {{ test_data_common.deploy | bool }}
+
+# ------------------------------
+# Network 2
+# ------------------------------
+- net_name: "{{ test_data_common.net2 }}"
+  vrf_name: "{{ test_data_common.net2_vrf }}"
+  net_id: "{{ test_data_common.net2_net_id }}"
+  net_template: "{{ test_data_common.net2_default_net_template }}"
+  net_extension_template: "{{ test_data_common.net2_net_extension_template }}"
+  vlan_id: "{{ test_data_common.net2_vlan_id }}"
+  gw_ip_subnet: "{{ test_data_common.net2_gw_ip_subnet }}"
+  attach:
+  - ip_address: "{{ test_data_common.sw2 }}"
+    ports: ["{{ test_data_common.sw2_int3 }}", "{{ test_data_common.sw2_int4 }}"]
+  deploy: {{ test_data_common.deploy | bool }}

tests/integration/targets/dcnm_network/templates/merged/dcnm_network_merged_double_net_conf.j2

@@ -0,0 +1,35 @@
+---
+# This NDFC test data structure is auto-generated
+# DO NOT EDIT MANUALLY
+#
+
+
+# ------------------------------
+# Network 1
+# ------------------------------
+- net_name: "{{ test_data_common.net1 }}"
+  vrf_name: "{{ test_data_common.net1_vrf }}"
+  net_id: "{{ test_data_common.net1_net_id }}"
+  net_template: "{{ test_data_common.net1_default_net_template }}"
+  net_extension_template: "{{ test_data_common.net1_net_extension_template }}"
+  vlan_id: "{{ test_data_common.net1_vlan_id }}"
+  gw_ip_subnet: "{{ test_data_common.net1_gw_ip_subnet }}"
+  attach:
+  - ip_address: "{{ test_data_common.sw1 }}"
+    ports: ["{{ test_data_common.sw1_int1 }}", "{{ test_data_common.sw1_int2 }}"]
+  deploy: {{ test_data_common.deploy | bool }}
+
+# ------------------------------
+# Network 2
+# ------------------------------
+- net_name: "{{ test_data_common.net2 }}"
+  vrf_name: "{{ test_data_common.net2_vrf }}"
+  net_id: "{{ test_data_common.net2_net_id }}"
+  net_template: "{{ test_data_common.net2_default_net_template }}"
+  net_extension_template: "{{ test_data_common.net2_net_extension_template }}"
+  vlan_id: "{{ test_data_common.net2_vlan_id }}"
+  gw_ip_subnet: "{{ test_data_common.net2_gw_ip_subnet }}"
+  attach:
+  - ip_address: "{{ test_data_common.sw2 }}"
+    ports: ["{{ test_data_common.sw2_int3 }}", "{{ test_data_common.sw2_int4 }}"]
+  deploy: {{ test_data_common.deploy | bool }}

tests/integration/targets/dcnm_network/templates/merged/dcnm_network_merged_l2_l3_net_conf.j2

@@ -0,0 +1,53 @@
+# ------------------------------
+# Network 1
+# ------------------------------
+- net_name: "{{ test_data_common.net1 }}"
+  net_id: "{{ test_data_common.net1_net_id }}"
+  net_template: "{{ test_data_common.net1_default_net_template }}"
+  net_extension_template: "{{ test_data_common.net1_net_extension_template }}"
+  vlan_id: "{{ test_data_common.net1_vlan_id }}"
+  gw_ip_subnet: "{{ test_data_common.net1_gw_ip_subnet }}"
+  mtu_l3intf: 7600
+  arp_suppress: False
+  # Leave 'arp_suppress' this false to avoid TCAM issue
+  # Please configure TCAM region for Ingress ARP-Ether ACL before configuring ARP supression
+  int_desc: 'test interface'
+  is_l2only: True
+  vlan_name: testvlan
+  dhcp_srvr1_ip: '1.1.1.1'
+  dhcp_srvr2_ip: '2.2.2.2'
+  dhcp_srvr3_ip: '3.3.3.3'
+  dhcp_srvr1_vrf: one
+  dhcp_srvr2_vrf: two
+  dhcp_srvr3_vrf: three
+  attach:
+  - ip_address: "{{ test_data_common.sw1 }}"
+    ports: ["{{ test_data_common.sw1_int1 }}", "{{ test_data_common.sw1_int2 }}"]
+  deploy: {{ test_data_common.deploy | bool }}
+
+# ------------------------------
+# Network 2
+# ------------------------------
+- net_name: "{{ test_data_common.net2 }}"
+  vrf_name: "{{ test_data_common.net2_vrf }}"
+  net_id: "{{ test_data_common.net2_net_id }}"
+  net_template: "{{ test_data_common.net2_default_net_template }}"
+  net_extension_template: "{{ test_data_common.net2_net_extension_template }}"
+  vlan_id: "{{ test_data_common.net2_vlan_id }}"
+  gw_ip_subnet: "{{ test_data_common.net2_gw_ip_subnet }}"
+  mtu_l3intf: 7600
+  # Leave 'arp_suppress' this false to avoid TCAM issue
+  # Please configure TCAM region for Ingress ARP-Ether ACL before configuring ARP supression
+  arp_suppress: False
+  int_desc: 'test interface 1'
+  vlan_name: testvlan1
+  dhcp_srvr1_ip: '1.1.1.1'
+  dhcp_srvr2_ip: '2.2.2.2'
+  dhcp_srvr3_ip: '3.3.3.3'
+  dhcp_srvr1_vrf: one
+  dhcp_srvr2_vrf: two
+  dhcp_srvr3_vrf: three
+  attach:
+  - ip_address: "{{ test_data_common.sw2 }}"
+    ports: ["{{ test_data_common.sw2_int3 }}", "{{ test_data_common.sw2_int4 }}"]
+  deploy: {{ test_data_common.deploy | bool }}

tests/integration/targets/dcnm_network/templates/merged/dcnm_network_merged_l2only_net_conf.j2

@@ -0,0 +1,30 @@
+---
+# This NDFC test data structure is auto-generated
+# DO NOT EDIT MANUALLY
+#
+
+
+# ------------------------------
+# L2 only Network 1
+# ------------------------------
+- net_name: "{{ test_data_common.net1 }}"
+  net_id: "{{ test_data_common.net1_net_id }}"
+  net_template: "{{ test_data_common.net1_default_net_template }}"
+  net_extension_template: "{{ test_data_common.net1_net_extension_template }}"
+  vlan_id: "{{ test_data_common.net1_vlan_id }}"
+  gw_ip_subnet: "{{ test_data_common.net1_gw_ip_subnet }}"
+  mtu_l3intf: 7600
+  arp_suppress: True
+  int_desc: 'test interface'
+  is_l2only: True
+  vlan_name: testvlan
+  dhcp_srvr1_ip: '1.1.1.1'
+  dhcp_srvr2_ip: '2.2.2.2'
+  dhcp_srvr3_ip: '3.3.3.3'
+  dhcp_srvr1_vrf: one
+  dhcp_srvr2_vrf: two
+  dhcp_srvr3_vrf: three
+  attach:
+  - ip_address: "{{ test_data_common.sw1 }}"
+    ports: ["{{ test_data_common.sw1_int1 }}", "{{ test_data_common.sw1_int2 }}"]
+  deploy: {{ test_data_common.deploy | bool }}

tests/integration/targets/dcnm_network/templates/merged/dcnm_network_merged_l3_net_conf.j2

@@ -0,0 +1,25 @@
+---
+# This NDFC test data structure is auto-generated
+# DO NOT EDIT MANUALLY
+#
+
+
+# ------------------------------
+# Network 1
+# ------------------------------
+- net_name: "{{ test_data_common.net1 }}"
+  net_id: "{{ test_data_common.net1_net_id }}"
+  vrf_name: "{{ test_data_common.net1_vrf }}"
+  net_template: "{{ test_data_common.net1_default_net_template }}"
+  net_extension_template: "{{ test_data_common.net1_net_extension_template }}"
+  vlan_id: "{{ test_data_common.net1_vlan_id }}"
+  gw_ip_subnet: "{{ test_data_common.net1_gw_ip_subnet }}"
+  int_desc: 'test interface'
+  vlan_name: testvlan
+  dhcp_srvr1_ip: '1.1.1.1'
+  dhcp_srvr2_ip: '2.2.2.2'
+  dhcp_srvr3_ip: '3.3.3.3'
+  dhcp_srvr1_vrf: one
+  dhcp_srvr2_vrf: two
+  dhcp_srvr3_vrf: three
+  deploy: {{ test_data_common.deploy | bool }}

tests/integration/targets/dcnm_network/templates/merged/dcnm_network_merged_l3_net_final_conf.j2

@@ -0,0 +1,28 @@
+---
+# This NDFC test data structure is auto-generated
+# DO NOT EDIT MANUALLY
+#
+
+
+# ------------------------------
+# Network 1
+# ------------------------------
+- net_name: "{{ test_data_common.net1 }}"
+  net_id: "{{ test_data_common.net1_net_id }}"
+  vrf_name: "{{ test_data_common.net1_vrf }}"
+  net_template: "{{ test_data_common.net1_default_net_template }}"
+  net_extension_template: "{{ test_data_common.net1_net_extension_template }}"
+  vlan_id: "{{ test_data_common.net1_vlan_id }}"
+  gw_ip_subnet: "{{ test_data_common.net1_gw_ip_subnet }}"
+  int_desc: 'test interface'
+  vlan_name: testvlan
+  dhcp_srvr1_ip: '1.1.1.1'
+  dhcp_srvr2_ip: '2.2.2.2'
+  dhcp_srvr3_ip: '3.3.3.3'
+  dhcp_srvr1_vrf: one
+  dhcp_srvr2_vrf: two
+  dhcp_srvr3_vrf: three
+  attach:
+  - ip_address: "{{ test_data_common.sw1 }}"
+    ports: ["{{ test_data_common.sw1_int1 }}", "{{ test_data_common.sw1_int2 }}"]
+  deploy: {{ test_data_common.deploy | bool }}

tests/integration/targets/dcnm_network/templates/merged/dcnm_network_merged_l3_net_only_attach_conf.j2

@@ -0,0 +1,15 @@
+---
+# This NDFC test data structure is auto-generated
+# DO NOT EDIT MANUALLY
+#
+
+
+# ------------------------------
+# L2 only Network 1
+# ------------------------------
+- net_name: "{{ test_data_common.net1 }}"
+  vrf_name: "{{ test_data_common.net1_vrf }}"
+  attach:
+  - ip_address: "{{ test_data_common.sw1 }}"
+    ports: ["{{ test_data_common.sw1_int1 }}", "{{ test_data_common.sw1_int2 }}"]
+  deploy: {{ test_data_common.deploy | bool }}