Devel ios vrf submode (#41659)

* add submode features ipv4/ipv6

* add tests for submode

* the parameters both_ * were not idempotent, in what is read in the configuration we have a list with 2 entries (import, export) while the input parameter has only one parameter which will be applied twice

* add docstring to the ios_vrf module + provide a fix for: https://github.com/ansible/ansible/issues/41581

* complete tests

* add missing description

* fix KeyError for address-family ipv*

* fix both_* tests

* fix E231, W292, W293

* fix W293

* remove set has it doesn't preserve order of routes

* fix E106

* remove ImportError: cannot import name OrderedDict

* We should be able to mix the parameters for the routes targets , while remaining imdepotent. During the first implementation we did not take this into account, which did not correspond to the reality of the needs in production (to be able to use each parameter indifemently together)

* remove epdb reference

* FIX E111, E106

* FIX E302

* using loop produce a result who  was not imdepotent

* FIX E241

* fix: used pass intead of list

lib/ansible/modules/network/ios/ios_vrf.py

@@ -15,7 +15,6 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
-
 ANSIBLE_METADATA = {'metadata_version': '1.1',
                     'status': ['preview'],
                     'supported_by': 'network'}
@@ -103,6 +102,30 @@ options:
     description:
       - Adds an import list of extended route target communities to the VRF.
     version_added: "2.5"
+  route_both_ipv4:
+    description:
+      - Adds an export and import list of extended route target communities in address-family configuration submode to the VRF.
+    version_added: "2.7"
+  route_export_ipv4:
+    description:
+      - Adds an export list of extended route target communities in address-family configuration submode to the VRF.
+    version_added: "2.7"
+  route_import_ipv4:
+    description:
+      - Adds an import list of extended route target communities in address-family configuration submode to the VRF.
+    version_added: "2.7"
+  route_both_ipv6:
+    description:
+      - Adds an export and import list of extended route target communities in address-family configuration submode to the VRF.
+    version_added: "2.7"
+  route_export_ipv6:
+    description:
+      - Adds an export list of extended route target communities in address-family configuration submode to the VRF.
+    version_added: "2.7"
+  route_import_ipv6:
+    description:
+      - Adds an import list of extended route target communities in address-family configuration submode to the VRF.
+    version_added: "2.7"
 
 """
 EXAMPLES = """
@@ -134,6 +157,22 @@ EXAMPLES = """
       - 1:100
       - 3:100
 
+- name: Creates a list of import RTs in address-family configuration submode for the VRF with the same parameters
+  ios_vrf:
+    name: test_import_ipv4
+    rd: 1:100
+    route_import_ipv4:
+      - 1:100
+      - 3:100
+
+- name: Creates a list of import RTs in address-family configuration submode for the VRF with the same parameters
+  ios_vrf:
+    name: test_import_ipv6
+    rd: 1:100
+    route_import_ipv6:
+      - 1:100
+      - 3:100
+
 - name: Creates a list of export RTs for the VRF with the same parameters
   ios_vrf:
     name: test_export
@@ -142,6 +181,22 @@ EXAMPLES = """
       - 1:100
       - 3:100
 
+- name: Creates a list of export RTs in address-family configuration submode for the VRF with the same parameters
+  ios_vrf:
+    name: test_export_ipv4
+    rd: 1:100
+    route_export_ipv4:
+      - 1:100
+      - 3:100
+
+- name: Creates a list of export RTs in address-family configuration submode for the VRF with the same parameters
+  ios_vrf:
+    name: test_export_ipv6
+    rd: 1:100
+    route_export_ipv6:
+      - 1:100
+      - 3:100
+
 - name: Creates a list of import and export route targets for the VRF with the same parameters
   ios_vrf:
     name: test_both
@@ -149,6 +204,23 @@ EXAMPLES = """
     route_both:
       - 1:100
       - 3:100
+
+- name: Creates a list of import and export route targets in address-family configuration submode for the VRF with the same parameters
+  ios_vrf:
+    name: test_both_ipv4
+    rd: 1:100
+    route_both_ipv4:
+      - 1:100
+      - 3:100
+
+- name: Creates a list of import and export route targets in address-family configuration submode for the VRF with the same parameters
+  ios_vrf:
+    name: test_both_ipv6
+    rd: 1:100
+    route_both_ipv6:
+      - 1:100
+      - 3:100
+
 """
 
 RETURN = """
@@ -220,12 +292,13 @@ def add_command_to_vrf(name, cmd, commands):
 
 def map_obj_to_commands(updates, module):
     commands = list()
-    state = module.params['state']  # FIXME NOT USED
 
     for update in updates:
         want, have = update
 
         def needs_update(want, have, x):
+            if isinstance(want.get(x), list) and isinstance(have.get(x), list):
+                return want.get(x) and (want.get(x) != have.get(x)) and not all(elem in have.get(x) for elem in want.get(x))
             return want.get(x) and (want.get(x) != have.get(x))
 
         if want['state'] == 'absent':
@@ -257,9 +330,40 @@ def map_obj_to_commands(updates, module):
                 cmd = 'route-target export %s' % route
                 add_command_to_vrf(want['name'], cmd, commands)
 
-        if needs_update(want, have, 'route_both'):
+        if needs_update(want, have, 'route_import_ipv4'):
-            for route in want['route_both']:
+                cmd = 'address-family ipv4'
-                cmd = 'route-target both %s' % route
+                add_command_to_vrf(want['name'], cmd, commands)
+                for route in want['route_import_ipv4']:
+                    cmd = 'route-target import %s' % route
+                    add_command_to_vrf(want['name'], cmd, commands)
+                cmd = 'exit-address-family'
+                add_command_to_vrf(want['name'], cmd, commands)
+
+        if needs_update(want, have, 'route_export_ipv4'):
+                cmd = 'address-family ipv4'
+                add_command_to_vrf(want['name'], cmd, commands)
+                for route in want['route_export_ipv4']:
+                    cmd = 'route-target export %s' % route
+                    add_command_to_vrf(want['name'], cmd, commands)
+                cmd = 'exit-address-family'
+                add_command_to_vrf(want['name'], cmd, commands)
+
+        if needs_update(want, have, 'route_import_ipv6'):
+                cmd = 'address-family ipv6'
+                add_command_to_vrf(want['name'], cmd, commands)
+                for route in want['route_import_ipv6']:
+                    cmd = 'route-target import %s' % route
+                    add_command_to_vrf(want['name'], cmd, commands)
+                cmd = 'exit-address-family'
+                add_command_to_vrf(want['name'], cmd, commands)
+
+        if needs_update(want, have, 'route_export_ipv6'):
+                cmd = 'address-family ipv6'
+                add_command_to_vrf(want['name'], cmd, commands)
+                for route in want['route_export_ipv6']:
+                    cmd = 'route-target export %s' % route
+                    add_command_to_vrf(want['name'], cmd, commands)
+                cmd = 'exit-address-family'
                 add_command_to_vrf(want['name'], cmd, commands)
 
         if want['interfaces'] is not None:
@@ -324,15 +428,77 @@ def parse_export(configobj, name):
     return matches
 
 
-def parse_both(configobj, name):
+def parse_both(configobj, name, address_family='global'):
+    rd_pattern = re.compile('(?P<rd>.+:.+)')
     matches = list()
+    export_match = None
+    import_match = None
+    if address_family == "global":
         export_match = parse_export(configobj, name)
         import_match = parse_import(configobj, name)
-    matches.extend(export_match)
+    elif address_family == "ipv4":
-    matches.extend(import_match)
+        export_match = parse_export_ipv4(configobj, name)
+        import_match = parse_import_ipv4(configobj, name)
+    elif address_family == "ipv6":
+        export_match = parse_export_ipv6(configobj, name)
+        import_match = parse_import_ipv6(configobj, name)
+    if import_match and export_match:
+        for ex in export_match:
+            exrd = rd_pattern.search(ex)
+            exrd = exrd.groupdict().get('rd')
+            for im in import_match:
+                imrd = rd_pattern.search(im)
+                imrd = imrd.groupdict().get('rd')
+                if exrd == imrd:
+                    matches.extend([exrd]) if exrd not in matches else None
+                    matches.extend([imrd]) if imrd not in matches else None
     return matches
 
 
+def parse_import_ipv4(configobj, name):
+    cfg = configobj['vrf definition %s' % name]
+    try:
+        subcfg = cfg['address-family ipv4']
+        subcfg = '\n'.join(subcfg.children)
+        matches = re.findall(r'route-target\s+import\s+(.+)', subcfg, re.M)
+        return matches
+    except KeyError:
+        pass
+
+
+def parse_export_ipv4(configobj, name):
+    cfg = configobj['vrf definition %s' % name]
+    try:
+        subcfg = cfg['address-family ipv4']
+        subcfg = '\n'.join(subcfg.children)
+        matches = re.findall(r'route-target\s+export\s+(.+)', subcfg, re.M)
+        return matches
+    except KeyError:
+        pass
+
+
+def parse_import_ipv6(configobj, name):
+    cfg = configobj['vrf definition %s' % name]
+    try:
+        subcfg = cfg['address-family ipv6']
+        subcfg = '\n'.join(subcfg.children)
+        matches = re.findall(r'route-target\s+import\s+(.+)', subcfg, re.M)
+        return matches
+    except KeyError:
+        pass
+
+
+def parse_export_ipv6(configobj, name):
+    cfg = configobj['vrf definition %s' % name]
+    try:
+        subcfg = cfg['address-family ipv6']
+        subcfg = '\n'.join(subcfg.children)
+        matches = re.findall(r'route-target\s+export\s+(.+)', subcfg, re.M)
+        return matches
+    except KeyError:
+        pass
+
+
 def map_config_to_obj(module):
     config = get_config(module)
     configobj = NetworkConfig(indent=1, contents=config)
@@ -341,7 +507,6 @@ def map_config_to_obj(module):
         return list()
 
     instances = list()
-
     for item in set(match):
         obj = {
             'name': item,
@@ -351,7 +516,13 @@ def map_config_to_obj(module):
             'interfaces': parse_interfaces(configobj, item),
             'route_import': parse_import(configobj, item),
             'route_export': parse_export(configobj, item),
-            'route_both': parse_both(configobj, item)
+            'route_both': parse_both(configobj, item),
+            'route_import_ipv4': parse_import_ipv4(configobj, item),
+            'route_export_ipv4': parse_export_ipv4(configobj, item),
+            'route_both_ipv4': parse_both(configobj, item, address_family='ipv4'),
+            'route_import_ipv6': parse_import_ipv6(configobj, item),
+            'route_export_ipv6': parse_export_ipv6(configobj, item),
+            'route_both_ipv6': parse_both(configobj, item, address_family='ipv6'),
         }
         instances.append(obj)
     return instances
@@ -396,7 +567,6 @@ def map_params_to_obj(module):
                 collection.append(item)
 
     objects = list()
-
     for item in collection:
         get_value = partial(get_param_value, item=item, module=module)
         item['description'] = get_value('description')
@@ -406,6 +576,21 @@ def map_params_to_obj(module):
         item['route_import'] = get_value('route_import')
         item['route_export'] = get_value('route_export')
         item['route_both'] = get_value('route_both')
+        item['route_import_ipv4'] = get_value('route_import_ipv4')
+        item['route_export_ipv4'] = get_value('route_export_ipv4')
+        item['route_both_ipv4'] = get_value('route_both_ipv4')
+        item['route_import_ipv6'] = get_value('route_import_ipv6')
+        item['route_export_ipv6'] = get_value('route_export_ipv6')
+        item['route_both_ipv6'] = get_value('route_both_ipv6')
+        both_addresses_family = ["", "_ipv6", "_ipv4"]
+        for address_family in both_addresses_family:
+            if item["route_both%s" % address_family]:
+                if not item["route_export%s" % address_family]:
+                    item["route_export%s" % address_family] = list()
+                if not item["route_import%s" % address_family]:
+                    item["route_import%s" % address_family] = list()
+                item["route_export%s" % address_family].extend(get_value("route_both%s" % address_family))
+                item["route_import%s" % address_family].extend(get_value("route_both%s" % address_family))
         item['associated_interfaces'] = get_value('associated_interfaces')
         objects.append(item)
 
@@ -421,6 +606,7 @@ def update_objects(want, have):
         else:
             for key, value in iteritems(entry):
                 if value:
+                    try:
                         if isinstance(value, list):
                             if sorted(value) != sorted(item[key]):
                                 if (entry, item) not in updates:
@@ -428,6 +614,8 @@ def update_objects(want, have):
                         elif value != item[key]:
                             if (entry, item) not in updates:
                                 updates.append((entry, item))
+                    except TypeError:
+                        pass
     return updates
 
 
@@ -469,6 +657,13 @@ def main():
         route_export=dict(type='list'),
         route_import=dict(type='list'),
         route_both=dict(type='list'),
+        route_export_ipv4=dict(type='list'),
+        route_import_ipv4=dict(type='list'),
+        route_both_ipv4=dict(type='list'),
+        route_export_ipv6=dict(type='list'),
+        route_import_ipv6=dict(type='list'),
+        route_both_ipv6=dict(type='list'),
+
 
         interfaces=dict(type='list'),
         associated_interfaces=dict(type='list'),
@@ -480,7 +675,7 @@ def main():
 
     argument_spec.update(ios_argument_spec)
 
-    mutually_exclusive = [('name', 'vrfs'), ('route_import', 'route_both'), ('route_export', 'route_both')]
+    mutually_exclusive = [('name', 'vrfs')]
     module = AnsibleModule(argument_spec=argument_spec,
                            mutually_exclusive=mutually_exclusive,
                            supports_check_mode=True)
@@ -493,7 +688,6 @@ def main():
 
     want = map_params_to_obj(module)
     have = map_config_to_obj(module)
-
     commands = map_obj_to_commands(update_objects(want, have), module)
 
     if module.params['purge']:

test/units/modules/network/ios/fixtures/ios_vrf_config.cfg

@@ -8,6 +8,70 @@ vrf definition test_2
 !
 vrf definition test_3
 !
+vrf definition test_17
+ rd 2:100
+ !
+ address-family ipv4
+ exit-address-family
+ !
+ address-family ipv6
+  route-target export 168.0.0.15:100
+  route-target export 4:100
+  route-target export 2:100
+  route-target export 168.0.0.13:100
+  route-target import 168.0.0.14:100
+  route-target import 2:100
+  route-target import 168.0.0.13:100
+ exit-address-family
+!
+vrf definition test_18
+ rd 168.0.0.9:100
+ !
+ address-family ipv4
+  route-target export 168.0.0.10:100
+  route-target export 168.0.0.9:100
+  route-target export 3:100
+  route-target import 168.0.0.9:100
+  route-target import 3:100
+  route-target import 168.0.0.10:600
+ exit-address-family
+ !
+ address-family ipv6
+ exit-address-family
+!
+vrf definition test_19
+ rd 10:700
+ route-target export 2:102
+ route-target export 2:103
+ route-target export 2:100
+ route-target export 2:101
+ route-target import 2:104
+ route-target import 2:105
+ route-target import 2:100
+ route-target import 2:101
+ !
+ address-family ipv4
+  route-target export 2:102
+  route-target export 2:103
+  route-target export 2:100
+  route-target export 2:101
+  route-target import 2:104
+  route-target import 2:105
+  route-target import 2:100
+  route-target import 2:101
+ exit-address-family
+ !
+ address-family ipv6
+  route-target export 2:102
+  route-target export 2:103
+  route-target export 2:100
+  route-target export 2:101
+  route-target import 2:104
+  route-target import 2:105
+  route-target import 2:100
+  route-target import 2:101
+ exit-address-family
+!
 interface Ethernet1
  ip address 1.2.3.4/5
 !

test/units/modules/network/ios/test_ios_vrf.py

@@ -18,6 +18,7 @@
 
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
+
 __metaclass__ = type
 
 from ansible.compat.tests.mock import patch
@@ -83,12 +84,14 @@ class TestIosVrfModule(TestIosModule):
 
     def test_ios_vrf_purge_all(self):
         set_module_args(dict(purge=True))
-        commands = ['no vrf definition test_1', 'no vrf definition test_2', 'no vrf definition test_3']
+        commands = ['no vrf definition test_1', 'no vrf definition test_2', 'no vrf definition test_3', 'no vrf definition test_17',
+                    'no vrf definition test_18', 'no vrf definition test_19']
         self.execute_module(changed=True, commands=commands)
 
     def test_ios_vrf_purge_all_but_one(self):
         set_module_args(dict(name='test_1', purge=True))
-        commands = ['no vrf definition test_2', 'no vrf definition test_3']
+        commands = ['no vrf definition test_2', 'no vrf definition test_3', 'no vrf definition test_17', 'no vrf definition test_18',
+                    'no vrf definition test_19']
         self.execute_module(changed=True, commands=commands)
 
     def test_ios_vrfs_no_purge(self):
@@ -101,7 +104,7 @@ class TestIosVrfModule(TestIosModule):
         vrfs = [{'name': 'test_1'}, {'name': 'test_4'}]
         set_module_args(dict(vrfs=vrfs, purge=True))
         commands = ['vrf definition test_4', 'address-family ipv4', 'exit', 'address-family ipv6', 'exit', 'no vrf definition test_2',
-                    'no vrf definition test_3']
+                    'no vrf definition test_3', 'no vrf definition test_17', 'no vrf definition test_18', 'no vrf definition test_19']
         self.execute_module(changed=True, commands=commands)
 
     def test_ios_vrfs_global_arg(self):
@@ -126,8 +129,8 @@ class TestIosVrfModule(TestIosModule):
 
     def test_ios_vrf_route_both(self):
         set_module_args(dict(name='test_5', rd='2:100', route_both=['2:100', '3:100']))
-        commands = ['vrf definition test_5', 'address-family ipv4', 'exit', 'address-family ipv6', 'exit', 'rd 2:100', 'route-target both 2:100',
+        commands = ['vrf definition test_5', 'address-family ipv4', 'exit', 'address-family ipv6', 'exit', 'rd 2:100', 'route-target import 2:100',
-                    'route-target both 3:100']
+                    'route-target import 3:100', 'route-target export 2:100', 'route-target export 3:100']
         self.execute_module(changed=True, commands=commands, sort=False)
 
     def test_ios_vrf_route_import(self):
@@ -142,6 +145,68 @@ class TestIosVrfModule(TestIosModule):
                     'route-target export 4:100']
         self.execute_module(changed=True, commands=commands, sort=False)
 
-    def test_ios_vrf_route_both_exclusive(self):
+    def test_ios_vrf_route_both_mixed(self):
         set_module_args(dict(name='test_8', rd='5:100', route_both=['3:100', '4:100'], route_export=['3:100', '4:100']))
-        self.execute_module(failed=True)
+        self.execute_module(changed=True)
+
+    def test_ios_vrf_route_both_ipv4(self):
+        set_module_args(dict(name='test_9', rd='168.0.0.9:100', route_both_ipv4=['168.0.0.9:100', '3:100']))
+        commands = ['vrf definition test_9', 'address-family ipv4', 'exit', 'address-family ipv6', 'exit', 'rd 168.0.0.9:100', 'address-family ipv4',
+                    'route-target import 168.0.0.9:100', 'route-target import 3:100', 'exit-address-family', 'address-family ipv4',
+                    'route-target export 168.0.0.9:100', 'route-target export 3:100', 'exit-address-family']
+        self.execute_module(changed=True, commands=commands, sort=False)
+
+    def test_ios_vrf_route_import_ipv4(self):
+        set_module_args(dict(name='test_10', rd='168.0.0.10:100', route_import_ipv4=['168.0.0.10:100', '3:100']))
+        commands = ['vrf definition test_10', 'address-family ipv4', 'exit', 'address-family ipv6', 'exit', 'rd 168.0.0.10:100', 'address-family ipv4',
+                    'route-target import 168.0.0.10:100', 'route-target import 3:100', 'exit-address-family']
+        self.execute_module(changed=True, commands=commands, sort=False)
+
+    def test_ios_vrf_route_export_ipv4(self):
+        set_module_args(dict(name='test_11', rd='168.0.0.11:100', route_export_ipv4=['168.0.0.11:100', '3:100']))
+        commands = ['vrf definition test_11', 'address-family ipv4', 'exit', 'address-family ipv6', 'exit', 'rd 168.0.0.11:100', 'address-family ipv4',
+                    'route-target export 168.0.0.11:100', 'route-target export 3:100', 'exit-address-family']
+        self.execute_module(changed=True, commands=commands, sort=False)
+
+    def test_ios_vrf_route_both_ipv4_mixed(self):
+        set_module_args(dict(name='test_12', rd='168.0.0.12:100', route_both_ipv4=['168.0.0.12:100', '3:100'], route_export_ipv4=['168.0.0.15:100', '6:100']))
+        self.execute_module(changed=True)
+
+    def test_ios_vrf_route_both_ipv6(self):
+        set_module_args(dict(name='test_13', rd='2:100', route_both_ipv6=['2:100', '168.0.0.13:100']))
+        commands = ['vrf definition test_13', 'address-family ipv4', 'exit', 'address-family ipv6', 'exit', 'rd 2:100', 'address-family ipv6',
+                    'route-target import 2:100', 'route-target import 168.0.0.13:100', 'exit-address-family', 'address-family ipv6',
+                    'route-target export 2:100', 'route-target export 168.0.0.13:100', 'exit-address-family']
+        self.execute_module(changed=True, commands=commands, sort=False)
+
+    def test_ios_vrf_route_import_ipv6(self):
+        set_module_args(dict(name='test_14', rd='3:100', route_import_ipv6=['3:100', '168.0.0.14:100']))
+        commands = ['vrf definition test_14', 'address-family ipv4', 'exit', 'address-family ipv6', 'exit', 'rd 3:100', 'address-family ipv6',
+                    'route-target import 3:100', 'route-target import 168.0.0.14:100', 'exit-address-family']
+        self.execute_module(changed=True, commands=commands, sort=False)
+
+    def test_ios_vrf_route_export_ipv6(self):
+        set_module_args(dict(name='test_15', rd='4:100', route_export_ipv6=['168.0.0.15:100', '4:100']))
+        commands = ['vrf definition test_15', 'address-family ipv4', 'exit', 'address-family ipv6', 'exit', 'rd 4:100', 'address-family ipv6',
+                    'route-target export 168.0.0.15:100', 'route-target export 4:100', 'exit-address-family']
+        self.execute_module(changed=True, commands=commands, sort=False)
+
+    def test_ios_vrf_route_both_ipv6_mixed(self):
+        set_module_args(dict(name='test_16', rd='5:100', route_both_ipv6=['168.0.0.9:100', '4:100'], route_export_ipv6=['168.0.0.12:100', '6:100']))
+        self.execute_module(changed=True)
+
+    def test_ios_vrf_route_both_ipv6_mixed_idempotent(self):
+        set_module_args(dict(name='test_17', rd='2:100', route_import_ipv6=['168.0.0.14:100'], route_both_ipv6=['2:100', '168.0.0.13:100'],
+                             route_export_ipv6=['168.0.0.15:100', '4:100']))
+        self.execute_module(changed=False, commands=[], sort=False)
+
+    def test_ios_vrf_route_both_ipv4_mixed_idempotent(self):
+        set_module_args(dict(name='test_18', rd='168.0.0.9:100', route_import_ipv4=['168.0.0.10:600'], route_export_ipv4=['168.0.0.10:100'],
+                             route_both_ipv4=['168.0.0.9:100', '3:100']))
+        self.execute_module(changed=False, commands=[], sort=False)
+
+    def test_ios_vrf_all_route_both_idempotent(self):
+        set_module_args(dict(name='test_19', rd='10:700', route_both=['2:100', '2:101'], route_export=['2:102', '2:103'], route_import=['2:104', '2:105'],
+                             route_both_ipv4=['2:100', '2:101'], route_export_ipv4=['2:102', '2:103'], route_import_ipv4=['2:104', '2:105'],
+                             route_both_ipv6=['2:100', '2:101'], route_export_ipv6=['2:102', '2:103'], route_import_ipv6=['2:104', '2:105']))
+        self.execute_module(changed=False, commands=[], sort=False)