Merge pull request #197 from napalm-automation/develop

Release 0.12.0

napalm_junos/junos.py

@@ -23,6 +23,7 @@
 import logging
 import collections
 from copy import deepcopy
+from collections import OrderedDict
 
 # import third party lib
 from lxml.builder import E
@@ -211,9 +212,9 @@ def _load_candidate(self, filename, config, overwrite):
                                 ignore_warning=self.ignore_warning)
         except ConfigLoadError as e:
             if self.config_replace:
-                raise ReplaceConfigException(e.message)
+                raise ReplaceConfigException(e.errs)
             else:
-                raise MergeConfigException(e.message)
+                raise MergeConfigException(e.errs)
 
     def load_replace_candidate(self, filename=None, config=None):
         """Open the candidate config and merge."""
@@ -236,7 +237,7 @@ def compare_config(self):
 
     def commit_config(self):
         """Commit configuration."""
-        self.device.cu.commit()
+        self.device.cu.commit(ignore_warning=self.ignore_warning)
         if not self.config_lock:
             self._unlock()
 
@@ -327,10 +328,20 @@ def get_environment(self):
         routing_engine.get()
         temperature_thresholds.get()
         environment_data = {}
+        current_class = None
 
         for sensor_object, object_data in environment.items():
             structured_object_data = {k: v for k, v in object_data}
 
+            if structured_object_data['class']:
+                # If current object has a 'class' defined, store it for use
+                # on subsequent unlabeled lines.
+                current_class = structured_object_data['class']
+            else:
+                # Juniper doesn't label the 2nd+ lines of a given class with a
+                # class name.  In that case, we use the most recent class seen.
+                structured_object_data['class'] = current_class
+
             if structured_object_data['class'] == 'Power':
                 # Create a dict for the 'power' key
                 try:
@@ -557,24 +568,39 @@ def _get_bgp_neighbors_core(neighbor_data, instance=None, uptime_table_items=Non
                         bgp_neighbor_data[instance_name]['peers'][neighbor] = {}
                     bgp_neighbor_data[instance_name]['peers'][neighbor]['uptime'] = uptime[0][1]
 
-        old_junos = napalm_base.helpers.convert(
-            int, self.device.facts.get('version', '0.0').split('.')[0], 0) < 15
-
-        if old_junos:
-            instances = junos_views.junos_route_instance_table(self.device).get()
-            for instance, instance_data in instances.items():
-                if instance.startswith('__'):
-                    # junos internal instances
-                    continue
-                bgp_neighbor_data[instance] = {'peers': {}}
-                instance_neighbors = bgp_neighbors_table.get(instance=instance).items()
-                uptime_table_items = uptime_table.get(instance=instance).items()
-                _get_bgp_neighbors_core(instance_neighbors,
-                                        instance=instance,
-                                        uptime_table_items=uptime_table_items)
-        else:
-            instance_neighbors = bgp_neighbors_table.get().items()
-            _get_bgp_neighbors_core(instance_neighbors)
+        # Commenting out the following sections, till Junos
+        #   will provide a way to identify the routing instance name
+        #   from the details of the BGP neighbor
+        #   currently, there are Junos 15 version having a field called `peer_fwd_rti`
+        #   but unfortunately, this is not consistent.
+        # Junos 17 might have this fixed, but this needs to be revisited later.
+        # In the definition below, `old_junos` means a version that does not provide
+        #   the forwarding RTI information.
+        #
+        # old_junos = napalm_base.helpers.convert(
+        #     int, self.device.facts.get('version', '0.0').split('.')[0], 0) < 15
+
+        # if old_junos:
+        instances = junos_views.junos_route_instance_table(self.device).get()
+        for instance, instance_data in instances.items():
+            if instance.startswith('__'):
+                # junos internal instances
+                continue
+            bgp_neighbor_data[instance] = {'peers': {}}
+            instance_neighbors = bgp_neighbors_table.get(instance=instance).items()
+            uptime_table_items = uptime_table.get(instance=instance).items()
+            _get_bgp_neighbors_core(instance_neighbors,
+                                    instance=instance,
+                                    uptime_table_items=uptime_table_items)
+        # If the OS provides the `peer_fwd_rti` or any way to identify the
+        #   rotuing instance name (see above), the performances of this getter
+        #   can be significantly improved, as we won't execute one request
+        #   for each an every RT.
+        # However, this improvement would only be beneficial for multi-VRF envs.
+        #
+        # else:
+        #     instance_neighbors = bgp_neighbors_table.get().items()
+        #     _get_bgp_neighbors_core(instance_neighbors)
         bgp_tmp_dict = {}
         for k, v in bgp_neighbor_data.items():
             if bgp_neighbor_data[k]['peers']:
@@ -655,13 +681,112 @@ def cli(self, commands):
         """Execute raw CLI commands and returns their output."""
         cli_output = {}
 
+        def _count(txt, none):  # Second arg for consistency only. noqa
+            '''
+            Return the exact output, as Junos displays
+            e.g.:
+            > show system processes extensive | match root | count
+            Count: 113 lines
+            '''
+            count = len(txt.splitlines())
+            return 'Count: {count} lines'.format(count=count)
+
+        def _trim(txt, length):
+            '''
+            Trim specified number of columns from start of line.
+            '''
+            try:
+                newlines = []
+                for line in txt.splitlines():
+                    newlines.append(line[int(length):])
+                return '\n'.join(newlines)
+            except ValueError:
+                return txt
+
+        def _except(txt, pattern):
+            '''
+            Show only text that does not match a pattern.
+            '''
+            rgx = '^.*({pattern}).*$'.format(pattern=pattern)
+            unmatched = [
+                line for line in txt.splitlines()
+                if not re.search(rgx, line, re.I)
+            ]
+            return '\n'.join(unmatched)
+
+        def _last(txt, length):
+            '''
+            Display end of output only.
+            '''
+            try:
+                return '\n'.join(
+                    txt.splitlines()[(-1)*int(length):]
+                )
+            except ValueError:
+                return txt
+
+        def _match(txt, pattern):
+            '''
+            Show only text that matches a pattern.
+            '''
+            rgx = '^.*({pattern}).*$'.format(pattern=pattern)
+            matched = [
+                line for line in txt.splitlines()
+                if re.search(rgx, line, re.I)
+            ]
+            return '\n'.join(matched)
+
+        def _process_pipe(cmd, txt):
+            '''
+            Process CLI output from Juniper device that
+            doesn't allow piping the output.
+            '''
+            if not txt:
+                return txt
+            _OF_MAP = OrderedDict()
+            _OF_MAP['except'] = _except
+            _OF_MAP['match'] = _match
+            _OF_MAP['last'] = _last
+            _OF_MAP['trim'] = _trim
+            _OF_MAP['count'] = _count
+            # the operations order matter in this case!
+            exploded_cmd = cmd.split('|')
+            pipe_oper_args = {}
+            for pipe in exploded_cmd[1:]:
+                exploded_pipe = pipe.split()
+                pipe_oper = exploded_pipe[0]  # always there
+                pipe_args = ''.join(exploded_pipe[1:2])
+                # will not throw error when there's no arg
+                pipe_oper_args[pipe_oper] = pipe_args
+            for oper in _OF_MAP.keys():
+                # to make sure the operation sequence is correct
+                if oper not in pipe_oper_args.keys():
+                    continue
+                txt = _OF_MAP[oper](txt, pipe_oper_args[oper])
+            return txt
+
         if not isinstance(commands, list):
             raise TypeError('Please enter a valid list of commands!')
-
+        _PIPE_BLACKLIST = ['save']
+        # Preprocessing to avoid forbidden commands
         for command in commands:
+            exploded_cmd = command.split('|')
+            command_safe_parts = []
+            for pipe in exploded_cmd[1:]:
+                exploded_pipe = pipe.split()
+                pipe_oper = exploded_pipe[0]  # always there
+                if pipe_oper in _PIPE_BLACKLIST:
+                    continue
+                pipe_args = ''.join(exploded_pipe[1:2])
+                safe_pipe = pipe_oper if not pipe_args else '{fun} {args}'.format(fun=pipe_oper,
+                                                                                  args=pipe_args)
+                command_safe_parts.append(safe_pipe)
+            safe_command = exploded_cmd[0] if not command_safe_parts else\
+                '{base} | {pipes}'.format(base=exploded_cmd[0],
+                                          pipes=' | '.join(command_safe_parts))
+            raw_txt = self.device.cli(safe_command, warning=False)
             cli_output[py23_compat.text_type(command)] = py23_compat.text_type(
-                self.device.cli(command))
-
+                _process_pipe(command, raw_txt))
         return cli_output
 
     def get_bgp_config(self, group='', neighbor=''):
@@ -999,23 +1124,23 @@ def _bgp_iter_core(neighbor_data, instance=None):
                 neighbor_details.update(neighbor_rib_details)
                 bgp_neighbors[instance_name][remote_as].append(neighbor_details)
 
-        old_junos = napalm_base.helpers.convert(
-            int, self.device.facts.get('version', '0.0').split('.')[0], 0) < 15
+        # old_junos = napalm_base.helpers.convert(
+        #     int, self.device.facts.get('version', '0.0').split('.')[0], 0) < 15
         bgp_neighbors_table = junos_views.junos_bgp_neighbors_table(self.device)
 
-        if old_junos:
-            instances = junos_views.junos_route_instance_table(self.device)
-            for instance, instance_data in instances.get().items():
-                if instance.startswith('__'):
-                    # junos internal instances
-                    continue
-                neighbor_data = bgp_neighbors_table.get(instance=instance,
-                                                        neighbor_address=neighbor_address).items()
-                _bgp_iter_core(neighbor_data, instance=instance)
-        else:
-            bgp_neighbors_table = junos_views.junos_bgp_neighbors_table(self.device)
-            neighbor_data = bgp_neighbors_table.get(neighbor_address=neighbor_address).items()
-            _bgp_iter_core(neighbor_data)
+        # if old_junos:
+        instances = junos_views.junos_route_instance_table(self.device)
+        for instance, instance_data in instances.get().items():
+            if instance.startswith('__'):
+                # junos internal instances
+                continue
+            neighbor_data = bgp_neighbors_table.get(instance=instance,
+                                                    neighbor_address=neighbor_address).items()
+            _bgp_iter_core(neighbor_data, instance=instance)
+        # else:
+        #     bgp_neighbors_table = junos_views.junos_bgp_neighbors_table(self.device)
+        #     neighbor_data = bgp_neighbors_table.get(neighbor_address=neighbor_address).items()
+        #     _bgp_iter_core(neighbor_data)
         return bgp_neighbors
 
     def get_arp_table(self):

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name="napalm-junos",
-    version="0.11.0",
+    version="0.12.0",
     packages=find_packages(),
     author="David Barroso, Mircea Ulinic",
     author_email="dbarrosop@dravetech.com, mircea@cloudflare.com",