Merge pull request #135 from netreplica/dev

Version 0.6.0

.github/workflows/pypi.yml

@@ -1,4 +1,4 @@
-name: Publish Python distribution to PyPI and TestPyPI
+name: Publish Python distribution to PyPI
 
 on:
     push:
@@ -13,7 +13,7 @@ jobs:
         steps:
           - uses: actions/checkout@v4
           - name: Set up Python
-            uses: actions/setup-python@v4
+            uses: actions/setup-python@v5
             with:
                 python-version: "3.x"
           - name: Install pypa/build
@@ -25,7 +25,7 @@ jobs:
           - name: Build a binary wheel and a source tarball
             run: make build
           - name: Store the distribution packages
-            uses: actions/upload-artifact@v3
+            uses: actions/upload-artifact@v4
             with:
                 name: python-package-distributions
                 path: dist/
@@ -45,9 +45,9 @@ jobs:
 
         steps:
           - name: Download all the dists
-            uses: actions/download-artifact@v3
+            uses: actions/download-artifact@v4
             with:
                 name: python-package-distributions
                 path: dist/
-          - name: Publish distribution to TestPyPI
+          - name: Publish distribution to PyPI
             uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/testpypi.yml

@@ -0,0 +1,57 @@
+name: Publish Python distribution to TestPyPI
+
+on:
+    push:
+        branches: [ "dev" ]
+    pull_request:
+        branches: [ "dev" ]
+    workflow_dispatch:
+
+jobs:
+    build:
+        name: Build distribution
+        runs-on: ubuntu-latest
+
+        steps:
+          - uses: actions/checkout@v4
+          - name: Set up Python
+            uses: actions/setup-python@v4
+            with:
+                python-version: "3.x"
+          - name: Install pypa/build
+            run: >-
+                python3 -m
+                pip install
+                build
+                --user
+          - name: Build a binary wheel and a source tarball
+            run: make build
+          - name: Store the distribution packages
+            uses: actions/upload-artifact@v3
+            with:
+                name: python-package-distributions
+                path: dist/
+
+    publish-to-pypi:
+        name: Publish to TestPyPI
+        needs:
+          - build
+        runs-on: ubuntu-latest
+
+        environment:
+            name: devtest
+            url: https://test.pypi.org/p/nrx
+
+        permissions:
+            id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+        steps:
+          - name: Download all the dists
+            uses: actions/download-artifact@v3
+            with:
+                name: python-package-distributions
+                path: dist/
+          - name: Publish distribution to TestPyPI
+            uses: pypa/gh-action-pypi-publish@release/v1
+            with:
+              repository-url: https://test.pypi.org/legacy/

Makefile

@@ -27,7 +27,7 @@ test-d2: test-dc1-cyjs-2-d2
 
 test-dc1: test-dc1-nb-2-cyjs-previous test-dc1-nb-2-cyjs-current test-dc1-nb-2-cyjs-latest test-dc1-cyjs-2-clab test-dc1-cyjs-2-clab-custom-platform-map test-dc1-cyjs-2-graphite test-dc1-cyjs-2-d2
 test-dc2: test-dc2-nb-2-cyjs-previous test-dc2-nb-2-cyjs-current test-dc2-nb-2-cyjs-latest test-dc2-cyjs-2-cml test-dc2-cyjs-2-graphite
-test-colo: test-colo-nb-2-cyjs-previous test-colo-nb-2-cyjs-current test-colo-nb-2-cyjs-latest
+test-colo: test-colo-nb-2-cyjs-previous test-colo-nb-2-cyjs-current test-colo-nb-2-cyjs-latest test-colo-nb-2-cyjs-interface-tags
 test-site1: test-site1-nb-2-cyjs-previous test-site1-nb-2-cyjs-current test-site1-nb-2-cyjs-latest test-site1-cyjs-2-clab test-site1-cyjs-2-clab-rename
 test-h88: test-h88-nb-2-cyjs-previous test-h88-nb-2-cyjs-current test-h88-nb-2-cyjs-latest test-h88-nb-2-cyjs-latest-noconfigs test-h88-cyjs-2-clab
 test-lrg: test-lrg-nb-2-cyjs-current test-lrg-nb-2-cyjs-latest test-lrg-cyjs-2-graphite
@@ -215,7 +215,7 @@ test-colo-nb-2-cyjs-previous:
 	mkdir -p tests/colo/test && cd tests/colo/test && rm -rf * && \
 	source ../../.env_previous && \
 	../../../nrx -c ../nrx.conf -o cyjs -d && \
-	diff colo.cyjs ../data/colo.cyjs
+	diff colo.cyjs ../data/colo.previous.cyjs
 	@echo
 
 test-colo-nb-2-cyjs-current:
@@ -225,7 +225,7 @@ test-colo-nb-2-cyjs-current:
 	mkdir -p tests/colo/test && cd tests/colo/test && rm -rf * && \
 	source ../../.env_current && \
 	../../../nrx -c ../nrx.conf -o cyjs -d && \
-	diff colo.cyjs ../data/colo.cyjs
+	diff colo.cyjs ../data/colo.current.cyjs
 	@echo
 
 test-colo-nb-2-cyjs-latest:
@@ -235,7 +235,17 @@ test-colo-nb-2-cyjs-latest:
 	mkdir -p tests/colo/test && cd tests/colo/test && rm -rf * && \
 	source ../../.env_latest && \
 	../../../nrx -c ../nrx.conf -o cyjs -d && \
-	diff colo.cyjs ../data/colo.cyjs
+	diff colo.cyjs ../data/colo.latest.cyjs
+	@echo
+
+test-colo-nb-2-cyjs-interface-tags:
+	@echo "#################################################################"
+	@echo "# Colo: read from NetBox latest version and export as CYJS using interface tags"
+	@echo "#################################################################"
+	mkdir -p tests/colo/test && cd tests/colo/test && rm -rf * && \
+	source ../../.env_latest && \
+	../../../nrx -c ../nrx.conf -o cyjs --interface-tags one,two,four -d && \
+	diff colo.cyjs ../data/colo-tags.cyjs
 	@echo
 
 test-site1-nb-2-cyjs-previous:

README.md

@@ -21,6 +21,8 @@ This project is in early phase. We're experimenting with the best ways to automa
 # Latest capabilities added
 
 The latest releases have a significant set of the new capabilities:
+* `0.6.0` NetBox `v4.0` compatibility
+* `0.6.0` Filter links between devices via interface tags
 * `0.5.0` PyPA packaging and distribution: `pip install nrx`
 * `0.4.0` Ability to create new output formats without a need for **nrx** code changes
 * `0.4.0` Mapping between NetBox platform values and node parameters via [`platform_map.yaml`](docs/platform_map.md) file
@@ -81,7 +83,7 @@ Export capabilities:
 
 The following software versions were tested for compatibility with `nrx`:
 
-* NetBox `v3.4`-`v3.6`. For device configuration export, `v3.5` is the minimum version.
+* NetBox `v3.6`-`v4.0`. We no longer run tests with previously supported `v3.4-3.5`
 * Containerlab `v0.39`, but earlier and later versions should work fine
 * Cisco Modeling Labs `v2.5`
 * Netreplica Graphite `v0.4.0`
@@ -157,6 +159,7 @@ optional arguments:
   -s, --site SITE           netbox site to export, cannot be combined with --sites
       --sites SITES         netbox sites to export, for multiple tags use a comma-separated list: site1,site2,site3 (uses OR logic)
   -t, --tags TAGS           netbox tags to export, for multiple tags use a comma-separated list: tag1,tag2,tag3 (uses AND logic)
+      --interface-tags TAGS netbox tags to filter interfaces to export, for multiple tags use a comma-separated list: tag1,tag2,tag3 (uses OR logic)
   -n, --name NAME           name of the exported topology (site name or tags by default)
       --noconfigs           disable device configuration export (enabled by default)
   -k, --insecure            allow insecure server connections when using TLS

renovate.json

@@ -9,5 +9,6 @@
   },
   "pip_requirements": {
     "fileMatch": ["requirements.txt", "requirements-dev.txt"]
-  }
+  },
+  "baseBranches": ["dev"]
 }

requirements-dev.txt

@@ -1,4 +1,4 @@
-pylint
+pylint ~= 3.2.6
 rich
 build
 twine

requirements.txt

@@ -1,11 +1,12 @@
-certifi==2024.2.2
+certifi==2024.7.4
 charset-normalizer==3.3.2
-idna==3.6
-Jinja2==3.1.3
+idna==3.7
+Jinja2==3.1.4
 MarkupSafe==2.1.5
 networkx==3.2.1
-pynetbox==7.3.3
-requests==2.31.0
+pynetbox==7.3.4
+requests==2.32.3
 toml==0.10.2
-urllib3==2.2.1
+urllib3==2.2.2
 pyyaml==6.0.1
+packaging==23.2

src/nrx/__about__.py

@@ -19,4 +19,4 @@
 """
 Metadata for the nrx package
 """
-__version__ = "0.5.2"
+__version__ = "0.6.0"

src/nrx/nrx.py

@@ -46,6 +46,7 @@
 import networkx as nx
 import jinja2
 import yaml
+from packaging import version
 
 # Single source version
 from nrx.__about__ import __version__
@@ -212,6 +213,7 @@ def __init__(self, config):
         self.nb_session = pynetbox.api(self.config['nb_api_url'],
                                        token=self.config['nb_api_token'],
                                        threading=True)
+        self.nb_api_version = version.parse(self.nb_session.version)
         self.nb_sites = None
         if not config['tls_validate']:
             self.nb_session.http_session.verify = False
@@ -305,7 +307,16 @@ def _get_nb_interfaces(self, block_size = 4):
                                                                          cabled=True,
                                                                          connected=True)):
                 if "base" in interface.type.value: # only ethernet interfaces
-                    debug(interface.device, ":", interface, ":", interface.type.value)
+                    if len(self.config['export_interface_tags']) > 0:
+                        tag_match = False
+                    for tag in interface.tags:
+                        if tag.name in self.config['export_interface_tags']: # implementing OR tag matching logic
+                            tag_match = True
+                            break
+                    if len(self.config['export_interface_tags']) > 0 and not tag_match:
+                        debug(f"{interface.device} : {interface} skipping, doesn't have any of the required tags")
+                        continue
+                    debug(f"{interface.device} : {interface} adding as {interface.type.value}")
                     i = {
                         "id": interface.id,
                         "type": "interface",
@@ -355,11 +366,17 @@ def _init_device(self, device):
             if device.device_type.manufacturer is not None:
                 d["vendor"] = device.device_type.manufacturer.slug
                 d["vendor_name"] = device.device_type.manufacturer.name
-        if device.device_role is not None:
-            d["role"] = device.device_role.slug
-            d["role_name"] = device.device_role.name
+
+        if device.role is not None:
+            if self.nb_api_version >= version.parse("4.0"):
+                d["role"] = device.role.slug
+                d["role_name"] = device.role.name
+            else:
+                d["role"] = device.device_role.slug
+                d["role_name"] = device.device_role.name
             if d["name"] is None:
                 d["name"] = f"{d['role']}-{device.id}"
+
         if device.primary_ip4 is not None:
             d["primary_ip4"] = device.primary_ip4.address
         if device.primary_ip6 is not None:
@@ -597,8 +614,8 @@ def _append_if_node_is_interface(self, n):
         """Append an interface node to the topology"""
         if self.G.nodes[n]['type'] == 'interface':
             int_name = self.G.nodes[n]['interface']['name']
-            dev_name, dev_node_id = None, None
-            peer_name, peer_dev_name, peer_dev_node_id = None, None, None
+            dev_name, dev_node_id, dev_index = None, None, None
+            peer_name, peer_dev_name, peer_dev_node_id, peer_dev_index = None, None, None, None
             for a_adj in self.G.adj[n].items():
                 if self.G.nodes[a_adj[0]]['type'] == 'device':
                     dev_name = self.G.nodes[a_adj[0]]['device']['name']
@@ -1014,6 +1031,9 @@ def parse_args():
                                                                           site1,site2,site3 (uses OR logic)')
     args_parser.add_argument('-t', '--tags',        required=False, help='netbox tags to export, for multiple tags use a comma-separated list: \
                                                                           tag1,tag2,tag3 (uses AND logic)')
+    args_parser.add_argument('--interface-tags',    required=False, help='netbox tags to filter interfaces to export, for multiple tags use a comma-separated list: \
+                                                                          tag1,tag2,tag3 (uses OR logic)',
+                                                                    metavar='TAGS')
     args_parser.add_argument('-n', '--name',        required=False, help='name of the exported topology (site name or tags by default)')
     args_parser.add_argument(      '--noconfigs',   required=False, help='disable device configuration export (enabled by default)',
                                                         action=argparse.BooleanOptionalAction)
@@ -1162,6 +1182,7 @@ def load_toml_config(filename):
         },
         'export_sites': [],
         'export_tags': [],
+        'export_interface_tags': [],
         'topology_name': '',
         'export_configs': True,
         'templates_path': ["./templates", f"{nrx_config_dir()}/templates"],
@@ -1212,6 +1233,9 @@ def config_apply_netbox_args(config, args):
     if args.tags is not None and len(args.tags) > 0:
         config['export_tags'] = args.tags.split(',')
         debug(f"List of tags to filter devices for export: {config['export_tags']}")
+    if args.interface_tags is not None and len(args.interface_tags) > 0:
+        config['export_interface_tags'] = args.interface_tags.split(',')
+        debug(f"List of tags to filter interfaces for export: {config['export_interface_tags']}")
     if len(config['export_sites']) == 0 and len(config['export_tags']) == 0:
         error("Need a Site name or Tags to export. Use --sites/--tags arguments, or EXPORT_SITE/EXPORT_TAGS key in --config file")
     if args.noconfigs is not None: