diff --git a/.copier-answers.yml b/.copier-answers.yml index 9e94ced..6d770d0 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,20 +1,31 @@ # Autogenerated. Do not edit this by hand, use `copier update`. --- -_commit: 0.2.6 +_commit: 0.4.4 _src_path: https://github.com/lkubb/salt-extension-copier author: EITR Technologies, LLC author_email: devops@eitr.tech -docs_url: '' +coc_contact: devops@eitr.tech +copyright_begin: 2022 +deploy_docs: never +docs_url: https://saltext-azurerm.readthedocs.io/en/latest/ +integration_name: Azure Resource Manager license: apache -loaders: [] -max_salt_version: 3006 +loaders: + - cloud + - fileserver + - module + - state +max_salt_version: 3007 no_saltext_namespace: false package_name: azurerm project_name: azurerm python_requires: '3.8' +relax_pylint: true salt_version: '3005' source_url: https://github.com/salt-extensions/saltext-azurerm ssh_fixtures: false summary: Salt Extension for interacting with Microsoft Azure +test_containers: false tracker_url: https://github.com/salt-extensions/saltext-azurerm/issues url: https://github.com/salt-extensions/saltext-azurerm +workflows: org diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..716c153 --- /dev/null +++ b/.envrc @@ -0,0 +1,7 @@ +layout_saltext() { + VIRTUAL_ENV="$(python3 tools/initialize.py --print-venv)" + PATH_add "$VIRTUAL_ENV/bin" + export VIRTUAL_ENV +} + +layout_saltext diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3a04455..30f3b8e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,3 +1,4 @@ +--- name: Pull Request or Push on: @@ -13,7 +14,9 @@ jobs: name: CI uses: salt-extensions/central-artifacts/.github/workflows/ci.yml@main with: - setup-vault: true + deploy-docs: false permissions: contents: write + id-token: write + pages: write pull-requests: read diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index abdd5e7..0367a0b 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -1,3 +1,4 @@ +--- name: Tagged Releases on: @@ -16,17 +17,18 @@ jobs: - name: Extract tag name id: get_version - run: echo "::set-output name=version::$(echo ${GITHUB_REF#refs/tags/v})" + run: echo "version=${GITHUB_REF#refs/tags/v}" >> "$GITHUB_OUTPUT" call_central_workflow: needs: get_tag_version uses: salt-extensions/central-artifacts/.github/workflows/ci.yml@main with: - setup-vault: true + deploy-docs: false release: true version: ${{ needs.get_tag_version.outputs.version }} permissions: contents: write id-token: write + pages: write pull-requests: read secrets: inherit diff --git a/.gitignore b/.gitignore index 45bf30e..3173eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,7 @@ celerybeat.pid *.sage.py # Environments +!.envrc .env .venv env/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e039c42..96293f8 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,16 +2,16 @@ minimum_pre_commit_version: 2.4.0 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - - id: check-merge-conflict # Check for files that contain merge conflict strings. - - id: trailing-whitespace # Trims trailing whitespace. + - id: check-merge-conflict # Check for files that contain merge conflict strings. + args: [--assume-in-merge] + - id: trailing-whitespace # Trim trailing whitespace. args: [--markdown-linebreak-ext=md] - - id: mixed-line-ending # Replaces or checks mixed line ending. + - id: mixed-line-ending # Ensure files use UNIX-style newlines only. args: [--fix=lf] - - id: end-of-file-fixer # Makes sure files end in a newline and only a newline. - - id: check-merge-conflict # Check for files that contain merge conflict strings. - - id: check-ast # Simply check whether files parse as valid python. + - id: end-of-file-fixer # Ensure files end with a newline. + - id: check-ast # Check whether files parse as valid Python. # ----- Formatting ----------------------------------------------------------------------------> - repo: https://github.com/saltstack/pre-commit-remove-import-headers @@ -24,7 +24,7 @@ repos: - id: check-cli-examples name: Check CLI examples on execution modules entry: python .pre-commit-hooks/check-cli-examples.py - language: system + language: python files: ^src/saltext/azurerm/modules/.*\.py$ - repo: local @@ -32,7 +32,7 @@ repos: - id: check-docs name: Check rST doc files exist for modules/states entry: python .pre-commit-hooks/make-autodocs.py - language: system + language: python pass_filenames: false - repo: https://github.com/s0undt3ch/salt-rewrite @@ -56,7 +56,7 @@ repos: args: [--silent, -E, fix_docstrings] - repo: https://github.com/asottile/pyupgrade - rev: v2.37.2 + rev: v3.16.0 hooks: - id: pyupgrade name: Rewrite Code to be Py3.8+ @@ -65,35 +65,35 @@ repos: ] exclude: src/saltext/azurerm/version.py - - repo: https://github.com/asottile/reorder_python_imports - rev: v3.10.0 + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 hooks: - - id: reorder-python-imports + - id: isort args: [ - --py38-plus, + --py 38, ] - exclude: src/saltext/azurerm/version.py + exclude: src/saltext/azurerm/(__init__|version).py - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 24.8.0 hooks: - id: black args: [-l 100] exclude: src/saltext/azurerm/version.py - repo: https://github.com/adamchainz/blacken-docs - rev: v1.12.1 + rev: 1.18.0 hooks: - id: blacken-docs args: [--skip-errors] files: ^(docs/.*\.rst|src/saltext/azurerm/.*\.py)$ additional_dependencies: - - black==22.6.0 + - black==24.2.0 # <---- Formatting ----------------------------------------------------------------------------- # ----- Security ------------------------------------------------------------------------------> - repo: https://github.com/PyCQA/bandit - rev: "1.7.4" + rev: 1.7.9 hooks: - id: bandit alias: bandit-salt @@ -101,7 +101,7 @@ repos: args: [--silent, -lll, --skip, B701] exclude: src/saltext/azurerm/version.py - repo: https://github.com/PyCQA/bandit - rev: "1.7.4" + rev: 1.7.9 hooks: - id: bandit alias: bandit-tests @@ -111,29 +111,35 @@ repos: # <---- Security ------------------------------------------------------------------------------- # ----- Code Analysis -------------------------------------------------------------------------> - - repo: https://github.com/saltstack/mirrors-nox - rev: v2021.6.12 + + - repo: local hooks: - id: nox alias: lint-src name: Lint Source Code + language: python + entry: nox -e lint-code-pre-commit -- files: ^((setup|noxfile)|src/.*)\.py$ require_serial: true - args: - - -e - - lint-code-pre-commit - - -- + additional_dependencies: + - nox==2024.4.15 + - uv==0.4.0 # Makes this hook much faster - - repo: https://github.com/saltstack/mirrors-nox - rev: v2021.6.12 - hooks: - id: nox alias: lint-tests name: Lint Tests + language: python + entry: nox -e lint-tests-pre-commit -- files: ^tests/.*\.py$ require_serial: true - args: - - -e - - lint-tests-pre-commit - - -- + additional_dependencies: + - nox==2024.4.15 + - uv==0.4.0 # Makes this hook much faster + + - repo: https://github.com/Mateusz-Grzelinski/actionlint-py + rev: 1ca29a1b5d949b3586800190ad6cc98317cb43b8 # v1.7.1.15 + hooks: + - id: actionlint + additional_dependencies: + - shellcheck-py>=0.9.0.5 # <---- Code Analysis -------------------------------------------------------------------------- diff --git a/.pre-commit-hooks/make-autodocs.py b/.pre-commit-hooks/make-autodocs.py index a5d4b62..b7d4487 100644 --- a/.pre-commit-hooks/make-autodocs.py +++ b/.pre-commit-hooks/make-autodocs.py @@ -3,7 +3,6 @@ import subprocess from pathlib import Path - repo_path = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode().strip()) src_dir = repo_path / "src" / "saltext" / "azurerm" doc_dir = repo_path / "docs" @@ -34,9 +33,14 @@ def write_module(rst_path, path, use_virtualname=True): virtualname = "``" + _find_virtualname(path) + "``" else: virtualname = make_import_path(path) + header_len = len(virtualname) + # The check-merge-conflict pre-commit hook chokes here: + # https://github.com/pre-commit/pre-commit-hooks/issues/100 + if header_len == 7: + header_len += 1 module_contents = f"""\ {virtualname} -{'='*len(virtualname)} +{'='*header_len} .. automodule:: {make_import_path(path)} :members: diff --git a/.pylintrc b/.pylintrc index 5692f3b..55234bc 100755 --- a/.pylintrc +++ b/.pylintrc @@ -39,7 +39,7 @@ extension-pkg-whitelist= fail-on= # Specify a score threshold under which the program will exit with error. -fail-under=10 +fail-under=10.0 # Interpret the stdin as a python script, whose filename needs to be passed as # the module_or_package argument. @@ -59,10 +59,11 @@ ignore-paths= # Emacs file locks ignore-patterns=^\.# -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. +# List of module names for which member attributes should not be checked and +# will not be imported (useful for modules/projects where namespaces are +# manipulated during runtime and thus existing member attributes cannot be +# deduced by static analysis). It supports qualified module names, as well as +# Unix pattern matching. ignored-modules= # Python code to execute, usually for sys.path manipulation such as @@ -86,9 +87,13 @@ load-plugins= # Pickle collected data for later comparisons. persistent=yes +# Resolve imports to .pyi stubs if available. May reduce no-member messages and +# increase not-an-iterable messages. +prefer-stubs=no + # Minimum Python version to use for version dependent checks. Will default to # the version used to run pylint. -py-version=3.10 +py-version=3.8 # Discover python modules and packages in the file system subtree. recursive=no @@ -285,19 +290,19 @@ exclude-too-few-public-methods= ignored-parents= # Maximum number of arguments for function / method. -max-args=15 +max-args=35 # Maximum number of attributes for a class (see R0902). -max-attributes=7 +max-attributes=15 # Maximum number of boolean expressions in an if statement (see R0916). -max-bool-expr=5 +max-bool-expr=8 # Maximum number of branch for function / method body. -max-branches=12 +max-branches=48 # Maximum number of locals for function / method body. -max-locals=15 +max-locals=40 # Maximum number of parents for a class (see R0901). max-parents=7 @@ -306,10 +311,10 @@ max-parents=7 max-public-methods=25 # Maximum number of return / yield for function / method body. -max-returns=6 +max-returns=10 # Maximum number of statements in function / method body. -max-statements=50 +max-statements=100 # Minimum number of public methods for a class (see R0903). min-public-methods=2 @@ -324,7 +329,7 @@ overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= +expected-line-ending-format=LF # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ @@ -337,10 +342,10 @@ indent-after-paren=4 indent-string=' ' # Maximum number of characters on a single line. -max-line-length=100 +max-line-length=120 # Maximum number of lines in a module. -max-module-lines=2000 +max-module-lines=3000 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. @@ -421,43 +426,33 @@ confidence=HIGH, # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=R, - locally-disabled, - file-ignored, - unexpected-special-method-signature, - import-error, - no-member, - unsubscriptable-object, - blacklisted-name, - invalid-name, - missing-docstring, - empty-docstring, - unidiomatic-typecheck, - wrong-import-order, - ungrouped-imports, - wrong-import-position, - bad-mcs-method-argument, - bad-mcs-classmethod-argument, - line-too-long, - too-many-lines, - bad-continuation, - exec-used, - attribute-defined-outside-init, - protected-access, - reimported, - fixme, - global-statement, - unused-variable, - unused-argument, - redefined-outer-name, - redefined-builtin, - undefined-loop-variable, - logging-format-interpolation, - invalid-format-index, - line-too-long, - import-outside-toplevel, - deprecated-method, - keyword-arg-before-vararg, +disable=consider-using-f-string, + duplicate-code, + fixme, + inconsistent-return-statements, + keyword-arg-before-vararg, + line-too-long, + logging-fstring-interpolation, + missing-class-docstring, + missing-function-docstring, + missing-module-docstring, + no-else-return, + no-member, + protected-access, + redefined-argument-from-local, + redefined-builtin, + redefined-outer-name, + too-few-public-methods, + too-many-boolean-expressions, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-return-statements, + too-many-statements, + ungrouped-imports, + wrong-import-position # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -495,6 +490,11 @@ max-nested-blocks=5 # printed. never-returning-functions=sys.exit,argparse.parse_error +# Let 'consider-using-join' be raised when the separator to join on would be +# non-empty (resulting in expected fixes of the type: ``"- " + " - +# ".join(items)``) +suggest-join-with-non-empty-separator=yes + [REPORTS] @@ -509,8 +509,9 @@ evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor # used to format the message information. See doc for all details. msg-template= -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. +# Set the output format. Available formats are: text, parseable, colorized, +# json2 (improved json format), json (old json format) and msvs (visual +# studio). You can also give a reporter class, e.g. # mypackage.mymodule.MyReporterClass. #output-format= @@ -544,8 +545,8 @@ min-similarity-lines=4 # Limits count of emitted suggestions for spelling mistakes. max-spelling-suggestions=4 -# Spelling dictionary name. No available dictionaries : You need to install the -# system dependency for enchant to work.. +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work. spelling-dict= # List of comma separated words that should be considered directives if they @@ -633,27 +634,27 @@ signature-mutators= # List of additional names supposed to be defined in builtins. Remember that # you should avoid defining new builtins when possible. additional-builtins=__opts__, - __salt__, - __pillar__, - __grains__, - __context__, - __runner__, - __ret__, - __env__, - __low__, - __states__, - __lowstate__, - __running__, - __active_provider_name__, - __master_opts__, - __jid_event__, - __instance_id__, - __salt_system_encoding__, - __proxy__, - __serializers__, - __reg__, - __executors__, - __events__ + __salt__, + __pillar__, + __grains__, + __context__, + __runner__, + __ret__, + __env__, + __low__, + __states__, + __lowstate__, + __running__, + __active_provider_name__, + __master_opts__, + __jid_event__, + __instance_id__, + __salt_system_encoding__, + __proxy__, + __serializers__, + __reg__, + __executors__, + __events__ # Tells whether unused global variables should be treated as a violation. allow-global-unused-variables=yes diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..7467394 --- /dev/null +++ b/NOTICE @@ -0,0 +1,10 @@ +Salt Extension Modules for Azure Resource Manager +Copyright 2022 EITR Technologies, LLC + +This product is licensed to you under the Apache 2.0 license (the "License"). +You may not use this product except in compliance with the Apache 2.0 License. + +This product may include a number of subcomponents with separate copyright +notices and license terms. Your use of these subcomponents is subject to the +terms and conditions of the subcomponent's license, as noted in the LICENSE +file. diff --git a/README.md b/README.md index 82fa3cf..14cea68 100644 --- a/README.md +++ b/README.md @@ -6,81 +6,98 @@ Salt Extension for interacting with Microsoft Azure ## Security -If you think you've found a security vulnerability, see -[Salt's security guide][security]. +If you discover a security vulnerability, please refer +to [Salt's security guide][security]. ## User Documentation -This README is more for contributing to the project. If you just want to get -started, check out the [User Documentation][docs]. +For setup and usage instructions, please refer to the +[User Documentation][docs]. ## Contributing -The saltext-azurerm project team welcomes contributions from the community. +The saltext-azurerm project welcomes contributions from anyone! -The [Salt Contributing guide][salt-contributing] has a lot of relevant -information, but if you'd like to jump right in here's how to get started: +The [Salt Extensions guide][salt-extensions-guide] provides comprehensive instructions on all aspects +of Salt extension development, including [writing tests][writing-tests], [running tests][running-tests], +[writing documentation][writing-docs] and [rendering the docs][rendering-docs]. +### Quickstart + +To get started contributing, first clone this repository (or your fork): ```bash # Clone the repo -git clone --origin salt git@github.com:salt-extensions/saltext-azurerm.git +git clone --origin upstream git@github.com:salt-extensions/saltext-azurerm.git # Change to the repo dir cd saltext-azurerm +``` -# Create a new venv -python3 -m venv env --prompt saltext-azurerm -source env/bin/activate +#### Automatic +If you have installed [direnv][direnv], allowing the project's `.envrc` ensures +a proper development environment is present and the virtual environment is active. -# On mac, you may need to upgrade pip -python -m pip install --upgrade pip +Without `direnv`, you can still run the automation explicitly: -# On WSL or some flavors of linux you may need to install the `enchant` -# library in order to build the docs -sudo apt-get install -y enchant +```bash +python3 tools/initialize.py +source .venv/bin/activate +``` -# Install extension + test/dev/doc dependencies into your environment -python -m pip install -e '.[tests,dev,docs]' +#### Manual +Please follow the [first steps][first-steps], skipping the repository initialization and first commit. -# Run tests! -python -m nox -e tests-3 +### Pull request + +Always make changes in a feature branch: + +```bash +git switch -c my-feature-branch +``` -# skip requirements install for next time -export SKIP_REQUIREMENTS_INSTALL=1 +To [submit a Pull Request][submitting-pr], you'll need a fork of this repository in +your own GitHub account. If you followed the instructions above, +set your fork as the `origin` remote now: -# Build the docs, serve, and view in your web browser: -python -m nox -e docs && (cd docs/_build/html; python -m webbrowser localhost:8000; python -m http.server; cd -) +```bash +git remote add origin git@github.com:.git ``` -Writing code isn't the only way to contribute! We value contributions in any of -these areas: +Ensure you followed the [first steps][first-steps] and commit your changes, fixing any +failing `pre-commit` hooks. Then push the feature branch to your fork and submit a PR. + +### Ways to contribute + +Contributions come in many forms, and they’re all valuable! Here are some ways you can help +without writing code: -* Documentation - especially examples of how to use this module to solve - specific problems. -* Triaging [issues][issues] and participating in [discussions][discussions] -* Reviewing [Pull Requests][PRs] (we really like - [Conventional Comments][comments]!) +* **Documentation**: Especially examples showing how to use this project + to solve specific problems. +* **Triaging issues**: Help manage [issues][issues] and participate in [discussions][discussions]. +* **Reviewing [Pull Requests][PRs]**: We especially appreciate reviews using [Conventional Comments][comments]. -You could also contribute in other ways: +You can also contribute by: * Writing blog posts -* Posting on social media about how you used Salt+Azurerm to solve your - problems, including videos +* Sharing your experiences using Salt + Azure Resource Manager + on social media * Giving talks at conferences * Publishing videos -* Asking/answering questions in IRC, Slack, or email groups +* Engaging in IRC, Discord or email groups Any of these things are super valuable to our community, and we sincerely appreciate every contribution! - -For more information, build the docs and head over to http://localhost:8000/ — -that's where you'll find the rest of the documentation. - - [security]: https://github.com/saltstack/salt/blob/master/SECURITY.md -[salt-contributing]: https://docs.saltproject.io/en/master/topics/development/contributing.html +[salt-extensions-guide]: https://salt-extensions.github.io/salt-extension-copier/ +[writing-tests]: https://salt-extensions.github.io/salt-extension-copier/topics/testing/writing.html +[running-tests]: https://salt-extensions.github.io/salt-extension-copier/topics/testing/running.html +[writing-docs]: https://salt-extensions.github.io/salt-extension-copier/topics/documenting/writing.html +[rendering-docs]: https://salt-extensions.github.io/salt-extension-copier/topics/documenting/building.html +[first-steps]: https://salt-extensions.github.io/salt-extension-copier/topics/creation.html#initialize-the-python-virtual-environment +[submitting-pr]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork +[direnv]: https://direnv.net [issues]: https://github.com/salt-extensions/saltext-azurerm/issues [PRs]: https://github.com/salt-extensions/saltext-azurerm/pulls [discussions]: https://github.com/salt-extensions/saltext-azurerm/discussions diff --git a/docs/_ext/saltdomain.py b/docs/_ext/saltdomain.py new file mode 100644 index 0000000..7a85489 --- /dev/null +++ b/docs/_ext/saltdomain.py @@ -0,0 +1,18 @@ +""" +Copied/distilled from Salt doc/_ext/saltdomain.py in order to be able +to use Salt's custom doc refs. +""" + + +def setup(app): + app.add_crossref_type( + directivename="conf_master", + rolename="conf_master", + indextemplate="pair: %s; conf/master", + ) + app.add_crossref_type( + directivename="conf_minion", + rolename="conf_minion", + indextemplate="pair: %s; conf/minion", + ) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/conf.py b/docs/conf.py index 7e439fc..de8713d 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,12 +42,12 @@ # -- Project information ----------------------------------------------------- this_year = datetime.datetime.today().year -if this_year == 2021: - copyright_year = 2021 +if this_year == 2022: + copyright_year = "2022" else: - copyright_year = f"2021 - {this_year}" + copyright_year = f"2022 - {this_year}" project = dist.metadata["Summary"] -author = dist.metadata["Author"] +author = dist.metadata.get("Author") if author is None: # Core metadata is serialized differently with pyproject.toml: @@ -79,6 +79,8 @@ # -- General configuration --------------------------------------------------- +linkcheck_ignore = [r"http://localhost:\d+"] + # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. @@ -92,6 +94,7 @@ "sphinx.ext.coverage", "sphinx_copybutton", "sphinxcontrib.spelling", + "saltdomain", "sphinxcontrib.towncrier.ext", "myst_parser", "sphinx_inline_tabs", diff --git a/docs/topics/installation.md b/docs/topics/installation.md index c3f4773..d04b1a8 100644 --- a/docs/topics/installation.md +++ b/docs/topics/installation.md @@ -22,15 +22,7 @@ pip install saltext-azurerm ``` ::: -:::{important} -Currently, there is [an issue][issue-second-saltext] where the installation of a Saltext fails silently -if the environment already has another one installed. You can workaround this by -removing all Saltexts and reinstalling them in one transaction. -::: - :::{hint} Saltexts are not distributed automatically via the fileserver like custom modules, they need to be installed on each node you want them to be available on. ::: - -[issue-second-saltext]: https://github.com/saltstack/salt/issues/65433 diff --git a/noxfile.py b/noxfile.py index d3957ad..75a338f 100755 --- a/noxfile.py +++ b/noxfile.py @@ -1,11 +1,10 @@ -# pylint: disable=missing-module-docstring,import-error,protected-access,missing-function-docstring import datetime import json import os -import pathlib import shutil import sys import tempfile +from importlib import metadata from pathlib import Path import nox @@ -17,18 +16,21 @@ nox.options.reuse_existing_virtualenvs = True # Don't fail on missing interpreters nox.options.error_on_missing_interpreters = False +# Speed up all sessions by using uv if possible +if tuple(map(int, metadata.version("nox").split("."))) >= (2024, 3): + nox.options.default_venv_backend = "uv|virtualenv" # Python versions to test against -PYTHON_VERSIONS = ("3", "3.8", "3.9", "3.10") +PYTHON_VERSIONS = ("3", "3.8", "3.9", "3.10", "3.11", "3.12") # Be verbose when running under a CI context CI_RUN = ( os.environ.get("JENKINS_URL") or os.environ.get("CI") or os.environ.get("DRONE") is not None ) PIP_INSTALL_SILENT = CI_RUN is False -SKIP_REQUIREMENTS_INSTALL = "SKIP_REQUIREMENTS_INSTALL" in os.environ +SKIP_REQUIREMENTS_INSTALL = os.environ.get("SKIP_REQUIREMENTS_INSTALL", "0") == "1" EXTRA_REQUIREMENTS_INSTALL = os.environ.get("EXTRA_REQUIREMENTS_INSTALL") -COVERAGE_VERSION_REQUIREMENT = "coverage==5.2" +COVERAGE_REQUIREMENT = os.environ.get("COVERAGE_REQUIREMENT") or "coverage==7.5.1" SALT_REQUIREMENT = os.environ.get("SALT_REQUIREMENT") or "salt>=3005" if SALT_REQUIREMENT == "salt==master": SALT_REQUIREMENT = "git+https://github.com/saltstack/salt.git@master" @@ -37,7 +39,7 @@ os.environ["PYTHONDONTWRITEBYTECODE"] = "1" # Global Path Definitions -REPO_ROOT = pathlib.Path(__file__).resolve().parent +REPO_ROOT = Path(__file__).resolve().parent # Change current directory to REPO_ROOT os.chdir(str(REPO_ROOT)) @@ -85,16 +87,17 @@ def _install_requirements( install_extras=None, ): install_extras = install_extras or [] + no_progress = "--progress-bar=off" + if isinstance(session._runner.venv, VirtualEnv) and session._runner.venv.venv_backend == "uv": + no_progress = "--no-progress" if SKIP_REQUIREMENTS_INSTALL is False: # Always have the wheel package installed - session.install("--progress-bar=off", "wheel", silent=PIP_INSTALL_SILENT) + session.install(no_progress, "wheel", silent=PIP_INSTALL_SILENT) if install_coverage_requirements: - session.install( - "--progress-bar=off", COVERAGE_VERSION_REQUIREMENT, silent=PIP_INSTALL_SILENT - ) + session.install(no_progress, COVERAGE_REQUIREMENT, silent=PIP_INSTALL_SILENT) if install_salt: - session.install("--progress-bar=off", SALT_REQUIREMENT, silent=PIP_INSTALL_SILENT) + session.install(no_progress, SALT_REQUIREMENT, silent=PIP_INSTALL_SILENT) if install_test_requirements: install_extras.append("tests") @@ -106,7 +109,7 @@ def _install_requirements( "EXTRA_REQUIREMENTS_INSTALL='%s'", EXTRA_REQUIREMENTS_INSTALL, ) - install_command = ["--progress-bar=off"] + install_command = [no_progress] install_command += [req.strip() for req in EXTRA_REQUIREMENTS_INSTALL.split()] session.install(*install_command, silent=PIP_INSTALL_SILENT) @@ -172,7 +175,7 @@ def tests(session): if arg.startswith(f"tests{os.sep}"): break try: - pathlib.Path(arg).resolve().relative_to(REPO_ROOT / "tests") + Path(arg).resolve().relative_to(REPO_ROOT / "tests") break except ValueError: continue @@ -246,7 +249,7 @@ def _lint(session, rcfile, flags, paths, tee_output=True): install_salt=False, install_coverage_requirements=False, install_test_requirements=False, - install_extras=["dev", "tests"], + install_extras=["lint", "tests"], ) if tee_output: @@ -309,12 +312,25 @@ def _lint_pre_commit(session, rcfile, flags, paths): ) # Let's patch nox to make it run inside the pre-commit virtualenv - session._runner.venv = VirtualEnv( - os.environ["VIRTUAL_ENV"], - interpreter=session._runner.func.python, - reuse_existing=True, - venv=True, - ) + try: + # nox >= 2024.03.02 + # pylint: disable=unexpected-keyword-arg + venv = VirtualEnv( + os.environ["VIRTUAL_ENV"], + interpreter=session._runner.func.python, + reuse_existing=True, + venv_backend="venv", + ) + except TypeError: + # nox < 2024.03.02 + # pylint: disable=unexpected-keyword-arg + venv = VirtualEnv( + os.environ["VIRTUAL_ENV"], + interpreter=session._runner.func.python, + reuse_existing=True, + venv=True, + ) + session._runner.venv = venv _lint(session, rcfile, flags, paths, tee_output=False) @@ -346,7 +362,7 @@ def lint_tests(session): Run PyLint against the test suite. Set PYLINT_REPORT to a path to capture output. """ flags = [ - "--disable=I,redefined-outer-name,missing-function-docstring,no-member,missing-module-docstring" + "--disable=I,redefined-outer-name,no-member,missing-module-docstring,missing-function-docstring,missing-class-docstring,attribute-defined-outside-init,inconsistent-return-statements,too-few-public-methods,too-many-public-methods,unused-argument", ] if session.posargs: paths = session.posargs @@ -374,7 +390,7 @@ def lint_tests_pre_commit(session): Run PyLint against the code and the test suite. Set PYLINT_REPORT to a path to capture output. """ flags = [ - "--disable=I,redefined-outer-name,missing-function-docstring,no-member,missing-module-docstring", + "--disable=I,redefined-outer-name,no-member,missing-module-docstring,missing-function-docstring,missing-class-docstring,attribute-defined-outside-init,inconsistent-return-statements,too-few-public-methods,too-many-public-methods,unused-argument", ] if session.posargs: paths = session.posargs @@ -409,37 +425,8 @@ def docs(session): os.chdir(str(REPO_ROOT)) -@nox.session(name="docs-html", python="3") -@nox.parametrize("clean", [False, True]) -@nox.parametrize("include_api_docs", [False, True]) -def docs_html(session, clean, include_api_docs): - """ - Build Sphinx HTML Documentation - - TODO: Add option for `make linkcheck` and `make coverage` - calls via Sphinx. Ran into problems with two when - using Furo theme and latest Sphinx. - """ - _install_requirements( - session, - install_coverage_requirements=False, - install_test_requirements=False, - install_source=True, - install_extras=["docs"], - ) - if include_api_docs: - gen_api_docs(session) - build_dir = Path("docs", "_build", "html") - sphinxopts = "-Wn" - if clean: - sphinxopts += "E" - args = [sphinxopts, "--keep-going", "docs", str(build_dir)] - session.run("sphinx-build", *args, external=True) - - @nox.session(name="docs-dev", python="3") -@nox.parametrize("clean", [False, True]) -def docs_dev(session, clean) -> None: +def docs_dev(session): """ Build and serve the Sphinx HTML documentation, with live reloading on file changes, via sphinx-autobuild. @@ -454,10 +441,18 @@ def docs_dev(session, clean) -> None: install_extras=["docs", "docsauto"], ) - # Launching LIVE reloading Sphinx session build_dir = Path("docs", "_build", "html") - args = ["--watch", ".", "--open-browser", "docs", str(build_dir)] - if clean and build_dir.exists(): + + # Allow specifying sphinx-autobuild options, like --host. + args = ["--watch", "."] + session.posargs + if not any(arg.startswith("--host") for arg in args): + # If the user is overriding the host to something other than localhost, + # it's likely they are rendering on a remote/headless system and don't + # want the browser to open. + args.append("--open-browser") + args += ["docs", str(build_dir)] + + if build_dir.exists(): shutil.rmtree(build_dir) session.run("sphinx-autobuild", *args) @@ -498,30 +493,3 @@ def docs_crosslink_info(session): "python", "-m", "sphinx.ext.intersphinx", mapping_entry[0].rstrip("/") + "/objects.inv" ) os.chdir(str(REPO_ROOT)) - - -@nox.session(name="gen-api-docs", python="3") -def gen_api_docs(session): - """ - Generate API Docs - """ - _install_requirements( - session, - install_coverage_requirements=False, - install_test_requirements=False, - install_source=True, - install_extras=["docs"], - ) - try: - shutil.rmtree("docs/ref") - except FileNotFoundError: - pass - session.run( - "sphinx-apidoc", - "--implicit-namespaces", - "--module-first", - "-o", - "docs/ref/", - "src/saltext", - "src/saltext/azurerm/config/schemas", - ) diff --git a/pyproject.toml b/pyproject.toml index d9d693b..4faafbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", @@ -70,16 +72,15 @@ content-type = "text/markdown" [project.urls] Homepage = "https://github.com/salt-extensions/saltext-azurerm" +Documentation = "https://saltext-azurerm.readthedocs.io/en/latest/" Source = "https://github.com/salt-extensions/saltext-azurerm" Tracker = "https://github.com/salt-extensions/saltext-azurerm/issues" [project.optional-dependencies] changelog = ["towncrier==22.12.0"] dev = [ - "nox", - "pre-commit>=2.4.0", - "pylint", - "saltpylint", + "nox[uv]>=2024.3", + "pre-commit>=2.21.0", ] docs = [ "sphinx", @@ -94,13 +95,12 @@ docs = [ ] docsauto = ["sphinx-autobuild"] lint = [ - "pylint", - "saltpylint", + "pylint==3.2.6", ] tests = [ - "pytest>=6.1.0", + "pytest>=7.2.0", + "pytest-salt-factories>=1.0.0", "pytest-ordering>=0.6", - "pytest-salt-factories>=1.0.0rc19", ] [project.entry-points."salt.loader"] @@ -130,42 +130,48 @@ build_dir = "build/sphinx" [tool.black] line-length = 100 +[tool.isort] +force_single_line = true +skip = ["src/saltext/azurerm/__init__.py"] +profile = "black" +line_length = 100 + [tool.towncrier] - package = "saltext.azurerm" - filename = "CHANGELOG.md" - template = "changelog/.template.jinja" - directory = "changelog/" - start_string = "# Changelog\n" - underlines = ["", "", ""] - title_format = "## {version} ({project_date})" - issue_format = "[#{issue}](https://github.com/salt-extensions/saltext-azurerm/issues/{issue})" - - [[tool.towncrier.type]] - directory = "removed" - name = "Removed" - showcontent = true - - [[tool.towncrier.type]] - directory = "deprecated" - name = "Deprecated" - showcontent = true - - [[tool.towncrier.type]] - directory = "changed" - name = "Changed" - showcontent = true - - [[tool.towncrier.type]] - directory = "fixed" - name = "Fixed" - showcontent = true - - [[tool.towncrier.type]] - directory = "added" - name = "Added" - showcontent = true - - [[tool.towncrier.type]] - directory = "security" - name = "Security" - showcontent = true +package = "saltext.azurerm" +filename = "CHANGELOG.md" +template = "changelog/.template.jinja" +directory = "changelog/" +start_string = "# Changelog\n" +underlines = ["", "", ""] +title_format = "## {version} ({project_date})" +issue_format = "[#{issue}](https://github.com/salt-extensions/saltext-azurerm/issues/{issue})" + +[[tool.towncrier.type]] +directory = "removed" +name = "Removed" +showcontent = true + +[[tool.towncrier.type]] +directory = "deprecated" +name = "Deprecated" +showcontent = true + +[[tool.towncrier.type]] +directory = "changed" +name = "Changed" +showcontent = true + +[[tool.towncrier.type]] +directory = "fixed" +name = "Fixed" +showcontent = true + +[[tool.towncrier.type]] +directory = "added" +name = "Added" +showcontent = true + +[[tool.towncrier.type]] +directory = "security" +name = "Security" +showcontent = true diff --git a/src/saltext/azurerm/clouds/azurerm.py b/src/saltext/azurerm/clouds/azurerm.py index 555bb6a..95489b0 100644 --- a/src/saltext/azurerm/clouds/azurerm.py +++ b/src/saltext/azurerm/clouds/azurerm.py @@ -89,6 +89,7 @@ **Note:** review the details of Service Principals. Owner role is more than you normally need, and you can restrict scope to a resource group or individual resources. """ + import importlib import logging import os.path @@ -97,27 +98,27 @@ from multiprocessing import cpu_count from multiprocessing.pool import ThreadPool -import salt.cache # pylint: disable=import-error -import salt.config as config # pylint: disable=import-error -import salt.utils.cloud # pylint: disable=import-error -import salt.utils.files # pylint: disable=import-error -import salt.utils.stringutils # pylint: disable=import-error -import salt.utils.yaml # pylint: disable=import-error -import salt.version # pylint: disable=import-error -import saltext.azurerm.utils.azurerm -from salt.exceptions import SaltCloudConfigError # pylint: disable=import-error -from salt.exceptions import SaltCloudExecutionFailure # pylint: disable=import-error -from salt.exceptions import SaltCloudExecutionTimeout # pylint: disable=import-error -from salt.exceptions import SaltCloudSystemExit # pylint: disable=import-error +import salt.cache +import salt.utils.cloud +import salt.utils.files +import salt.utils.stringutils +import salt.utils.yaml +import salt.version +from salt import config +from salt.exceptions import SaltCloudConfigError +from salt.exceptions import SaltCloudExecutionFailure +from salt.exceptions import SaltCloudExecutionTimeout +from salt.exceptions import SaltCloudSystemExit +import saltext.azurerm.utils.azurerm HAS_LIBS = False try: import azure.mgmt.compute.models as compute_models import azure.mgmt.network.models as network_models - - from azure.storage.blob import BlobServiceClient, ContainerClient from azure.core.exceptions import HttpResponseError + from azure.storage.blob import BlobServiceClient + from azure.storage.blob import ContainerClient HAS_LIBS = True except ImportError: @@ -126,7 +127,7 @@ try: __salt__ # pylint: disable=used-before-assignment except NameError: - import salt.loader # pylint: disable=import-error + import salt.loader __opts__ = salt.config.minion_config("/etc/salt/minion") __utils__ = salt.loader.utils(__opts__) @@ -1706,7 +1707,7 @@ def create_or_update_vmextension(call=None, kwargs=None): # pylint: disable=unu if not isinstance(settings, dict): raise SaltCloudSystemExit("VM extension settings are not valid") - elif "commandToExecute" not in settings and "script" not in settings: + if "commandToExecute" not in settings and "script" not in settings: raise SaltCloudSystemExit( "VM extension settings are not valid. Either commandToExecute or script" " must be specified." diff --git a/src/saltext/azurerm/fileserver/__init__.py b/src/saltext/azurerm/fileserver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/saltext/azurerm/fileserver/azurefs.py b/src/saltext/azurerm/fileserver/azurefs.py index 9420bf7..e768bfe 100644 --- a/src/saltext/azurerm/fileserver/azurefs.py +++ b/src/saltext/azurerm/fileserver/azurefs.py @@ -44,6 +44,7 @@ Do not include the leading ? for sas_token if generated from the web """ + import base64 import logging import os diff --git a/src/saltext/azurerm/loader.py b/src/saltext/azurerm/loader.py index 0b81ee7..3299ae0 100644 --- a/src/saltext/azurerm/loader.py +++ b/src/saltext/azurerm/loader.py @@ -2,6 +2,7 @@ Define the required entry-points functions in order for Salt to know what and from where it should load this extension's loaders """ + from . import PACKAGE_ROOT # pylint: disable=unused-import diff --git a/src/saltext/azurerm/modules/azurerm_compute.py b/src/saltext/azurerm/modules/azurerm_compute.py index 9738b96..6d2f07b 100644 --- a/src/saltext/azurerm/modules/azurerm_compute.py +++ b/src/saltext/azurerm/modules/azurerm_compute.py @@ -33,10 +33,10 @@ * ``AZURE_US_GOV_CLOUD`` * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging - # Azure libs HAS_LIBS = False try: diff --git a/src/saltext/azurerm/modules/azurerm_compute_availability_set.py b/src/saltext/azurerm/modules/azurerm_compute_availability_set.py index a03e410..87ac655 100644 --- a/src/saltext/azurerm/modules/azurerm_compute_availability_set.py +++ b/src/saltext/azurerm/modules/azurerm_compute_availability_set.py @@ -33,6 +33,7 @@ * ``AZURE_US_GOV_CLOUD`` * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging @@ -42,11 +43,9 @@ HAS_LIBS = False try: import azure.mgmt.compute.models # pylint: disable=unused-import - from azure.core.exceptions import ( - ResourceNotFoundError, - HttpResponseError, - SerializationError, - ) + from azure.core.exceptions import HttpResponseError + from azure.core.exceptions import ResourceNotFoundError + from azure.core.exceptions import SerializationError HAS_LIBS = True except ImportError: diff --git a/src/saltext/azurerm/modules/azurerm_compute_disk.py b/src/saltext/azurerm/modules/azurerm_compute_disk.py index 1825192..e6c48c3 100644 --- a/src/saltext/azurerm/modules/azurerm_compute_disk.py +++ b/src/saltext/azurerm/modules/azurerm_compute_disk.py @@ -33,6 +33,7 @@ * ``AZURE_US_GOV_CLOUD`` * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging @@ -42,9 +43,7 @@ HAS_LIBS = False try: import azure.mgmt.compute.models # pylint: disable=unused-import - from azure.core.exceptions import ( - HttpResponseError, - ) + from azure.core.exceptions import HttpResponseError HAS_LIBS = True except ImportError: diff --git a/src/saltext/azurerm/modules/azurerm_compute_image.py b/src/saltext/azurerm/modules/azurerm_compute_image.py index e6bbb64..9025c80 100644 --- a/src/saltext/azurerm/modules/azurerm_compute_image.py +++ b/src/saltext/azurerm/modules/azurerm_compute_image.py @@ -33,6 +33,7 @@ * ``AZURE_US_GOV_CLOUD`` * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging @@ -42,10 +43,8 @@ HAS_LIBS = False try: import azure.mgmt.compute.models # pylint: disable=unused-import - from azure.core.exceptions import ( - HttpResponseError, - SerializationError, - ) + from azure.core.exceptions import HttpResponseError + from azure.core.exceptions import SerializationError from azure.mgmt.core.tools import is_valid_resource_id HAS_LIBS = True diff --git a/src/saltext/azurerm/modules/azurerm_compute_virtual_machine.py b/src/saltext/azurerm/modules/azurerm_compute_virtual_machine.py index 0599fc7..eee73ef 100644 --- a/src/saltext/azurerm/modules/azurerm_compute_virtual_machine.py +++ b/src/saltext/azurerm/modules/azurerm_compute_virtual_machine.py @@ -33,6 +33,7 @@ * ``AZURE_US_GOV_CLOUD`` * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging import os @@ -43,12 +44,11 @@ HAS_LIBS = False try: import azure.mgmt.compute.models # pylint: disable=unused-import - from azure.core.exceptions import ( - ResourceNotFoundError, - HttpResponseError, - SerializationError, - ) - from azure.mgmt.core.tools import is_valid_resource_id, parse_resource_id + from azure.core.exceptions import HttpResponseError + from azure.core.exceptions import ResourceNotFoundError + from azure.core.exceptions import SerializationError + from azure.mgmt.core.tools import is_valid_resource_id + from azure.mgmt.core.tools import parse_resource_id HAS_LIBS = True except ImportError: @@ -109,7 +109,7 @@ def create_or_update( host_group=None, extensions_time_budget=None, **kwargs, -): +): # pylint: disable=too-many-arguments """ .. versionadded:: 2.1.0 @@ -660,7 +660,7 @@ def create_or_update( "password", ) connection_profile = {x: kwargs[x] for x in auth_kwargs if x in kwargs} - is_linux = True if result["storage_profile"]["os_disk"]["os_type"] == "Linux" else False + is_linux = result["storage_profile"]["os_disk"]["os_type"] == "Linux" extension_info = {} # attach custom script extension for userdata diff --git a/src/saltext/azurerm/modules/azurerm_compute_virtual_machine_extension.py b/src/saltext/azurerm/modules/azurerm_compute_virtual_machine_extension.py index 44bf771..c89fe05 100644 --- a/src/saltext/azurerm/modules/azurerm_compute_virtual_machine_extension.py +++ b/src/saltext/azurerm/modules/azurerm_compute_virtual_machine_extension.py @@ -33,6 +33,7 @@ * ``AZURE_US_GOV_CLOUD`` * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging @@ -42,9 +43,7 @@ HAS_LIBS = False try: import azure.mgmt.compute.models # pylint: disable=unused-import - from azure.core.exceptions import ( - HttpResponseError, - ) + from azure.core.exceptions import HttpResponseError HAS_LIBS = True except ImportError: diff --git a/src/saltext/azurerm/modules/azurerm_compute_virtual_machine_image.py b/src/saltext/azurerm/modules/azurerm_compute_virtual_machine_image.py index b283d43..f6ef27c 100644 --- a/src/saltext/azurerm/modules/azurerm_compute_virtual_machine_image.py +++ b/src/saltext/azurerm/modules/azurerm_compute_virtual_machine_image.py @@ -33,6 +33,7 @@ * ``AZURE_US_GOV_CLOUD`` * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging diff --git a/src/saltext/azurerm/modules/azurerm_dns.py b/src/saltext/azurerm/modules/azurerm_dns.py index e12b362..f1f5ee5 100644 --- a/src/saltext/azurerm/modules/azurerm_dns.py +++ b/src/saltext/azurerm/modules/azurerm_dns.py @@ -37,6 +37,7 @@ * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging @@ -47,7 +48,9 @@ try: import azure.mgmt.dns.models # pylint: disable=unused-import import azure.mgmt.privatedns.models # pylint: disable=unused-import - from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, SerializationError + from azure.core.exceptions import HttpResponseError + from azure.core.exceptions import ResourceNotFoundError + from azure.core.exceptions import SerializationError HAS_LIBS = True except ImportError: diff --git a/src/saltext/azurerm/modules/azurerm_keyvault_key.py b/src/saltext/azurerm/modules/azurerm_keyvault_key.py index 8537c36..ac2d3fc 100644 --- a/src/saltext/azurerm/modules/azurerm_keyvault_key.py +++ b/src/saltext/azurerm/modules/azurerm_keyvault_key.py @@ -34,6 +34,7 @@ * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import datetime import logging @@ -43,13 +44,11 @@ # Azure libs HAS_LIBS = False try: + from azure.core.exceptions import HttpResponseError + from azure.core.exceptions import ResourceExistsError + from azure.core.exceptions import ResourceNotFoundError + from azure.core.exceptions import SerializationError from azure.keyvault.keys import KeyClient - from azure.core.exceptions import ( - ResourceNotFoundError, - HttpResponseError, - ResourceExistsError, - SerializationError, - ) HAS_LIBS = True except ImportError: diff --git a/src/saltext/azurerm/modules/azurerm_keyvault_secret.py b/src/saltext/azurerm/modules/azurerm_keyvault_secret.py index 8fc8596..a173e39 100644 --- a/src/saltext/azurerm/modules/azurerm_keyvault_secret.py +++ b/src/saltext/azurerm/modules/azurerm_keyvault_secret.py @@ -34,6 +34,7 @@ * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import datetime import logging @@ -43,13 +44,11 @@ # Azure libs HAS_LIBS = False try: + from azure.core.exceptions import HttpResponseError + from azure.core.exceptions import ResourceExistsError + from azure.core.exceptions import ResourceNotFoundError + from azure.core.exceptions import SerializationError from azure.keyvault.secrets import SecretClient - from azure.core.exceptions import ( - ResourceNotFoundError, - HttpResponseError, - ResourceExistsError, - SerializationError, - ) HAS_LIBS = True except ImportError: diff --git a/src/saltext/azurerm/modules/azurerm_keyvault_vault.py b/src/saltext/azurerm/modules/azurerm_keyvault_vault.py index 0fa89ee..29f8941 100644 --- a/src/saltext/azurerm/modules/azurerm_keyvault_vault.py +++ b/src/saltext/azurerm/modules/azurerm_keyvault_vault.py @@ -34,6 +34,7 @@ * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging @@ -43,7 +44,8 @@ HAS_LIBS = False try: import azure.mgmt.keyvault.models # pylint: disable=unused-import - from azure.core.exceptions import HttpResponseError, ResourceNotFoundError + from azure.core.exceptions import HttpResponseError + from azure.core.exceptions import ResourceNotFoundError HAS_LIBS = True except ImportError: diff --git a/src/saltext/azurerm/modules/azurerm_network.py b/src/saltext/azurerm/modules/azurerm_network.py index b506856..881a1eb 100644 --- a/src/saltext/azurerm/modules/azurerm_network.py +++ b/src/saltext/azurerm/modules/azurerm_network.py @@ -33,18 +33,22 @@ * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging import salt.config # pylint: disable=import-error, unused-import import salt.loader # pylint: disable=import-error, unused-import + import saltext.azurerm.utils.azurerm # Azure libs HAS_LIBS = False try: import azure.mgmt.network.models # pylint: disable=unused-import - from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, SerializationError + from azure.core.exceptions import HttpResponseError + from azure.core.exceptions import ResourceNotFoundError + from azure.core.exceptions import SerializationError from azure.mgmt.core.tools import is_valid_resource_id HAS_LIBS = True @@ -1500,10 +1504,9 @@ def network_interface_create_or_update( errmsg = "The provided Backend Pool ID is not a valid resource ID string." log.error(errmsg) - elif ( - "load_balancer_name" - and "backend_address_pool_name" - in ipconfig["load_balancer_backend_address_pools"][idx] + elif all( + key in ipconfig["load_balancer_backend_address_pools"][idx] + for key in ("load_balancer_name", "backend_address_pool_name") ): try: lbbep_data = ( diff --git a/src/saltext/azurerm/modules/azurerm_resource.py b/src/saltext/azurerm/modules/azurerm_resource.py index e8f8e2a..e47d1fc 100644 --- a/src/saltext/azurerm/modules/azurerm_resource.py +++ b/src/saltext/azurerm/modules/azurerm_resource.py @@ -33,19 +33,23 @@ * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging from json.decoder import JSONDecodeError import salt.utils.files # pylint: disable=import-error import salt.utils.json # pylint: disable=import-error + import saltext.azurerm.utils.azurerm # Azure libs HAS_LIBS = False try: import azure.mgmt.resource.resources.models # pylint: disable=unused-import - from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, SerializationError + from azure.core.exceptions import HttpResponseError + from azure.core.exceptions import ResourceNotFoundError + from azure.core.exceptions import SerializationError HAS_LIBS = True except ImportError: @@ -939,12 +943,10 @@ def policy_assignment_create(name, scope, definition_name, **kwargs): # Delete this section when the ticket above is resolved. # BEGIN definition_list = policy_definitions_list(**kwargs) - if definition_name in definition_list: - definition = definition_list[definition_name] - else: - definition = { - "error": f'The policy definition named "{definition_name}" could not be found.' - } + definition = definition_list.get( + definition_name, + {"error": f'The policy definition named "{definition_name}" could not be found.'}, + ) # END if "error" not in definition: diff --git a/src/saltext/azurerm/states/azurerm_compute.py b/src/saltext/azurerm/states/azurerm_compute.py index 04f0704..d9a2c09 100644 --- a/src/saltext/azurerm/states/azurerm_compute.py +++ b/src/saltext/azurerm/states/azurerm_compute.py @@ -73,6 +73,7 @@ - connection_auth: {{ profile }} """ + # Python libs import logging @@ -99,7 +100,7 @@ def availability_set_present( virtual_machines=None, sku=None, connection_auth=None, - **kwargs + **kwargs, ): """ .. versionadded:: 2019.2.0 diff --git a/src/saltext/azurerm/states/azurerm_compute_availability_set.py b/src/saltext/azurerm/states/azurerm_compute_availability_set.py index d8bae61..67456da 100644 --- a/src/saltext/azurerm/states/azurerm_compute_availability_set.py +++ b/src/saltext/azurerm/states/azurerm_compute_availability_set.py @@ -55,6 +55,7 @@ """ + # Python libs import logging @@ -233,10 +234,10 @@ def present( ret["comment"] = f"Availability set {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create availability set {}! ({})".format( # pylint: disable=consider-using-f-string - name, aset.get("error") + ret["comment"] = ( + "Failed to create availability set {}! ({})".format( # pylint: disable=consider-using-f-string + name, aset.get("error") + ) ) return ret diff --git a/src/saltext/azurerm/states/azurerm_compute_virtual_machine.py b/src/saltext/azurerm/states/azurerm_compute_virtual_machine.py index 1921718..48627c7 100644 --- a/src/saltext/azurerm/states/azurerm_compute_virtual_machine.py +++ b/src/saltext/azurerm/states/azurerm_compute_virtual_machine.py @@ -51,6 +51,7 @@ password: 123pass """ + # Python libs import logging @@ -120,7 +121,7 @@ def present( tags=None, connection_auth=None, **kwargs, -): +): # pylint: disable=too-many-arguments """ .. versionadded:: 2.1.0 @@ -622,10 +623,10 @@ def present( ret["comment"] = f"Virtual machine {name} has been {action}d." return ret - ret[ - "comment" - ] = "Failed to {} virtual machine {}! ({})".format( # pylint: disable=consider-using-f-string - action, name, virt_mach.get("error") + ret["comment"] = ( + "Failed to {} virtual machine {}! ({})".format( # pylint: disable=consider-using-f-string + action, name, virt_mach.get("error") + ) ) if not ret["result"]: ret["changes"] = {} diff --git a/src/saltext/azurerm/states/azurerm_dns.py b/src/saltext/azurerm/states/azurerm_dns.py index cadb9be..b27af16 100644 --- a/src/saltext/azurerm/states/azurerm_dns.py +++ b/src/saltext/azurerm/states/azurerm_dns.py @@ -95,6 +95,7 @@ - connection_auth: {{ profile }} """ + import logging import salt.utils.dictdiffer # pylint: disable=import-error @@ -298,10 +299,10 @@ def zone_present( ret["comment"] = f"DNS zone {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create DNS zone {}! ({})".format( # pylint: disable=consider-using-f-string - name, zone.get("error") + ret["comment"] = ( + "Failed to create DNS zone {}! ({})".format( # pylint: disable=consider-using-f-string + name, zone.get("error") + ) ) return ret @@ -562,10 +563,10 @@ def record_set_present( continue if record_str[-1] != "s": if not isinstance(record, dict): - ret[ - "comment" - ] = "{} record information must be specified as a dictionary!".format( # pylint: disable=consider-using-f-string - record_str + ret["comment"] = ( + "{} record information must be specified as a dictionary!".format( # pylint: disable=consider-using-f-string + record_str + ) ) return ret for key, val in record.items(): @@ -573,10 +574,10 @@ def record_set_present( ret["changes"] = {"new": {record_str: record}} elif record_str[-1] == "s": if not isinstance(record, list): - ret[ - "comment" - ] = "{} record information must be specified as a list of dictionaries!".format( # pylint: disable=consider-using-f-string - record_str + ret["comment"] = ( + "{} record information must be specified as a list of dictionaries!".format( # pylint: disable=consider-using-f-string + record_str + ) ) return ret local, remote = (sorted(config) for config in (record, rec_set[record_str])) @@ -657,10 +658,10 @@ def record_set_present( ret["comment"] = f"Record set {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create record set {}! ({})".format( # pylint: disable=consider-using-f-string - name, rec_set.get("error") + ret["comment"] = ( + "Failed to create record set {}! ({})".format( # pylint: disable=consider-using-f-string + name, rec_set.get("error") + ) ) return ret diff --git a/src/saltext/azurerm/states/azurerm_keyvault_key.py b/src/saltext/azurerm/states/azurerm_keyvault_key.py index 52b2d37..3560017 100644 --- a/src/saltext/azurerm/states/azurerm_keyvault_key.py +++ b/src/saltext/azurerm/states/azurerm_keyvault_key.py @@ -35,6 +35,7 @@ * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging @@ -108,9 +109,9 @@ def present( action = "create" if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via acct or connection_auth dictionary!" + ret["comment"] = ( + "Connection information must be specified via acct or connection_auth dictionary!" + ) return ret key = __salt__["azurerm_keyvault_key.get_key"]( @@ -242,9 +243,9 @@ def absent(name, vault_url, connection_auth=None): ret = {"name": name, "result": False, "comment": "", "changes": {}} if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via acct or connection_auth dictionary!" + ret["comment"] = ( + "Connection information must be specified via acct or connection_auth dictionary!" + ) return ret key = __salt__["azurerm_keyvault_key.get_key"]( diff --git a/src/saltext/azurerm/states/azurerm_keyvault_secret.py b/src/saltext/azurerm/states/azurerm_keyvault_secret.py index 6eba502..6cd6a2d 100644 --- a/src/saltext/azurerm/states/azurerm_keyvault_secret.py +++ b/src/saltext/azurerm/states/azurerm_keyvault_secret.py @@ -35,6 +35,7 @@ * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging @@ -105,9 +106,9 @@ def present( action = "create" if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via acct or connection_auth dictionary!" + ret["comment"] = ( + "Connection information must be specified via acct or connection_auth dictionary!" + ) return ret secret = __salt__["azurerm_keyvault_secret.get_secret"]( @@ -216,10 +217,10 @@ def present( ret["comment"] = f"Secret {name} has been {action}d." return ret - ret[ - "comment" - ] = "Failed to {} Secret {}! ({})".format( # pylint: disable=consider-using-f-string - action, name, secret.get("error") + ret["comment"] = ( + "Failed to {} Secret {}! ({})".format( # pylint: disable=consider-using-f-string + action, name, secret.get("error") + ) ) if not ret["result"]: ret["changes"] = {} @@ -259,11 +260,12 @@ def absent(name, vault_url, purge=False, wait=False, connection_auth=None): """ ret = {"name": name, "result": False, "comment": "", "changes": {}} action = "delete" + deleted = False if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via acct or connection_auth dictionary!" + ret["comment"] = ( + "Connection information must be specified via acct or connection_auth dictionary!" + ) return ret secret = __salt__["azurerm_keyvault_secret.get_secret"]( diff --git a/src/saltext/azurerm/states/azurerm_keyvault_vault.py b/src/saltext/azurerm/states/azurerm_keyvault_vault.py index db41fdf..51774b4 100644 --- a/src/saltext/azurerm/states/azurerm_keyvault_vault.py +++ b/src/saltext/azurerm/states/azurerm_keyvault_vault.py @@ -35,6 +35,7 @@ * ``AZURE_GERMAN_CLOUD`` """ + # Python libs import logging from operator import itemgetter @@ -377,10 +378,10 @@ def present( ret["comment"] = f"Key Vault {name} has been {action}d." return ret - ret[ - "comment" - ] = "Failed to {} Key Vault {}! ({})".format( # pylint: disable=consider-using-f-string - action, name, vault.get("error") + ret["comment"] = ( + "Failed to {} Key Vault {}! ({})".format( # pylint: disable=consider-using-f-string + action, name, vault.get("error") + ) ) if not ret["result"]: ret["changes"] = {} diff --git a/src/saltext/azurerm/states/azurerm_network.py b/src/saltext/azurerm/states/azurerm_network.py index f416c9d..7ad0e54 100644 --- a/src/saltext/azurerm/states/azurerm_network.py +++ b/src/saltext/azurerm/states/azurerm_network.py @@ -75,9 +75,11 @@ - connection_auth: {{ profile }} """ + import logging import salt.utils.dictdiffer # pylint: disable=import-error + import saltext.azurerm.utils.azurerm __virtualname__ = "azurerm_network" @@ -240,10 +242,10 @@ def virtual_network_present( ret["comment"] = f"Virtual network {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create virtual network {}! ({})".format( # pylint: disable=consider-using-f-string - name, vnet.get("error") + ret["comment"] = ( + "Failed to create virtual network {}! ({})".format( # pylint: disable=consider-using-f-string + name, vnet.get("error") + ) ) return ret @@ -436,10 +438,10 @@ def subnet_present( ret["comment"] = f"Subnet {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create subnet {}! ({})".format( # pylint: disable=consider-using-f-string - name, snet.get("error") + ret["comment"] = ( + "Failed to create subnet {}! ({})".format( # pylint: disable=consider-using-f-string + name, snet.get("error") + ) ) return ret @@ -585,10 +587,10 @@ def network_security_group_present( ) if comp_ret.get("comment"): - ret[ - "comment" - ] = '"security_rules" {}'.format( # pylint: disable=consider-using-f-string - comp_ret["comment"] + ret["comment"] = ( + '"security_rules" {}'.format( # pylint: disable=consider-using-f-string + comp_ret["comment"] + ) ) return ret @@ -637,10 +639,10 @@ def network_security_group_present( ret["comment"] = f"Network security group {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create network security group {}! ({})".format( # pylint: disable=consider-using-f-string - name, nsg.get("error") + ret["comment"] = ( + "Failed to create network security group {}! ({})".format( # pylint: disable=consider-using-f-string + name, nsg.get("error") + ) ) return ret @@ -826,10 +828,10 @@ def security_rule_present( for params in exclusive_params: # pylint: disable=eval-used if not eval(params[0]) and not eval(params[1]): - ret[ - "comment" - ] = "Either the {} or {} parameter must be provided!".format( # pylint: disable=consider-using-f-string - params[0], params[1] + ret["comment"] = ( + "Either the {} or {} parameter must be provided!".format( # pylint: disable=consider-using-f-string + params[0], params[1] + ) ) return ret # pylint: disable=eval-used @@ -1024,10 +1026,10 @@ def security_rule_present( ret["comment"] = f"Security rule {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create security rule {}! ({})".format( # pylint: disable=consider-using-f-string - name, rule.get("error") + ret["comment"] = ( + "Failed to create security rule {}! ({})".format( # pylint: disable=consider-using-f-string + name, rule.get("error") + ) ) return ret @@ -1302,10 +1304,10 @@ def load_balancer_present( ) if comp_ret.get("comment"): - ret[ - "comment" - ] = '"frontend_ip_configurations" {}'.format( # pylint: disable=consider-using-f-string - comp_ret["comment"] + ret["comment"] = ( + '"frontend_ip_configurations" {}'.format( # pylint: disable=consider-using-f-string + comp_ret["comment"] + ) ) return ret @@ -1319,10 +1321,10 @@ def load_balancer_present( ) if comp_ret.get("comment"): - ret[ - "comment" - ] = '"backend_address_pools" {}'.format( # pylint: disable=consider-using-f-string - comp_ret["comment"] + ret["comment"] = ( + '"backend_address_pools" {}'.format( # pylint: disable=consider-using-f-string + comp_ret["comment"] + ) ) return ret @@ -1353,10 +1355,10 @@ def load_balancer_present( ) if comp_ret.get("comment"): - ret[ - "comment" - ] = '"load_balancing_rules" {}'.format( # pylint: disable=consider-using-f-string - comp_ret["comment"] + ret["comment"] = ( + '"load_balancing_rules" {}'.format( # pylint: disable=consider-using-f-string + comp_ret["comment"] + ) ) return ret @@ -1372,10 +1374,10 @@ def load_balancer_present( ) if comp_ret.get("comment"): - ret[ - "comment" - ] = '"inbound_nat_rules" {}'.format( # pylint: disable=consider-using-f-string - comp_ret["comment"] + ret["comment"] = ( + '"inbound_nat_rules" {}'.format( # pylint: disable=consider-using-f-string + comp_ret["comment"] + ) ) return ret @@ -1391,10 +1393,10 @@ def load_balancer_present( ) if comp_ret.get("comment"): - ret[ - "comment" - ] = '"inbound_nat_pools" {}'.format( # pylint: disable=consider-using-f-string - comp_ret["comment"] + ret["comment"] = ( + '"inbound_nat_pools" {}'.format( # pylint: disable=consider-using-f-string + comp_ret["comment"] + ) ) return ret @@ -1410,10 +1412,10 @@ def load_balancer_present( ) if comp_ret.get("comment"): - ret[ - "comment" - ] = '"outbound_nat_rules" {}'.format( # pylint: disable=consider-using-f-string - comp_ret["comment"] + ret["comment"] = ( + '"outbound_nat_rules" {}'.format( # pylint: disable=consider-using-f-string + comp_ret["comment"] + ) ) return ret @@ -1475,10 +1477,10 @@ def load_balancer_present( ret["comment"] = f"Load balancer {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create load balancer {}! ({})".format( # pylint: disable=consider-using-f-string - name, load_bal.get("error") + ret["comment"] = ( + "Failed to create load balancer {}! ({})".format( # pylint: disable=consider-using-f-string + name, load_bal.get("error") + ) ) return ret @@ -1727,10 +1729,10 @@ def public_ip_address_present( ret["comment"] = f"Public IP address {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create public IP address {}! ({})".format( # pylint: disable=consider-using-f-string - name, pub_ip.get("error") + ret["comment"] = ( + "Failed to create public IP address {}! ({})".format( # pylint: disable=consider-using-f-string + name, pub_ip.get("error") + ) ) return ret @@ -1981,10 +1983,10 @@ def network_interface_present( ) if comp_ret.get("comment"): - ret[ - "comment" - ] = '"ip_configurations" {}'.format( # pylint: disable=consider-using-f-string - comp_ret["comment"] + ret["comment"] = ( + '"ip_configurations" {}'.format( # pylint: disable=consider-using-f-string + comp_ret["comment"] + ) ) return ret @@ -2048,10 +2050,10 @@ def network_interface_present( ret["comment"] = f"Network interface {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create network interface {}! ({})".format( # pylint: disable=consider-using-f-string - name, iface.get("error") + ret["comment"] = ( + "Failed to create network interface {}! ({})".format( # pylint: disable=consider-using-f-string + name, iface.get("error") + ) ) return ret @@ -2252,10 +2254,10 @@ def route_table_present( ret["comment"] = f"Route table {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create route table {}! ({})".format( # pylint: disable=consider-using-f-string - name, rt_tbl.get("error") + ret["comment"] = ( + "Failed to create route table {}! ({})".format( # pylint: disable=consider-using-f-string + name, rt_tbl.get("error") + ) ) return ret @@ -2444,10 +2446,10 @@ def route_present( ret["comment"] = f"Route {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create route {}! ({})".format( # pylint: disable=consider-using-f-string - name, route.get("error") + ret["comment"] = ( + "Failed to create route {}! ({})".format( # pylint: disable=consider-using-f-string + name, route.get("error") + ) ) return ret diff --git a/src/saltext/azurerm/states/azurerm_resource.py b/src/saltext/azurerm/states/azurerm_resource.py index 9872003..a019d88 100644 --- a/src/saltext/azurerm/states/azurerm_resource.py +++ b/src/saltext/azurerm/states/azurerm_resource.py @@ -69,6 +69,7 @@ - connection_auth: {{ profile }} """ + import json import logging @@ -181,10 +182,10 @@ def resource_group_present( ret["changes"] = {"old": {}, "new": group} return ret - ret[ - "comment" - ] = "Failed to create resource group {}! ({})".format( # pylint: disable=consider-using-f-string - name, group.get("error") + ret["comment"] = ( + "Failed to create resource group {}! ({})".format( # pylint: disable=consider-using-f-string + name, group.get("error") + ) ) return ret @@ -360,15 +361,15 @@ def policy_definition_present( return ret if not policy_rule and not policy_rule_json and not policy_rule_file: - ret[ - "comment" - ] = 'One of "policy_rule", "policy_rule_json", or "policy_rule_file" is required!' + ret["comment"] = ( + 'One of "policy_rule", "policy_rule_json", or "policy_rule_file" is required!' + ) return ret if sum(x is not None for x in [policy_rule, policy_rule_json, policy_rule_file]) > 1: - ret[ - "comment" - ] = 'Only one of "policy_rule", "policy_rule_json", or "policy_rule_file" is allowed!' + ret["comment"] = ( + 'Only one of "policy_rule", "policy_rule_json", or "policy_rule_file" is allowed!' + ) return ret if (policy_rule_json or policy_rule_file) and ( @@ -406,10 +407,10 @@ def policy_definition_present( **kwargs, ) except Exception as exc: # pylint: disable=broad-except - ret[ - "comment" - ] = 'Unable to locate policy rule file "{}"! ({})'.format( # pylint: disable=consider-using-f-string - policy_rule_file, exc + ret["comment"] = ( + 'Unable to locate policy rule file "{}"! ({})'.format( # pylint: disable=consider-using-f-string + policy_rule_file, exc + ) ) return ret @@ -421,10 +422,10 @@ def policy_definition_present( with salt.utils.files.fopen(sfn, "r") as prf: temp_rule = json.load(prf) except Exception as exc: # pylint: disable=broad-except - ret[ - "comment" - ] = 'Unable to load policy rule file "{}"! ({})'.format( # pylint: disable=consider-using-f-string - policy_rule_file, exc + ret["comment"] = ( + 'Unable to load policy rule file "{}"! ({})'.format( # pylint: disable=consider-using-f-string + policy_rule_file, exc + ) ) return ret @@ -541,10 +542,10 @@ def policy_definition_present( ret["comment"] = f"Policy definition {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create policy definition {}! ({})".format( # pylint: disable=consider-using-f-string - name, policy.get("error") + ret["comment"] = ( + "Failed to create policy definition {}! ({})".format( # pylint: disable=consider-using-f-string + name, policy.get("error") + ) ) return ret @@ -742,10 +743,10 @@ def policy_assignment_present( ret["comment"] = f"Policy assignment {name} has been created." return ret - ret[ - "comment" - ] = "Failed to create policy assignment {}! ({})".format( # pylint: disable=consider-using-f-string - name, policy.get("error") + ret["comment"] = ( + "Failed to create policy assignment {}! ({})".format( # pylint: disable=consider-using-f-string + name, policy.get("error") + ) ) return ret diff --git a/src/saltext/azurerm/utils/azurerm.py b/src/saltext/azurerm/utils/azurerm.py index 9f54bb7..375518a 100644 --- a/src/saltext/azurerm/utils/azurerm.py +++ b/src/saltext/azurerm/utils/azurerm.py @@ -19,6 +19,7 @@ :platform: linux """ + import importlib import logging import os @@ -33,17 +34,13 @@ from salt.exceptions import SaltSystemExit # pylint: disable=import-error try: - from azure.identity import ( - AzureAuthorityHosts, - DefaultAzureCredential, - KnownAuthorities, - ) from azure.core.exceptions import ClientAuthenticationError - from msrestazure.azure_cloud import ( - MetadataEndpointError, - get_cloud_from_metadata_endpoint, - ) from azure.core.pipeline.policies import UserAgentPolicy + from azure.identity import AzureAuthorityHosts + from azure.identity import DefaultAzureCredential + from azure.identity import KnownAuthorities + from msrestazure.azure_cloud import MetadataEndpointError + from msrestazure.azure_cloud import get_cloud_from_metadata_endpoint HAS_AZURE = True except ImportError: @@ -197,8 +194,6 @@ def log_cloud_error(client, message, **kwargs): message, ) - return - def paged_object_to_list(paged_object): """ diff --git a/tests/conftest.py b/tests/conftest.py index d1d59a0..4aab3cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,9 @@ import os import pytest -from saltext.azurerm import PACKAGE_ROOT from saltfactories.utils import random_string +from saltext.azurerm import PACKAGE_ROOT # Reset the root logger to its default level(because salt changed it) logging.root.setLevel(logging.WARNING) @@ -30,10 +30,26 @@ def salt_factories_config(): @pytest.fixture(scope="package") -def master(salt_factories): - return salt_factories.salt_master_daemon(random_string("master-")) +def master_config(): + """ + Salt master configuration overrides for integration tests. + """ + return {} + + +@pytest.fixture(scope="package") +def master(salt_factories, master_config): + return salt_factories.salt_master_daemon(random_string("master-"), overrides=master_config) + + +@pytest.fixture(scope="package") +def minion_config(): + """ + Salt minion configuration overrides for integration tests. + """ + return {} @pytest.fixture(scope="package") -def minion(master): - return master.salt_minion_daemon(random_string("minion-")) +def minion(master, minion_config): + return master.salt_minion_daemon(random_string("minion-"), overrides=minion_config) diff --git a/tests/functional/clouds/__init__.py b/tests/functional/clouds/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/fileserver/__init__.py b/tests/functional/fileserver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/modules/__init__.py b/tests/functional/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/states/__init__.py b/tests/functional/states/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/fileserver/__init__.py b/tests/integration/fileserver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3e3fa4a..b3b6ced 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,3 +1,5 @@ +import os + import pytest import salt.config @@ -17,6 +19,7 @@ def minion_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/minion.log" + opts["conf_file"] = os.path.join(opts["conf_dir"], "minion") return opts @@ -35,6 +38,7 @@ def master_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/master.log" + opts["conf_file"] = os.path.join(opts["conf_dir"], "master") return opts @@ -54,4 +58,5 @@ def syndic_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/syndic.log" + opts["conf_file"] = os.path.join(opts["conf_dir"], "syndic") return opts diff --git a/tests/unit/fileserver/__init__.py b/tests/unit/fileserver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/utils/test_azurerm.py b/tests/unit/utils/test_azurerm.py index 868068d..475f307 100644 --- a/tests/unit/utils/test_azurerm.py +++ b/tests/unit/utils/test_azurerm.py @@ -3,12 +3,14 @@ from unittest.mock import patch import pytest -import saltext.azurerm.utils.azurerm from azure.mgmt.resource.resources import ResourceManagementClient +import saltext.azurerm.utils.azurerm + try: from salt._logging.impl import SaltLoggingClass - from salt.exceptions import SaltSystemExit, SaltInvocationError + from salt.exceptions import SaltInvocationError + from salt.exceptions import SaltSystemExit except ImportError: pass @@ -21,9 +23,8 @@ class FakeCredential: """ def get_token(self, *scopes, **kwargs): # pylint: disable=unused-argument - from azure.core.credentials import ( # pylint: disable=import-outside-toplevel - AccessToken, - ) + # pylint: disable=import-outside-toplevel + from azure.core.credentials import AccessToken return AccessToken("fake_token", 2527537086) diff --git a/tools/helpers/__init__.py b/tools/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/helpers/cmd.py b/tools/helpers/cmd.py new file mode 100644 index 0000000..9e8eb80 --- /dev/null +++ b/tools/helpers/cmd.py @@ -0,0 +1,286 @@ +""" +Polyfill for very basic ``plumbum`` functionality, no external libs required. +Makes scripts that call a lot of CLI commands much more pleasant to write. +""" + +import os +import platform +import shlex +import shutil +import subprocess +from contextlib import contextmanager +from dataclasses import dataclass +from dataclasses import field +from pathlib import Path + + +class CommandNotFound(RuntimeError): + """ + Raised when a command cannot be found in $PATH + """ + + +@dataclass(frozen=True) +class ProcessResult: + """ + The full process result, returned by ``.run`` methods. + The ``__call__`` ones just return stdout. + """ + + retcode: int + stdout: str | bytes + stderr: str | bytes + argv: tuple + + def check(self, retcode=None): + """ + Check if the retcode is expected. retcode can be a list. + """ + if retcode is None: + expected = [0] + elif not isinstance(retcode, (list, tuple)): + expected = [retcode] + if self.retcode not in expected: + raise ProcessExecutionError(self.argv, self.retcode, self.stdout, self.stderr) + + def __str__(self): + msg = [ + "Process execution result:", + f"Command: {shlex.join(self.argv)}", + f"Retcode: {self.retcode}", + "Stdout: |", + ] + msg += [" " * 10 + "| " + line for line in str(self.stdout).splitlines()] + msg.append("Stderr: |") + msg += [" " * 10 + "| " + line for line in str(self.stderr).splitlines()] + return "\n".join(msg) + + +class ProcessExecutionError(OSError): + """ + Raised by ProcessResult.check when an unexpected retcode was returned. + """ + + def __init__(self, argv, retcode, stdout, stderr): + self.argv = argv + self.retcode = retcode + if isinstance(stdout, bytes): + stdout = ascii(stdout) + if isinstance(stderr, bytes): + stderr = ascii(stderr) + self.stdout = stdout + self.stderr = stderr + + def __str__(self): + msg = [ + "Process finished with unexpected exit code", + f"Retcode: {self.retcode}", + f"Command: {shlex.join(self.argv)}", + "Stdout: |", + ] + msg += [" " * 10 + "| " + line for line in str(self.stdout).splitlines()] + msg.append("Stderr: |") + msg += [" " * 10 + "| " + line for line in str(self.stderr).splitlines()] + return "\n".join(msg) + + +class Local: + """ + Glue for command environment defaults. + Should be treated as a singleton. + + Example: + + local = Local() + + some_cmd = local["some_cmd"] + with local.cwd(some_path), local.env(FOO="bar"): + some_cmd("baz") + + # A changed $PATH requires to rediscover commands. + with local.prepend_path(important_path): + local["other_cmd"]() + with local.venv(venv_path): + local["python"]("-m", "pip", "install", "salt") + + """ + + def __init__(self): + # Explicitly cast values to strings to avoid problems on Windows + self._env = {k: str(v) for k, v in os.environ.items()} + + def __getitem__(self, exe): + """ + Return a LocalCommand in this context. + """ + return LocalCommand(exe, _local=self) + + @property + def path(self): + """ + List of paths in the context's $PATH. + """ + return self._env.get("PATH", "").split(os.pathsep) + + @contextmanager + def cwd(self, path): + """ + Set the default current working directory for commands inside this context. + """ + prev = Path(os.getcwd()) + new = prev / path + os.cwd(new) + try: + yield + finally: + os.cwd(prev) + + @contextmanager + def env(self, **kwargs): + """ + Override default env vars (sourced from the current process' environment) + for commands inside this context. + """ + prev = self._env.copy() + self._env.update((k, str(v)) for k, v in kwargs.items()) + try: + yield + finally: + self._env = prev + + @contextmanager + def path_prepend(self, *args): + """ + Prepend paths to $PATH for commands inside this context. + + Note: If you have saved a reference to an already requested command, + its $PATH will be updated, but it might not be the command + that would have been returned by a new request. + """ + new_path = [str(arg) for arg in args] + self.path + with self.env(PATH=os.pathsep.join(new_path)): + yield + + @contextmanager + def venv(self, venv_dir): + """ + Enter a Python virtual environment. Effectively prepends its bin dir + to $PATH and sets ``VIRTUAL_ENV``. + """ + venv_dir = Path(venv_dir) + if not venv_dir.is_dir() or not (venv_dir / "pyvenv.cfg").exists(): + raise ValueError(f"Not a virtual environment: {venv_dir}") + venv_bin_dir = venv_dir / "bin" + if platform.system() == "Windows": + venv_bin_dir = venv_dir / "Scripts" + with self.path_prepend(venv_bin_dir), self.env(VIRTUAL_ENV=str(venv_dir)): + yield + + +@dataclass(frozen=True) +class Executable: + """ + Utility class used to avoid repeated command lookups. + """ + + _exe: str + + def __str__(self): + return self._exe + + def __repr__(self): + return f"Executable <{self._exe}>" + + +@dataclass(frozen=True) +class Command: + """ + A command object, can be instantiated directly. Does not follow ``Local``. + """ + + exe: Executable | str + args: tuple[str, ...] = () + + def __post_init__(self): + if not isinstance(self.exe, Executable): + if not (full_exe := self._which(self.exe)): + raise CommandNotFound(self.exe) + object.__setattr__(self, "exe", Executable(full_exe)) + + def _which(self, exe): + return shutil.which(exe) + + def _get_env(self, overrides=None): + base = {k: str(v) for k, v in os.environ.items()} + base.update(overrides or {}) + return base + + def __getitem__(self, arg_or_args): + """ + Returns a subcommand with bound parameters. + + Example: + + git = Command("git")["-c", "commit.gpgsign=0"] + # ... + git("add", ".") + git("commit", "-m", "testcommit") + + """ + if not isinstance(arg_or_args, tuple): + arg_or_args = (arg_or_args,) + return type(self)(self.exe, tuple(*self.args, *arg_or_args), _local=self._local) + + def __call__(self, *args, **kwargs): + """ + Run this command and return stdout. + """ + return self.run(*args, **kwargs).stdout + + def __str__(self): + return shlex.join([self.exe] + list(self.args)) + + def __repr__(self): + return f"Command<{self.exe}, {self.args!r}>" + + def run(self, *args, check=True, env=None, **kwargs): + """ + Run this command and return the full output. + """ + kwargs.setdefault("stdout", subprocess.PIPE) + kwargs.setdefault("stderr", subprocess.PIPE) + kwargs.setdefault("text", True) + argv = [str(self.exe), *self.args, *args] + proc = subprocess.run(argv, check=False, env=self._get_env(env), **kwargs) + ret = ProcessResult( + retcode=proc.returncode, + stdout=proc.stdout, + stderr=proc.stderr, + argv=argv, + ) + if check: + ret.check() + return ret + + +@dataclass(frozen=True) +class LocalCommand(Command): + """ + Command returned by Local()["some_command"]. Follows local contexts. + """ + + _local: Local = field(kw_only=True, repr=False) + + def _which(self, exe): + return shutil.which(exe, path=self._local._env.get("PATH", "")) + + def _get_env(self, overrides=None): + base = self._local._env.copy() + base.update(overrides or {}) + return base + + +# Should be imported from here. +local = Local() +# We must assume git is installed +git = local["git"] diff --git a/tools/helpers/copier.py b/tools/helpers/copier.py new file mode 100644 index 0000000..91cf291 --- /dev/null +++ b/tools/helpers/copier.py @@ -0,0 +1,68 @@ +import sys +from functools import wraps +from pathlib import Path + +from . import prompt + +try: + # In case we have it, use it. + # It's always installed in the Copier environment, so if you ensure you + # call this via ``copier_python``, this will work. + import yaml +except ImportError: + yaml = None + + +COPIER_ANSWERS = Path(".copier-answers.yml").resolve() + + +def _needs_answers(func): + @wraps(func) + def _wrapper(*args, **kwargs): + if not COPIER_ANSWERS.exists(): + raise RuntimeError(f"Missing answers file at {COPIER_ANSWERS}") + return func(*args, **kwargs) + + return _wrapper + + +@_needs_answers +def load_answers(): + """ + Load the complete answers file. Depends on PyYAML. + """ + if not yaml: + raise RuntimeError("Missing pyyaml in environment") + with open(COPIER_ANSWERS) as f: + return yaml.safe_load(f) + + +@_needs_answers +def discover_project_name(): + """ + Specifically discover project name. No dependency. + """ + for line in COPIER_ANSWERS.read_text().splitlines(): + if line.startswith("project_name"): + return line.split(":", maxsplit=1)[1].strip() + raise RuntimeError("Failed discovering project name") + + +def finish_task(msg, success, err_exit=False, extra=None): + """ + Print final conclusion of task (migration) run in Copier. + + We usually want to exit with 0, even when something fails, + because a failing task/migration should not crash Copier. + """ + print("\n", file=sys.stderr) + if success: + prompt.pprint(f"\n ✓ {msg}", bold=True, bg=prompt.DARKGREEN, stream=sys.stderr) + elif success is None: + prompt.pprint( + f"\n ✓ {msg}", bold=True, fg=prompt.YELLOW, bg=prompt.DARKGREEN, stream=sys.stderr + ) + success = True + else: + prompt.warn(f" ✗ {msg}", extra) + raise SystemExit(int(not success and err_exit)) diff --git a/tools/helpers/git.py b/tools/helpers/git.py new file mode 100644 index 0000000..5336031 --- /dev/null +++ b/tools/helpers/git.py @@ -0,0 +1,30 @@ +from pathlib import Path + +from .cmd import git + + +def ensure_git(): + """ + Ensure the repository has been initialized. + """ + if Path(".git").is_dir(): + return + git("init", "--initial-branch", "main") + + +def list_untracked(): + """ + List untracked files. + """ + for path in git("ls-files", "-z", "-o", "--exclude-standard").split("\x00"): + if path: + yield path + + +def list_conflicted(): + """ + List files with merge conflicts. + """ + for path in git("diff", "-z", "--name-only", "--diff-filter=U", "--relative").split("\x00"): + if path: + yield path diff --git a/tools/helpers/pre_commit.py b/tools/helpers/pre_commit.py new file mode 100644 index 0000000..bcab18a --- /dev/null +++ b/tools/helpers/pre_commit.py @@ -0,0 +1,107 @@ +import re + +from . import prompt +from .cmd import ProcessExecutionError +from .cmd import git +from .cmd import local +from .git import list_untracked + +PRE_COMMIT_TEST_REGEX = re.compile( + r"^(?P[^\n]+?)\.{4,}.*(?PFailed|Passed|Skipped)$" +) +NON_IDEMPOTENT_HOOKS = ( + "trim trailing whitespace", + "mixed line ending", + "fix end of files", + "Remove Python Import Header Comments", + "Check rST doc files exist for modules/states", + "Salt extensions docstrings auto-fixes", + "Rewrite the test suite", + "Rewrite Code to be Py3.", + "isort", + "black", + "blacken-docs", +) + + +def parse_pre_commit(data): + """ + Parse pre-commit output into a list of passing hooks and a mapping of + failing hooks to their output. + """ + passing = [] + failing = {} + cur = None + for line in data.splitlines(): + if match := PRE_COMMIT_TEST_REGEX.match(line): + cur = None + if match.group("resolution") != "Failed": + passing.append(match.group("test")) + continue + cur = match.group("test") + failing[cur] = [] + continue + try: + failing[cur].append(line) + except KeyError: + # in case the parsing logic fails, let's not crash everything + continue + return passing, {test: "\n".join(output).strip() for test, output in failing.items()} + + +def check_pre_commit_rerun(data): + """ + Check if we can expect failing hooks to turn green during a rerun. + """ + _, failing = parse_pre_commit(data) + for hook in failing: + if hook.startswith(NON_IDEMPOTENT_HOOKS): + return True + return False + + +def run_pre_commit(venv, retries=2): + """ + Run pre-commit in a loop until it passes, there is no chance of + autoformatting to make it pass or a maximum number of runs is reached. + + Usually, a maximum of two runs is necessary (if a hook reformats the + output of another later one again). + """ + new_files = set() + + def _run_pre_commit_loop(retries_left): + untracked_files = set(map(str, list_untracked())) + nonlocal new_files + new_files = new_files.union(untracked_files) + # Ensure pre-commit runs on all paths. + # We don't want to git add . because this removes merge conflicts + git("add", "--intent-to-add", *untracked_files) + with local.venv(venv): + try: + local["python"]("-m", "pre_commit", "run", "--all-files") + except ProcessExecutionError as err: + if retries_left > 0 and check_pre_commit_rerun(err.stdout): + return _run_pre_commit_loop(retries_left - 1) + raise + + prompt.status( + "Running pre-commit hooks against all files. This can take a minute, please be patient" + ) + + try: + _run_pre_commit_loop(retries) + return True + except ProcessExecutionError as err: + _, failing = parse_pre_commit(err.stdout) + if failing: + msg = f"Please fix all ({len(failing)}) failing hooks" + else: + msg = f"Output: {err.stderr or err.stdout}" + prompt.warn(f"Pre-commit is failing. {msg}") + for i, failing_hook in enumerate(failing): + prompt.warn(f"✗ Failing hook ({i + 1}): {failing_hook}", failing[failing_hook]) + finally: + # Undo git add --intent-to-add to allow RenovateBot to detect new files correctly + git("restore", "--staged", *new_files) + return False diff --git a/tools/helpers/prompt.py b/tools/helpers/prompt.py new file mode 100644 index 0000000..d0f38c3 --- /dev/null +++ b/tools/helpers/prompt.py @@ -0,0 +1,51 @@ +import platform +import sys + +DARKGREEN = (0, 100, 0) +DARKRED = (139, 0, 0) +YELLOW = (255, 255, 0) + + +def ensure_utf8(): + """ + On Windows, ensure stdout/stderr output uses UTF-8 encoding. + """ + if platform.system() != "Windows": + return + for stream in (sys.stdout, sys.stderr): + if stream.encoding != "utf-8": + stream.reconfigure(encoding="utf-8") + + +def pprint(msg, bold=False, fg=None, bg=None, stream=None): + """ + Ugly helper for printing a bit more fancy output. + Stand-in for questionary/prompt_toolkit. + """ + out = "" + if bold: + out += "\033[1m" + if fg: + red, green, blue = fg + out += f"\033[38;2;{red};{green};{blue}m" + if bg: + red, green, blue = bg + out += f"\033[48;2;{red};{green};{blue}m" + out += msg + if bold or fg or bg: + out += "\033[0m" + print(out, file=stream or sys.stdout) + + +def status(msg, message=None): + out = f"\n → {msg}" + pprint(out, bold=True, fg=DARKGREEN, stream=sys.stderr) + if message: + pprint(message, stream=sys.stderr) + + +def warn(header, message=None): + out = f"\n{header}" + pprint(out, bold=True, bg=DARKRED, stream=sys.stderr) + if message: + pprint(message, stream=sys.stderr) diff --git a/tools/helpers/venv.py b/tools/helpers/venv.py new file mode 100644 index 0000000..bf04216 --- /dev/null +++ b/tools/helpers/venv.py @@ -0,0 +1,96 @@ +from pathlib import Path + +from . import prompt +from .cmd import CommandNotFound +from .cmd import local +from .copier import discover_project_name + +# Should follow the version used for relenv packages, see +# https://github.com/saltstack/salt/blob/master/cicd/shared-gh-workflows-context.yml +RECOMMENDED_PYVER = "3.10" +# For discovery of existing virtual environment, descending priority. +VENV_DIRS = ( + ".venv", + "venv", + ".env", + "env", +) + + +try: + uv = local["uv"] +except CommandNotFound: + uv = None + + +def is_venv(path): + if (venv_path := Path(path)).is_dir and (venv_path / "pyvenv.cfg").exists(): + return venv_path + return False + + +def discover_venv(project_root="."): + base = Path(project_root).resolve() + for name in VENV_DIRS: + if found := is_venv(base / name): + return found + raise RuntimeError(f"No venv found in {base}") + + +def create_venv(project_root=".", directory=None): + base = Path(project_root).resolve() + venv = (base / (directory or VENV_DIRS[0])).resolve() + if is_venv(venv): + raise RuntimeError(f"Venv at {venv} already exists") + prompt.status(f"Creating virtual environment at {venv}") + if uv is not None: + prompt.status("Found `uv`. Creating venv") + uv( + "venv", + "--python", + RECOMMENDED_PYVER, + f"--prompt=saltext-{discover_project_name()}", + ) + prompt.status("Installing pip into venv") + # Ensure there's still a `pip` (+ setuptools/wheel) inside the venv for compatibility + uv("venv", "--seed") + else: + prompt.status("Did not find `uv`. Falling back to `venv`") + try: + python = local[f"python{RECOMMENDED_PYVER}"] + except CommandNotFound: + python = local["python3"] + version = python("--version").split(" ")[1] + if not version.startswith(RECOMMENDED_PYVER): + raise RuntimeError( + f"No `python{RECOMMENDED_PYVER}` executable found in $PATH, exiting" + ) + python("-m", "venv", VENV_DIRS[0], f"--prompt=saltext-{discover_project_name()}") + return venv + + +def ensure_project_venv(project_root=".", reinstall=True): + exists = False + try: + venv = discover_venv(project_root) + prompt.status(f"Found existing virtual environment at {venv}") + exists = True + except RuntimeError: + venv = create_venv(project_root) + if not reinstall: + return venv + prompt.status(("Reinstalling" if exists else "Installing") + " project and dependencies") + with local.venv(venv): + if uv is not None: + uv("pip", "install", "-e", ".[dev,tests,docs]") + else: + try: + # We install uv into the virtualenv, so it might be available now. + # It speeds up this step a lot. + local["uv"]("pip", "install", "-e", ".[dev,tests,docs]") + except CommandNotFound: + local["python"]("-m", "pip", "install", "-e", ".[dev,tests,docs]") + if not exists or not (Path(project_root) / ".git" / "hooks" / "pre-commit").exists(): + prompt.status("Installing pre-commit hooks") + local["python"]("-m", "pre_commit", "install", "--install-hooks") + return venv diff --git a/tools/initialize.py b/tools/initialize.py new file mode 100644 index 0000000..1c07f68 --- /dev/null +++ b/tools/initialize.py @@ -0,0 +1,25 @@ +import sys + +from helpers import prompt +from helpers.copier import finish_task +from helpers.git import ensure_git +from helpers.venv import ensure_project_venv + +if __name__ == "__main__": + try: + prompt.ensure_utf8() + ensure_git() + venv = ensure_project_venv() + except Exception as err: # pylint: disable=broad-except + finish_task( + f"Failed initializing environment: {err}", + False, + True, + extra=( + "No worries, just follow the manual steps documented here: " + "https://salt-extensions.github.io/salt-extension-copier/topics/creation.html#first-steps" + ), + ) + if len(sys.argv) > 1 and sys.argv[1] == "--print-venv": + print(venv) + finish_task("Successfully initialized environment", True)