From cc9ce8218013041efbbb1d48a31dc1784453f2c3 Mon Sep 17 00:00:00 2001 From: Levi Naden Date: Tue, 10 Nov 2020 10:44:22 -0500 Subject: [PATCH 1/2] Update GHA to use Conda-Incubator Implements the Conda-Incubator to GitHub Actions instead of old activate script. Removes the older helper scripts. --- .github/workflows/CI.yaml | 60 +++++++------ devtools/github-actions/initialize_conda.sh | 10 --- devtools/scripts/create_conda_env.py | 95 --------------------- 3 files changed, 32 insertions(+), 133 deletions(-) delete mode 100644 devtools/github-actions/initialize_conda.sh delete mode 100644 devtools/scripts/create_conda_env.py diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 665ea866..e2707380 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -23,7 +23,7 @@ jobs: fail-fast: false matrix: os: [macOS-latest, ubuntu-latest, windows-latest] - python-version: [3.7, 3.8] + python-version: [3.8, 3.9] env: PYVER: ${{ matrix.python-version }} CI_OS: ${{ matrix.os }} @@ -46,34 +46,33 @@ jobs: run: | sudo chown -R $(id -u):$(id -g) ${CONDA} - - name: Create environment for package - shell: bash - run: | - . devtools/github-actions/initialize_conda.sh - conda activate - conda info - python devtools/scripts/create_conda_env.py -n=test -p=$PYVER devtools/conda-envs/test_env.yaml + - name: Setup Conda + uses: conda-incubator/setup-miniconda@v1 + with: + python-version: ${{ matrix.python-version }} + environment-file: devtools/conda-envs/test_env.yaml + + channels: conda-forge,defaults + + activate-environment: test + auto-update-conda: false + auto-activate-base: false + show-channel-urls: true - name: Install package - shell: bash + shell: bash -l {0} run: | - . devtools/github-actions/initialize_conda.sh - conda activate test python -m pip install . --no-deps conda list --show-channel-urls - name: Run tests (pytest) - shell: bash + shell: bash -l {0} run: | - . devtools/github-actions/initialize_conda.sh - conda activate test pytest -v --cov=$PACKAGE --cov-report=xml --color=yes --doctest-modules $PACKAGE/ - name: Run examples - shell: bash + shell: bash -l {0} run: | - . devtools/github-actions/initialize_conda.sh - conda activate test ### constant-force-optical-trap cd examples/constant-force-optical-trap python extract-data.py @@ -111,27 +110,32 @@ jobs: - name: Checkout the code uses: actions/checkout@v1 + - name: Setup Conda + uses: conda-incubator/setup-miniconda@v1 + with: + python-version: ${{ matrix.python-version }} + environment-file: devtools/conda-envs/test_env.yaml + + channels: conda-forge,defaults + + activate-environment: test + auto-update-conda: false + auto-activate-base: false + show-channel-urls: true + - name: Install linter and formatter - shell: bash + shell: bash -l {0} run: | - . devtools/github-actions/initialize_conda.sh - conda activate - python devtools/scripts/create_conda_env.py -n=test -p=$PYVER devtools/conda-envs/test_env.yaml - conda activate test conda install pylint black - name: Run pylint - shell: bash + shell: bash -l {0} run: | - . devtools/github-actions/initialize_conda.sh - conda activate test pylint $PACKAGE/ # TODO: Remove --exclude pymbar/old_mbar.py after deprecation - name: Run black check - shell: bash + shell: bash -l {0} if: always() run: | - . devtools/github-actions/initialize_conda.sh - conda activate test black --check -l 99 $PACKAGE/ examples/ --exclude pymbar/old_mbar.py diff --git a/devtools/github-actions/initialize_conda.sh b/devtools/github-actions/initialize_conda.sh deleted file mode 100644 index 91a66ec3..00000000 --- a/devtools/github-actions/initialize_conda.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -# $CI_OS is $matrix.os, as exported in GHA *.yaml -# $CONDA (miniconda installation path) is always defined in the GHA virtual environments -case ${CI_OS} in - windows*) - eval "$(${CONDA}/condabin/conda.bat shell.bash hook)";; - *) - eval "$(${CONDA}/condabin/conda shell.bash hook)";; -esac diff --git a/devtools/scripts/create_conda_env.py b/devtools/scripts/create_conda_env.py deleted file mode 100644 index bc063dc1..00000000 --- a/devtools/scripts/create_conda_env.py +++ /dev/null @@ -1,95 +0,0 @@ -import argparse -import os -import re -import glob -import shutil -import subprocess as sp -from tempfile import TemporaryDirectory -from contextlib import contextmanager -# YAML imports -try: - import yaml # PyYAML - loader = yaml.load -except ImportError: - try: - import ruamel_yaml as yaml # Ruamel YAML - except ImportError: - try: - # Load Ruamel YAML from the base conda environment - from importlib import util as import_util - CONDA_BIN = os.path.dirname(os.environ['CONDA_EXE']) - ruamel_yaml_path = glob.glob(os.path.join(CONDA_BIN, '..', - 'lib', 'python*.*', 'site-packages', - 'ruamel_yaml', '__init__.py'))[0] - # Based on importlib example, but only needs to load_module since its the whole package, not just - # a module - spec = import_util.spec_from_file_location('ruamel_yaml', ruamel_yaml_path) - yaml = spec.loader.load_module() - except (KeyError, ImportError, IndexError): - raise ImportError("No YAML parser could be found in this or the conda environment. " - "Could not find PyYAML or Ruamel YAML in the current environment, " - "AND could not find Ruamel YAML in the base conda environment through CONDA_EXE path. " - "Environment not created!") - loader = yaml.YAML(typ="safe").load # typ="safe" avoids odd typing on output - - -@contextmanager -def temp_cd(): - """Temporary CD Helper""" - cwd = os.getcwd() - with TemporaryDirectory() as td: - try: - os.chdir(td) - yield - finally: - os.chdir(cwd) - - -# Args -parser = argparse.ArgumentParser(description='Creates a conda environment from file for a given Python version.') -parser.add_argument('-n', '--name', type=str, - help='The name of the created Python environment') -parser.add_argument('-p', '--python', type=str, - help='The version of the created Python environment') -parser.add_argument('conda_file', - help='The file for the created Python environment') - -args = parser.parse_args() - -# Open the base file -with open(args.conda_file, "r") as handle: - yaml_script = loader(handle.read()) - -python_replacement_string = "python {}*".format(args.python) - -try: - for dep_index, dep_value in enumerate(yaml_script['dependencies']): - if re.match('python([ ><=*]+[0-9.*]*)?$', dep_value): # Match explicitly 'python' and its formats - yaml_script['dependencies'].pop(dep_index) - break # Making the assumption there is only one Python entry, also avoids need to enumerate in reverse -except (KeyError, TypeError): - # Case of no dependencies key, or dependencies: None - yaml_script['dependencies'] = [] -finally: - # Ensure the python version is added in. Even if the code does not need it, we assume the env does - yaml_script['dependencies'].insert(0, python_replacement_string) - -# Figure out conda path -if "CONDA_EXE" in os.environ: - conda_path = os.environ["CONDA_EXE"] -else: - conda_path = shutil.which("conda") -if conda_path is None: - raise RuntimeError("Could not find a conda binary in CONDA_EXE variable or in executable search path") - -print("CONDA ENV NAME {}".format(args.name)) -print("PYTHON VERSION {}".format(args.python)) -print("CONDA FILE NAME {}".format(args.conda_file)) -print("CONDA PATH {}".format(conda_path)) - -# Write to a temp directory which will always be cleaned up -with temp_cd(): - temp_file_name = "temp_script.yaml" - with open(temp_file_name, 'w') as f: - f.write(yaml.dump(yaml_script)) - sp.call("{} env create -n {} -f {}".format(conda_path, args.name, temp_file_name), shell=True) \ No newline at end of file From 603cc197c1c07069abfd62d21eda807c1347fb5d Mon Sep 17 00:00:00 2001 From: Levi Naden Date: Tue, 10 Nov 2020 12:06:10 -0500 Subject: [PATCH 2/2] Simplify lint CI, try to rerun and see if numexpr assert is deterministic --- .github/workflows/CI.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index e2707380..e85ca80e 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -101,10 +101,11 @@ jobs: lint-format: runs-on: ubuntu-latest + matrix: + os: [ubuntu-latest] + python-version: [3.8] env: - CI_OS: ubuntu-latest PACKAGE: "pymbar" - PYVER: "3.7" steps: - name: Checkout the code