2024 refresh

.flake8

@@ -1,4 +1,7 @@
 [flake8]
-ignore = E203, E266, E501, W503, E211, E731
-max-line-length = 88
-select = B,C,E,F,W,T4,B9
+max-line-length = 100
+select = B,C,E,F,W,T4
+exclude = cwltool/schemas
+extend-ignore = E203,E501,E704,B905,W503
+# when Python 3.10 is the minimum version, re-enable check B905 for zip + strict
+extend-select = B9

.github/workflows/ci-tests.yml

@@ -0,0 +1,115 @@
+name: CI Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+concurrency:
+  group: build-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  TOX_SKIP_MISSING_INTERPRETERS: False
+  # Rich (pip)
+  FORCE_COLOR: 1
+  # Tox
+  PY_COLORS: 1
+  # Mypy (see https://github.com/python/mypy/issues/7771)
+  TERM: xterm-color
+  MYPY_FORCE_COLOR: 1
+  MYPY_FORCE_TERMINAL_WIDTH: 200
+  # Pytest
+  PYTEST_ADDOPTS: --color=yes
+
+jobs:
+
+  tox:
+    name: Tox
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        py-ver-major: [3]
+        py-ver-minor: [8, 9, 10, 11, 12]
+        step: [lint, unit, bandit, mypy]
+
+    env:
+      py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}
+      TOXENV: ${{ format('py{0}{1}-{2}', matrix.py-ver-major, matrix.py-ver-minor, matrix.step) }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.py-semver }}
+          allow-prereleases: true
+          cache: pip
+
+      - name: Install libcurl-dev for pycurl (needed by arvados)
+        if: ${{ matrix.step == 'mypy' }}
+        run: sudo apt-get install -y --no-install-recommends libcurl4-gnutls-dev gnutls-dev
+
+      - name: Upgrade setuptools and install tox
+        run: |
+          pip install -U pip setuptools wheel
+          pip install "tox<4" "tox-gh-actions<3"
+
+      - name: MyPy cache
+        if: ${{ matrix.step == 'mypy' }}
+        uses: actions/cache@v4
+        with:
+          path: .mypy_cache/${{ env.py-semver }}
+          key: mypy-${{ env.py-semver }}
+
+      - name: Test with tox
+        run: tox
+
+      - name: Upload coverage to Codecov
+        if: ${{ matrix.step == 'unit' }}
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  tox-style:
+    name: CI linters via Tox
+
+    runs-on: ubuntu-22.04
+
+    strategy:
+      matrix:
+        step: [lintreadme, pydocstyle]
+
+    env:
+      py-semver: "3.12"
+      TOXENV: ${{ format('py312-{0}', matrix.step) }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.py-semver }}
+          cache: pip
+
+      - name: Upgrade setuptools and install tox
+        run: |
+          pip install -U pip setuptools wheel
+          pip install "tox<4" "tox-gh-actions<3"
+
+      - if: ${{ matrix.step == 'pydocstyle' && github.event_name == 'pull_request'}}
+        name: Create local branch for diff-quality for PRs
+        run: git branch ${{github.base_ref}} origin/${{github.base_ref}}
+
+      - name: Test with tox
+        run: tox

Dockerfile

@@ -12,7 +12,7 @@ RUN apt-get update && \
     sh -c 'echo deb https://oss-binaries.phusionpassenger.com/apt/passenger buster main > /etc/apt/sources.list.d/passenger.list'
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends passenger python3-setuptools build-essential python3-dev python3-pip git && \
+    apt-get install -y --no-install-recommends passenger python3-setuptools build-essential python3-dev python3-pip python3-wheel git && \
     pip3 install pip==21.1.3
 
 RUN apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev

Makefile

@@ -0,0 +1,197 @@
+# This file is part of workflow-service
+# https://github.com/common-workflow-language/workflow-service/, and is
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Contact: common-workflow-language@googlegroups.com
+
+# make format to fix most python formatting errors
+# make pylint to check Python code for enhanced compliance including naming
+#  and documentation
+# make coverage-report to check coverage of the python scripts by the tests
+
+MODULE1=wes_client
+MODULE2=wes_service
+PACKAGE=wes-service
+EXTRAS=[toil]
+
+# `SHELL=bash` doesn't work for some, so don't use BASH-isms like
+# `[[` conditional expressions.
+PYSOURCES=$(shell find $(MODULE1) -name "*.py") $(shell find $(MODULE2) -name "*.py")\
+	  $(wildcard test/*.py) $(wildcard *.py)
+DEVPKGS=build diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv \
+	wheel autoflake pyupgrade bandit auto-walrus \
+	-rlint-requirements.txt -rtest-requirements.txt -rmypy-requirements.txt
+DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
+	   python-flake8 python-mock shellcheck
+VERSION=v$(shell grep version pyproject.toml | awk -F\" '{print $2}')
+mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+UNAME_S=$(shell uname -s)
+
+## all                    : default task (install wes-service in dev mode)
+all: dev
+
+## help                   : print this help message and exit
+help: Makefile
+	@sed -n 's/^##//p' $<
+
+## cleanup                : shortcut for "make sort_imports format flake8 diff_pydocstyle_report"
+cleanup: sort_imports format flake8 diff_pydocstyle_report
+
+## install-dep            : install most of the development dependencies via pip
+install-dep: install-dependencies
+
+install-dependencies:
+	pip install -U pip setuptools wheel
+	pip install --upgrade $(DEVPKGS)
+
+## install-deb-dep        : install many of the dev dependencies via apt-get
+install-deb-dep:
+	sudo apt-get install $(DEBDEVPKGS)
+
+## install                : install the wes-service package and the scripts
+install: FORCE
+	pip install .$(EXTRAS)
+
+## dev                    : install the wes-service package in dev mode
+dev: install-dep
+	pip install -U pip setuptools wheel
+	pip install -e .$(EXTRAS)
+
+## dist                   : create a module package for distribution
+dist: FORCE
+	python -m build
+
+## clean                  : clean up all temporary / machine-generated files
+clean: FORCE
+	rm -f ${MODULE}/*.pyc tests/*.pyc
+	rm -Rf .coverage
+	rm -f diff-cover.html
+
+# Linting and code style related targets
+## sort_import            : sorting imports using isort: https://github.com/timothycrosley/isort
+sort_imports: $(PYSOURCES)
+	isort $^
+
+remove_unused_imports: $(PYSOURCES)
+	autoflake --in-place --remove-all-unused-imports $^
+
+pep257: pydocstyle
+## pydocstyle             : check Python docstring style
+pydocstyle: $(PYSOURCES) FORCE
+	pydocstyle --add-ignore=D100,D101,D102,D103 $^ || true
+
+pydocstyle_report.txt: $(PYSOURCES) FORCE
+	pydocstyle $^ > $@ 2>&1 || true
+
+## diff_pydocstyle_report : check Python docstring style for changed files only
+diff_pydocstyle_report: pydocstyle_report.txt
+	diff-quality --compare-branch=main --violations=pydocstyle --fail-under=100 $^
+
+## codespell              : check for common misspellings
+codespell:
+	codespell -w $(shell git ls-files | grep -v mypy-stubs)
+
+## format                 : check/fix all code indentation and formatting (runs black)
+format: $(PYSOURCES) FORCE
+	black $(PYSOURCES)
+
+format-check: $(PYSOURCES)
+	black --diff --check $^
+
+## pylint                 : run static code analysis on Python code
+pylint: $(PYSOURCES)
+	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
+                $^ -j0|| true
+
+pylint_report.txt: $(PYSOURCES)
+	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
+		$^ -j0> $@ || true
+
+diff_pylint_report: pylint_report.txt
+	diff-quality --compare-branch=main --violations=pylint pylint_report.txt
+
+.coverage: testcov
+
+coverage: .coverage
+	coverage report
+
+coverage.xml: .coverage
+	coverage xml
+
+coverage.html: htmlcov/index.html
+
+htmlcov/index.html: .coverage
+	coverage html
+	@echo Test coverage of the Python code is now in htmlcov/index.html
+
+coverage-report: .coverage
+	coverage report
+
+diff-cover: coverage.xml
+	diff-cover --compare-branch=main $^
+
+diff-cover.html: coverage.xml
+	diff-cover --compare-branch=main $^ --html-report $@
+
+## test                   : run the wes-service test suite
+test: $(PYSOURCES) FORCE
+	python -m pytest -rsx ${PYTEST_EXTRA}
+
+## testcov                : run the wes-service test suite and collect coverage
+testcov: $(PYSOURCES)
+	python -m pytest -rsx --cov ${PYTEST_EXTRA}
+
+sloccount.sc: $(PYSOURCES) Makefile
+	sloccount --duplicates --wide --details $^ > $@
+
+## sloccount              : count lines of code
+sloccount: $(PYSOURCES) Makefile
+	sloccount $^
+
+list-author-emails:
+	@echo 'name, E-Mail Address'
+	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
+
+mypy3: mypy
+mypy: ${PYSOURCES}
+	MYPYPATH=$$MYPYPATH:mypy-stubs mypy $^
+
+shellcheck: FORCE
+	shellcheck release-test.sh
+
+pyupgrade: $(PYSOURCES)
+	pyupgrade --exit-zero-even-if-changed --py38-plus $^
+	auto-walrus $^
+
+release-test: FORCE
+	git diff-index --quiet HEAD -- || ( echo You have uncommitted changes, please commit them and try again; false )
+	./release-test.sh
+
+release: release-test
+	. testenv2/bin/activate && \
+		pip install build && \
+		python -m build testenv2/src/${PACKAGE} && \
+		pip install twine && \
+		twine upload testenv2/src/${PACKAGE}/dist/* && \
+		git tag ${VERSION} && git push --tags
+
+flake8: $(PYSOURCES)
+	flake8 $^
+
+FORCE:
+
+# Use this to print the value of a Makefile variable
+# Example `make print-VERSION`
+# From https://www.cmcrossroads.com/article/printing-value-makefile-variable
+print-%  : ; @echo $* = $($*)

README.md

@@ -5,7 +5,7 @@ Execution Service](https://github.com/ga4gh/workflow-execution-schemas) 1.0.0 AP
 
 It provides [Arvados](https://arvados.org/) and [Toil](http://toil.ucsc-cgl.org/) backends.  It
 also works with any `cwl-runner` that supports the CWL standard command line
-interface: http://www.commonwl.org/v1.0/CommandLineTool.html#Executing_CWL_documents_as_scripts
+interface: <http://www.commonwl.org/v1.0/CommandLineTool.html#Executing_CWL_documents_as_scripts>
 
 ## Installation:
 
@@ -42,7 +42,9 @@ $ wes-client --info
 Attachments must be accessible from the filesystem.  Workflow runners may also support http URLs or other storage systems.
 
 ```
-$ wes-client --attachments="testdata/dockstore-tool-md5sum.cwl,testdata/md5sum.input" testdata/md5sum.cwl testdata/md5sum.cwl.json
+$ wes-client --host localhost:8080 --proto http                             \
+   --attachments="testdata/dockstore-tool-md5sum.cwl,testdata/md5sum.input" \
+   testdata/md5sum.cwl testdata/md5sum.cwl.json
 ```
 
 ### List workflows
@@ -89,7 +91,7 @@ $ wes-server --backend=wes_service.toil_wes --opt extra=--clean=never
 
 ```
 $ pip install cwltool
-$ wes-server --backend=wes_service.cwl_runner --opt runner=cwltool --opt extra=--logLevel=CRITICAL
+$ wes-server --backend=wes_service.cwl_runner --opt runner=cwltool --opt extra=--debug
 ```
 
 ### Pass parameters to cwl-runner