Switch to pyctdev for project maintenance (#61)

Includes:
  * packaging metadata declared in setup.cfg (including run time dependencies)
  * automatic versioning (uses autover via param)
  * build time dependencies declared in pyproject.toml
  * conda build is templated (avoids duplicating metadata) except for build time dependencies (doesn't support pyproject.toml)
  * user command (parambokeh examples) for copying examples (using pyct.cmd)
  * cleaned up packaging and testing (adds pypi+wheel, adds package testing; all test commands etc are described in tox.ini)

Also:
  * Syntax fix for python2

.appveyor.yml

@@ -1,22 +1,21 @@
 environment:
+  global:
+    CHANS_DEV: "-c pyviz/label/dev"
   matrix:
-    - PY: "3.6"
-      CONDA: "C:\\Miniconda3-x64"
-      DOIT: "doit"
     - PY: "2.7"
       CONDA: "C:\\Miniconda-x64"
-      DOIT: "doit==0.29.0"
+    - PY: "3.6"
+      CONDA: "C:\\Miniconda36-x64"
 
 install:
   - "SET PATH=%CONDA%;%CONDA%\\Scripts;%PATH%"
-  - pip install %DOIT% ioamdoit
-  - doit create_env --python=%PY%
-  - activate test-environment
-  - doit install_test_dependencies
-  - doit capture_conda_env
-  - doit develop_install
+  - "conda install -y -c pyviz/label/dev pyctdev && doit ecosystem_setup"
+  - "doit env_create %CHANS_DEV% --name=test --python=%PY%"
+  - "activate test"
+  - "doit develop_install -o examples -o tests %CHANS_DEV%"
+  - "doit env_capture"
 
 build: off
 
 test_script:
-  - doit all_tests
+  - "doit test_all"

.gitattributes

@@ -1 +1,2 @@
- __init__.py export-subst
+__init__.py export-subst
+setup.cfg export-subst

.gitignore

@@ -64,4 +64,5 @@ target/
 # Editor files
 *~
 
-.doit.db.db
+*.doit*
+

.travis.yml

@@ -10,82 +10,122 @@ sudo: false
 
 env:
   global:
-    secure: qDzvmjcR6eLAmHx0ziN3Y5M8h65+OM+eBNERQsC3WSkeaD45rEE6a8w59dBdUzZBTLKO2RP8pKlgzxBZGOBlnLbTyN4RD/q2dA+gm5c0SDWWT4kVCiogJVsrzyRHVgMDmqDu56HPbQzhsW+5Qh2JKjVkkI7GwvGZRFB9WKQC8yjW/RfLvQ7bzX6jXbaHDYg9dywjI5u7da7HZKXyt/6nhebdaI19RdQHuESUqTADFEgdCKdzrNC+NqbJ8SUnNBBmkb1ZqoQ3/p1/Z/+uDi/2ThozE7Lz6gl8HtxVRnBiwNCDvZRE8uTB79+FlW0FJCVZ7qGO8jVWlVqy+WEEFh/6s9RP2PLd73CmItfF7BzSdxdurOYfnizboqCideSasDQK7XdeDiX4fPrgqbeesn8+XJ9vhxRAmpsNdZIU+2pivgviJwUZUwKwgzt3hf23ld9GpJg60wCfQRT8hiZc3/XsUPrzDjci7hwrRESqobPcuWk6xsFPTvJsTeOSLejP1wDOGAhufLTQIys26BSwHiaj6aI4RPK3Y4sy644m4IOfVvi3d+YfyN2Q5NO7QIwk5PjQBUEdwcIi3lAk2eAIjJamlDYFCRrnwuEoFinuFlUjtqpkuVTLgonTbk9KkIYnXeP3wYERrt4uOJtxDRCojBC0FI5rDbkhC3wP8Wef4EToQ+s=
+    - PYENV_VERSION=3.6
+    - CHANS_DEV="-c pyviz/label/dev"
+    - CHANS_REL="-c pyviz"
+    - LABELS_DEV="--label dev"
+    - LABELS_REL="--label dev --label main"
+    - PKG_TEST_PYTHON="--test-python=py27 --test-python=py36"
+
+cache:
+  directories:
+    - $HOME/miniconda
+
+before_cache:
+  - rm -rf $HOME/miniconda/pkgs
+  - rm -rf $HOME/miniconda/conda-bld/*
+  - rm -rf $HOME/miniconda/envs/*/conda-bld  
 
 stages:
   - test
-  - doc
-  - name: anaconda_deploy
-    if: branch = master AND type != pull_request
+  - name: conda_dev_package
+    if: tag =~ ^v(\d+|\.)*[a-z]\d*$
+  - name: pip_dev_package
+    if: tag =~ ^v(\d+|\.)*[a-z]\d*$
+  - name: conda_package
+    if: tag =~ ^v(\d+|\.)*[^a-z]\d*$
+  - name: pip_package
+    if: tag =~ ^v(\d+|\.)*[^a-z]\d*$
+  - name: docs
+    if: tag =~ ^v(\d+|\.).*$
+
 
 jobs:
   include:
-    - &default
+
+    ########## DEVELOPER INSTALL ##########
+
+    - &conda_default
       stage: test
-      env: PYTHON_VERSION="2.7"
+      env: DESC="dev test_all"
       before_install:
-        #######################
-        ## install miniconda ##
-        - easy_install --user doit==0.29.0 ioamdoit
-        - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-            ~/Library/Python/2.7/bin/doit install_miniconda;
-          else
-            ~/.local/bin/doit install_miniconda;
-          fi
-        - rm -f .doit.db
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        #######################
-        - pip install ioamdoit
-        - doit create_env --python=$PYTHON_VERSION
+        # install doit/pyctdev and use to install miniconda...
+        - pip install pyctdev && doit miniconda_install && pip uninstall -y doit pyctdev
+        - export PATH="$HOME/miniconda/bin:$PATH" && hash -r
+        - conda config --set always_yes True
+        # ...and now install doit/pyctdev into miniconda
+        - conda install -c pyviz/label/dev pyctdev && doit ecosystem_setup
+      install:
+        - doit env_create $CHANS_DEV --python=$PYENV_VERSION
         - source activate test-environment
+        - doit develop_install -o examples -o tests $CHANS_DEV
+        - doit env_capture
+      script: doit test_all
 
-      install:
-        - doit install_test_dependencies
-        - doit capture_conda_env
-        - doit develop_install
-      script:
-        - doit all_tests
+    # python 2 flake checking typically catches python 2 syntax
+    # errors where python 3's been assumed...
+    - <<: *conda_default
+      env: DESC="py2 flakes" PYENV_VERSION=2.7
+      script: doit test_lint
+
+    ########## END-USER PACKAGES ##########
 
-    - <<: *default
-      env: PYTHON_VERSION="3.6"
+    ## dev packages
 
-    - <<: *default
-      stage: doc
-      env: PYTHON_VERSION="3.6"
+    - &pip_default
+      env: TRAVIS_NOCACHE=$TRAVIS_JOB_ID PYPI=testpypi PYPIUSER=$TPPU PYPIPASS=$TPPP
+      stage: pip_dev_package
+      before_install: pip install pyctdev && doit ecosystem=pip ecosystem_setup
       install:
-        # TODO: could (build and) use conda package; to be cleaned up
-        # once auto versioning/package building is finalized
-        - doit install_doc_dependencies
-        - doit capture_conda_env
-        - doit develop_install
+        - unset PYENV_VERSION && pyenv global 3.6 2.7
+        - doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=unit --sdist-install-build-deps
+        - doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=examples --sdist-install-build-deps
+      script: doit ecosystem=pip package_upload -u $PYPIUSER -p $PYPIPASS --pypi ${PYPI}
+
+    - &conda_pkg
+      <<: *conda_default
+      stage: conda_dev_package
+      env: DESC="" TRAVIS_NOCACHE=$TRAVIS_JOB_ID LABELS=$LABELS_DEV CHANS=$CHANS_DEV
+      install:
+        - doit package_build $CHANS $PKG_TEST_PYTHON --test-group=unit
+        - doit package_test $CHANS $PKG_TEST_PYTHON --test-group=examples --test-requires=examples
+      script: doit package_upload --token=$ANACONDA_TOKEN $LABELS
+
+    ## release packages
+
+    - <<: *pip_default
+      env: TRAVIS_NOCACHE=$TRAVIS_JOB_ID PYPI=pypi PYPIUSER=$PPU PYPIPASS=$PPP
+      stage: pip_package
+
+    - <<: *conda_pkg
+      stage: conda_package
+      env: DESC="" TRAVIS_NOCACHE=$TRAVIS_JOB_ID LABELS=$LABELS_REL CHANS=$CHANS_REL
+
+
+    ########## DOCS ##########
+
+    - <<: *conda_default
+      stage: docs
+      env: DESC="docs"
       script:
+        - doit develop_install $CHANS_DEV -o doc -o examples -c conda-forge # phantomjs still not on defaults
         - doit docs
       deploy:
         - provider: pages
           skip_cleanup: true
           github_token: $GITHUB_TOKEN
-          fqdn: parambokeh.pyviz.org
           local_dir: ./doc/_build/html
+          repo: ioam-docs/parambokeh-dev
           on:
-            # TODO: depending on what we do for auto versioning/packaging,
-            # should make this deploy somewhere for PRs etc, and only to the
-            # main site on specific tag format.
             tags: true
+            all_branches: true
+            condition: $TRAVIS_TAG =~ [0-9]+[a-z][0-9]*$
         - provider: pages
           skip_cleanup: true
           github_token: $GITHUB_TOKEN
           local_dir: ./doc/_build/html
-          repo: ioam-docs/parambokeh-master
-
-    - <<: *default
-      stage: anaconda_deploy
-      env: PYTHON_VERSION="3.6"
-      install:
-        - source deactivate
-        - conda update -q -y conda
-        - conda install -q -y conda-build anaconda-client
-      script:
-        - export VERSIONHACK=$(python -c "import subprocess;desc=subprocess.check_output(['git','describe','--long']).decode('utf8');v,commits=desc.split('-')[0:2];newv=[int(x) for x in v[1::].split('.')];newv[-1]+=1;print('.'.join(str(x) for x in newv)+'.dev'+commits)")
-        - conda build conda.recipe/
-        # only upload if package doesn't exist (as e.g. there are cron builds)
-        - anaconda show pyviz/param/$VERSIONHACK || anaconda --token $CONDA_UPLOAD_TOKEN_AGAIN upload --user pyviz --label dev $(conda build --output conda.recipe)
+          fqdn: parambokeh.pyviz.org
+          on:
+            tags: true
+            all_branches: true
+            condition: $TRAVIS_TAG =~ [0-9]+[^a-z][0-9]*$

MANIFEST.in

@@ -0,0 +1,8 @@
+include LICENSE.txt
+include README.md
+include parambokeh/.version
+graft examples
+graft parambokeh/examples
+global-exclude *.py[co]
+global-exclude *~
+global-exclude *.ipynb_checkpoints/*

conda.recipe/meta.yaml

@@ -1,28 +1,45 @@
+{% set sdata = load_setup_py_data() %}
+
 package:
   name: parambokeh
-  version: {{ os.environ.get("VERSIONHACK") }}
+  version: {{ sdata['version'] }}
 
 source:
   path: ..
 
 build:
   noarch: python
   script: python setup.py install --single-version-externally-managed --record=record.txt
+  entry_points:
+    {% for group,epoints in sdata.get("entry_points",{}).items() %}
+    {% for entry_point in epoints %}
+    - {{ entry_point }}
+    {% endfor %}
+    {% endfor %}  
 
 requirements:
-  build:
+  host:
+    # duplicates pyproject.toml (not supported in conda build)
     - python
+    - setuptools >=30.3.0
+    - param >=1.7.0
+    - pyct-core >=0.4.4
   run:
-    - python
-    - param
-    - pyviz_comms
-    - bokeh >=0.12.10
+    - python {{ sdata['python_requires'] }}
+    {% for dep in sdata.get('install_requires',{}) %}
+    - "{{ dep }}"
+    {% endfor %}
 
 test:
   imports:
     - parambokeh
+  requires:
+    {% for dep in sdata['extras_require']['tests'] %}
+    - "{{ dep }}"
+    {% endfor %}
 
 about:
-  home: https://github.com/ioam/parambokeh
-  summary: ParamBokeh provides an easy way to generate a UI for param based classes in the notebook or on bokeh server.
-  license: BSD 3-Clause
+  home: {{ sdata['url'] }}
+  summary: {{ sdata['description'] }}
+  license: {{ sdata['license'] }}
+#  license_file: {{ sdata['license_file'] }}

doc/conf.py

@@ -28,7 +28,7 @@
     'PROJECT': project,
     'DESCRIPTION': description,
     'AUTHOR': authors,
-    'WEBSITE_URL': 'https://ioam.github.io/parambokeh',
+    'WEBSITE_URL': 'https://parambokeh.pyviz.org',
     'VERSION': version,
     'NAV': _NAV,
     'LINKS': _NAV,

dodo.py

@@ -1,51 +1,14 @@
-DOIT_CONFIG = {'verbosity': 2}
+import os
+if "PYCTDEV_ECOSYSTEM" not in os.environ:
+    os.environ["PYCTDEV_ECOSYSTEM"] = "conda"
 
-from ioamdoit import *
+from pyctdev import *  # noqa: api
 
-# The aim would be to not have anything much here, but right now
-# that's not easy because of awkward installation/specification of
-# dependencies across projects.
-
-def task_install_required_dependencies():
-    return {'actions': ['conda install -y -q -c conda-forge -c pyviz param "bokeh>=0.12.10" pyviz_comms']}
-
-def task_install_test_dependencies():
-    return {
-        'actions': [
-            'conda install -y -q -c conda-forge "holoviews>=1.9.0" pandas notebook flake8 pyparsing pytest',
-            'pip install pytest-nbsmoke'],
-        'task_dep': ['install_required_dependencies']
-        }
-
-def task_install_doc_dependencies():
-    # would not need to exist if nbsite had conda package
-    return {
-        'actions': [
-            'conda install -y -q -c conda-forge notebook ipython sphinx=1.6 beautifulsoup4 graphviz selenium phantomjs',
-            'pip install https://github.com/pyviz/nbsite/archive/master.zip sphinx_ioam_theme'],
-        'task_dep': ['install_test_dependencies']
-        }
-
-def task_lint():
-    return {'actions': ['flake8 --ignore E,W parambokeh']}
-
-
-def task_lint_nb():
-    return {'actions': ['pytest --nbsmoke-lint examples/']}
-
-
-def task_test_nb():
-    return {'actions': ['pytest --nbsmoke-run examples/']}
-
-
-def task_all_tests():
-    return {'actions': [],
-            'task_dep': ['lint','lint_nb','test_nb']}
 
+############################################################
+# Website building tasks; will move out to pyct
 
 def task_docs():
-    # TODO: could do better than this, or nbsite could itself use doit
-    # (providing a dodo file for docs, or using doit internally).
     return {'actions': [
         'nbsite_nbpagebuild.py ioam parambokeh ./examples ./doc',
         'sphinx-build -b html ./doc ./doc/_build/html',

examples/apps/django2/manage.py

@@ -11,5 +11,5 @@
             "Couldn't import Django. Are you sure it's installed and "
             "available on your PYTHONPATH environment variable? Did you "
             "forget to activate a virtual environment?"
-        ) from exc
+        )
     execute_from_command_line(sys.argv)

parambokeh/__init__.py

@@ -29,11 +29,25 @@
 from .util import named_objs, get_method_owner
 from .view import _View
 
+from param.version import Version
+__version__ = str(param.Version(fpath=__file__,archive_commit="$Format:%h$",reponame="parambokeh"))
+del Version
+
+##
+# make pyct's example/data commands available if possible
+from functools import partial
 try:
-    __version__ = param.Version(release=(0,2,1), fpath=__file__,
-                                commit="$Format:%h$", reponame='parambokeh')
-except:
-    __version__ = '0.2.1-unknown'
+    from pyct.cmd import copy_examples as _copy, fetch_data as _fetch, examples as _examples
+    copy_examples = partial(_copy, 'parambokeh')
+    fetch_data = partial(_fetch, 'parambokeh')
+    examples = partial(_examples, 'parambokeh')
+except ImportError:
+    def _missing_cmd(*args,**kw): return("install pyct to enable this command (e.g. `conda install pyct` or `pip install pyct[cmd]`)")
+    _copy = _fetch = _examples = _missing_cmd
+    def _err(): raise ValueError(_missing_cmd())
+    fetch_data = copy_examples = examples = _err
+del partial, _examples, _copy, _fetch
+##
 
 
 def notebook_show(obj, doc, comm):