From 19ea8fe01fe8c68c2d8d1a0979eff19818c3cd08 Mon Sep 17 00:00:00 2001 From: James Busche Date: Tue, 2 Apr 2024 13:22:19 -0700 Subject: [PATCH 1/3] Build Wheel Action Signed-off-by: James Busche release version in build workflow Signed-off-by: James Busche release version in build workflow Signed-off-by: James Busche fix build workflow for version Signed-off-by: James Busche space fix in build workflow for version Signed-off-by: James Busche Add publish to pypi Signed-off-by: James Busche condense publish to pypi Signed-off-by: James Busche cleanup build yaml Signed-off-by: James Busche Switch publish to release or tag Signed-off-by: James Busche adding py311 and fix dependencies Signed-off-by: James Busche remove version from setup.py and add debugging Signed-off-by: James Busche adding PiPY publish Signed-off-by: James Busche --- .github/workflows/build-and-publish.yaml | 52 ++++++++++++++++++++++++ pyproject.toml | 17 +++++--- setup.py | 2 +- setup_requirements.txt | 3 +- tox.ini | 15 +++++++ 5 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-and-publish.yaml diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml new file mode 100644 index 000000000..195b6b42c --- /dev/null +++ b/.github/workflows/build-and-publish.yaml @@ -0,0 +1,52 @@ +# Copyright The FMS HF Tuning Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Build and Publish FMS-hf-tuning Library + +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - setup: "3.11" + tox: "py311" + + environment: + name: pypi + url: https://pypi.org/p/fms-hf-tuning + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version.setup }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version.setup }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r setup_requirements.txt + - name: Build and test with tox + run: tox -e ${{ matrix.python-version.tox }} + - name: Build and check wheel package + run: + tox -e build,twinecheck + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/pyproject.toml b/pyproject.toml index f915662c7..be4bf620f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,12 @@ [build-system] +build-backend = "setuptools.build_meta" requires = [ - "setuptools>=61", + "setuptools>=60", "setuptools-scm>=8.0"] [project] name = "fms-hf-tuning" -version = "0.0.1" +dynamic = ["version", "dependencies"] description = "FMS HF Tuning" authors = [ {name = "Sukriti Sharma", email = "sukriti.sharma4@ibm.com"}, @@ -16,11 +17,14 @@ readme = "README.md" requires-python = "~=3.9" keywords = ['fms-hf-tuning', 'python', 'tuning'] classifiers=[ - "License :: OSI Approved :: Apache Software License" + "License :: OSI Approved :: Apache Software License", + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] -dynamic = ["dependencies"] - [project.optional-dependencies] dev = ["wheel", "packaging", "ninja", "scikit-learn>=1.0, <2.0"] flash-attn = ["flash-attn"] @@ -29,6 +33,9 @@ aim = ["aim==3.18.1"] [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} +[tool.setuptools_scm] +version_file = "tuning/_version.py" + [project.urls] Homepage = "https://github.com/foundation-model-stack/fms-hf-tuning" Repository = "https://github.com/foundation-model-stack/fms-hf-tuning" diff --git a/setup.py b/setup.py index 0d42e68fe..d3cb2fd03 100644 --- a/setup.py +++ b/setup.py @@ -15,4 +15,4 @@ # Third Party from setuptools import find_packages, setup -setup(name="fms-hf-tuning", version="0.0.1", packages=find_packages()) +setup(name="fms-hf-tuning", packages=find_packages()) diff --git a/setup_requirements.txt b/setup_requirements.txt index 519d362a1..a1329fac1 100644 --- a/setup_requirements.txt +++ b/setup_requirements.txt @@ -1,4 +1,5 @@ pre-commit>=3.0.4,<4.0 pylint>=2.16.2,<4.0 pydeps>=1.12.12,<2 -tox>=4.4.2,<5 \ No newline at end of file +tox>=4.4.2,<5 +build>=0.10.0,<2.0 diff --git a/tox.ini b/tox.ini index 74f49d4eb..3476945d5 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ description = run unit tests deps = pytest>=7 -r requirements.txt + wheel>=0.38.4 commands = pytest {posargs:tests} @@ -22,3 +23,17 @@ deps = pylint>=2.16.2,<=3.1.0 -r requirements.txt commands = pylint tuning scripts/*.py build/*.py tests allowlist_externals = pylint + +[testenv:build] +description = build wheel +deps = + build +commands = python -m build +skip_install = True + +[testenv:twinecheck] +description = check wheel +deps = + twine +commands = twine check dist/* +skip_install = True From be99d6410470bee0d439a80dfc21d835a0e98afb Mon Sep 17 00:00:00 2001 From: James Busche Date: Wed, 10 Apr 2024 13:46:09 -0700 Subject: [PATCH 2/3] dependency handling and exclude tests Signed-off-by: James Busche --- CONTRIBUTING.md | 7 +++++++ pyproject.toml | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e22ee76fc..2017e2670 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,6 +35,13 @@ We require new unit tests to be contributed with any new functionality added. Before sending pull requests, make sure your changes pass formatting, linting and unit tests. These checks will run with the pull request builds. Alternatively, you can run the checks manually on your local machine [as specified below](#development). +#### Dependencies +If additional new Python module dependencies are required, think about where to put them: + +- If they're required for fms-hf-tuning, then append them to the end of the requirements.txt file. +- If they're optional dependencies for additional functionality, then put them in the pyproject.toml file like were done for [flash-attn](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L26) or [aim](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L27). +- If it's an additional dependency for development, then add it to the [dev](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L25) dependencies. + #### Code Review Once you've [created a pull request](#how-can-i-contribute), maintainers will review your code and may make suggestions to fix before merging. It will be easier for your pull request to receive reviews if you consider the criteria the reviewers follow while working. Remember to: diff --git a/pyproject.toml b/pyproject.toml index be4bf620f..d396a5460 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,10 @@ aim = ["aim==3.18.1"] [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} +[tool.setuptools.packages.find] +exclude = ["tests", "tests.*"] +namespaces = false + [tool.setuptools_scm] version_file = "tuning/_version.py" From 10fad2de48c168c00645512e81cebd9a30cf76bc Mon Sep 17 00:00:00 2001 From: James Busche Date: Wed, 10 Apr 2024 15:26:06 -0700 Subject: [PATCH 3/3] Build wheel without src file Signed-off-by: James Busche --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3476945d5..b126ed79e 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ allowlist_externals = pylint description = build wheel deps = build -commands = python -m build +commands = python -m build -w skip_install = True [testenv:twinecheck]