refactor: slight change in dyn batch queue #18860

Workflow file for this run

	name: CI

	on:
	pull_request:
	paths-ignore:
	- 'docs/**'
	- 'README.md'

	#on:
	# push:
	# branches-ignore:
	# - '**' # temporally ignore all
	jobs:
	commit-lint:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	with:
	fetch-depth: 0
	- uses: wagoid/commitlint-github-action@v4

	lint-flake-8:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- name: Lint with flake8
	run: \|
	pip install flake8
	# stop the build if there are Python syntax errors or undefined names
	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/,jina/proto/docarray_v1,jina/proto/docarray_v2,jina/serve/consensus/add_voter/pb,jina/serve/consensus/add_voter/pb2
	# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
	flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/proto/docarray_v1,jina/proto/docarray_v2,jina/serve/consensus/add_voter/pb,jina/serve/consensus/add_voter/pb2

	check-docstring:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	with:
	fetch-depth: 0
	- name: Set up Python 3.8
	uses: actions/setup-python@v1
	with:
	python-version: 3.8
	- id: file_changes
	uses: Ana06/get-changed-files@v1.2
	- name: docstring check with darglint and pydocstyle
	run: ./scripts/docstrings_lint.sh
	env:
	CHANGED_FILES: ${{ steps.file_changes.outputs.added_modified }}

	check-black:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	with:
	fetch-depth: 0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- id: file_changes
	uses: Ana06/get-changed-files@v1.2
	- name: check black
	run: ./scripts/black.sh
	env:
	CHANGED_FILES: ${{ steps.file_changes.outputs.added_modified }}

	code-injection:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	if: ${{ !github.event.pull_request.head.repo.fork }}
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	ref: ${{ github.event.pull_request.head.sha }}
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	if: ${{ !github.event.pull_request.head.repo.fork }}
	with:
	python-version: 3.8
	- name: Styling
	id: styling
	if: ${{ !github.event.pull_request.head.repo.fork }}
	run: \|
	git config --local user.email "dev-bot@jina.ai"
	git config --local user.name "Jina Dev Bot"
	pip install ".[devel,black]"
	pip install setuptools
	cd scripts && bash ./devbot.sh && cd -
	if [[ -z $(git status -s) ]]
	then
	echo "tree is clean"
	else
	git add -u
	git commit -m "style: fix overload and cli autocomplete"
	echo "IS_CHANGED=YES" >> $GITHUB_OUTPUT
	fi
	- name: Push changes to head ref
	uses: ad-m/github-push-action@v0.6.0
	if: ${{ !github.event.pull_request.head.repo.fork }}
	with:
	github_token: ${{ secrets.JINA_DEV_BOT }}
	branch: ${{ github.head_ref }}
	- name: Fail this pipelint
	if: ${{ steps.styling.outputs.IS_CHANGED == 'YES' && !github.event.pull_request.head.repo.fork }}
	run: exit 1

	docker-image-test:
	needs: [code-injection]
	runs-on: ubuntu-latest
	services:
	registry:
	image: registry:2
	ports:
	- 5000:5000
	strategy:
	fail-fast: false
	matrix:
	test-arch: ["linux/amd64", "linux/arm64"]
	steps:
	# - name: Cancel Previous Runs
	# uses: styfle/cancel-workflow-action@0.9.0
	# with:
	# access_token: ${{ github.token }}
	- uses: actions/checkout@v2.5.0
	- name: Set up Docker Buildx
	id: buildx
	uses: docker/setup-buildx-action@v1
	with:
	install: true
	driver-opts: network=host
	- run: \|
	docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64
	docker run --rm --privileged tonistiigi/binfmt --install all
	- name: Build and test
	uses: docker/build-push-action@v2
	with:
	context: .
	file: Dockerfiles/debianx.Dockerfile
	platforms: ${{ matrix.test-arch }}
	push: true
	tags: localhost:5000/jina/multiarch:latest
	target: jina
	- run: \|
	docker run --platform ${{ matrix.test-arch }} localhost:5000/jina/multiarch:latest -v

	hub-test:
	runs-on: ubuntu-latest
	needs: [lint-flake-8, code-injection]
	if: ${{ !github.event.pull_request.head.repo.fork }}
	steps:
	# - name: Cancel Previous Runs
	# uses: styfle/cancel-workflow-action@0.9.0
	# with:
	# access_token: ${{ github.token }}
	- uses: actions/checkout@v2.5.0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- name: Test hubapp with hubpods
	run: \|
	./tests/jinahub/test_integration.sh
	timeout-minutes: 30
	env:
	JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }}
	JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }}

	k8s-flow-test:
	needs: [lint-flake-8, code-injection]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- name: Prepare enviroment
	run: \|
	docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
	python -m pip install --upgrade pip
	python -m pip install wheel
	pip install ".[all]" --no-cache-dir
	pip install docarray==0.21.0
	jina
	export JINA_LOG_LEVEL="ERROR"
	curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
	sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- name: Test k8s Flow
	run: \|
	export LINKERD2_VERSION=stable-2.11.4
	curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install \| sh
	pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_flow.py
	timeout-minutes: 30
	- name: Check codecov file
	id: check_files
	uses: andstor/file-existence-action@v1
	with:
	files: "coverage.xml"
	- name: Upload coverage from test to Codecov
	uses: codecov/codecov-action@v3.1.1
	if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
	with:
	file: coverage.xml
	name: ${{ matrix.test-path }}-codecov
	flags: ${{ steps.test.outputs.codecov_flag }}
	fail_ci_if_error: false

	k8s-deployment-test:
	needs: [lint-flake-8, code-injection]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- name: Prepare enviroment
	run: \|
	docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
	python -m pip install --upgrade pip
	python -m pip install wheel
	pip install ".[all]" --no-cache-dir
	pip install docarray==0.21.0
	jina
	export JINA_LOG_LEVEL="ERROR"
	curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
	sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- name: Test k8s Deployment
	run: \|
	export LINKERD2_VERSION=stable-2.11.4
	curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install \| sh
	pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_deployment.py
	timeout-minutes: 30
	- name: Check codecov file
	id: check_files
	uses: andstor/file-existence-action@v1
	with:
	files: "coverage.xml"
	- name: Upload coverage from test to Codecov
	uses: codecov/codecov-action@v3.1.1
	if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
	with:
	file: coverage.xml
	name: ${{ matrix.test-path }}-codecov
	flags: ${{ steps.test.outputs.codecov_flag }}
	fail_ci_if_error: false

	k8s-graceful-test:
	needs: [lint-flake-8, code-injection]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- name: Prepare enviroment
	run: \|
	docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
	python -m pip install --upgrade pip
	python -m pip install wheel
	pip install ".[all]" --no-cache-dir
	pip install docarray==0.21.0
	jina
	export JINA_LOG_LEVEL="ERROR"
	curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
	sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- name: Test graceful request handling on k8s
	run: \|
	export LINKERD2_VERSION=stable-2.11.4
	curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install \| sh
	pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_graceful_request_handling.py
	timeout-minutes: 45
	- name: Check codecov file
	id: check_files
	uses: andstor/file-existence-action@v1
	with:
	files: "coverage.xml"
	- name: Upload coverage from test to Codecov
	uses: codecov/codecov-action@v3.1.1
	if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
	with:
	file: coverage.xml
	name: ${{ matrix.test-path }}-codecov
	flags: ${{ steps.test.outputs.codecov_flag }}
	fail_ci_if_error: false

	k8s-failures-test:
	needs: [ lint-flake-8, code-injection ]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- name: Prepare enviroment
	run: \|
	docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
	python -m pip install --upgrade pip
	python -m pip install wheel
	pip install ".[all]" --no-cache-dir
	pip install docarray==0.21.0
	jina
	export JINA_LOG_LEVEL="ERROR"
	curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
	sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- name: Test k8s failures
	run: \|
	export LINKERD2_VERSION=stable-2.11.4
	curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install \| sh
	curl --proto '=https' --tlsv1.2 -sSfL https://linkerd.github.io/linkerd-smi/install \| sh
	pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_failures.py
	timeout-minutes: 45
	- name: Check codecov file
	id: check_files
	uses: andstor/file-existence-action@v1
	with:
	files: "coverage.xml"
	- name: Upload coverage from test to Codecov
	uses: codecov/codecov-action@v3.1.1
	if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
	with:
	file: coverage.xml
	name: ${{ matrix.test-path }}-codecov
	flags: ${{ steps.test.outputs.codecov_flag }}
	fail_ci_if_error: false

	k8s-otel-test:
	needs: [lint-flake-8, code-injection]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- name: Prepare enviroment
	run: \|
	docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
	python -m pip install --upgrade pip
	python -m pip install wheel
	pip install ".[all]" --no-cache-dir
	pip install docarray==0.21.0
	jina
	export JINA_LOG_LEVEL="ERROR"
	curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
	sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- name: Test K8s with OpenTelemetry
	run: \|
	pytest -v -s --cov=jina --cov-report=xml ./tests/k8s_otel
	timeout-minutes: 30
	- name: Check codecov file
	id: check_files
	uses: andstor/file-existence-action@v1
	with:
	files: "coverage.xml"
	- name: Upload coverage from test to Codecov
	uses: codecov/codecov-action@v3.1.1
	if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
	with:
	file: coverage.xml
	name: ${{ matrix.test-path }}-codecov
	flags: ${{ steps.test.outputs.codecov_flag }}
	fail_ci_if_error: false


	docker-compose-test:
	needs: [lint-flake-8, code-injection]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- name: Prepare enviroment
	run: \|
	docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
	python -m pip install --upgrade pip
	python -m pip install wheel
	pip install ".[all]" --no-cache-dir
	pip install docarray==0.21.0
	jina
	export JINA_LOG_LEVEL="ERROR"
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- name: Test docker compose
	run: \|
	pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/docker_compose/test_*.py
	timeout-minutes: 30
	- name: Check codecov file
	id: check_files
	uses: andstor/file-existence-action@v1
	with:
	files: "coverage.xml"
	- name: Upload coverage from test to Codecov
	uses: codecov/codecov-action@v3.1.1
	if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
	with:
	file: coverage.xml
	name: ${{ matrix.test-path }}-codecov
	flags: ${{ steps.test.outputs.codecov_flag }}
	fail_ci_if_error: false

	prep-testbed:
	runs-on: ubuntu-latest
	needs: [ lint-flake-8, code-injection]
	steps:
	- uses: actions/checkout@v2.5.0
	- id: set-matrix
	run: \|
	sudo apt-get install jq
	export value=$(bash scripts/get-all-test-paths.sh)
	echo "matrix=$value" >> $GITHUB_OUTPUT
	outputs:
	matrix: ${{ steps.set-matrix.outputs.matrix }}

	docarray-v-two-test:
	needs: prep-testbed
	runs-on: ubuntu-latest
	env:
	JINA_RANDOM_PORT_MIN: 16384
	strategy:
	fail-fast: false
	matrix:
	python-version: [3.8]
	protobuf-version: ['==3.19.6', '']
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Set up Golang
	uses: actions/setup-go@v2
	with:
	go-version: 1.19.5

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install setuptools wheel
	pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
	pip install cibuildwheel

	- name: Build wheels with setuptools-golang-build-manylinux-wheel
	run: \|
	setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
	- name: Prepare environment
	run: \|
	docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
	python -m pip install --upgrade pip
	python -m pip install wheel
	WHEEL_FILE=$(ls dist/*whl)
	pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
	if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then
	pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5
	else
	pip install -U protobuf${{ matrix.protobuf-version }}
	fi
	jina
	export JINA_LOG_LEVEL="ERROR"
	- name: Test
	id: test
	run: \|
	pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/unit/serve/runtimes/test_helper.py
	pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_v2.py
	pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/deployment_http_composite
	pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_singleton.py
	pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_parameters_as_pydantic.py
	pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_streaming.py
	pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/csp
	pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/docker
	echo "flag it as jina for codeoverage"
	echo "codecov_flag=jina" >> $GITHUB_OUTPUT
	timeout-minutes: 45
	env:
	JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
	- name: Check codecov file
	id: check_files
	uses: andstor/file-existence-action@v1
	with:
	files: "coverage.xml"
	- name: Upload coverage from test to Codecov
	uses: codecov/codecov-action@v3.1.1
	if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
	with:
	file: coverage.xml
	name: ${{ matrix.test-path }}-codecov
	flags: ${{ steps.test.outputs.codecov_flag }}
	fail_ci_if_error: false


	stateful-docarray-v-two-test:
	needs: prep-testbed
	runs-on: ubuntu-latest
	env:
	JINA_RANDOM_PORT_MIN: 16384
	strategy:
	fail-fast: false
	matrix:
	python-version: [3.8]
	protobuf-version: ['==3.19.6', '']
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Set up Golang
	uses: actions/setup-go@v2
	with:
	go-version: 1.19.5

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install setuptools wheel
	pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
	pip install cibuildwheel

	- name: Build wheels with setuptools-golang-build-manylinux-wheel
	run: \|
	setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
	- name: Prepare environment
	run: \|
	docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
	python -m pip install --upgrade pip
	python -m pip install wheel
	WHEEL_FILE=$(ls dist/*whl)
	pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
	if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then
	pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5
	else
	pip install -U protobuf${{ matrix.protobuf-version }}
	fi
	jina
	export JINA_LOG_LEVEL="ERROR"
	- name: Test stateful
	id: test_stateful
	run: \|
	JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --maxfail 1 --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful
	echo "flag it as jina for codeoverage"
	echo "codecov_flag=jina" >> $GITHUB_OUTPUT
	timeout-minutes: 30
	env:
	JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
	- name: Check codecov file
	id: check_files
	uses: andstor/file-existence-action@v1
	with:
	files: "coverage.xml"
	- name: Upload coverage from test to Codecov
	uses: codecov/codecov-action@v3.1.1
	if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
	with:
	file: coverage.xml
	name: ${{ matrix.test-path }}-codecov
	flags: ${{ steps.test.outputs.codecov_flag }}
	fail_ci_if_error: false

	# Build the wheels for Linux and macOS for Python 3.8 and newer
	build-wheels:
	runs-on: ${{ matrix.os }}
	strategy:
	# Ensure that a wheel builder finishes even if another fails
	fail-fast: false
	matrix:
	include:
	# linux
	- os: ubuntu-latest
	python: '3.7'
	python-manylinux-tag: "cp37-cp37m"
	- os: ubuntu-latest
	python: '3.8'
	python-manylinux-tag: "cp38-cp38"
	- os: ubuntu-latest
	python: '3.9'
	python-manylinux-tag: "cp39-cp39"
	- os: ubuntu-latest
	python: '3.10'
	python-manylinux-tag: "cp310-cp310"
	- os: ubuntu-latest
	python: '3.11'
	python-manylinux-tag: "cp311-cp311"

	# MacOS emulated
	- os: macos-latest
	python: '3.8'
	python-cibuildwheels: '38'
	platform_id: macosx_x86_64
	arch: x86_64
	goarch: amd64
	- os: macos-latest
	python: '3.9'
	python-cibuildwheels: '39'
	platform_id: macosx_x86_64
	arch: x86_64
	goarch: amd64
	- os: macos-latest
	python: '3.10'
	python-cibuildwheels: '310'
	platform_id: macosx_x86_64
	arch: x86_64
	goarch: amd64
	- os: macos-latest
	python: '3.11'
	python-cibuildwheels: '311'
	platform_id: macosx_x86_64
	arch: x86_64
	goarch: amd64

	# MacOS native
	- os: macos-latest
	python: '3.8'
	python-cibuildwheels: '38'
	platform_id: macosx_arm64
	arch: arm64
	goarch: arm64
	- os: macos-latest
	python: '3.9'
	python-cibuildwheels: '39'
	platform_id: macosx_arm64
	arch: arm64
	goarch: arm64
	- os: macos-latest
	python: '3.10'
	python-cibuildwheels: '310'
	platform_id: macosx_arm64
	arch: arm64
	goarch: arm64
	- os: macos-latest
	python: '3.11'
	python-cibuildwheels: '311'
	platform_id: macosx_arm64
	arch: arm64
	goarch: arm64

	steps:
	- name: Check out repository
	uses: actions/checkout@v2

	- name: Set up Python ${{ matrix.python }}
	uses: actions/setup-python@v2
	with:
	python-version: ${{ matrix.python }}

	- name: Set up Golang
	uses: actions/setup-go@v2
	with:
	go-version: 1.19.5

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install setuptools wheel
	pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
	pip install cibuildwheel

	- name: Build wheels with setuptools-golang-build-manylinux-wheel
	if: ${{ matrix.os == 'ubuntu-latest' }}
	run: \|
	setuptools-golang-build-manylinux-wheels --pythons ${{ matrix.python-manylinux-tag }}

	- name: Build wheels with cibuildwheels on macos
	env:
	CIBW_BUILD: cp${{ matrix.python-cibuildwheels }}-${{ matrix.platform_id }}
	CIBW_ARCHS: ${{ matrix.arch }}
	CIBW_TEST_COMMAND: python -c "import jina; import jraft"
	CIBW_BUILD_VERBOSITY: 1
	GOARCH: ${{ matrix.goarch }}
	CGO_ENABLED: 1
	if: ${{ matrix.os == 'macos-latest' }}
	run: \|
	python -m cibuildwheel --output-dir dist

	- name: Upload wheels as artifacts
	uses: actions/upload-artifact@v2
	with:
	path: dist/*.whl

	core-test:
	needs: [prep-testbed]
	runs-on: ubuntu-latest
	env:
	JINA_RANDOM_PORT_MIN: 16384
	strategy:
	fail-fast: false
	matrix:
	python-version: [3.8]
	test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}}
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Set up Golang
	uses: actions/setup-go@v2
	with:
	go-version: 1.19.5

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install setuptools wheel
	pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
	pip install cibuildwheel

	- name: Build wheels with setuptools-golang-build-manylinux-wheel
	run: \|
	setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
	- name: Prepare environment
	run: \|
	docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
	python -m pip install --upgrade pip
	python -m pip install wheel
	WHEEL_FILE=$(ls dist/*whl)
	pip install "$WHEEL_FILE[all]" --no-cache-dir
	pip install docarray==0.21.0
	jina
	export JINA_LOG_LEVEL="ERROR"
	- name: Test
	id: test
	run: \|
	pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/docarray_v2/' --ignore-glob='tests/integration/stateful/' --ignore-glob='tests/integration/hub_usage/dummyhub*' ${{ matrix.test-path }}
	if [[ "${{ matrix.test-path }}" == *"tests/integration/instrumentation/" ]]
	then
	echo "sleeping to let open telemetry export daemon thread expire by itself"
	sleep 2
	fi
	echo "flag it as jina for codeoverage"
	echo "codecov_flag=jina" >> $GITHUB_OUTPUT
	timeout-minutes: 30
	env:
	JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
	- name: Check codecov file
	id: check_files
	uses: andstor/file-existence-action@v1
	with:
	files: "coverage.xml"
	- name: Upload coverage from test to Codecov
	uses: codecov/codecov-action@v3.1.1
	if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
	with:
	file: coverage.xml
	name: ${{ matrix.test-path }}-codecov
	flags: ${{ steps.test.outputs.codecov_flag }}
	fail_ci_if_error: false

	import-test:
	runs-on: ubuntu-latest
	needs: [lint-flake-8, code-injection ]
	strategy:
	fail-fast: false
	matrix:
	core: ['', 'true']
	perf: ['', 'true']
	python-env: ['3.7', '3.8', '3.9', '3.10', '3.11']
	exclude:
	- core: 'true'
	perf: 'true'
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Prepare enviroment
	run: \|
	python -m pip install --upgrade pip
	python -m pip install wheel
	pip install --no-cache-dir .
	env:
	JINA_PIP_INSTALL_CORE: ${{ matrix.core }}
	JINA_PIP_INSTALL_PERF: ${{ matrix.perf }}
	- name: Test basic import
	run: python -c 'from jina import Executor,requests,Client,Flow,dynamic_batching,Document,DocumentArray'
	- name: Test import all
	run: python -c 'from jina import *'

	install-jina-ecosystem-test:
	runs-on: ubuntu-latest
	needs: [lint-flake-8, code-injection ]
	strategy:
	fail-fast: false
	steps:
	- uses: actions/checkout@v2.5.0
	- name: Set up Python 3.8
	uses: actions/setup-python@v4
	with:
	python-version: 3.8
	- name: Prepare enviroment
	run: \|
	python -m pip install --upgrade pip
	python -m pip install wheel

	HUBBLE_VERSION=$(curl -L -s "https://pypi.org/pypi/jina-hubble-sdk/json" \| jq -r '.releases \| keys \| .[]\| select(startswith("0."))' \| sort -V \| tail -1)
	DOCARRAY_VERSION=$(curl -L -s "https://pypi.org/pypi/docarray/json" \| jq -r '.releases \| keys \| .[]\| select(startswith("0."))' \| sort -V \| tail -1)
	JCLOUD_VERSION=$(curl -L -s "https://pypi.org/pypi/jcloud/json" \| jq -r '.releases \| keys \| .[]\| select(startswith("0."))' \| sort -V \| tail -1)

	pip install . jcloud==$JCLOUD_VERSION docarray==$DOCARRAY_VERSION jina-hubble-sdk==$HUBBLE_VERSION
	env:
	JINA_PIP_INSTALL_CORE: ${{ matrix.core }}
	JINA_PIP_INSTALL_PERF: ${{ matrix.perf }}

	- name: Test basic import
	run: python -c 'from jina import Executor,requests'
	- name: Test import all
	run: python -c 'from jina import *'


	# just for blocking the merge until all parallel core-test are successful
	success-all-test:
	runs-on: ubuntu-latest
	needs: [commit-lint, core-test, docarray-v-two-test, stateful-docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, check-docstring, check-black, code-injection, build-wheels]

	if: always()
	steps:
	- uses: technote-space/workflow-conclusion-action@v2
	- name: Check Failure
	if: env.WORKFLOW_CONCLUSION == 'failure'
	run: exit 1
	- name: Success
	if: ${{ success() }}
	run: echo "All Done"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

refactor: slight change in dyn batch queue #18860

Workflow file

refactor: slight change in dyn batch queue #18860

Jobs

Run details

Workflow file for this run