Build Docker Images / Push Programs To Server #568
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Docker Images / Push Programs To Server | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 */16 * * *' | |
| jobs: | |
| push_deb11_to_hub: | |
| name: Push Debian 11 to Docker Hub | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@master | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/linux_model_deb_11 | |
| - name: Build and push Debian 11 Docker Image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/midori-ai-pip-deb11-dockerfile | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Clean up Docker Cache | |
| run: | | |
| docker image prune -f | |
| Build-Subsystem-Programs: | |
| name: Update Subsystem Programs | |
| runs-on: ubuntu-latest | |
| needs: push_deb11_to_hub | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Update Version Name | |
| run: | | |
| mkdir -p /tmp/artifacts | |
| export MIDORI_AI_API_KEY_TEMP=123456789 | |
| - name: Move everything to working dir | |
| run: | | |
| curl -k --disable --disable-eprt -s https://raw.githubusercontent.com/lunamidori5/Midori-AI/master/Webserver/Programs/Downloader/helper_app.py > helper_app.py | |
| curl -k --disable --disable-eprt -s https://raw.githubusercontent.com/lunamidori5/Midori-AI/master/Webserver/Programs/Login_program/midori_ai_login_app.py > midori_ai_login_app.py | |
| curl -k --disable --disable-eprt -s https://raw.githubusercontent.com/lunamidori5/Midori-AI-Subsystem-Manager/master/model_installer/yaml_edit.py > yaml_edit.py | |
| curl -k --disable --disable-eprt -s https://raw.githubusercontent.com/lunamidori5/Midori-AI-Subsystem-Manager/master/midori_ai_manager/huggingface_downloader.py > huggingface_downloader.py | |
| - name: Install Dependencies | |
| run: | | |
| pip install pyinstaller | |
| pip install tor-proxy | |
| pip install -r ./Subsystem-Manager/model_installer/requirements.txt | |
| - name: Download the Uploader | |
| run: | | |
| python3 helper_app.py git_uploader.py | |
| - name: Build Programs | |
| run: | | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ helper_app.py | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ midori_ai_login_app.py | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ yaml_edit.py | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ huggingface_downloader.py | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ git_uploader.py | |
| - name: Move Exec | |
| run: | | |
| mv /tmp/artifacts/helper_app midori_ai_downloader | |
| mv /tmp/artifacts/midori_ai_login_app midori_ai_login | |
| mv /tmp/artifacts/yaml_edit yaml_edit | |
| mv /tmp/artifacts/huggingface_downloader hf-downloader | |
| mv /tmp/artifacts/git_uploader git_uploader | |
| - name: Upload Raw Python | |
| run: | | |
| chmod +x git_uploader | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "helper_app.py" --filename "helper_app.py" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "yaml_edit.py" --filename "yaml_edit.py" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "huggingface_downloader.py" --filename "huggingface_downloader.py" | |
| - name: Upload Programs | |
| run: | | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "midori_ai_downloader" --filename "midori_ai_downloader" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "yaml_edit" --filename "yaml_edit" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "hf-downloader" --filename "hf-downloader" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "midori_ai_login" --filename "midori_ai_login" | |
| push_subsystem_manager_to_hub: | |
| name: Push Subsystem (Ubuntu Rolling) to Docker Hub | |
| runs-on: ubuntu-latest | |
| needs: Build-Subsystem-Programs | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@master | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| logout: false | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/subsystem_manager | |
| - name: Build and push Subsystem (other os) Docker Image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/midori-ai-subsystem-manager-dockerfile | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Clean up Docker Cache | |
| run: | | |
| docker image prune -f | |
| push_pixelarch_subsystem_topaz_to_hub: | |
| name: Push Subsystem (PixelArch Rolling) to Docker Hub | |
| runs-on: ubuntu-latest | |
| needs: Build-Subsystem-Programs | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@master | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| logout: false | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/pixelarch | |
| tags: | | |
| type=raw,value=subsystem | |
| - name: Build and push PixelArch OS (Subsystem Manager) Docker Image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/midori-ai-subsystem-manager-pixelarch-dockerfile | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| push_cpu_to_hub: | |
| name: Push LocalAI CPU to Docker Hub | |
| runs-on: self-hosted | |
| needs: Build-Subsystem-Programs | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@master | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| logout: false | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/midori_ai_subsystem_localai_cpu | |
| - name: Make Ver Tags (tags, labels) for Docker | |
| id: sha | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/midori_ai_subsystem_localai_cpu | |
| tags: | | |
| type=sha,enable=true,priority=100,prefix={{date 'YYYYMMDD-HHmmss' tz='Asia/Tokyo'}}-sha-,suffix=,format=short | |
| - name: Build and push CPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-cpu | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Build and push CPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-cpu | |
| push: true | |
| tags: ${{ steps.sha.outputs.tags }} | |
| labels: ${{ steps.sha.outputs.labels }} | |
| - name: Clean up Docker Cache | |
| run: | | |
| docker image prune -a -f | |
| docker system prune -a -f | |
| push_gpu_to_hub: | |
| name: Push LocalAI GPUs to Docker Hub | |
| strategy: | |
| matrix: | |
| gpu_type: [intelf16, intelf32, hipblas, nvidia] | |
| needs: Build-Subsystem-Programs | |
| runs-on: self-hosted | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@master | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| logout: false | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/midori_ai_subsystem_localai_${{ matrix.gpu_type }}_gpu | |
| - name: Make Ver Tags (tags, labels) for Docker | |
| id: sha | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/midori_ai_subsystem_localai_${{ matrix.gpu_type }}_gpu | |
| tags: | | |
| type=sha,enable=true,priority=100,prefix={{date 'YYYYMMDD-HHmmss' tz='Asia/Tokyo'}}-sha-,suffix=,format=short | |
| - name: Build and push GPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-${{ matrix.gpu_type }}-gpu | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Build and push GPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-${{ matrix.gpu_type }}-gpu | |
| push: true | |
| tags: ${{ steps.sha.outputs.tags }} | |
| labels: ${{ steps.sha.outputs.labels }} | |
| clean_docker_mess: | |
| name: Clean up Docker Cache | |
| strategy: | |
| matrix: | |
| gpu_type: [1, 2, 3] | |
| for_each: [1, 2, 3] | |
| needs: push_gpu_to_hub | |
| runs-on: self-hosted | |
| steps: | |
| - name: Clean up Docker Cache | |
| run: | | |
| nohup docker image prune -a -f & | |
| nohup docker system prune -a -f & |