From 6021e0309f17508a2d653451c99a5139b8d15a99 Mon Sep 17 00:00:00 2001 From: Minh141120 Date: Wed, 20 Aug 2025 21:02:41 +0700 Subject: [PATCH] ci: add autoqa reliability macos and linux jobs --- .github/workflows/autoqa-reliability.yml | 197 ++++++++++++++++++++++- 1 file changed, 194 insertions(+), 3 deletions(-) diff --git a/.github/workflows/autoqa-reliability.yml b/.github/workflows/autoqa-reliability.yml index 759e93717..97ed44c21 100644 --- a/.github/workflows/autoqa-reliability.yml +++ b/.github/workflows/autoqa-reliability.yml @@ -18,12 +18,12 @@ on: description: 'Ubuntu .deb URL path' required: true type: string - default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_amd64.deb' + default: 'https://catalog.jan.ai/linux/Jan_0.6.8_amd64.deb' jan_app_macos_source: description: 'macOS .dmg URL path' required: true type: string - default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_universal.dmg' + default: 'https://catalog.jan.ai/macos/Jan_0.6.8_universal.dmg' is_nightly: description: 'Is the app a nightly build?' required: true @@ -37,7 +37,7 @@ on: default: development reliability_runs: description: 'Custom runs (0 uses phase default)' - required: true + required: false type: number default: 0 reliability_test_path: @@ -119,3 +119,194 @@ jobs: shell: powershell run: | .\autoqa\scripts\windows_post_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}" + + reliability-ubuntu: + runs-on: ubuntu-22-04-nvidia-gpu + timeout-minutes: 60 + env: + DEFAULT_JAN_APP_URL: 'https://catalog.jan.ai/linux/Jan_0.6.8_amd64.deb' + DEFAULT_IS_NIGHTLY: 'false' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python 3.13 + uses: actions/setup-python@v4 + with: + python-version: '3.13' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + x11-utils \ + python3-tk \ + python3-dev \ + wmctrl \ + xdotool \ + libnss3-dev \ + libgconf-2-4 \ + libxss1 \ + libasound2 \ + libxtst6 \ + libgtk-3-0 \ + libgbm-dev \ + libxshmfence1 \ + libxrandr2 \ + libpangocairo-1.0-0 \ + libatk1.0-0 \ + libcairo-gobject2 \ + libgdk-pixbuf2.0-0 \ + gnome-screenshot + + - name: Setup script permissions + run: | + chmod +x autoqa/scripts/setup_permissions.sh + ./autoqa/scripts/setup_permissions.sh + + - name: Clean existing Jan installations + run: | + ./autoqa/scripts/ubuntu_cleanup.sh + + - name: Download/Prepare Jan app + run: | + ./autoqa/scripts/ubuntu_download.sh \ + "${{ inputs.jan_app_ubuntu_source }}" \ + "${{ inputs.is_nightly }}" \ + "${{ vars.JAN_APP_URL_LINUX }}" \ + "${{ vars.IS_NIGHTLY }}" \ + "$DEFAULT_JAN_APP_URL" \ + "$DEFAULT_IS_NIGHTLY" + # Set env for install and runtime + echo "JAN_APP_PATH=/tmp/jan-installer.deb" >> $GITHUB_ENV + if [ "${{ inputs.is_nightly }}" = "true" ]; then + echo "JAN_PROCESS_NAME=Jan-nightly" >> $GITHUB_ENV + echo "RUNTIME_JAN_BIN=/usr/bin/Jan-nightly" >> $GITHUB_ENV + else + echo "JAN_PROCESS_NAME=Jan" >> $GITHUB_ENV + echo "RUNTIME_JAN_BIN=/usr/bin/Jan" >> $GITHUB_ENV + fi + + - name: Install Jan app + run: | + ./autoqa/scripts/ubuntu_install.sh "$IS_NIGHTLY" + + - name: Install Python dependencies + working-directory: autoqa + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run reliability tests + working-directory: autoqa + env: + DISPLAY: ${{ env.DISPLAY }} + run: | + runs="${{ inputs.reliability_runs }}" + runsArg="" + if [ "${runs}" -gt 0 ]; then runsArg="--reliability-runs ${runs}"; fi + processName="${JAN_PROCESS_NAME}" + janBin="${RUNTIME_JAN_BIN}" + python main.py --enable-reliability-test --reliability-phase "${{ inputs.reliability_phase }}" --reliability-test-path "${{ inputs.reliability_test_path }}" ${runsArg} --jan-process-name "${processName}" --jan-app-path "${janBin}" + + - name: Upload screen recordings + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-recordings-${{ github.run_number }}-ubuntu + path: autoqa/recordings/ + + - name: Upload trajectories + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-trajectories-${{ github.run_number }}-ubuntu + path: autoqa/trajectories/ + + - name: Cleanup after tests + if: always() + run: | + ./autoqa/scripts/ubuntu_post_cleanup.sh "$IS_NIGHTLY" + + reliability-macos: + runs-on: macos-selfhosted-15-arm64-cua + timeout-minutes: 60 + env: + DEFAULT_JAN_APP_URL: 'https://catalog.jan.ai/macos/Jan_0.6.8_universal.dmg' + DEFAULT_IS_NIGHTLY: 'false' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python 3.13 + uses: actions/setup-python@v4 + with: + python-version: '3.13' + + - name: Setup script permissions + run: | + chmod +x autoqa/scripts/setup_permissions.sh + ./autoqa/scripts/setup_permissions.sh + + - name: Clean existing Jan installations + run: | + ./autoqa/scripts/macos_cleanup.sh + + - name: Download/Prepare Jan app + run: | + ./autoqa/scripts/macos_download.sh \ + "${{ inputs.jan_app_macos_source }}" \ + "${{ inputs.is_nightly }}" \ + "${{ vars.JAN_APP_URL }}" \ + "${{ vars.IS_NIGHTLY }}" \ + "$DEFAULT_JAN_APP_URL" \ + "$DEFAULT_IS_NIGHTLY" + echo "JAN_APP_PATH=/tmp/jan-installer.dmg" >> $GITHUB_ENV + if [ "${{ inputs.is_nightly }}" = "true" ]; then + echo "PROCESS_NAME=Jan-nightly" >> $GITHUB_ENV + echo "RUNTIME_JAN_BIN=/Applications/Jan-nightly.app/Contents/MacOS/Jan-nightly" >> $GITHUB_ENV + else + echo "PROCESS_NAME=Jan" >> $GITHUB_ENV + echo "RUNTIME_JAN_BIN=/Applications/Jan.app/Contents/MacOS/Jan" >> $GITHUB_ENV + fi + + - name: Install Jan app + run: | + ./autoqa/scripts/macos_install.sh + + - name: Install Python dependencies + working-directory: autoqa + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run reliability tests + working-directory: autoqa + run: | + runs="${{ inputs.reliability_runs }}" + runsArg="" + if [ "${runs}" -gt 0 ]; then runsArg="--reliability-runs ${runs}"; fi + python main.py --enable-reliability-test --reliability-phase "${{ inputs.reliability_phase }}" --reliability-test-path "${{ inputs.reliability_test_path }}" ${runsArg} --jan-process-name "${PROCESS_NAME}" --jan-app-path "${RUNTIME_JAN_BIN}" + + - name: Upload screen recordings + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-recordings-${{ github.run_number }}-macos + path: autoqa/recordings/ + + - name: Upload trajectories + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-trajectories-${{ github.run_number }}-macos + path: autoqa/trajectories/ + + - name: Cleanup after tests + if: always() + run: | + ./autoqa/scripts/macos_post_cleanup.sh