From 4059d7a7ece3d3e5524ed4d90c5f8b676b375d5a Mon Sep 17 00:00:00 2001 From: Minh141120 Date: Wed, 20 Aug 2025 18:03:27 +0700 Subject: [PATCH 1/2] ci: add autoqa reliability workflow for windows --- .github/workflows/autoqa-reliability.yml | 156 +++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 .github/workflows/autoqa-reliability.yml diff --git a/.github/workflows/autoqa-reliability.yml b/.github/workflows/autoqa-reliability.yml new file mode 100644 index 000000000..d7a4c006b --- /dev/null +++ b/.github/workflows/autoqa-reliability.yml @@ -0,0 +1,156 @@ +name: AutoQA Reliability (Manual) + +on: + workflow_dispatch: + inputs: + source_type: + description: 'App source type (url or local)' + required: true + type: choice + options: [url, local] + default: url + jan_app_windows_source: + description: 'Windows installer URL or local path (used when source_type=url or to select artifact)' + required: true + type: string + default: 'https://catalog.jan.ai/windows/Jan_0.6.8_x64-setup.exe' + jan_app_ubuntu_source: + description: 'Ubuntu .deb URL or local path' + required: true + type: string + default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_amd64.deb' + jan_app_macos_source: + description: 'macOS .dmg URL or local path' + required: true + type: string + default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_universal.dmg' + is_nightly: + description: 'Is the app a nightly build?' + required: true + type: boolean + default: true + reliability_phase: + description: 'Reliability phase' + required: true + type: choice + options: [development, deployment] + default: development + reliability_runs: + description: 'Custom runs (0 uses phase default)' + required: true + type: number + default: 0 + reliability_test_path: + description: 'Test file path (relative to autoqa working directory)' + required: true + type: string + default: 'tests/base/settings/app-data.txt' + artifact_name_windows: + description: 'Windows artifact name (only for source_type=local)' + required: false + type: string + default: '' + artifact_name_ubuntu: + description: 'Ubuntu artifact name (only for source_type=local)' + required: false + type: string + default: '' + artifact_name_macos: + description: 'macOS artifact name (only for source_type=local)' + required: false + type: string + default: '' + +jobs: + reliability-windows: + runs-on: windows-11-nvidia-gpu + timeout-minutes: 60 + env: + DEFAULT_JAN_APP_URL: 'https://catalog.jan.ai/windows/Jan_0.6.8_x64-setup.exe' + DEFAULT_IS_NIGHTLY: 'false' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python 3.13 + uses: actions/setup-python@v4 + with: + python-version: '3.13' + + - name: Download artifact (if source_type is local) + if: inputs.source_type == 'local' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact_name_windows }} + path: ${{ runner.temp }}/windows-artifact + + - name: Clean existing Jan installations + shell: powershell + run: | + .\autoqa\scripts\windows_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}" + + - name: Download/Prepare Jan app + shell: powershell + run: | + if ("${{ inputs.source_type }}" -eq "local") { + $exeFile = Get-ChildItem -Path "${{ runner.temp }}/windows-artifact" -Recurse -Filter "*.exe" | Select-Object -First 1 + if ($exeFile) { + Write-Host "[SUCCESS] Found local installer: $($exeFile.FullName)" + Copy-Item -Path $exeFile.FullName -Destination "$env:TEMP\jan-installer.exe" -Force + Write-Host "[SUCCESS] Installer copied to: $env:TEMP\jan-installer.exe" + echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $env:GITHUB_ENV + } else { + Write-Error "[FAILED] No .exe file found in artifact" + exit 1 + } + } else { + .\autoqa\scripts\windows_download.ps1 ` + -WorkflowInputUrl "${{ inputs.jan_app_windows_source }}" ` + -WorkflowInputIsNightly "${{ inputs.is_nightly }}" ` + -RepoVariableUrl "${{ vars.JAN_APP_URL }}" ` + -RepoVariableIsNightly "${{ vars.IS_NIGHTLY }}" ` + -DefaultUrl "$env:DEFAULT_JAN_APP_URL" ` + -DefaultIsNightly "$env:DEFAULT_IS_NIGHTLY" + } + + - name: Install Jan app + shell: powershell + run: | + .\autoqa\scripts\windows_install.ps1 -IsNightly "$env:IS_NIGHTLY" + + - name: Install Python dependencies + working-directory: autoqa + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run reliability tests + working-directory: autoqa + shell: powershell + run: | + $runs = "${{ inputs.reliability_runs }}" + $runsArg = "" + if ([int]$runs -gt 0) { $runsArg = "--reliability-runs $runs" } + python main.py --enable-reliability-test --reliability-phase "${{ inputs.reliability_phase }}" --reliability-test-path "${{ inputs.reliability_test_path }}" $runsArg + + - name: Upload screen recordings + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-recordings-${{ github.run_number }}-${{ runner.os }} + path: autoqa/recordings/ + + - name: Upload trajectories + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-trajectories-${{ github.run_number }}-${{ runner.os }} + path: autoqa/trajectories/ + + - name: Cleanup after tests + if: always() + shell: powershell + run: | + .\autoqa\scripts\windows_post_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}" From 76dfe027abc521308326509053899506b1cf23fd Mon Sep 17 00:00:00 2001 From: Minh141120 Date: Wed, 20 Aug 2025 18:08:38 +0700 Subject: [PATCH 2/2] chore: remove local artifact inputs --- .github/workflows/autoqa-reliability.yml | 59 +++++------------------- 1 file changed, 12 insertions(+), 47 deletions(-) diff --git a/.github/workflows/autoqa-reliability.yml b/.github/workflows/autoqa-reliability.yml index d7a4c006b..759e93717 100644 --- a/.github/workflows/autoqa-reliability.yml +++ b/.github/workflows/autoqa-reliability.yml @@ -4,23 +4,23 @@ on: workflow_dispatch: inputs: source_type: - description: 'App source type (url or local)' + description: 'App source type (url)' required: true type: choice - options: [url, local] + options: [url] default: url jan_app_windows_source: - description: 'Windows installer URL or local path (used when source_type=url or to select artifact)' + description: 'Windows installer URL path (used when source_type=url or to select artifact)' required: true type: string default: 'https://catalog.jan.ai/windows/Jan_0.6.8_x64-setup.exe' jan_app_ubuntu_source: - description: 'Ubuntu .deb URL or local path' + description: 'Ubuntu .deb URL path' required: true type: string default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_amd64.deb' jan_app_macos_source: - description: 'macOS .dmg URL or local path' + description: 'macOS .dmg URL path' required: true type: string default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_universal.dmg' @@ -45,21 +45,6 @@ on: required: true type: string default: 'tests/base/settings/app-data.txt' - artifact_name_windows: - description: 'Windows artifact name (only for source_type=local)' - required: false - type: string - default: '' - artifact_name_ubuntu: - description: 'Ubuntu artifact name (only for source_type=local)' - required: false - type: string - default: '' - artifact_name_macos: - description: 'macOS artifact name (only for source_type=local)' - required: false - type: string - default: '' jobs: reliability-windows: @@ -77,13 +62,6 @@ jobs: with: python-version: '3.13' - - name: Download artifact (if source_type is local) - if: inputs.source_type == 'local' - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.artifact_name_windows }} - path: ${{ runner.temp }}/windows-artifact - - name: Clean existing Jan installations shell: powershell run: | @@ -92,26 +70,13 @@ jobs: - name: Download/Prepare Jan app shell: powershell run: | - if ("${{ inputs.source_type }}" -eq "local") { - $exeFile = Get-ChildItem -Path "${{ runner.temp }}/windows-artifact" -Recurse -Filter "*.exe" | Select-Object -First 1 - if ($exeFile) { - Write-Host "[SUCCESS] Found local installer: $($exeFile.FullName)" - Copy-Item -Path $exeFile.FullName -Destination "$env:TEMP\jan-installer.exe" -Force - Write-Host "[SUCCESS] Installer copied to: $env:TEMP\jan-installer.exe" - echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $env:GITHUB_ENV - } else { - Write-Error "[FAILED] No .exe file found in artifact" - exit 1 - } - } else { - .\autoqa\scripts\windows_download.ps1 ` - -WorkflowInputUrl "${{ inputs.jan_app_windows_source }}" ` - -WorkflowInputIsNightly "${{ inputs.is_nightly }}" ` - -RepoVariableUrl "${{ vars.JAN_APP_URL }}" ` - -RepoVariableIsNightly "${{ vars.IS_NIGHTLY }}" ` - -DefaultUrl "$env:DEFAULT_JAN_APP_URL" ` - -DefaultIsNightly "$env:DEFAULT_IS_NIGHTLY" - } + .\autoqa\scripts\windows_download.ps1 ` + -WorkflowInputUrl "${{ inputs.jan_app_windows_source }}" ` + -WorkflowInputIsNightly "${{ inputs.is_nightly }}" ` + -RepoVariableUrl "${{ vars.JAN_APP_URL }}" ` + -RepoVariableIsNightly "${{ vars.IS_NIGHTLY }}" ` + -DefaultUrl "$env:DEFAULT_JAN_APP_URL" ` + -DefaultIsNightly "$env:DEFAULT_IS_NIGHTLY" - name: Install Jan app shell: powershell