diff --git a/.github/workflows/autoqa-reliability.yml b/.github/workflows/autoqa-reliability.yml new file mode 100644 index 000000000..d7a4c006b --- /dev/null +++ b/.github/workflows/autoqa-reliability.yml @@ -0,0 +1,156 @@ +name: AutoQA Reliability (Manual) + +on: + workflow_dispatch: + inputs: + source_type: + description: 'App source type (url or local)' + required: true + type: choice + options: [url, local] + default: url + jan_app_windows_source: + description: 'Windows installer URL or local path (used when source_type=url or to select artifact)' + required: true + type: string + default: 'https://catalog.jan.ai/windows/Jan_0.6.8_x64-setup.exe' + jan_app_ubuntu_source: + description: 'Ubuntu .deb URL or local path' + required: true + type: string + default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_amd64.deb' + jan_app_macos_source: + description: 'macOS .dmg URL or local path' + required: true + type: string + default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_universal.dmg' + is_nightly: + description: 'Is the app a nightly build?' + required: true + type: boolean + default: true + reliability_phase: + description: 'Reliability phase' + required: true + type: choice + options: [development, deployment] + default: development + reliability_runs: + description: 'Custom runs (0 uses phase default)' + required: true + type: number + default: 0 + reliability_test_path: + description: 'Test file path (relative to autoqa working directory)' + required: true + type: string + default: 'tests/base/settings/app-data.txt' + artifact_name_windows: + description: 'Windows artifact name (only for source_type=local)' + required: false + type: string + default: '' + artifact_name_ubuntu: + description: 'Ubuntu artifact name (only for source_type=local)' + required: false + type: string + default: '' + artifact_name_macos: + description: 'macOS artifact name (only for source_type=local)' + required: false + type: string + default: '' + +jobs: + reliability-windows: + runs-on: windows-11-nvidia-gpu + timeout-minutes: 60 + env: + DEFAULT_JAN_APP_URL: 'https://catalog.jan.ai/windows/Jan_0.6.8_x64-setup.exe' + DEFAULT_IS_NIGHTLY: 'false' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python 3.13 + uses: actions/setup-python@v4 + with: + python-version: '3.13' + + - name: Download artifact (if source_type is local) + if: inputs.source_type == 'local' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact_name_windows }} + path: ${{ runner.temp }}/windows-artifact + + - name: Clean existing Jan installations + shell: powershell + run: | + .\autoqa\scripts\windows_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}" + + - name: Download/Prepare Jan app + shell: powershell + run: | + if ("${{ inputs.source_type }}" -eq "local") { + $exeFile = Get-ChildItem -Path "${{ runner.temp }}/windows-artifact" -Recurse -Filter "*.exe" | Select-Object -First 1 + if ($exeFile) { + Write-Host "[SUCCESS] Found local installer: $($exeFile.FullName)" + Copy-Item -Path $exeFile.FullName -Destination "$env:TEMP\jan-installer.exe" -Force + Write-Host "[SUCCESS] Installer copied to: $env:TEMP\jan-installer.exe" + echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $env:GITHUB_ENV + } else { + Write-Error "[FAILED] No .exe file found in artifact" + exit 1 + } + } else { + .\autoqa\scripts\windows_download.ps1 ` + -WorkflowInputUrl "${{ inputs.jan_app_windows_source }}" ` + -WorkflowInputIsNightly "${{ inputs.is_nightly }}" ` + -RepoVariableUrl "${{ vars.JAN_APP_URL }}" ` + -RepoVariableIsNightly "${{ vars.IS_NIGHTLY }}" ` + -DefaultUrl "$env:DEFAULT_JAN_APP_URL" ` + -DefaultIsNightly "$env:DEFAULT_IS_NIGHTLY" + } + + - name: Install Jan app + shell: powershell + run: | + .\autoqa\scripts\windows_install.ps1 -IsNightly "$env:IS_NIGHTLY" + + - name: Install Python dependencies + working-directory: autoqa + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run reliability tests + working-directory: autoqa + shell: powershell + run: | + $runs = "${{ inputs.reliability_runs }}" + $runsArg = "" + if ([int]$runs -gt 0) { $runsArg = "--reliability-runs $runs" } + python main.py --enable-reliability-test --reliability-phase "${{ inputs.reliability_phase }}" --reliability-test-path "${{ inputs.reliability_test_path }}" $runsArg + + - name: Upload screen recordings + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-recordings-${{ github.run_number }}-${{ runner.os }} + path: autoqa/recordings/ + + - name: Upload trajectories + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-trajectories-${{ github.run_number }}-${{ runner.os }} + path: autoqa/trajectories/ + + - name: Cleanup after tests + if: always() + shell: powershell + run: | + .\autoqa\scripts\windows_post_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}"