diff --git a/.github/workflows/autoqa-migration.yml b/.github/workflows/autoqa-migration.yml new file mode 100644 index 000000000..86a6a40ca --- /dev/null +++ b/.github/workflows/autoqa-migration.yml @@ -0,0 +1,330 @@ +name: AutoQA Migration (Manual) + +on: + workflow_dispatch: + inputs: + old_windows_installer: + description: 'Windows OLD installer URL or path (.exe)' + required: true + type: string + new_windows_installer: + description: 'Windows NEW installer URL or path (.exe)' + required: true + type: string + old_ubuntu_installer: + description: 'Ubuntu OLD installer URL or path (.deb)' + required: false + type: string + default: '' + new_ubuntu_installer: + description: 'Ubuntu NEW installer URL or path (.deb)' + required: false + type: string + default: '' + old_macos_installer: + description: 'macOS OLD installer URL or path (.dmg)' + required: false + type: string + default: '' + new_macos_installer: + description: 'macOS NEW installer URL or path (.dmg)' + required: false + type: string + default: '' + migration_test_case: + description: 'Specific migration test case key (leave empty to run all)' + required: false + type: string + default: '' + max_turns: + description: 'Maximum turns per test phase' + required: false + type: number + default: 65 + +jobs: + migration-windows: + runs-on: windows-11-nvidia-gpu + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python 3.13 + uses: actions/setup-python@v4 + with: + python-version: '3.13' + + - name: Clean existing Jan installations + shell: powershell + run: | + .\autoqa\scripts\windows_cleanup.ps1 -IsNightly $false + + - name: Download OLD and NEW installers + shell: powershell + run: | + # Download OLD installer using existing script + .\autoqa\scripts\windows_download.ps1 ` + -WorkflowInputUrl "${{ inputs.old_windows_installer }}" ` + -WorkflowInputIsNightly "false" ` + -RepoVariableUrl "" ` + -RepoVariableIsNightly "" ` + -DefaultUrl "" ` + -DefaultIsNightly "" + + $oldSrc = Join-Path $env:TEMP 'jan-installer.exe' + $oldOut = Join-Path $env:TEMP 'jan-old.exe' + Copy-Item -Path $oldSrc -Destination $oldOut -Force + + # Download NEW installer using existing script + .\autoqa\scripts\windows_download.ps1 ` + -WorkflowInputUrl "${{ inputs.new_windows_installer }}" ` + -WorkflowInputIsNightly "false" ` + -RepoVariableUrl "" ` + -RepoVariableIsNightly "" ` + -DefaultUrl "" ` + -DefaultIsNightly "" + + $newSrc = Join-Path $env:TEMP 'jan-installer.exe' + $newOut = Join-Path $env:TEMP 'jan-new.exe' + Copy-Item -Path $newSrc -Destination $newOut -Force + + Write-Host "OLD installer: $oldOut" + Write-Host "NEW installer: $newOut" + echo "OLD_VERSION=$oldOut" | Out-File -FilePath $env:GITHUB_ENV -Append + echo "NEW_VERSION=$newOut" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Install Python dependencies + working-directory: autoqa + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run migration tests (Windows) + working-directory: autoqa + shell: powershell + env: + RP_TOKEN: ${{ secrets.RP_TOKEN }} + ENABLE_REPORTPORTAL: 'true' + RP_ENDPOINT: 'https://reportportal.menlo.ai' + RP_PROJECT: 'default_personal' + run: | + $case = "${{ inputs.migration_test_case }}" + $caseArg = "" + if ($case -and $case.Trim() -ne "") { $caseArg = "--migration-test-case `"$case`"" } + python main.py --enable-migration-test --old-version "$env:OLD_VERSION" --new-version "$env:NEW_VERSION" --max-turns ${{ inputs.max_turns }} $caseArg + + - name: Upload screen recordings + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: migration-recordings-${{ github.run_number }}-windows + path: autoqa/recordings/ + + - name: Upload trajectories + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: migration-trajectories-${{ github.run_number }}-windows + path: autoqa/trajectories/ + + - name: Cleanup after tests + if: always() + shell: powershell + run: | + .\autoqa\scripts\windows_post_cleanup.ps1 -IsNightly $false + + migration-ubuntu: + if: inputs.old_ubuntu_installer != '' && inputs.new_ubuntu_installer != '' + runs-on: ubuntu-22-04-nvidia-gpu + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python 3.13 + uses: actions/setup-python@v4 + with: + python-version: '3.13' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + x11-utils \ + python3-tk \ + python3-dev \ + wmctrl \ + xdotool \ + libnss3-dev \ + libgconf-2-4 \ + libxss1 \ + libasound2 \ + libxtst6 \ + libgtk-3-0 \ + libgbm-dev \ + libxshmfence1 \ + libxrandr2 \ + libpangocairo-1.0-0 \ + libatk1.0-0 \ + libcairo-gobject2 \ + libgdk-pixbuf2.0-0 \ + gnome-screenshot \ + xvfb + + - name: Setup script permissions + run: | + chmod +x autoqa/scripts/setup_permissions.sh || true + ./autoqa/scripts/setup_permissions.sh || true + + - name: Clean existing Jan installations + run: | + ./autoqa/scripts/ubuntu_cleanup.sh + + - name: Download OLD and NEW installers + run: | + set -e + # Download OLD installer using existing script + ./autoqa/scripts/ubuntu_download.sh \ + "${{ inputs.old_ubuntu_installer }}" \ + "false" \ + "" \ + "" \ + "" \ + "" + cp /tmp/jan-installer.deb /tmp/jan-old.deb + + # Download NEW installer using existing script + ./autoqa/scripts/ubuntu_download.sh \ + "${{ inputs.new_ubuntu_installer }}" \ + "false" \ + "" \ + "" \ + "" \ + "" + cp /tmp/jan-installer.deb /tmp/jan-new.deb + + echo "OLD_VERSION=/tmp/jan-old.deb" >> $GITHUB_ENV + echo "NEW_VERSION=/tmp/jan-new.deb" >> $GITHUB_ENV + + - name: Install Python dependencies + working-directory: autoqa + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run migration tests (Ubuntu) + working-directory: autoqa + run: | + case="${{ inputs.migration_test_case }}" + caseArg="" + if [ -n "${case}" ]; then caseArg="--migration-test-case \"${case}\""; fi + xvfb-run -a python main.py --enable-migration-test --old-version "${OLD_VERSION}" --new-version "${NEW_VERSION}" --max-turns ${{ inputs.max_turns }} ${caseArg} + + - name: Upload screen recordings + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: migration-recordings-${{ github.run_number }}-ubuntu + path: autoqa/recordings/ + + - name: Upload trajectories + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: migration-trajectories-${{ github.run_number }}-ubuntu + path: autoqa/trajectories/ + + - name: Cleanup after tests + if: always() + run: | + ./autoqa/scripts/ubuntu_post_cleanup.sh "false" + + migration-macos: + if: inputs.old_macos_installer != '' && inputs.new_macos_installer != '' + runs-on: macos-selfhosted-15-arm64-cua + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python 3.13 + uses: actions/setup-python@v4 + with: + python-version: '3.13' + + - name: Setup script permissions + run: | + chmod +x autoqa/scripts/setup_permissions.sh || true + ./autoqa/scripts/setup_permissions.sh || true + + - name: Clean existing Jan installations + run: | + ./autoqa/scripts/macos_cleanup.sh + + - name: Download OLD and NEW installers + run: | + set -e + # Download OLD installer using existing script + ./autoqa/scripts/macos_download.sh \ + "${{ inputs.old_macos_installer }}" \ + "false" \ + "" \ + "" \ + "" \ + "" + cp /tmp/jan-installer.dmg /tmp/jan-old.dmg + + # Download NEW installer using existing script + ./autoqa/scripts/macos_download.sh \ + "${{ inputs.new_macos_installer }}" \ + "false" \ + "" \ + "" \ + "" \ + "" + cp /tmp/jan-installer.dmg /tmp/jan-new.dmg + + echo "OLD_VERSION=/tmp/jan-old.dmg" >> $GITHUB_ENV + echo "NEW_VERSION=/tmp/jan-new.dmg" >> $GITHUB_ENV + + - name: Install Python dependencies + working-directory: autoqa + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run migration tests (macOS) + working-directory: autoqa + run: | + case="${{ inputs.migration_test_case }}" + caseArg="" + if [ -n "${case}" ]; then caseArg="--migration-test-case \"${case}\""; fi + python main.py --enable-migration-test --old-version "${OLD_VERSION}" --new-version "${NEW_VERSION}" --max-turns ${{ inputs.max_turns }} ${caseArg} + + - name: Upload screen recordings + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: migration-recordings-${{ github.run_number }}-macos + path: autoqa/recordings/ + + - name: Upload trajectories + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: migration-trajectories-${{ github.run_number }}-macos + path: autoqa/trajectories/ + + - name: Cleanup after tests + if: always() + run: | + ./autoqa/scripts/macos_post_cleanup.sh + +