472 lines
16 KiB
YAML
472 lines
16 KiB
YAML
name: Auto QA Test Runner Template
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
jan_app_windows_source:
|
|
description: 'Windows app source - can be URL or local path'
|
|
required: true
|
|
type: string
|
|
jan_app_ubuntu_source:
|
|
description: 'Ubuntu app source - can be URL or local path'
|
|
required: true
|
|
type: string
|
|
jan_app_macos_source:
|
|
description: 'macOS app source - can be URL or local path'
|
|
required: true
|
|
type: string
|
|
is_nightly:
|
|
description: 'Is this a nightly build?'
|
|
required: true
|
|
type: boolean
|
|
default: true
|
|
source_type:
|
|
description: 'Source type: url or local'
|
|
required: true
|
|
type: string
|
|
default: 'url'
|
|
artifact_name_windows:
|
|
description: 'Windows artifact name (only needed for local)'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
artifact_name_ubuntu:
|
|
description: 'Ubuntu artifact name (only needed for local)'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
artifact_name_macos:
|
|
description: 'macOS artifact name (only needed for local)'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
secrets:
|
|
RP_TOKEN:
|
|
description: 'ReportPortal API token'
|
|
required: true
|
|
|
|
jobs:
|
|
windows:
|
|
runs-on: windows-11-nvidia-gpu
|
|
timeout-minutes: 60
|
|
|
|
env:
|
|
DEFAULT_JAN_APP_URL: 'https://catalog.jan.ai/windows/Jan-nightly_0.6.5-758_x64-setup.exe'
|
|
DEFAULT_IS_NIGHTLY: 'true'
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Python 3.13
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.13'
|
|
|
|
- name: Download artifact (if source_type is local)
|
|
if: inputs.source_type == 'local'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: ${{ inputs.artifact_name_windows }}
|
|
path: ${{ runner.temp }}/windows-artifact
|
|
|
|
- name: Clean existing Jan installations
|
|
shell: powershell
|
|
run: |
|
|
.\autoqa\scripts\windows_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}"
|
|
|
|
- name: Download/Prepare Jan app
|
|
shell: powershell
|
|
run: |
|
|
if ("${{ inputs.source_type }}" -eq "local") {
|
|
# Find the exe file in the artifact
|
|
$exeFile = Get-ChildItem -Path "${{ runner.temp }}/windows-artifact" -Recurse -Filter "*.exe" | Select-Object -First 1
|
|
if ($exeFile) {
|
|
Write-Host "[SUCCESS] Found local installer: $($exeFile.FullName)"
|
|
Copy-Item -Path $exeFile.FullName -Destination "$env:TEMP\jan-installer.exe" -Force
|
|
Write-Host "[SUCCESS] Installer copied to: $env:TEMP\jan-installer.exe"
|
|
# Don't set JAN_APP_PATH here - let the install script set it to the correct installed app path
|
|
echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $env:GITHUB_ENV
|
|
} else {
|
|
Write-Error "[FAILED] No .exe file found in artifact"
|
|
exit 1
|
|
}
|
|
} else {
|
|
# Use the existing download script for URLs
|
|
.\autoqa\scripts\windows_download.ps1 `
|
|
-WorkflowInputUrl "${{ inputs.jan_app_windows_source }}" `
|
|
-WorkflowInputIsNightly "${{ inputs.is_nightly }}" `
|
|
-RepoVariableUrl "${{ vars.JAN_APP_URL }}" `
|
|
-RepoVariableIsNightly "${{ vars.IS_NIGHTLY }}" `
|
|
-DefaultUrl "$env:DEFAULT_JAN_APP_URL" `
|
|
-DefaultIsNightly "$env:DEFAULT_IS_NIGHTLY"
|
|
}
|
|
|
|
- name: Install Jan app
|
|
shell: powershell
|
|
run: |
|
|
.\autoqa\scripts\windows_install.ps1 -IsNightly "$env:IS_NIGHTLY"
|
|
|
|
- name: Install Python dependencies
|
|
working-directory: autoqa
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
|
|
- name: Run Auto QA Tests
|
|
working-directory: autoqa
|
|
shell: powershell
|
|
env:
|
|
RP_TOKEN: ${{ secrets.RP_TOKEN }}
|
|
ENABLE_REPORTPORTAL: 'true'
|
|
RP_ENDPOINT: 'https://reportportal.menlo.ai'
|
|
RP_PROJECT: 'default_personal'
|
|
MAX_TURNS: '50'
|
|
DELAY_BETWEEN_TESTS: '3'
|
|
LAUNCH_NAME: 'CI AutoQA Run Windows - ${{ github.run_number }} - ${{ github.ref_name }}'
|
|
run: |
|
|
.\scripts\run_tests.ps1 -JanAppPath "$env:JAN_APP_PATH" -ProcessName "$env:JAN_PROCESS_NAME" -RpToken "$env:RP_TOKEN"
|
|
|
|
- name: Collect Jan logs for artifact upload
|
|
if: always()
|
|
shell: powershell
|
|
run: |
|
|
$logDirs = @(
|
|
"$env:APPDATA\Jan-nightly\data\logs",
|
|
"$env:APPDATA\Jan\data\logs"
|
|
)
|
|
$dest = "autoqa\jan-logs"
|
|
mkdir $dest -Force | Out-Null
|
|
foreach ($dir in $logDirs) {
|
|
if (Test-Path $dir) {
|
|
Copy-Item "$dir\*.log" $dest -Force -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
|
|
- name: Upload screen recordings
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
continue-on-error: true
|
|
with:
|
|
name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-recordings-${{ github.run_number }}-${{ runner.os }}
|
|
path: autoqa/recordings/
|
|
|
|
- name: Upload Jan logs
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-logs-${{ github.run_number }}-${{ runner.os }}
|
|
path: autoqa/jan-logs/
|
|
|
|
- name: Cleanup after tests
|
|
if: always()
|
|
shell: powershell
|
|
run: |
|
|
.\autoqa\scripts\windows_post_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}"
|
|
|
|
ubuntu:
|
|
runs-on: ubuntu-22-04-nvidia-gpu
|
|
timeout-minutes: 60
|
|
|
|
env:
|
|
DEFAULT_JAN_APP_URL: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_amd64.deb'
|
|
DEFAULT_IS_NIGHTLY: 'true'
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Python 3.13
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.13'
|
|
|
|
- name: Download artifact (if source_type is local)
|
|
if: inputs.source_type == 'local'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: ${{ inputs.artifact_name_ubuntu }}
|
|
path: ${{ runner.temp }}/ubuntu-artifact
|
|
|
|
- name: Install system dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y \
|
|
x11-utils \
|
|
python3-tk \
|
|
python3-dev \
|
|
wmctrl \
|
|
xdotool \
|
|
libnss3-dev \
|
|
libgconf-2-4 \
|
|
libxss1 \
|
|
libasound2 \
|
|
libxtst6 \
|
|
libgtk-3-0 \
|
|
libgbm-dev \
|
|
libxshmfence1 \
|
|
libxrandr2 \
|
|
libpangocairo-1.0-0 \
|
|
libatk1.0-0 \
|
|
libcairo-gobject2 \
|
|
libgdk-pixbuf2.0-0 \
|
|
gnome-screenshot
|
|
|
|
- name: Setup script permissions
|
|
run: |
|
|
chmod +x autoqa/scripts/setup_permissions.sh
|
|
./autoqa/scripts/setup_permissions.sh
|
|
|
|
- name: Clean existing Jan installations
|
|
run: |
|
|
./autoqa/scripts/ubuntu_cleanup.sh
|
|
|
|
- name: Download/Prepare Jan app
|
|
run: |
|
|
if [ "${{ inputs.source_type }}" = "local" ]; then
|
|
# Find the deb file in the artifact
|
|
DEB_FILE=$(find "${{ runner.temp }}/ubuntu-artifact" -name "*.deb" -type f | head -1)
|
|
if [ -n "$DEB_FILE" ]; then
|
|
echo "[SUCCESS] Found local installer: $DEB_FILE"
|
|
cp "$DEB_FILE" "/tmp/jan-installer.deb"
|
|
echo "[SUCCESS] Installer copied to: /tmp/jan-installer.deb"
|
|
echo "JAN_APP_PATH=/tmp/jan-installer.deb" >> $GITHUB_ENV
|
|
echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $GITHUB_ENV
|
|
if [ "${{ inputs.is_nightly }}" = "true" ]; then
|
|
echo "JAN_PROCESS_NAME=Jan-nightly" >> $GITHUB_ENV
|
|
else
|
|
echo "JAN_PROCESS_NAME=Jan" >> $GITHUB_ENV
|
|
fi
|
|
else
|
|
echo "[FAILED] No .deb file found in artifact"
|
|
exit 1
|
|
fi
|
|
else
|
|
# Use the existing download script for URLs
|
|
./autoqa/scripts/ubuntu_download.sh \
|
|
"${{ inputs.jan_app_ubuntu_source }}" \
|
|
"${{ inputs.is_nightly }}" \
|
|
"${{ vars.JAN_APP_URL_LINUX }}" \
|
|
"${{ vars.IS_NIGHTLY }}" \
|
|
"$DEFAULT_JAN_APP_URL" \
|
|
"$DEFAULT_IS_NIGHTLY"
|
|
|
|
# Set the correct environment variables for the test runner
|
|
echo "JAN_APP_PATH=/tmp/jan-installer.deb" >> $GITHUB_ENV
|
|
if [ "${{ inputs.is_nightly }}" = "true" ]; then
|
|
echo "JAN_PROCESS_NAME=Jan-nightly" >> $GITHUB_ENV
|
|
else
|
|
echo "JAN_PROCESS_NAME=Jan" >> $GITHUB_ENV
|
|
fi
|
|
fi
|
|
|
|
- name: Install Jan app
|
|
run: |
|
|
./autoqa/scripts/ubuntu_install.sh "$IS_NIGHTLY"
|
|
|
|
- name: Install Python dependencies
|
|
working-directory: autoqa
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
|
|
- name: Run Auto QA Tests
|
|
working-directory: autoqa
|
|
env:
|
|
RP_TOKEN: ${{ secrets.RP_TOKEN }}
|
|
ENABLE_REPORTPORTAL: 'true'
|
|
RP_ENDPOINT: 'https://reportportal.menlo.ai'
|
|
RP_PROJECT: 'default_personal'
|
|
MAX_TURNS: '50'
|
|
DELAY_BETWEEN_TESTS: '3'
|
|
LAUNCH_NAME: 'CI AutoQA Run Ubuntu - ${{ github.run_number }} - ${{ github.ref_name }}'
|
|
run: |
|
|
./scripts/run_tests.sh "$JAN_APP_PATH" "$JAN_PROCESS_NAME" "$RP_TOKEN" "ubuntu"
|
|
|
|
- name: Collect Jan logs for artifact upload
|
|
if: always()
|
|
run: |
|
|
mkdir -p autoqa/jan-logs
|
|
cp ~/.local/share/Jan-nightly/data/logs/*.log autoqa/jan-logs/ 2>/dev/null || true
|
|
cp ~/.local/share/Jan/data/logs/*.log autoqa/jan-logs/ 2>/dev/null || true
|
|
|
|
- name: Upload screen recordings
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
continue-on-error: true
|
|
with:
|
|
name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-recordings-${{ github.run_number }}-${{ runner.os }}
|
|
path: autoqa/recordings/
|
|
|
|
- name: Upload Jan logs
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-logs-${{ github.run_number }}-${{ runner.os }}
|
|
path: autoqa/jan-logs/
|
|
|
|
- name: Cleanup after tests
|
|
if: always()
|
|
run: |
|
|
./autoqa/scripts/ubuntu_post_cleanup.sh "$IS_NIGHTLY"
|
|
|
|
macos:
|
|
runs-on: macos-selfhosted-15-arm64-cua
|
|
timeout-minutes: 60
|
|
|
|
env:
|
|
DEFAULT_JAN_APP_URL: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_universal.dmg'
|
|
DEFAULT_IS_NIGHTLY: 'true'
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Python 3.13
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.13'
|
|
|
|
- name: Download artifact (if source_type is local)
|
|
if: inputs.source_type == 'local'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: ${{ inputs.artifact_name_macos }}
|
|
path: ${{ runner.temp }}/macos-artifact
|
|
|
|
- name: Setup script permissions
|
|
run: |
|
|
chmod +x autoqa/scripts/setup_permissions.sh
|
|
./autoqa/scripts/setup_permissions.sh
|
|
|
|
- name: Clean existing Jan installations
|
|
run: |
|
|
./autoqa/scripts/macos_cleanup.sh
|
|
|
|
- name: Download/Prepare Jan app
|
|
run: |
|
|
if [ "${{ inputs.source_type }}" = "local" ]; then
|
|
# Find the dmg file in the artifact
|
|
DMG_FILE=$(find "${{ runner.temp }}/macos-artifact" -name "*.dmg" -type f | head -1)
|
|
if [ -n "$DMG_FILE" ]; then
|
|
echo "[SUCCESS] Found local installer: $DMG_FILE"
|
|
cp "$DMG_FILE" "/tmp/jan-installer.dmg"
|
|
echo "[SUCCESS] Installer copied to: /tmp/jan-installer.dmg"
|
|
echo "JAN_APP_PATH=/tmp/jan-installer.dmg" >> $GITHUB_ENV
|
|
echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $GITHUB_ENV
|
|
if [ "${{ inputs.is_nightly }}" = "true" ]; then
|
|
echo "PROCESS_NAME=Jan-nightly" >> $GITHUB_ENV
|
|
else
|
|
echo "PROCESS_NAME=Jan" >> $GITHUB_ENV
|
|
fi
|
|
else
|
|
echo "[FAILED] No .dmg file found in artifact"
|
|
exit 1
|
|
fi
|
|
else
|
|
# Use the existing download script for URLs
|
|
./autoqa/scripts/macos_download.sh \
|
|
"${{ inputs.jan_app_macos_source }}" \
|
|
"${{ inputs.is_nightly }}" \
|
|
"${{ vars.JAN_APP_URL }}" \
|
|
"${{ vars.IS_NIGHTLY }}" \
|
|
"$DEFAULT_JAN_APP_URL" \
|
|
"$DEFAULT_IS_NIGHTLY"
|
|
|
|
# Set the correct environment variables for the test runner
|
|
echo "JAN_APP_PATH=/tmp/jan-installer.dmg" >> $GITHUB_ENV
|
|
if [ "${{ inputs.is_nightly }}" = "true" ]; then
|
|
echo "PROCESS_NAME=Jan-nightly" >> $GITHUB_ENV
|
|
else
|
|
echo "PROCESS_NAME=Jan" >> $GITHUB_ENV
|
|
fi
|
|
fi
|
|
|
|
- name: Install Jan app
|
|
run: |
|
|
./autoqa/scripts/macos_install.sh
|
|
|
|
- name: Install system dependencies
|
|
run: |
|
|
echo "Installing system dependencies for macOS..."
|
|
|
|
# Check if Homebrew is available
|
|
if command -v brew >/dev/null 2>&1; then
|
|
echo "Homebrew is available"
|
|
|
|
# Install python-tk if not available
|
|
python3 -c "import tkinter" 2>/dev/null || {
|
|
echo "Installing python-tk via Homebrew..."
|
|
brew install python-tk || true
|
|
}
|
|
else
|
|
echo "Homebrew not available, checking if tkinter works..."
|
|
python3 -c "import tkinter" || {
|
|
echo "[WARNING] tkinter not available and Homebrew not found"
|
|
echo "This may cause issues with mouse control"
|
|
}
|
|
fi
|
|
|
|
echo "System dependencies check completed"
|
|
|
|
- name: Install Python dependencies
|
|
run: |
|
|
cd autoqa
|
|
echo "Installing Python dependencies..."
|
|
pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
echo "[SUCCESS] Python dependencies installed"
|
|
|
|
- name: Setup ReportPortal environment
|
|
run: |
|
|
echo "Setting up ReportPortal environment..."
|
|
echo "RP_TOKEN=${{ secrets.RP_TOKEN }}" >> $GITHUB_ENV
|
|
echo "ReportPortal environment configured"
|
|
|
|
- name: Run E2E tests
|
|
env:
|
|
RP_TOKEN: ${{ secrets.RP_TOKEN }}
|
|
ENABLE_REPORTPORTAL: 'true'
|
|
RP_ENDPOINT: 'https://reportportal.menlo.ai'
|
|
RP_PROJECT: 'default_personal'
|
|
MAX_TURNS: '50'
|
|
DELAY_BETWEEN_TESTS: '3'
|
|
LAUNCH_NAME: 'CI AutoQA Run Macos - ${{ github.run_number }} - ${{ github.ref_name }}'
|
|
run: |
|
|
cd autoqa
|
|
echo "Starting E2E test execution..."
|
|
|
|
echo "Environment variables:"
|
|
echo "JAN_APP_PATH: $JAN_APP_PATH"
|
|
echo "PROCESS_NAME: $PROCESS_NAME"
|
|
echo "IS_NIGHTLY: $IS_NIGHTLY"
|
|
|
|
./scripts/run_tests.sh "$JAN_APP_PATH" "$PROCESS_NAME" "$RP_TOKEN" "macos"
|
|
|
|
- name: Collect Jan logs for artifact upload
|
|
if: always()
|
|
run: |
|
|
mkdir -p autoqa/jan-logs
|
|
cp ~/Library/Application\ Support/Jan-nightly/data/logs/*.log autoqa/jan-logs/ 2>/dev/null || true
|
|
cp ~/Library/Application\ Support/Jan/data/logs/*.log autoqa/jan-logs/ 2>/dev/null || true
|
|
|
|
- name: Upload screen recordings
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
continue-on-error: true
|
|
with:
|
|
name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-recordings-${{ github.run_number }}-${{ runner.os }}
|
|
path: autoqa/recordings/
|
|
|
|
- name: Upload Jan logs
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-logs-${{ github.run_number }}-${{ runner.os }}
|
|
path: autoqa/jan-logs/
|
|
|
|
- name: Cleanup after tests
|
|
if: always()
|
|
run: |
|
|
./autoqa/scripts/macos_post_cleanup.sh
|