499 lines
20 KiB
Python
499 lines
20 KiB
Python
import asyncio
|
|
import logging
|
|
import os
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from utils import force_close_jan, is_jan_running
|
|
from migration_utils import install_jan_version, prepare_migration_environment
|
|
from test_runner import run_single_test_with_timeout
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Migration test case definitions - organized by QA checklist categories
|
|
MIGRATION_TEST_CASES = {
|
|
"models": {
|
|
"name": "Model Downloads Migration",
|
|
"setup_test": "models/setup-download-model.txt",
|
|
"verify_test": "models/verify-model-persistence.txt",
|
|
"description": "Tests that downloaded models persist after upgrade"
|
|
},
|
|
"appearance_dark-theme": {
|
|
"name": "Dark Theme Appearance Migration",
|
|
"setup_test": "appearance/setup-dark-theme-appearance.txt",
|
|
"verify_test": "appearance/verify-dark-theme-appearance-persistence.txt",
|
|
"description": "Tests that the Dark theme setting persists after upgrade"
|
|
},
|
|
"appearance_light-theme": {
|
|
"name": "Light Theme Appearance Migration",
|
|
"setup_test": "appearance/setup-light-theme-appearance.txt",
|
|
"verify_test": "appearance/verify-light-theme-appearance-persistence.txt",
|
|
"description": "Tests that the Light theme setting persists after upgrade"
|
|
},
|
|
"appearance_font-size": {
|
|
"name": "Font Size Appearance Migration",
|
|
"setup_test": "appearance/setup-font-size-appearance.txt",
|
|
"verify_test": "appearance/verify-font-size-appearance-persistence.txt",
|
|
"description": "Tests that the Font Size setting (Extra Large) persists after upgrade"
|
|
},
|
|
"assistants": {
|
|
"name": "Custom Assistants Migration",
|
|
"setup_test": "assistants/setup-create-assistants.txt",
|
|
"verify_test": "assistants/verify-create-assistant-persistence.txt",
|
|
"description": "Tests that custom assistants persist after upgrade"
|
|
},
|
|
# "assistants-complete": {
|
|
# "name": "Complete Assistants Migration (Create + Chat)",
|
|
# "setup_tests": [
|
|
# "assistants/setup-create-assistants.txt",
|
|
# "assistants/setup-chat-with-assistant.txt"
|
|
# ],
|
|
# "verify_tests": [
|
|
# "assistants/verify-create-assistant-persistence.txt",
|
|
# "assistants/verify-chat-with-assistant-persistence.txt"
|
|
# ],
|
|
# "description": "Tests that custom assistants creation and chat functionality persist after upgrade (batch mode only)"
|
|
# },
|
|
"modify-local-api-server": {
|
|
"name": "Modify Local API Server Settings Migration",
|
|
"setup_test": "settings/setup-local-api-server.txt",
|
|
"verify_test": "settings/verify-local-api-server-persistence.txt",
|
|
"description": "Tests that local API server settings (port and API prefix) persist after upgrade"
|
|
},
|
|
"modify-https-proxy": {
|
|
"name": "Modify HTTPS Proxy Settings Migration",
|
|
"setup_test": "settings/setup-https-proxy.txt",
|
|
"verify_test": "settings/verify-https-proxy-persistence.txt",
|
|
"description": "Tests that HTTPS proxy settings persist after upgrade"
|
|
},
|
|
"models_disable-model-providers": {
|
|
"name": "Model Providers Enabled/Disabled State Migration",
|
|
"setup_test": "models/setup-model-providers.txt",
|
|
"verify_test": "models/verify-model-providers-persistence.txt",
|
|
"description": "Ensures enabled/disabled state for providers persists after upgrade (Disabled: Llama.cpp, OpenAI, Anthropic; Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face)"
|
|
}
|
|
}
|
|
|
|
async def run_individual_migration_test(computer, test_case_key, old_version_path, new_version_path,
|
|
rp_client=None, launch_id=None, max_turns=30, agent_config=None,
|
|
enable_reportportal=False):
|
|
"""
|
|
Run a single migration test case
|
|
|
|
Args:
|
|
computer: Computer agent instance
|
|
test_case_key: Key identifying the test case (e.g., "models", "chat-threads")
|
|
old_version_path: Path to old version installer
|
|
new_version_path: Path to new version installer
|
|
rp_client: ReportPortal client (optional)
|
|
launch_id: ReportPortal launch ID (optional)
|
|
max_turns: Maximum turns per test phase
|
|
agent_config: Agent configuration
|
|
enable_reportportal: Whether to upload to ReportPortal
|
|
"""
|
|
if test_case_key not in MIGRATION_TEST_CASES:
|
|
raise ValueError(f"Unknown test case: {test_case_key}")
|
|
|
|
test_case = MIGRATION_TEST_CASES[test_case_key]
|
|
|
|
logger.info("=" * 80)
|
|
logger.info(f"RUNNING MIGRATION TEST: {test_case['name'].upper()}")
|
|
logger.info("=" * 80)
|
|
logger.info(f"Description: {test_case['description']}")
|
|
logger.info(f"Setup Test: tests/migration/{test_case['setup_test']}")
|
|
logger.info(f"Verify Test: tests/migration/{test_case['verify_test']}")
|
|
logger.info("")
|
|
logger.info("Test Flow:")
|
|
logger.info(" 1. Install OLD version → Run SETUP test")
|
|
logger.info(" 2. Install NEW version → Run VERIFY test")
|
|
logger.info(" 3. Cleanup and prepare for next test")
|
|
logger.info("")
|
|
|
|
migration_result = {
|
|
"test_case": test_case_key,
|
|
"test_name": test_case["name"],
|
|
"overall_success": False,
|
|
"old_version_setup": False,
|
|
"new_version_install": False,
|
|
"upgrade_verification": False,
|
|
"error_message": None
|
|
}
|
|
|
|
try:
|
|
# Prepare migration environment
|
|
env_setup = prepare_migration_environment()
|
|
logger.info(f"Migration environment prepared: {env_setup}")
|
|
|
|
# Phase 1: Install old version and run setup test
|
|
logger.info("PHASE 1: Installing old version and running setup test")
|
|
logger.info("-" * 60)
|
|
|
|
install_jan_version(old_version_path, "old")
|
|
time.sleep(10) # Wait for Jan to be ready
|
|
|
|
# Load and run setup test
|
|
setup_test_path = f"tests/migration/{test_case['setup_test']}"
|
|
if not os.path.exists(setup_test_path):
|
|
raise FileNotFoundError(f"Setup test file not found: {setup_test_path}")
|
|
|
|
with open(setup_test_path, "r", encoding="utf-8") as f:
|
|
setup_content = f.read()
|
|
|
|
setup_test_data = {
|
|
"path": test_case['setup_test'],
|
|
"prompt": setup_content
|
|
}
|
|
|
|
setup_result = await run_single_test_with_timeout(
|
|
computer=computer,
|
|
test_data=setup_test_data,
|
|
rp_client=rp_client,
|
|
launch_id=launch_id,
|
|
max_turns=max_turns,
|
|
jan_app_path=None, # Auto-detect
|
|
jan_process_name="Jan.exe",
|
|
agent_config=agent_config,
|
|
enable_reportportal=enable_reportportal
|
|
)
|
|
|
|
migration_result["old_version_setup"] = setup_result.get("success", False) if setup_result else False
|
|
logger.info(f"Setup phase result: {migration_result['old_version_setup']}")
|
|
|
|
if not migration_result["old_version_setup"]:
|
|
migration_result["error_message"] = f"Failed to setup {test_case['name']} on old version"
|
|
return migration_result
|
|
|
|
# Phase 2: Install new version (upgrade)
|
|
logger.info("PHASE 2: Installing new version (upgrade)")
|
|
logger.info("-" * 60)
|
|
|
|
# Force close Jan before installing new version
|
|
force_close_jan("Jan.exe")
|
|
force_close_jan("Jan-nightly.exe")
|
|
time.sleep(5)
|
|
|
|
# Install new version
|
|
install_jan_version(new_version_path, "new")
|
|
migration_result["new_version_install"] = True
|
|
time.sleep(10) # Wait for new version to be ready
|
|
|
|
# Phase 3: Run verification test on new version (includes data integrity check)
|
|
logger.info("PHASE 3: Running verification test on new version")
|
|
logger.info("-" * 60)
|
|
|
|
# Load and run verification test
|
|
verify_test_path = f"tests/migration/{test_case['verify_test']}"
|
|
if not os.path.exists(verify_test_path):
|
|
raise FileNotFoundError(f"Verification test file not found: {verify_test_path}")
|
|
|
|
with open(verify_test_path, "r", encoding="utf-8") as f:
|
|
verify_content = f.read()
|
|
|
|
verify_test_data = {
|
|
"path": test_case['verify_test'],
|
|
"prompt": verify_content
|
|
}
|
|
|
|
verify_result = await run_single_test_with_timeout(
|
|
computer=computer,
|
|
test_data=verify_test_data,
|
|
rp_client=rp_client,
|
|
launch_id=launch_id,
|
|
max_turns=max_turns,
|
|
jan_app_path=None, # Auto-detect
|
|
jan_process_name="Jan.exe",
|
|
agent_config=agent_config,
|
|
enable_reportportal=enable_reportportal
|
|
)
|
|
|
|
migration_result["upgrade_verification"] = verify_result.get("success", False) if verify_result else False
|
|
logger.info(f"Verification phase result: {migration_result['upgrade_verification']}")
|
|
|
|
# Overall success check
|
|
migration_result["overall_success"] = (
|
|
migration_result["old_version_setup"] and
|
|
migration_result["new_version_install"] and
|
|
migration_result["upgrade_verification"]
|
|
)
|
|
|
|
logger.info("=" * 80)
|
|
logger.info(f"MIGRATION TEST COMPLETED: {test_case['name'].upper()}")
|
|
logger.info("=" * 80)
|
|
logger.info(f"Overall Success: {migration_result['overall_success']}")
|
|
logger.info(f"Old Version Setup: {migration_result['old_version_setup']}")
|
|
logger.info(f"New Version Install: {migration_result['new_version_install']}")
|
|
logger.info(f"Upgrade Verification: {migration_result['upgrade_verification']}")
|
|
|
|
return migration_result
|
|
|
|
except Exception as e:
|
|
logger.error(f"Migration test {test_case['name']} failed with exception: {e}")
|
|
migration_result["error_message"] = str(e)
|
|
return migration_result
|
|
finally:
|
|
# Cleanup: Force close any remaining Jan processes
|
|
force_close_jan("Jan.exe")
|
|
force_close_jan("Jan-nightly.exe")
|
|
|
|
async def run_assistant_batch_migration_test(computer, old_version_path, new_version_path,
|
|
rp_client=None, launch_id=None, max_turns=30, agent_config=None,
|
|
enable_reportportal=False):
|
|
"""
|
|
Run both assistant test cases in batch mode:
|
|
- Setup both assistant tests on old version
|
|
- Upgrade to new version
|
|
- Verify both assistant tests on new version
|
|
"""
|
|
assistant_test_cases = ["assistants", "assistant-chat"]
|
|
|
|
logger.info("=" * 100)
|
|
logger.info("RUNNING ASSISTANT BATCH MIGRATION TESTS")
|
|
logger.info("=" * 100)
|
|
logger.info(f"Test cases: {', '.join(assistant_test_cases)}")
|
|
logger.info("Approach: Setup Both → Upgrade → Verify Both")
|
|
logger.info("")
|
|
|
|
batch_result = {
|
|
"overall_success": False,
|
|
"setup_phase_success": False,
|
|
"upgrade_success": False,
|
|
"verification_phase_success": False,
|
|
"setup_results": {},
|
|
"verify_results": {},
|
|
"error_message": None
|
|
}
|
|
|
|
try:
|
|
# Prepare migration environment
|
|
env_setup = prepare_migration_environment()
|
|
logger.info(f"Migration environment prepared: {env_setup}")
|
|
|
|
# PHASE 1: Install old version and run BOTH setup tests
|
|
logger.info("=" * 80)
|
|
logger.info("PHASE 1: BATCH SETUP ON OLD VERSION")
|
|
logger.info("=" * 80)
|
|
|
|
install_jan_version(old_version_path, "old")
|
|
time.sleep(15) # Extra wait time for stability
|
|
|
|
setup_failures = 0
|
|
|
|
for i, test_case_key in enumerate(assistant_test_cases, 1):
|
|
test_case = MIGRATION_TEST_CASES[test_case_key]
|
|
logger.info(f"[{i}/{len(assistant_test_cases)}] Running setup: {test_case['name']}")
|
|
|
|
# Load and run setup test
|
|
setup_test_path = f"tests/migration/{test_case['setup_test']}"
|
|
if not os.path.exists(setup_test_path):
|
|
logger.error(f"Setup test file not found: {setup_test_path}")
|
|
batch_result["setup_results"][test_case_key] = False
|
|
setup_failures += 1
|
|
continue
|
|
|
|
with open(setup_test_path, "r") as f:
|
|
setup_content = f.read()
|
|
|
|
setup_test_data = {
|
|
"path": test_case['setup_test'],
|
|
"prompt": setup_content
|
|
}
|
|
|
|
setup_result = await run_single_test_with_timeout(
|
|
computer=computer,
|
|
test_data=setup_test_data,
|
|
rp_client=rp_client,
|
|
launch_id=launch_id,
|
|
max_turns=max_turns,
|
|
jan_app_path=None,
|
|
jan_process_name="Jan.exe",
|
|
agent_config=agent_config,
|
|
enable_reportportal=enable_reportportal
|
|
)
|
|
|
|
success = setup_result.get("success", False) if setup_result else False
|
|
batch_result["setup_results"][test_case_key] = success
|
|
|
|
if success:
|
|
logger.info(f"✅ Setup {test_case_key}: SUCCESS")
|
|
else:
|
|
logger.error(f"❌ Setup {test_case_key}: FAILED")
|
|
setup_failures += 1
|
|
|
|
# Small delay between setups
|
|
time.sleep(3)
|
|
|
|
batch_result["setup_phase_success"] = setup_failures == 0
|
|
logger.info(f"Setup phase complete: {len(assistant_test_cases) - setup_failures}/{len(assistant_test_cases)} successful")
|
|
|
|
# PHASE 2: Upgrade to new version
|
|
logger.info("=" * 80)
|
|
logger.info("PHASE 2: UPGRADING TO NEW VERSION")
|
|
logger.info("=" * 80)
|
|
|
|
force_close_jan("Jan.exe")
|
|
force_close_jan("Jan-nightly.exe")
|
|
time.sleep(5)
|
|
|
|
install_jan_version(new_version_path, "new")
|
|
batch_result["upgrade_success"] = True
|
|
time.sleep(15) # Extra wait time after upgrade
|
|
|
|
# PHASE 3: Run BOTH verification tests on new version
|
|
logger.info("=" * 80)
|
|
logger.info("PHASE 3: BATCH VERIFICATION ON NEW VERSION")
|
|
logger.info("=" * 80)
|
|
|
|
verify_failures = 0
|
|
|
|
for i, test_case_key in enumerate(assistant_test_cases, 1):
|
|
test_case = MIGRATION_TEST_CASES[test_case_key]
|
|
logger.info(f"[{i}/{len(assistant_test_cases)}] Running verification: {test_case['name']}")
|
|
|
|
# Load and run verification test
|
|
verify_test_path = f"tests/migration/{test_case['verify_test']}"
|
|
if not os.path.exists(verify_test_path):
|
|
logger.error(f"Verification test file not found: {verify_test_path}")
|
|
batch_result["verify_results"][test_case_key] = False
|
|
verify_failures += 1
|
|
continue
|
|
|
|
with open(verify_test_path, "r") as f:
|
|
verify_content = f.read()
|
|
|
|
verify_test_data = {
|
|
"path": test_case['verify_test'],
|
|
"prompt": verify_content
|
|
}
|
|
|
|
verify_result = await run_single_test_with_timeout(
|
|
computer=computer,
|
|
test_data=verify_test_data,
|
|
rp_client=rp_client,
|
|
launch_id=launch_id,
|
|
max_turns=max_turns,
|
|
jan_app_path=None,
|
|
jan_process_name="Jan.exe",
|
|
agent_config=agent_config,
|
|
enable_reportportal=enable_reportportal
|
|
)
|
|
|
|
success = verify_result.get("success", False) if verify_result else False
|
|
batch_result["verify_results"][test_case_key] = success
|
|
|
|
if success:
|
|
logger.info(f"✅ Verify {test_case_key}: SUCCESS")
|
|
else:
|
|
logger.error(f"❌ Verify {test_case_key}: FAILED")
|
|
verify_failures += 1
|
|
|
|
# Small delay between verifications
|
|
time.sleep(3)
|
|
|
|
batch_result["verification_phase_success"] = verify_failures == 0
|
|
logger.info(f"Verification phase complete: {len(assistant_test_cases) - verify_failures}/{len(assistant_test_cases)} successful")
|
|
|
|
# Overall success calculation
|
|
batch_result["overall_success"] = (
|
|
batch_result["setup_phase_success"] and
|
|
batch_result["upgrade_success"] and
|
|
batch_result["verification_phase_success"]
|
|
)
|
|
|
|
# Final summary
|
|
logger.info("=" * 100)
|
|
logger.info("ASSISTANT BATCH MIGRATION TEST SUMMARY")
|
|
logger.info("=" * 100)
|
|
logger.info(f"Overall Success: {batch_result['overall_success']}")
|
|
logger.info(f"Setup Phase: {batch_result['setup_phase_success']} ({len(assistant_test_cases) - setup_failures}/{len(assistant_test_cases)})")
|
|
logger.info(f"Upgrade Phase: {batch_result['upgrade_success']}")
|
|
logger.info(f"Verification Phase: {batch_result['verification_phase_success']} ({len(assistant_test_cases) - verify_failures}/{len(assistant_test_cases)})")
|
|
logger.info("")
|
|
logger.info("Detailed Results:")
|
|
for test_case_key in assistant_test_cases:
|
|
setup_status = "✅" if batch_result["setup_results"].get(test_case_key, False) else "❌"
|
|
verify_status = "✅" if batch_result["verify_results"].get(test_case_key, False) else "❌"
|
|
logger.info(f" {test_case_key.ljust(20)}: Setup {setup_status} | Verify {verify_status}")
|
|
|
|
return batch_result
|
|
|
|
except Exception as e:
|
|
logger.error(f"Assistant batch migration test failed with exception: {e}")
|
|
batch_result["error_message"] = str(e)
|
|
return batch_result
|
|
finally:
|
|
# Cleanup
|
|
force_close_jan("Jan.exe")
|
|
force_close_jan("Jan-nightly.exe")
|
|
|
|
async def run_all_migration_tests(computer, old_version_path, new_version_path, rp_client=None,
|
|
launch_id=None, max_turns=30, agent_config=None, enable_reportportal=False,
|
|
test_cases=None):
|
|
"""
|
|
Run multiple migration test cases
|
|
|
|
Args:
|
|
test_cases: List of test case keys to run. If None, runs all test cases.
|
|
"""
|
|
if test_cases is None:
|
|
test_cases = list(MIGRATION_TEST_CASES.keys())
|
|
|
|
logger.info("=" * 100)
|
|
logger.info("RUNNING ALL MIGRATION TESTS")
|
|
logger.info("=" * 100)
|
|
logger.info(f"Test cases to run: {', '.join(test_cases)}")
|
|
|
|
results = {}
|
|
overall_success = True
|
|
|
|
for i, test_case_key in enumerate(test_cases, 1):
|
|
logger.info(f"\n[{i}/{len(test_cases)}] Starting migration test: {test_case_key}")
|
|
|
|
result = await run_individual_migration_test(
|
|
computer=computer,
|
|
test_case_key=test_case_key,
|
|
old_version_path=old_version_path,
|
|
new_version_path=new_version_path,
|
|
rp_client=rp_client,
|
|
launch_id=launch_id,
|
|
max_turns=max_turns,
|
|
agent_config=agent_config,
|
|
enable_reportportal=enable_reportportal
|
|
)
|
|
|
|
results[test_case_key] = result
|
|
if not result["overall_success"]:
|
|
overall_success = False
|
|
|
|
# Add delay between test cases
|
|
if i < len(test_cases):
|
|
logger.info("Waiting 30 seconds before next migration test...")
|
|
time.sleep(30)
|
|
|
|
# Final summary
|
|
logger.info("=" * 100)
|
|
logger.info("MIGRATION TESTS SUMMARY")
|
|
logger.info("=" * 100)
|
|
|
|
passed = sum(1 for r in results.values() if r["overall_success"])
|
|
failed = len(results) - passed
|
|
|
|
logger.info(f"Total tests: {len(results)}")
|
|
logger.info(f"Passed: {passed}")
|
|
logger.info(f"Failed: {failed}")
|
|
logger.info(f"Overall success: {overall_success}")
|
|
|
|
for test_case_key, result in results.items():
|
|
status = "PASS" if result["overall_success"] else "FAIL"
|
|
logger.info(f" {test_case_key}: {status}")
|
|
if result["error_message"]:
|
|
logger.info(f" Error: {result['error_message']}")
|
|
|
|
return {
|
|
"overall_success": overall_success,
|
|
"total_tests": len(results),
|
|
"passed": passed,
|
|
"failed": failed,
|
|
"results": results
|
|
}
|