test: add testcases for light and dark theme on apperance

autoqa/individual_migration_runner.py

@@ -19,6 +19,18 @@ MIGRATION_TEST_CASES = {
         "verify_test": "models/verify-model-persistence.txt", 
         "description": "Tests that downloaded models persist after upgrade"
     },
+    "appearance_dark-theme": {
+        "name": "Dark Theme Appearance Migration",
+        "setup_test": "appearance/setup-dark-theme-appearance.txt",
+        "verify_test": "appearance/verify-dark-theme-appearance-persistence.txt",
+        "description": "Tests that the Dark theme setting persists after upgrade"
+    },
+    "appearance_light-theme": {
+        "name": "Light Theme Appearance Migration",
+        "setup_test": "appearance/setup-light-theme-appearance.txt",
+        "verify_test": "appearance/verify-light-theme-appearance-persistence.txt",
+        "description": "Tests that the Light theme setting persists after upgrade"
+    },
     "assistants": {
         "name": "Custom Assistants Migration",
         "setup_test": "assistants/setup-create-assistants.txt",
@@ -48,6 +60,12 @@ MIGRATION_TEST_CASES = {
         "setup_test": "settings/setup-https-proxy.txt",
         "verify_test": "settings/verify-https-proxy-persistence.txt",
         "description": "Tests that HTTPS proxy settings persist after upgrade"
+    },
+    "models_disable-model-providers": {
+        "name": "Model Providers Enabled/Disabled State Migration",
+        "setup_test": "models/setup-model-providers.txt",
+        "verify_test": "models/verify-model-providers-persistence.txt",
+        "description": "Ensures enabled/disabled state for providers persists after upgrade (Disabled: Llama.cpp, OpenAI, Anthropic; Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face)"
     }
 }
 

autoqa/tests/migration/appearance/setup-dark-theme-appearance.txt

@@ -28,9 +28,9 @@ call_user() # Submit the task and call the user when the task is unsolvable, or
 
 ## User Instruction
 
-You are changing the Appearance theme in the OLD version of the Jan application.
+You are changing the Appearance theme to Dark in the OLD version of the Jan application.
 
-PHASE: SETUP APPEARANCE THEME (OLD VERSION)
+PHASE: SETUP DARK THEME APPEARANCE (OLD VERSION)
 
 Step-by-step instructions:
 
@@ -42,7 +42,7 @@ Step-by-step instructions:
    - In the bottom-left menu, click **Settings**.
    - In the left sidebar, click **Appearance**.
 
-3. Change the theme:
+3. Change the theme to Dark:
    - In the **Theme** field, click the current theme menu (e.g., **System**, **Light**, or **Dark**).
    - Select **Dark** (only change if it is not already Dark).
 

autoqa/tests/migration/appearance/setup-light-theme-appearance.txt

@@ -0,0 +1,69 @@
+prompt = """
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+```
+Thought: ...
+Action: ...
+```
+
+## Action Space
+
+click(start_box='<|box_start|>(x1,y1)<|box_end|>')
+left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
+right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
+drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
+hotkey(key='')
+type(content='') #If you want to submit your input, use "\
+" at the end of `content`.
+scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished()
+call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
+
+
+## Note
+- Use Chinese in `Thought` part.
+- Summarize your next action (with its target element) in one sentence in `Thought` part.
+
+## User Instruction
+
+You are changing the Appearance theme to Light in the OLD version of the Jan application.
+
+PHASE: SETUP LIGHT THEME APPEARANCE (OLD VERSION)
+
+Step-by-step instructions:
+
+1. Open the Jan application (OLD version).
+   - If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing.
+   - If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing.
+
+2. Navigate to Appearance settings:
+   - In the bottom-left menu, click **Settings**.
+   - In the left sidebar, click **Appearance**.
+
+3. Change the theme to Light:
+   - In the **Theme** field, click the current theme menu (e.g., **System**, **Light**, or **Dark**).
+   - Select **Light** (only change if it is not already Light).
+
+4. Return result:
+   - If the Theme is set to **Light**, return:
+     {"result": True, "phase": "setup_complete"}
+   - If the Theme cannot be set to **Light**, return:
+     {"result": False, "phase": "setup_failed"}
+
+CRITICAL INSTRUCTIONS FOR FINAL RESPONSE:
+- You MUST respond in English only, not any other language
+- You MUST return ONLY the JSON format below, nothing else
+- Do NOT add any explanations, thoughts, or additional text
+
+   - If the configuration is successful, return:
+     {"result": True, "phase": "setup_complete"}
+   - If there are any issues, return:
+     {"result": False, "phase": "setup_failed"}
+
+IMPORTANT: 
+- Your response must be ONLY the JSON above
+- Do NOT add any other text before or after the JSON
+"""
+

autoqa/tests/migration/appearance/verify-dark-theme-appearance-persistence.txt

@@ -28,9 +28,9 @@ call_user() # Submit the task and call the user when the task is unsolvable, or
 
 ## User Instruction
 
-You are verifying that the Appearance theme persists after upgrading to the NEW version of the Jan application.
+You are verifying that the Dark theme Appearance setting persists after upgrading to the NEW version of the Jan application.
 
-PHASE: VERIFY APPEARANCE THEME PERSISTENCE (NEW VERSION)
+PHASE: VERIFY DARK THEME APPEARANCE PERSISTENCE (NEW VERSION)
 
 Step-by-step instructions:
 
@@ -42,7 +42,7 @@ Step-by-step instructions:
    - In the bottom-left menu, click **Settings**.
    - In the left sidebar, click **Appearance**.
 
-3. Verify theme persistence:
+3. Verify Dark theme persistence:
    - Check the **Theme** field menu and read the current theme value.
    - Confirm the theme is **Dark**.
 

autoqa/tests/migration/appearance/verify-light-theme-appearance-persistence.txt

@@ -0,0 +1,69 @@
+prompt = """
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+```
+Thought: ...
+Action: ...
+```
+
+## Action Space
+
+click(start_box='<|box_start|>(x1,y1)<|box_end|>')
+left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
+right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
+drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
+hotkey(key='')
+type(content='') #If you want to submit your input, use "\
+" at the end of `content`.
+scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished()
+call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
+
+
+## Note
+- Use Chinese in `Thought` part.
+- Summarize your next action (with its target element) in one sentence in `Thought` part.
+
+## User Instruction
+
+You are verifying that the Light theme Appearance setting persists after upgrading to the NEW version of the Jan application.
+
+PHASE: VERIFY LIGHT THEME APPEARANCE PERSISTENCE (NEW VERSION)
+
+Step-by-step instructions:
+
+1. Open the Jan application (NEW version).
+   - If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing.
+   - If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing.
+
+2. Navigate to Appearance settings:
+   - In the bottom-left menu, click **Settings**.
+   - In the left sidebar, click **Appearance**.
+
+3. Verify Light theme persistence:
+   - Check the **Theme** field menu and read the current theme value.
+   - Confirm the theme is **Light**.
+
+4. Return result:
+   - If the current theme is **Light**, return:
+     {"result": True, "phase": "verification_complete"}
+   - Otherwise, return:
+     {"result": False, "phase": "verification_failed"}
+
+CRITICAL INSTRUCTIONS FOR FINAL RESPONSE:
+- You MUST respond in English only, not any other language
+- You MUST return ONLY the JSON format below, nothing else
+- Do NOT add any explanations, thoughts, or additional text
+
+   - If the settings persist and are correct, return:
+     {"result": True, "phase": "verification_complete"}
+   - If there are any issues, return:
+     {"result": False, "phase": "verification_failed"}
+
+IMPORTANT: 
+- Your response must be ONLY the JSON above
+- Do NOT add any other text before or after the JSON
+"""
+

autoqa/tests/migration/models/setup-model-providers.txt

@@ -0,0 +1,81 @@
+prompt = """
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+```
+Thought: ...
+Action: ...
+```
+
+## Action Space
+
+click(start_box='<|box_start|>(x1,y1)<|box_end|>')
+left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
+right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
+drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
+hotkey(key='')
+type(content='') #If you want to submit your input, use "\
+" at the end of `content`.
+scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished()
+call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
+
+
+## Note
+- Use Chinese in `Thought` part.
+- Summarize your next action (with its target element) in one sentence in `Thought` part.
+
+## User Instruction
+
+You are configuring Model Providers in the OLD version of the Jan application to prepare for migration verification.
+
+PHASE: SETUP MODEL PROVIDERS (OLD VERSION)
+
+Desired state:
+- Disabled: Llama.cpp, OpenAI, Anthropic
+- Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face
+
+Step-by-step instructions:
+
+1. Open the Jan application (OLD version).
+   - If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing.
+   - If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing.
+
+2. Navigate to Model Providers settings:
+   - In the bottom-left menu, click **Settings**.
+   - In the left sidebar, click **Model Providers**.
+
+3. Set provider states:
+   - For each provider in the right panel, use the chip/toggle on the right side of the row to set the desired state:
+     - Llama.cpp: disable (toggle off if enabled), then call `wait()` once
+     - OpenAI: disable (toggle off if enabled), then call `wait()` once
+     - Anthropic: disable (toggle off if enabled), then call `wait()` once
+     - Cohere: enable (toggle on if disabled), then call `wait()` once
+     - OpenRouter: enable (toggle on if disabled), then call `wait()` once
+     - Mistral: enable (toggle on if disabled), then call `wait()` once
+     - Groq: enable (toggle on if disabled), then call `wait()` once
+     - Gemini: enable (toggle on if disabled), then call `wait()` once
+     - Hugging Face: enable (toggle on if disabled), then call `wait()` once
+
+4. Return result:
+   - If all providers are set to the desired states (Disabled: Llama.cpp, OpenAI, Anthropic; Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face), return:
+     {"result": True, "phase": "setup_complete"}
+   - If any provider cannot be set to the desired state, return:
+     {"result": False, "phase": "setup_failed"}
+
+CRITICAL INSTRUCTIONS FOR FINAL RESPONSE:
+- You MUST respond in English only, not any other language
+- You MUST return ONLY the JSON format below, nothing else
+- Do NOT add any explanations, thoughts, or additional text
+
+   - If the configuration is successful, return:
+     {"result": True, "phase": "setup_complete"}
+   - If there are any issues, return:
+     {"result": False, "phase": "setup_failed"}
+
+IMPORTANT: 
+- Your response must be ONLY the JSON above
+- Do NOT add any other text before or after the JSON
+"""
+

autoqa/tests/migration/models/verify-model-providers-persistence.txt

@@ -0,0 +1,81 @@
+prompt = """
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+```
+Thought: ...
+Action: ...
+```
+
+## Action Space
+
+click(start_box='<|box_start|>(x1,y1)<|box_end|>')
+left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
+right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
+drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
+hotkey(key='')
+type(content='') #If you want to submit your input, use "\
+" at the end of `content`.
+scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished()
+call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
+
+
+## Note
+- Use Chinese in `Thought` part.
+- Summarize your next action (with its target element) in one sentence in `Thought` part.
+
+## User Instruction
+
+You are verifying that the Model Providers enabled/disabled state persists after upgrading to the NEW version of the Jan application.
+
+PHASE: VERIFY MODEL PROVIDERS PERSISTENCE (NEW VERSION)
+
+Expected persisted state:
+- Disabled: Llama.cpp, OpenAI, Anthropic
+- Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face
+
+Step-by-step instructions:
+
+1. Open the Jan application (NEW version).
+   - If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing.
+   - If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing.
+
+2. Navigate to Model Providers settings:
+   - In the bottom-left menu, click **Settings**.
+   - In the left sidebar, click **Model Providers**.
+
+3. Verify provider states:
+   - For each provider in the right panel, read the chip/toggle state and confirm it matches the expected persisted state:
+     - Llama.cpp: disabled
+     - OpenAI: disabled
+     - Anthropic: disabled
+     - Cohere: enabled
+     - OpenRouter: enabled
+     - Mistral: enabled
+     - Groq: enabled
+     - Gemini: enabled
+     - Hugging Face: enabled
+
+4. Return result:
+   - If all providers match the expected states, return:
+     {"result": True, "phase": "verification_complete"}
+   - If any provider does not match the expected state, return:
+     {"result": False, "phase": "verification_failed"}
+
+CRITICAL INSTRUCTIONS FOR FINAL RESPONSE:
+- You MUST respond in English only, not any other language
+- You MUST return ONLY the JSON format below, nothing else
+- Do NOT add any explanations, thoughts, or additional text
+
+   - If the settings persist and are correct, return:
+     {"result": True, "phase": "verification_complete"}
+   - If there are any issues, return:
+     {"result": False, "phase": "verification_failed"}
+
+IMPORTANT: 
+- Your response must be ONLY the JSON above
+- Do NOT add any other text before or after the JSON
+"""
+