Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: add testcases for light and dark theme on apperance

Browse Source
This commit is contained in:
Minh141120 2025-09-04 09:06:15 +07:00
parent a1007b51b6
commit 7e41e1b0b6
7 changed files with 324 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
autoqa/individual_migration_runner.py
View File

@ -19,6 +19,18 @@ MIGRATION_TEST_CASES = {
"verify_test": "models/verify-model-persistence.txt",
"description": "Tests that downloaded models persist after upgrade"
},
"appearance_dark-theme": {
"name": "Dark Theme Appearance Migration",
"setup_test": "appearance/setup-dark-theme-appearance.txt",
"verify_test": "appearance/verify-dark-theme-appearance-persistence.txt",
"description": "Tests that the Dark theme setting persists after upgrade"
},
"appearance_light-theme": {
"name": "Light Theme Appearance Migration",
"setup_test": "appearance/setup-light-theme-appearance.txt",
"verify_test": "appearance/verify-light-theme-appearance-persistence.txt",
"description": "Tests that the Light theme setting persists after upgrade"
},
"assistants": {
"name": "Custom Assistants Migration",
"setup_test": "assistants/setup-create-assistants.txt",
@ -48,6 +60,12 @@ MIGRATION_TEST_CASES = {
"setup_test": "settings/setup-https-proxy.txt",
"verify_test": "settings/verify-https-proxy-persistence.txt",
"description": "Tests that HTTPS proxy settings persist after upgrade"
},
"models_disable-model-providers": {
"name": "Model Providers Enabled/Disabled State Migration",
"setup_test": "models/setup-model-providers.txt",
"verify_test": "models/verify-model-providers-persistence.txt",
"description": "Ensures enabled/disabled state for providers persists after upgrade (Disabled: Llama.cpp, OpenAI, Anthropic; Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face)"
}
}

6
autoqa/tests/migration/appearance/setup-appearance.txt → autoqa/tests/migration/appearance/setup-dark-theme-appearance.txt
View File

@ -28,9 +28,9 @@ call_user() # Submit the task and call the user when the task is unsolvable, or
## User Instruction
You are changing the Appearance theme in the OLD version of the Jan application.
You are changing the Appearance theme to Dark in the OLD version of the Jan application.
PHASE: SETUP APPEARANCE THEME (OLD VERSION)
PHASE: SETUP DARK THEME APPEARANCE (OLD VERSION)
Step-by-step instructions:
@ -42,7 +42,7 @@ Step-by-step instructions:
- In the bottom-left menu, click **Settings**.
- In the left sidebar, click **Appearance**.
3. Change the theme:
3. Change the theme to Dark:
- In the **Theme** field, click the current theme menu (e.g., **System**, **Light**, or **Dark**).
- Select **Dark** (only change if it is not already Dark).

69
autoqa/tests/migration/appearance/setup-light-theme-appearance.txt Normal file
View File

@ -0,0 +1,69 @@
prompt = """
You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
## Output Format
```
Thought: ...
Action: ...
```
## Action Space
click(start_box='<|box_start|>(x1,y1)<|box_end|>')
left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
hotkey(key='')
type(content='') #If you want to submit your input, use "\
" at the end of `content`.
scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
wait() #Sleep for 5s and take a screenshot to check for any changes.
finished()
call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
## Note
- Use Chinese in `Thought` part.
- Summarize your next action (with its target element) in one sentence in `Thought` part.
## User Instruction
You are changing the Appearance theme to Light in the OLD version of the Jan application.
PHASE: SETUP LIGHT THEME APPEARANCE (OLD VERSION)
Step-by-step instructions:
1. Open the Jan application (OLD version).
- If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing.
- If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing.
2. Navigate to Appearance settings:
- In the bottom-left menu, click **Settings**.
- In the left sidebar, click **Appearance**.
3. Change the theme to Light:
- In the **Theme** field, click the current theme menu (e.g., **System**, **Light**, or **Dark**).
- Select **Light** (only change if it is not already Light).
4. Return result:
- If the Theme is set to **Light**, return:
{"result": True, "phase": "setup_complete"}
- If the Theme cannot be set to **Light**, return:
{"result": False, "phase": "setup_failed"}
CRITICAL INSTRUCTIONS FOR FINAL RESPONSE:
- You MUST respond in English only, not any other language
- You MUST return ONLY the JSON format below, nothing else
- Do NOT add any explanations, thoughts, or additional text
- If the configuration is successful, return:
{"result": True, "phase": "setup_complete"}
- If there are any issues, return:
{"result": False, "phase": "setup_failed"}
IMPORTANT:
- Your response must be ONLY the JSON above
- Do NOT add any other text before or after the JSON
"""

6
autoqa/tests/migration/appearance/verify-appearance-persistence.txt → autoqa/tests/migration/appearance/verify-dark-theme-appearance-persistence.txt
View File

@ -28,9 +28,9 @@ call_user() # Submit the task and call the user when the task is unsolvable, or
## User Instruction
You are verifying that the Appearance theme persists after upgrading to the NEW version of the Jan application.
You are verifying that the Dark theme Appearance setting persists after upgrading to the NEW version of the Jan application.
PHASE: VERIFY APPEARANCE THEME PERSISTENCE (NEW VERSION)
PHASE: VERIFY DARK THEME APPEARANCE PERSISTENCE (NEW VERSION)
Step-by-step instructions:
@ -42,7 +42,7 @@ Step-by-step instructions:
- In the bottom-left menu, click **Settings**.
- In the left sidebar, click **Appearance**.
3. Verify theme persistence:
3. Verify Dark theme persistence:
- Check the **Theme** field menu and read the current theme value.
- Confirm the theme is **Dark**.

69
autoqa/tests/migration/appearance/verify-light-theme-appearance-persistence.txt Normal file
View File

@ -0,0 +1,69 @@
prompt = """
You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
## Output Format
```
Thought: ...
Action: ...
```
## Action Space
click(start_box='<|box_start|>(x1,y1)<|box_end|>')
left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
hotkey(key='')
type(content='') #If you want to submit your input, use "\
" at the end of `content`.
scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
wait() #Sleep for 5s and take a screenshot to check for any changes.
finished()
call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
## Note
- Use Chinese in `Thought` part.
- Summarize your next action (with its target element) in one sentence in `Thought` part.
## User Instruction
You are verifying that the Light theme Appearance setting persists after upgrading to the NEW version of the Jan application.
PHASE: VERIFY LIGHT THEME APPEARANCE PERSISTENCE (NEW VERSION)
Step-by-step instructions:
1. Open the Jan application (NEW version).
- If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing.
- If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing.
2. Navigate to Appearance settings:
- In the bottom-left menu, click **Settings**.
- In the left sidebar, click **Appearance**.
3. Verify Light theme persistence:
- Check the **Theme** field menu and read the current theme value.
- Confirm the theme is **Light**.
4. Return result:
- If the current theme is **Light**, return:
{"result": True, "phase": "verification_complete"}
- Otherwise, return:
{"result": False, "phase": "verification_failed"}
CRITICAL INSTRUCTIONS FOR FINAL RESPONSE:
- You MUST respond in English only, not any other language
- You MUST return ONLY the JSON format below, nothing else
- Do NOT add any explanations, thoughts, or additional text
- If the settings persist and are correct, return:
{"result": True, "phase": "verification_complete"}
- If there are any issues, return:
{"result": False, "phase": "verification_failed"}
IMPORTANT:
- Your response must be ONLY the JSON above
- Do NOT add any other text before or after the JSON
"""

81
autoqa/tests/migration/models/setup-model-providers.txt Normal file
View File

@ -0,0 +1,81 @@
prompt = """
You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
## Output Format
```
Thought: ...
Action: ...
```
## Action Space
click(start_box='<|box_start|>(x1,y1)<|box_end|>')
left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
hotkey(key='')
type(content='') #If you want to submit your input, use "\
" at the end of `content`.
scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
wait() #Sleep for 5s and take a screenshot to check for any changes.
finished()
call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
## Note
- Use Chinese in `Thought` part.
- Summarize your next action (with its target element) in one sentence in `Thought` part.
## User Instruction
You are configuring Model Providers in the OLD version of the Jan application to prepare for migration verification.
PHASE: SETUP MODEL PROVIDERS (OLD VERSION)
Desired state:
- Disabled: Llama.cpp, OpenAI, Anthropic
- Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face
Step-by-step instructions:
1. Open the Jan application (OLD version).
- If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing.
- If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing.
2. Navigate to Model Providers settings:
- In the bottom-left menu, click **Settings**.
- In the left sidebar, click **Model Providers**.
3. Set provider states:
- For each provider in the right panel, use the chip/toggle on the right side of the row to set the desired state:
- Llama.cpp: disable (toggle off if enabled), then call `wait()` once
- OpenAI: disable (toggle off if enabled), then call `wait()` once
- Anthropic: disable (toggle off if enabled), then call `wait()` once
- Cohere: enable (toggle on if disabled), then call `wait()` once
- OpenRouter: enable (toggle on if disabled), then call `wait()` once
- Mistral: enable (toggle on if disabled), then call `wait()` once
- Groq: enable (toggle on if disabled), then call `wait()` once
- Gemini: enable (toggle on if disabled), then call `wait()` once
- Hugging Face: enable (toggle on if disabled), then call `wait()` once
4. Return result:
- If all providers are set to the desired states (Disabled: Llama.cpp, OpenAI, Anthropic; Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face), return:
{"result": True, "phase": "setup_complete"}
- If any provider cannot be set to the desired state, return:
{"result": False, "phase": "setup_failed"}
CRITICAL INSTRUCTIONS FOR FINAL RESPONSE:
- You MUST respond in English only, not any other language
- You MUST return ONLY the JSON format below, nothing else
- Do NOT add any explanations, thoughts, or additional text
- If the configuration is successful, return:
{"result": True, "phase": "setup_complete"}
- If there are any issues, return:
{"result": False, "phase": "setup_failed"}
IMPORTANT:
- Your response must be ONLY the JSON above
- Do NOT add any other text before or after the JSON
"""

81
autoqa/tests/migration/models/verify-model-providers-persistence.txt Normal file
View File

@ -0,0 +1,81 @@
prompt = """
You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
## Output Format
```
Thought: ...
Action: ...
```
## Action Space
click(start_box='<|box_start|>(x1,y1)<|box_end|>')
left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
hotkey(key='')
type(content='') #If you want to submit your input, use "\
" at the end of `content`.
scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
wait() #Sleep for 5s and take a screenshot to check for any changes.
finished()
call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
## Note
- Use Chinese in `Thought` part.
- Summarize your next action (with its target element) in one sentence in `Thought` part.
## User Instruction
You are verifying that the Model Providers enabled/disabled state persists after upgrading to the NEW version of the Jan application.
PHASE: VERIFY MODEL PROVIDERS PERSISTENCE (NEW VERSION)
Expected persisted state:
- Disabled: Llama.cpp, OpenAI, Anthropic
- Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face
Step-by-step instructions:
1. Open the Jan application (NEW version).
- If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing.
- If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing.
2. Navigate to Model Providers settings:
- In the bottom-left menu, click **Settings**.
- In the left sidebar, click **Model Providers**.
3. Verify provider states:
- For each provider in the right panel, read the chip/toggle state and confirm it matches the expected persisted state:
- Llama.cpp: disabled
- OpenAI: disabled
- Anthropic: disabled
- Cohere: enabled
- OpenRouter: enabled
- Mistral: enabled
- Groq: enabled
- Gemini: enabled
- Hugging Face: enabled
4. Return result:
- If all providers match the expected states, return:
{"result": True, "phase": "verification_complete"}
- If any provider does not match the expected state, return:
{"result": False, "phase": "verification_failed"}
CRITICAL INSTRUCTIONS FOR FINAL RESPONSE:
- You MUST respond in English only, not any other language
- You MUST return ONLY the JSON format below, nothing else
- Do NOT add any explanations, thoughts, or additional text
- If the settings persist and are correct, return:
{"result": True, "phase": "verification_complete"}
- If there are any issues, return:
{"result": False, "phase": "verification_failed"}
IMPORTANT:
- Your response must be ONLY the JSON above
- Do NOT add any other text before or after the JSON
"""