From 7e41e1b0b6ea35ba89fb4ab4321e463b2e33bfaa Mon Sep 17 00:00:00 2001 From: Minh141120 Date: Thu, 4 Sep 2025 09:06:15 +0700 Subject: [PATCH] test: add testcases for light and dark theme on apperance --- autoqa/individual_migration_runner.py | 18 +++++ ...ce.txt => setup-dark-theme-appearance.txt} | 6 +- .../setup-light-theme-appearance.txt | 69 ++++++++++++++++ ...ify-dark-theme-appearance-persistence.txt} | 6 +- ...ify-light-theme-appearance-persistence.txt | 69 ++++++++++++++++ .../models/setup-model-providers.txt | 81 +++++++++++++++++++ .../verify-model-providers-persistence.txt | 81 +++++++++++++++++++ 7 files changed, 324 insertions(+), 6 deletions(-) rename autoqa/tests/migration/appearance/{setup-appearance.txt => setup-dark-theme-appearance.txt} (93%) create mode 100644 autoqa/tests/migration/appearance/setup-light-theme-appearance.txt rename autoqa/tests/migration/appearance/{verify-appearance-persistence.txt => verify-dark-theme-appearance-persistence.txt} (91%) create mode 100644 autoqa/tests/migration/appearance/verify-light-theme-appearance-persistence.txt create mode 100644 autoqa/tests/migration/models/setup-model-providers.txt create mode 100644 autoqa/tests/migration/models/verify-model-providers-persistence.txt diff --git a/autoqa/individual_migration_runner.py b/autoqa/individual_migration_runner.py index 17834ceb7..fb13b86fd 100644 --- a/autoqa/individual_migration_runner.py +++ b/autoqa/individual_migration_runner.py @@ -19,6 +19,18 @@ MIGRATION_TEST_CASES = { "verify_test": "models/verify-model-persistence.txt", "description": "Tests that downloaded models persist after upgrade" }, + "appearance_dark-theme": { + "name": "Dark Theme Appearance Migration", + "setup_test": "appearance/setup-dark-theme-appearance.txt", + "verify_test": "appearance/verify-dark-theme-appearance-persistence.txt", + "description": "Tests that the Dark theme setting persists after upgrade" + }, + "appearance_light-theme": { + "name": "Light Theme Appearance Migration", + "setup_test": "appearance/setup-light-theme-appearance.txt", + "verify_test": "appearance/verify-light-theme-appearance-persistence.txt", + "description": "Tests that the Light theme setting persists after upgrade" + }, "assistants": { "name": "Custom Assistants Migration", "setup_test": "assistants/setup-create-assistants.txt", @@ -48,6 +60,12 @@ MIGRATION_TEST_CASES = { "setup_test": "settings/setup-https-proxy.txt", "verify_test": "settings/verify-https-proxy-persistence.txt", "description": "Tests that HTTPS proxy settings persist after upgrade" + }, + "models_disable-model-providers": { + "name": "Model Providers Enabled/Disabled State Migration", + "setup_test": "models/setup-model-providers.txt", + "verify_test": "models/verify-model-providers-persistence.txt", + "description": "Ensures enabled/disabled state for providers persists after upgrade (Disabled: Llama.cpp, OpenAI, Anthropic; Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face)" } } diff --git a/autoqa/tests/migration/appearance/setup-appearance.txt b/autoqa/tests/migration/appearance/setup-dark-theme-appearance.txt similarity index 93% rename from autoqa/tests/migration/appearance/setup-appearance.txt rename to autoqa/tests/migration/appearance/setup-dark-theme-appearance.txt index 32334b339..418f18022 100644 --- a/autoqa/tests/migration/appearance/setup-appearance.txt +++ b/autoqa/tests/migration/appearance/setup-dark-theme-appearance.txt @@ -28,9 +28,9 @@ call_user() # Submit the task and call the user when the task is unsolvable, or ## User Instruction -You are changing the Appearance theme in the OLD version of the Jan application. +You are changing the Appearance theme to Dark in the OLD version of the Jan application. -PHASE: SETUP APPEARANCE THEME (OLD VERSION) +PHASE: SETUP DARK THEME APPEARANCE (OLD VERSION) Step-by-step instructions: @@ -42,7 +42,7 @@ Step-by-step instructions: - In the bottom-left menu, click **Settings**. - In the left sidebar, click **Appearance**. -3. Change the theme: +3. Change the theme to Dark: - In the **Theme** field, click the current theme menu (e.g., **System**, **Light**, or **Dark**). - Select **Dark** (only change if it is not already Dark). diff --git a/autoqa/tests/migration/appearance/setup-light-theme-appearance.txt b/autoqa/tests/migration/appearance/setup-light-theme-appearance.txt new file mode 100644 index 000000000..b9cfbf82a --- /dev/null +++ b/autoqa/tests/migration/appearance/setup-light-theme-appearance.txt @@ -0,0 +1,69 @@ +prompt = """ +You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. + +## Output Format +``` +Thought: ... +Action: ... +``` + +## Action Space + +click(start_box='<|box_start|>(x1,y1)<|box_end|>') +left_double(start_box='<|box_start|>(x1,y1)<|box_end|>') +right_single(start_box='<|box_start|>(x1,y1)<|box_end|>') +drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>') +hotkey(key='') +type(content='') #If you want to submit your input, use "\ +" at the end of `content`. +scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left') +wait() #Sleep for 5s and take a screenshot to check for any changes. +finished() +call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help. + + +## Note +- Use Chinese in `Thought` part. +- Summarize your next action (with its target element) in one sentence in `Thought` part. + +## User Instruction + +You are changing the Appearance theme to Light in the OLD version of the Jan application. + +PHASE: SETUP LIGHT THEME APPEARANCE (OLD VERSION) + +Step-by-step instructions: + +1. Open the Jan application (OLD version). + - If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing. + - If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing. + +2. Navigate to Appearance settings: + - In the bottom-left menu, click **Settings**. + - In the left sidebar, click **Appearance**. + +3. Change the theme to Light: + - In the **Theme** field, click the current theme menu (e.g., **System**, **Light**, or **Dark**). + - Select **Light** (only change if it is not already Light). + +4. Return result: + - If the Theme is set to **Light**, return: + {"result": True, "phase": "setup_complete"} + - If the Theme cannot be set to **Light**, return: + {"result": False, "phase": "setup_failed"} + +CRITICAL INSTRUCTIONS FOR FINAL RESPONSE: +- You MUST respond in English only, not any other language +- You MUST return ONLY the JSON format below, nothing else +- Do NOT add any explanations, thoughts, or additional text + + - If the configuration is successful, return: + {"result": True, "phase": "setup_complete"} + - If there are any issues, return: + {"result": False, "phase": "setup_failed"} + +IMPORTANT: +- Your response must be ONLY the JSON above +- Do NOT add any other text before or after the JSON +""" + diff --git a/autoqa/tests/migration/appearance/verify-appearance-persistence.txt b/autoqa/tests/migration/appearance/verify-dark-theme-appearance-persistence.txt similarity index 91% rename from autoqa/tests/migration/appearance/verify-appearance-persistence.txt rename to autoqa/tests/migration/appearance/verify-dark-theme-appearance-persistence.txt index 19ba31617..a7559d1cb 100644 --- a/autoqa/tests/migration/appearance/verify-appearance-persistence.txt +++ b/autoqa/tests/migration/appearance/verify-dark-theme-appearance-persistence.txt @@ -28,9 +28,9 @@ call_user() # Submit the task and call the user when the task is unsolvable, or ## User Instruction -You are verifying that the Appearance theme persists after upgrading to the NEW version of the Jan application. +You are verifying that the Dark theme Appearance setting persists after upgrading to the NEW version of the Jan application. -PHASE: VERIFY APPEARANCE THEME PERSISTENCE (NEW VERSION) +PHASE: VERIFY DARK THEME APPEARANCE PERSISTENCE (NEW VERSION) Step-by-step instructions: @@ -42,7 +42,7 @@ Step-by-step instructions: - In the bottom-left menu, click **Settings**. - In the left sidebar, click **Appearance**. -3. Verify theme persistence: +3. Verify Dark theme persistence: - Check the **Theme** field menu and read the current theme value. - Confirm the theme is **Dark**. diff --git a/autoqa/tests/migration/appearance/verify-light-theme-appearance-persistence.txt b/autoqa/tests/migration/appearance/verify-light-theme-appearance-persistence.txt new file mode 100644 index 000000000..557b2ecca --- /dev/null +++ b/autoqa/tests/migration/appearance/verify-light-theme-appearance-persistence.txt @@ -0,0 +1,69 @@ +prompt = """ +You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. + +## Output Format +``` +Thought: ... +Action: ... +``` + +## Action Space + +click(start_box='<|box_start|>(x1,y1)<|box_end|>') +left_double(start_box='<|box_start|>(x1,y1)<|box_end|>') +right_single(start_box='<|box_start|>(x1,y1)<|box_end|>') +drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>') +hotkey(key='') +type(content='') #If you want to submit your input, use "\ +" at the end of `content`. +scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left') +wait() #Sleep for 5s and take a screenshot to check for any changes. +finished() +call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help. + + +## Note +- Use Chinese in `Thought` part. +- Summarize your next action (with its target element) in one sentence in `Thought` part. + +## User Instruction + +You are verifying that the Light theme Appearance setting persists after upgrading to the NEW version of the Jan application. + +PHASE: VERIFY LIGHT THEME APPEARANCE PERSISTENCE (NEW VERSION) + +Step-by-step instructions: + +1. Open the Jan application (NEW version). + - If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing. + - If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing. + +2. Navigate to Appearance settings: + - In the bottom-left menu, click **Settings**. + - In the left sidebar, click **Appearance**. + +3. Verify Light theme persistence: + - Check the **Theme** field menu and read the current theme value. + - Confirm the theme is **Light**. + +4. Return result: + - If the current theme is **Light**, return: + {"result": True, "phase": "verification_complete"} + - Otherwise, return: + {"result": False, "phase": "verification_failed"} + +CRITICAL INSTRUCTIONS FOR FINAL RESPONSE: +- You MUST respond in English only, not any other language +- You MUST return ONLY the JSON format below, nothing else +- Do NOT add any explanations, thoughts, or additional text + + - If the settings persist and are correct, return: + {"result": True, "phase": "verification_complete"} + - If there are any issues, return: + {"result": False, "phase": "verification_failed"} + +IMPORTANT: +- Your response must be ONLY the JSON above +- Do NOT add any other text before or after the JSON +""" + diff --git a/autoqa/tests/migration/models/setup-model-providers.txt b/autoqa/tests/migration/models/setup-model-providers.txt new file mode 100644 index 000000000..155a2c051 --- /dev/null +++ b/autoqa/tests/migration/models/setup-model-providers.txt @@ -0,0 +1,81 @@ +prompt = """ +You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. + +## Output Format +``` +Thought: ... +Action: ... +``` + +## Action Space + +click(start_box='<|box_start|>(x1,y1)<|box_end|>') +left_double(start_box='<|box_start|>(x1,y1)<|box_end|>') +right_single(start_box='<|box_start|>(x1,y1)<|box_end|>') +drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>') +hotkey(key='') +type(content='') #If you want to submit your input, use "\ +" at the end of `content`. +scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left') +wait() #Sleep for 5s and take a screenshot to check for any changes. +finished() +call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help. + + +## Note +- Use Chinese in `Thought` part. +- Summarize your next action (with its target element) in one sentence in `Thought` part. + +## User Instruction + +You are configuring Model Providers in the OLD version of the Jan application to prepare for migration verification. + +PHASE: SETUP MODEL PROVIDERS (OLD VERSION) + +Desired state: +- Disabled: Llama.cpp, OpenAI, Anthropic +- Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face + +Step-by-step instructions: + +1. Open the Jan application (OLD version). + - If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing. + - If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing. + +2. Navigate to Model Providers settings: + - In the bottom-left menu, click **Settings**. + - In the left sidebar, click **Model Providers**. + +3. Set provider states: + - For each provider in the right panel, use the chip/toggle on the right side of the row to set the desired state: + - Llama.cpp: disable (toggle off if enabled), then call `wait()` once + - OpenAI: disable (toggle off if enabled), then call `wait()` once + - Anthropic: disable (toggle off if enabled), then call `wait()` once + - Cohere: enable (toggle on if disabled), then call `wait()` once + - OpenRouter: enable (toggle on if disabled), then call `wait()` once + - Mistral: enable (toggle on if disabled), then call `wait()` once + - Groq: enable (toggle on if disabled), then call `wait()` once + - Gemini: enable (toggle on if disabled), then call `wait()` once + - Hugging Face: enable (toggle on if disabled), then call `wait()` once + +4. Return result: + - If all providers are set to the desired states (Disabled: Llama.cpp, OpenAI, Anthropic; Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face), return: + {"result": True, "phase": "setup_complete"} + - If any provider cannot be set to the desired state, return: + {"result": False, "phase": "setup_failed"} + +CRITICAL INSTRUCTIONS FOR FINAL RESPONSE: +- You MUST respond in English only, not any other language +- You MUST return ONLY the JSON format below, nothing else +- Do NOT add any explanations, thoughts, or additional text + + - If the configuration is successful, return: + {"result": True, "phase": "setup_complete"} + - If there are any issues, return: + {"result": False, "phase": "setup_failed"} + +IMPORTANT: +- Your response must be ONLY the JSON above +- Do NOT add any other text before or after the JSON +""" + diff --git a/autoqa/tests/migration/models/verify-model-providers-persistence.txt b/autoqa/tests/migration/models/verify-model-providers-persistence.txt new file mode 100644 index 000000000..01ba18a5c --- /dev/null +++ b/autoqa/tests/migration/models/verify-model-providers-persistence.txt @@ -0,0 +1,81 @@ +prompt = """ +You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. + +## Output Format +``` +Thought: ... +Action: ... +``` + +## Action Space + +click(start_box='<|box_start|>(x1,y1)<|box_end|>') +left_double(start_box='<|box_start|>(x1,y1)<|box_end|>') +right_single(start_box='<|box_start|>(x1,y1)<|box_end|>') +drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>') +hotkey(key='') +type(content='') #If you want to submit your input, use "\ +" at the end of `content`. +scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left') +wait() #Sleep for 5s and take a screenshot to check for any changes. +finished() +call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help. + + +## Note +- Use Chinese in `Thought` part. +- Summarize your next action (with its target element) in one sentence in `Thought` part. + +## User Instruction + +You are verifying that the Model Providers enabled/disabled state persists after upgrading to the NEW version of the Jan application. + +PHASE: VERIFY MODEL PROVIDERS PERSISTENCE (NEW VERSION) + +Expected persisted state: +- Disabled: Llama.cpp, OpenAI, Anthropic +- Enabled: Cohere, OpenRouter, Mistral, Groq, Gemini, Hugging Face + +Step-by-step instructions: + +1. Open the Jan application (NEW version). + - If a dialog appears in the bottom-right corner titled **"Help Us Improve Jan"**, click **Deny** to dismiss it before continuing. + - If **New Version Available** popup appears on app launch (older versions), click **Remind Me Later** to dismiss it before continuing. + +2. Navigate to Model Providers settings: + - In the bottom-left menu, click **Settings**. + - In the left sidebar, click **Model Providers**. + +3. Verify provider states: + - For each provider in the right panel, read the chip/toggle state and confirm it matches the expected persisted state: + - Llama.cpp: disabled + - OpenAI: disabled + - Anthropic: disabled + - Cohere: enabled + - OpenRouter: enabled + - Mistral: enabled + - Groq: enabled + - Gemini: enabled + - Hugging Face: enabled + +4. Return result: + - If all providers match the expected states, return: + {"result": True, "phase": "verification_complete"} + - If any provider does not match the expected state, return: + {"result": False, "phase": "verification_failed"} + +CRITICAL INSTRUCTIONS FOR FINAL RESPONSE: +- You MUST respond in English only, not any other language +- You MUST return ONLY the JSON format below, nothing else +- Do NOT add any explanations, thoughts, or additional text + + - If the settings persist and are correct, return: + {"result": True, "phase": "verification_complete"} + - If there are any issues, return: + {"result": False, "phase": "verification_failed"} + +IMPORTANT: +- Your response must be ONLY the JSON above +- Do NOT add any other text before or after the JSON +""" +