[#249] Add e2e test cases (#266)

* chore: add e2e tests

* chore: do not retry failed test cases

electron/package.json

@@ -32,6 +32,7 @@
   },
   "scripts": {
     "lint": "eslint . --ext \".js,.jsx,.ts,.tsx\"",
+    "test:e2e": "playwright test --workers=2",
     "dev": "tsc -p . && electron .",
     "build": "tsc -p . && electron-builder -p never -m",
     "build:darwin": "tsc -p . && electron-builder -p never -m --x64 --arm64",
@@ -51,10 +52,12 @@
     "request-progress": "^3.0.0"
   },
   "devDependencies": {
+    "@playwright/test": "^1.38.1",
     "@typescript-eslint/eslint-plugin": "^6.7.3",
     "@typescript-eslint/parser": "^6.7.3",
     "electron": "26.2.1",
     "electron-builder": "^24.6.4",
+    "electron-playwright-helpers": "^1.6.0",
     "eslint-plugin-react": "^7.33.2"
   },
   "installConfig": {

electron/playwright.config.ts

@@ -0,0 +1,9 @@
+import { PlaywrightTestConfig } from "@playwright/test";
+
+const config: PlaywrightTestConfig = {
+  testDir: "./tests",
+  testIgnore: "./core/**",
+  retries: 0
+};
+
+export default config;

electron/tests/explore.e2e..spec.ts

@@ -0,0 +1,47 @@
+import { _electron as electron } from "playwright";
+import { ElectronApplication, Page, expect, test } from "@playwright/test";
+
+import {
+  findLatestBuild,
+  parseElectronApp,
+  stubDialog,
+} from "electron-playwright-helpers";
+
+let electronApp: ElectronApplication;
+let page: Page;
+
+test.beforeAll(async () => {
+  process.env.CI = "e2e";
+
+  const latestBuild = findLatestBuild("dist");
+  expect(latestBuild).toBeTruthy();
+
+  // parse the packaged Electron app and find paths and other info
+  const appInfo = parseElectronApp(latestBuild);
+  expect(appInfo).toBeTruthy();
+
+  electronApp = await electron.launch({
+    args: [appInfo.main], // main file from package.json
+    executablePath: appInfo.executable, // path to the Electron executable
+  });
+  await stubDialog(electronApp, "showMessageBox", { response: 1 });
+
+  page = await electronApp.firstWindow();
+});
+
+test.afterAll(async () => {
+  await electronApp.close();
+  await page.close();
+});
+
+test("explores models", async () => {
+  await page.getByRole("button", { name: "Explore Models" }).first().click();
+  const header = await page
+    .getByRole("heading")
+    .filter({ hasText: "Explore Models" })
+    .first()
+    .isDisabled();
+  expect(header).toBe(false);
+
+  //   More test cases here...
+});

electron/tests/main.e2e.spec.ts

@@ -0,0 +1,59 @@
+import { _electron as electron } from "playwright";
+import { ElectronApplication, Page, expect, test } from "@playwright/test";
+
+import {
+  findLatestBuild,
+  parseElectronApp,
+  stubDialog,
+} from "electron-playwright-helpers";
+
+let electronApp: ElectronApplication;
+let page: Page;
+
+test.beforeAll(async () => {
+  process.env.CI = "e2e";
+
+  const latestBuild = findLatestBuild("dist");
+  expect(latestBuild).toBeTruthy();
+
+  // parse the packaged Electron app and find paths and other info
+  const appInfo = parseElectronApp(latestBuild);
+  expect(appInfo).toBeTruthy();
+  expect(appInfo.arch).toBeTruthy();
+  expect(appInfo.arch).toBe(process.arch);
+  expect(appInfo.asar).toBe(true);
+  expect(appInfo.executable).toBeTruthy();
+  expect(appInfo.main).toBeTruthy();
+  expect(appInfo.name).toBe("jan-electron");
+  expect(appInfo.packageJson).toBeTruthy();
+  expect(appInfo.packageJson.name).toBe("jan-electron");
+  expect(appInfo.platform).toBeTruthy();
+  expect(appInfo.platform).toBe(process.platform);
+  expect(appInfo.resourcesDir).toBeTruthy();
+
+  electronApp = await electron.launch({
+    args: [appInfo.main], // main file from package.json
+    executablePath: appInfo.executable, // path to the Electron executable
+  });
+  await stubDialog(electronApp, "showMessageBox", { response: 1 });
+
+  page = await electronApp.firstWindow();
+});
+
+test.afterAll(async () => {
+  await electronApp.close();
+  await page.close();
+});
+
+test("renders the home page", async () => {
+  expect(page).toBeDefined();
+
+  // Welcome text is available
+  const welcomeText = await page
+    .locator(".text-5xl", {
+      hasText: "Welcome,let’s download your first model",
+    })
+    .first()
+    .isDisabled();
+  expect(welcomeText).toBe(false);
+});

electron/tests/my-models.e2e.spec.ts

@@ -0,0 +1,46 @@
+import { _electron as electron } from "playwright";
+import { ElectronApplication, Page, expect, test } from "@playwright/test";
+
+import {
+  findLatestBuild,
+  parseElectronApp,
+  stubDialog,
+} from "electron-playwright-helpers";
+
+let electronApp: ElectronApplication;
+let page: Page;
+
+test.beforeAll(async () => {
+  process.env.CI = "e2e";
+
+  const latestBuild = findLatestBuild("dist");
+  expect(latestBuild).toBeTruthy();
+
+  // parse the packaged Electron app and find paths and other info
+  const appInfo = parseElectronApp(latestBuild);
+  expect(appInfo).toBeTruthy();
+
+  electronApp = await electron.launch({
+    args: [appInfo.main], // main file from package.json
+    executablePath: appInfo.executable, // path to the Electron executable
+  });
+  await stubDialog(electronApp, "showMessageBox", { response: 1 });
+
+  page = await electronApp.firstWindow();
+});
+
+test.afterAll(async () => {
+  await electronApp.close();
+  await page.close();
+});
+
+test("shows my models", async () => {
+  await page.getByRole("button", { name: "My Models" }).first().click();
+  const header = await page
+    .getByRole("heading")
+    .filter({ hasText: "My Models" })
+    .first()
+    .isDisabled();
+  expect(header).toBe(false);
+  //   More test cases here...
+});

electron/tests/navigation.e2e.spec.ts

@@ -0,0 +1,81 @@
+import { _electron as electron } from "playwright";
+import { ElectronApplication, Page, expect, test } from "@playwright/test";
+
+import {
+  findLatestBuild,
+  parseElectronApp,
+  stubDialog,
+} from "electron-playwright-helpers";
+
+let electronApp: ElectronApplication;
+let page: Page;
+
+test.beforeAll(async () => {
+  process.env.CI = "e2e";
+
+  const latestBuild = findLatestBuild("dist");
+  expect(latestBuild).toBeTruthy();
+
+  // parse the packaged Electron app and find paths and other info
+  const appInfo = parseElectronApp(latestBuild);
+  expect(appInfo).toBeTruthy();
+
+  electronApp = await electron.launch({
+    args: [appInfo.main], // main file from package.json
+    executablePath: appInfo.executable, // path to the Electron executable
+  });
+  await stubDialog(electronApp, "showMessageBox", { response: 1 });
+
+  page = await electronApp.firstWindow();
+});
+
+test.afterAll(async () => {
+  await electronApp.close();
+  await page.close();
+});
+
+test("renders left navigation panel", async () => {
+  // Chat History section is available
+  const chatSection = await page
+    .getByRole("heading")
+    .filter({ hasText: "CHAT HISTORY" })
+    .first()
+    .isDisabled();
+  expect(chatSection).toBe(false);
+
+  // Home actions
+  const newChatBtn = await page
+    .getByRole("button", { name: "New Chat" })
+    .first()
+    .isEnabled();
+  const exploreBtn = await page
+    .getByRole("button", { name: "Explore Models" })
+    .first()
+    .isEnabled();
+  const startConversation = await page
+    .getByRole("button", { name: "Start a Conversation" })
+    .first()
+    .isEnabled();
+  const discordBtn = await page
+    .getByRole("button", { name: "Discord" })
+    .first()
+    .isEnabled();
+  const myModelsBtn = await page
+    .getByRole("button", { name: "My Models" })
+    .first()
+    .isEnabled();
+  const settingsBtn = await page
+    .getByRole("button", { name: "Settings" })
+    .first()
+    .isEnabled();
+  expect(
+    [
+      newChatBtn,
+      exploreBtn,
+      startConversation,
+      discordBtn,
+      myModelsBtn,
+      settingsBtn,
+    ].filter((e) => !e).length
+  ).toBe(0);
+});

electron/tests/settings.e2e.spec.ts

@@ -0,0 +1,42 @@
+import { _electron as electron } from "playwright";
+import { ElectronApplication, Page, expect, test } from "@playwright/test";
+
+import {
+  findLatestBuild,
+  parseElectronApp,
+  stubDialog,
+} from "electron-playwright-helpers";
+
+let electronApp: ElectronApplication;
+let page: Page;
+
+test.beforeAll(async () => {
+  process.env.CI = "e2e";
+
+  const latestBuild = findLatestBuild("dist");
+  expect(latestBuild).toBeTruthy();
+
+  // parse the packaged Electron app and find paths and other info
+  const appInfo = parseElectronApp(latestBuild);
+  expect(appInfo).toBeTruthy();
+
+  electronApp = await electron.launch({
+    args: [appInfo.main], // main file from package.json
+    executablePath: appInfo.executable, // path to the Electron executable
+  });
+  await stubDialog(electronApp, "showMessageBox", { response: 1 });
+
+  page = await electronApp.firstWindow();
+});
+
+test.afterAll(async () => {
+  await electronApp.close();
+  await page.close();
+});
+
+test("shows settings", async () => {
+  await page.getByRole("button", { name: "Settings" }).first().click();
+
+  const pluginList = await page.getByTestId("plugin-item").count();
+  expect(pluginList).toBe(4);
+});

electron/tsconfig.json

@@ -2,13 +2,17 @@
   "compilerOptions": {
     "target": "es5",
     "module": "commonjs",
+    "noImplicitAny": true,
     "sourceMap": true,
     "strict": true,
     "outDir": "./build",
     "rootDir": "./",
     "noEmitOnError": true,
+    "baseUrl": ".",
     "allowJs": true,
+    "paths": { "*": ["node_modules/*"] },
     "typeRoots": ["node_modules/@types"]
   },
-  "exclude": ["core", "build", "node_modules"]
+  "include": ["./**/*.ts"],
+  "exclude": ["core", "build", "dist", "tests"]
 }

package.json

@@ -14,6 +14,8 @@
     ]
   },
   "scripts": {
+    "lint": "yarn workspace jan-electron lint && yarn workspace jan-web lint",
+    "test": "yarn workspace jan-electron test:e2e",
     "dev:electron": "yarn workspace jan-electron dev",
     "dev:web": "yarn workspace jan-web dev",
     "dev": "concurrently --kill-others \"yarn dev:web\" \"wait-on http://localhost:3000 && yarn dev:electron\"",

web/app/_components/Preferences.tsx

@@ -198,6 +198,7 @@ export const Preferences = () => {
               .map((e) => (
                 <div
                   key={e.name}
+                  data-testid="plugin-item"
                   className="flex flex-col h-full p-6 bg-white border border-gray-200 rounded-sm dark:border-gray-300"
                 >
                   <div className="flex flex-row space-x-2 items-center">