feat: Playwright capture screenshot of Electron desktop app (Jan) on failures (#1934)

* feat: Apply Screenshot on failures

* feat: set timeout by default

* chore: clean up import

electron/playwright.config.ts

@@ -1,9 +1,16 @@
 import { PlaywrightTestConfig } from '@playwright/test'
 
 const config: PlaywrightTestConfig = {
-  testDir: './tests',
+  testDir: './tests/e2e',
   retries: 0,
   globalTimeout: 300000,
+  use: {
+    screenshot: 'only-on-failure',
+    video: 'retain-on-failure',
+    trace: 'retain-on-failure',
+  },
+
+  reporter: [['html', { outputFolder: './playwright-report' }]],
 }
 
 export default config

electron/tests/e2e/hub.e2e.spec.ts

@@ -0,0 +1,34 @@
+import {
+  page,
+  test,
+  setupElectron,
+  teardownElectron,
+  TIMEOUT,
+} from '../pages/basePage'
+import { expect } from '@playwright/test'
+
+test.beforeAll(async () => {
+  const appInfo = await setupElectron()
+  expect(appInfo.asar).toBe(true)
+  expect(appInfo.executable).toBeTruthy()
+  expect(appInfo.main).toBeTruthy()
+  expect(appInfo.name).toBe('jan')
+  expect(appInfo.packageJson).toBeTruthy()
+  expect(appInfo.packageJson.name).toBe('jan')
+  expect(appInfo.platform).toBeTruthy()
+  expect(appInfo.platform).toBe(process.platform)
+  expect(appInfo.resourcesDir).toBeTruthy()
+})
+
+test.afterAll(async () => {
+  await teardownElectron()
+})
+
+test('explores hub', async () => {
+  await page.getByTestId('Hub').first().click({
+    timeout: TIMEOUT,
+  })
+  await page.getByTestId('hub-container-test-id').isVisible({
+    timeout: TIMEOUT,
+  })
+})

electron/tests/e2e/navigation.e2e.spec.ts

@@ -0,0 +1,38 @@
+import { expect } from '@playwright/test'
+import {
+  page,
+  setupElectron,
+  TIMEOUT,
+  test,
+  teardownElectron,
+} from '../pages/basePage'
+
+test.beforeAll(async () => {
+  await setupElectron()
+})
+
+test.afterAll(async () => {
+  await teardownElectron()
+})
+
+test('renders left navigation panel', async () => {
+  const systemMonitorBtn = await page
+    .getByTestId('System Monitor')
+    .first()
+    .isEnabled({
+      timeout: TIMEOUT,
+    })
+  const settingsBtn = await page
+    .getByTestId('Thread')
+    .first()
+    .isEnabled({ timeout: TIMEOUT })
+  expect([systemMonitorBtn, settingsBtn].filter((e) => !e).length).toBe(0)
+  // Chat section should be there
+  await page.getByTestId('Local API Server').first().click({
+    timeout: TIMEOUT,
+  })
+  const localServer = page.getByTestId('local-server-testid').first()
+  await expect(localServer).toBeVisible({
+    timeout: TIMEOUT,
+  })
+})

electron/tests/e2e/settings.e2e.spec.ts

@@ -0,0 +1,23 @@
+import { expect } from '@playwright/test'
+
+import {
+  setupElectron,
+  teardownElectron,
+  test,
+  page,
+  TIMEOUT,
+} from '../pages/basePage'
+
+test.beforeAll(async () => {
+  await setupElectron()
+})
+
+test.afterAll(async () => {
+  await teardownElectron()
+})
+
+test('shows settings', async () => {
+  await page.getByTestId('Settings').first().click({ timeout: TIMEOUT })
+  const settingDescription = page.getByTestId('testid-setting-description')
+  await expect(settingDescription).toBeVisible({ timeout: TIMEOUT })
+})

electron/tests/hub.e2e.spec.ts

@@ -1,48 +0,0 @@
-import { _electron as electron } from 'playwright'
-import { ElectronApplication, Page, expect, test } from '@playwright/test'
-
-import {
-  findLatestBuild,
-  parseElectronApp,
-  stubDialog,
-} from 'electron-playwright-helpers'
-
-let electronApp: ElectronApplication
-let page: Page
-const TIMEOUT: number = parseInt(process.env.TEST_TIMEOUT || '300000')
-
-test.beforeAll(async () => {
-  process.env.CI = 'e2e'
-
-  const latestBuild = findLatestBuild('dist')
-  expect(latestBuild).toBeTruthy()
-
-  // parse the packaged Electron app and find paths and other info
-  const appInfo = parseElectronApp(latestBuild)
-  expect(appInfo).toBeTruthy()
-
-  electronApp = await electron.launch({
-    args: [appInfo.main], // main file from package.json
-    executablePath: appInfo.executable, // path to the Electron executable
-  })
-  await stubDialog(electronApp, 'showMessageBox', { response: 1 })
-
-  page = await electronApp.firstWindow({
-    timeout: TIMEOUT,
-  })
-})
-
-test.afterAll(async () => {
-  await electronApp.close()
-  await page.close()
-})
-
-test('explores hub', async () => {
-  test.setTimeout(TIMEOUT)
-  await page.getByTestId('Hub').first().click({
-    timeout: TIMEOUT,
-  })
-  await page.getByTestId('hub-container-test-id').isVisible({
-    timeout: TIMEOUT,
-  })
-})

electron/tests/navigation.e2e.spec.ts

@@ -1,61 +0,0 @@
-import { _electron as electron } from 'playwright'
-import { ElectronApplication, Page, expect, test } from '@playwright/test'
-
-import {
-  findLatestBuild,
-  parseElectronApp,
-  stubDialog,
-} from 'electron-playwright-helpers'
-
-let electronApp: ElectronApplication
-let page: Page
-const TIMEOUT: number = parseInt(process.env.TEST_TIMEOUT || '300000')
-
-test.beforeAll(async () => {
-  process.env.CI = 'e2e'
-
-  const latestBuild = findLatestBuild('dist')
-  expect(latestBuild).toBeTruthy()
-
-  // parse the packaged Electron app and find paths and other info
-  const appInfo = parseElectronApp(latestBuild)
-  expect(appInfo).toBeTruthy()
-
-  electronApp = await electron.launch({
-    args: [appInfo.main], // main file from package.json
-    executablePath: appInfo.executable, // path to the Electron executable
-  })
-  await stubDialog(electronApp, 'showMessageBox', { response: 1 })
-
-  page = await electronApp.firstWindow({
-    timeout: TIMEOUT,
-  })
-})
-
-test.afterAll(async () => {
-  await electronApp.close()
-  await page.close()
-})
-
-test('renders left navigation panel', async () => {
-  test.setTimeout(TIMEOUT)
-  const systemMonitorBtn = await page
-    .getByTestId('System Monitor')
-    .first()
-    .isEnabled({
-      timeout: TIMEOUT,
-    })
-  const settingsBtn = await page
-    .getByTestId('Thread')
-    .first()
-    .isEnabled({ timeout: TIMEOUT })
-  expect([systemMonitorBtn, settingsBtn].filter((e) => !e).length).toBe(0)
-  // Chat section should be there
-  await page.getByTestId('Local API Server').first().click({
-    timeout: TIMEOUT,
-  })
-  const localServer = await page.getByTestId('local-server-testid').first()
-  await expect(localServer).toBeVisible({
-    timeout: TIMEOUT,
-  })
-})

electron/tests/pages/basePage.ts

@@ -0,0 +1,67 @@
+import {
+  expect,
+  test as base,
+  _electron as electron,
+  ElectronApplication,
+  Page,
+} from '@playwright/test'
+import {
+  findLatestBuild,
+  parseElectronApp,
+  stubDialog,
+} from 'electron-playwright-helpers'
+
+export const TIMEOUT: number = parseInt(process.env.TEST_TIMEOUT || '300000')
+
+export let electronApp: ElectronApplication
+export let page: Page
+
+export async function setupElectron() {
+  process.env.CI = 'e2e'
+
+  const latestBuild = findLatestBuild('dist')
+  expect(latestBuild).toBeTruthy()
+
+  // parse the packaged Electron app and find paths and other info
+  const appInfo = parseElectronApp(latestBuild)
+  expect(appInfo).toBeTruthy()
+
+  electronApp = await electron.launch({
+    args: [appInfo.main], // main file from package.json
+    executablePath: appInfo.executable, // path to the Electron executable
+  })
+  await stubDialog(electronApp, 'showMessageBox', { response: 1 })
+
+  page = await electronApp.firstWindow({
+    timeout: TIMEOUT,
+  })
+  // Return appInfo for future use
+  return appInfo
+}
+
+export async function teardownElectron() {
+  await page.close()
+  await electronApp.close()
+}
+
+export const test = base.extend<{
+  attachScreenshotsToReport: void
+}>({
+  attachScreenshotsToReport: [
+    async ({ request }, use, testInfo) => {
+      await use()
+
+      // After the test, we can check whether the test passed or failed.
+      if (testInfo.status !== testInfo.expectedStatus) {
+        const screenshot = await page.screenshot()
+        await testInfo.attach('screenshot', {
+          body: screenshot,
+          contentType: 'image/png',
+        })
+      }
+    },
+    { auto: true },
+  ],
+})
+
+test.setTimeout(TIMEOUT)

electron/tests/settings.e2e.spec.ts

@@ -1,45 +0,0 @@
-import { _electron as electron } from 'playwright'
-import { ElectronApplication, Page, expect, test } from '@playwright/test'
-
-import {
-  findLatestBuild,
-  parseElectronApp,
-  stubDialog,
-} from 'electron-playwright-helpers'
-
-let electronApp: ElectronApplication
-let page: Page
-const TIMEOUT: number = parseInt(process.env.TEST_TIMEOUT || '300000')
-
-test.beforeAll(async () => {
-  process.env.CI = 'e2e'
-
-  const latestBuild = findLatestBuild('dist')
-  expect(latestBuild).toBeTruthy()
-
-  // parse the packaged Electron app and find paths and other info
-  const appInfo = parseElectronApp(latestBuild)
-  expect(appInfo).toBeTruthy()
-
-  electronApp = await electron.launch({
-    args: [appInfo.main], // main file from package.json
-    executablePath: appInfo.executable, // path to the Electron executable
-  })
-  await stubDialog(electronApp, 'showMessageBox', { response: 1 })
-
-  page = await electronApp.firstWindow({
-    timeout: TIMEOUT,
-  })
-})
-
-test.afterAll(async () => {
-  await electronApp.close()
-  await page.close()
-})
-
-test('shows settings', async () => {
-  test.setTimeout(TIMEOUT)
-  await page.getByTestId('Settings').first().click({ timeout: TIMEOUT })
-  const settingDescription = page.getByTestId('testid-setting-description')
-  await expect(settingDescription).toBeVisible({ timeout: TIMEOUT })
-})