fix: turborepo extensions (#2392)

* fix: turborepo extensions Update package.json Update Makefile Update Makefile Update Makefile Update Makefile Update Makefile Update package.json * chore: turbo cache * fix: install extensions in parallel * fix: timeout issue * Turbo cache using s3 * Remove cache task --------- Co-authored-by: Hien To <tominhhien97@gmail.com> Co-authored-by: Service Account <service@jan.ai>
2024-03-22 17:53:20 +07:00 · 2024-03-22 17:53:20 +07:00 · 254a79ccbe
commit 254a79ccbe
parent 36405fe894
8 changed files with 77 additions and 134 deletions
--- a/.github/workflows/codeQL-analysis.yml
+++ b/.github/workflows/codeQL-analysis.yml
@ -1,104 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches:
-      - main
-      - dev
-    paths:
-      - "electron/**"
-      - .github/workflows/jan-electron-linter-and-test.yml
-      - "web/**"
-      - "uikit/**"
-      - "package.json"
-      - "node_modules/**"
-      - "yarn.lock"
-      - "core/**"
-      - "extensions/**"
-      - "!README.md"
-      - "Makefile"
-
-  pull_request:
-    branches:
-      - main
-      - dev
-    paths:
-      - "electron/**"
-      - .github/workflows/jan-electron-linter-and-test.yml
-      - "web/**"
-      - "uikit/**"
-      - "package.json"
-      - "node_modules/**"
-      - "yarn.lock"
-      - "Makefile"
-
-jobs:
-  analyze:
-    name: Analyze
-    # Runner size impacts CodeQL analysis time. To learn more, please see:
-    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
-    #   - https://gh.io/supported-runners-and-hardware-resources
-    #   - https://gh.io/using-larger-runners
-    # Consider using larger runners for possible analysis time improvements.
-    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
-    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ["javascript-typescript"]
-        # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
-        # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
-        # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-
-          # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-          # queries: security-extended,security-and-quality
-
-      # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-      #   If the Autobuild fails above, remove it and uncomment the following three lines.
-      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-      # - run: |
-      #     echo "Run, Build Application using script"
-      #     ./location_of_script_within_repo/buildscript.sh
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: "/language:${{matrix.language}}"
--- a/.github/workflows/jan-electron-linter-and-test.yml
+++ b/.github/workflows/jan-electron-linter-and-test.yml
@ -51,7 +51,7 @@ jobs:
        uses: actions/checkout@v3

      - name: Installing node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
        with:
          node-version: 20

@ -87,7 +87,7 @@ jobs:
        uses: actions/checkout@v3

      - name: Installing node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
        with:
          node-version: 20

@ -149,7 +149,7 @@ jobs:
        uses: actions/checkout@v3

      - name: Installing node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
        with:
          node-version: 20

@ -157,7 +157,7 @@ jobs:
        continue-on-error: true
        run: |
          make clean
-
+            
      - name: Linter and test
        run: |
          export DISPLAY=$(w -h | awk 'NR==1 {print $2}')
--- a/5
+++ b/5
@ -17,6 +17,7 @@ install-and-build: build-uikit
 ifeq ($(OS),Windows_NT)
 	yarn config set network-timeout 300000
 endif
+	yarn global add turbo
 	yarn build:core
 	yarn build:server
 	yarn install
@ -24,9 +25,9 @@ endif

 check-file-counts: install-and-build
 ifeq ($(OS),Windows_NT)
-	powershell -Command "if ((Get-ChildItem -Path pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }"
+	powershell -Command "if ((Get-ChildItem -Path pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Where-Object Name -like *-extension* | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in pre-install does not match the number of subdirectories in extensions with package.json'; exit 1 } else { Write-Host 'Extension build successful' }"
 else
-	@tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi
+	@tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d -exec test -e '{}/package.json' \; -print | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi
 endif

 dev: check-file-counts
--- a/extensions/package.json
+++ b/extensions/package.json
@ -0,0 +1,11 @@
+{
+  "private": true,
+  "workspaces": {
+    "packages": [
+      "**"
+    ],
+    "nohoist": [
+      "**"
+    ]
+  }
+}
--- a/extensions/turbo.json
+++ b/extensions/turbo.json
@ -0,0 +1,15 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": [".dist/**", "*.tgz"]
+    },
+    "build:publish": {
+      "dependsOn": ["^build"]
+    },
+    "dev": {
+      "cache": false
+    },
+    "type-check": {}
+  }
+}
--- a/package.json
+++ b/package.json
@ -18,31 +18,30 @@
    ]
  },
  "scripts": {
-    "lint": "yarn workspace jan lint && yarn workspace jan-web lint",
+    "lint": "yarn workspace jan lint && yarn workspace @janhq/web lint",
    "test:unit": "yarn workspace @janhq/core test",
    "test": "yarn workspace jan test:e2e",
+    "test-local": "yarn lint && yarn build:test && yarn test",
+    "pre-install:darwin": "find extensions -type f -path \"**/*.tgz\" -exec cp {} pre-install \\;",
+    "pre-install:linux": "find extensions -type f -path \"**/*.tgz\" -exec cp {} pre-install \\;",
+    "pre-install:win32": "powershell -Command \"Get-ChildItem -Path \"extensions\" -Recurse -File -Filter \"*.tgz\" | ForEach-Object { Copy-Item -Path $_.FullName -Destination \"pre-install\" }\"",
+    "pre-install": "run-script-os",
    "copy:assets": "cpx \"models/**\" \"electron/models/\" && cpx \"pre-install/*.tgz\" \"electron/pre-install/\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"",
    "dev:electron": "yarn copy:assets && yarn workspace jan dev",
-    "dev:web": "yarn workspace jan-web dev",
+    "dev:web": "yarn workspace @janhq/web dev",
    "dev:server": "yarn copy:assets && yarn workspace @janhq/server dev",
-    "dev": "concurrently --kill-others \"yarn dev:web\" \"wait-on http://localhost:3000 && yarn dev:electron\"",
-    "test-local": "yarn lint && yarn build:test && yarn test",
+    "dev": "turbo run dev --parallel --filter=!@janhq/server",
    "dev:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit dev",
    "build:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build",
    "build:server": "yarn copy:assets && cd server && yarn install && yarn run build",
    "build:core": "cd core && yarn install && yarn run build",
-    "build:web": "yarn workspace jan-web build && cpx \"web/out/**\" \"electron/renderer/\"",
+    "build:web": "yarn workspace @janhq/web build && cpx \"web/out/**\" \"electron/renderer/\"",
    "build:electron": "yarn copy:assets && yarn workspace jan build",
    "build:electron:test": "yarn workspace jan build:test",
-    "build:extensions:windows": "rimraf ./pre-install/*.tgz && powershell -command \"$jobs = Get-ChildItem -Path './extensions' -Directory | ForEach-Object { Start-Job -Name ($_.Name) -ScriptBlock { param($_dir); try { Set-Location $_dir; yarn; yarn build:publish; Write-Output 'Build successful in ' + $_dir } catch { Write-Error 'Error in ' + $_dir; throw } } -ArgumentList $_.FullName }; $jobs | Wait-Job; $jobs | ForEach-Object { Receive-Job -Job $_ -Keep } | ForEach-Object { Write-Host $_ }; $failed = $jobs | Where-Object { $_.State -ne 'Completed' -or $_.ChildJobs[0].JobStateInfo.State -ne 'Completed' }; if ($failed) { Exit 1 }\"",
-    "build:extensions:linux": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {}; yarn; yarn build:publish'",
-    "build:extensions:darwin": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {}; yarn; yarn build:publish'",
-    "build:extensions:server": "yarn workspace build:extensions ",
-    "build:extensions": "run-script-os",
-    "build:test": "yarn copy:assets && yarn build:web && yarn workspace jan build:test",
+    "build:extensions": "rimraf ./pre-install/*.tgz && turbo run @janhq/core#build && cd extensions && yarn install && turbo run build:publish && cd .. && yarn pre-install",
+    "build:test": "yarn copy:assets && turbo run @janhq/web#build && cpx \"web/out/**\" \"electron/renderer/\" && turbo run build:test",
    "build": "yarn build:web && yarn build:electron",
-    "build:publish": "yarn copy:assets && yarn build:web && yarn workspace jan build:publish",
-    "turbo:electron": "turbo run dev --parallel --filter=!@janhq/server"
+    "build:publish": "yarn copy:assets && yarn build:web && yarn workspace jan build:publish"
  },
  "devDependencies": {
    "concurrently": "^8.2.1",
--- a/turbo.json
+++ b/turbo.json
@ -2,27 +2,48 @@
  "$schema": "https://turbo.build/schema.json",
  "pipeline": {
    "build": {
-      "outputs": [".next/**", "!.next/cache/**"]
+      "outputs": [".next/**", "!.next/cache/**", "dist/**"]
    },
    "dev": {
      "cache": false
    },
-    "web#build": {
-      "dependsOn": ["@janhq/core#build"]
-    },
-    "web:dev": {
+    "@janhq/web#dev": {
      "cache": false,
      "persistent": true,
      "dependsOn": ["@janhq/core#build", "@janhq/uikit#build"]
    },
-    "electron:dev": {
+    "@janhq/server#build": {
+      "outputs": ["dist/**"],
+      "dependsOn": ["@janhq/core#build"]
+    },
+    "jan#dev": {
      "cache": false,
      "persistent": true,
-      "dependsOn": ["@janhq/core#build", "@janhq/server#build", "jan#compile"]
+      "dependsOn": ["@janhq/core#build", "@janhq/server#build"]
    },
-    "electron#build": {
-      "dependsOn": ["web#build", "server#build", "core#build"],
-      "cache": false
+    "@janhq/core#build": {
+      "outputs": ["dist/**"]
+    },
+    "@janhq/web#build": {
+      "outputs": ["out/**"],
+      "dependsOn": ["@janhq/core#build", "@janhq/uikit#build"]
+    },
+    "jan#build": {
+      "outputs": ["dist/**"],
+      "dependsOn": [
+        "@janhq/core#build",
+        "@janhq/server#build",
+        "@janhq/web#build"
+      ]
+    },
+    "jan#build:test": {
+      "outputs": ["dist/**"],
+      "cache": false,
+      "dependsOn": [
+        "@janhq/core#build",
+        "@janhq/server#build",
+        "@janhq/web#build"
+      ]
    },
    "type-check": {}
  }
--- a/web/package.json
+++ b/web/package.json
@ -1,5 +1,5 @@
 {
-  "name": "jan-web",
+  "name": "@janhq/web",
  "version": "0.1.0",
  "private": true,
  "homepage": "./",