Feature autoupdater for nightly build (#1068)

* CI nightly build enable auto-updater * Add jobs dependencies * add continue on error for task delete latest folder on R2 * Switch to use aws s3api * Switch to use bash for make build command * Fix error region aws s3cli command * Fix makefile command error on windows * Change content type of yaml file upload to R2 * add retry curl github api * Fix error command aws s3api delete-objects * quit from tty for aws s3api command * CI nightly build enable auto-updater * Add jobs dependencies * add continue on error for task delete latest folder on R2 * Switch to use aws s3api * Switch to use bash for make build command * Fix error region aws s3cli command * Fix makefile command error on windows * Change content type of yaml file upload to R2 * add retry curl github api * Fix error command aws s3api delete-objects * quit from tty for aws s3api command * Correct yml file for auto updater electron * Add channel for auto-updater * Add channel for auto-updater --------- Co-authored-by: Hien To <tominhhien97@gmail.com> Co-authored-by: Service Account <service@jan.ai>
2023-12-19 00:06:09 +07:00 · 2023-12-19 00:06:09 +07:00 · a296cb2c85
commit a296cb2c85
parent aa69bc5f46
3 changed files with 147 additions and 8 deletions
--- a/.github/workflows/jan-electron-build-nightly.yml
+++ b/.github/workflows/jan-electron-build-nightly.yml
@ -7,8 +7,36 @@ on:
  workflow_dispatch:

 jobs:
+  delete-cloudflare-r2-folder:
+    runs-on: ubuntu-latest
+    environment: production
+    steps:
+      - name: install-aws-cli-action
+        uses: unfor19/install-aws-cli-action@v1
+
+      - name: Delete cloudflare-r2 folder using awscli s3api
+        continue-on-error: true
+        run: |
+          # Get the list of objects in the 'latest' folder
+          OBJECTS=$(aws s3api list-objects --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --prefix "latest/" --query 'Contents[].{Key: Key}' --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com | jq -c .)
+          
+          # Create a JSON file for the delete operation
+          echo "{\"Objects\": $OBJECTS, \"Quiet\": false}" > delete.json
+          
+          # Delete the objects
+          echo q | aws s3api delete-objects --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --delete file://delete.json --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          
+          # Remove the JSON file
+          rm delete.json
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          AWS_EC2_METADATA_DISABLED: "true"
+
  build-macos:
    runs-on: macos-latest
+    needs: delete-cloudflare-r2-folder
    environment: production
    permissions:
      contents: write
@ -28,8 +56,28 @@ jobs:
      - name: Update app version based on latest release tag with build number
        id: version_update
        run: |
+          # Function to get the latest release tag
+          get_latest_tag() {
+            local retries=0
+            local max_retries=3
+            local tag
+            while [ $retries -lt $max_retries ]; do
+              tag=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+              if [ -n "$tag" ] && [ "$tag" != "null" ]; then
+                echo $tag
+                return
+              else
+                let retries++
+                echo "Retrying... ($retries/$max_retries)"
+                sleep 2
+              fi
+            done
+            echo "Failed to fetch latest tag after $max_retries attempts."
+            exit 1
+          }
+      
          # Get the latest release tag from GitHub API
-          LATEST_TAG=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+          LATEST_TAG=$(get_latest_tag)
          
          # Remove the 'v' and append the build number to the version
          NEW_VERSION="${LATEST_TAG#v}-${GITHUB_RUN_NUMBER}"
@ -40,6 +88,9 @@ jobs:
          mv /tmp/package.json electron/package.json
          echo "::set-output name=new_version::$NEW_VERSION"

+          jq '.build.publish = [{"provider": "generic", "url": "${{ secrets.CLOUDFLARE_R2_PUBLIC_URL }}", "channel": "latest"}]' electron/package.json > /tmp/package.json
+          mv /tmp/package.json electron/package.json
+
      - name: Get Cer for code signing
        run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
        shell: bash
@ -77,8 +128,25 @@ jobs:
          name: jan-mac-arm64-${{ steps.version_update.outputs.new_version }}
          path: ./electron/dist/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg

+      - name: put-object using awscli s3api
+        continue-on-error: true
+        run: |
+          ls -al ./electron/dist
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/jan-mac-x64-${{ steps.version_update.outputs.new_version }}.dmg" --body "./electron/dist/jan-mac-x64-${{ steps.version_update.outputs.new_version }}.dmg" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg" --body "./electron/dist/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/jan-mac-x64-${{ steps.version_update.outputs.new_version }}.dmg" --body "./electron/dist/jan-mac-x64-${{ steps.version_update.outputs.new_version }}.dmg" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg" --body "./electron/dist/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/latest-mac.yml" --body "./electron/dist/latest-mac.yml" --content-type "text/yaml"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/latest-mac.yml" --body "./electron/dist/latest-mac.yml" --content-type "text/yaml"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          AWS_EC2_METADATA_DISABLED: "true"
+
  build-windows-x64:
    runs-on: windows-latest
+    needs: delete-cloudflare-r2-folder
    permissions:
      contents: write
    steps:
@ -97,8 +165,28 @@ jobs:
        id: version_update
        shell: bash
        run: |
+          # Function to get the latest release tag
+          get_latest_tag() {
+            local retries=0
+            local max_retries=3
+            local tag
+            while [ $retries -lt $max_retries ]; do
+              tag=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+              if [ -n "$tag" ] && [ "$tag" != "null" ]; then
+                echo $tag
+                return
+              else
+                let retries++
+                echo "Retrying... ($retries/$max_retries)"
+                sleep 2
+              fi
+            done
+            echo "Failed to fetch latest tag after $max_retries attempts."
+            exit 1
+          }
+      
          # Get the latest release tag from GitHub API
-          LATEST_TAG=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+          LATEST_TAG=$(get_latest_tag)
          
          # Remove the 'v' and append the build number to the version
          NEW_VERSION="${LATEST_TAG#v}-${GITHUB_RUN_NUMBER}"
@ -110,8 +198,10 @@ jobs:

          echo "::set-output name=new_version::$NEW_VERSION"

+          jq '.build.publish = [{"provider": "generic", "url": "${{ secrets.CLOUDFLARE_R2_PUBLIC_URL }}", "channel": "latest"}]' electron/package.json > /tmp/package.json
+          mv /tmp/package.json electron/package.json
+
      - name: Build app
-        shell: cmd
        run: |
          make build

@ -126,9 +216,24 @@ jobs:
        with:
          name: jan-win-x64-${{ steps.version_update.outputs.new_version }}
          path: ./electron/dist/*.exe
+      
+      - name: put-object using awscli s3api
+        shell: bash
+        run: |
+          ls -al ./electron/dist
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/jan-win-x64-${{ steps.version_update.outputs.new_version }}.exe" --body "./electron/dist/jan-win-x64-${{ steps.version_update.outputs.new_version }}.exe" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/jan-win-x64-${{ steps.version_update.outputs.new_version }}.exe" --body "./electron/dist/jan-win-x64-${{ steps.version_update.outputs.new_version }}.exe" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/latest.yml" --body "./electron/dist/latest.yml" --content-type "text/yaml"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/latest.yml" --body "./electron/dist/latest.yml" --content-type "text/yaml"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          AWS_EC2_METADATA_DISABLED: "true"
          
  build-linux-x64:
    runs-on: ubuntu-latest
+    needs: delete-cloudflare-r2-folder
    environment: production
    env:
      SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_TOKEN }}
@ -149,8 +254,28 @@ jobs:
      - name: Update app version base on tag
        id: version_update
        run: |
+          # Function to get the latest release tag
+          get_latest_tag() {
+            local retries=0
+            local max_retries=3
+            local tag
+            while [ $retries -lt $max_retries ]; do
+              tag=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+              if [ -n "$tag" ] && [ "$tag" != "null" ]; then
+                echo $tag
+                return
+              else
+                let retries++
+                echo "Retrying... ($retries/$max_retries)"
+                sleep 2
+              fi
+            done
+            echo "Failed to fetch latest tag after $max_retries attempts."
+            exit 1
+          }
+      
          # Get the latest release tag from GitHub API
-          LATEST_TAG=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+          LATEST_TAG=$(get_latest_tag)
          
          # Remove the 'v' and append the build number to the version
          NEW_VERSION="${LATEST_TAG#v}-${GITHUB_RUN_NUMBER}"
@ -160,6 +285,8 @@ jobs:
          jq --arg version "$NEW_VERSION" '.version = $version' electron/package.json > /tmp/package.json
          mv /tmp/package.json electron/package.json
          echo "::set-output name=new_version::$NEW_VERSION"
+          jq '.build.publish = [{"provider": "generic", "url": "${{ secrets.CLOUDFLARE_R2_PUBLIC_URL }}", "channel": "latest"}]' electron/package.json > /tmp/package.json
+          mv /tmp/package.json electron/package.json

      - name: Build and publish app
        run: |
@ -172,9 +299,22 @@ jobs:
        with:
          name: jan-linux-amd64-${{ steps.version_update.outputs.new_version }}
          path: ./electron/dist/*.deb
+      
+      - name: put-object using awscli s3api
+        run: |
+          ls -al ./electron/dist
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/jan-linux-amd64-${{ steps.version_update.outputs.new_version }}.deb" --body "./electron/dist/jan-linux-amd64-${{ steps.version_update.outputs.new_version }}.deb" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/jan-linux-amd64-${{ steps.version_update.outputs.new_version }}.deb" --body "./electron/dist/jan-linux-amd64-${{ steps.version_update.outputs.new_version }}.deb" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/latest-linux.yml" --body "./electron/dist/latest-linux.yml" --content-type "text/yaml"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/latest-linux.yml" --body "./electron/dist/latest-linux.yml" --content-type "text/yaml"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          AWS_EC2_METADATA_DISABLED: "true"

  noti-discord-nightly-and-update-url-readme:
-    needs: [build-macos, build-windows-x64, build-linux-x64]
+    needs: [build-macos, build-windows-x64, build-linux-x64, delete-cloudflare-r2-folder]
    environment: production
    if: github.event_name == 'schedule'
    runs-on: ubuntu-latest
@ -204,7 +344,7 @@ jobs:
          GITHUB_RUN_ID: ${{ github.run_id }}

  noti-discord-manual-and-update-url-readme:
-    needs: [build-macos, build-windows-x64, build-linux-x64]
+    needs: [build-macos, build-windows-x64, build-linux-x64, delete-cloudflare-r2-folder]
    environment: production
    if: github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
--- a/.github/workflows/jan-electron-build.yml
+++ b/.github/workflows/jan-electron-build.yml
@ -124,7 +124,6 @@ jobs:
          VERSION_TAG: ${{ steps.tag.outputs.tag }}

      - name: Build app
-        shell: cmd
        run: |
          make build
        env:
--- a/2
+++ b/2
@ -23,7 +23,7 @@ endif

 check-file-counts: install-and-build
 ifeq ($(OS),Windows_NT)
-	powershell -Command "$$tgz_count = (Get-ChildItem -Path electron/pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count); $$dir_count = (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count); if ($$tgz_count -ne $$dir_count) { Write-Host 'Number of .tgz files in electron/pre-install (' + $$tgz_count + ') does not match the number of subdirectories in extension (' + $$dir_count + ')'; exit 1 } else { Write-Host 'Extension build successful' }"
+	powershell -Command "if ((Get-ChildItem -Path electron/pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in electron/pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }"
 else
 	@tgz_count=$$(find electron/pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in electron/pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi
 endif