Feature autoupdater for nightly build (#1068)

* CI nightly build enable auto-updater

* Add jobs dependencies

* add continue on error for task delete latest folder on R2

* Switch to use aws s3api

* Switch to use bash for make build command

* Fix error region aws s3cli command

* Fix makefile command error on windows

* Change content type of yaml file upload to R2

* add retry curl github api

* Fix error command aws s3api delete-objects

* quit from tty for aws s3api command

* CI nightly build enable auto-updater

* Add jobs dependencies

* add continue on error for task delete latest folder on R2

* Switch to use aws s3api

* Switch to use bash for make build command

* Fix error region aws s3cli command

* Fix makefile command error on windows

* Change content type of yaml file upload to R2

* add retry curl github api

* Fix error command aws s3api delete-objects

* quit from tty for aws s3api command

* Correct yml file for auto updater electron

* Add channel for auto-updater

* Add channel for auto-updater

---------

Co-authored-by: Hien To <tominhhien97@gmail.com>
Co-authored-by: Service Account <service@jan.ai>

.github/workflows/jan-electron-build-nightly.yml

@@ -7,8 +7,36 @@ on:
   workflow_dispatch:
 
 jobs:
+  delete-cloudflare-r2-folder:
+    runs-on: ubuntu-latest
+    environment: production
+    steps:
+      - name: install-aws-cli-action
+        uses: unfor19/install-aws-cli-action@v1
+
+      - name: Delete cloudflare-r2 folder using awscli s3api
+        continue-on-error: true
+        run: |
+          # Get the list of objects in the 'latest' folder
+          OBJECTS=$(aws s3api list-objects --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --prefix "latest/" --query 'Contents[].{Key: Key}' --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com | jq -c .)
+          
+          # Create a JSON file for the delete operation
+          echo "{\"Objects\": $OBJECTS, \"Quiet\": false}" > delete.json
+          
+          # Delete the objects
+          echo q | aws s3api delete-objects --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --delete file://delete.json --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          
+          # Remove the JSON file
+          rm delete.json
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          AWS_EC2_METADATA_DISABLED: "true"
+
   build-macos:
     runs-on: macos-latest
+    needs: delete-cloudflare-r2-folder
     environment: production
     permissions:
       contents: write
@@ -28,8 +56,28 @@ jobs:
       - name: Update app version based on latest release tag with build number
         id: version_update
         run: |
+          # Function to get the latest release tag
+          get_latest_tag() {
+            local retries=0
+            local max_retries=3
+            local tag
+            while [ $retries -lt $max_retries ]; do
+              tag=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+              if [ -n "$tag" ] && [ "$tag" != "null" ]; then
+                echo $tag
+                return
+              else
+                let retries++
+                echo "Retrying... ($retries/$max_retries)"
+                sleep 2
+              fi
+            done
+            echo "Failed to fetch latest tag after $max_retries attempts."
+            exit 1
+          }
+      
           # Get the latest release tag from GitHub API
-          LATEST_TAG=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+          LATEST_TAG=$(get_latest_tag)
           
           # Remove the 'v' and append the build number to the version
           NEW_VERSION="${LATEST_TAG#v}-${GITHUB_RUN_NUMBER}"
@@ -40,6 +88,9 @@ jobs:
           mv /tmp/package.json electron/package.json
           echo "::set-output name=new_version::$NEW_VERSION"
 
+          jq '.build.publish = [{"provider": "generic", "url": "${{ secrets.CLOUDFLARE_R2_PUBLIC_URL }}", "channel": "latest"}]' electron/package.json > /tmp/package.json
+          mv /tmp/package.json electron/package.json
+
       - name: Get Cer for code signing
         run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
         shell: bash
@@ -77,8 +128,25 @@ jobs:
           name: jan-mac-arm64-${{ steps.version_update.outputs.new_version }}
           path: ./electron/dist/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg
 
+      - name: put-object using awscli s3api
+        continue-on-error: true
+        run: |
+          ls -al ./electron/dist
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/jan-mac-x64-${{ steps.version_update.outputs.new_version }}.dmg" --body "./electron/dist/jan-mac-x64-${{ steps.version_update.outputs.new_version }}.dmg" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg" --body "./electron/dist/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/jan-mac-x64-${{ steps.version_update.outputs.new_version }}.dmg" --body "./electron/dist/jan-mac-x64-${{ steps.version_update.outputs.new_version }}.dmg" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg" --body "./electron/dist/jan-mac-arm64-${{ steps.version_update.outputs.new_version }}.dmg" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/latest-mac.yml" --body "./electron/dist/latest-mac.yml" --content-type "text/yaml"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/latest-mac.yml" --body "./electron/dist/latest-mac.yml" --content-type "text/yaml"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          AWS_EC2_METADATA_DISABLED: "true"
+
   build-windows-x64:
     runs-on: windows-latest
+    needs: delete-cloudflare-r2-folder
     permissions:
       contents: write
     steps:
@@ -97,8 +165,28 @@ jobs:
         id: version_update
         shell: bash
         run: |
+          # Function to get the latest release tag
+          get_latest_tag() {
+            local retries=0
+            local max_retries=3
+            local tag
+            while [ $retries -lt $max_retries ]; do
+              tag=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+              if [ -n "$tag" ] && [ "$tag" != "null" ]; then
+                echo $tag
+                return
+              else
+                let retries++
+                echo "Retrying... ($retries/$max_retries)"
+                sleep 2
+              fi
+            done
+            echo "Failed to fetch latest tag after $max_retries attempts."
+            exit 1
+          }
+      
           # Get the latest release tag from GitHub API
-          LATEST_TAG=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+          LATEST_TAG=$(get_latest_tag)
           
           # Remove the 'v' and append the build number to the version
           NEW_VERSION="${LATEST_TAG#v}-${GITHUB_RUN_NUMBER}"
@@ -110,8 +198,10 @@ jobs:
 
           echo "::set-output name=new_version::$NEW_VERSION"
 
+          jq '.build.publish = [{"provider": "generic", "url": "${{ secrets.CLOUDFLARE_R2_PUBLIC_URL }}", "channel": "latest"}]' electron/package.json > /tmp/package.json
+          mv /tmp/package.json electron/package.json
+
       - name: Build app
-        shell: cmd
         run: |
           make build
 
@@ -127,8 +217,23 @@ jobs:
           name: jan-win-x64-${{ steps.version_update.outputs.new_version }}
           path: ./electron/dist/*.exe
       
+      - name: put-object using awscli s3api
+        shell: bash
+        run: |
+          ls -al ./electron/dist
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/jan-win-x64-${{ steps.version_update.outputs.new_version }}.exe" --body "./electron/dist/jan-win-x64-${{ steps.version_update.outputs.new_version }}.exe" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/jan-win-x64-${{ steps.version_update.outputs.new_version }}.exe" --body "./electron/dist/jan-win-x64-${{ steps.version_update.outputs.new_version }}.exe" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/latest.yml" --body "./electron/dist/latest.yml" --content-type "text/yaml"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/latest.yml" --body "./electron/dist/latest.yml" --content-type "text/yaml"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          AWS_EC2_METADATA_DISABLED: "true"
+          
   build-linux-x64:
     runs-on: ubuntu-latest
+    needs: delete-cloudflare-r2-folder
     environment: production
     env:
       SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_TOKEN }}
@@ -149,8 +254,28 @@ jobs:
       - name: Update app version base on tag
         id: version_update
         run: |
+          # Function to get the latest release tag
+          get_latest_tag() {
+            local retries=0
+            local max_retries=3
+            local tag
+            while [ $retries -lt $max_retries ]; do
+              tag=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+              if [ -n "$tag" ] && [ "$tag" != "null" ]; then
+                echo $tag
+                return
+              else
+                let retries++
+                echo "Retrying... ($retries/$max_retries)"
+                sleep 2
+              fi
+            done
+            echo "Failed to fetch latest tag after $max_retries attempts."
+            exit 1
+          }
+      
           # Get the latest release tag from GitHub API
-          LATEST_TAG=$(curl -s https://api.github.com/repos/janhq/jan/releases/latest | jq -r .tag_name)
+          LATEST_TAG=$(get_latest_tag)
           
           # Remove the 'v' and append the build number to the version
           NEW_VERSION="${LATEST_TAG#v}-${GITHUB_RUN_NUMBER}"
@@ -160,6 +285,8 @@ jobs:
           jq --arg version "$NEW_VERSION" '.version = $version' electron/package.json > /tmp/package.json
           mv /tmp/package.json electron/package.json
           echo "::set-output name=new_version::$NEW_VERSION"
+          jq '.build.publish = [{"provider": "generic", "url": "${{ secrets.CLOUDFLARE_R2_PUBLIC_URL }}", "channel": "latest"}]' electron/package.json > /tmp/package.json
+          mv /tmp/package.json electron/package.json
 
       - name: Build and publish app
         run: |
@@ -173,8 +300,21 @@ jobs:
           name: jan-linux-amd64-${{ steps.version_update.outputs.new_version }}
           path: ./electron/dist/*.deb
       
+      - name: put-object using awscli s3api
+        run: |
+          ls -al ./electron/dist
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/jan-linux-amd64-${{ steps.version_update.outputs.new_version }}.deb" --body "./electron/dist/jan-linux-amd64-${{ steps.version_update.outputs.new_version }}.deb" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/jan-linux-amd64-${{ steps.version_update.outputs.new_version }}.deb" --body "./electron/dist/jan-linux-amd64-${{ steps.version_update.outputs.new_version }}.deb" --content-type "application/octet-stream"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "latest/latest-linux.yml" --body "./electron/dist/latest-linux.yml" --content-type "text/yaml"
+          echo "q" | aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "${{ steps.version_update.outputs.new_version }}/latest-linux.yml" --body "./electron/dist/latest-linux.yml" --content-type "text/yaml"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          AWS_EC2_METADATA_DISABLED: "true"
+
   noti-discord-nightly-and-update-url-readme:
-    needs: [build-macos, build-windows-x64, build-linux-x64]
+    needs: [build-macos, build-windows-x64, build-linux-x64, delete-cloudflare-r2-folder]
     environment: production
     if: github.event_name == 'schedule'
     runs-on: ubuntu-latest
@@ -204,7 +344,7 @@ jobs:
           GITHUB_RUN_ID: ${{ github.run_id }}
 
   noti-discord-manual-and-update-url-readme:
-    needs: [build-macos, build-windows-x64, build-linux-x64]
+    needs: [build-macos, build-windows-x64, build-linux-x64, delete-cloudflare-r2-folder]
     environment: production
     if: github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest

.github/workflows/jan-electron-build.yml

@@ -124,7 +124,6 @@ jobs:
           VERSION_TAG: ${{ steps.tag.outputs.tag }}
 
       - name: Build app
-        shell: cmd
         run: |
           make build
         env:

Makefile

@@ -23,7 +23,7 @@ endif
 
 check-file-counts: install-and-build
 ifeq ($(OS),Windows_NT)
-	powershell -Command "$$tgz_count = (Get-ChildItem -Path electron/pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count); $$dir_count = (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count); if ($$tgz_count -ne $$dir_count) { Write-Host 'Number of .tgz files in electron/pre-install (' + $$tgz_count + ') does not match the number of subdirectories in extension (' + $$dir_count + ')'; exit 1 } else { Write-Host 'Extension build successful' }"
+	powershell -Command "if ((Get-ChildItem -Path electron/pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in electron/pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }"
 else
 	@tgz_count=$$(find electron/pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in electron/pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi
 endif