Add workflow test openai api (#2778)

* Add workflow test openai api

* chore: add filename as attribute to each test case

* chore: correct label

* feat: create pytest.ini

* chore: remote extra "

* chore: rename workflow to group similar type

* chore: remove auto build on push to main

* chore: rename job

---------

Co-authored-by: Hien To <tominhhien97@gmail.com>
Co-authored-by: Van-QA <van@jan.ai>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -25,16 +25,13 @@ If applicable, add screenshots to help explain your issue.
 
 **Environment details**
 - Operating System: [Specify your OS. e.g., MacOS Sonoma 14.2.1, Windows 11, Ubuntu 22, etc]
-- Jan Version: [e.g., 0.4.3]
+- Jan Version: [e.g., 0.4.xxx nightly or manual]
 - Processor: [e.g., Apple M1, Intel Core i7, AMD Ryzen 5, etc]
 - RAM: [e.g., 8GB, 16GB]
 - Any additional relevant hardware specifics: [e.g., Graphics card, SSD/HDD]
 
 **Logs**
-If the cause of the error is not clear, kindly provide your usage logs:
+If the cause of the error is not clear, kindly provide your usage logs: https://jan.ai/docs/troubleshooting#how-to-get-error-logs
-- `tail -n 50 ~/jan/logs/app.log` if you are using the UI
-- `tail -n 50 ~/jan/logs/server.log` if you are using the local api server
-Making sure to redact any private information.
 
 **Additional context**
 Add any other context or information that could be helpful in diagnosing the problem.

.github/workflows/jan-electron-build-nightly.yml

@@ -1,12 +1,6 @@
-name: Jan Build Electron App Nightly or Manual
+name: Electron Builder - Nightly / Manual
 
 on:
-  push:
-    branches:
-      - main
-    paths-ignore:
-      - 'README.md'
-      - 'docs/**'
   schedule:
     - cron: '0 20 * * 1,2,3' # At 8 PM UTC on Monday, Tuesday, and Wednesday which is 3 AM UTC+7 Tuesday, Wednesday, and Thursday
   workflow_dispatch:

.github/workflows/jan-electron-build.yml

@@ -1,4 +1,4 @@
-name: Jan Build Electron App
+name: Electron Builder - Tag
 
 on:
   push:

.github/workflows/jan-electron-linter-and-test.yml

@@ -1,4 +1,4 @@
-name: Jan Electron Linter & Test
+name: Test - Linter & Playwright
 on:
   workflow_dispatch:
   push:

.github/workflows/jan-openai-api-test.yml

@@ -0,0 +1,90 @@
+name: Test - OpenAI API Pytest collection
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - dev
+      - release/**
+    paths:
+      - "docs/**"
+
+  pull_request:
+    branches:
+      - main
+      - dev
+      - release/**
+    paths:
+      - "docs/**"
+
+jobs:
+  openai-python-tests:
+    runs-on: [self-hosted, Linux, ubuntu-desktop]
+    if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Getting the repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Installing node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: "Cleanup cache"
+        continue-on-error: true
+        run: |
+          rm -rf ~/jan
+          make clean
+
+      - name: install dependencies
+        run: |
+          npm install -g @stoplight/prism-cli
+
+      - name: create python virtual environment and run test
+        run: |
+          python3 -m venv /tmp/jan
+          source /tmp/jan/bin/activate
+          # Clone openai-api-python repo
+          OPENAI_API_PYTHON_TAG=$(cat docs/openapi/version.txt)
+          git clone https://github.com/openai/openai-python.git
+          cd openai-python
+          git checkout $OPENAI_API_PYTHON_TAG
+
+          python3 -m venv /tmp/jan
+          source /tmp/jan/bin/activate
+          pip install -r requirements-dev.lock
+          pip install pytest-reportportal pytest-html
+          
+          # Create pytest.ini file with content
+          cat ../docs/tests/pytest.ini >> pytest.ini
+          echo "rp_api_key = ${{ secrets.REPORT_PORTAL_API_KEY }}" >> pytest.ini
+          echo "rp_endpoint = ${{ secrets.REPORT_PORTAL_URL_PYTEST }}" >> pytest.ini
+          cat pytest.ini
+          
+          # Append to conftest.py
+          cat ../docs/tests/conftest.py >> tests/conftest.py
+
+          # start mock server and run test then stop mock server
+          prism mock ../docs/openapi/jan.yaml > prism.log & prism_pid=$! && pytest --reportportal --html=report.html && kill $prism_pid
+          deactivate
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: report
+          path: |
+            openai-python/report.html
+            openai-python/assets
+            openai-python/prism.log
+
+      - name: clean up
+        if: always()
+        run: |
+          rm -rf /tmp/jan
+          rm -rf openai-python
+          rm -rf report.html
+          rm -rf report.zip
+  
+

.github/workflows/jan-server-build-nightly.yml

@@ -1,4 +1,4 @@
-name: Jan Build Docker Nightly or Manual
+name: Docker Builder - Nightly / Manual
 
 on:
   push:

.github/workflows/jan-server-build.yml

@@ -1,4 +1,4 @@
-name: Jan Build Docker
+name: Docker Builder - Tag
 
 on:
   push:

docs/openapi/version.txt

@@ -0,0 +1 @@
+v1.23.2

docs/tests/conftest.py

@@ -0,0 +1,6 @@
+def pytest_collection_modifyitems(items):
+    for item in items:
+        # add the name of the file (without extension) as a marker
+        filename = item.nodeid.split("::")[0].split("/")[-1].replace(".py", "")
+        marker = pytest.mark.file(filename)
+        item.add_marker(marker)

docs/tests/pytest.ini

@@ -0,0 +1,8 @@
+[pytest]
+rp_project = openai-api-test
+rp_launch = OpenAI Collection Test
+rp_launch_description = Full collection to ensure compatibility with OpenAI API
+rp_launch_attributes = 'CI'
+filterwarnings = ignore::pytest.PytestUnknownMarkWarning
+log_format = %(asctime)s %(levelname)s %(message)s
+log_date_format = %Y-%m-%d %H:%M:%S