From ae499c7ac417d7781c69e8511d8b045ed1be7646 Mon Sep 17 00:00:00 2001 From: Van Pham <64197333+Van-QA@users.noreply.github.com> Date: Mon, 20 May 2024 23:45:37 +0700 Subject: [PATCH] Feat/oai endpoints mapper using JSON (#2929) * feat: test mapper * chore: temporally add other runner * chore: temp remove ubuntu-18-04-openai-api-collection-test * chore: test json file * chore: test json file * Correct path endpoints_mapping.json * feat: running via endpoints * feat: running multiple endpoint * feat: use endpoint value from workflow dispatch * feat: add mapper between endpoint and python test file * feat: config run all * feat: config run all --------- Co-authored-by: Van-QA Co-authored-by: Hien To --- .github/workflows/jan-openai-api-test.yml | 21 +++++-- docs/tests/conftest.py | 42 +++++++++++-- docs/tests/endpoints_mapping.json | 75 +++++++++++++++++++++++ 3 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 docs/tests/endpoints_mapping.json diff --git a/.github/workflows/jan-openai-api-test.yml b/.github/workflows/jan-openai-api-test.yml index 9964a41d5..b7e2717b3 100644 --- a/.github/workflows/jan-openai-api-test.yml +++ b/.github/workflows/jan-openai-api-test.yml @@ -1,6 +1,13 @@ name: Test - OpenAI API Pytest collection on: workflow_dispatch: + inputs: + endpoints: + description: 'comma-separated list (see available at endpoints_mapping.json e.g. GET /users,POST /transform)' + required: false + default: all + type: string + push: branches: - main @@ -38,11 +45,11 @@ jobs: rm -rf ~/jan make clean - - name: install dependencies + - name: Install dependencies run: | npm install -g @stoplight/prism-cli - - name: create python virtual environment and run test + - name: Create python virtual environment and run test run: | python3 -m venv /tmp/jan source /tmp/jan/bin/activate @@ -65,10 +72,14 @@ jobs: # Append to conftest.py cat ../docs/tests/conftest.py >> tests/conftest.py - + cat ../docs/tests/endpoints_mapping.json >> tests/endpoints_mapping.json + # start mock server and run test then stop mock server - prism mock ../docs/openapi/jan.yaml > prism.log & prism_pid=$! && pytest --reportportal --html=report.html && kill $prism_pid + prism mock ../docs/openapi/jan.yaml > prism.log & prism_pid=$! && + pytest --endpoint "$ENDPOINTS" --reportportal --html=report.html && kill $prism_pid deactivate + env: + ENDPOINTS: ${{ github.event.inputs.endpoints }} - name: Upload Artifact uses: actions/upload-artifact@v2 @@ -79,7 +90,7 @@ jobs: openai-python/assets openai-python/prism.log - - name: clean up + - name: Clean up if: always() run: | rm -rf /tmp/jan diff --git a/docs/tests/conftest.py b/docs/tests/conftest.py index 86b6c422f..bec60cf0b 100644 --- a/docs/tests/conftest.py +++ b/docs/tests/conftest.py @@ -1,6 +1,40 @@ +import json + + +def pytest_addoption(parser): + parser.addoption( + "--endpoint", action="store", default="all", help="my option: endpoints" + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "endpoint(endpoint): this mark select the test based on endpoint" + ) + + +def pytest_runtest_setup(item): + getoption = item.config.getoption("--endpoint").split(",") + if getoption not in (["all"], [''], [""]): + endpoint_names = [mark.args[0] for mark in item.iter_markers(name="endpoint")] + if not endpoint_names or not set(getoption).intersection(set(endpoint_names)): + pytest.skip("Test skipped because endpoint is {!r}".format(endpoint_names)) + + def pytest_collection_modifyitems(items): + # load the JSON file + with open("tests/endpoints_mapping.json", "r") as json_file: + endpoints_file_mapping = json.load(json_file) + + # create a dictionary to map filenames to endpoints + filename_to_endpoint = {} + for endpoint, files in endpoints_file_mapping.items(): + for filename in files: + filename_to_endpoint[filename] = endpoint + + # add the markers based on the JSON file for item in items: - # add the name of the file (without extension) as a marker - filename = item.nodeid.split("::")[0].split("/")[-1].replace(".py", "") - marker = pytest.mark.file(filename) - item.add_marker(marker) + # map the name of the file to endpoint, else use default value + filename = item.fspath.basename + marker = filename_to_endpoint.get(filename, filename) + item.add_marker(pytest.mark.endpoint(marker, filename=filename)) diff --git a/docs/tests/endpoints_mapping.json b/docs/tests/endpoints_mapping.json new file mode 100644 index 000000000..5800abc32 --- /dev/null +++ b/docs/tests/endpoints_mapping.json @@ -0,0 +1,75 @@ +{ + "/embeddings": [ + "test_embedding.py" + ], + "/audio/translations": [ + "test_translations.py" + ], + "/audio/transcriptions": [ + "test_transcriptions.py" + ], + "/moderations": [ + "test_moderations.py" + ], + "/images/generations": [ + "test_images.py" + ], + "/batches": [ + "test_batches.py" + ], + "/vector_stores": [ + "test_vector_stores.py" + ], + "/fine_tuning/jobs": [ + "test_jobs.py", + "test_checkpoints.py" + ], + "/assistants": [ + "test_assistants.py" + ], + "/threads/{thread_id}/runs": [ + "test_runs.py" + ], + "/threads/{thread_id}/runs/{run_id}/steps": [ + "test_steps.py" + ], + "/vector_stores/{vector_store_id}/file_batches": [ + "test_file_batches.py" + ], + "/messages": [ + "test_messages.py" + ], + "/vector_stores/{vector_store_id}/files": [ + "test_files.py" + ], + "/chat/completions": [ + "test_completions.py" + ], + "/threads": [ + "test_threads.py" + ], + "/audio/speech": [ + "test_speech.py" + ], + "/models": [ + "test_models.py" + ], + "native_client_sdk_only": [ + "test_streaming.py" + ], + "utils": [ + "test_response.py", + "test_client.py", + "test_extract_files.py", + "test_typing.py", + "test_legacy_response.py", + "test_module_client.py", + "test_old_api.py", + "test_proxy.py", + "test_qs.py", + "test_required_args.py", + "test_transform.py", + "test_azure.py", + "test_deepcopy.py" + ] +}