From 80912a0684ec5e7cfcf75758b46b70ca6866455c Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 9 Nov 2026 20:44:42 +0000 Subject: [PATCH] Switch to the shared staged deploy workflows (LLM variant) Replace the legacy test-and-deploy.yml with the four workflows used by survey and the other evaluation functions (test-lint, staging-deploy, production-deploy, pre_production_tests), calling evaluation-function-workflows deploy.yml@main. LLM-backed function: CI runs lint only (tests call model providers) and production deploys keep run-database-tests: false. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pre_production_tests.yml | 49 ++++++ .github/workflows/production-deploy.yml | 63 +++++++ .github/workflows/staging-deploy.yml | 62 +++++++ .github/workflows/test-and-deploy.yml | 195 --------------------- .github/workflows/test-lint.yml | 43 +++++ pytest.ini | 4 + 6 files changed, 221 insertions(+), 195 deletions(-) create mode 100644 .github/workflows/pre_production_tests.yml create mode 100644 .github/workflows/production-deploy.yml create mode 100644 .github/workflows/staging-deploy.yml delete mode 100755 .github/workflows/test-and-deploy.yml create mode 100644 .github/workflows/test-lint.yml create mode 100644 pytest.ini diff --git a/.github/workflows/pre_production_tests.yml b/.github/workflows/pre_production_tests.yml new file mode 100644 index 0000000..2daabd1 --- /dev/null +++ b/.github/workflows/pre_production_tests.yml @@ -0,0 +1,49 @@ +name: Run Pre-Production Validation Tests + +on: + workflow_dispatch: + inputs: + eval_function: + type: string + description: "The name of the evaluation function to test" + required: true + sql_limit: + type: number + description: "The maximum number of SQL test cases to run" + required: false + default: 500 + seed: + type: string + description: "Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate." + required: false + default: '' + request_delay: + type: string + description: "Delay (seconds) between dispatching requests" + required: false + default: '0' + max_concurrency: + type: string + description: "Max concurrent requests (lower for GPT-backed functions)" + required: false + default: '5' + +jobs: + run-pre-production-tests: + name: 🧪 Run Staging Validation Tests + uses: lambda-feedback/Database-Testing/.github/workflows/test_evaluation_function.yml@main + with: + eval_function: ${{ inputs.eval_function }} + sql_limit: ${{ inputs.sql_limit }} + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} + secrets: + TEST_API_ENDPOINT: ${{ secrets.TEST_API_ENDPOINT }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_HOST: ${{ secrets.DB_HOST }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_NAME: ${{ secrets.DB_NAME }} + GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_DB_PROJECT_ID }} \ No newline at end of file diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml new file mode 100644 index 0000000..82c13e0 --- /dev/null +++ b/.github/workflows/production-deploy.yml @@ -0,0 +1,63 @@ +name: Deploy to Production + +on: + workflow_dispatch: + inputs: + version-bump: + description: 'Version bump type' + required: true + type: choice + options: + - patch + - minor + - major + default: patch + branch: + description: 'Branch to release from' + required: true + type: string + default: 'main' + seed: + description: 'Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate.' + required: false + type: string + default: '' + request_delay: + description: 'Delay (seconds) between dispatching requests' + required: false + type: string + default: '0' + max_concurrency: + description: 'Max concurrent requests (lower for GPT-backed functions)' + required: false + type: string + default: '5' +jobs: + deploy: + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@main + with: + template-repository-name: 'lambda-feedback/evaluation-function-boilerplate-python' + build-file: "app/Dockerfile" + build-context: "./app" + environment: "production" + version-bump: ${{ inputs.version-bump }} + branch: ${{ inputs.branch }} + run-database-tests: false # LLM-backed: database tests call OpenAI + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} + + secrets: + aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key-staging: ${{ secrets.FUNCTION_ADMIN_API_KEY_STAGING }} + function-admin-api-key-production: ${{ secrets.FUNCTION_ADMIN_API_KEY_PRODUCTION }} + gcp_credentials: ${{ secrets.GCP_DEPLOY }} + TEST_API_ENDPOINT: ${{ secrets.TEST_API_ENDPOINT }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_HOST: ${{ secrets.DB_HOST }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_NAME: ${{ secrets.DB_NAME }} + GCP_DB_CREDS: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} \ No newline at end of file diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml new file mode 100644 index 0000000..98fee7b --- /dev/null +++ b/.github/workflows/staging-deploy.yml @@ -0,0 +1,62 @@ +name: Deploy to Staging + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m venv .venv + .venv/bin/pip install pytest + .venv/bin/pip install -r app/requirements.txt + + + - name: Lint with flake8 + run: | + source .venv/bin/activate + .venv/bin/pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 ./app --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./app --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + # Tests are disabled in CI: they call the OpenAI API. + deploy: + needs: test + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@main + with: + template-repository-name: "lambda-feedback/evaluation-function-boilerplate-python" + build-file: "app/Dockerfile" + build-context: "./app" + build-platforms: "aws" + environment: "staging" + lfs: false + secrets: + aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key-staging: ${{ secrets.FUNCTION_ADMIN_API_KEY_STAGING }} + gcp_credentials: ${{ secrets.GCP_DEPLOY }} diff --git a/.github/workflows/test-and-deploy.yml b/.github/workflows/test-and-deploy.yml deleted file mode 100755 index febffd7..0000000 --- a/.github/workflows/test-and-deploy.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Test & Deploy Evaluation Function to AWS Lambda - -on: - push: - branches: - - master - - main - workflow_dispatch: - -jobs: - test: - name: Test - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: [3.8] - - defaults: - run: - working-directory: app/ - - env: - SCHEMAS_URL: https://raw.githubusercontent.com/lambda-feedback/request-response-schemas/master/ - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - python -m pip install -r requirements.txt - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Test Evaluation Function - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - run: | - pytest -v evaluation_tests.py::TestEvaluationFunction - - - name: Test Preview Function - run: | - pytest -v preview_tests.py::TestPreviewFunction - - deploy-staging: - name: Deploy Staging - needs: test - runs-on: ubuntu-latest - environment: production - env: - ECR_REPOSITORY: lambda-feedback-staging-functions-repository - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set config.json output - id: set_config_var - run: | - content=`cat ./config.json` - # the following lines are only required for multi line json - content="${content//'%'/'%25'}" - content="${content//$'\n'/'%0A'}" - content="${content//$'\r'/'%0D'}" - # end of optional handling for multi line json - echo "::set-output name=configJson::$content" - - - name: set Evaluation Function Name - id: set_function_name - run: | - functionName="${{fromJson(steps.set_config_var.outputs.configJson).EvaluationFunctionName}}" - [[ -z "$functionName" ]] && { echo "Add EvaluationFunctionName to config.json" ; exit 1; } - echo "::set-output name=function_name::$functionName" - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} - aws-secret-access-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET }} - aws-region: eu-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - # Build docker image from algorithm, schema and requirements - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG app/ - # Push image to ECR - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" - - - name: deploy evaluation function - id: deploy-evaluation-function - env: - BACKEND_API_URL: https://staging-api.lambdafeedback.com - API_KEY: ${{ secrets.FUNCTION_ADMIN_API_KEY }} - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - curl --location --request POST "$BACKEND_API_URL/grading-function/ensure" \ - --header 'content-type: application/json' \ - --data-raw "{ - \"apiKey\": \"$API_KEY\", - \"dockerImageUri\": \"$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG\", - \"functionName\": \"$IMAGE_TAG\" - }" - - deploy-production: - name: Deploy Production - needs: deploy-staging - runs-on: ubuntu-latest - environment: production - env: - ECR_REPOSITORY: lambda-feedback-production-functions-repository - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set config.json output - id: set_config_var - run: | - content=`cat ./config.json` - # the following lines are only required for multi line json - content="${content//'%'/'%25'}" - content="${content//$'\n'/'%0A'}" - content="${content//$'\r'/'%0D'}" - # end of optional handling for multi line json - echo "::set-output name=configJson::$content" - - - name: set Evaluation Function Name - id: set_function_name - run: | - functionName="${{fromJson(steps.set_config_var.outputs.configJson).EvaluationFunctionName}}" - [[ -z "$functionName" ]] && { echo "Add EvaluationFunctionName to config.json" ; exit 1; } - echo "::set-output name=function_name::$functionName" - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} - aws-secret-access-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET }} - aws-region: eu-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - # Build docker image from algorithm, schema and requirements - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG app/ - # Push image to ECR - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" - - - name: deploy evaluation function - id: deploy-evaluation-function - env: - BACKEND_API_URL: https://prod-api.lambdafeedback.com - API_KEY: ${{ secrets.FUNCTION_ADMIN_API_KEY }} - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - curl --location --request POST "$BACKEND_API_URL/grading-function/ensure" \ - --header 'content-type: application/json' \ - --data-raw "{ - \"apiKey\": \"$API_KEY\", - \"dockerImageUri\": \"$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG\", - \"functionName\": \"$IMAGE_TAG\" - }" diff --git a/.github/workflows/test-lint.yml b/.github/workflows/test-lint.yml new file mode 100644 index 0000000..1ca434e --- /dev/null +++ b/.github/workflows/test-lint.yml @@ -0,0 +1,43 @@ +name: Test and Lint + +on: + pull_request: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m venv .venv + .venv/bin/pip install pytest + .venv/bin/pip install -r app/requirements.txt + + - name: Lint with flake8 + run: | + source .venv/bin/activate + .venv/bin/pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 ./app --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./app --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + # Tests are disabled in CI: they call the OpenAI API. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..589497f --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +# evaluation_tests.py is the name the base layer's healthcheck imports, so +# include it alongside pytest's default patterns. +python_files = test_*.py *_test.py *_tests.py