litellm/.github/workflows/run_observatory_tests.yml

name: Run Observatory Tests
on:
  workflow_dispatch:
    inputs:
      tag:
        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
        required: true
        type: string
      commit_hash:
        description: "Commit hash (defaults to HEAD of current branch)"
        required: false
        type: string
  workflow_call:
    inputs:
      tag:
        description: "Docker image tag to test"
        required: true
        type: string
      commit_hash:
        description: "Commit hash of the release"
        required: true
        type: string

permissions:
  contents: read

env:
  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}

jobs:
  observatory-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout repository
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Validate tag input
        env:
          TAG: ${{ inputs.tag }}
        run: |
          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
            exit 1
          fi

      - name: Start LiteLLM container
        env:
          TAG: ${{ inputs.tag }}
          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
          WORKSPACE: ${{ github.workspace }}
        run: |
          docker run -d \
            --name litellm-rc \
            -p 4000:4000 \
            -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
            -e AZURE_API_KEY="${AZURE_API_KEY}" \
            -e AZURE_API_BASE="${AZURE_API_BASE}" \
            "litellm/litellm:${TAG}" \
            --config /app/config.yaml --port 4000

      - name: Wait for LiteLLM health check
        run: |
          echo "Waiting for LiteLLM to be ready..."
          for i in $(seq 1 30); do
            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
              echo "LiteLLM is healthy"
              exit 0
            fi
            echo "Attempt $i/30 - not ready yet, waiting 10s..."
            sleep 10
          done
          echo "LiteLLM failed to start within 5 minutes"
          docker logs litellm-rc
          exit 1

      - name: Start cloudflared tunnel
        run: |
          # Install cloudflared (pinned version + checksum)
          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
          echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8  /usr/local/bin/cloudflared" | sha256sum -c -
          chmod +x /usr/local/bin/cloudflared

          # Start a quick tunnel (no account needed) and capture the URL
          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
          CLOUDFLARED_PID=$!
          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV

          # Wait for tunnel URL to appear in logs
          echo "Waiting for tunnel URL..."
          for i in $(seq 1 30); do
            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
            if [ -n "$TUNNEL_URL" ]; then
              echo "Tunnel URL: $TUNNEL_URL"
              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
              exit 0
            fi
            sleep 2
          done
          echo "Failed to get tunnel URL"
          cat /tmp/cloudflared.log
          exit 1

      - name: Verify tunnel connectivity
        run: |
          echo "Testing tunnel at ${TUNNEL_URL}..."
          # Quick tunnels need time for DNS propagation; retry to avoid
          # transient NXDOMAIN (curl exit code 6) on first attempt.
          for i in $(seq 1 10); do
            if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then
              echo "Tunnel is working (attempt $i)"
              exit 0
            fi
            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
            sleep 5
          done
          echo "Tunnel failed to become reachable after 50s"
          cat /tmp/cloudflared.log
          exit 1

      - name: Trigger observatory test run
        id: trigger
        env:
          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
        run: |
          PAYLOAD=$(jq -n \
            --arg url "${TUNNEL_URL}" \
            --arg key "${LITELLM_MASTER_KEY}" \
            '{
              deployment_url: $url,
              api_key: $key,
              test_suite: "TestOAIAzureRelease",
              models: ["gpt-4o-mini", "gpt-4o"]
            }')
          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
            -H "Content-Type: application/json" \
            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
            -d "$PAYLOAD")
          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
          BODY=$(echo "$RESPONSE" | head -n -1)
          echo "Response ($HTTP_CODE): $BODY"
          if [ "$HTTP_CODE" -ge 400 ]; then
            echo "Failed to trigger test run"
            exit 1
          fi

          # Extract request_id for polling this specific run
          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
            echo "Failed to extract request_id from response"
            exit 1
          fi
          echo "Request ID: $REQUEST_ID"
          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT

      - name: Poll for test completion
        id: poll
        env:
          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
        run: |
          TIMEOUT=900  # 15 minutes
          INTERVAL=30
          ELAPSED=0
          while [ $ELAPSED -lt $TIMEOUT ]; do
            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"

            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
              echo "Test finished with status: $RUN_STATUS"
              echo "$STATUS" > /tmp/observatory_result.json
              exit 0
            fi

            sleep $INTERVAL
            ELAPSED=$((ELAPSED + INTERVAL))
          done
          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
          exit 1

      - name: Verify test results
        run: |
          RESULT=$(cat /tmp/observatory_result.json)
          echo "Full result: $RESULT"

          STATUS=$(echo "$RESULT" | jq -r '.status')
          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
          ERROR=$(echo "$RESULT" | jq -r '.error // empty')

          echo "Status: $STATUS"
          echo "Test passed: $TEST_PASSED"
          echo "Failure rate: $FAILURE_RATE"

          if [ -n "$ERROR" ]; then
            echo "Error: $ERROR"
          fi

          if [ "$STATUS" = "failed" ]; then
            echo "Test run failed"
            exit 1
          fi

          if [ "$TEST_PASSED" != "true" ]; then
            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
            exit 1
          fi

          echo "All tests passed!"

      - name: Print LiteLLM logs on failure
        if: failure()
        run: |
          docker logs litellm-rc 2>/dev/null || true
          cat /tmp/cloudflared.log 2>/dev/null || true

      - name: Cleanup
        if: always()
        run: |
          kill "$CLOUDFLARED_PID" 2>/dev/null || true
          docker rm -f litellm-rc 2>/dev/null || true