promptflow-sdk-cli-test.yml

name: promptflow-sdk-cli-test
on:
  schedule:
    - cron: "40 18 * * *" # Every day starting at 2:40 BJT
  pull_request:
    paths:
      - src/promptflow-core/**
      - src/promptflow-devkit/**
      - src/promptflow/**
      - src/promptflow-tracing/**
      - scripts/building/**
      - .github/workflows/promptflow-sdk-cli-test.yml
      - src/promptflow-recording/**
  workflow_dispatch:
    inputs:
      filepath:
        description: file or paths you want to trigger a test
        required: true
        default: "./tests/sdk_cli_test ./tests/sdk_pfs_test"
        type: string
permissions:
  id-token: write
  contents: read
env:
  IS_IN_CI_PIPELINE: "true"
  PF_DISABLE_TRACING: "false"
  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
jobs:
  sdk_cli_tests:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        pythonVersion: ['3.9', '3.10', '3.11']
    environment:
      internal
    runs-on: ${{ matrix.os }}
    steps:
    - name: set test mode
      run: |
        echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
        echo "FILE_PATHS=$(if [[ "${{ inputs.filepath }}" == "" ]]; then echo "./tests/sdk_cli_test ./tests/sdk_pfs_test"; else echo ${{ inputs.filepath }}; fi)" >> $GITHUB_ENV
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.pythonVersion }}
    - uses: snok/install-poetry@v1
    - name: install test dependency group
      run: |
        set -xe
        poetry install -E pyarrow --with ci,test

        poetry run pip show promptflow-tracing
        poetry run pip show promptflow-core
        poetry run pip show promptflow-devkit
        poetry run pip show promptflow-tools
      working-directory: ${{ env.WORKING_DIRECTORY }}
    - name: Azure login (non pull_request workflow)
      if: github.event_name != 'pull_request'
      uses: azure/login@v1
      with:
        subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
        tenant-id: ${{secrets.AZURE_TENANT_ID}}
        client-id: ${{secrets.AZURE_CLIENT_ID}}
    - name: generate live test resources (non pull_request workflow)
      if: github.event_name != 'pull_request'
      uses: "./.github/actions/step_generate_configs"
      with:
        targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
    - name: generate live test resources (pull_request workflow)
      if: github.event_name == 'pull_request'
      working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
      run: |
        cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
    - name: run devkit tests
      run: |
        poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \
        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \
        --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py --tb=short
      working-directory: ${{ env.WORKING_DIRECTORY }}
    - name: Upload Test Results
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
        path: |
          ${{ env.WORKING_DIRECTORY }}/*.xml
          ${{ env.WORKING_DIRECTORY }}/htmlcov/
          ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json
    - run: poetry install -E executable
      working-directory: ${{ env.WORKING_DIRECTORY }}
    - name: run devkit executable tests
      run: |
        poetry run pytest -n auto -m "unittest or e2etest" ./tests/sdk_cli_test/e2etests/test_executable.py --tb=short
      working-directory: ${{ env.WORKING_DIRECTORY }}


  publish-test-results-sdk-cli-test:
    needs: sdk_cli_tests
    runs-on: ubuntu-latest
    permissions:
      checks: write
      pull-requests: write
      contents: read
      issues: read
    if: always()

    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: Publish Test Results
      uses: "./.github/actions/step_publish_test_results"
      with:
        testActionFileName: promptflow-sdk-cli-test.yml
        testResultTitle: SDK CLI Test Result
        osVersion: ubuntu-latest
        pythonVersion: 3.9
        coverageThreshold: 40
        context: test/sdk_cli