diff --git a/.github/workflows/check-pod.yaml b/.github/workflows/check-pod.yaml deleted file mode 100644 index cbae547af..000000000 --- a/.github/workflows/check-pod.yaml +++ /dev/null @@ -1,58 +0,0 @@ -name: Check Pod - -on: - pull_request: - push: - branches: - - main - -jobs: - check: - runs-on: macos-13 - steps: - - uses: actions/checkout@v2 - - - name: Select Specific Xcode Version (15.1) - run: | - sudo xcode-select -s /Applications/Xcode_15.1.app - echo "Selected Xcode version:" - xcodebuild -version - - # Run the steps we document in the Release Process. - # unzip commands included as proof-of-life for the Carthage output. - - name: Print Ruby version - run: ruby --version - - name: Print Carthage version - run: 'echo -n "carthage version: " && carthage version' - - name: Print CocoaPods version - run: 'echo -n "pod version: " && pod --version --verbose' - - name: Print Make version - run: make --version - - name: Build Carthage dependencies - run: make update - - name: Build Ably framework - run: make carthage_package - - name: Print contents of generated ZIP file - run: | - unzip -l Ably.framework.zip - unzip -l Ably.framework.zip | grep 'Mac/Ably.framework' - unzip -l Ably.framework.zip | grep 'tvOS/Ably.framework' - unzip -l Ably.framework.zip | grep 'iOS/Ably.framework' - - name: Validate pod - run: pod lib lint - # We move Ably.framework.zip into a directory. This is because, by - # default, macOS’s Archive Utility unzips directly-nested zip files, so - # if Ably.framework.zip were at the top level of the zip file that - # actions/upload-artifact creates, then Archive Utility would unzip - # Ably.framework.zip too, which we don’t want, since we want this file - # to be kept intact so that we can upload it to GitHub releases as - # described in CONTRIBUTING.md. - - name: Prepare built framework for archiving - run: | - mkdir -p carthage-built-framework-artifact-contents/carthage-built-framework - mv Ably.framework.zip carthage-built-framework-artifact-contents/carthage-built-framework - - name: Archive built framework - uses: actions/upload-artifact@v3 - with: - name: carthage-built-framework - path: carthage-built-framework-artifact-contents diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 081cd98fb..000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Docs Generation - -on: - pull_request: - push: - branches: - - main - tags: - - '*' - -jobs: - build: - runs-on: macos-latest - - permissions: - deployments: write - id-token: write - - steps: - - uses: actions/checkout@v2 - - - name: Select Specific Xcode Version (15.1) - run: | - sudo xcode-select -s /Applications/Xcode_15.1.app - echo "Selected Xcode version:" - xcodebuild -version - - - name: Install Dependencies - run: | - make submodules - bundle install - make update_carthage_dependencies_macos - - - name: Build Documentation - run: | - ./Scripts/jazzy.sh - ls -al Docs/jazzy - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: eu-west-2 - role-to-assume: arn:aws:iam::${{ secrets.ABLY_AWS_ACCOUNT_ID_SDK }}:role/ably-sdk-builds-ably-cocoa - role-session-name: "${{ github.run_id }}-${{ github.run_number }}" - - - name: Upload Documentation - uses: ably/sdk-upload-action@v1 - with: - sourcePath: Docs/jazzy - githubToken: ${{ secrets.GITHUB_TOKEN }} - artifactName: jazzydoc - diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml deleted file mode 100644 index d418daddb..000000000 --- a/.github/workflows/examples.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: Examples Test - -on: - pull_request: - push: - branches: - - main - -jobs: - check: - runs-on: macos-13 - - env: - LC_CTYPE: en_US.UTF-8 - LANG: en_US.UTF-8 - ABLY_ENV: sandbox - - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Select Xcode (15.1) - run: | - sudo xcode-select -s /Applications/Xcode_15.1.app - echo "Selected Xcode version:" - xcodebuild -version - - - name: Environment Info - run: ./Scripts/log-environment-information.sh - - - name: Reset Simulators - run: xcrun simctl erase all - - - name: Install Dependencies - run: | - make submodules - bundle install - make update_carthage_dependencies_ios - - - name: Run Examples Tests - working-directory: ./Examples/Tests - run: | - pod repo update - pod install - bundle exec fastlane scan -s Tests --output-directory "fastlane/test_output/examples/test_iOS17_2" - - - name: Carthage Installation - working-directory: ./Examples/AblyCarthage - run: | - echo 'Installing Carthage dependencies...' - carthage update --use-xcframeworks --platform iOS --no-use-binaries - echo 'Building AblyCarthage example...' - xcodebuild build -scheme "AblyCarthage" -destination "platform=iOS Simulator,name=iPhone 15" -configuration "Debug" - - - name: SPM Installation - working-directory: ./ - run: | - echo 'Current Branch: ' $GITHUB_HEAD_REF - echo 'Current Revision (SHA):' $GITHUB_SHA - echo Current Path: $(pwd) - export PACKAGE_URL=file://$(pwd) - export PACKAGE_BRANCH_NAME=$GITHUB_HEAD_REF - export PACKAGE_REVISION=$GITHUB_SHA - swift test --package-path Examples/SPM -v - - - name: Build APNS Example - working-directory: ./Examples/AblyPush - run: | - xcodebuild build -scheme "AblyPushExample" -destination "platform=iOS Simulator,name=iPhone 15" -configuration "Debug" diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml deleted file mode 100644 index d1c123738..000000000 --- a/.github/workflows/features.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Features - -on: - pull_request: - push: - branches: - - main - -jobs: - build: - uses: ably/features/.github/workflows/sdk-features.yml@main - with: - repository-name: ably-cocoa - secrets: inherit diff --git a/.github/workflows/integration-test-iOS17_2.yaml b/.github/workflows/integration-test-iOS17_2.yaml new file mode 100644 index 000000000..485df62e8 --- /dev/null +++ b/.github/workflows/integration-test-iOS17_2.yaml @@ -0,0 +1,70 @@ +name: "Integration Test: iOS 17.2" + +on: + pull_request: + push: + branches: + - main + +# IMPORTANT NOTES: +# - Changes made to this file needs to replicated across other integration-test-*.yaml files. +# - The Fastlane lane name is duplicated in more than one place within this workflow. + +jobs: + check: + runs-on: macos-13 + + env: + LC_CTYPE: en_US.UTF-8 + LANG: en_US.UTF-8 + ABLY_ENV: sandbox + + steps: + - name: Check out SDK repo + uses: actions/checkout@v2 + + - name: Select Specific Xcode Version (15.1) + run: | + sudo xcode-select -s /Applications/Xcode_15.1.app + echo "Selected Xcode version:" + xcodebuild -version + + - name: Log environment information + run: ./Scripts/log-environment-information.sh + + - name: Check out xcparse repo + uses: actions/checkout@v3 + with: + repository: ably-forks/xcparse + ref: emit-test-case-info + path: xcparse + + - id: get-xcparse-commit-sha + name: Get xcparse commit SHA + run: | + cd xcparse + echo "::set-output name=sha::$(git rev-parse HEAD)" + + - name: "actions/cache@v3 (xcparse binary)" + uses: actions/cache@v3 + with: + path: xcparse/.build/debug/xcparse + key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }} + + - name: Install Dependencies and Run Tests Continuously + env: + TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }} + run: | + brew install xcbeautify + brew install coreutils # for `timeout` + make submodules + bundle install + make update_carthage_dependencies_ios + Scripts/continuously-run-tests-and-upload-results.sh --lane test_iOS17_2 + + - name: Upload .xcresult bundles + uses: actions/upload-artifact@v3 + if: always() + with: + name: xcresult-bundles.tar.gz + path: xcresult-bundles.tar.gz diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml deleted file mode 100644 index bbe44efbc..000000000 --- a/.github/workflows/integration-test.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: "Integration Test" - -on: - pull_request: - push: - branches: - - main - -# IMPORTANT NOTES: -# - Changes made to this file needs to replicated across other integration-test-*.yaml files. -# - The Fastlane lane name is duplicated in more than one place within this workflow. - -jobs: - check: - runs-on: macos-13 - - strategy: - fail-fast: false - matrix: - include: - - - platform: iOS - lane: test_iOS17_2 - - - platform: tvOS - lane: test_tvOS17_2 - - - platform: macOS - lane: test_macOS - - env: - LC_CTYPE: en_US.UTF-8 - LANG: en_US.UTF-8 - ABLY_ENV: sandbox - - steps: - - name: Check out repo - uses: actions/checkout@v2 - - - name: Select Xcode (15.1) - run: | - sudo xcode-select -s /Applications/Xcode_15.1.app - echo "Selected Xcode version:" - xcodebuild -version - - - name: Environment Info - run: ./Scripts/log-environment-information.sh - - - name: Check out xcparse repo - uses: actions/checkout@v3 - with: - repository: ably-forks/xcparse - ref: emit-test-case-info - path: xcparse - - - id: get-xcparse-commit-sha - name: Get xcparse commit SHA - run: | - cd xcparse - echo "::set-output name=sha::$(git rev-parse HEAD)" - - - name: "actions/cache@v3 (xcparse binary)" - uses: actions/cache@v3 - with: - path: xcparse/.build/debug/xcparse - key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }} - - - name: Reset Simulators - run: xcrun simctl erase all - - - name: Install Dependencies - run: | - brew install xcbeautify - make submodules - bundle install - carthage update --use-xcframeworks --platform ${{ matrix.platform }} --no-use-binaries - - - name: Run Tests - run: bundle exec fastlane ${{ matrix.lane }} - - - name: Check Static Analyzer Output - id: analyzer-output - run: | - if [[ -z $(find ./derived_data -name "report-*.html") ]]; then - echo "Static Analyzer found no issues." - else - echo "Static Analyzer found some issues. HTML report will be available in Artifacts section. Failing build." - exit 1 - fi - - - name: Upload Static Analyzer Reports - if: ${{ failure() && steps.analyzer-output.outcome == 'failure' }} - uses: actions/upload-artifact@v2 - with: - name: static-analyzer-reports-${{ matrix.lane }} - path: ./derived_data/**/report-*.html - - - name: Upload Xcodebuild Logs - if: always() - uses: actions/upload-artifact@v2 - with: - name: xcodebuild-logs - path: ~/Library/Developer/Xcode/DerivedData/*/Logs - - - name: Upload Test Output - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: fastlane/test_output - - - name: Upload Test Results - if: always() - env: - TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - Scripts/upload_test_results.sh --job-name "check (${{ matrix.platform }}, ${{ matrix.lane }})" - diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh new file mode 100755 index 000000000..2b9346109 --- /dev/null +++ b/Scripts/continuously-run-tests-and-upload-results.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +set -e + +# 1. Check dependencies. + +if ! which timeout > /dev/null +then + echo "You need to install timeout (\`brew install coreutils\` on macOS)." 2>&1 + exit 1 +fi + +# 2. Grab command-line options. + +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash +while [[ "$#" -gt 0 ]]; do + case $1 in + -l|--lane) lane="$2"; shift ;; + -u|--upload-server-base-url) upload_server_base_url="$2"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +if [[ -z $lane ]] +then + echo "You need to specify the Fastlane lane to run (-l / --lane)." 2>&1 + exit 1 +fi + +# 3. Capture the time at which we started, to make sure we don’t exceed the +# maximum job running time. +started_at=`date +%s` +# https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration +let github_job_maximum_execution_seconds=6*60*60 +# We assume that the part of the job that ran before this script took at most 10 minutes, and that uploading the artifacts will take 30 minutes. +let must_end_by=$((started_at + github_job_maximum_execution_seconds - (10 + 30) * 60)) + +echo "We’ll make sure this script ends by `date -r${must_end_by}`." 2>&1 + +# 4. Run the tests in a loop and report the results. + +end_iteration_with_exit_value() { + if [[ -e xcresult-bundles ]] + then + echo "There are `du -d0 -h xcresult-bundles | awk -F '\t' '{print $1}'` of xcresult bundles to be uploaded." + tar --create --gzip xcresult-bundles > xcresult-bundles.tar.gz + echo "The file xcresult-bundles.tar.gz that will be uploaded as an artifact is `du -d0 -h xcresult-bundles.tar.gz | awk -F '\t' '{print $1}'`." + else + echo "There are no xcresult bundles to be uploaded." + fi + + exit $1 +} + +declare -i iteration=1 +while true +do + echo "BEGIN ITERATION ${iteration}" 2>&1 + + rm -rf fastlane/test_output + rm -rf xcodebuild_output + xcrun simctl erase all + + set +e + let allowed_execution_time=$must_end_by-`date +%s` + set -e + + if [[ $allowed_execution_time -le 0 ]]; then + echo "ITERATION ${iteration}: Allowed execution time reached. Exiting." 2>&1 + end_iteration_with_exit_value 0 + fi + + echo "ITERATION ${iteration}: Running fastlane with a timeout of ${allowed_execution_time} seconds." 2>&1 + + set +e + timeout --kill-after=20 ${allowed_execution_time} bundle exec fastlane --verbose $lane + tests_exit_value=$? + set -e + + if [[ tests_exit_value -eq 124 || tests_exit_value -eq 137 ]]; then + # Execution timed out. + echo "ITERATION ${iteration}: Cancelled the execution of fastlane since it exceeded timeout imposed by maximum GitHub running time. Terminating this script." + end_iteration_with_exit_value 0 + fi + + if [[ tests_exit_value -eq 0 ]] + then + echo "ITERATION ${iteration}: Tests passed." + else + echo "ITERATION ${iteration}: Tests failed (exit value ${tests_exit_value})." + fi + + echo "ITERATION ${iteration}: BEGIN xcodebuild raw output." + ls xcodebuild_output + cat xcodebuild_output/** + echo "ITERATION ${iteration}: END xcodebuild raw output." + + echo "ITERATION ${iteration}: Uploading results to observability server." + + # https://unix.stackexchange.com/questions/446847/conditionally-pass-params-to-a-script + optional_params=() + + if [[ ! -z $upload_server_base_url ]] + then + optional_params+=(--upload-server-base-url "${upload_server_base_url}") + fi + + set +e + ./Scripts/upload_test_results.sh \ + --iteration $iteration \ + "${optional_params[@]}" + # We defer failing the script until after copying the .xcresult bundle. + upload_exit_value=$? + set -e + + if [[ upload_exit_value -eq 0 ]] + then + echo "ITERATION ${iteration}: Upload succeeded." + else + echo "ITERATION ${iteration}: Upload failed (exit value ${upload_exit_value}). Will exit after copying result bundle." + fi + + # Find the .xcresult bundle and copy it to the directory that will eventually be saved as an artifact. + + result_bundles=$(find fastlane/test_output/sdk -name '*.xcresult') + if [[ -z $result_bundles ]] + then + number_of_result_bundles=0 + else + number_of_result_bundles=$(echo "${result_bundles}" | wc -l) + fi + + if [[ $number_of_result_bundles -eq 0 ]] + then + echo "ITERATION ${iteration}: No result bundles found." + end_iteration_with_exit_value 1 + fi + + if [[ $number_of_result_bundles -gt 1 ]] + then + echo -e "ITERATION ${iteration}: Multiple result bundles found:\n${result_bundles}" + end_iteration_with_exit_value 1 + fi + + echo "ITERATION ${iteration}: Report bundle found: ${result_bundles}" + + if [[ ! -d xcresult-bundles ]]; then + mkdir xcresult-bundles + fi + + mkdir "xcresult-bundles/${iteration}" + cp -r "${result_bundles}" "xcresult-bundles/${iteration}" + + echo "ITERATION ${iteration}: Copied result bundle to xcresult-bundles/${iteration}." + + if [[ upload_exit_value -ne 0 ]] + then + echo "ITERATION ${iteration}: Terminating due to failed upload." + end_iteration_with_exit_value $upload_exit_value + fi + + echo "END ITERATION ${iteration}" 2>&1 + + iteration+=1 +done diff --git a/Scripts/fetch-test-logs.sh b/Scripts/fetch-test-logs.sh new file mode 100755 index 000000000..60caa5e4b --- /dev/null +++ b/Scripts/fetch-test-logs.sh @@ -0,0 +1,493 @@ +#!/bin/bash + +# Retrieves the raw xcodebuild output for one or more test observability server +# uploads. +# +# Only works for tests that were run using the +# continuously-run-tests-and-upload-results script in this directory. + +# Usage: +# ./fetch-test-logs.sh --repo ably/ably-cocoa --test-case-id --filter [filter] +# +# or +# +# ./fetch-test-logs.sh --repo ably/ably-cocoa --upload-id + +# Options: +# +# -r / --repo : The 'org/name'-formatted name of the GitHub repo, for +# example 'ably/ably-cocoa'. +# +# -t / --test-case-id : The ID of a test case saved on the test +# observability server. Will fetch all uploads that match the filter specified +# using the --filter option, and then save the results inside the directory +# specified by the --output-directory option, in the following hierarchy, where +# uploads are split into those where the test case failed and those where it +# didn’t (which doesn’t necessarily imply that the test case passed; it may not +# have run at all): +# +# +# ├── info.json (contains metadata about the results in this directory) +# ├── upload_logs +# │   ├── failed +# │ │   └── xcodebuild-logs-upload-.txt, ... +# │   └── not_failed +# │ └── xcodebuild-logs-upload-.txt, ... +# └── test-case-logs (unless --no-extract-test-case-logs specified) +# ├── failed +# │   └── xcodebuild-logs-upload-.txt, ... +# └── not_failed +# └── xcodebuild-logs-upload-.txt, ... +# +# The upload_logs directory contains the full logs for that upload, and the +# test_case_logs directory contains just the segments of the logs that +# correspond to the specific test case. +# +# -d / --output-directory : Where to output the logs generated by the +# --test-case-id option to. Defaults to ./xcodebuild-logs-test-case--. +# +# -f / --filter : A URL query string describing a filter to be applied +# to the uploads fetched when using the --test-case-id option. For example, +# "branches[]=main&createdBefore=2022-02-20". +# +# -n / --no-extract-test-case-logs: Will cause the --test-case-id option to not +# attempt to extract the segment of the upload log that corresponds to the test +# case. +# +# -i / --upload-id : The ID of a upload saved on the test observability +# server. +# +# -u / --upload-server-base-url : Allows you to specify a URL to use as +# the upload server base URL. Defaults to https://test-observability.herokuapp.com. +# +# -o / --output-file : Where to output the logs generated by the +# --upload-id option to. Defaults to ./xcodebuild-logs-upload-.txt. +# +# -c / --cache-directory : Where to cache the GitHub logs. Defaults to +# ~/Library/Caches/com.ably.testObservabilityLogs. Will be created if doesn’t +# exist. +# +# -a / --no-use-github-auth: Will not prompt the user for an access token to be +# used for making requests to the GitHub API. Useful if all the required GitHub +# job logs are already cached locally. + +set -e + +check_dependencies() { + if ! which jq >/dev/null; then + echo "You need to install jq." 2>&1 + exit 1 + fi +} + +get_github_access_token() { + # https://stackoverflow.com/questions/3980668/how-to-get-a-password-from-a-shell-script-without-echoing#comment4260181_3980904 + read -s -p "Enter your GitHub access token (this will be used to fetch logs from the GitHub API): " github_access_token + + echo + + if [[ -z $github_access_token ]]; then + echo "You need to specify a GitHub access token." 2>&1 + exit 1 + fi + + echo +} + +# Args: +# $1: JSON representation of the test observability server upload +# $2: Path to write the logs to +fetch_and_write_logs_for_upload() { + upload_json=$1 + output_file=$2 + + # (TIL I learned that `echo` will interpret backslash sequences, which we + # don’t want. Appparently in general printf is recommended over echo.) + # https://stackoverflow.com/questions/43528202/prevent-echo-from-interpreting-backslash-escapes + github_repository=$(printf '%s' $upload_json | jq --raw-output '.githubRepository') + github_run_id=$(printf '%s' $upload_json | jq --raw-output '.githubRunId') + github_run_attempt=$(printf '%s' $upload_json | jq --raw-output '.githubRunAttempt') + github_job=$(printf '%s' $upload_json | jq --raw-output '.githubJob') + iteration=$(printf '%s' $upload_json | jq --raw-output '.iteration') + + echo "Upload comes from GitHub repository ${github_repository}. It has GitHub run ID ${github_run_id}, run attempt number ${github_run_attempt}, and job name ${github_job}. It corresponds to loop iteration ${iteration}." + + # Check whether we have a cached log for this job. + # (We cache the job logs because when running the tests continuously, with + # verbose logging enabled, a job log can be ~1.5GB.) + + log_file_name="github-log-${github_repository//\//-}-run-${github_run_id}-attempt-${github_run_attempt}-job-${github_job}" + log_file_path="${cache_directory}/${log_file_name}" + + if [[ -f "${log_file_path}" ]]; then + echo "GitHub job log file already exists at ${log_file_path}. Skipping download." 2>&1 + else + echo "GitHub job log file not yet downloaded." 2>&1 + + # (I wonder if this information that I’m fetching from GitHub is stuff that + # I should have just had in the upload in the first place? Not that + # important right now.) + + github_api_base_url="https://api.github.com" + + # From the GitHub API, fetch the jobs for this workflow run attempt. + # https://docs.github.com/en/rest/reference/actions#list-jobs-for-a-workflow-run-attempt + github_jobs_json=$(curl \ + --fail \ + -H "Accept: application/vnd.github.v3+json" \ + "${github_auth_curl_args[@]}" \ + "${github_api_base_url}/repos/${github_repository}/actions/runs/${github_run_id}/attempts/${github_run_attempt}/jobs") + + # From this list of jobs, find the one that corresponds to our upload. + github_job_id=$(printf "%s" $github_jobs_json | jq \ + --arg jobName "${github_job}" \ + '.jobs[] | select(.name == $jobName) | .id') + + if [[ -z $github_job_id ]]; then + echo "Could not find job with name ${github_job} in attempt ${github_run_attempt} of run ${github_run_id} in GitHub repository ${github_repository}." 2>&1 + exit 1 + fi + + echo "Upload corresponds to GitHub job ID ${github_job_id}. Downloading logs. This may take a while." + + # From the GitHub API, fetch the logs for this job and cache them. + # https://docs.github.com/en/rest/reference/actions#download-job-logs-for-a-workflow-run + + if [[ ! -d "${cache_directory}" ]]; then + mkdir -p "${cache_directory}" + fi + + curl \ + --fail \ + --location \ + -H "Accept: application/vnd.github.v3+json" \ + "${github_auth_curl_args[@]}" \ + "${github_api_base_url}/repos/${github_repository}/actions/jobs/${github_job_id}/logs" >"${log_file_path}.partial" + + mv "${log_file_path}.partial" "${log_file_path}" + + echo "Saved GitHub job logs to ${log_file_path}." + fi + + # Extract the part of the logs that corresponds to the raw xcodebuild output for this iteration. + # https://stackoverflow.com/a/18870500 + + echo "Finding xcodebuild output for iteration ${iteration}." + + xcodebuild_output_start_marker="ITERATION ${iteration}: BEGIN xcodebuild raw output" + xcodebuild_output_start_line_number=$(sed -n "/${xcodebuild_output_start_marker}/=" "${log_file_path}") + + if [[ -z "${xcodebuild_output_start_line_number}" ]]; then + echo "Couldn’t find start of xcodebuild raw output (couldn’t find marker \"${xcodebuild_output_start_marker}\")." 2>&1 + echo "This may be because the GitHub job hasn’t finished yet, or because the tests are not being run in a loop, or it may be an upload created before this functionality was implemented." 2>&1 + echo "You may need to delete the cached log file ${log_file_path}." 2>&1 + exit 1 + fi + + xcodebuild_output_end_marker="ITERATION ${iteration}: END xcodebuild raw output" + xcodebuild_output_end_line_number=$(sed -n "/${xcodebuild_output_end_marker}/=" "${log_file_path}") + + if [[ -z "${xcodebuild_output_end_line_number}" ]]; then + echo "Couldn’t find end of xcodebuild raw output (couldn’t find marker \"${xcodebuild_output_end_marker}\")." 2>&1 + exit 1 + fi + + # Strip the GitHub-added timestamps (which just correspond to the time that `cat` was executed on the log file, and hence aren’t of any use) from the start of each line. + + echo "Stripping GitHub timestamps." + + # https://arkit.co.in/print-given-range-of-lines-using-awk-perl-head-tail-and-python/ + sed -n "${xcodebuild_output_start_line_number},${xcodebuild_output_end_line_number} p" "${log_file_path}" | sed -e 's/^[^ ]* //' >"${output_file}" + + echo "Wrote xcodebuild output to ${output_file}." 2>&1 +} + +default_output_file_for_upload_id() { + echo "xcodebuild-logs-upload-$1.txt" +} + +run_for_test_case() { + # From the test observability server API, fetch the test case and extract its + # properties. + + echo "Fetching test case ${test_case_id} from ${upload_server_base_url}." 2>&1 + + test_case_json=$(curl --fail --header "Accept: application/json" "${upload_server_base_url}/repos/${repo}/test_cases/${test_case_id}") + + test_class_name=$(printf '%s' $test_case_json | jq --raw-output '.testClassName') + test_case_name=$(printf '%s' $test_case_json | jq --raw-output '.testCaseName') + + printf "Test case ${test_case_id} has test class name ${test_class_name} and test case name ${test_case_name}.\n\n" + + # From the test observability server API, fetch the filtered uploads. + + if [[ -z $filter ]]; then + filter_description="no filter" + filter_query="" + else + filter_description="filter ${filter}" + filter_query="?${filter}" + fi + + echo "Fetching uploads for test case ${test_case_id}, with ${filter_description}, from ${upload_server_base_url}." 2>&1 + + uploads_json=$(curl --fail --header "Accept: application/json" "${upload_server_base_url}/repos/${repo}/test_cases/${test_case_id}/uploads${filter_query}") + + number_of_uploads=$(printf '%s' $uploads_json | jq '. | length') + + if [[ ${number_of_uploads} -eq 1 ]]; then + echo "There is 1 upload". 2>&1 + else + echo "There are ${number_of_uploads} uploads". 2>&1 + fi + + echo + + mkdir "${output_directory}" + mkdir "${output_directory}/upload_logs" + mkdir "${output_directory}/test_case_logs" + + failed_upload_logs_output_directory="${output_directory}/upload_logs/failed" + mkdir "${failed_upload_logs_output_directory}" + not_failed_upload_logs_output_directory="${output_directory}/upload_logs/not_failed" + mkdir "${not_failed_upload_logs_output_directory}" + + failed_test_case_logs_output_directory="${output_directory}/test_case_logs/failed" + mkdir "${failed_test_case_logs_output_directory}" + not_failed_test_case_logs_output_directory="${output_directory}/test_case_logs/not_failed" + mkdir "${not_failed_test_case_logs_output_directory}" + + jq -n \ + --arg testCaseId "${test_case_id}" \ + --arg filter "${filter}" \ + --arg uploadServerBaseUrl "${upload_server_base_url}" \ + '{ fetchedAt: (now | todateiso8601), testCaseId: $testCaseId, filter: $filter, uploadServerBaseUrl: $uploadServerBaseUrl }' \ + >"${output_directory}/info.json" + + for ((i = 0; i < number_of_uploads; i += 1)); do + failed=$(printf '%s' $uploads_json | jq ".[${i}].failed") + upload_json=$(printf '%s' $uploads_json | jq ".[${i}].upload") + + upload_id=$(printf '%s' $upload_json | jq --raw-output '.id') + + echo "[$((i + 1)) of ${number_of_uploads}] Processing upload ${upload_id}." 2>&1 + + output_file_without_directory=$(default_output_file_for_upload_id "${upload_id}") + + if [[ $failed == "true" ]]; then + upload_log_output_file="${failed_upload_logs_output_directory}/${output_file_without_directory}" + test_case_log_output_file="${failed_test_case_logs_output_directory}/${output_file_without_directory}" + else + upload_log_output_file="${not_failed_upload_logs_output_directory}/${output_file_without_directory}" + test_case_log_output_file="${not_failed_test_case_logs_output_directory}/${output_file_without_directory}" + fi + + fetch_and_write_logs_for_upload "${upload_json}" "${upload_log_output_file}" + + if [[ -z "${no_extract_test_case_logs}" ]]; then + extract_logs_for_test_case "${test_class_name}" "${test_case_name}" "${upload_log_output_file}" "${test_case_log_output_file}" + fi + + echo + done +} + +# Args: +# $1: Test class name e.g. RealtimeClientPresenceTests +# $2: Test case name e.g. test__037__Presence__update__should_update_the_data_for_the_present_member_with_a_value() +# $3: Path of the xcodebuild logs for the entire test suite run +# $4: Path of where to write the test case logs to. +extract_logs_for_test_case() { + # Extract the part of the logs that corresponds to the raw xcodebuild output for this iteration. (We have similar code in fetch_and_write_logs_for_upload.) + + test_class_name=$1 + test_case_name=$2 + upload_log_file=$3 + output_file=$4 + + # (For some reason, the test case name in the observability server has + # trailing (), but in the xcodebuild logs it doesn’t. So strip them.) + sanitised_test_case_name="${test_case_name//[()]/}" + + echo "Finding logs for test class ${test_class_name}, test case ${test_case_name} in ${upload_log_file}." 2>&1 + + test_case_log_start_marker="Test Case.*${test_class_name} ${sanitised_test_case_name}.*started" + test_case_log_start_line_number=$(sed -n "/${test_case_log_start_marker}/=" "${upload_log_file}") + + if [[ -z "${test_case_log_start_line_number}" ]]; then + echo "Couldn’t find start of test case output (couldn’t find marker \"${test_case_log_start_marker}\")." 2>&1 + exit 1 + fi + + test_case_log_end_marker="Test Case.*${test_class_name} ${sanitised_test_case_name}.*(passed|failed)|Restarting after unexpected exit, crash, or test timeout in ${test_class_name}\/${sanitised_test_case_name}\(\)" + test_case_log_end_line_number=$(sed -En "/${test_case_log_end_marker}/=" "${upload_log_file}") + + if [[ -z "${test_case_log_end_line_number}" ]]; then + echo "Couldn’t find end of test case output (couldn’t find marker \"${test_case_log_end_marker}\")." 2>&1 + exit 1 + fi + + sed -n "${test_case_log_start_line_number},${test_case_log_end_line_number} p" "${upload_log_file}" >"${output_file}" + + echo "Wrote test case log to ${output_file}." 2>&1 +} + +run_for_upload() { + # From the test observability server API, fetch the upload, to find the + # GitHub run ID, attempt number, job name, and iteration. + + echo "Fetching upload ${upload_id} from ${upload_server_base_url}." 2>&1 + + upload_json=$(curl --fail --header "Accept: application/json" "${upload_server_base_url}/repos/${repo}/uploads/${upload_id}") + + fetch_and_write_logs_for_upload "${upload_json}" "${output_file}" +} + +check_dependencies + +# Grab and validate command-line options, and apply defaults. + +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash +while [[ "$#" -gt 0 ]]; do + case $1 in + -r | --repo) + repo="$2" + shift + ;; + -t | --test-case-id) + if [[ -z "$2" ]]; then + echo "You must specify a test case ID when using the --test-case-id option." 2>&1 + exit 1 + fi + test_case_id="$2" + shift + ;; + -d | --output-directory) + if [[ -z "$2" ]]; then + echo "You must specify an output directory when using the --output-directory option." 2>&1 + exit 1 + fi + output_directory="$2" + shift + ;; + -f | --filter) + if [[ -z "$2" ]]; then + echo "You must specify a filter when using the --filter option." 2>&1 + exit 1 + fi + filter="$2" + shift + ;; + -n | --no-extract-test-case-logs) no_extract_test_case_logs="1" ;; + -i | --upload-id) + if [[ -z "$2" ]]; then + echo "You must specify an upload ID when using the --upload-id option." 2>&1 + exit 1 + fi + upload_id="$2" + shift + ;; + -u | --upload-server-base-url) + if [[ -z "$2" ]]; then + echo "You must specify a base URL when using the --upload-server-base-url option." 2>&1 + exit 1 + fi + upload_server_base_url="$2" + shift + ;; + -o | --output-file) + if [[ -z "$2" ]]; then + echo "You must specify an output file when using the --output-file option." 2>&1 + exit 1 + fi + output_file="$2" + shift + ;; + -c | --cache-directory) + if [[ -z "$2" ]]; then + echo "You must specify a cache directory when using the --cache-directory option." 2>&1 + exit 1 + fi + cache_directory="$2" + shift + ;; + -a | --no-use-github-auth) no_use_github_auth="1" ;; + *) + echo "Unknown parameter passed: $1" 2>&1 + exit 1 + ;; + esac + shift +done + +if [[ -z $repo ]]; then + echo "You need to specify a repo (-r / --repo)." 2>&1 + exit 1 +fi + +if [[ -z $test_case_id && -z $upload_id ]]; then + echo "You need to specify the test case ID (-t / --test-case-id) or upload ID (-i / --upload-id)." 2>&1 + exit 1 +fi + +if [[ -n $test_case_id && -n $upload_id ]]; then + echo "You cannot specify both a test case ID and an upload ID." 2>&1 + exit 1 +fi + +if [[ -n $test_case_id && -n $upload_id ]]; then + echo "You cannot specify both a test case ID and an upload ID." 2>&1 + exit 1 +fi + +if [[ -z $test_case_id && -n $output_directory ]]; then + echo "You can only specify an output directory with a test case ID (-t / --test-case-id)." 2>&1 + exit 1 +fi + +if [[ -z $output_directory ]]; then + output_directory="xcodebuild-logs-test-case-${test_case_id}-$(date -Iseconds)" +fi + +if [[ -z $test_case_id && -n $filter ]]; then + echo "You can only specify a filter with a test case ID (-t / --test-case-id)." 2>&1 + exit 1 +fi + +if [[ -z $test_case_id && -n $no_extract_test_case_logs ]]; then + echo "You can only specify the --no-extract-test-case-logs option with a test case ID (-t / --test-case-id)." 2>&1 + exit 1 +fi + +if [[ -z $upload_server_base_url ]]; then + upload_server_base_url="https://test-observability.herokuapp.com" +fi + +if [[ -z $upload_id && -n $output_file ]]; then + echo "You can only specify an output file with an upload ID (-i / --upload-id)." 2>&1 + exit 1 +fi + +if [[ -z $output_file ]]; then + output_file=$(default_output_file_for_upload_id "${upload_id}") +fi + +if [[ -z $cache_directory ]]; then + cache_directory="${HOME}/Library/Caches/com.ably.testObservabilityLogs" +fi + +github_auth_curl_args=() +if [[ -z $no_use_github_auth ]]; then + # Get the GitHub access token from the user. We don’t allow them to specify it on the command line. + github_access_token="" + get_github_access_token + github_auth_curl_args+=(-H "Authorization: token ${github_access_token}") +fi + +# Run the appropriate function based on arguments. + +if [[ -n $test_case_id ]]; then + run_for_test_case +elif [[ -n $upload_id ]]; then + run_for_upload +fi diff --git a/Scripts/set-ci-length-and-parallelism.sh b/Scripts/set-ci-length-and-parallelism.sh new file mode 100755 index 000000000..efd705441 --- /dev/null +++ b/Scripts/set-ci-length-and-parallelism.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -e + +# Usage: +# ./set-ci-length-and-parallelism.sh --workflows --jobs-per-workflow + +# Check dependencies. +if ! which yq > /dev/null; then + echo "You need to install yq." 2>&1 + exit 1 +fi + +# Grab and validate command-line options. + +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash +while [[ "$#" -gt 0 ]]; do + case $1 in + --workflows) + if [[ -z "$2" ]]; then + echo "You must specify the number of workflows." 2>&1 + exit 1 + fi + num_workflows="$2" + shift + ;; + --jobs-per-workflow) + if [[ -z "$2" ]]; then + echo "You must specify the number of jobs per workflow." 2>&1 + exit 1 + fi + jobs_per_workflow="$2" + shift + ;; + *) + echo "Unknown parameter passed: $1" 2>&1 + exit 1 + ;; + esac + shift +done + +if [[ -z $num_workflows ]]; then + echo "You need to specify the number of workflows (--workflows)." 2>&1 + exit 1 +fi + +if [[ ! $num_workflows =~ ^-?[0-9]+$ ]]; then + echo "The number of workflows must be a number." 2>&1 + exit 1 +fi + +if [[ $num_workflows -lt 1 ]]; then + echo "The number of workflows must be 1 or more." 2>&1 + exit 1 +fi + +if [[ -z $jobs_per_workflow ]]; then + echo "You need to specify the number of jobs per workflow (--jobs-per-workflow)." 2>&1 + exit 1 +fi + +if [[ ! $jobs_per_workflow =~ ^-?[0-9]+$ ]]; then + echo "The number of jobs per workflow must be a number." 2>&1 + exit 1 +fi + +if [[ $jobs_per_workflow -lt 1 ]]; then + echo "The number of jobs per workflow must be 1 or more." 2>&1 + exit 1 +fi + +workflow_file_without_extension=".github/workflows/integration-test-iOS16_2" +workflow_file_extension=".yaml" + +workflow_file="${workflow_file_without_extension}${workflow_file_extension}" +workflow_name=$(yq .name $workflow_file) + +# First, we apply the number of jobs per workflow. + +yq -i '(.jobs.check | key) = "check-1"' $workflow_file +yq -i "(.jobs.check-1.steps[] | select(.with.path == \"xcresult-bundles.tar.gz\")).with.name = \"xcresult-bundles-1.tar.gz\"" $workflow_file + +for ((i=2; i <= $jobs_per_workflow; i += 1)) +do + yq -i ".jobs.check-${i} = .jobs.check-$(($i-1))" $workflow_file + yq -i ".jobs.check-${i}.needs = [\"check-$(($i-1))\"]" $workflow_file + yq -i "(.jobs.check-${i}.steps[] | select(.with.path == \"xcresult-bundles.tar.gz\")).with.name = \"xcresult-bundles-${i}.tar.gz\"" $workflow_file +done + +# Now, we duplicate the workflow file the requested number of times. + +mv $workflow_file "${workflow_file_without_extension}-1${workflow_file_extension}" + +for ((i=1; i <= $num_workflows; i += 1)) +do + new_workflow_file="${workflow_file_without_extension}-${i}${workflow_file_extension}" + + if [[ $i -gt 1 ]]; then + cp "${workflow_file_without_extension}-$((i-1))${workflow_file_extension}" $new_workflow_file + fi + + yq -i ".name = \"${workflow_name} (workflow ${i})\"" $new_workflow_file +done diff --git a/Test/Test Utilities/TestUtilities.swift b/Test/Test Utilities/TestUtilities.swift index 2349a2368..09bb0ddac 100644 --- a/Test/Test Utilities/TestUtilities.swift +++ b/Test/Test Utilities/TestUtilities.swift @@ -1709,7 +1709,6 @@ extension ARTRealtime { guard let transport = self.internal.transport as? TestProxyTransport else { fail("TestProxyTransport is not set"); return } - channel.internal.presence.startSync() transport.send(syncMessage) } } diff --git a/Test/Tests/RealtimeClientPresenceTests.swift b/Test/Tests/RealtimeClientPresenceTests.swift index d259446c2..bbca9b79a 100644 --- a/Test/Tests/RealtimeClientPresenceTests.swift +++ b/Test/Tests/RealtimeClientPresenceTests.swift @@ -105,30 +105,7 @@ class RealtimeClientPresenceTests: XCTestCase { // RTP1 - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 - func skipped__test__009__Presence__ProtocolMessage_bit_flag__when_no_members_are_present() throws { - let test = Test() - let options = try AblyTests.commonAppSetup(for: test) - options.autoConnect = false - options.testOptions.transportFactory = TestProxyTransportFactory() - let client = ARTRealtime(options: options) - client.connect() - defer { client.dispose(); client.close() } - let channel = client.channels.get(test.uniqueChannelName()) - channel.attach() - - expect(channel.state).toEventually(equal(ARTRealtimeChannelState.attached), timeout: testTimeout) - - let transport = client.internal.transport as! TestProxyTransport - let attached = transport.protocolMessagesReceived.filter { $0.action == .attached }[0] - - XCTAssertEqual(attached.flags & 0x1, 0) - XCTAssertFalse(attached.hasPresence) - XCTAssertFalse(channel.presence.syncComplete) - XCTAssertFalse(channel.internal.presence.syncComplete) - } - - func test__FLAKY__010__Presence__ProtocolMessage_bit_flag__when_members_are_present() throws { + func test__010__Presence__ProtocolMessage_bit_flag__when_members_are_present() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) @@ -168,9 +145,8 @@ class RealtimeClientPresenceTests: XCTestCase { // RTP18 - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 - // RTP18a, RTP18b - func skipped__test__011__Presence__realtime_system_reserves_the_right_to_initiate_a_sync_of_the_presence_members_at_any_point_once_a_channel_is_attached__should_do_a_new_sync_whenever_a_SYNC_ProtocolMessage_is_received_with_a_channel_attribute_and_a_new_sync_sequence_identifier_in_the_channelSerial_attribute() throws { + // RTP18b + func test__011__Presence__realtime_system_reserves_the_right_to_initiate_a_sync_of_the_presence_members_at_any_point_once_a_channel_is_attached__should_do_a_new_sync_whenever_a_SYNC_ProtocolMessage_is_received_with_a_channel_attribute_and_a_new_sync_sequence_identifier_in_the_channelSerial_attribute() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let client = AblyTests.newRealtime(options).client @@ -187,8 +163,9 @@ class RealtimeClientPresenceTests: XCTestCase { guard let transport = client.internal.transport as? TestProxyTransport else { fail("TestProxyTransport is not set"); return } - - XCTAssertFalse(channel.internal.presence.syncInProgress) + + // Before starting artificial SYNC process we should wait for the initial one completed: + expect(channel.internal.presence.syncInProgress).toEventually(beFalse(), timeout: testTimeout) expect(channel.internal.presence.members).to(beEmpty()) waitUntil(timeout: testTimeout) { done in @@ -246,9 +223,8 @@ class RealtimeClientPresenceTests: XCTestCase { } } - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 // RTP18c, RTP18b - func skipped__test__012__Presence__realtime_system_reserves_the_right_to_initiate_a_sync_of_the_presence_members_at_any_point_once_a_channel_is_attached__when_a_SYNC_is_sent_with_no_channelSerial_attribute_then_the_sync_data_is_entirely_contained_within_that_ProtocolMessage() throws { + func test__012__Presence__realtime_system_reserves_the_right_to_initiate_a_sync_of_the_presence_members_at_any_point_once_a_channel_is_attached__when_a_SYNC_is_sent_with_no_channelSerial_attribute_then_the_sync_data_is_entirely_contained_within_that_ProtocolMessage() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let client = AblyTests.newRealtime(options).client @@ -266,7 +242,7 @@ class RealtimeClientPresenceTests: XCTestCase { fail("TestProxyTransport is not set"); return } - XCTAssertFalse(channel.internal.presence.syncInProgress) + expect(channel.internal.presence.syncInProgress).toEventually(beFalse(), timeout: testTimeout) expect(channel.internal.presence.members).to(beEmpty()) waitUntil(timeout: testTimeout) { done in @@ -307,7 +283,7 @@ class RealtimeClientPresenceTests: XCTestCase { // RTP19 - func test__FLAKY__013__Presence__PresenceMap_has_existing_members_when_a_SYNC_is_started__should_ensure_that_members_no_longer_present_on_the_channel_are_removed_from_the_local_PresenceMap_once_the_sync_is_complete() throws { + func test__013__Presence__PresenceMap_has_existing_members_when_a_SYNC_is_started__should_ensure_that_members_no_longer_present_on_the_channel_are_removed_from_the_local_PresenceMap_once_the_sync_is_complete() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let channelName = test.uniqueChannelName() @@ -331,27 +307,27 @@ class RealtimeClientPresenceTests: XCTestCase { } XCTAssertEqual(channel.internal.presence.members.count, 2) - // Inject a local member - let internalMember = ARTPresenceMessage(clientId: NSUUID().uuidString, action: .enter, connectionId: "another", id: "another:0:0") + // Inject a internal member + let internalMember = ARTPresenceMessage(clientId: "internal-member", action: .enter, connectionId: channel.internal.connectionId, id: "\(channel.internal.connectionId):0:0") channel.internal.presence.processMember(internalMember) XCTAssertEqual(channel.internal.presence.members.count, 3) + XCTAssertEqual(channel.internal.presence.internalMembers.count, 1) XCTAssertEqual(channel.internal.presence.members.filter { memberKey, _ in memberKey.contains(internalMember.clientId!) }.count, 1) waitUntil(timeout: testTimeout) { done in channel.presence.get { members, error in XCTAssertNil(error) - guard let members = members, members.count == 3 else { - fail("Should at least have 3 members"); done(); return - } - XCTAssertEqual(members.filter { $0.clientId == internalMember.clientId }.count, 1) + XCTAssertEqual(members?.count, 3) + XCTAssertEqual(members?.filter { $0.clientId == internalMember.clientId }.count, 1) done() } } waitUntil(timeout: testTimeout) { done in channel.presence.subscribe(.leave) { leave in - XCTAssertEqual(leave.clientId, internalMember.clientId) - done() + if leave.clientId == internalMember.clientId { + done() + } } client.requestPresenceSyncForChannel(channel) } @@ -359,18 +335,15 @@ class RealtimeClientPresenceTests: XCTestCase { waitUntil(timeout: testTimeout) { done in channel.presence.get { members, error in XCTAssertNil(error) - guard let members = members, members.count == 2 else { - fail("Should at least have 2 members"); done(); return - } - expect(members.filter { $0.clientId == internalMember.clientId }).to(beEmpty()) + XCTAssertEqual(members?.count, 2) + expect(members?.filter { $0.clientId == internalMember.clientId }).to(beEmpty()) done() } } } - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 // RTP19a - func skipped__test__014__Presence__PresenceMap_has_existing_members_when_a_SYNC_is_started__should_emit_a_LEAVE_event_for_each_existing_member_if_the_PresenceMap_has_existing_members_when_an_ATTACHED_message_is_received_without_a_HAS_PRESENCE_flag() throws { + func test__014__Presence__PresenceMap_has_existing_members_when_a_SYNC_is_started__should_emit_a_LEAVE_event_for_each_existing_member_if_the_PresenceMap_has_existing_members_when_an_ATTACHED_message_is_received_without_a_HAS_PRESENCE_flag() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let client = AblyTests.newRealtime(options).client @@ -388,9 +361,10 @@ class RealtimeClientPresenceTests: XCTestCase { waitUntil(timeout: testTimeout) { done in let partialDone = AblyTests.splitDone(4, done: done) - transport.setListenerAfterProcessingIncomingMessage { protocolMessage in + transport.setListenerBeforeProcessingIncomingMessage { protocolMessage in if protocolMessage.action == .attached { - XCTAssertFalse(protocolMessage.hasPresence) + // Ensure incoming ATTACH message has no HAS_PRESENCE flag + protocolMessage.flags = 0 partialDone() } } @@ -410,6 +384,7 @@ class RealtimeClientPresenceTests: XCTestCase { partialDone() } } + transport.setListenerBeforeProcessingIncomingMessage(nil) waitUntil(timeout: testTimeout) { done in channel.presence.get { members, error in @@ -421,7 +396,7 @@ class RealtimeClientPresenceTests: XCTestCase { } // RTP4 - func test__FLAKY__002__Presence__should_receive_all_250_members() throws { + func test__002__Presence__should_receive_all_250_members() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) var clientSource: ARTRealtime! @@ -431,7 +406,7 @@ class RealtimeClientPresenceTests: XCTestCase { clientSource = AblyTests.addMembersSequentiallyToChannel(channelName, members: 250, options: options) let clientTarget = ARTRealtime(options: options) - defer { clientTarget.close() } + defer { clientTarget.dispose(); clientTarget.close() } let channel = clientTarget.channels.get(channelName) waitUntil(timeout: testTimeout) { done in @@ -552,7 +527,7 @@ class RealtimeClientPresenceTests: XCTestCase { } } - func test__FLAKY__019__Presence__Channel_state_change_side_effects__if_the_channel_enters_the_FAILED_state__should_clear_the_PresenceMap_including_local_members_and_does_not_emit_any_presence_events() throws { + func test__019__Presence__Channel_state_change_side_effects__if_the_channel_enters_the_FAILED_state__should_clear_the_PresenceMap_including_local_members_and_does_not_emit_any_presence_events() throws { let test = Test() let client = ARTRealtime(options: try AblyTests.commonAppSetup(for: test)) defer { client.dispose(); client.close() } @@ -649,9 +624,8 @@ class RealtimeClientPresenceTests: XCTestCase { } } - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 // RTP5b - func skipped__test__017__Presence__Channel_state_change_side_effects__if_a_channel_enters_the_ATTACHED_state_then_all_queued_presence_messages_will_be_sent_immediately_and_a_presence_SYNC_may_be_initiated() throws { + func test__017__Presence__Channel_state_change_side_effects__if_a_channel_enters_the_ATTACHED_state_then_all_queued_presence_messages_will_be_sent_immediately_and_a_presence_SYNC_may_be_initiated() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let client1 = AblyTests.newRealtime(options).client @@ -693,13 +667,7 @@ class RealtimeClientPresenceTests: XCTestCase { XCTAssertFalse(channel2.presence.syncComplete) XCTAssertEqual(channel2.internal.presence.members.count, 0) } - - guard let transport = client2.internal.transport as? TestProxyTransport else { - fail("Transport should be a test proxy"); return - } - - XCTAssertEqual(transport.protocolMessagesReceived.filter { $0.action == .sync }.count, 1) - + expect(channel2.presence.syncComplete).toEventually(beTrue(), timeout: testTimeout) XCTAssertEqual(channel2.internal.presence.members.count, 2) } @@ -874,19 +842,19 @@ class RealtimeClientPresenceTests: XCTestCase { // RTP8 // RTP8a - func skipped__test__024__Presence__enter__should_enter_the_current_client__optionally_with_the_data_provided() throws { + func test__024__Presence__enter__should_enter_the_current_client__optionally_with_the_data_provided() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) options.clientId = "john" let client1 = ARTRealtime(options: options) - defer { client1.close() } + defer { client1.dispose(); client1.close() } let channelName = test.uniqueChannelName() let channel1 = client1.channels.get(channelName) let client2 = ARTRealtime(options: options) - defer { client2.close() } + defer { client2.dispose(); client2.close() } let channel2 = client2.channels.get(channelName) waitUntil(timeout: testTimeout) { done in @@ -1027,21 +995,20 @@ class RealtimeClientPresenceTests: XCTestCase { // RTP8 - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 // RTP8b - func test__FLAKY__030__Presence__enter__optionally_a_callback_can_be_provided_that_is_called_for_success() throws { + func test__030__Presence__enter__optionally_a_callback_can_be_provided_that_is_called_for_success() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) options.clientId = "john" let client1 = ARTRealtime(options: options) - defer { client1.close() } + defer { client1.dispose(); client1.close() } let channelName = test.uniqueChannelName() let channel1 = client1.channels.get(channelName) let client2 = ARTRealtime(options: options) - defer { client2.close() } + defer { client2.dispose(); client2.close() } let channel2 = client2.channels.get(channelName) waitUntil(timeout: testTimeout) { done in @@ -1056,23 +1023,24 @@ class RealtimeClientPresenceTests: XCTestCase { } } - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 // RTP8b - func skipped__test__031__Presence__enter__optionally_a_callback_can_be_provided_that_is_called_for_failure() throws { + func test__031__Presence__enter__optionally_a_callback_can_be_provided_that_is_called_for_failure() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) options.clientId = "john" - - let client1 = ARTRealtime(options: options) - defer { client1.close() } - let channelName = test.uniqueChannelName() - let channel1 = client1.channels.get(channelName) - + let client1 = ARTRealtime(options: options) let client2 = ARTRealtime(options: options) - defer { client2.close() } + let channel1 = client1.channels.get(channelName) let channel2 = client2.channels.get(channelName) - + + defer { + client1.dispose(); client1.close() + client2.dispose(); client2.close() + expect(client1.connection.state).toEventually(equal(.closed), timeout: testTimeout) + expect(client2.connection.state).toEventually(equal(.closed), timeout: testTimeout) + } + waitUntil(timeout: testTimeout) { done in channel1.presence.subscribe(.enter) { _ in fail("shouldn't be called") @@ -1082,8 +1050,8 @@ class RealtimeClientPresenceTests: XCTestCase { XCTAssertTrue(error === protocolError.error) done() } - delay(0.1) { - channel2.internal.onError(protocolError) + channel2.internalAsync { _internal in + _internal.onError(protocolError) } } } @@ -1264,7 +1232,7 @@ class RealtimeClientPresenceTests: XCTestCase { } // RTP9a - func skipped__test__038__Presence__update__should_update_the_data_for_the_present_member_with_null() throws { + func test__038__Presence__update__should_update_the_data_for_the_present_member_with_null() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) options.clientId = "john" @@ -1291,9 +1259,8 @@ class RealtimeClientPresenceTests: XCTestCase { // RTP9 - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 // RTP9b - func skipped__test__039__Presence__update__should_enter_current_client_into_the_channel_if_the_client_was_not_already_entered() throws { + func test__039__Presence__update__should_enter_current_client_into_the_channel_if_the_client_was_not_already_entered() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) options.clientId = "john" @@ -1388,7 +1355,7 @@ class RealtimeClientPresenceTests: XCTestCase { // RTP10 // RTP10a - func skipped__test__043__Presence__leave__should_leave_the_current_client_from_the_channel_and_the_data_will_be_updated_with_the_value_provided() throws { + func test__043__Presence__leave__should_leave_the_current_client_from_the_channel_and_the_data_will_be_updated_with_the_value_provided() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) options.clientId = "john" @@ -1444,7 +1411,7 @@ class RealtimeClientPresenceTests: XCTestCase { } // RTP2 - func test__FLAKY__003__Presence__should_be_used_a_PresenceMap_to_maintain_a_list_of_members() throws { + func test__003__Presence__should_be_used_a_PresenceMap_to_maintain_a_list_of_members() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) var clientSecondary: ARTRealtime! @@ -1477,11 +1444,10 @@ class RealtimeClientPresenceTests: XCTestCase { } } - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 // RTP2 // RTP2a - func skipped__test__045__Presence__PresenceMap__all_incoming_presence_messages_must_be_compared_for_newness_with_the_matching_member_already_in_the_PresenceMap() throws { + func test__045__Presence__PresenceMap__all_incoming_presence_messages_must_be_compared_for_newness_with_the_matching_member_already_in_the_PresenceMap() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let client = ARTRealtime(options: options) @@ -1493,7 +1459,7 @@ class RealtimeClientPresenceTests: XCTestCase { let partialDone = AblyTests.splitDone(2, done: done) channel.presence.subscribe { presence in XCTAssertEqual(presence.clientId, "tester") - XCTAssertEqual(presence.action, .enter) + XCTAssertTrue(presence.action == .enter || presence.action == .present) channel.presence.unsubscribe() partialDone() } @@ -1509,9 +1475,9 @@ class RealtimeClientPresenceTests: XCTestCase { XCTAssertEqual(intialPresenceMessage.memberKey(), "\(client.connection.id!):tester") - var compareForNewnessMethodCalls = 0 + var compareForNewnessMethodCalled = false let hook = channel.internal.presence.testSuite_injectIntoMethod(after: #selector(ARTRealtimePresenceInternal.member(_:isNewerThan:))) { - compareForNewnessMethodCalls += 1 + compareForNewnessMethodCalled = true } waitUntil(timeout: testTimeout) { done in @@ -1528,7 +1494,7 @@ class RealtimeClientPresenceTests: XCTestCase { XCTAssertEqual(intialPresenceMessage.memberKey(), updatedPresenceMessage.memberKey()) expect(intialPresenceMessage.timestamp).to(beLessThan(updatedPresenceMessage.timestamp)) - XCTAssertEqual(compareForNewnessMethodCalls, 1) + XCTAssertTrue(compareForNewnessMethodCalled) hook.remove() } @@ -1692,16 +1658,14 @@ class RealtimeClientPresenceTests: XCTestCase { } // RTP2c - - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 - func skipped__test__054__Presence__PresenceMap__all_presence_messages_from_a_SYNC_must_also_be_compared_for_newness_in_the_same_way_as_they_would_from_a_PRESENCE__discard_members_where_messages_have_arrived_before_the_SYNC() throws { + func test__054__Presence__PresenceMap__all_presence_messages_from_a_SYNC_must_also_be_compared_for_newness_in_the_same_way_as_they_would_from_a_PRESENCE__discard_members_where_messages_have_arrived_before_the_SYNC() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) - let timeBeforeSync = NSDate() + let timeBeforeSync = Date() let channelName = test.uniqueChannelName() var clientMembers: ARTRealtime? defer { clientMembers?.dispose(); clientMembers?.close() } - clientMembers = AblyTests.addMembersSequentiallyToChannel(channelName, members: 120, options: options) + clientMembers = AblyTests.addMembersSequentiallyToChannel(channelName, members: 20, options: options) guard let membersConnectionId = clientMembers?.connection.id else { fail("Members client isn't connected"); return @@ -1714,22 +1678,20 @@ class RealtimeClientPresenceTests: XCTestCase { guard let transport = client.internal.transport as? TestProxyTransport else { fail("TestProxyTransport is not set"); return } - - channel.presence.subscribe(.leave) { leave in - XCTAssertEqual(leave.clientId, "user110") - fail("Should not fire Leave event for member `user110` because it's out of date") + + var leaveEvents = [ARTPresenceMessage]() + channel.presence.subscribe(.leave) { message in + leaveEvents.append(message) } waitUntil(timeout: testTimeout) { done in let partialDone = AblyTests.splitDone(3, done: done) transport.setBeforeIncomingMessageModifier { protocolMessage in if protocolMessage.action == .sync { - let injectLeave = ARTPresenceMessage() - injectLeave.action = .leave - injectLeave.connectionId = membersConnectionId - injectLeave.clientId = "user110" - injectLeave.timestamp = timeBeforeSync as Date - protocolMessage.presence?.append(injectLeave) + protocolMessage.presence?.append(contentsOf: [ + ARTPresenceMessage(clientId: "user10", action: .leave, connectionId: membersConnectionId, id: "synthesized:9:0", timestamp: timeBeforeSync), + ARTPresenceMessage(clientId: "user12", action: .leave, connectionId: membersConnectionId, id: "synthesized:11:0", timestamp: Date() + 1) + ]) transport.setBeforeIncomingMessageModifier(nil) partialDone() } @@ -1737,61 +1699,9 @@ class RealtimeClientPresenceTests: XCTestCase { } channel.internal.presence.testSuite_injectIntoMethod(after: #selector(ARTRealtimePresenceInternal.endSync)) { XCTAssertFalse(channel.internal.presence.syncInProgress) - XCTAssertEqual(channel.internal.presence.members.count, 120) - XCTAssertEqual(channel.internal.presence.members.filter { _, presence in presence.clientId == "user110" && presence.action == .present }.count, 1) - partialDone() - } - channel.attach { error in - XCTAssertNil(error) - partialDone() - } - } - } - - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 - func skipped__test__055__Presence__PresenceMap__all_presence_messages_from_a_SYNC_must_also_be_compared_for_newness_in_the_same_way_as_they_would_from_a_PRESENCE__accept_members_where_message_have_arrived_after_the_SYNC() throws { - let test = Test() - let options = try AblyTests.commonAppSetup(for: test) - let channelName = test.uniqueChannelName() - var clientMembers: ARTRealtime? - defer { clientMembers?.dispose(); clientMembers?.close() } - clientMembers = AblyTests.addMembersSequentiallyToChannel(channelName, members: 120, options: options) - - guard let membersConnectionId = clientMembers?.connection.id else { - fail("Members client isn't connected"); return - } - - let client = AblyTests.newRealtime(options).client - defer { client.dispose(); client.close() } - let channel = client.channels.get(channelName) - - guard let transport = client.internal.transport as? TestProxyTransport else { - fail("TestProxyTransport is not set"); return - } - - waitUntil(timeout: testTimeout.multiplied(by: 2)) { done in - let partialDone = AblyTests.splitDone(4, done: done) - channel.presence.subscribe(.leave) { leave in - XCTAssertEqual(leave.clientId, "user110") - partialDone() - } - transport.setBeforeIncomingMessageModifier { protocolMessage in - if protocolMessage.action == .sync { - let injectLeave = ARTPresenceMessage() - injectLeave.action = .leave - injectLeave.connectionId = membersConnectionId - injectLeave.clientId = "user110" - injectLeave.timestamp = Date() + 1 - protocolMessage.presence?.append(injectLeave) - transport.setBeforeIncomingMessageModifier(nil) - partialDone() - } - return protocolMessage - } - channel.internal.presence.testSuite_injectIntoMethod(after: #selector(ARTRealtimePresenceInternal.endSync)) { - XCTAssertFalse(channel.internal.presence.syncInProgress) - XCTAssertEqual(channel.internal.presence.members.count, 119) - expect(channel.internal.presence.members.filter { _, presence in presence.clientId == "user110" }).to(beEmpty()) + XCTAssertEqual(channel.internal.presence.members.count, 19) + XCTAssertEqual(channel.internal.presence.members.filter { _, presence in presence.clientId == "user10" && presence.action == .present }.count, 1) // LEAVE for user10 is ignored, because it's timestamped before SYNC + XCTAssertEqual(channel.internal.presence.members.filter { _, presence in presence.clientId == "user12" && presence.action == .present }.count, 0) // LEAVE for user12 is not ignored, because it's timestamped after SYNC partialDone() } channel.attach { error in @@ -1799,10 +1709,12 @@ class RealtimeClientPresenceTests: XCTestCase { partialDone() } } + XCTAssertTrue(leaveEvents.contains(where: { $0.clientId == "user12" })) + XCTAssertFalse(leaveEvents.contains(where: { $0.clientId == "user10" })) } // RTP2d - func skipped__test__046__Presence__PresenceMap__if_action_of_ENTER_arrives__it_should_be_added_to_the_presence_map_with_the_action_set_to_PRESENT() throws { + func test__047__Presence__PresenceMap__if_action_of_UPDATE_arrives__it_should_be_added_to_the_presence_map_with_the_action_set_to_PRESENT() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let client = ARTRealtime(options: options) @@ -1812,41 +1724,15 @@ class RealtimeClientPresenceTests: XCTestCase { waitUntil(timeout: testTimeout) { done in let partialDone = AblyTests.splitDone(2, done: done) - channel.presence.subscribe(.enter) { _ in - partialDone() - } - channel.presence.enterClient("tester", data: nil) { error in - XCTAssertNil(error) - partialDone() - } - } - - XCTAssertEqual(channel.internal.presence.members.filter { _, presence in presence.action == .present }.count, 1) - expect(channel.internal.presence.members.filter { _, presence in presence.action == .enter }).to(beEmpty()) - } - - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 - // RTP2d - func test__FLAKY__047__Presence__PresenceMap__if_action_of_UPDATE_arrives__it_should_be_added_to_the_presence_map_with_the_action_set_to_PRESENT() throws { - let test = Test() - let options = try AblyTests.commonAppSetup(for: test) - let client = ARTRealtime(options: options) - defer { client.dispose(); client.close() } - let channelName = test.uniqueChannelName() - let channel = client.channels.get(channelName) - - waitUntil(timeout: testTimeout) { done in - let partialDone = AblyTests.splitDone(3, done: done) channel.presence.subscribe(.update) { _ in partialDone() } channel.presence.enterClient("tester", data: nil) { error in XCTAssertNil(error) - partialDone() - } - channel.presence.updateClient("tester", data: nil) { error in - XCTAssertNil(error) - partialDone() + channel.presence.updateClient("tester", data: nil) { error in + XCTAssertNil(error) + partialDone() + } } } @@ -1884,7 +1770,7 @@ class RealtimeClientPresenceTests: XCTestCase { } // RTP2e - func test__FLAKY__049__Presence__PresenceMap__if_a_SYNC_is_not_in_progress__then_when_a_presence_message_with_an_action_of_LEAVE_arrives__that_memberKey_should_be_deleted_from_the_presence_map__if_present() throws { + func test__049__Presence__PresenceMap__if_a_SYNC_is_not_in_progress__then_when_a_presence_message_with_an_action_of_LEAVE_arrives__that_memberKey_should_be_deleted_from_the_presence_map__if_present() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) @@ -1953,8 +1839,11 @@ class RealtimeClientPresenceTests: XCTestCase { channel.presence.subscribe(.leave) { leave in XCTAssertEqual(leave.clientId, "user11") let absentMember = channel.internal.presence.members.first { _, m in m.clientId == "user11" }.map { $0.value } - XCTAssertTrue(channel.internal.presence.syncInProgress) - XCTAssertEqual(absentMember?.action, .absent) + if channel.internal.presence.syncInProgress { + XCTAssertEqual(absentMember?.action, .absent) + } else { + XCTAssertEqual(absentMember?.action, .leave) + } partialDone() } @@ -1970,7 +1859,9 @@ class RealtimeClientPresenceTests: XCTestCase { leaveMessage.presence = [ ARTPresenceMessage(clientId: "user11", action: .leave, connectionId: "another", id: "another:123:0", timestamp: Date()), ] - transport.receive(leaveMessage) + channel.internalAsync { _ in + transport.receive(leaveMessage) + } partialDone() } } @@ -1984,8 +1875,8 @@ class RealtimeClientPresenceTests: XCTestCase { XCTAssertEqual(channel.internal.presence.members.count, 20) } - // RTP2g - func test__FLAKY__051__Presence__PresenceMap__any_incoming_presence_message_that_passes_the_newness_check_should_be_emitted_on_the_Presence_object__with_an_event_name_set_to_its_original_action() throws { + // RTP2d (ENTER), RTP2g + func test__051__Presence__PresenceMap__any_incoming_presence_message_that_passes_the_newness_check_should_be_emitted_on_the_Presence_object__with_an_event_name_set_to_its_original_action() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let client = ARTRealtime(options: options) @@ -1995,11 +1886,11 @@ class RealtimeClientPresenceTests: XCTestCase { waitUntil(timeout: testTimeout) { done in let partialDone = AblyTests.splitDone(2, done: done) - channel.presence.enterClient("tester", data: nil) { error in - XCTAssertNil(error) + channel.presence.subscribe(.enter) { _ in partialDone() } - channel.presence.subscribe(.enter) { _ in + channel.presence.enterClient("tester", data: nil) { error in + XCTAssertNil(error) partialDone() } } @@ -2588,10 +2479,10 @@ class RealtimeClientPresenceTests: XCTestCase { } } - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 // RTP17 - func skipped__test__080__Presence__private_and_internal_PresenceMap_containing_only_members_that_match_the_current_connectionId__any_ENTER__PRESENT__UPDATE_or_LEAVE_event_that_matches_the_current_connectionId_should_be_applied_to_this_object() throws { + // RTP17b + func test__080__Presence__private_and_internal_PresenceMap_containing_only_members_that_match_the_current_connectionId__any_ENTER__PRESENT__UPDATE_or_LEAVE_event_that_matches_the_current_connectionId_should_be_applied_to_this_object() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let channelName = test.uniqueChannelName() @@ -2613,7 +2504,7 @@ class RealtimeClientPresenceTests: XCTestCase { guard let currentConnectionId = clientA.connection.id else { fail("ClientA should be connected"); partialDone(); return } - XCTAssertEqual(presence.action, ARTPresenceAction.enter) + XCTAssertTrue(presence.action == ARTPresenceAction.enter || presence.action == ARTPresenceAction.present) XCTAssertEqual(presence.connectionId, currentConnectionId) XCTAssertEqual(channelA.internal.presence.members.count, 1) XCTAssertEqual(channelA.internal.presence.internalMembers.count, 1) @@ -2624,7 +2515,7 @@ class RealtimeClientPresenceTests: XCTestCase { guard let currentConnectionId = clientB.connection.id else { fail("ClientB should be connected"); partialDone(); return } - expect(presence.action).to(equal(ARTPresenceAction.enter) || equal(ARTPresenceAction.present)) + XCTAssertTrue(presence.action == ARTPresenceAction.enter || presence.action == ARTPresenceAction.present) XCTAssertNotEqual(presence.connectionId, currentConnectionId) XCTAssertEqual(channelB.internal.presence.members.count, 1) XCTAssertEqual(channelB.internal.presence.internalMembers.count, 0) @@ -2701,7 +2592,12 @@ class RealtimeClientPresenceTests: XCTestCase { XCTAssertEqual(presence.action, ARTPresenceAction.leave) XCTAssertEqual(presence.data as? String, "bye") XCTAssertNotEqual(presence.connectionId, currentConnectionId) - XCTAssertEqual(channelA.internal.presence.members.count, 1) + if channelA.internal.presence.syncInProgress { + XCTAssertEqual(channelA.internal.presence.members.filter({ $0.value.action != .present }).count, 1) + XCTAssertEqual(channelA.internal.presence.members.filter({ $0.value.action != .absent }).count, 1) + } else { + XCTAssertEqual(channelA.internal.presence.members.count, 1) + } XCTAssertEqual(channelA.internal.presence.internalMembers.count, 1) channelA.presence.unsubscribe() partialDone() @@ -2713,8 +2609,13 @@ class RealtimeClientPresenceTests: XCTestCase { XCTAssertEqual(presence.action, ARTPresenceAction.leave) XCTAssertEqual(presence.data as? String, "bye") XCTAssertEqual(presence.connectionId, currentConnectionId) - XCTAssertEqual(channelB.internal.presence.members.count, 1) - XCTAssertEqual(channelB.internal.presence.internalMembers.count, 0) + if channelB.internal.presence.syncInProgress { + XCTAssertEqual(channelB.internal.presence.members.filter({ $0.value.action != .present }).count, 1) + XCTAssertEqual(channelB.internal.presence.members.filter({ $0.value.action != .absent }).count, 1) + } else { + XCTAssertEqual(channelB.internal.presence.members.count, 1) + } + XCTAssertEqual(channelB.internal.presence.internalMembers.count, 0) // was removed bc not a 'synthesized leave' (RTP17b) channelB.presence.unsubscribe() partialDone() } @@ -2723,7 +2624,7 @@ class RealtimeClientPresenceTests: XCTestCase { } // RTP17a - func test__FLAKY__081__Presence__private_and_internal_PresenceMap_containing_only_members_that_match_the_current_connectionId__all_members_belonging_to_the_current_connection_are_published_as_a_PresenceMessage_on_the_Channel_by_the_server_irrespective_of_whether_the_client_has_permission_to_subscribe_or_the_Channel_is_configured_to_publish_presence_events() throws { + func test__081__Presence__private_and_internal_PresenceMap_containing_only_members_that_match_the_current_connectionId__all_members_belonging_to_the_current_connection_are_published_as_a_PresenceMessage_on_the_Channel_by_the_server_irrespective_of_whether_the_client_has_permission_to_subscribe_or_the_Channel_is_configured_to_publish_presence_events() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) let channelName = test.uniqueChannelName() @@ -2759,105 +2660,6 @@ class RealtimeClientPresenceTests: XCTestCase { } } } - - // RTP17b - - func skipped__test__082__Presence__private_and_internal_PresenceMap_containing_only_members_that_match_the_current_connectionId__events_applied_to_presence_map__should_be_applied_to_ENTER__PRESENT_or_UPDATE_events_with_a_connectionId_that_matches_the_current_client_s_connectionId() throws { - let test = Test() - let options = try AblyTests.commonAppSetup(for: test) - let client = AblyTests.newRealtime(options).client - defer { client.dispose(); client.close() } - - let channel = client.channels.get(test.uniqueChannelName()) - waitUntil(timeout: testTimeout) { done in - let partialDone = AblyTests.splitDone(2, done: done) - channel.presence.subscribe(.enter) { presence in - XCTAssertEqual(presence.clientId, "one") - channel.presence.unsubscribe() - partialDone() - } - channel.presence.enterClient("one", data: nil) { error in - XCTAssertNil(error) - partialDone() - } - } - - guard let connectionId = client.connection.id else { - fail("connectionId is empty"); return - } - - channel.internalSync { _internal in - XCTAssertEqual(_internal.presence.internalMembers.count, 1) - } - - let additionalMember = ARTPresenceMessage( - clientId: "two", - action: .enter, - connectionId: connectionId, - id: connectionId + ":0:0" - ) - - // Inject an additional member into the myMember set, then force a suspended state - client.simulateSuspended(beforeSuspension: { done in - channel.internal.presence.internalMembers[additionalMember.clientId!] = additionalMember - done() - }) - expect(client.connection.state).toEventually(equal(.suspended), timeout: testTimeout) - - XCTAssertEqual(channel.internal.presence.internalMembers.count, 2) - - channel.internalSync { _internal in - XCTAssertEqual(_internal.presence.internalMembers.count, 2) - } - - waitUntil(timeout: testTimeout) { done in - let partialDone = AblyTests.splitDone(3, done: done) - - channel.once(.attached) { stateChange in - XCTAssertNil(stateChange.reason) - partialDone() - } - - // Await Sync - channel.internal.presence.onceSyncEnds { _ in - // Should remove the "two" member that was added manually because the connectionId - // doesn't match and it's not synthesized, it will be re-entered. - XCTAssertEqual(channel.internal.presence.internalMembers.count, 1) - - partialDone() - } - - channel.presence.subscribe(.enter) { presence in - XCTAssertEqual(presence.clientId, "two") - channel.presence.unsubscribe() - partialDone() - } - - // Reconnect - client.connect() - } - - // Wait for server - waitUntil(timeout: testTimeout) { done in - delay(1, closure: done) - } - - client.requestPresenceSyncForChannel(channel) - - XCTAssertFalse(channel.presence.syncComplete) - waitUntil(timeout: testTimeout) { done in - channel.presence.get { presences, error in - XCTAssertNil(error) - guard let presences = presences else { - fail("Presences is nil"); done(); return - } - XCTAssertTrue(channel.presence.syncComplete) - XCTAssertEqual(presences.count, 2) - expect(presences.map { $0.clientId }).to(contain(["one", "two"])) - done() - } - } - } // RTP17i, RTP17g func test__200__Presence__PresenceMap_should_perform_re_entry_whenever_a_channel_moves_into_the_attached_state_and_presence_message_consists_of_enter_action_with_client_id_and_data() throws { @@ -2941,59 +2743,6 @@ class RealtimeClientPresenceTests: XCTestCase { expect(client1PresenceMessage.data as? String).to(equal(firstClientData)) expect(client2PresenceMessage.data as? String).to(equal(secondClientData)) } - - func skipped__test__083__Presence__private_and_internal_PresenceMap_containing_only_members_that_match_the_current_connectionId__events_applied_to_presence_map__should_be_applied_to_any_LEAVE_event_with_a_connectionId_that_matches_the_current_client_s_connectionId_and_is_not_a_synthesized() throws { - let test = Test() - let options = try AblyTests.commonAppSetup(for: test) - let client = ARTRealtime(options: options) - client.internal.shouldImmediatelyReconnect = false - defer { client.dispose(); client.close() } - - let channel = client.channels.get(test.uniqueChannelName()) - waitUntil(timeout: testTimeout) { done in - let partialDone = AblyTests.splitDone(2, done: done) - channel.presence.subscribe(.enter) { presence in - XCTAssertEqual(presence.clientId, "one") - channel.presence.unsubscribe() - partialDone() - } - channel.presence.enterClient("one", data: nil) { error in - XCTAssertNil(error) - partialDone() - } - } - - waitUntil(timeout: .seconds(20)) { done in - channel.internal.presence.onceSyncEnds { _ in - // Synthesized leave - expect(channel.internal.presence.internalMembers).to(beEmpty()) - done() - } - client.internal.onDisconnected() - } - - waitUntil(timeout: testTimeout) { done in - channel.presence.subscribe(.enter) { presence in - // Re-entering... - XCTAssertEqual(presence.clientId, "one") - channel.presence.unsubscribe() - done() - } - } - - waitUntil(timeout: testTimeout) { done in - channel.presence.get { presences, error in - XCTAssertNil(error) - guard let presences = presences else { - fail("Presences is nil"); done(); return - } - XCTAssertTrue(channel.internal.presence.syncComplete) - XCTAssertEqual(presences.count, 1) - expect(presences.map { $0.clientId }).to(contain(["one"])) - done() - } - } - } // RTP15d func test__004__Presence__callback_can_be_provided_that_will_be_called_upon_success() throws { @@ -3359,9 +3108,8 @@ class RealtimeClientPresenceTests: XCTestCase { XCTAssertTrue(ARTRealtimePresenceQuery().waitForSync) } - // FIXME: Fix flaky presence tests and re-enable. See https://ably-real-time.slack.com/archives/C030C5YLY/p1623172436085700 // RTP11a - func test__FLAKY__100__Presence__get__should_return_a_list_of_current_members_on_the_channel() throws { + func test__100__Presence__get__should_return_a_list_of_current_members_on_the_channel() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) @@ -3592,40 +3340,33 @@ class RealtimeClientPresenceTests: XCTestCase { // RTP11c // RTP11c1 - func skipped__test__110__Presence__get__Query__set_of_params___waitForSync_is_true__should_wait_until_SYNC_is_complete_before_returning_a_list_of_members() throws { + func test__110__Presence__get__Query__set_of_params___waitForSync_is_true__should_wait_until_SYNC_is_complete_before_returning_a_list_of_members() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) - var clientSecondary: ARTRealtime! - defer { clientSecondary.dispose(); clientSecondary.close() } - let channelName = test.uniqueChannelName() - clientSecondary = AblyTests.addMembersSequentiallyToChannel(channelName, members: 150, options: options) - + let clientSecondary = AblyTests.addMembersSequentiallyToChannel(channelName, members: 150, options: options) let client = AblyTests.newRealtime(options).client - defer { client.dispose(); client.close() } + defer { + clientSecondary.dispose(); clientSecondary.close() + client.dispose(); client.close() + expect(clientSecondary.connection.state).toEventually(equal(.closed), timeout: testTimeout) + expect(client.connection.state).toEventually(equal(.closed), timeout: testTimeout) + } let channel = client.channels.get(channelName) - - let query = ARTRealtimePresenceQuery() - XCTAssertTrue(query.waitForSync) - + expect(channel.internal.presence.syncInProgress).toEventually(beTrue(), timeout: testTimeout) + waitUntil(timeout: testTimeout) { done in - channel.attach { error in + let query = ARTRealtimePresenceQuery() + XCTAssertTrue(query.waitForSync) + XCTAssertEqual(channel.internal.presence.syncInProgress, true) + channel.presence.get(query) { members, error in XCTAssertNil(error) - let transport = client.internal.transport as! TestProxyTransport - transport.setListenerBeforeProcessingIncomingMessage { protocolMessage in - if protocolMessage.action == .sync { - XCTAssertEqual(protocolMessage.presence!.count, 100) - channel.presence.get(query) { members, error in - XCTAssertNil(error) - if let members { - XCTAssertEqual(members.count, 150) - } else { - XCTFail("Expected members to be non-nil") - } - done() - } - transport.setListenerBeforeProcessingIncomingMessage(nil) - } + if let members { + XCTAssertEqual(members.count, 150) + XCTAssertEqual(channel.internal.presence.syncInProgress, false) + done() + } else { + XCTFail("Expected members to be non-nil") } } } @@ -3940,7 +3681,7 @@ class RealtimeClientPresenceTests: XCTestCase { } // RTP13 - func skipped__test__008__Presence__Presence_syncComplete_returns_true_if_the_initial_SYNC_operation_has_completed() throws { + func test__008__Presence__Presence_syncComplete_returns_true_if_the_initial_SYNC_operation_has_completed() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) @@ -3969,7 +3710,7 @@ class RealtimeClientPresenceTests: XCTestCase { XCTAssertFalse(channel.presence.internal.syncComplete_nosync()) } } - + transport.setListenerBeforeProcessingIncomingMessage(nil) expect(channel.presence.syncComplete).toEventually(beTrue(), timeout: testTimeout) XCTAssertEqual(transport.protocolMessagesReceived.filter { $0.action == .sync }.count, 3) } @@ -3977,7 +3718,7 @@ class RealtimeClientPresenceTests: XCTestCase { // RTP14 // RTP14a, RTP14b, RTP14c, RTP14d - func skipped__test__116__Presence__enterClient__enters_into_presence_on_a_channel_on_behalf_of_another_clientId() throws { + func test__116__Presence__enterClient__enters_into_presence_on_a_channel_on_behalf_of_another_clientId() throws { let test = Test() let client = ARTRealtime(options: try AblyTests.commonAppSetup(for: test)) defer { client.dispose(); client.close() } @@ -3986,15 +3727,15 @@ class RealtimeClientPresenceTests: XCTestCase { let expectedData = ["test": 1] - var encodeNumberOfCalls = 0 + var encodeWasCalled = false let hookEncode = channel.internal.dataEncoder.testSuite_injectIntoMethod(after: #selector(ARTDataEncoder.encode(_:))) { - encodeNumberOfCalls += 1 + encodeWasCalled = true } defer { hookEncode.remove() } - var decodeNumberOfCalls = 0 + var decodeWasCalled = false let hookDecode = channel.internal.dataEncoder.testSuite_injectIntoMethod(after: #selector(ARTDataEncoder.decode(_:encoding:))) { - decodeNumberOfCalls += 1 + decodeWasCalled = true } defer { hookDecode.remove() } @@ -4022,8 +3763,8 @@ class RealtimeClientPresenceTests: XCTestCase { } } - XCTAssertEqual(encodeNumberOfCalls, 1) - XCTAssertEqual(decodeNumberOfCalls, 1) + XCTAssertTrue(encodeWasCalled) + XCTAssertTrue(decodeWasCalled) } // RTP14d diff --git a/fastlane/Scanfile b/fastlane/Scanfile index fea35296b..da002c455 100644 --- a/fastlane/Scanfile +++ b/fastlane/Scanfile @@ -1,8 +1,10 @@ open_report false -clean true +clean false skip_slack true ensure_devices_found true output_types "junit" # I'm being explicit about this because I want to make sure it's being used, to make sure that trainer is used to generate the JUnit report xcodebuild_formatter "xcbeautify" result_bundle true +# Just for printing inside these loop jobs +buildlog_path "xcodebuild_output"