diff --git a/IntegrationTests/plugin_echo.sh b/IntegrationTests/plugin_echo.sh new file mode 100644 index 00000000..6eed284f --- /dev/null +++ b/IntegrationTests/plugin_echo.sh @@ -0,0 +1,58 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +function plugin_echo_test_suite_begin() { + echo "Running test suite '$1'" +} + +function plugin_echo_test_suite_end() { + true +} + +# test_name +function plugin_echo_test_begin() { + echo -n "Running test '$1'... " +} + +function plugin_echo_test_skip() { + echo "Skipping test '$1'" +} + +function plugin_echo_test_ok() { + echo "OK (${1}s)" +} + +function plugin_echo_test_fail() { + echo "FAILURE ($1)" + echo "--- OUTPUT BEGIN ---" + cat "$2" + echo "--- OUTPUT END ---" +} + +function plugin_echo_test_end() { + true +} + +function plugin_echo_summary_ok() { + echo "OK (ran $1 tests successfully)" +} + +function plugin_echo_summary_fail() { + echo "FAILURE (oks: $1, failures: $2)" +} + +function plugin_echo_init() { + true +} diff --git a/IntegrationTests/plugin_junit_xml.sh b/IntegrationTests/plugin_junit_xml.sh new file mode 100644 index 00000000..d88a51c7 --- /dev/null +++ b/IntegrationTests/plugin_junit_xml.sh @@ -0,0 +1,119 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +junit_testsuite_time=0 + +function junit_output_write() { + extra_flags="" + if [[ "$1" == "-n" ]]; then + extra_flags="-n" + shift + fi + test -n "$junit_xml_output" + echo $extra_flags "$*" >> "$junit_xml_output" +} + +function junit_output_cat() { + cat "$@" >> "$junit_xml_output" +} + +# search, replace +function junit_output_replace() { + test -n "$junit_xml_output" + case "$(uname -s)" in + Linux) + sed -i "s/$1/$2/g" "$junit_xml_output" + ;; + *) + sed -i "" "s/$1/$2/g" "$junit_xml_output" + ;; + esac +} + +function plugin_junit_xml_test_suite_begin() { + junit_testsuite_time=0 + junit_output_write "" +} + +function plugin_junit_xml_test_suite_end() { + junit_repl_success_and_fail "$1" "$2" + junit_output_write "" +} + +# test_name +function plugin_junit_xml_test_begin() { + junit_output_write -n " " + junit_testsuite_time=$((junit_testsuite_time + time_ms)) +} + +function plugin_junit_xml_test_fail() { + time_ms=$1 + junit_output_write " time='$time_ms'>" + junit_output_write " " + junit_output_write " " + junit_output_write ' ' + junit_output_write " " + junit_output_write " " +} + +function plugin_junit_xml_test_end() { + junit_output_write " " +} + +function junit_repl_success_and_fail() { + junit_output_replace XXX-TESTS-XXX "$(($1 + $2))" + junit_output_replace XXX-FAILURES-XXX "$2" + junit_output_replace XXX-TIME-XXX "$junit_testsuite_time" +} + +function plugin_junit_xml_summary_ok() { + junit_output_write "" +} + +function plugin_junit_xml_summary_fail() { + junit_output_write "" +} + +function plugin_junit_xml_init() { + junit_xml_output="" + for f in "$@"; do + if [[ "$junit_xml_output" = "PLACEHOLDER" ]]; then + junit_xml_output="$f" + fi + if [[ "$f" == "--junit-xml" && -z "$junit_xml_output" ]]; then + junit_xml_output="PLACEHOLDER" + fi + done + + if [[ -z "$junit_xml_output" || "$junit_xml_output" = "PLACEHOLDER" ]]; then + echo >&2 "ERROR: you need to specify the output after the --junit-xml argument" + false + fi + echo "" > "$junit_xml_output" +} diff --git a/IntegrationTests/run-single-test.sh b/IntegrationTests/run-single-test.sh new file mode 100755 index 00000000..36501029 --- /dev/null +++ b/IntegrationTests/run-single-test.sh @@ -0,0 +1,33 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +( +# this sub-shell is where the actual test is run +set -eu +set -x +set -o pipefail + +test="$1" +tmp="$2" +root="$3" +g_show_info="$4" +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +source "$here/test_functions.sh" +source "$test" +wait +) +exit_code=$? +exit $exit_code diff --git a/IntegrationTests/run-tests.sh b/IntegrationTests/run-tests.sh new file mode 100755 index 00000000..9250b83e --- /dev/null +++ b/IntegrationTests/run-tests.sh @@ -0,0 +1,159 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set -eu + +shopt -s nullglob + +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +tmp=$(mktemp -d /tmp/.swift-nio-http1-server-sh-tests_XXXXXX) + +# start_time +function time_diff_to_now() { + echo "$(( $(date +%s) - $1 ))" +} + +function plugins_do() { + local method + method="$1" + shift + for plugin in $plugins; do + cd "$orig_cwd" + "plugin_${plugin}_${method}" "$@" + cd - > /dev/null + done +} + +source "$here/plugin_echo.sh" +source "$here/plugin_junit_xml.sh" + +plugins="echo" +plugin_opts_ind=0 +if [[ "${1-default}" == "--junit-xml" ]]; then + plugins="echo junit_xml" + plugin_opts_ind=2 +fi + +function usage() { + echo >&2 "Usage: $0 [OPTIONS]" + echo >&2 + echo >&2 "OPTIONS:" + echo >&2 " -f FILTER: Only run tests matching FILTER (regex)" +} + +orig_cwd=$(pwd) +cd "$here" + +plugins_do init "$@" +shift $plugin_opts_ind + +filter="." +verbose=false +show_info=false +debug=false +while getopts "f:vid" opt; do + case $opt in + f) + filter="$OPTARG" + ;; + v) + verbose=true + ;; + i) + show_info=true + ;; + d) + debug=true + ;; + \?) + usage + exit 1 + ;; + esac +done + +function run_test() { + if $verbose; then + "$@" 2>&1 | tee -a "$out" + # we need to return the return value of the first command + return ${PIPESTATUS[0]} + else + "$@" >> "$out" 2>&1 + fi +} + +exec 3>&1 4>&2 # copy stdout/err to fd 3/4 to we can output control messages +cnt_ok=0 +cnt_fail=0 +for f in tests_*; do + suite_ok=0 + suite_fail=0 + plugins_do test_suite_begin "$f" + start_suite=$(date +%s) + cd "$f" + for t in test_*.sh; do + if [[ ! "$f/$t" =~ $filter ]]; then + plugins_do test_skip "$t" + continue + fi + out=$(mktemp "$tmp/test.out_XXXXXX") + test_tmp=$(mktemp -d "$tmp/test.tmp_XXXXXX") + plugins_do test_begin "$t" "$f" + start=$(date +%s) + if run_test "$here/run-single-test.sh" "$here/$f/$t" "$test_tmp" "$here/.." "$show_info"; then + plugins_do test_ok "$(time_diff_to_now $start)" + suite_ok=$((suite_ok+1)) + if $verbose; then + cat "$out" + fi + else + plugins_do test_fail "$(time_diff_to_now $start)" "$out" + suite_fail=$((suite_fail+1)) + fi + if ! $debug; then + rm "$out" + rm -rf "$test_tmp" + fi + plugins_do test_end + done + cnt_ok=$((cnt_ok + suite_ok)) + cnt_fail=$((cnt_fail + suite_fail)) + cd .. + plugins_do test_suite_end "$(time_diff_to_now $start_suite)" "$suite_ok" "$suite_fail" +done + +if ! $debug; then + rm -rf "$tmp" +else + echo >&2 "debug mode, not deleting '$tmp'" +fi + + +# report +if [[ $cnt_fail > 0 ]]; then + # kill leftovers (the whole process group) + trap '' TERM + kill 0 + + plugins_do summary_fail "$cnt_ok" "$cnt_fail" +else + plugins_do summary_ok "$cnt_ok" "$cnt_fail" +fi + +if [[ $cnt_fail > 0 ]]; then + exit 1 +else + exit 0 +fi diff --git a/IntegrationTests/test_functions.sh b/IntegrationTests/test_functions.sh new file mode 100644 index 00000000..2eafeb71 --- /dev/null +++ b/IntegrationTests/test_functions.sh @@ -0,0 +1,78 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## +function fail() { + echo >&2 "FAILURE: $*" + false +} + +function assert_equal() { + if [[ "$1" != "$2" ]]; then + fail "expected '$1', got '$2' ${3-}" + fi +} + +function assert_equal_files() { + if ! cmp -s "$1" "$2"; then + diff -u "$1" "$2" || true + echo + echo "--- SNIP ($1, size=$(wc "$1"), SHA=$(shasum "$1")) ---" + cat "$1" + echo "--- SNAP ($1)---" + echo "--- SNIP ($2, size=$(wc "$2"), SHA=$(shasum "$2")) ---" + cat "$2" + echo "--- SNAP ($2) ---" + fail "file '$1' not equal to '$2'" + fi +} + +function assert_less_than() { + if [[ ! "$1" -lt "$2" ]]; then + fail "assertion '$1' < '$2' failed" + fi +} + +function assert_less_than_or_equal() { + if [[ ! "$1" -le "$2" ]]; then + fail "assertion '$1' <= '$2' failed" + fi +} + +function assert_greater_than() { + if [[ ! "$1" -gt "$2" ]]; then + fail "assertion '$1' > '$2' failed" + fi +} + +function assert_greater_than_or_equal() { + if [[ ! "$1" -ge "$2" ]]; then + fail "assertion '$1' >= '$2' failed" + fi +} + +g_has_previously_infoed=false + +function info() { + if $g_show_info; then + if ! $g_has_previously_infoed; then + echo >&3 || true # echo an extra newline so it looks better + g_has_previously_infoed=true + fi + echo >&3 "info: $*" || true + fi +} + +function warn() { + echo >&4 "warning: $*" +} diff --git a/IntegrationTests/tests_01_allocation_counters/defines.sh b/IntegrationTests/tests_01_allocation_counters/defines.sh new file mode 100644 index 00000000..c99a5218 --- /dev/null +++ b/IntegrationTests/tests_01_allocation_counters/defines.sh @@ -0,0 +1,14 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_allocation_counts.sh b/IntegrationTests/tests_01_allocation_counters/test_01_allocation_counts.sh new file mode 100644 index 00000000..74a823ca --- /dev/null +++ b/IntegrationTests/tests_01_allocation_counters/test_01_allocation_counts.sh @@ -0,0 +1,52 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +source defines.sh + +set -eu +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +all_tests=() +for file in "$here/test_01_resources/"test_*.swift; do + test_name=$(basename "$file") + test_name=${test_name#test_*} + test_name=${test_name%*.swift} + all_tests+=( "$test_name" ) +done + +"$here/test_01_resources/run-nio-http2-alloc-counter-tests.sh" -t "$tmp" > "$tmp/output" + +for test in "${all_tests[@]}"; do + cat "$tmp/output" # helps debugging + total_allocations=$(grep "^test_$test.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') + not_freed_allocations=$(grep "^test_$test.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') + max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test" + + info "$test: allocations not freed: $not_freed_allocations" + info "$test: total number of mallocs: $total_allocations" + + assert_less_than "$not_freed_allocations" 5 # allow some slack + assert_greater_than "$not_freed_allocations" -5 # allow some slack + assert_greater_than "$total_allocations" 1000 + if [[ -z "${!max_allowed_env_name+x}" ]]; then + if [[ -z "${!max_allowed_env_name+x}" ]]; then + warn "no reference number of allocations set (set to \$$max_allowed_env_name)" + warn "to set current number:" + warn " export $max_allowed_env_name=$total_allocations" + fi + else + assert_less_than_or_equal "$total_allocations" "${!max_allowed_env_name}" + fi +done diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_resources/run-nio-http2-alloc-counter-tests.sh b/IntegrationTests/tests_01_allocation_counters/test_01_resources/run-nio-http2-alloc-counter-tests.sh new file mode 100755 index 00000000..15d85053 --- /dev/null +++ b/IntegrationTests/tests_01_allocation_counters/test_01_resources/run-nio-http2-alloc-counter-tests.sh @@ -0,0 +1,50 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set -eu +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +tmp_dir="/tmp" + +function die() { + echo >&2 "ERROR: $*" + exit 1 +} + +while getopts "t:" opt; do + case "$opt" in + t) + tmp_dir="$OPTARG" + ;; + \?) + die "unknown option $opt" + ;; + esac +done + +nio_checkout=$(mktemp -d "$tmp_dir/.swift-nio_XXXXXX") +( +cd "$nio_checkout" +git clone --depth 1 https://github.com/apple/swift-nio +) + +"$nio_checkout/swift-nio/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh" \ + -p "$here/../../.." \ + -m NIO \ + -m NIOHTTP1 \ + -m NIOHTTP2 \ + -t "$tmp_dir" \ + -d <( echo '.package(url: "https://github.com/apple/swift-nio.git", from: "2.0.0"),' ) \ + "$here"/test_*.swift diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_create_client_stream_channel.swift b/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_create_client_stream_channel.swift new file mode 100644 index 00000000..549cb400 --- /dev/null +++ b/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_create_client_stream_channel.swift @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIO +import NIOHPACK +import NIOHTTP1 +import NIOHTTP2 + +func run(identifier: String) { + let channel = EmbeddedChannel(handler: NIOHTTP2Handler(mode: .client)) + let multiplexer = HTTP2StreamMultiplexer(mode: .client, channel: channel) + try! channel.pipeline.addHandler(multiplexer).wait() + try! channel.connect(to: SocketAddress(ipAddress: "1.2.3.4", port: 5678)).wait() + + measure(identifier: identifier) { + var sumOfStreamIDs = 0 + + for _ in 0..<1000 { + let promise = channel.eventLoop.makePromise(of: Channel.self) + multiplexer.createStreamChannel(promise: promise) { (channel, streamID) in + return channel.eventLoop.makeSucceededFuture(()) + } + channel.embeddedEventLoop.run() + let child = try! promise.futureResult.wait() + let streamID = try! Int(child.getOption(HTTP2StreamChannelOptions.streamID).wait()) + + sumOfStreamIDs += streamID + let closeFuture = child.close() + channel.embeddedEventLoop.run() + try! closeFuture.wait() + } + + return sumOfStreamIDs + } +} diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_hpack_decoding.swift b/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_hpack_decoding.swift new file mode 100644 index 00000000..55949438 --- /dev/null +++ b/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_hpack_decoding.swift @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIO +import NIOHTTP2 +import NIOHPACK + +func run(identifier: String) { + var buffer = ByteBufferAllocator().buffer(capacity: 128) + buffer.writeBytes([0x82, 0x86, 0x84, 0x41, 0x0f, 0x77, 0x77, 0x77, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d]) + let request1 = buffer + buffer.clear() + buffer.writeBytes([0x82, 0x86, 0x84, 0xbe, 0x58, 0x08, 0x6e, 0x6f, 0x2d, 0x63, 0x61, 0x63, 0x68, 0x65]) + let request2 = buffer + buffer.clear() + buffer.writeBytes([0x82, 0x87, 0x85, 0xbf, 0x40, 0x0a, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x2d, 0x6b, 0x65, 0x79, 0x0c, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x2d, 0x76, 0x61, 0x6c, 0x75, 0x65]) + let request3 = buffer + + measure(identifier: identifier) { + var sum = 0 + for _ in 0..<1000 { + var request1Buffer = request1 + var request2Buffer = request2 + var request3Buffer = request3 + var decoder = HPACKDecoder(allocator: ByteBufferAllocator()) + let decoded1 = try! decoder.decodeHeaders(from: &request1Buffer) + sum += decoded1.count + let decoded2 = try! decoder.decodeHeaders(from: &request2Buffer) + sum += decoded2.count + let decoded3 = try! decoder.decodeHeaders(from: &request3Buffer) + sum += decoded3.count + } + return sum + } +} diff --git a/docker/docker-compose.1804.50.yaml b/docker/docker-compose.1804.50.yaml index 05bade93..e88b3190 100644 --- a/docker/docker-compose.1804.50.yaml +++ b/docker/docker-compose.1804.50.yaml @@ -10,8 +10,18 @@ services: swift_version: "5.0" h2spec_version: "2.2.1" + integration-tests: + image: swift-nio-http2:18.04-5.0 + environment: + - MAX_ALLOCS_ALLOWED_create_client_stream_channel=70010 + - MAX_ALLOCS_ALLOWED_hpack_decoding=5050 + test: image: swift-nio-http2:18.04-5.0 + command: /bin/bash -cl "swift test && ./scripts/integration_tests.sh" + environment: + - MAX_ALLOCS_ALLOWED_create_client_stream_channel=70010 + - MAX_ALLOCS_ALLOWED_hpack_decoding=5050 h2spec: image: swift-nio-http2:18.04-5.0 diff --git a/scripts/integration_tests.sh b/scripts/integration_tests.sh new file mode 100755 index 00000000..083c59bf --- /dev/null +++ b/scripts/integration_tests.sh @@ -0,0 +1,17 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +mkdir -p .build # for the junit.xml file +./IntegrationTests/run-tests.sh --junit-xml .build/junit-sh-tests.xml -i