diff --git a/docker/docker-compose.1804.50.yaml b/docker/docker-compose.1804.50.yaml
index 10a464f9..a11f7368 100644
--- a/docker/docker-compose.1804.50.yaml
+++ b/docker/docker-compose.1804.50.yaml
@@ -20,6 +20,9 @@ services:
- MAX_ALLOCS_ALLOWED_hpack_decoding=5050
- MAX_ALLOCS_ALLOWED_client_server_request_response=360000
+ performance-test:
+ image: swift-nio-http2:18.04-5.0
+
h2spec:
image: swift-nio-http2:18.04-5.0
@@ -29,3 +32,6 @@ services:
- MAX_ALLOCS_ALLOWED_create_client_stream_channel=70010
- MAX_ALLOCS_ALLOWED_hpack_decoding=5050
- MAX_ALLOCS_ALLOWED_client_server_request_response=360000
+
+ shell:
+ image: swift-nio-http2:18.04-5.0
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index 5aaa1c3e..94f4d616 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -31,6 +31,10 @@ services:
<<: *common
command: /bin/bash -cl "./scripts/integration_tests.sh"
+ performance-test:
+ <<: *common
+ command: /bin/bash -cl "swift build -c release && ./.build/release/NIOHTTP2PerformanceTester"
+
h2spec:
<<: *common
command: /bin/bash -cl "./scripts/test_h2spec.sh"
@@ -38,3 +42,9 @@ services:
test:
<<: *common
command: /bin/bash -cl "swift test -Xswiftc -warnings-as-errors && ./scripts/integration_tests.sh && ./scripts/test_h2spec.sh"
+
+ # util
+
+ shell:
+ <<: *common
+ entrypoint: /bin/bash
diff --git a/scripts/analyze_performance_results.rb b/scripts/analyze_performance_results.rb
new file mode 100755
index 00000000..f5481a2d
--- /dev/null
+++ b/scripts/analyze_performance_results.rb
@@ -0,0 +1,184 @@
+#!/usr/bin/env ruby
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftNIO open source project
+##
+## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+require 'optparse'
+
+METRIC="min" # used for comparison
+
+module Enumerable
+ def sum
+ return self.inject(0){|accum, i| accum + i }
+ end
+
+ def mean
+ return self.sum / self.length.to_f
+ end
+
+ def sample_variance
+ m = self.mean
+ sum = self.inject(0){|accum, i| accum + (i - m) ** 2 }
+ return sum / (self.length - 1).to_f
+ end
+
+ def standard_deviation
+ return Math.sqrt(self.sample_variance)
+ end
+end
+
+def parse_results(file)
+ results = {}
+ File.open(file, "r") do |f|
+ f.each_line do |line|
+ parts = line.split(':').collect(&:strip)
+ throw "invalid data format" unless parts.length == 3
+ key = parts[1]
+ values = parts[2].split(',').collect(&:strip).map(&:to_f)
+ results[key] = {}
+ results[key]["values"] = values
+ results[key]["max"] = values.max
+ results[key]["min"] = values.min
+ results[key]["mean"] = values.mean
+ results[key]["std"] = values.standard_deviation
+ end
+ end
+ results
+end
+
+def compare_results(current, previous)
+ results = {}
+ current.keys.each do |key|
+ results[key] = {}
+ results[key]["previous"] = previous[key] || { ::METRIC => "n/a" }
+ results[key]["current"] = current[key]
+ if previous[key]
+ current_value = current[key][::METRIC]
+ previous_value = previous[key][::METRIC]
+ delta = current_value - previous_value
+ results[key]["delta"] = delta
+ results[key]["winner"] = current_value <= previous_value ? "current" : "previous"
+ results[key]["diff"] = (delta / previous_value * 100).to_i
+ else
+ results[key]["winner"] = "n/a"
+ results[key]["diff"] = "n/a"
+ end
+ end
+ results
+end
+
+def print_results_markdown(results)
+ columns = ["min", "max", "mean", "std"]
+ puts "| name | #{columns.join(" | ")} |"
+ puts "|#{Array.new(columns.size+1, '--').join("|")}|"
+ results.keys.each do |key|
+ print "| #{key}"
+ columns.each do |column|
+ print " | #{results[key][column]}"
+ end
+ puts " |\n"
+ end
+end
+
+def print_results_html(results)
+ columns = ["min", "max", "mean", "std"]
+ puts "
"
+ puts "name | #{columns.join(" | ")} |
"
+ results.keys.each do |key|
+ puts ""
+ puts "#{key} | "
+ columns.each do |column|
+ puts "#{results[key][column]} | "
+ end
+ puts "
"
+ end
+ puts "
"
+end
+
+def print_results_csv(results)
+ puts results.keys.join(",")
+ puts results.keys.map{ |key| results[key][::METRIC] }.join(",")
+end
+
+def print_comparison_markdown(results)
+ puts "| name | current | previous | winner | diff |"
+ puts "|#{Array.new(5, '--').join("|")}|"
+ results.keys.each do |key|
+ puts "| #{key} | #{results[key]["current"]["mean"]} | #{results[key]["previous"]["mean"]} | #{results[key]["winner"]} | #{results[key]["diff"]}% |"
+ end
+end
+
+def print_comparison_html(results)
+ puts ""
+ puts "
+ name |
+ current |
+ previous |
+ winner |
+ diff |
+
"
+ results.keys.each do |key|
+ puts "
+ #{key} |
+ #{results[key]["current"]["mean"]} |
+ #{results[key]["previous"]["mean"]} |
+ #{results[key]["winner"]} |
+ #{results[key]["diff"]}% |
+
"
+ end
+ puts "
"
+end
+
+
+ARGV << '-h' if ARGV.empty?
+
+options = {}
+OptionParser.new do |opt|
+ opt.on('-f', '--file file', 'file to process') { |o| options[:file] = o }
+ opt.on('-p', '--previous previous', 'previous file to process') { |o| options[:previous] = o }
+ opt.on('-o', '--output output', 'output format') { |o| options[:output] = o }
+ opt.on_tail("-h", "--help", "show this message") do
+ puts opt
+ end
+end.parse!
+
+if options.has_key?(:file) && options.has_key?(:previous)
+ current = parse_results(options[:file])
+ previous = parse_results(options[:previous])
+ results = compare_results(current, previous)
+
+ case options[:output]
+ when "html"
+ print_comparison_html(results)
+ when "markdown", nil
+ print_comparison_markdown(results)
+ else
+ throw "invalid output format #{options[:output]}"
+ end
+
+elsif options.has_key?(:file)
+ results = parse_results(options[:file])
+ case options[:output]
+ when "csv"
+ print_results_csv(results)
+ when "html", nil
+ print_results_html(results)
+ when "markdown", nil
+ print_results_markdown(results)
+ else
+ throw "invalid output format #{options[:output]}"
+ end
+
+else
+ throw "invalid arguemnts"
+end