add performance analysis docker-compose setup and script (apple#141)

motivation: prepare to run performance tests in ci

changes:

add performance-test docker-compose task
add script that can parse the performance test suite results and compare between two result-sets
add generic shell docker-compose task

docker/docker-compose.1804.50.yaml

@@ -20,6 +20,9 @@ services:
       - MAX_ALLOCS_ALLOWED_hpack_decoding=5050
       - MAX_ALLOCS_ALLOWED_client_server_request_response=360000
 
+  performance-test:
+    image: swift-nio-http2:18.04-5.0
+
   h2spec:
     image: swift-nio-http2:18.04-5.0
 
@@ -29,3 +32,6 @@ services:
       - MAX_ALLOCS_ALLOWED_create_client_stream_channel=70010
       - MAX_ALLOCS_ALLOWED_hpack_decoding=5050
       - MAX_ALLOCS_ALLOWED_client_server_request_response=360000
+
+  shell:
+    image: swift-nio-http2:18.04-5.0

docker/docker-compose.yaml

@@ -31,10 +31,20 @@ services:
     <<: *common
     command: /bin/bash -cl "./scripts/integration_tests.sh"
 
+  performance-test:
+    <<: *common
+    command: /bin/bash -cl "swift build -c release && ./.build/release/NIOHTTP2PerformanceTester"
+
   h2spec:
     <<: *common
     command: /bin/bash -cl "./scripts/test_h2spec.sh"
 
   test:
     <<: *common
     command: /bin/bash -cl "swift test -Xswiftc -warnings-as-errors && ./scripts/integration_tests.sh && ./scripts/test_h2spec.sh"
+
+  # util
+
+  shell:
+    <<: *common
+    entrypoint: /bin/bash

scripts/analyze_performance_results.rb

@@ -0,0 +1,184 @@
+#!/usr/bin/env ruby
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftNIO open source project
+##
+## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+require 'optparse'
+
+METRIC="min" # used for comparison
+
+module Enumerable
+  def sum
+    return self.inject(0){|accum, i| accum + i }
+  end
+
+  def mean
+    return self.sum / self.length.to_f
+  end
+
+  def sample_variance
+    m = self.mean
+    sum = self.inject(0){|accum, i| accum + (i - m) ** 2 }
+    return sum / (self.length - 1).to_f
+  end
+
+  def standard_deviation
+    return Math.sqrt(self.sample_variance)
+  end
+end
+
+def parse_results(file)
+  results = {}
+  File.open(file, "r") do |f|
+    f.each_line do |line|
+      parts = line.split(':').collect(&:strip)
+      throw "invalid data format" unless parts.length == 3
+      key = parts[1]
+      values = parts[2].split(',').collect(&:strip).map(&:to_f)
+      results[key] = {}
+      results[key]["values"] = values
+      results[key]["max"] = values.max
+      results[key]["min"] = values.min
+      results[key]["mean"] = values.mean
+      results[key]["std"] = values.standard_deviation
+    end
+  end
+  results
+end
+
+def compare_results(current, previous)
+  results = {}
+  current.keys.each do |key|
+    results[key] = {}
+    results[key]["previous"] = previous[key] || { ::METRIC => "n/a" }
+    results[key]["current"] = current[key]
+    if previous[key]
+      current_value = current[key][::METRIC]
+      previous_value = previous[key][::METRIC]
+      delta = current_value - previous_value
+      results[key]["delta"] = delta
+      results[key]["winner"] = current_value <= previous_value ? "current" : "previous"
+      results[key]["diff"] = (delta / previous_value * 100).to_i
+    else
+      results[key]["winner"] = "n/a"
+      results[key]["diff"] = "n/a"
+    end
+  end
+  results
+end
+
+def print_results_markdown(results)
+  columns = ["min", "max", "mean", "std"]
+  puts "| name | #{columns.join(" | ")} |"
+  puts "|#{Array.new(columns.size+1, '--').join("|")}|"
+  results.keys.each do |key|
+    print "| #{key}"
+    columns.each do |column|
+      print " | #{results[key][column]}"
+    end
+    puts " |\n"
+  end
+end
+
+def print_results_html(results)
+  columns = ["min", "max", "mean", "std"]
+  puts "<table border=\"1\">"
+  puts "<tr><td>name</td><td>#{columns.join("</td><td>")}</td></tr>"
+  results.keys.each do |key|
+    puts "<tr>"
+    puts "<td>#{key}</td>"
+    columns.each do |column|
+      puts "<td>#{results[key][column]}</td>"
+    end
+    puts "</tr>"
+  end
+  puts "</table>"
+end
+
+def print_results_csv(results)
+  puts results.keys.join(",")
+  puts results.keys.map{ |key| results[key][::METRIC] }.join(",")
+end
+
+def print_comparison_markdown(results)
+  puts "| name | current | previous | winner | diff |"
+  puts "|#{Array.new(5, '--').join("|")}|"
+  results.keys.each do |key|
+    puts "| #{key} | #{results[key]["current"]["mean"]} | #{results[key]["previous"]["mean"]} | #{results[key]["winner"]} | #{results[key]["diff"]}% |"
+  end
+end
+
+def print_comparison_html(results)
+  puts "<table border=\"1\">"
+  puts "  <tr>
+    <td>name</td>
+    <td>current</td>
+    <td>previous</td>
+    <td>winner</td>
+    <td>diff</td>
+  </tr>"
+  results.keys.each do |key|
+    puts "  <tr>
+    <td>#{key}</td>
+    <td>#{results[key]["current"]["mean"]}</td>
+    <td>#{results[key]["previous"]["mean"]}</td>
+    <td>#{results[key]["winner"]}</td>
+    <td>#{results[key]["diff"]}%</td>
+  </tr>"
+  end
+  puts "</table>"
+end
+
+
+ARGV << '-h' if ARGV.empty?
+
+options = {}
+OptionParser.new do |opt|
+  opt.on('-f', '--file file', 'file to process') { |o| options[:file] = o }
+  opt.on('-p', '--previous previous', 'previous file to process') { |o| options[:previous] = o }
+  opt.on('-o', '--output output', 'output format') { |o| options[:output] = o }
+  opt.on_tail("-h", "--help", "show this message") do
+    puts opt
+  end
+end.parse!
+
+if options.has_key?(:file) && options.has_key?(:previous)
+  current = parse_results(options[:file])
+  previous = parse_results(options[:previous])
+  results = compare_results(current, previous)
+
+  case options[:output]
+  when "html"
+    print_comparison_html(results)
+  when "markdown", nil
+    print_comparison_markdown(results)
+  else
+    throw "invalid output format #{options[:output]}"
+  end
+
+elsif options.has_key?(:file)
+  results = parse_results(options[:file])
+  case options[:output]
+  when "csv"
+    print_results_csv(results)
+  when "html", nil
+    print_results_html(results)
+  when "markdown", nil
+    print_results_markdown(results)
+  else
+    throw "invalid output format #{options[:output]}"
+  end
+
+else
+  throw "invalid arguemnts"
+end