From 3d0b38268ecda6ba0e7a1d5aca1c3c5a20f7c42a Mon Sep 17 00:00:00 2001 From: tomer doron Date: Tue, 16 Jul 2019 03:04:26 -0700 Subject: [PATCH] add performance analysis docker-compose setup and script (#141) motivation: prepare to run performance tests in ci changes: add performance-test docker-compose task add script that can parse the performance test suite results and compare between two result-sets add generic shell docker-compose task --- docker/docker-compose.1804.50.yaml | 6 + docker/docker-compose.yaml | 10 ++ scripts/analyze_performance_results.rb | 184 +++++++++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100755 scripts/analyze_performance_results.rb diff --git a/docker/docker-compose.1804.50.yaml b/docker/docker-compose.1804.50.yaml index 10a464f9..a11f7368 100644 --- a/docker/docker-compose.1804.50.yaml +++ b/docker/docker-compose.1804.50.yaml @@ -20,6 +20,9 @@ services: - MAX_ALLOCS_ALLOWED_hpack_decoding=5050 - MAX_ALLOCS_ALLOWED_client_server_request_response=360000 + performance-test: + image: swift-nio-http2:18.04-5.0 + h2spec: image: swift-nio-http2:18.04-5.0 @@ -29,3 +32,6 @@ services: - MAX_ALLOCS_ALLOWED_create_client_stream_channel=70010 - MAX_ALLOCS_ALLOWED_hpack_decoding=5050 - MAX_ALLOCS_ALLOWED_client_server_request_response=360000 + + shell: + image: swift-nio-http2:18.04-5.0 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 5aaa1c3e..94f4d616 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -31,6 +31,10 @@ services: <<: *common command: /bin/bash -cl "./scripts/integration_tests.sh" + performance-test: + <<: *common + command: /bin/bash -cl "swift build -c release && ./.build/release/NIOHTTP2PerformanceTester" + h2spec: <<: *common command: /bin/bash -cl "./scripts/test_h2spec.sh" @@ -38,3 +42,9 @@ services: test: <<: *common command: /bin/bash -cl "swift test -Xswiftc -warnings-as-errors && ./scripts/integration_tests.sh && ./scripts/test_h2spec.sh" + + # util + + shell: + <<: *common + entrypoint: /bin/bash diff --git a/scripts/analyze_performance_results.rb b/scripts/analyze_performance_results.rb new file mode 100755 index 00000000..f5481a2d --- /dev/null +++ b/scripts/analyze_performance_results.rb @@ -0,0 +1,184 @@ +#!/usr/bin/env ruby +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +require 'optparse' + +METRIC="min" # used for comparison + +module Enumerable + def sum + return self.inject(0){|accum, i| accum + i } + end + + def mean + return self.sum / self.length.to_f + end + + def sample_variance + m = self.mean + sum = self.inject(0){|accum, i| accum + (i - m) ** 2 } + return sum / (self.length - 1).to_f + end + + def standard_deviation + return Math.sqrt(self.sample_variance) + end +end + +def parse_results(file) + results = {} + File.open(file, "r") do |f| + f.each_line do |line| + parts = line.split(':').collect(&:strip) + throw "invalid data format" unless parts.length == 3 + key = parts[1] + values = parts[2].split(',').collect(&:strip).map(&:to_f) + results[key] = {} + results[key]["values"] = values + results[key]["max"] = values.max + results[key]["min"] = values.min + results[key]["mean"] = values.mean + results[key]["std"] = values.standard_deviation + end + end + results +end + +def compare_results(current, previous) + results = {} + current.keys.each do |key| + results[key] = {} + results[key]["previous"] = previous[key] || { ::METRIC => "n/a" } + results[key]["current"] = current[key] + if previous[key] + current_value = current[key][::METRIC] + previous_value = previous[key][::METRIC] + delta = current_value - previous_value + results[key]["delta"] = delta + results[key]["winner"] = current_value <= previous_value ? "current" : "previous" + results[key]["diff"] = (delta / previous_value * 100).to_i + else + results[key]["winner"] = "n/a" + results[key]["diff"] = "n/a" + end + end + results +end + +def print_results_markdown(results) + columns = ["min", "max", "mean", "std"] + puts "| name | #{columns.join(" | ")} |" + puts "|#{Array.new(columns.size+1, '--').join("|")}|" + results.keys.each do |key| + print "| #{key}" + columns.each do |column| + print " | #{results[key][column]}" + end + puts " |\n" + end +end + +def print_results_html(results) + columns = ["min", "max", "mean", "std"] + puts "" + puts "" + results.keys.each do |key| + puts "" + puts "" + columns.each do |column| + puts "" + end + puts "" + end + puts "
name#{columns.join("")}
#{key}#{results[key][column]}
" +end + +def print_results_csv(results) + puts results.keys.join(",") + puts results.keys.map{ |key| results[key][::METRIC] }.join(",") +end + +def print_comparison_markdown(results) + puts "| name | current | previous | winner | diff |" + puts "|#{Array.new(5, '--').join("|")}|" + results.keys.each do |key| + puts "| #{key} | #{results[key]["current"]["mean"]} | #{results[key]["previous"]["mean"]} | #{results[key]["winner"]} | #{results[key]["diff"]}% |" + end +end + +def print_comparison_html(results) + puts "" + puts " + + + + + + " + results.keys.each do |key| + puts " + + + + + + " + end + puts "
namecurrentpreviouswinnerdiff
#{key}#{results[key]["current"]["mean"]}#{results[key]["previous"]["mean"]}#{results[key]["winner"]}#{results[key]["diff"]}%
" +end + + +ARGV << '-h' if ARGV.empty? + +options = {} +OptionParser.new do |opt| + opt.on('-f', '--file file', 'file to process') { |o| options[:file] = o } + opt.on('-p', '--previous previous', 'previous file to process') { |o| options[:previous] = o } + opt.on('-o', '--output output', 'output format') { |o| options[:output] = o } + opt.on_tail("-h", "--help", "show this message") do + puts opt + end +end.parse! + +if options.has_key?(:file) && options.has_key?(:previous) + current = parse_results(options[:file]) + previous = parse_results(options[:previous]) + results = compare_results(current, previous) + + case options[:output] + when "html" + print_comparison_html(results) + when "markdown", nil + print_comparison_markdown(results) + else + throw "invalid output format #{options[:output]}" + end + +elsif options.has_key?(:file) + results = parse_results(options[:file]) + case options[:output] + when "csv" + print_results_csv(results) + when "html", nil + print_results_html(results) + when "markdown", nil + print_results_markdown(results) + else + throw "invalid output format #{options[:output]}" + end + +else + throw "invalid arguemnts" +end