From 0e8a56ad1a66c9a5d699aed764bdfdab6898b45e Mon Sep 17 00:00:00 2001 From: stfeng Date: Wed, 28 Jan 2015 21:46:32 -0800 Subject: [PATCH] Incorporate storage api based tests into object_storage_benchmarks. After this commit, we will have: 1. single byte upload and download API tests. 2. List after write consistency tests In addition to this, I also did a number of refactoring of the existing object_storage_benchmark code. And I also add the pkb/scripts/ directory to the default resource loader. --- MANIFEST.in | 1 + README.md | 28 ++ .../benchmarks/object_storage_benchmark.py | 341 +++++++++++++++--- perfkitbenchmarker/data/__init__.py | 10 +- .../data/cloud-storage-workload.sh | 18 +- perfkitbenchmarker/scripts/__init__.py | 13 + .../scripts/object_storage_api_tests.py | 30 +- 7 files changed, 369 insertions(+), 72 deletions(-) create mode 100644 perfkitbenchmarker/scripts/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in index 278a9be496..0841950630 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include perfkitbenchmarker/data/* +include perfkitbenchmarker/scripts/* include LICENSE include README.md include ez_setup.py diff --git a/README.md b/README.md index a05fe104c5..d16c06fd00 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,34 @@ Test that azure is installed correctly ``` $ azure vm list ``` +## Create and Configure a .BOTO file for object storage benchmarks + +In order to run object storage benchmark tests, you need to have a properly configured ~/.boto file. + +Here is how: + +* Create the ~/.boto file (If you already have ~/.boto, you can skip this step. Consider making a backup copy of your existing .boto file.) + +To create a new ~/.boto file, issue the following command and follow the instructions given by this command: +``` +$ gsutil config +``` +As a result, a .boto file is created under your home directory. + +* Open the .boto file and edit the following fields: +1. In the [Credentials] section: + +gs_oauth2_refresh_token: set it to be the same as the "refresh_token" field in your gcloud credential file (~/.config/gcloud/credentials), which was setup as part of the 'gcloud auth login' step. + +aws_access_key_id, aws_secret_access_key: set these to be the AWS access keys you intend to use for these tests, or you can use the same keys as those in your existing AWS credentials file (~/.aws/credentials). + +2. In the [gsutil] section: + +default_project_id: if it is not already set, set it to be the google cloud storage project ID you intend to use for this test. (If you used 'gsutil config' to generate the .boto file, you should have been prompted to supply this information at this step). + +3. In the [OAuth2] section: +client_id, client_secret: set these to be the same as those in your gcloud credentials file (~/.config/gcloud/credentials), which was setup as part of the 'gcloud auth login' step. + ## Install PerfKit dependencies ``` diff --git a/perfkitbenchmarker/benchmarks/object_storage_benchmark.py b/perfkitbenchmarker/benchmarks/object_storage_benchmark.py index 6a4de09484..81898c2a5a 100644 --- a/perfkitbenchmarker/benchmarks/object_storage_benchmark.py +++ b/perfkitbenchmarker/benchmarks/object_storage_benchmark.py @@ -12,10 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Using storage tools from providers to upload/download files in directory. +"""Object (blob) Storage benchmark tests. -Benchmarks here use CLI tools to communicate with storage providers, this -simulates a set of common use cases that are based on CLI tools. +There are two categories of tests here: 1) tests based on CLI tools, and 2) +tests that use APIs to access storage provider. + +For 1), we aim to simulate one typical use case of common user using storage +provider: upload and downloads a set of files from/to a local directory. + +For 2), we aim to measure more directly the performance of a storage provider +by accessing them via APIs. Here are the main scenarios covered in this +category: + a: Single byte object upload and download, measures latency. + b: List-after-write and list-after-update consistency measurement. + c: Single stream large object upload and download, measures throughput. Naming Conventions (X refers to cloud providers): PrepareX: Prepare vm with necessary storage tools from cloud providers. @@ -24,6 +34,7 @@ Documentation: https://goto.google.com/perfkitbenchmarker-storage """ +import json import logging import os import re @@ -42,13 +53,16 @@ flags.DEFINE_string('object_storage_credential_file', None, 'Directory of credential file.') +flags.DEFINE_string('boto_file_location', None, + 'The location of the boto file.') + FLAGS = flags.FLAGS # User a scratch disk here to simulate what most users would do when they # use CLI tools to interact with the storage provider. BENCHMARK_INFO = {'name': 'object_storage_benchmark', 'description': - 'Benchmark upload/download to Cloud Storage using CLI tools.', + 'Object/blob storage benchmarks.', 'scratch_disk': True, 'num_machines': 1} @@ -56,6 +70,8 @@ GCE_CREDENTIAL_LOCATION = '.config/gcloud' AZURE_CREDENTIAL_LOCATION = '.azure' +DEFAULT_BOTO_LOCATION = '~/.boto' + OBJECT_STORAGE_CREDENTIAL_DEFAULT_LOCATION = { benchmark_spec_class.GCP: '~/' + GCE_CREDENTIAL_LOCATION, benchmark_spec_class.AWS: '~/' + AWS_CREDENTIAL_LOCATION, @@ -66,13 +82,115 @@ DATA_FILE = 'cloud-storage-workload.sh' # size of all data -DATA_SIZE_IN_MB = 2561 +DATA_SIZE_IN_MB = 256.1 + +API_TEST_SCRIPT = 'object_storage_api_tests.py' + +# The default number of iterations to run for the list consistency benchmark. +LIST_CONSISTENCY_ITERATIONS = 200 + +# Various constants to name the result metrics. +THROUGHPUT_UNIT = 'MB/sec' +LATENCY_UNIT = 'seconds' +NA_UNIT = 'na' +PERCENTILES_LIST = ['p50', 'p90', 'p99', 'p99.9'] + +UPLOAD_THROUGHPUT_VIA_CLI = 'upload throughput via cli' +DOWNLOAD_THROUGHPUT_VIA_CLI = 'download throughput via cli' + +ONE_BYTE_LATENCY = 'one byte %s latency' +LAW_CONSISTENCY_PERCENTAGE = 'list-after-write consistency percentage' +LAW_INCONSISTENCY_WINDOW = 'list-after-write inconsistency window' +CONSISTENT_LIST_LATENCY = 'consistent list latency' def GetInfo(): return BENCHMARK_INFO +def S3orGCSApiBasedBenchmarks(results, metadata, vm, storage, test_script_path, + bucket_name): + """Runs the api based benchmarks for s3 or GCS + + Args: + vm: The vm being used to run the benchmark. + results: The results array to append to. + storage: The storage provider to run: S3 or GCS + test_script_path: The complete path to the test script on the target VM. + + Raises: + ValueError: unexpected test outcome is found from the API test script. + """ + if storage is not 'S3' and storage is not 'GCS': + raise ValueError("Storage must be S3 or GCS to invoke this function") + + # One byte RW latency + one_byte_rw_cmd = ('%s --bucket=%s --storage=%s --scenario=OneByteRW') % ( + test_script_path, bucket_name, storage) # noqa + + _, raw_result = vm.RemoteCommand(one_byte_rw_cmd) + logging.info('OneByteRW raw result is %s' % raw_result) + + for up_and_down in ['upload', 'download']: + search_string = 'One byte %s - (.*)' % up_and_down + result_string = re.findall(search_string, raw_result) + + if len(result_string) > 0: + result = json.loads(result_string[0]) + for percentile in PERCENTILES_LIST: + results.append([('%s %s') % (ONE_BYTE_LATENCY % up_and_down, + percentile), + float(result[percentile]), + LATENCY_UNIT, + metadata]) + else: + raise ValueError( + 'Unexpected test outcome from OneByteRW api test: %s.' % raw_result) + + # list-after-write consistency metrics + list_consistency_cmd = ('%s --bucket=%s --storage=%s --iterations=%d ' + '--scenario=ListConsistency') % (test_script_path, + bucket_name, # noqa + storage, # noqa + LIST_CONSISTENCY_ITERATIONS) # noqa + + _, raw_result = vm.RemoteCommand(list_consistency_cmd) + logging.info('ListConsistency raw result is %s' % raw_result) + + search_string = 'List consistency percentage: (.*)' + result_string = re.findall(search_string, raw_result) + if len(result_string) > 0: + results.append([LAW_CONSISTENCY_PERCENTAGE, + (float)(result_string[0]), + NA_UNIT, + metadata]) + else: + raise ValueError( + 'Cannot get percentage from ListConsistency test.') + + # Parse the list inconsistency window if there is any. + search_string = 'List inconsistency window: (.*)' + result_string = re.findall(search_string, raw_result) + if len(result_string) > 0: + result = json.loads(result_string[0]) + for percentile in PERCENTILES_LIST: + results.append([('%s %s') % (LAW_INCONSISTENCY_WINDOW, percentile), + (float)(result[percentile]), + LATENCY_UNIT, + metadata]) + + # Also report the list latency from those lists that are consistent + search_string = 'List latency: (.*)' + result_string = re.findall(search_string, raw_result) + if len(result_string) > 0: + result = json.loads(result_string[0]) + for percentile in PERCENTILES_LIST: + results.append([('%s %s') % (CONSISTENT_LIST_LATENCY, percentile), + (float)(result[percentile]), + LATENCY_UNIT, + metadata]) + + class S3StorageBenchmark(object): """S3 version of storage benchmark.""" @@ -83,38 +201,77 @@ def Prepare(self, vm): Args: vm: The vm being used to run the benchmark. """ - vm.InstallPackage('python-setuptools') + vm.InstallPackage(' '.join(['gcc', + 'python-setuptools', + 'python-dev', + 'libffi-dev'])) + vm.RemoteCommand('sudo easy_install -U pip') vm.RemoteCommand('sudo pip install awscli') + vm.RemoteCommand('sudo pip install python-gflags==2.0') + vm.RemoteCommand('sudo pip install gcs-oauth2-boto-plugin==1.8') + vm.PushFile(FLAGS.object_storage_credential_file, AWS_CREDENTIAL_LOCATION) + vm.PushFile(FLAGS.boto_file_location, DEFAULT_BOTO_LOCATION) + + self.bucket_name = 'pkb%s' % FLAGS.run_uri vm.RemoteCommand( - 'aws s3 mb s3://pkb%s --region=us-east-1' % - FLAGS.run_uri) + 'aws s3 mb s3://%s --region=us-east-1' % self.bucket_name) - def Run(self, vm, result): + def Run(self, vm, metadata): """Run upload/download on vm with s3 tool. Args: vm: The vm being used to run the benchmark. - result: The result variable to store resutls. + metadata: the metadata to be stored with the results. + + Returns: + A list of lists containing results of the tests. Each scenario outputs + results to a list of the following format: + name of the scenario, value, unit of the value, metadata + e.g., + 'one byte object upload latency p50', 0.800, 'seconds', 'storage=gcs' + + Then the final return value is the list of the above list that reflect + the results of all scenarios run here. """ - vm.RemoteCommand('aws s3 rm s3://pkb%s --recursive' - % FLAGS.run_uri, ignore_failure=True) + vm.RemoteCommand('aws s3 rm s3://%s --recursive' + % self.bucket_name, ignore_failure=True) scratch_dir = vm.GetScratchDir() _, res = vm.RemoteCommand('time aws s3 sync %s/run/data/ ' - 's3://pkb%s/' % (scratch_dir, FLAGS.run_uri)) + 's3://%s/' % (scratch_dir, self.bucket_name)) logging.info(res) time_used = vm_util.ParseTimeCommandResult(res) - result[0][1] = DATA_SIZE_IN_MB / time_used + + results = [] + + upload_results = [UPLOAD_THROUGHPUT_VIA_CLI, + DATA_SIZE_IN_MB / time_used, + THROUGHPUT_UNIT, + metadata] + results.append(upload_results) vm.RemoteCommand('rm %s/run/data/*' % scratch_dir) _, res = vm.RemoteCommand('time aws s3 sync ' - 's3://pkb%s/ %s/run/data/' - % (FLAGS.run_uri, scratch_dir)) + 's3://%s/ %s/run/data/' + % (self.bucket_name, scratch_dir)) logging.info(res) time_used = vm_util.ParseTimeCommandResult(res) - result[1][1] = DATA_SIZE_IN_MB / time_used + + download_results = [DOWNLOAD_THROUGHPUT_VIA_CLI, + DATA_SIZE_IN_MB / time_used, + THROUGHPUT_UNIT, + metadata] + + results.append(download_results) + + # Now tests the storage provider via APIs + test_script_path = '%s/run/%s' % (scratch_dir, API_TEST_SCRIPT) + S3orGCSApiBasedBenchmarks(results, metadata, vm, 'S3', test_script_path, + self.bucket_name) + + return results def Cleanup(self, vm): """Clean up S3 bucket and uninstall packages on vm. @@ -122,10 +279,13 @@ def Cleanup(self, vm): Args: vm: The vm needs cleanup. """ - vm.RemoteCommand('aws s3 rm s3://pkb%s --recursive' - % FLAGS.run_uri, ignore_failure=True) - vm.RemoteCommand('aws s3 rb s3://pkb%s' % FLAGS.run_uri) + vm.RemoteCommand('aws s3 rm s3://%s --recursive' + % self.bucket_name, ignore_failure=True) + vm.RemoteCommand('aws s3 rb s3://%s' % self.bucket_name) + vm.RemoteCommand('/usr/bin/yes | sudo pip uninstall awscli') + vm.RemoteCommand('/usr/bin/yes | sudo pip uninstall python-gflags') + vm.RemoteCommand('/usr/bin/yes | sudo pip uninstall gcs-oauth2-boto-plugin') vm.RemoteCommand('sudo easy_install -m pip') @@ -163,12 +323,23 @@ def Prepare(self, vm): vm.RemoteCommand('azure storage blob list pkb%s %s' % ( FLAGS.run_uri, vm.azure_command_suffix)) - def Run(self, vm, result): + def Run(self, vm, metadata): """Run upload/download on vm with azure CLI tool. Args: vm: The vm being used to run the benchmark. - result: The result variable to store results. + metadata: the metadata to be stored with the results. + + Returns: + A list of lists containing results of the tests. Each scenario outputs + results to a list of the following format: + name of the scenario, value, unit of the value, metadata + e.g., + 'one byte object upload latency p50', 0.800, 'seconds', 'storage=gcs' + + Then the final return value is the list of the above list that reflect + the results of all scenarios run here. + """ vm.RemoteCommand('for i in {0..99}; do azure storage blob delete ' 'pkb%s file-$i.dat %s; done' % @@ -183,7 +354,15 @@ def Run(self, vm, result): vm.azure_command_suffix)) print res time_used = vm_util.ParseTimeCommandResult(res) - result[0][1] = DATA_SIZE_IN_MB / time_used + + results = [] + + upload_results = [UPLOAD_THROUGHPUT_VIA_CLI, + DATA_SIZE_IN_MB / time_used, + THROUGHPUT_UNIT, + metadata] + results.append(upload_results) + vm.RemoteCommand('rm %s/run/data/*' % scratch_dir) _, res = vm.RemoteCommand('time for i in {0..99}; do azure storage blob ' 'download pkb%s ' @@ -192,7 +371,14 @@ def Run(self, vm, result): vm.azure_command_suffix)) print res time_used = vm_util.ParseTimeCommandResult(res) - result[1][1] = DATA_SIZE_IN_MB / time_used + + download_results = [DOWNLOAD_THROUGHPUT_VIA_CLI, + DATA_SIZE_IN_MB / time_used, + THROUGHPUT_UNIT, + metadata] + results.append(download_results) + + return results def Cleanup(self, vm): """Clean up Azure storage container and uninstall packages on vm. @@ -235,43 +421,87 @@ def Prepare(self, vm): '--rc-path=.bash_profile ' '--path-update=true ' '--bash-completion=true') + + vm.InstallPackage(' '.join(['gcc', + 'python-setuptools', + 'python-dev', + 'libffi-dev'])) + + vm.RemoteCommand('sudo easy_install -U pip') + vm.RemoteCommand('sudo pip install python-gflags==2.0') + vm.RemoteCommand('sudo pip install gcs-oauth2-boto-plugin==1.8') + try: vm.RemoteCommand('mkdir .config') except errors.VmUtil.SshConnectionError: # If ran on existing machines, .config folder may already exists. pass vm.PushFile(FLAGS.object_storage_credential_file, '.config/') + vm.PushFile(FLAGS.boto_file_location, DEFAULT_BOTO_LOCATION) + vm.gsutil_path, _ = vm.RemoteCommand('which gsutil', login_shell=True) vm.gsutil_path = vm.gsutil_path.split()[0] - vm.RemoteCommand('%s mb gs://pkb%s' % - (vm.gsutil_path, FLAGS.run_uri)) - def Run(self, vm, result): + self.bucket_name = 'pkb%s' % FLAGS.run_uri + vm.RemoteCommand('%s mb gs://%s' % (vm.gsutil_path, self.bucket_name)) + + + def Run(self, vm, metadata): """Run upload/download on vm with gsutil tool. Args: vm: The vm being used to run the benchmark. - result: The result variable to store results. + metadata: the metadata to be stored with the results. + + Returns: + A list of lists containing results of the tests. Each scenario outputs + results to a list of the following format: + name of the scenario, value, unit of the value, metadata + e.g., + 'one byte object upload latency p50', 0.800, 'seconds', 'storage=gcs' + + Then the final return value is the list of the above list that reflect + the results of all scenarios run here. """ - vm.RemoteCommand('%s rm gs://pkb%s/*' % - (vm.gsutil_path, FLAGS.run_uri), ignore_failure=True) + vm.RemoteCommand('%s rm gs://%s/*' % + (vm.gsutil_path, self.bucket_name), ignore_failure=True) scratch_dir = vm.GetScratchDir() _, res = vm.RemoteCommand('time %s -m cp %s/run/data/* ' - 'gs://pkb%s/' % (vm.gsutil_path, scratch_dir, - FLAGS.run_uri)) + 'gs://%s/' % (vm.gsutil_path, scratch_dir, + self.bucket_name)) print res time_used = vm_util.ParseTimeCommandResult(res) - result[0][1] = DATA_SIZE_IN_MB / time_used + + results = [] + + upload_results = [UPLOAD_THROUGHPUT_VIA_CLI, + DATA_SIZE_IN_MB / time_used, + THROUGHPUT_UNIT, + metadata] + results.append(upload_results) + vm.RemoteCommand('rm %s/run/data/*' % scratch_dir) _, res = vm.RemoteCommand('time %s -m cp ' - 'gs://pkb%s/* ' - '%s/run/data/' % (vm.gsutil_path, FLAGS.run_uri, + 'gs://%s/* ' + '%s/run/data/' % (vm.gsutil_path, + self.bucket_name, scratch_dir)) print res time_used = vm_util.ParseTimeCommandResult(res) - result[1][1] = DATA_SIZE_IN_MB / time_used + + download_results = [DOWNLOAD_THROUGHPUT_VIA_CLI, + DATA_SIZE_IN_MB / time_used, + THROUGHPUT_UNIT, + metadata] + results.append(download_results) + + test_script_path = '%s/run/%s' % (scratch_dir, API_TEST_SCRIPT) + S3orGCSApiBasedBenchmarks(results, metadata, vm, 'GCS', test_script_path, + self.bucket_name) + + return results def Cleanup(self, vm): """Clean up Google Cloud Storage bucket and uninstall packages on vm. @@ -279,10 +509,14 @@ def Cleanup(self, vm): Args: vm: The vm needs cleanup. """ - vm.RemoteCommand('%s rm gs://pkb%s/*' % - (vm.gsutil_path, FLAGS.run_uri)) - vm.RemoteCommand('%s rb gs://pkb%s' % - (vm.gsutil_path, FLAGS.run_uri)) + vm.RemoteCommand('%s rm -r gs://%s/*' % + (vm.gsutil_path, self.bucket_name)) + vm.RemoteCommand('%s rb gs://%s' % + (vm.gsutil_path, self.bucket_name)) + + vm.RemoteCommand('/usr/bin/yes | sudo pip uninstall python-gflags') + vm.RemoteCommand('/usr/bin/yes | sudo pip uninstall gcs-oauth2-boto-plugin') + vm.RemoteCommand('sudo easy_install -m pip') OBJECT_STORAGE_BENCHMARK_DICTIONARY = { @@ -311,15 +545,32 @@ def Prepare(benchmark_spec): raise errors.Benchmarks.MissingObjectCredentialException( 'Credential cannot be found in %s', FLAGS.object_storage_credential_file) + + if not FLAGS.boto_file_location: + FLAGS.boto_file_location = DEFAULT_BOTO_LOCATION + FLAGS.boto_file_location = os.path.expanduser(FLAGS.boto_file_location) + + if not os.path.isfile(FLAGS.boto_file_location): + if FLAGS.storage is not benchmark_spec_class.AZURE: + raise errors.Benchmarks.MissingObjectCredentialException( + 'Boto file cannot be found in %s but it is required for gcs or s3.', + FLAGS.boto_file_location) + + vms[0].RemoteCommand('sudo apt-get update') OBJECT_STORAGE_BENCHMARK_DICTIONARY[FLAGS.storage].Prepare(vms[0]) + # Prepare data on vm, create a run directory on scratch drive, and add # permission. scratch_dir = vms[0].GetScratchDir() vms[0].RemoteCommand('sudo mkdir %s/run/' % scratch_dir) vms[0].RemoteCommand('sudo chmod 777 %s/run/' % scratch_dir) + file_path = data.ResourcePath(DATA_FILE) vms[0].PushFile(file_path, '%s/run/' % scratch_dir) + api_test_script_path = data.ResourcePath(API_TEST_SCRIPT) + vms[0].PushFile(api_test_script_path, '%s/run/' % scratch_dir) + def Run(benchmark_spec): """Run storage benchmark and publish results. @@ -332,15 +583,15 @@ def Run(benchmark_spec): Total throughput in the form of tuple. The tuple contains the sample metric (string), value (float), unit (string). """ - value = 0.0 - unit = 'MB/sec' metadata = {'storage provider': FLAGS.storage} - results = [['storage upload', value, unit, metadata], - ['storage download', value, unit, metadata]] + vms = benchmark_spec.vms + + # The client tool based tests requires some provisioning on the VMs first. vms[0].RemoteCommand( 'cd %s/run/; bash cloud-storage-workload.sh' % vms[0].GetScratchDir()) - OBJECT_STORAGE_BENCHMARK_DICTIONARY[FLAGS.storage].Run(vms[0], results) + results = OBJECT_STORAGE_BENCHMARK_DICTIONARY[FLAGS.storage].Run(vms[0], + metadata) print results return results diff --git a/perfkitbenchmarker/data/__init__.py b/perfkitbenchmarker/data/__init__.py index 585c5a2466..1f0a25b797 100644 --- a/perfkitbenchmarker/data/__init__.py +++ b/perfkitbenchmarker/data/__init__.py @@ -15,8 +15,8 @@ Due to license restrictions, not all software dependences can be shipped with PerfKitBenchmarker. -Those that can be included in perfkitbenchmarker/data, and are loaded via a -PackageResourceLoader. +Those that can be included in perfkitbenchmarker/data, or + perfkitbenchmarker/scripts and are loaded via a PackageResourceLoader. Users can specify additional paths to search for required data files using the `--data_search_paths` flag. @@ -125,8 +125,10 @@ def ResourcePath(self, name): raise ResourceNotFound(name) return pkg_resources.resource_filename(self.package, name) - -DEFAULT_RESOURCE_LOADERS = [PackageResourceLoader(__name__)] +DATA_PACKAGE_NAME = 'perfkitbenchmarker.data' +SCRIPT_PACKAGE_NAME = 'perfkitbenchmarker.scripts' +DEFAULT_RESOURCE_LOADERS = [PackageResourceLoader(DATA_PACKAGE_NAME), + PackageResourceLoader(SCRIPT_PACKAGE_NAME)] def _GetResourceLoaders(): diff --git a/perfkitbenchmarker/data/cloud-storage-workload.sh b/perfkitbenchmarker/data/cloud-storage-workload.sh index a8c7aba98d..c5ec3e3278 100644 --- a/perfkitbenchmarker/data/cloud-storage-workload.sh +++ b/perfkitbenchmarker/data/cloud-storage-workload.sh @@ -15,14 +15,14 @@ # limitations under the License. # Prepare data for object_storage_benchmark and copy_benchmark. File size -# distribution simulates cloud storage workload. All files add up to ~2560MB. +# distribution simulates cloud storage workload. All files add up to ~256MB. mkdir data; cd data; -for i in {0..12}; do dd bs=10KB count=16 if=/dev/urandom of=file-$i.dat; done -for i in {13..26}; do dd bs=10KB count=32 if=/dev/urandom of=file-$i.dat; done -for i in {27..53}; do dd bs=10KB count=64 if=/dev/urandom of=file-$i.dat; done -for i in {54..62}; do dd bs=20KB count=64 if=/dev/urandom of=file-$i.dat; done -for i in {63..71}; do dd bs=80KB count=64 if=/dev/urandom of=file-$i.dat; done -for i in {72..89}; do dd bs=160KB count=64 if=/dev/urandom of=file-$i.dat; done -for i in {90..95}; do dd bs=10MB count=$((32-(99-i)*2)) if=/dev/urandom of=file-$i.dat; done -for i in {96..99}; do dd bs=10MB count=32 if=/dev/urandom of=file-$i.dat; done +for i in {0..12}; do dd bs=1KB count=16 if=/dev/urandom of=file-$i.dat; done +for i in {13..26}; do dd bs=1KB count=32 if=/dev/urandom of=file-$i.dat; done +for i in {27..53}; do dd bs=1KB count=64 if=/dev/urandom of=file-$i.dat; done +for i in {54..62}; do dd bs=2KB count=64 if=/dev/urandom of=file-$i.dat; done +for i in {63..71}; do dd bs=8KB count=64 if=/dev/urandom of=file-$i.dat; done +for i in {72..89}; do dd bs=16KB count=64 if=/dev/urandom of=file-$i.dat; done +for i in {90..95}; do dd bs=1MB count=$((32-(99-i)*2)) if=/dev/urandom of=file-$i.dat; done +for i in {96..99}; do dd bs=1MB count=32 if=/dev/urandom of=file-$i.dat; done diff --git a/perfkitbenchmarker/scripts/__init__.py b/perfkitbenchmarker/scripts/__init__.py new file mode 100644 index 0000000000..ff02c77b9c --- /dev/null +++ b/perfkitbenchmarker/scripts/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/perfkitbenchmarker/scripts/object_storage_api_tests.py b/perfkitbenchmarker/scripts/object_storage_api_tests.py index 330c87854e..1dc6b2f32b 100755 --- a/perfkitbenchmarker/scripts/object_storage_api_tests.py +++ b/perfkitbenchmarker/scripts/object_storage_api_tests.py @@ -94,10 +94,10 @@ def _PercentileCalculator(numbers): numbers_sorted = sorted(numbers) count = len(numbers_sorted) result = {} - result['tp50'] = numbers_sorted[int(count * 0.5)] - result['tp90'] = numbers_sorted[int(count * 0.9)] - result['tp99'] = numbers_sorted[int(count * 0.99)] - result['tp99.9'] = numbers_sorted[int(count * 0.999)] + result['p50'] = numbers_sorted[int(count * 0.5)] + result['p90'] = numbers_sorted[int(count * 0.9)] + result['p99'] = numbers_sorted[int(count * 0.99)] + result['p99.9'] = numbers_sorted[int(count * 0.999)] return result @@ -272,7 +272,7 @@ def ListConsistencyBenchmark(storage_schema, host_to_connect=None): thread.start() threads.append(thread) - logging.info('All threads started, waiting for them to end...') + logging.debug('All threads started, waiting for them to end...') for i in range(LIST_CONSISTENCY_THREAD_COUNT): try: @@ -280,7 +280,7 @@ def ListConsistencyBenchmark(storage_schema, host_to_connect=None): final_objects_written += per_thread_objects_written[i] except: logging.exception('Caught exception waiting for the %dth thread.', i) - logging.info('All threads ended...') + logging.debug('All threads ended...') write_finish_time = time.time() @@ -289,8 +289,8 @@ def ListConsistencyBenchmark(storage_schema, host_to_connect=None): raise LowAvailabilityError('Failed to provision required number of ' 'objects, exiting.') - logging.info('Done provisioning the objects, objects written %d. Now start ' - 'doing the lists...', final_count) + logging.debug('Done provisioning the objects, objects written %d. Now start ' + 'doing the lists...', final_count) # Now list this bucket under this prefix, compare the list results with # objects_written. If they are not the same, keep doing it until they @@ -318,21 +318,21 @@ def ListConsistencyBenchmark(storage_schema, host_to_connect=None): final_result = {} if result_consistent: - logging.info('Listed %d times until results are consistent.', list_count) + logging.debug('Listed %d times until results are consistent.', list_count) if list_count == 1: - logging.info('List latency is: %f' % list_latency) + logging.debug('List latency is: %f' % list_latency) final_result['is-list-consistent'] = True final_result['list-latency'] = list_latency else: - logging.info('List-after-write inconsistency window is: %f', - total_wait_time) + logging.debug('List-after-write inconsistency window is: %f', + total_wait_time) final_result['is-list-consistent'] = False final_result['inconsistency-window'] = total_wait_time else: - logging.info('Results are still inconsistent after waiting for the max ' - 'limit!') + logging.debug('Results are still inconsistent after waiting for the max ' + 'limit!') final_result['is-list-consistent'] = False final_result['inconsistency-window'] = LIST_CONSISTENCY_WAIT_TIME_LIMIT @@ -375,6 +375,8 @@ def Main(argv=sys.argv): list_inconsistency_window = [] inconsistent_list_count = 0.0 + logging.info('Running list consistency tests for %d iterations...' % + FLAGS.iterations) for _ in range(FLAGS.iterations): result = ListConsistencyBenchmark(storage_schema, host_to_connect) if result['is-list-consistent']: