Vdbench_1201

Added scripts to parse and merge all Vdbench results into an Excel
workbook with the Max BW (while under 20ms latency) highlighted for each
run

Convert_all_vdbench_flatfiles_to_csv.ps1

@@ -0,0 +1,129 @@
+[CmdletBinding()]
+    param (        
+    [parameter(Mandatory=$false,HelpMessage="Enter the name of this Testrun")]
+    [ValidateNotNullOrEmpty()]
+    [string]$Run
+    )
+<#
+    # This function converts flatfile.html into a Powershell CSV Object
+    # .Example1
+    # $CsvObj = (convert-VdbenchFlatfiletoCsv -File \path\to\flatfile.html)
+    #
+    # .Example2
+    # Using Pipeline
+    # $CSVObj = (get-content \path\to\flatfile.html).pspath | convert-VdbenchFlatfiletoCsv
+    #
+    # .Example3
+    # Query CSVObj
+    # $CSVObj | Where-Object {($_.resp -gt "19") -and ($_.resp -lt "20")} | export-csv -path .\flatfile.csv
+    #
+    # Next Steps: 
+    # Query CSV Object
+    # Export CSV Object to a file
+    # Convert all Vdbench flat.html's to CSV
+    # Merge all CSV files into a single Excel Workbook .xlsx
+    # Query data in Excel using Excel GUI, Macros, or Powershell
+    #> 
+    #
+##
+# Load module to convert flatfile.html > flatfile.csv
+#. .\convert-VdbenchFlatfiletoCsv.ps1
+#
+#
+function convert-VdbenchFlatfiletoCsv {
+	[CmdletBinding()]
+	param(
+		#[Parameter(Mandatory=$True,Position=1,ValueFromPipelineByPropertyName =$True)][string[]]$HTMLFlatFile
+        [Parameter(Mandatory=$True)][string[]]$File
+	)
+<#
+    # This function converts flatfile.html into a Powershell CSV Object
+    # .Example1
+    # $CsvObj = (convert-VdbenchFlatfiletoCsv -File \path\to\flatfile.html)
+    #
+    # .Example2
+    # Using Pipeline
+    # $CSVObj = (get-content \path\to\flatfile.html).pspath | convert-VdbenchFlatfiletoCsv
+    #
+    # .Example3
+    # Query CSVObj
+    # $CSVObj | Where-Object {($_.resp -gt "19") -and ($_.resp -lt "20")} | export-csv -path .\flatfile.csv
+    #
+    # Next Steps: 
+    # Query CSV Object
+    # Export CSV Object to a file
+    # Convert all Vdbench flat.html's to CSV
+    # Merge all CSV files into a single Excel Workbook .xlsx
+    # Query data in Excel using Excel GUI, Macros, or Powershell
+    #> 
+    #
+    # Logic expecting a single flatfile.html
+    $FlatFile = (Get-Item $File)
+    $NewFlatFile = ".\newflatfile.txt"
+    $Content = $Null
+    $NewContent = $Null
+    $NewCsv = ".\new.csv"
+
+    #Fetch the contents
+    $Content = (get-content $flatfile) -match '^[0-9]'
+
+    #Replace at least two spaces by a single tab:
+    $NewContent = $Content -replace ' {1,}', "`t"
+
+    #Write back to a file
+    $NewContent | Set-Content $NewFlatFile
+
+    #notepad $NewFlatFile
+    $CsvHeader = "date","Run","Interval","reqrate","rate","MB/sec","bytes/io","read%","resp","read_resp","write_resp","resp_max","resp_std","xfersize","threads","rdpct","rhpct","whpct","seekpct","lunsize","version","compratio","dedupratio","queue_depth","cpu_used","cpu_user","cpu_kernel","cpu_wait","cpu_idle"
+    $CsvObj = (Import-Csv -Delimiter "`t" -Path $Newflatfile -Header $CsvHeader)
+
+    #$CsvObj | Export-Csv -Path $NewCsv -NoTypeInformation
+    return $CsvObj
+
+}
+#
+#
+#
+Write-Verbose ""
+Write-Verbose "Setting Variables"
+#
+$BaseDir = "."
+$Output = "$BaseDir\output"
+#$TestRun = (Get-ChildItem $Results)
+$File = "flatfile.html"
+$Flatfiles = (Get-ChildItem $File -Recurse)
+#
+Write-Verbose "BaseDir = $BaseDir"
+Write-Verbose "OutputDir = $Output"
+Write-Verbose "OutputCSVDir = $OutputCSV"
+
+## Get Flatfiles, convert flatfile to csv
+$Flatfiles | foreach{Write-Verbose $_.FullName}
+
+## Process
+Write-Verbose ""
+write-verbose "Beginning Conversion"
+# Find all flatfile.html files recursively from $basedir
+# Convert each flatfile.html to flatfile.csv
+# Save flatfile.csv in same directory as originating flatfile.html
+#
+foreach($ffile in $flatfiles){
+    $OutFile = $ffile.fullname -replace ".html",".csv"
+    Write-Verbose $OutFile
+    # $CsvObj = (convert-VdbenchFlatfiletoCsv -File .\flatfile.html)    
+    $CSVObj = convert-VdbenchFlatfiletoCsv -File $ffile
+    ## Filter results (If desired or later in excel)
+    # (e.g. Less than 20 ms latency)   
+    # Query CSV Object and output to csv $OutFile
+    # $CSVObj | Where-Object {($_.resp -gt "19") -and ($_.resp -lt "20")} | export-csv -path $OutFile
+    $CSVObj | export-csv -NoTypeInformation -Path $Outfile
+}
+
+# Next Step:
+# Merge all CSVs into a single Excel workbook
+# $get-content $csv | Export-Excel .\test.xlsx -WorkSheetname csv2 -AutoSize -Show
+
+
+
+
+

Create-VdbenchResultsXLSX.ps1

@@ -0,0 +1,141 @@
+function Merge-vdbenchCSV {
+   [cmdletbinding(SupportsShouldProcess)]
+        param(
+            [Parameter(Mandatory=$false)][String]$Source = ".\output",
+            [Parameter(Mandatory=$false)][String]$Destination=".\results.xlsx"
+            #[Parameter(Mandatory=$false)][Switch]$Show
+    )
+
+    <#
+
+    .SYNOPSIS
+    Merge CSV created by Convert-VdbenchFlatfiletoCsv.ps1 into an Excel Workbook
+
+    .DESCRIPTION
+    Create a new Excel workbook
+    Import each Workload results CSV file as a new worksheet
+    Identify the maximum bandwidth while under 20ms 
+
+    Loops through vdbench flatfiles in a single directory that have been converted to csv by the 
+    convert-VdbenchFlatfiletoCsv -File
+    Merges all CSV files found in $path
+    converts to .xlsx in the output directory
+
+    .EXAMPLE
+    Merge-vdbenchCSV
+
+    .EXAMPLE
+    Merge-vdbenchCSV -whatif
+
+    .EXAMPLE
+    Merge-vdbenchCSV -Verbose
+
+    .EXAMPLE
+    Merge-vdbenchCSV -Source .\output -Destination .\allresults.xlsx -WhatIf
+
+    .EXAMPLE
+    Merge-vdbenchCSV -Source .\output -Destination .\allresults.xlsx -Verbose
+
+    .PARAMETER Source
+    The computer name to query. Just one.
+
+    .PARAMETER Destination
+    The name and path to the excel file to write to.
+
+    #>
+
+    Begin {
+        # Check if module is installed, 
+        # if installed, load module
+        # if not, install module, then load module
+        # Load Modules
+        #. .\ConvertCSV-ToExcel.ps1
+        # Get-InstalledModule importexcel
+        if(!(Get-InstalledModule importexcel)) {
+            Try {
+            Install-Module importexcel
+            Import-Module importexcel
+            } catch {
+                $_.Exception.GetType().FullName, $_.Exception.Message | Write-Warning
+                Write-Warning "Automatic installation of the importexcel module failed"
+                Write-Warning "Please install the importexcel module to continue"
+                Write-Warning "https://www.powershellgallery.com/packages/ImportExcel/2.2.9"
+                Write-Warnning "Install-Module -Name ImportExcel"
+                break
+            }
+        } else {
+            Try {
+            Import-Module importexcel
+            } catch {
+                $_.Exception.GetType().FullName, $_.Exception.Message | Write-Warning
+                Write-Warning "Automatic installation of the importexcel module failed"
+                Write-Warning "Please install the importexcel module to continue"
+                Write-Warning "https://www.powershellgallery.com/packages/ImportExcel/2.2.9"
+                Write-Warnning "Install-Module -Name ImportExcel"
+                break
+            }
+        }
+        #
+        $BaseDir = "."
+        $Output = "$BaseDir\output"
+        $File = "flatfile.csv"
+        $Workbook = $Destination
+        #
+        function Get-VdbenchCSV {
+            #$CSVfiles = (Get-ChildItem "$output\$File" -Recurse)
+            $CSVFiles = Get-ChildItem "$output\$File" -Recurse | 
+            sort @{Expression={($_.fullname -split '\\')[-2]}; Ascending=$false},@{Expression={($_.fullname -split '\\')[-3]}; Ascending=$true} | Select-Object fullname
+            $CSVCount=$CSVfiles.Count
+            Write-Verbose “Detected ($CSVCount) CSV files”
+            return $CSVFiles
+        }
+        $CSVFiles = Get-VdbenchCSV
+    }
+
+    Process {
+
+        Write-Verbose "Creating: $Workbook"
+
+        ## PROCESS
+
+        $CSVfiles | foreach{
+
+            #$CsvHeader = "date","Run","Interval","reqrate","rate","MB/sec","bytes/io","read%","resp","read_resp","write_resp","resp_max","resp_std","xfersize","threads","rdpct","rhpct","whpct","seekpct","lunsize","version","compratio","dedupratio","queue_depth","cpu_used","cpu_user","cpu_kernel","cpu_wait","cpu_idle"
+            $CsvObj = (Import-Csv -Delimiter "," -Path $_.fullname)
+
+            $workload = (($_.fullname -split '\\')[-2])
+            $sd = ($_.fullname -split '\\')[-3]
+            $worksheetName = (($workload+$sd) -replace "thread_curve|threads_curve","t_")
+
+            #Write-Verbose "Adding $Workload to $Workbook"
+
+            If ($Pscmdlet.ShouldProcess("$SD-$Workload","Merge")) {
+                # Filtering out names longer than 31 characters for Excel worksheet name limit
+                If($worksheetName.length -lt "31" ){
+                    #Write-Host "adding $worksheetname"
+                    #Write-Host "get-content $_.fullname | Export-Excel $Workbook -WorkSheetname $worksheetName -AutoSize"
+                    # TODO: Find Max value of CsvObj rate colum where latency -lt 20
+
+                    $MAX = ($CSVObj | Where-Object {$_.resp -lt "20" }  | Sort-Object  -Property "MB/sec" -Descending)[0]
+
+                    $CsvObj | 
+                    Export-Excel $Workbook -WorkSheetname $worksheetName -BoldTopRow -AutoFilter -FreezeTopRow -AutoSize -ConditionalText $( New-ConditionalText $max.date blue cyan )
+                    #(Col_X = QueueDepth)
+                    #(Col_E = Rate)
+                    #(Col_F = MB/s)
+                    #(Col_I = Latency(resp))
+                }else{
+                    write-host "skipping $worksheetname"
+                }
+            }
+        }
+    }
+}
+
+# Example Commands (Uncomment to Test)
+#Merge-vdbenchCSV
+#Merge-vdbenchCSV -whatif
+Merge-vdbenchCSV -Verbose
+#Merge-vdbenchCSV -Source .\output -Destination .\allresults.xlsx -WhatIf
+#Merge-vdbenchCSV -Source .\output -Destination .\allresults.xlsx -Verbose
+

README.txt

@@ -0,0 +1,79 @@
+
+
+1.Retrieve/Download Vdbench and PreRequisites Software
+(or copy testing package from /se/temp/
+2.Install Vdbench and PreRequisite softare
+(or unzip testing package)
+install java
+3.Configure Testing Environment
+
+4.Execute Tests
+5.Report Results
+
+
+1.Retrieve/Download Vdbench and PreRequisites Software
+Java
+Vdbench
+Workload definitions and scripts
+
+2.Install Vdbench and PreRequisites
+Java
+Vdbench
+Validation package
+- Vdbench Workload definitions
+- Scripts
+
+3.Configure Testing Environment
+Set-VDBenchSD.ps1
+
+4.Execute Tests
+Start-TestRun.ps1
+
+5.Report Results
+(Collect/Query/Transform/View/Report Results)
+
+5.1 Create-VdbenchExcel.ps1
+5.1.1 ./convert-VdbenchFlatfiletoCsv.ps1
+5.1.1 ./convert_all_vdbench_flatfiles_to_csv.ps1
+5.1.1. ./merge_vdbench_csv_to_excel.ps1
+
+
+
+
+For each workload, Convert Flatfile.html to Flatfile.csv
+Run Convert_all_vdbench_flatfiles_to_csv.ps1
+Merge all flatfile.csv into single excel spreadsheet
+Merge all CSV to XLS
+
+
+
+
+
+Requirements:
+	1. Powershell v.X
+	2. Import-Excel module required for output to Excel
+	3. Vdbench 12-01 package required on test machine
+
+To use:
+	1. Provision storage to Test VM
+	2. Copy Vdbench package to Test VM
+	3. Simply drop any workloads from ./Workload/Templates/ into ./Workload/Run/ 
+	./Workloads/Run directory will be ran
+	./Workloads/Templates houses Vdbench workload templates
+
+	4. Set-VdbenchSD.ps1	# Sets the storage device to be used in the Vdbench workloads in ./Workloads/Run
+	5. Start-TestRun.ps1	# Starts tests configured in ./Workloads/Run
+	6. Convert_all_vdbench_flatfiles_to_csv.ps1 < 6b. Convert-Vdbenchflatfiletocsv.ps1
+	7. csvmerge.ps1/merge-VdbenchCSV
+	merges CSVs into a single excel document
+	provides a filter point for data
+    Applies conditional formatting, pivot tables, and charts to Excel worksheet
+
+	Example: 
+	Identify Max Bandwidth while under 20ms latency
+    Worksheet containing Rollup of all max data from all runs
+
+
+
+
+

Set-VdbenchSD.ps1

@@ -34,5 +34,5 @@ ForEach ($w in $workload_profiles){
 	}
 
 
-
-#gc .\workloads\curves\*.cfg | sls ",size="
+# Run the following command to validate the change was made
+#Get-Content .\workloads\curves\*.cfg | sls ",size="

start-testrun.ps1 → Start-VdbenchTestRun.ps1

@@ -115,7 +115,7 @@ function start-testrun {
 
         Foreach ($workload in Get-Childitem $workload_profiles){
         $workload_name = [System.IO.Path]::GetFileNameWithoutExtension($workload)
-        $workloaddir = ($rundir + "\" + $workload_name)
+        $workloaddir = ($outputdir + "\" + $run + "\" + $workload_name)
 
         $arguments = "-f $workload -o $workloaddir"
         Write-Verbose ""