Initial upload

Working copy of powershell scripts and vdbench workloads

Set-VdbenchSD.ps1

@@ -0,0 +1,38 @@
+###########################################################################
+$basedir = Split-Path -parent $MyInvocation.MyCommand.Definition 
+$workload_profiles = $(Get-Childitem $basedir\workloads\curves\*.cfg)
+$outputdir = $basedir + "\output"
+$vdbench = $basedir + "\vdbench50403\vdbench.bat"
+$timestamp=Get-Date -Format 'yyyyMMdd-HHmmss'   #20161008-101629
+#$rundir = ($outputdir + "\" + $run + "\")
+#md $rundir -ErrorAction SilentlyContinue
+###########################################################################
+$regex = "^(?=.*?\b'sd'\b)(?!'*'.)*$."
+Function Get-VdbenchSD () {
+	#TODO: Use Get-PSDrive provider property and select non-os drive to avoid wiping out OS
+	$DiskDrives = Get-WmiObject WIn32_DiskDrive | Select Index, DeviceID, Size | Sort Index
+	$SD = $(foreach ($Disk in $DiskDrives) {"sd=Disk$($Disk.Index),lun=$($Disk.DeviceID),openflags=directio,size=$([Math]::Floor($Disk.Size / 1GB))g"})
+	return $SD[1]
+}
+
+Function Set-VdbenchSD ($PathToFile, $StringToFind, $StringToReplace) {
+	(Get-Content $PathToFile) |
+	Foreach-Object {$_ -replace $StringToFind, $StringToReplace } |
+	Set-Content $PathToFile
+}
+
+$SD = Get-VdbenchSD
+
+Write-Verbose ""
+Write-Verbose "Setting Vdbench Workload Storage Device parameter"
+ForEach ($w in $workload_profiles){
+	$regex = "^sd.*?$"
+    write-verbose "$w.name"    
+	Set-VdbenchSD $w $regex $sd
+    gc $w | sls ",size=" | Write-Verbose
+    Write-Verbose "----------------------"
+	}
+
+
+
+#gc .\workloads\curves\*.cfg | sls ",size="

start-testrun.ps1

@@ -0,0 +1,144 @@
+[CmdletBinding()]
+    param (
+
+    [parameter(Mandatory=$True,HelpMessage="Enter the name of this testrun")]
+    [ValidateNotNullOrEmpty()]
+    [string]$Run
+    )
+
+clear-host
+
+function start-testrun {
+	[CmdletBinding()]
+	param (
+
+        [parameter(Mandatory=$True,HelpMessage="Enter the name of this testrun")]
+        [ValidateNotNullOrEmpty()]
+		[string]$Run
+
+	)
+    Begin {
+        function Get-Timestamp {
+            $timestamp=Get-Date -Format 'yyyyMMdd-HHmmss'   #20161008-101629
+        return $timestamp
+        }
+
+        function Set-PowerPlan {
+	    [CmdletBinding(SupportsShouldProcess = $True)]
+	    param (
+		    [ValidateSet("High performance", "Balanced", "Power saver")]
+		    [ValidateNotNullOrEmpty()]
+		    [string] $PreferredPlan = "High Performance"
+	    )
+
+	    Write-Verbose "Setting power plan to `"$PreferredPlan`""
+	    $guid = (Get-WmiObject -Class Win32_PowerPlan -Namespace root\cimv2\power -Filter "ElementName='$PreferredPlan'").InstanceID.ToString()
+	    $regex = [regex]"{(.*?)}$"
+	    $plan = $regex.Match($guid).groups[1].value
+
+	    powercfg -S $plan
+	    $Output = "Power plan set to "
+	    $Output += "`"" + ((Get-WmiObject -Class Win32_PowerPlan -Namespace root\cimv2\power -Filter "IsActive='$True'").ElementName) + "`""
+	    Write-Verbose $Output
+    }
+
+    Set-PowerPlan "High Performance"
+
+    <#
+    Runs whatever workloads (*.cfg) are in the workloads directory
+    #>
+    ###########################################################################
+    # Notes:
+    <# $DiskDrives = Get-WmiObject WIn32_DiskDrive | Select Index, DeviceID, Size | Sort Index
+    # foreach ($Disk in $DiskDrives) {"sd=Disk$($Disk.Index),lun=$($Disk.DeviceID),openflags=directio,size=$([Math]::Floor($Disk.Size / 1GB))g"}
+    # get-item *.cfg | ForEach {(get-content -path $_.FullName).Replace("6143g","2147g") | Set-Content -Path $_.FullName}
+    # 2147g
+    # Curve adjustment for HDD testing
+    # get-item *.cfg | ForEach {(get-content -path $_.FullName).Replace("(1,10,20,30,40,50,60,70,80,85,90,92,94,96,98)","(1,5,10,15,20,25,30,35,40,45,50,75)") | Set-Content -Path $_.FullName}
+    # ^(?=.*?\b<text_to_replace>\b)(?!#.)*$.
+    # (1,5,10,15,20,25,30,35,40,45,50,75)
+    # get-item *.cfg | ForEach {(get-content -path $_.FullName | Select-String "curve=")}
+    #>
+    <#
+    #rd=lblkseqiorun,wd=lblkseqio,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #rd=lblkseqiorun,wd=lblkseqio,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #rd=exchcurverun,wd=exchcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #rd=smallrun1,wd=small,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15
+    #rd=max4kcurverun,wd=max4kcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #rd=max4kcurverun,wd=max4kcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #rd=oltpcurverun,wd=oltpcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #*rd=myrd,wd=wd1,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #rd=sqlrun,wd=sqlcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #*rd=myrd,wd=wd1,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #rd=webcurverun,wd=webcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,75),interval=1,elapsed=60,warmup=15,threads=512
+    #>
+    ###########################################################################
+    # Setting up environment variables for test run
+    ###########################################################################
+    #$basedir = Split-Path -parent $MyInvocation.MyCommand.Definition 
+    $basedir = '.'
+    $workload_profiles = $(Get-Childitem $basedir\workloads\curves\*.cfg)
+    $outputdir = $basedir + "\output"
+    $vdbench = $basedir + "\vdbench50403\vdbench.bat"
+    #$timestamp=Get-Date -Format 'yyyyMMdd-HHmmss'   #20161008-101629
+    $timestamp = Get-Timestamp
+    $rundir = ($outputdir + "\" + $timestamp + '-'+$run)
+    Write-Verbose ""
+    Write-Verbose "Variables Used"
+    Write-Verbose "BaseDir = $basedir"
+    Write-Verbose "Outputdir = $outputdir"
+    Write-Verbose "Vdbench = $vdbench"
+    Write-Verbose "Timestamp = $timestamp"
+    Write-Verbose "Rundir = $rundir"
+    #
+    # Create Output Directory $rundir
+    if(!(test-path $rundir)){
+        try {
+            New-Item -Path $rundir -ItemType directory -Force | out-null
+        } catch [Exception] {
+            $_.Exception.GetType().FullName, $_.Exception.Message | Write-Warning
+            break
+        } finally {
+
+        }
+    } else {
+    Write-Warning $Rundr already exists
+    break
+    }
+    Write-Verbose ""
+    Write-Verbose "Calling .\Set-VdbenchSD.ps1"
+    .\Set-VdbenchSD.ps1
+
+    }
+
+    Process {
+
+        Foreach ($workload in Get-Childitem $workload_profiles){
+        $workload_name = [System.IO.Path]::GetFileNameWithoutExtension($workload)
+        $workloaddir = ($rundir + "\" + $workload_name)
+
+        $arguments = "-f $workload -o $workloaddir"
+        Write-Verbose ""
+        Write-Verbose "Starting Vdbench"
+        Write-Verbose "$vdbench $arguments -NoNewWindow -Wait"
+        Start-Process $vdbench $arguments -NoNewWindow -Wait
+
+    <#
+    TODO: format disk, wipe disk between runs
+    clear-disk
+    Reset-PhysicalDisk (if unhealthy)
+    Write-VolumeCache #enables you to forcibly empty, or flush, the write cache by writing it to disk
+    Update-HostStorageCache or VDS Rescan when PG groups change
+    #>
+    Write-Verbose "Sleeping for 60 seconds between workloads"
+    Start-Sleep 60
+    }
+
+    }
+
+    End {
+    Write-Verbose "Test Run Complete"
+    }
+}
+Write-Verbose "Starting Test Run"
+start-testrun -Run $Run

workloads/curves/512k_100_read_seq_512threads_curve.cfg

@@ -0,0 +1,13 @@
+********************************************************************************************
+*large block sequential io
+*TODO: Add random (seekpct) for large block io
+*wd=large,sd=Disk1,xfersize=52488,rdpct=100
+*rd=largerun,wd=large,iorate=100,elapsed=10,warmup=15,interval=1,threads=512
+********************************************************************************************
+* Single disk, 100% random read of 512k blocks
+* Adjust curve for HDD testing (1,5,10,15,20,25,30,35,40,45,50)
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=8191g
+wd=lblkseqread,sd=Disk1,xfersize=524288,rdpct=100
+rd=lblkseqreadrun,wd=lblkseqread,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,55,60,70,75),interval=1,elapsed=60,warmup=15,threads=512  
+
+

workloads/curves/512k_100_write_seq_512threads_curve.cfg

@@ -0,0 +1,12 @@
+********************************************************************************************
+*large block sequential io
+*TODO: Add random (seekpct) for large block io
+*wd=large,sd=Disk1,xfersize=52488,rdpct=100
+*rd=largerun,wd=large,iorate=100,elapsed=10,warmup=15,interval=1,threads=512
+********************************************************************************************
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=8191g
+wd=lblkseqwrite,sd=Disk1,xfersize=524288,rdpct=0
+rd=lblkseqwriterun,wd=lblkseqwrite,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,55,60,70,75),interval=1,elapsed=60,warmup=15,threads=512  
+
+
+

workloads/curves/exch_curve.cfg

@@ -0,0 +1,7 @@
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=8191g
+wd=exchcurve,sd=Disk1,xfersize=8192,rdpct=55,seekpct=80
+*rd=exchcurverun,wd=exchcurve,iorate=curve,curve=(75,76,77,78,79,80,81,82,83,84,85),interval=1,elapsed=60,warmup=15,threads=512
+rd=exchcurverun,wd=exchcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,55,60,70,75),interval=1,elapsed=60,threads=512
+
+
+

workloads/curves/max_4k_100_read_1thread_curve.cfg

@@ -0,0 +1,5 @@
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=8191g
+wd=small,sd=Disk1,xfersize=4096,rdpct=100,seekpct=80
+rd=smallrun1,wd=small,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,55,60,70,75),interval=1,elapsed=60,warmup=15
+
+

workloads/curves/max_4k_100_read_512threads_curve.cfg

@@ -0,0 +1,6 @@
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=8191g
+wd=max4kcurve,sd=Disk1,xfersize=4096,rdpct=100,seekpct=80
+rd=max4kcurverun,wd=max4kcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,55,60,70,75),interval=1,elapsed=60,warmup=15,threads=512  
+
+
+

workloads/curves/max_4k_100_write_512threads_curve.cfg

@@ -0,0 +1,6 @@
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=8191g
+wd=max4kwritecurve,sd=Disk1,xfersize=4096,rdpct=100,seekpct=80
+rd=max4kwritecurverun,wd=max4kwritecurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,55,60,70,75),interval=1,elapsed=60,warmup=15,threads=512  
+
+
+

workloads/curves/oltp_curve.cfg

@@ -0,0 +1,5 @@
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=8191g
+wd=oltpcurve,sd=Disk1,xfersize=8192,rdpct=70,seekpct=100
+rd=oltpcurverun,wd=oltpcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,55,60,70,75),interval=1,elapsed=60,warmup=15,threads=512
+
+

workloads/curves/parsing_curve_results.txt

@@ -0,0 +1,34 @@
+PS C:\Users\administrator.SETO\desktop\output\3\web_curve> gci logfile.html -Recurse | Select-String "starting rd","avg" | sls name -notmat
+
+logfile.html:45:02:59:18.002 Starting RD=webcurverun; I/O rate: Uncontrolled curve; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:133:03:00:33.018 avg_16-75    2298.15    17.95    8192  94.96  222.809  224.987  181.737 1048.310   65.379 512.0   7.0   4.3
+logfile.html:138:03:00:35.000 Starting RD=webcurverun_(1%); I/O rate: 30; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:226:03:01:50.011 avg_16-75      29.98     0.23    8192  95.39   10.061   10.520    0.580  133.462   11.862   0.3   4.6   1.1
+logfile.html:231:03:01:52.000 Starting RD=webcurverun_(10%); I/O rate: 230; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:319:03:03:07.009 avg_16-75     228.55     1.79    8192  95.12    8.702    9.118    0.592  162.483    8.189   2.0   2.0   0.9
+logfile.html:324:03:03:09.001 Starting RD=webcurverun_(20%); I/O rate: 460; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:412:03:04:24.008 avg_16-75     460.58     3.60    8192  94.81    7.800    8.194    0.597   81.897    4.322   3.6   2.8   1.6
+logfile.html:417:03:04:26.001 Starting RD=webcurverun_(30%); I/O rate: 690; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:505:03:05:41.009 avg_16-75     689.78     5.39    8192  94.84    8.976    9.432    0.600   96.916    5.394   6.2   5.0   2.6
+logfile.html:510:03:05:43.001 Starting RD=webcurverun_(40%); I/O rate: 920; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:598:03:06:58.010 avg_16-75     929.27     7.26    8192  95.04   10.142   10.638    0.627   99.065    6.432   9.4   4.3   2.6
+logfile.html:603:03:06:59.001 Starting RD=webcurverun_(50%); I/O rate: 1200; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:691:03:08:14.011 avg_16-75    1201.53     9.39    8192  95.24   11.928   12.495    0.587  130.592    8.488  14.3   4.6   2.6
+logfile.html:696:03:08:16.001 Starting RD=webcurverun_(60%); I/O rate: 1400; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:784:03:09:31.009 avg_16-75    1401.88    10.95    8192  94.97   13.810   14.507    0.670  174.195   10.741  19.4   5.0   3.2
+logfile.html:789:03:09:33.001 Starting RD=webcurverun_(70%); I/O rate: 1700; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:877:03:10:48.009 avg_16-75    1699.75    13.28    8192  94.95   17.365   18.251    0.704  227.419   15.106  29.5   6.2   4.1
+logfile.html:882:03:10:50.000 Starting RD=webcurverun_(80%); I/O rate: 1900; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:970:03:12:05.009 avg_16-75    1896.65    14.82    8192  95.03   21.307   22.378    0.800  318.443   20.371  40.4   8.3   4.5
+logfile.html:975:03:12:07.001 Starting RD=webcurverun_(85%); I/O rate: 2000; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:1063:03:13:22.009 avg_16-75    2005.65    15.67    8192  95.02   24.182   25.408    0.808  505.708   24.038  48.5   6.6   4.3
+logfile.html:1068:03:13:24.000 Starting RD=webcurverun_(90%); I/O rate: 2100; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:1156:03:14:39.008 avg_16-75    2094.38    16.36    8192  95.08   26.568   27.900    0.848  405.756   26.909  55.7   6.3   4.0
+logfile.html:1161:03:14:41.000 Starting RD=webcurverun_(92%); I/O rate: 2200; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:1249:03:15:56.009 avg_16-75    2203.43    17.21    8192  95.01   30.580   32.132    1.015  444.960   31.558  67.4   7.7   4.8
+logfile.html:1254:03:15:58.001 Starting RD=webcurverun_(94%); I/O rate: 2200; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:1342:03:17:13.009 avg_16-75    2203.47    17.21    8192  94.93   30.603   32.183    1.022  446.774   31.622  67.4   6.6   4.2
+logfile.html:1347:03:17:15.006 Starting RD=webcurverun_(96%); I/O rate: 2300; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:1435:03:18:30.009 avg_16-75    2292.47    17.91    8192  95.03   33.857   35.541    1.682  549.699   34.717  77.6   6.5   4.0
+logfile.html:1440:03:18:32.001 Starting RD=webcurverun_(98%); I/O rate: 2300; elapsed=60 warmup=15; For loops: threads=512
+logfile.html:1528:03:19:47.009 avg_16-75    2300.00    17.97    8192  94.94   35.267   37.033    2.120  549.729   35.663  81.1   6.0   4.0

workloads/curves/pool/512k_100_read_80_random_512threads_curve.cfg

@@ -0,0 +1,12 @@
+*Single disk, 100% random read of 8k records at i/o rate of 100 for 10 seconds
+*sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=2136g
+*wd=wd1,sd=Disk1,xfersize=8192,rdpct=55
+*rd=run1,wd=wd1,iorate=100,elapsed=10,interval=1
+*
+* Single disk, 100% random read of 4k blocks with 80% random
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=6143g
+wd=max512kcurve,sd=Disk1,xfersize=524288,rdpct=100,seekpct=80
+rd=max512kcurverun,wd=max4kcurve,iorate=curve,curve=(1,10,20,30,40,50,60,70,80,85,90,92,94,96,98),interval=1,elapsed=60,warmup=15,threads=512  
+
+
+

workloads/curves/pool/512k_100_read_seq_512threads_curve.cfg

@@ -0,0 +1,13 @@
+********************************************************************************************
+*large block sequential io
+*TODO: Add random (seekpct) for large block io
+*wd=large,sd=Disk1,xfersize=52488,rdpct=100
+*rd=largerun,wd=large,iorate=100,elapsed=10,warmup=15,interval=1,threads=512
+********************************************************************************************
+* Single disk, 100% random read of 512k blocks
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=6143g
+wd=lblkseqio,sd=Disk1,xfersize=524288,rdpct=100
+rd=lblkseqiorun,wd=lblkseqio,iorate=curve,curve=(1,10,20,30,40,50,60,70,80,85,90,92,94,96,98),interval=1,elapsed=60,warmup=15,threads=512  
+
+
+

workloads/curves/readme.txt

@@ -0,0 +1,10 @@
+# Vdbench Curve Runs
+Identify X_IOPS at Y_Latency
+First run idenfies max io
+Subsequent runs, run at the % listed in the curve=() param
+e.g. curve=(25,50,75)
+will tell Vdbench to first establish the maximum iops for the workload
+then run the same workload at 25%, 50%, and 75% of that max
+* use 512 threads and Vdbench will only consume what it needs
+iorate=curve,curve=(1,10,20,30,40,50,60,70,80,85,90,92,94,96,98),interval=1,elapsed=60,warmup=15,threads=512
+

workloads/curves/sql_curve.cfg

@@ -0,0 +1,6 @@
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=8191g
+wd=sqlcurve,sd=Disk1,xfersize=65536,rdpct=66,seekpct=100
+rd=sqlrun,wd=sqlcurve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,55,60,70,75),interval=1,elapsed=60,warmup=15,threads=512
+
+
+

workloads/curves/web_curve.cfg

@@ -0,0 +1,5 @@
+sd=Disk1,lun=\\.\PHYSICALDRIVE1,openflags=directio,size=8191g
+wd=webcurve,sd=Disk1,xfersize=8192,rdpct=95,seekpct=75
+rd=webcurverun,wd=webcurve,iorate=curve,iorate=curve,curve=(1,5,10,15,20,25,30,35,40,45,50,55,60,70,75),interval=1,elapsed=60,threads=512
+
+