Skip to content

Commit

Permalink
kram - scripts, update mem dump
Browse files Browse the repository at this point in the history
  • Loading branch information
alecazam committed Nov 5, 2023
1 parent cf071f7 commit 65e3d53
Show file tree
Hide file tree
Showing 3 changed files with 153 additions and 27 deletions.
64 changes: 37 additions & 27 deletions scripts/GpuMemDumpPerfetto.py
100644 → 100755
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#!/usr/bin/env python3

# kram - Copyright 2020-2023 by Alec Miller. - MIT License
# The license and copyright notice shall be included
# in all copies or substantial portions of the Software.
Expand Down Expand Up @@ -33,6 +35,9 @@

PROGRAM_VERSION = 'Vulkan/D3D12 Memory Allocator Dump Perfetto 3.0.3'

# TODO: Perfetto can't handle empty string for name on the allocs
# so had to set them to 'M'

# dx12 or vulkan
currentApi = ""

Expand All @@ -41,8 +46,8 @@

# now convert the dictonaries to new dictionaries, and then out to json
perfettoDict = {
'displayTimeUnit': 'ms', # TODO: /1000, not quite kb
'systemTraceEvents': 'systemTraceEvents',
'displayTimeUnit': 'ns', # TODO: /1000, not quite kb, ns or ms only
'systemTraceEvents': 'SystemTraceData',
'traceEvents': [],
}

Expand Down Expand Up @@ -72,7 +77,8 @@ def ProcessBlock(poolData, block):
for alloc in block[1]['Suballocations']:
allocData = {'Type': alloc['Type'],
'Size': int(alloc['Size']),
'Usage': int(alloc['Usage']) if 'Usage' in alloc else 0 }
'Usage': int(alloc['Usage']) if 'Usage' in alloc else 0,
'Name': alloc['Name'] if 'Name' in alloc else 'M' }
blockInfo['Suballocations'].append(allocData)
poolData['Blocks'].append(blockInfo)

Expand Down Expand Up @@ -120,7 +126,10 @@ def AllocTypeToCategory(type, usage):
if type == 'BUFFER':
if (usage & 0x1C0) != 0: # INDIRECT_BUFFER | VERTEX_BUFFER | INDEX_BUFFER
isVB = True # TODO: split up
return ifelse(isVB, "VB", "IB")
if isVB:
return "VB"
else:
return "IB"
elif (usage & 0x28) != 0: # STORAGE_BUFFER | STORAGE_TEXEL_BUFFER
return "SB"
elif (usage & 0x14) != 0: # UNIFORM_BUFFER | UNIFORM_TEXEL_BUFFER
Expand Down Expand Up @@ -165,21 +174,16 @@ def AddTraceEventsAlloc(alloc, address, blockCounter):
size = alloc['Size']
type = AllocTypeToCategory(alloc['Type'], alloc['Usage'])

# begin/end events for Perfetto, address is treated as a time value for plotting rects
perfettoDict['traceEvents'].add({
'name': alloc['Name'], # TODO: conditionally add only if name present
'ph': 'B',
# complete event is much less data than B/E
perfettoDict['traceEvents'].append({
'name': alloc['Name'],
'ph': 'X',
'ts': int(address),
'dur': int(size),
'tid': int(blockCounter),
#'pid': 0,
'cat': type
})
perfettoDict['traceEvents'].add(
{
'ph': 'E',
'ts': int(address+size),
'tid': int(blockCounter),
'cat': type # TODO: needed if begin already has it spec'd?
})

def AddTraceEventsBlock(block, blockCounter):
global perfettoDict
Expand All @@ -188,14 +192,15 @@ def AddTraceEventsBlock(block, blockCounter):
AddTraceEventsAlloc(alloc, address, blockCounter)
address += alloc['Size']

def AddBlockName(blockCounter, blockName):
def AddBlockName(blockName, blockCounter):
global perfettoDict

# TODO: are these just loose added into the object ?
perfettoDict.add({
# TODO: would be nice to add at top, rather than intermingled
perfettoDict['traceEvents'].append({
'name': 'thread_name',
'ph': 'M',
'tid': int(blockCounter),
#'pid': 0,
'args': {
'name': blockName
}
Expand All @@ -213,39 +218,39 @@ def AddTraceEvents():
blockIndex = 0
for block in poolData['Blocks']:
blockName = "p{} block{} {}".format(poolIndex, blockIndex, block['ID'])
AddTraceEventsBlock(block, blockCounter)
AddBlockName(blockName, blockCounter)
AddTraceEventsBlock(block, blockCounter)
blockCounter += 1
blockIndex += 1

# dedicated allocs
allocationIndex = 0
for dedicatedAlloc in poolData['DedicatedAllocations']:
blockName = 'p{} alloc{} {}'.format(poolIndex, allocationIndex, dedicatedAlloc['Name'])
AddTraceEventsAlloc(dedicatedAlloc, 0, blockCounter)
AddBlockName(blockName, blockCounter)
AddTraceEventsAlloc(dedicatedAlloc, 0, blockCounter)
blockCounter += 1
allocationIndex += 1

# repeat for custom pools
for customPoolData in poolData['CustomPools'].values():
customPoolName = customPoolData['Name']
customPoolName = "" # TODO: hook up, but it's not a field customPoolData['Name']

# pool block allocs
blockIndex = 0
for block in customPoolData['Blocks']:
blockName = 'p{} {} block{} {}'.format(poolIndex, customPoolName, blockIndex, block['ID'])
AddTraceEventsBlock(block, blockCounter)
AddBlockName(blockName, blockCounter)
AddTraceEventsBlock(block, blockCounter)
blockCounter += 1
blockIndex += 1

# pool dedicated allocs
allocationIndex = 0
for dedicatedAlloc in customPoolData['DedicatedAllocations']:
blockName = 'p{} {} alloc{} {}'.format(poolIndex, customPoolName, dedicatedAlloc['Name'])
AddTraceEventsAlloc(dedicatedAlloc, 0, blockCounter)
blockName = 'p{} {} alloc{} {}'.format(poolIndex, customPoolName, allocationIndex, dedicatedAlloc['Name'])
AddBlockName(blockName, blockCounter)
AddTraceEventsAlloc(dedicatedAlloc, 0, blockCounter)
blockCounter += 1
allocationIndex += 1

Expand All @@ -269,7 +274,8 @@ def AddTraceEvents():
for dedicatedAlloc in memoryPool[1]['DedicatedAllocations']:
allocData = {'Type': dedicatedAlloc['Type'],
'Size': int(dedicatedAlloc['Size']),
'Usage': int(dedicatedAlloc['Usage'])}
'Usage': int(dedicatedAlloc['Usage']),
'Name': dedicatedAlloc['Name'] if 'Name' in dedicatedAlloc else 'M'}
poolData['DedicatedAllocations'].append(allocData)
# Get allocations in block vectors
for block in memoryPool[1]['Blocks'].items():
Expand All @@ -286,7 +292,8 @@ def AddTraceEvents():
for dedicatedAlloc in pool['DedicatedAllocations']:
allocData = {'Type': dedicatedAlloc['Type'],
'Size': int(dedicatedAlloc['Size']),
'Usage': int(dedicatedAlloc['Usage'])}
'Usage': int(dedicatedAlloc['Usage']),
'Name': dedicatedAlloc['Name'] if 'Name' in dedicatedAlloc else 'M'}
poolData['CustomPools'][poolName]['DedicatedAllocations'].append(allocData)
# Get allocations in block vectors
for block in pool['Blocks'].items():
Expand All @@ -300,7 +307,8 @@ def AddTraceEvents():

AddTraceEvents()

perfettoJson = json.dumps(perfettoDict, indent=4)
# perfettoJson = json.dumps(perfettoDict, indent=4)
perfettoJson = json.dumps(perfettoDict, indent=0)

with open(args.output, "w") as outfile:
outfile.write(perfettoJson)
Expand All @@ -311,6 +319,8 @@ def AddTraceEvents():
- Fixed key 'Type'. Value is string.
- Fixed key 'Size'. Value is int.
- Fixed key 'Usage'. Value is int.
- Key 'Name' optional, Value is string
- Key'CustomData' optional
- Fixed key 'Blocks'. Value is list of objects, each containing dictionary with:
- Fixed key 'ID'. Value is int.
- Fixed key 'Size'. Value is int.
Expand Down
3 changes: 3 additions & 0 deletions scripts/GpuMemDumpVis.py
100644 → 100755
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#!/usr/bin/env python3
#
# Copyright (c) 2018-2023 Advanced Micro Devices, Inc. All rights reserved.
#
Expand Down Expand Up @@ -323,6 +324,8 @@ def DrawDedicatedAllocationBlock(draw, y, dedicatedAlloc, pixelsPerByte):
- Fixed key 'Type'. Value is string.
- Fixed key 'Size'. Value is int.
- Fixed key 'Usage'. Value is int.
- Key 'Name' optional, Value is string
- Key'CustomData' optional
- Fixed key 'Blocks'. Value is list of objects, each containing dictionary with:
- Fixed key 'ID'. Value is int.
- Fixed key 'Size'. Value is int.
Expand Down
113 changes: 113 additions & 0 deletions scripts/open_trace_in_ui.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# taken from here, to open a .trace file in Perfetto in the browser. Runs
# an http server at a port, and then browser can read the file from that
# since fileURL are often blocked. Use a system call to run this, and it
# will change the working directory to that of the file.
# from https://github.com/google/perfetto/blob/master/tools/open_trace_in_ui

import argparse
import http.server
import os
import socketserver
import sys
import webbrowser


class ANSI:
END = '\033[0m'
BOLD = '\033[1m'
RED = '\033[91m'
BLACK = '\033[30m'
BLUE = '\033[94m'
BG_YELLOW = '\033[43m'
BG_BLUE = '\033[44m'


# HTTP Server used to open the trace in the browser.
class HttpHandler(http.server.SimpleHTTPRequestHandler):

def end_headers(self):
self.send_header('Access-Control-Allow-Origin', self.server.allow_origin)
self.send_header('Cache-Control', 'no-cache')
super().end_headers()

def do_GET(self):
if self.path != '/' + self.server.expected_fname:
self.send_error(404, "File not found")
return

self.server.fname_get_completed = True
super().do_GET()

def do_POST(self):
self.send_error(404, "File not found")


def prt(msg, colors=ANSI.END):
print(colors + msg + ANSI.END)


def open_trace(path, open_browser, origin):
# We reuse the HTTP+RPC port because it's the only one allowed by the CSP.
PORT = 9001
path = os.path.abspath(path)
os.chdir(os.path.dirname(path))
fname = os.path.basename(path)
socketserver.TCPServer.allow_reuse_address = True
with socketserver.TCPServer(('127.0.0.1', PORT), HttpHandler) as httpd:
address = f'{origin}/#!/?url=http://127.0.0.1:{PORT}/{fname}'
if open_browser:
webbrowser.open_new_tab(address)
else:
print(f'Open URL in browser: {address}')

httpd.expected_fname = fname
httpd.fname_get_completed = None
httpd.allow_origin = origin
while httpd.fname_get_completed is None:
httpd.handle_request()


def main():
examples = '\n'.join(
[ANSI.BOLD + 'Usage:' + ANSI.END, ' -i path/trace_file_name [-n]'])
parser = argparse.ArgumentParser(
epilog=examples, formatter_class=argparse.RawTextHelpFormatter)

help = 'Input trace filename'
parser.add_argument('-i', '--trace', help=help)
parser.add_argument(
'-n', '--no-open-browser', action='store_true', default=False)
parser.add_argument('--origin', default='https://ui.perfetto.dev')

args = parser.parse_args()
trace_file = args.trace
open_browser = not args.no_open_browser

if trace_file is None:
prt('Please specify trace file name with -i/--trace argument', ANSI.RED)
sys.exit(1)
elif not os.path.exists(trace_file):
prt('%s not found ' % trace_file, ANSI.RED)
sys.exit(1)

prt('Opening the trace (%s) in the browser' % trace_file)
open_trace(trace_file, open_browser, args.origin)


if __name__ == '__main__':
sys.exit(main())

0 comments on commit 65e3d53

Please sign in to comment.