From 65e3d532a32ebd2b145513864fb1b7d1a85bf6d1 Mon Sep 17 00:00:00 2001 From: Alec Miller Date: Sun, 5 Nov 2023 13:13:02 -0800 Subject: [PATCH] kram - scripts, update mem dump --- scripts/GpuMemDumpPerfetto.py | 64 +++++++++++-------- scripts/GpuMemDumpVis.py | 3 + scripts/open_trace_in_ui.py | 113 ++++++++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 27 deletions(-) mode change 100644 => 100755 scripts/GpuMemDumpPerfetto.py mode change 100644 => 100755 scripts/GpuMemDumpVis.py create mode 100644 scripts/open_trace_in_ui.py diff --git a/scripts/GpuMemDumpPerfetto.py b/scripts/GpuMemDumpPerfetto.py old mode 100644 new mode 100755 index b467c8e5..43128c9a --- a/scripts/GpuMemDumpPerfetto.py +++ b/scripts/GpuMemDumpPerfetto.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + # kram - Copyright 2020-2023 by Alec Miller. - MIT License # The license and copyright notice shall be included # in all copies or substantial portions of the Software. @@ -33,6 +35,9 @@ PROGRAM_VERSION = 'Vulkan/D3D12 Memory Allocator Dump Perfetto 3.0.3' +# TODO: Perfetto can't handle empty string for name on the allocs +# so had to set them to 'M' + # dx12 or vulkan currentApi = "" @@ -41,8 +46,8 @@ # now convert the dictonaries to new dictionaries, and then out to json perfettoDict = { - 'displayTimeUnit': 'ms', # TODO: /1000, not quite kb - 'systemTraceEvents': 'systemTraceEvents', + 'displayTimeUnit': 'ns', # TODO: /1000, not quite kb, ns or ms only + 'systemTraceEvents': 'SystemTraceData', 'traceEvents': [], } @@ -72,7 +77,8 @@ def ProcessBlock(poolData, block): for alloc in block[1]['Suballocations']: allocData = {'Type': alloc['Type'], 'Size': int(alloc['Size']), - 'Usage': int(alloc['Usage']) if 'Usage' in alloc else 0 } + 'Usage': int(alloc['Usage']) if 'Usage' in alloc else 0, + 'Name': alloc['Name'] if 'Name' in alloc else 'M' } blockInfo['Suballocations'].append(allocData) poolData['Blocks'].append(blockInfo) @@ -120,7 +126,10 @@ def AllocTypeToCategory(type, usage): if type == 'BUFFER': if (usage & 0x1C0) != 0: # INDIRECT_BUFFER | VERTEX_BUFFER | INDEX_BUFFER isVB = True # TODO: split up - return ifelse(isVB, "VB", "IB") + if isVB: + return "VB" + else: + return "IB" elif (usage & 0x28) != 0: # STORAGE_BUFFER | STORAGE_TEXEL_BUFFER return "SB" elif (usage & 0x14) != 0: # UNIFORM_BUFFER | UNIFORM_TEXEL_BUFFER @@ -165,21 +174,16 @@ def AddTraceEventsAlloc(alloc, address, blockCounter): size = alloc['Size'] type = AllocTypeToCategory(alloc['Type'], alloc['Usage']) - # begin/end events for Perfetto, address is treated as a time value for plotting rects - perfettoDict['traceEvents'].add({ - 'name': alloc['Name'], # TODO: conditionally add only if name present - 'ph': 'B', + # complete event is much less data than B/E + perfettoDict['traceEvents'].append({ + 'name': alloc['Name'], + 'ph': 'X', 'ts': int(address), + 'dur': int(size), 'tid': int(blockCounter), + #'pid': 0, 'cat': type }) - perfettoDict['traceEvents'].add( - { - 'ph': 'E', - 'ts': int(address+size), - 'tid': int(blockCounter), - 'cat': type # TODO: needed if begin already has it spec'd? - }) def AddTraceEventsBlock(block, blockCounter): global perfettoDict @@ -188,14 +192,15 @@ def AddTraceEventsBlock(block, blockCounter): AddTraceEventsAlloc(alloc, address, blockCounter) address += alloc['Size'] -def AddBlockName(blockCounter, blockName): +def AddBlockName(blockName, blockCounter): global perfettoDict - # TODO: are these just loose added into the object ? - perfettoDict.add({ + # TODO: would be nice to add at top, rather than intermingled + perfettoDict['traceEvents'].append({ 'name': 'thread_name', 'ph': 'M', 'tid': int(blockCounter), + #'pid': 0, 'args': { 'name': blockName } @@ -213,8 +218,8 @@ def AddTraceEvents(): blockIndex = 0 for block in poolData['Blocks']: blockName = "p{} block{} {}".format(poolIndex, blockIndex, block['ID']) - AddTraceEventsBlock(block, blockCounter) AddBlockName(blockName, blockCounter) + AddTraceEventsBlock(block, blockCounter) blockCounter += 1 blockIndex += 1 @@ -222,30 +227,30 @@ def AddTraceEvents(): allocationIndex = 0 for dedicatedAlloc in poolData['DedicatedAllocations']: blockName = 'p{} alloc{} {}'.format(poolIndex, allocationIndex, dedicatedAlloc['Name']) - AddTraceEventsAlloc(dedicatedAlloc, 0, blockCounter) AddBlockName(blockName, blockCounter) + AddTraceEventsAlloc(dedicatedAlloc, 0, blockCounter) blockCounter += 1 allocationIndex += 1 # repeat for custom pools for customPoolData in poolData['CustomPools'].values(): - customPoolName = customPoolData['Name'] + customPoolName = "" # TODO: hook up, but it's not a field customPoolData['Name'] # pool block allocs blockIndex = 0 for block in customPoolData['Blocks']: blockName = 'p{} {} block{} {}'.format(poolIndex, customPoolName, blockIndex, block['ID']) - AddTraceEventsBlock(block, blockCounter) AddBlockName(blockName, blockCounter) + AddTraceEventsBlock(block, blockCounter) blockCounter += 1 blockIndex += 1 # pool dedicated allocs allocationIndex = 0 for dedicatedAlloc in customPoolData['DedicatedAllocations']: - blockName = 'p{} {} alloc{} {}'.format(poolIndex, customPoolName, dedicatedAlloc['Name']) - AddTraceEventsAlloc(dedicatedAlloc, 0, blockCounter) + blockName = 'p{} {} alloc{} {}'.format(poolIndex, customPoolName, allocationIndex, dedicatedAlloc['Name']) AddBlockName(blockName, blockCounter) + AddTraceEventsAlloc(dedicatedAlloc, 0, blockCounter) blockCounter += 1 allocationIndex += 1 @@ -269,7 +274,8 @@ def AddTraceEvents(): for dedicatedAlloc in memoryPool[1]['DedicatedAllocations']: allocData = {'Type': dedicatedAlloc['Type'], 'Size': int(dedicatedAlloc['Size']), - 'Usage': int(dedicatedAlloc['Usage'])} + 'Usage': int(dedicatedAlloc['Usage']), + 'Name': dedicatedAlloc['Name'] if 'Name' in dedicatedAlloc else 'M'} poolData['DedicatedAllocations'].append(allocData) # Get allocations in block vectors for block in memoryPool[1]['Blocks'].items(): @@ -286,7 +292,8 @@ def AddTraceEvents(): for dedicatedAlloc in pool['DedicatedAllocations']: allocData = {'Type': dedicatedAlloc['Type'], 'Size': int(dedicatedAlloc['Size']), - 'Usage': int(dedicatedAlloc['Usage'])} + 'Usage': int(dedicatedAlloc['Usage']), + 'Name': dedicatedAlloc['Name'] if 'Name' in dedicatedAlloc else 'M'} poolData['CustomPools'][poolName]['DedicatedAllocations'].append(allocData) # Get allocations in block vectors for block in pool['Blocks'].items(): @@ -300,7 +307,8 @@ def AddTraceEvents(): AddTraceEvents() - perfettoJson = json.dumps(perfettoDict, indent=4) + # perfettoJson = json.dumps(perfettoDict, indent=4) + perfettoJson = json.dumps(perfettoDict, indent=0) with open(args.output, "w") as outfile: outfile.write(perfettoJson) @@ -311,6 +319,8 @@ def AddTraceEvents(): - Fixed key 'Type'. Value is string. - Fixed key 'Size'. Value is int. - Fixed key 'Usage'. Value is int. + - Key 'Name' optional, Value is string + - Key'CustomData' optional - Fixed key 'Blocks'. Value is list of objects, each containing dictionary with: - Fixed key 'ID'. Value is int. - Fixed key 'Size'. Value is int. diff --git a/scripts/GpuMemDumpVis.py b/scripts/GpuMemDumpVis.py old mode 100644 new mode 100755 index e27b4cd8..a8b945fd --- a/scripts/GpuMemDumpVis.py +++ b/scripts/GpuMemDumpVis.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # # Copyright (c) 2018-2023 Advanced Micro Devices, Inc. All rights reserved. # @@ -323,6 +324,8 @@ def DrawDedicatedAllocationBlock(draw, y, dedicatedAlloc, pixelsPerByte): - Fixed key 'Type'. Value is string. - Fixed key 'Size'. Value is int. - Fixed key 'Usage'. Value is int. + - Key 'Name' optional, Value is string + - Key'CustomData' optional - Fixed key 'Blocks'. Value is list of objects, each containing dictionary with: - Fixed key 'ID'. Value is int. - Fixed key 'Size'. Value is int. diff --git a/scripts/open_trace_in_ui.py b/scripts/open_trace_in_ui.py new file mode 100644 index 00000000..269e71e9 --- /dev/null +++ b/scripts/open_trace_in_ui.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# taken from here, to open a .trace file in Perfetto in the browser. Runs +# an http server at a port, and then browser can read the file from that +# since fileURL are often blocked. Use a system call to run this, and it +# will change the working directory to that of the file. +# from https://github.com/google/perfetto/blob/master/tools/open_trace_in_ui + +import argparse +import http.server +import os +import socketserver +import sys +import webbrowser + + +class ANSI: + END = '\033[0m' + BOLD = '\033[1m' + RED = '\033[91m' + BLACK = '\033[30m' + BLUE = '\033[94m' + BG_YELLOW = '\033[43m' + BG_BLUE = '\033[44m' + + +# HTTP Server used to open the trace in the browser. +class HttpHandler(http.server.SimpleHTTPRequestHandler): + + def end_headers(self): + self.send_header('Access-Control-Allow-Origin', self.server.allow_origin) + self.send_header('Cache-Control', 'no-cache') + super().end_headers() + + def do_GET(self): + if self.path != '/' + self.server.expected_fname: + self.send_error(404, "File not found") + return + + self.server.fname_get_completed = True + super().do_GET() + + def do_POST(self): + self.send_error(404, "File not found") + + +def prt(msg, colors=ANSI.END): + print(colors + msg + ANSI.END) + + +def open_trace(path, open_browser, origin): + # We reuse the HTTP+RPC port because it's the only one allowed by the CSP. + PORT = 9001 + path = os.path.abspath(path) + os.chdir(os.path.dirname(path)) + fname = os.path.basename(path) + socketserver.TCPServer.allow_reuse_address = True + with socketserver.TCPServer(('127.0.0.1', PORT), HttpHandler) as httpd: + address = f'{origin}/#!/?url=http://127.0.0.1:{PORT}/{fname}' + if open_browser: + webbrowser.open_new_tab(address) + else: + print(f'Open URL in browser: {address}') + + httpd.expected_fname = fname + httpd.fname_get_completed = None + httpd.allow_origin = origin + while httpd.fname_get_completed is None: + httpd.handle_request() + + +def main(): + examples = '\n'.join( + [ANSI.BOLD + 'Usage:' + ANSI.END, ' -i path/trace_file_name [-n]']) + parser = argparse.ArgumentParser( + epilog=examples, formatter_class=argparse.RawTextHelpFormatter) + + help = 'Input trace filename' + parser.add_argument('-i', '--trace', help=help) + parser.add_argument( + '-n', '--no-open-browser', action='store_true', default=False) + parser.add_argument('--origin', default='https://ui.perfetto.dev') + + args = parser.parse_args() + trace_file = args.trace + open_browser = not args.no_open_browser + + if trace_file is None: + prt('Please specify trace file name with -i/--trace argument', ANSI.RED) + sys.exit(1) + elif not os.path.exists(trace_file): + prt('%s not found ' % trace_file, ANSI.RED) + sys.exit(1) + + prt('Opening the trace (%s) in the browser' % trace_file) + open_trace(trace_file, open_browser, args.origin) + + +if __name__ == '__main__': + sys.exit(main())