From 3673608b7d5a833870951b38a118d40407c28283 Mon Sep 17 00:00:00 2001 From: Emery Berger Date: Mon, 18 Dec 2023 18:35:07 -0500 Subject: [PATCH 1/3] Type cleanup and refactoring. --- scalene/replacement_pjoin.py | 4 +- scalene/scalene_analysis.py | 3 +- scalene/scalene_json.py | 4 +- scalene/scalene_output.py | 2 +- scalene/scalene_profiler.py | 131 ++++++++++++++++------------------- scalene/scalene_utility.py | 8 +-- scalene/time_info.py | 26 +++++++ 7 files changed, 97 insertions(+), 81 deletions(-) create mode 100644 scalene/time_info.py diff --git a/scalene/replacement_pjoin.py b/scalene/replacement_pjoin.py index af2298382..90aaf807f 100644 --- a/scalene/replacement_pjoin.py +++ b/scalene/replacement_pjoin.py @@ -43,9 +43,9 @@ def replacement_process_join(self, timeout: float = -1) -> None: # type: ignore if timeout != -1: end_time = time.perf_counter() if end_time - start_time >= timeout: - from multiprocessing.process import ( + from multiprocessing.process import ( # type: ignore _children, - ) # type: ignore + ) _children.discard(self) return diff --git a/scalene/scalene_analysis.py b/scalene/scalene_analysis.py index 5eaf41712..38c65c403 100644 --- a/scalene/scalene_analysis.py +++ b/scalene/scalene_analysis.py @@ -109,6 +109,7 @@ def find_regions(src: str) -> Dict[int, Tuple[int, int]]: functions = {} classes = {} for node in ast.walk(tree): + assert node.end_lineno if isinstance(node, ast.ClassDef): for line in range(node.lineno, node.end_lineno + 1): classes[line] = (node.lineno, node.end_lineno) @@ -118,7 +119,7 @@ def find_regions(src: str) -> Dict[int, Tuple[int, int]]: if isinstance(node, ast.FunctionDef): for line in range(node.lineno, node.end_lineno + 1): functions[line] = (node.lineno, node.end_lineno) - for lineno, line in enumerate(srclines, 1): + for lineno, _ in enumerate(srclines, 1): if lineno in loops: regions[lineno] = loops[lineno] elif lineno in functions: diff --git a/scalene/scalene_json.py b/scalene/scalene_json.py index 9277a1d3e..e690a60fe 100644 --- a/scalene/scalene_json.py +++ b/scalene/scalene_json.py @@ -235,8 +235,8 @@ def output_profiles( pid: int, profile_this_code: Callable[[Filename, LineNumber], bool], python_alias_dir: Path, - program_path: Path, - entrypoint_dir: Path, + program_path: Filename, + entrypoint_dir: Filename, program_args: Optional[List[str]], profile_memory: bool = True, reduced_profile: bool = False, diff --git a/scalene/scalene_output.py b/scalene/scalene_output.py index 21b8b071b..9d5ec21e0 100644 --- a/scalene/scalene_output.py +++ b/scalene/scalene_output.py @@ -306,7 +306,7 @@ def output_profiles( pid: int, profile_this_code: Callable[[Filename, LineNumber], bool], python_alias_dir: Path, - program_path: Path, + program_path: Filename, program_args: Optional[List[str]], profile_memory: bool = True, reduced_profile: bool = False, diff --git a/scalene/scalene_profiler.py b/scalene/scalene_profiler.py index 9ea76a1db..49c5f238e 100644 --- a/scalene/scalene_profiler.py +++ b/scalene/scalene_profiler.py @@ -16,7 +16,7 @@ # Import cysignals early so it doesn't disrupt Scalene's use of signals; this allows Scalene to profile Sage. # See https://github.com/plasma-umass/scalene/issues/740. try: - import cysignals + import cysignals # type: ignore except ModuleNotFoundError: pass @@ -34,6 +34,7 @@ import os import pathlib import platform +import queue import re import signal import stat @@ -48,8 +49,9 @@ # For debugging purposes from rich.console import Console -from scalene.get_module_details import _get_module_details from scalene.find_browser import find_browser +from scalene.get_module_details import _get_module_details +from scalene.time_info import get_times, TimeInfo from scalene.redirect_python import redirect_python from collections import defaultdict @@ -61,6 +63,7 @@ Any, Callable, Dict, + Generator, List, Optional, Set, @@ -103,19 +106,11 @@ # Assigning to `nada` disables any console.log commands. def nada(*args: Any) -> None: pass -console.log = nada +console.log = nada # type: ignore MINIMUM_PYTHON_VERSION_MAJOR = 3 MINIMUM_PYTHON_VERSION_MINOR = 8 -@dataclass -class time_info: - virtual : float - wallclock : float - sys : float - user : float - - def require_python(version: Tuple[int, int]) -> None: assert ( sys.version_info >= version @@ -129,8 +124,6 @@ def require_python(version: Tuple[int, int]) -> None: # It also has partial support for Windows. # Install our profile decorator. - - def scalene_redirect_profile(func: Any) -> Any: """Handle @profile decorators. @@ -155,14 +148,14 @@ def start() -> None: def stop() -> None: """Stop profiling.""" Scalene.stop() - + class Scalene: """The Scalene profiler itself.""" # Get the number of available CPUs (preferring `os.sched_getaffinity`, if available). __availableCPUs: int try: - __availableCPUs = len(os.sched_getaffinity(0)) + __availableCPUs = len(os.sched_getaffinity(0)) # type: ignore except AttributeError: cpu_count = os.cpu_count() __availableCPUs = cpu_count if cpu_count else 1 @@ -176,11 +169,19 @@ class Scalene: __parent_pid = -1 __initialized: bool = False __last_profiled = [Filename("NADA"), LineNumber(0), ByteCodeIndex(0)] + + @staticmethod + def last_profiled_tuple() -> Tuple[Filename, LineNumber, ByteCodeIndex]: + """Helper function to type last profiled information.""" + return cast(Tuple[Filename, LineNumber, ByteCodeIndex], Scalene.__last_profiled) + + __last_profiled_invalidated = False __gui_dir = "scalene-gui" __profile_filename = Filename("profile.json") __profiler_html = Filename("profile.html") __error_message = "Error in program being profiled" + __windows_queue : queue.Queue[Any] = queue.Queue() # only used for Windows timer logic BYTES_PER_MB = 1024 * 1024 MALLOC_ACTION = "M" @@ -218,7 +219,7 @@ def get_original_lock() -> threading.Lock: return Scalene.__original_lock() # when did we last receive a signal? - __last_signal_time = time_info(0.0, 0.0, 0.0, 0.0) + __last_signal_time = TimeInfo() # path for the program being profiled __program_path = Filename("") @@ -310,6 +311,15 @@ def update_line() -> None: """Mark a new line by allocating the trigger number of bytes.""" bytearray(NEWLINE_TRIGGER_LENGTH) + @staticmethod + def update_profiled() -> None: + with Scalene.__invalidate_mutex: + last_prof_tuple = Scalene.last_profiled_tuple() + Scalene.__invalidate_queue.append( + (last_prof_tuple[0], last_prof_tuple[1]) + ) + Scalene.update_line() + @staticmethod def invalidate_lines_python( frame: FrameType, _event: str, _arg: str @@ -319,7 +329,7 @@ def invalidate_lines_python( # If we are still on the same line, return. ff = frame.f_code.co_filename fl = frame.f_lineno - (fname, lineno, lasti) = Scalene.__last_profiled + (fname, lineno, lasti) = Scalene.last_profiled_tuple() if (ff == fname) and (fl == lineno): return Scalene.invalidate_lines_python # Different line: stop tracing this frame. @@ -332,11 +342,7 @@ def invalidate_lines_python( return None # We are on a different line; stop tracing and increment the count. sys.settrace(None) - with Scalene.__invalidate_mutex: - Scalene.__invalidate_queue.append( - (Scalene.__last_profiled[0], Scalene.__last_profiled[1]) - ) - Scalene.update_line() + Scalene.update_profiled() Scalene.__last_profiled_invalidated = True Scalene.__last_profiled = [ @@ -442,6 +448,7 @@ def reset_thread_sleeping(tid: int) -> None: @staticmethod def windows_timer_loop() -> None: """For Windows, send periodic timer signals; launch as a background thread.""" + assert sys.platform == "win32" Scalene.timer_signals = True while Scalene.timer_signals: Scalene.__windows_queue.get() @@ -490,7 +497,7 @@ def malloc_signal_handler( if not Scalene.__args.memory: # This should never happen, but we fail gracefully. return - from scalene import pywhere + from scalene import pywhere # type: ignore if this_frame: Scalene.enter_function_meta(this_frame, Scalene.__stats) @@ -511,15 +518,11 @@ def malloc_signal_handler( # First, see if we have now executed a different line of code. # If so, increment. invalidated = pywhere.get_last_profiled_invalidated() - (fname, lineno, lasti) = Scalene.__last_profiled + (fname, lineno, lasti) = Scalene.last_profiled_tuple() if not invalidated and this_frame and not ( on_stack(this_frame, fname, lineno) ): - with Scalene.__invalidate_mutex: - Scalene.__invalidate_queue.append( - (Scalene.__last_profiled[0], Scalene.__last_profiled[1]) - ) - Scalene.update_line() + Scalene.update_profiled() pywhere.set_last_profiled_invalidated_false() Scalene.__last_profiled = [Filename(f.f_code.co_filename), LineNumber(f.f_lineno), @@ -628,10 +631,9 @@ def __init__( Scalene.__memcpy_sigq, ] Scalene.__invalidate_mutex = Scalene.get_original_lock() + + Scalene.__windows_queue = queue.Queue() if sys.platform == "win32": - import queue - - Scalene.__windows_queue = queue.Queue() if arguments.memory: print(f"Scalene warning: Memory profiling is not currently supported for Windows.") arguments.memory = False @@ -675,6 +677,8 @@ def __init__( "cli", "web", "no_browser", "reduced_profile"]: if getattr(arguments, arg): cmdline += f' --{arg.replace("_", "-")}' + # Add the --pid field so we can propagate it to the child. + cmdline += f" --pid={os.getpid()} ---" # Build the commands to pass along other arguments environ = ScalenePreload.get_preload_environ(arguments) if sys.platform == "win32": @@ -686,9 +690,6 @@ def __init__( "=".join((k, str(v))) for (k, v) in environ.items() ) - # Add the --pid field so we can propagate it to the child. - cmdline += f" --pid={os.getpid()} ---" - redirect_python(preface, cmdline, Scalene.__python_alias_dir) @@ -711,18 +712,7 @@ def cpu_signal_handler( """Handle CPU signals.""" try: # Get current time stats. - now_sys: float = 0 - now_user: float = 0 - if sys.platform != "win32": - # On Linux/Mac, use getrusage, which provides higher - # resolution values than os.times() for some reason. - ru = resource.getrusage(resource.RUSAGE_SELF) - now_sys = ru.ru_stime - now_user = ru.ru_utime - else: - time_info = os.times() - now_sys = time_info.system - now_user = time_info.user + now_sys, now_user = get_times() now_virtual = time.process_time() now_wallclock = time.perf_counter() if ( @@ -862,6 +852,15 @@ def output_profile(program_args: Optional[List[str]] = None) -> bool: ) return did_output + @staticmethod + @functools.lru_cache(None) + def get_line_info(fname: Filename) -> Generator[Tuple[list[str], int], None, None]: + line_info = ( + inspect.getsourcelines(fn) + for fn in Scalene.__functions_to_profile[fname] + ) + return line_info + @staticmethod def profile_this_code(fname: Filename, lineno: LineNumber) -> bool: # sourcery skip: inline-immediately-returned-variable @@ -871,10 +870,7 @@ def profile_this_code(fname: Filename, lineno: LineNumber) -> bool: if fname not in Scalene.__files_to_profile: return False # Now check to see if it's the right line range. - line_info = ( - inspect.getsourcelines(fn) - for fn in Scalene.__functions_to_profile[fname] - ) + line_info = Scalene.get_line_info(fname) found_function = any( line_start <= lineno < line_start + len(lines) for (lines, line_start) in line_info @@ -1494,7 +1490,7 @@ def should_trace(filename: Filename, func: str) -> bool: if filename.startswith("_ipython-input-"): # Profiling code created in a Jupyter cell: # create a file to hold the contents. - import IPython + import IPython # type: ignore if result := re.match(r"_ipython-input-([0-9]+)-.*", filename): # Write the cell's contents into the file. @@ -1662,7 +1658,7 @@ def profile_code( ) -> int: """Initiate execution and profiling.""" if Scalene.__args.memory: - from scalene import pywhere + from scalene import pywhere # type: ignore pywhere.populate_struct() # If --off is set, tell all children to not profile and stop profiling before we even start. @@ -1690,7 +1686,7 @@ def profile_code( # Leaving here in case of reversion # sys.settrace(None) stats = Scalene.__stats - (last_file, last_line, _) = Scalene.__last_profiled + (last_file, last_line, _) = Scalene.last_profiled_tuple() stats.memory_malloc_count[last_file][last_line] += 1 stats.memory_aggregate_footprint[last_file][ last_line @@ -1792,18 +1788,12 @@ def run_profiler( ) sys.exit(1) if sys.platform != "win32": - Scalene.__orig_signal( - Scalene.__signals.start_profiling_signal, - Scalene.start_signal_handler, - ) - Scalene.__orig_signal( - Scalene.__signals.stop_profiling_signal, - Scalene.stop_signal_handler, - ) - for sig in [Scalene.__signals.start_profiling_signal, - Scalene.__signals.stop_profiling_signal]: + for sig, handler in [(Scalene.__signals.start_profiling_signal, + Scalene.start_signal_handler), + (Scalene.__signals.stop_profiling_signal, + Scalene.stop_signal_handler)]: + Scalene.__orig_signal(sig, handler) Scalene.__orig_siginterrupt(sig, False) - Scalene.__orig_signal(signal.SIGINT, Scalene.interruption_handler) did_preload = ( False if is_jupyter else ScalenePreload.setup_preload(args) @@ -1964,12 +1954,11 @@ def run_profiler( sys.exit(1) finally: with contextlib.suppress(Exception): - Scalene.__malloc_mapfile.close() - Scalene.__memcpy_mapfile.close() - if not Scalene.__is_child: - # We are done with these files, so remove them. - Scalene.__malloc_mapfile.cleanup() - Scalene.__memcpy_mapfile.cleanup() + for mapfile in [Scalene.__malloc_mapfile, + Scalene.__memcpy_mapfile]: + mapfile.close() + if not Scalene.__is_child: + mapfile.cleanup() if not is_jupyter: sys.exit(exit_status) diff --git a/scalene/scalene_utility.py b/scalene/scalene_utility.py index 663cec1a3..ed1f94ce8 100644 --- a/scalene/scalene_utility.py +++ b/scalene/scalene_utility.py @@ -4,7 +4,7 @@ import sys from jinja2 import Environment, FileSystemLoader -from types import FrameType +from types import CodeType, FrameType from typing import ( Any, Callable, @@ -41,9 +41,9 @@ def __str__(self) -> str: def add_stack(frame: FrameType, should_trace: Callable[[Filename, str], bool], - stacks: Dict[Tuple[Any], int]) -> None: + stacks: Dict[Any, int]) -> None: """Add one to the stack starting from this frame.""" - stk = list() + stk : List[Tuple[str, str, int]] = list() f : Optional[FrameType] = frame while f: if should_trace(Filename(f.f_code.co_filename), f.f_code.co_name): @@ -73,7 +73,7 @@ def get_fully_qualified_name(frame: FrameType) -> Filename: version = sys.version_info if version.major >= 3 and version.minor >= 11: # Introduced in Python 3.11 - fn_name = Filename(frame.f_code.co_qualname) + fn_name = Filename(frame.f_code.co_qualname) # type: ignore return fn_name f = frame # Manually search for an enclosing class. diff --git a/scalene/time_info.py b/scalene/time_info.py new file mode 100644 index 000000000..2581829c0 --- /dev/null +++ b/scalene/time_info.py @@ -0,0 +1,26 @@ +import os +import sys + +from dataclasses import dataclass +from typing import Tuple + +@dataclass +class TimeInfo: + virtual : float = 0.0 + wallclock : float = 0.0 + sys : float = 0.0 + user : float = 0.0 + +def get_times() -> Tuple[float, float]: + if sys.platform != "win32": + # On Linux/Mac, use getrusage, which provides higher + # resolution values than os.times() for some reason. + import resource + ru = resource.getrusage(resource.RUSAGE_SELF) + now_sys = ru.ru_stime + now_user = ru.ru_utime + else: + time_info = os.times() + now_sys = time_info.system + now_user = time_info.user + return now_sys, now_user From a63304a4228516e325b11aa3107cef152f0c605e Mon Sep 17 00:00:00 2001 From: Emery Berger Date: Mon, 18 Dec 2023 18:37:15 -0500 Subject: [PATCH 2/3] Blackened. --- scalene/find_browser.py | 21 +++- scalene/get_module_details.py | 1 - scalene/redirect_python.py | 22 +++-- scalene/replacement_mp_lock.py | 7 +- scalene/replacement_pjoin.py | 2 +- scalene/replacement_sem_lock.py | 7 +- scalene/scalene_analysis.py | 10 +- scalene/scalene_gpu.py | 4 +- scalene/scalene_json.py | 6 +- scalene/scalene_jupyter.py | 1 + scalene/scalene_parseargs.py | 12 ++- scalene/scalene_profiler.py | 167 +++++++++++++++++++++----------- scalene/scalene_signals.py | 2 +- scalene/scalene_statistics.py | 2 +- scalene/scalene_utility.py | 36 ++++--- scalene/time_info.py | 11 ++- 16 files changed, 198 insertions(+), 113 deletions(-) diff --git a/scalene/find_browser.py b/scalene/find_browser.py index 2bda78a3b..352a93f41 100644 --- a/scalene/find_browser.py +++ b/scalene/find_browser.py @@ -1,17 +1,32 @@ import webbrowser from typing import Optional + def find_browser() -> Optional[webbrowser.BaseBrowser]: """Find the default browser if possible and if compatible.""" # Names of known graphical browsers as per Python's webbrowser documentation - graphical_browsers = ["windowsdefault", "macosx", "safari", "google-chrome", - "chrome", "chromium", "firefox", "opera", "edge", "mozilla", "netscape"] + graphical_browsers = [ + "windowsdefault", + "macosx", + "safari", + "google-chrome", + "chrome", + "chromium", + "firefox", + "opera", + "edge", + "mozilla", + "netscape", + ] try: # Get the default browser object browser = webbrowser.get() # Check if the browser's class name matches any of the known graphical browsers browser_class_name = str(type(browser)).lower() - if any(graphical_browser in browser_class_name for graphical_browser in graphical_browsers): + if any( + graphical_browser in browser_class_name + for graphical_browser in graphical_browsers + ): return browser else: return None diff --git a/scalene/get_module_details.py b/scalene/get_module_details.py index 437c9ca43..76abfa079 100644 --- a/scalene/get_module_details.py +++ b/scalene/get_module_details.py @@ -92,4 +92,3 @@ def _get_module_details( if code is None: raise error("No code object available for %s" % mod_name) return mod_name, spec, code - diff --git a/scalene/redirect_python.py b/scalene/redirect_python.py index 6f6ba71c1..f041a23cc 100644 --- a/scalene/redirect_python.py +++ b/scalene/redirect_python.py @@ -3,19 +3,26 @@ import stat import sys -def redirect_python(preface: str, cmdline: str, python_alias_dir: pathlib.Path) -> None: + +def redirect_python( + preface: str, cmdline: str, python_alias_dir: pathlib.Path +) -> None: # Likely names for the Python interpreter. base_python_extension = ".exe" if sys.platform == "win32" else "" all_python_names = [ "python" + base_python_extension, "python" + str(sys.version_info.major) + base_python_extension, - "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) + base_python_extension + "python" + + str(sys.version_info.major) + + "." + + str(sys.version_info.minor) + + base_python_extension, ] # if sys.platform == "win32": # base_python_name = re.sub(r'\.exe$', '', os.path.basename(sys.executable)) # else: # base_python_name = sys.executable - + # Don't show commands on Windows; regular shebang for # shell scripts on Linux/OS X shebang = "@echo off" if sys.platform == "win32" else "#!/bin/bash" @@ -31,7 +38,7 @@ def redirect_python(preface: str, cmdline: str, python_alias_dir: pathlib.Path) for name in all_python_names: fname = os.path.join(python_alias_dir, name) if sys.platform == "win32": - fname = re.sub(r'\.exe$', '.bat', fname) + fname = re.sub(r"\.exe$", ".bat", fname) with open(fname, "w") as file: file.write(payload) os.chmod(fname, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) @@ -39,9 +46,7 @@ def redirect_python(preface: str, cmdline: str, python_alias_dir: pathlib.Path) # Finally, insert this directory into the path. sys.path.insert(0, str(python_alias_dir)) os.environ["PATH"] = ( - str(python_alias_dir) - + os.pathsep - + os.environ["PATH"] + str(python_alias_dir) + os.pathsep + os.environ["PATH"] ) # Force the executable (if anyone invokes it later) to point to one of our aliases. sys.executable = os.path.join( @@ -49,5 +54,4 @@ def redirect_python(preface: str, cmdline: str, python_alias_dir: pathlib.Path) all_python_names[0], ) if sys.platform == "win32" and sys.executable.endswith(".exe"): - sys.executable = re.sub(r'\.exe$', '.bat', sys.executable) - + sys.executable = re.sub(r"\.exe$", ".bat", sys.executable) diff --git a/scalene/replacement_mp_lock.py b/scalene/replacement_mp_lock.py index 418303ab4..2058b0fa8 100644 --- a/scalene/replacement_mp_lock.py +++ b/scalene/replacement_mp_lock.py @@ -15,10 +15,9 @@ # timeout_obj is parsed as a double from scalene.replacement_sem_lock import ReplacementSemLock - + + @Scalene.shim def replacement_mp_semlock(scalene: Scalene) -> None: - ReplacementSemLock.__qualname__ = ( - "replacement_semlock.ReplacementSemLock" - ) + ReplacementSemLock.__qualname__ = "replacement_semlock.ReplacementSemLock" multiprocessing.synchronize.Lock = ReplacementSemLock # type: ignore diff --git a/scalene/replacement_pjoin.py b/scalene/replacement_pjoin.py index 90aaf807f..08cc3e851 100644 --- a/scalene/replacement_pjoin.py +++ b/scalene/replacement_pjoin.py @@ -43,7 +43,7 @@ def replacement_process_join(self, timeout: float = -1) -> None: # type: ignore if timeout != -1: end_time = time.perf_counter() if end_time - start_time >= timeout: - from multiprocessing.process import ( # type: ignore + from multiprocessing.process import ( # type: ignore _children, ) diff --git a/scalene/replacement_sem_lock.py b/scalene/replacement_sem_lock.py index 080c86d0a..755f1536d 100644 --- a/scalene/replacement_sem_lock.py +++ b/scalene/replacement_sem_lock.py @@ -6,15 +6,18 @@ from scalene.scalene_profiler import Scalene from typing import Any + def _recreate_replacement_sem_lock(): return ReplacementSemLock() + class ReplacementSemLock(multiprocessing.synchronize.Lock): def __init__(self, ctx=None): # Ensure to use the appropriate context while initializing if ctx is None: ctx = multiprocessing.get_context() super().__init__(ctx=ctx) + def __enter__(self) -> bool: max_timeout = sys.getswitchinterval() tident = threading.get_ident() @@ -26,8 +29,10 @@ def __enter__(self) -> bool: if acquired: return True else: - max_timeout *= 2 # Exponential backoff + max_timeout *= 2 # Exponential backoff + def __exit__(self, *args: Any) -> None: super().__exit__(*args) + def __reduce__(self): return (_recreate_replacement_sem_lock, ()) diff --git a/scalene/scalene_analysis.py b/scalene/scalene_analysis.py index 38c65c403..4a571ddca 100644 --- a/scalene/scalene_analysis.py +++ b/scalene/scalene_analysis.py @@ -157,7 +157,11 @@ def walk(node, current_outermost_region, outer_class): for child_node in ast.iter_child_nodes(node): walk(child_node, current_outermost_region, outer_class) if isinstance(node, ast.stmt): - outermost_is_loop = outer_class in [ast.For, ast.AsyncFor, ast.While] + outermost_is_loop = outer_class in [ + ast.For, + ast.AsyncFor, + ast.While, + ] curr_is_block_not_loop = node.__class__ in [ ast.With, ast.If, @@ -175,9 +179,7 @@ def walk(node, current_outermost_region, outer_class): # NOTE: this additionally accounts for the case in which `node` # is a loop. Any loop within another loop should take on the entirety of the # outermost loop too - regions[ - line - ] = current_outermost_region + regions[line] = current_outermost_region elif ( curr_is_block_not_loop and len(srclines[line - 1].strip()) > 0 diff --git a/scalene/scalene_gpu.py b/scalene/scalene_gpu.py index 2e66da61a..c22f11dbd 100644 --- a/scalene/scalene_gpu.py +++ b/scalene/scalene_gpu.py @@ -117,7 +117,9 @@ def gpu_memory_usage(self, pid: int) -> float: for i in range(self.__ngpus): handle = self.__handle[i] with contextlib.suppress(Exception): - for proc in pynvml.nvmlDeviceGetComputeRunningProcesses(handle): + for proc in pynvml.nvmlDeviceGetComputeRunningProcesses( + handle + ): # Only accumulate memory stats for the current pid. if proc.usedGpuMemory and proc.pid == pid: # First check is to protect against return of None diff --git a/scalene/scalene_json.py b/scalene/scalene_json.py index e690a60fe..5bae9aa1b 100644 --- a/scalene/scalene_json.py +++ b/scalene/scalene_json.py @@ -289,7 +289,7 @@ def output_profiles( # Convert stacks into a representation suitable for JSON dumping. stks = [] for stk in stats.stacks.keys(): - this_stk : List[str] = [] + this_stk: List[str] = [] this_stk.extend(stk) stks.append((this_stk, stats.stacks[stk])) @@ -449,7 +449,9 @@ def output_profiles( profile_line["end_region_line"] = enclosing_regions[ lineno ][1] - profile_line["start_outermost_loop"] = outer_loop[lineno][0] + profile_line["start_outermost_loop"] = outer_loop[lineno][ + 0 + ] profile_line["end_outermost_loop"] = outer_loop[lineno][1] # When reduced-profile set, only output if the payload for the line is non-zero. if reduced_profile: diff --git a/scalene/scalene_jupyter.py b/scalene/scalene_jupyter.py index 46106ab6a..9876b1f71 100644 --- a/scalene/scalene_jupyter.py +++ b/scalene/scalene_jupyter.py @@ -3,6 +3,7 @@ from threading import Thread from typing import Any, Optional + class ScaleneJupyter: @staticmethod def find_available_port(start_port: int, end_port: int) -> Optional[int]: diff --git a/scalene/scalene_parseargs.py b/scalene/scalene_parseargs.py index ff5efb82a..4b91ba116 100644 --- a/scalene/scalene_parseargs.py +++ b/scalene/scalene_parseargs.py @@ -208,16 +208,22 @@ def parse_args() -> Tuple[argparse.Namespace, List[str]]: + " [/blue])", ) if sys.platform == "win32": - memory_profile_message = "profile memory (not supported on this platform)" + memory_profile_message = ( + "profile memory (not supported on this platform)" + ) else: - memory_profile_message = "profile memory (default: [blue]" + (str(defaults.memory)) + " [/blue])" + memory_profile_message = ( + "profile memory (default: [blue]" + + (str(defaults.memory)) + + " [/blue])" + ) parser.add_argument( "--memory", dest="memory", action="store_const", const=True, default=None, - help= memory_profile_message + help=memory_profile_message, ) parser.add_argument( "--profile-all", diff --git a/scalene/scalene_profiler.py b/scalene/scalene_profiler.py index 49c5f238e..a59574bdc 100644 --- a/scalene/scalene_profiler.py +++ b/scalene/scalene_profiler.py @@ -16,7 +16,7 @@ # Import cysignals early so it doesn't disrupt Scalene's use of signals; this allows Scalene to profile Sage. # See https://github.com/plasma-umass/scalene/issues/740. try: - import cysignals # type: ignore + import cysignals # type: ignore except ModuleNotFoundError: pass @@ -106,11 +106,14 @@ # Assigning to `nada` disables any console.log commands. def nada(*args: Any) -> None: pass -console.log = nada # type: ignore + + +console.log = nada # type: ignore MINIMUM_PYTHON_VERSION_MAJOR = 3 MINIMUM_PYTHON_VERSION_MINOR = 8 + def require_python(version: Tuple[int, int]) -> None: assert ( sys.version_info >= version @@ -149,13 +152,14 @@ def stop() -> None: """Stop profiling.""" Scalene.stop() + class Scalene: """The Scalene profiler itself.""" # Get the number of available CPUs (preferring `os.sched_getaffinity`, if available). __availableCPUs: int try: - __availableCPUs = len(os.sched_getaffinity(0)) # type: ignore + __availableCPUs = len(os.sched_getaffinity(0)) # type: ignore except AttributeError: cpu_count = os.cpu_count() __availableCPUs = cpu_count if cpu_count else 1 @@ -173,15 +177,18 @@ class Scalene: @staticmethod def last_profiled_tuple() -> Tuple[Filename, LineNumber, ByteCodeIndex]: """Helper function to type last profiled information.""" - return cast(Tuple[Filename, LineNumber, ByteCodeIndex], Scalene.__last_profiled) + return cast( + Tuple[Filename, LineNumber, ByteCodeIndex], Scalene.__last_profiled + ) - __last_profiled_invalidated = False __gui_dir = "scalene-gui" __profile_filename = Filename("profile.json") __profiler_html = Filename("profile.html") __error_message = "Error in program being profiled" - __windows_queue : queue.Queue[Any] = queue.Queue() # only used for Windows timer logic + __windows_queue: queue.Queue[ + Any + ] = queue.Queue() # only used for Windows timer logic BYTES_PER_MB = 1024 * 1024 MALLOC_ACTION = "M" @@ -319,7 +326,7 @@ def update_profiled() -> None: (last_prof_tuple[0], last_prof_tuple[1]) ) Scalene.update_line() - + @staticmethod def invalidate_lines_python( frame: FrameType, _event: str, _arg: str @@ -497,7 +504,7 @@ def malloc_signal_handler( if not Scalene.__args.memory: # This should never happen, but we fail gracefully. return - from scalene import pywhere # type: ignore + from scalene import pywhere # type: ignore if this_frame: Scalene.enter_function_meta(this_frame, Scalene.__stats) @@ -519,14 +526,18 @@ def malloc_signal_handler( # If so, increment. invalidated = pywhere.get_last_profiled_invalidated() (fname, lineno, lasti) = Scalene.last_profiled_tuple() - if not invalidated and this_frame and not ( - on_stack(this_frame, fname, lineno) + if ( + not invalidated + and this_frame + and not (on_stack(this_frame, fname, lineno)) ): Scalene.update_profiled() pywhere.set_last_profiled_invalidated_false() - Scalene.__last_profiled = [Filename(f.f_code.co_filename), - LineNumber(f.f_lineno), - ByteCodeIndex(f.f_lasti)] + Scalene.__last_profiled = [ + Filename(f.f_code.co_filename), + LineNumber(f.f_lineno), + ByteCodeIndex(f.f_lasti), + ] Scalene.__alloc_sigq.put([0]) pywhere.enable_settrace() del this_frame @@ -576,7 +587,7 @@ def enable_signals_win32() -> None: Scalene.__windows_queue.put(None) Scalene.start_signal_queues() return - + @staticmethod def enable_signals() -> None: """Set up the signal handlers to handle interrupts for profiling and start the @@ -586,11 +597,13 @@ def enable_signals() -> None: return Scalene.start_signal_queues() # Set signal handlers for various events. - for sig, handler in [(Scalene.__signals.malloc_signal, Scalene.malloc_signal_handler), - (Scalene.__signals.free_signal, Scalene.free_signal_handler), - (Scalene.__signals.memcpy_signal, Scalene.memcpy_signal_handler), - (signal.SIGTERM, Scalene.term_signal_handler), - (Scalene.__signals.cpu_signal, Scalene.cpu_signal_handler)]: + for sig, handler in [ + (Scalene.__signals.malloc_signal, Scalene.malloc_signal_handler), + (Scalene.__signals.free_signal, Scalene.free_signal_handler), + (Scalene.__signals.memcpy_signal, Scalene.memcpy_signal_handler), + (signal.SIGTERM, Scalene.term_signal_handler), + (Scalene.__signals.cpu_signal, Scalene.cpu_signal_handler), + ]: Scalene.__orig_signal(sig, handler) # Set every signal to restart interrupted system calls. for s in Scalene.__signals.get_all_signals(): @@ -631,13 +644,15 @@ def __init__( Scalene.__memcpy_sigq, ] Scalene.__invalidate_mutex = Scalene.get_original_lock() - + Scalene.__windows_queue = queue.Queue() if sys.platform == "win32": if arguments.memory: - print(f"Scalene warning: Memory profiling is not currently supported for Windows.") + print( + f"Scalene warning: Memory profiling is not currently supported for Windows." + ) arguments.memory = False - + # Initialize the malloc related files; if for whatever reason # the files don't exist and we are supposed to be profiling # memory, exit. @@ -673,8 +688,17 @@ def __init__( # Pass along commands from the invoking command line. if "off" in arguments and arguments.off: cmdline += " --off" - for arg in ["use_virtual_time", "cpu_sampling_rate", "cpu", "gpu", "memory", - "cli", "web", "no_browser", "reduced_profile"]: + for arg in [ + "use_virtual_time", + "cpu_sampling_rate", + "cpu", + "gpu", + "memory", + "cli", + "web", + "no_browser", + "reduced_profile", + ]: if getattr(arguments, arg): cmdline += f' --{arg.replace("_", "-")}' # Add the --pid field so we can propagate it to the child. @@ -691,7 +715,6 @@ def __init__( ) redirect_python(preface, cmdline, Scalene.__python_alias_dir) - # Register the exit handler to run when the program terminates or we quit. atexit.register(Scalene.exit_handler) @@ -854,13 +877,15 @@ def output_profile(program_args: Optional[List[str]] = None) -> bool: @staticmethod @functools.lru_cache(None) - def get_line_info(fname: Filename) -> Generator[Tuple[list[str], int], None, None]: + def get_line_info( + fname: Filename, + ) -> Generator[Tuple[list[str], int], None, None]: line_info = ( inspect.getsourcelines(fn) for fn in Scalene.__functions_to_profile[fname] ) return line_info - + @staticmethod def profile_this_code(fname: Filename, lineno: LineNumber) -> bool: # sourcery skip: inline-immediately-returned-variable @@ -923,7 +948,7 @@ def process_cpu_sample( if not new_frames: # No new frames, so nothing to update. return - + # Here we take advantage of an ostensible limitation of Python: # it only delivers signals after the interpreter has given up # control. This seems to mean that sampling is limited to code @@ -994,7 +1019,9 @@ def process_cpu_sample( main_thread_frame = new_frames[0][0] if Scalene.__args.stacks: - add_stack(main_thread_frame, Scalene.should_trace, Scalene.__stats.stacks) + add_stack( + main_thread_frame, Scalene.should_trace, Scalene.__stats.stacks + ) average_python_time = python_time / total_frames average_c_time = c_time / total_frames @@ -1490,7 +1517,7 @@ def should_trace(filename: Filename, func: str) -> bool: if filename.startswith("_ipython-input-"): # Profiling code created in a Jupyter cell: # create a file to hold the contents. - import IPython # type: ignore + import IPython # type: ignore if result := re.match(r"_ipython-input-([0-9]+)-.*", filename): # Write the cell's contents into the file. @@ -1518,9 +1545,9 @@ def should_trace(filename: Filename, func: str) -> bool: return True # Profile anything in the program's directory or a child directory, # but nothing else, unless otherwise specified. - filename = Filename(os.path.normpath( - os.path.join(Scalene.__program_path, filename) - )) + filename = Filename( + os.path.normpath(os.path.join(Scalene.__program_path, filename)) + ) return Scalene.__program_path in filename __done = False @@ -1547,10 +1574,13 @@ def stop() -> None: Scalene.disable_signals() Scalene.__stats.stop_clock() if Scalene.__args.outfile: - Scalene.__profile_filename = os.path.join(os.path.dirname(Scalene.__args.outfile), - os.path.basename(Scalene.__profile_filename)) + Scalene.__profile_filename = os.path.join( + os.path.dirname(Scalene.__args.outfile), + os.path.basename(Scalene.__profile_filename), + ) - if (Scalene.__args.web + if ( + Scalene.__args.web and not Scalene.__args.cli and not Scalene.__is_child ): @@ -1570,7 +1600,7 @@ def stop() -> None: Scalene.__args.web = False # If so, set variables appropriately. - if (Scalene.__args.web and Scalene.in_jupyter()): + if Scalene.__args.web and Scalene.in_jupyter(): # Force JSON output to profile.json. Scalene.__args.json = True Scalene.__output.html = False @@ -1628,9 +1658,11 @@ def disable_signals(retry: bool = True) -> None: return try: Scalene.__orig_setitimer(Scalene.__signals.cpu_timer_signal, 0) - for sig in [Scalene.__signals.malloc_signal, - Scalene.__signals.free_signal, - Scalene.__signals.memcpy_signal]: + for sig in [ + Scalene.__signals.malloc_signal, + Scalene.__signals.free_signal, + Scalene.__signals.memcpy_signal, + ]: Scalene.__orig_signal(sig, signal.SIG_IGN) Scalene.stop_signal_queues() except Exception: @@ -1658,7 +1690,7 @@ def profile_code( ) -> int: """Initiate execution and profiling.""" if Scalene.__args.memory: - from scalene import pywhere # type: ignore + from scalene import pywhere # type: ignore pywhere.populate_struct() # If --off is set, tell all children to not profile and stop profiling before we even start. @@ -1708,7 +1740,9 @@ def profile_code( generate_html( profile_fname=Scalene.__profile_filename, - output_fname=Scalene.__args.outfile if Scalene.__args.outfile else Scalene.__profiler_html, + output_fname=Scalene.__args.outfile + if Scalene.__args.outfile + else Scalene.__profiler_html, ) if Scalene.in_jupyter(): from scalene.scalene_jupyter import ScaleneJupyter @@ -1717,7 +1751,9 @@ def profile_code( if not port: print("Scalene error: could not find an available port.") else: - ScaleneJupyter.display_profile(port, Scalene.__profiler_html) + ScaleneJupyter.display_profile( + port, Scalene.__profiler_html + ) else: if not Scalene.__args.no_browser: # Remove any interposition libraries from the environment before opening the browser. @@ -1725,17 +1761,20 @@ def profile_code( old_dyld = os.environ.pop("DYLD_INSERT_LIBRARIES", "") old_ld = os.environ.pop("LD_PRELOAD", "") if Scalene.__args.outfile: - output_fname=Scalene.__args.outfile + output_fname = Scalene.__args.outfile else: - output_fname=f"{os.getcwd()}/{Scalene.__profiler_html}" + output_fname = ( + f"{os.getcwd()}/{Scalene.__profiler_html}" + ) if Scalene.__pid == 0: # Only open a browser tab for the parent. - webbrowser.open( - f"file:///{output_fname}" - ) + webbrowser.open(f"file:///{output_fname}") # Restore them. os.environ.update( - {"DYLD_INSERT_LIBRARIES": old_dyld, "LD_PRELOAD": old_ld} + { + "DYLD_INSERT_LIBRARIES": old_dyld, + "LD_PRELOAD": old_ld, + } ) return exit_status @@ -1749,7 +1788,9 @@ def process_args(args: argparse.Namespace) -> None: ) Scalene.__output.html = args.html if args.outfile: - Scalene.__output.output_file = os.path.abspath(os.path.expanduser(args.outfile)) + Scalene.__output.output_file = os.path.abspath( + os.path.expanduser(args.outfile) + ) Scalene.__is_child = args.pid != 0 # the pid of the primary profiler Scalene.__parent_pid = args.pid if Scalene.__is_child else os.getpid() @@ -1788,10 +1829,16 @@ def run_profiler( ) sys.exit(1) if sys.platform != "win32": - for sig, handler in [(Scalene.__signals.start_profiling_signal, - Scalene.start_signal_handler), - (Scalene.__signals.stop_profiling_signal, - Scalene.stop_signal_handler)]: + for sig, handler in [ + ( + Scalene.__signals.start_profiling_signal, + Scalene.start_signal_handler, + ), + ( + Scalene.__signals.stop_profiling_signal, + Scalene.stop_signal_handler, + ), + ]: Scalene.__orig_signal(sig, handler) Scalene.__orig_siginterrupt(sig, False) Scalene.__orig_signal(signal.SIGINT, Scalene.interruption_handler) @@ -1883,9 +1930,9 @@ def run_profiler( Scalene.__entrypoint_dir = program_path # If a program path was specified at the command-line, use it. if len(args.program_path) > 0: - Scalene.__program_path = Filename(os.path.abspath( - args.program_path - )) + Scalene.__program_path = Filename( + os.path.abspath(args.program_path) + ) else: # Otherwise, use the invoked directory. Scalene.__program_path = program_path @@ -1954,8 +2001,10 @@ def run_profiler( sys.exit(1) finally: with contextlib.suppress(Exception): - for mapfile in [Scalene.__malloc_mapfile, - Scalene.__memcpy_mapfile]: + for mapfile in [ + Scalene.__malloc_mapfile, + Scalene.__memcpy_mapfile, + ]: mapfile.close() if not Scalene.__is_child: mapfile.cleanup() diff --git a/scalene/scalene_signals.py b/scalene/scalene_signals.py index 7a5d72b67..917314a0e 100644 --- a/scalene/scalene_signals.py +++ b/scalene/scalene_signals.py @@ -29,7 +29,7 @@ def __init__(self) -> None: self.malloc_signal = None self.free_signal = None - def set_timer_signals(self, use_virtual_time: bool=True) -> None: + def set_timer_signals(self, use_virtual_time: bool = True) -> None: """ Set up timer signals for CPU profiling. diff --git a/scalene/scalene_statistics.py b/scalene/scalene_statistics.py index 0f9668344..d1c2b38f7 100644 --- a/scalene/scalene_statistics.py +++ b/scalene/scalene_statistics.py @@ -447,7 +447,7 @@ def merge_stats(self, the_dir_name: pathlib.Path) -> None: for filename in self.per_line_footprint_samples: for lineno in self.per_line_footprint_samples[filename]: self.per_line_footprint_samples[filename][lineno].sort( - key=lambda x: x[0] # type: ignore + key=lambda x: x[0] # type: ignore ) self.increment_per_line_samples( self.memory_malloc_count, x.memory_malloc_count diff --git a/scalene/scalene_utility.py b/scalene/scalene_utility.py index ed1f94ce8..2b32d5b73 100644 --- a/scalene/scalene_utility.py +++ b/scalene/scalene_utility.py @@ -5,19 +5,8 @@ from jinja2 import Environment, FileSystemLoader from types import CodeType, FrameType -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Tuple, - cast -) -from scalene.scalene_statistics import ( - Filename, - LineNumber -) +from typing import Any, Callable, Dict, List, Optional, Tuple, cast +from scalene.scalene_statistics import Filename, LineNumber from scalene.scalene_version import scalene_version, scalene_date # These are here to simplify print debugging, a la C. @@ -28,6 +17,7 @@ def __str__(self) -> str: assert frame.f_back return str(frame.f_back.f_lineno) + class FileName: def __str__(self) -> str: frame = inspect.currentframe() @@ -36,21 +26,26 @@ def __str__(self) -> str: assert frame.f_back.f_code return str(frame.f_back.f_code.co_filename) + __LINE__ = LineNo() __FILE__ = FileName() -def add_stack(frame: FrameType, - should_trace: Callable[[Filename, str], bool], - stacks: Dict[Any, int]) -> None: + +def add_stack( + frame: FrameType, + should_trace: Callable[[Filename, str], bool], + stacks: Dict[Any, int], +) -> None: """Add one to the stack starting from this frame.""" - stk : List[Tuple[str, str, int]] = list() - f : Optional[FrameType] = frame + stk: List[Tuple[str, str, int]] = list() + f: Optional[FrameType] = frame while f: if should_trace(Filename(f.f_code.co_filename), f.f_code.co_name): stk.insert(0, (f.f_code.co_filename, f.f_code.co_name, f.f_lineno)) f = f.f_back stacks[tuple(stk)] += 1 + def on_stack( frame: FrameType, fname: Filename, lineno: LineNumber ) -> Optional[FrameType]: @@ -68,12 +63,13 @@ def on_stack( f = cast(FrameType, f.f_back) return None + def get_fully_qualified_name(frame: FrameType) -> Filename: # Obtain the fully-qualified name. version = sys.version_info if version.major >= 3 and version.minor >= 11: # Introduced in Python 3.11 - fn_name = Filename(frame.f_code.co_qualname) # type: ignore + fn_name = Filename(frame.f_code.co_qualname) # type: ignore return fn_name f = frame # Manually search for an enclosing class. @@ -93,6 +89,7 @@ def get_fully_qualified_name(frame: FrameType) -> Filename: f = f.f_back return fn_name + def flamegraph_format(stacks: Dict[Tuple[Any], int]) -> str: """Converts stacks to a string suitable for input to Brendan Gregg's flamegraph.pl script.""" output = "" @@ -104,6 +101,7 @@ def flamegraph_format(stacks: Dict[Tuple[Any], int]) -> str: output += "\n" return output + def generate_html(profile_fname: Filename, output_fname: Filename) -> None: """Apply a template to generate a single HTML payload containing the current profile.""" diff --git a/scalene/time_info.py b/scalene/time_info.py index 2581829c0..e37cb248c 100644 --- a/scalene/time_info.py +++ b/scalene/time_info.py @@ -4,18 +4,21 @@ from dataclasses import dataclass from typing import Tuple + @dataclass class TimeInfo: - virtual : float = 0.0 - wallclock : float = 0.0 - sys : float = 0.0 - user : float = 0.0 + virtual: float = 0.0 + wallclock: float = 0.0 + sys: float = 0.0 + user: float = 0.0 + def get_times() -> Tuple[float, float]: if sys.platform != "win32": # On Linux/Mac, use getrusage, which provides higher # resolution values than os.times() for some reason. import resource + ru = resource.getrusage(resource.RUSAGE_SELF) now_sys = ru.ru_stime now_user = ru.ru_utime From 2e1cf0443a96751744513152ef46f9f6868ffdc9 Mon Sep 17 00:00:00 2001 From: Emery Berger Date: Mon, 18 Dec 2023 18:40:37 -0500 Subject: [PATCH 3/3] Moved assertion into loops. --- scalene/scalene_analysis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scalene/scalene_analysis.py b/scalene/scalene_analysis.py index 4a571ddca..81f6bd8c0 100644 --- a/scalene/scalene_analysis.py +++ b/scalene/scalene_analysis.py @@ -109,14 +109,16 @@ def find_regions(src: str) -> Dict[int, Tuple[int, int]]: functions = {} classes = {} for node in ast.walk(tree): - assert node.end_lineno if isinstance(node, ast.ClassDef): + assert node.end_lineno for line in range(node.lineno, node.end_lineno + 1): classes[line] = (node.lineno, node.end_lineno) if isinstance(node, (ast.For, ast.While)): + assert node.end_lineno for line in range(node.lineno, node.end_lineno + 1): loops[line] = (node.lineno, node.end_lineno) if isinstance(node, ast.FunctionDef): + assert node.end_lineno for line in range(node.lineno, node.end_lineno + 1): functions[line] = (node.lineno, node.end_lineno) for lineno, _ in enumerate(srclines, 1):