From 5ef3cd735c7d7f41e730b97b254e3621a622b2a4 Mon Sep 17 00:00:00 2001 From: Comma Device Date: Sat, 15 Feb 2025 03:36:26 +0000 Subject: [PATCH] the start of something beautiful --- fixup_cams.sh | 18 ++++++++++++ repro_realign.py | 30 ++++++++++++++++++++ selfdrive/test/test_onroad.py | 52 +++++++++++++++++++---------------- tools/profiling/ftrace.sh | 11 ++++---- 4 files changed, 82 insertions(+), 29 deletions(-) create mode 100755 fixup_cams.sh create mode 100755 repro_realign.py diff --git a/fixup_cams.sh b/fixup_cams.sh new file mode 100755 index 00000000000000..429c14ef08c1b4 --- /dev/null +++ b/fixup_cams.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e +set -x + +for cam-cpas crm_workq-cam_fd_worker crm_workq-cam_ife_worke crm_workq-cam_lrme_devi +crm_workq-cam_lrme_hw_w +qcom,cam170-cpas-cdm0 +qcom,cam_virtual_cdm + +sudo su -c "echo 40 | tee /sys/devices/virtual/workqueue/*cam*/cpumask" +echo 40 | sudo tee /sys/devices/virtual/workqueue/*cam*/cpumask +sudo cat /sys/devices/virtual/workqueue/*cam*/cpumask + +for pid in $(ps -A | grep cam | awk '{print $1}'); do + chrt -p $pid + sudo chrt -f -p 1 $pid + sudo taskset -pc 6 $pid || echo "failed on $pid" +done diff --git a/repro_realign.py b/repro_realign.py new file mode 100755 index 00000000000000..cacee4b9f563be --- /dev/null +++ b/repro_realign.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +import time +from openpilot.system.manager.process_config import managed_processes +from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf +from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext, MonitoringModelFrame + +cnt = 0 +try: + while True: + """ + cl_context = CLContext() + vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True, cl_context) + while not vipc_client.connect(False): + time.sleep(0.1) + print("cnnctDD") + del vipc_client + time.sleep(0.5) + continue + """ + if (cnt % 5) == 0: + managed_processes['modeld'].stop(block=True) + managed_processes['modeld'].start() + if (cnt % 3) == 0: + managed_processes['dmonitoringmodeld'].stop(block=True) + managed_processes['dmonitoringmodeld'].start() + cnt += 1 + time.sleep(1) +except: + managed_processes['modeld'].stop(block=True) + managed_processes['dmonitoringmodeld'].stop(block=True) diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 98a800e9bd606b..38fd91f9a1ce03 100644 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -31,7 +31,7 @@ should not exceed MAX_TOTAL_CPU """ -TEST_DURATION = 25 +TEST_DURATION = 10 LOG_OFFSET = 8 MAX_TOTAL_CPU = 280. # total for all 8 cores @@ -114,13 +114,13 @@ def cputime_total(ct): @pytest.mark.tici class TestOnroad: - @classmethod - def setup_class(cls): + #@classmethod + def setup_method(self): if "DEBUG" in os.environ: segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog.zst")), Path(Paths.log_root()).iterdir()) segs = sorted(segs, key=lambda x: x.stat().st_mtime) print(segs[-3]) - cls.lr = list(LogReader(os.path.join(segs[-3], "rlog.zst"))) + self.lr = list(LogReader(os.path.join(segs[-3], "rlog.zst"))) return # setup env @@ -137,7 +137,7 @@ def setup_class(cls): proc = None try: manager_path = os.path.join(BASEDIR, "system/manager/manager.py") - cls.manager_st = time.monotonic() + self.manager_st = time.monotonic() proc = subprocess.Popen(["python", manager_path]) sm = messaging.SubMaster(['carState']) @@ -146,7 +146,7 @@ def setup_class(cls): sm.update(1000) route = None - cls.segments = [] + self.segments = [] with Timeout(300, "timed out waiting for logs"): while route is None: route = params.get("CurrentRoute", encoding="utf-8") @@ -155,36 +155,36 @@ def setup_class(cls): # test car params caching params.put("CarParamsCache", car.CarParams().to_bytes()) - while len(cls.segments) < 1: + while len(self.segments) < 1: segs = set() if Path(Paths.log_root()).exists(): segs = set(Path(Paths.log_root()).glob(f"{route}--*")) - cls.segments = sorted(segs, key=lambda s: int(str(s).rsplit('--')[-1])) + self.segments = sorted(segs, key=lambda s: int(str(s).rsplit('--')[-1])) time.sleep(0.01) time.sleep(TEST_DURATION) finally: - cls.gpu_procs = {psutil.Process(int(f.name)).name() for f in pathlib.Path('/sys/devices/virtual/kgsl/kgsl/proc/').iterdir() if f.is_dir()} + self.gpu_procs = {psutil.Process(int(f.name)).name() for f in pathlib.Path('/sys/devices/virtual/kgsl/kgsl/proc/').iterdir() if f.is_dir()} if proc is not None: proc.terminate() if proc.wait(60) is None: proc.kill() - cls.lrs = [list(LogReader(os.path.join(str(s), "rlog.zst"))) for s in cls.segments] + self.lrs = [list(LogReader(os.path.join(str(s), "rlog.zst"))) for s in self.segments] - cls.lr = list(LogReader(os.path.join(str(cls.segments[0]), "rlog.zst"))) - cls.log_path = cls.segments[0] + self.lr = list(LogReader(os.path.join(str(self.segments[0]), "rlog.zst"))) + self.log_path = self.segments[0] - cls.log_sizes = {} - for f in cls.log_path.iterdir(): + self.log_sizes = {} + for f in self.log_path.iterdir(): assert f.is_file() - cls.log_sizes[f] = f.stat().st_size / 1e6 + self.log_sizes[f] = f.stat().st_size / 1e6 - cls.msgs = defaultdict(list) - for m in cls.lr: - cls.msgs[m.which()].append(m) + self.msgs = defaultdict(list) + for m in self.lr: + self.msgs[m.which()].append(m) def test_service_frequencies(self, subtests): @@ -214,6 +214,7 @@ def test_cloudlog_size(self): assert len(big_logs) == 0, f"Log spam: {big_logs}" def test_log_sizes(self, subtests): + # TODO: this isn't super stable between different devices for f, sz in self.log_sizes.items(): rate = LOGS_SIZE[f.name]/60. minn = rate * TEST_DURATION * 0.5 @@ -313,21 +314,24 @@ def test_memory_usage(self): def test_gpu_usage(self): assert self.gpu_procs == {"weston", "ui", "camerad", "selfdrive.modeld.modeld", "selfdrive.modeld.dmonitoringmodeld"} - @pytest.mark.skip("TODO: enable once timings are fixed") def test_camera_frame_timings(self, subtests): result = "\n" result += "------------------------------------------------\n" - result += "----------------- SoF Timing ------------------\n" + result += "----------------- SOF Timing ------------------\n" result += "------------------------------------------------\n" for name in ['roadCameraState', 'wideRoadCameraState', 'driverCameraState']: ts = [getattr(m, m.which()).timestampSof for m in self.lr if name in m.which()] d_ms = np.diff(ts) / 1e6 d50 = np.abs(d_ms-50) - result += f"{name} sof delta vs 50ms: min {min(d50):.5f}s\n" - result += f"{name} sof delta vs 50ms: max {max(d50):.5f}s\n" - result += f"{name} sof delta vs 50ms: mean {d50.mean():.5f}s\n" + result += f"{name} sof delta vs 50ms: min {min(d50):.5f}ms\n" + result += f"{name} sof delta vs 50ms: max {max(d50):.5f}ms\n" + result += f"{name} sof delta vs 50ms: mean {d50.mean():.5f}ms\n" with subtests.test(camera=name): - assert max(d50) < 1.0, f"high SOF delta vs 50ms: {max(d50)}" + print("\n", "="*10, name, "="*10) + print("d_ms", list(np.round(d_ms, 1))) + print("d50", list(np.round(d50, 1))) + print("\n", "="*30) + assert max(d50) < 4.0, f"high SOF delta vs 50ms: {max(d50)}" result += "------------------------------------------------\n" print(result) diff --git a/tools/profiling/ftrace.sh b/tools/profiling/ftrace.sh index 7d5d1f5f9d6114..57c0944ae3af8e 100755 --- a/tools/profiling/ftrace.sh +++ b/tools/profiling/ftrace.sh @@ -8,14 +8,15 @@ echo boot > trace_clock echo 1000 > buffer_size_kb # /sys/kernel/tracing/available_events -echo 1 > events/irq/enable -echo 1 > events/sched/enable -echo 1 > events/kgsl/enable -echo 1 > events/camera/enable +echo 0 > events/enable +#echo 1 > events/irq/enable +#echo 1 > events/sched/enable +#echo 1 > events/kgsl/enable +#echo 1 > events/camera/enable echo 1 > events/workqueue/enable echo > trace -sleep 5 +sleep 2 echo 0 > tracing_on cp trace /tmp/trace