ruff rules for comprehensions and performance

common/swaglog.py

@@ -39,7 +39,7 @@ def _open(self):
     return stream
 
   def get_existing_logfiles(self):
-    log_files = list()
+    log_files = []
     base_dir = os.path.dirname(self.base_filename)
     for fn in os.listdir(base_dir):
       fp = os.path.join(base_dir, fn)

docs/hooks/glossary.py

@@ -49,7 +49,7 @@ def apply_tooltip(_term_key, _definition, pattern, html):
   )
 
 def tooltip_html(vocabulary, html):
-  for _category, terms in vocabulary.items():
+  for terms in vocabulary.values():
     for term_key, definition in terms.items():
       if definition.get("description"):
         pattern = rf"(?<!\w){re.escape(term_key.replace('_', ' ').title())}(?![^<]*<\/a>)(?!\([^)]*\))"

pyproject.toml

@@ -215,19 +215,19 @@ indent-width = 2
 lint.select = [
   "E", "F", "W", "PIE", "C4", "ISC", "A", "B",
   "NPY", # numpy
-  "UP",  # pyupgrade
-  "TRY302", "TRY400", "TRY401", # try/excepts
+  "PERF",
+  "PLR1704",
   "RUF008", "RUF100",
   "TID251",
-  "PLR1704",
+  "TRY203", "TRY400", "TRY401", # try/excepts
+  "UP",  # pyupgrade
 ]
 lint.ignore = [
-  "E741",
+  "B024",
+  "B027",
   "E402",
-  "C408",
+  "E741",
   "ISC003",
-  "B027",
-  "B024",
   "NPY002",  # new numpy random syntax is worse
   "UP038",   # (x, y) -> x|y for isinstance
 ]
@@ -246,6 +246,7 @@ exclude = [
   "third_party",
   "*.ipynb",
 ]
+lint.flake8-comprehensions.allow-dict-calls-with-keyword-arguments = true
 lint.flake8-implicit-str-concat.allow-multiline = false
 
 [tool.ruff.lint.flake8-tidy-imports.banned-api]

selfdrive/car/card.py

@@ -51,7 +51,7 @@ def can_recv(wait_for_one: bool = False) -> list[list[CanData]]:
     """
     ret = []
     for can in messaging.drain_sock(logcan, wait_for_one=wait_for_one):
-      ret.append([CanData(msg.address, msg.dat, msg.src) for msg in can.can])
+      ret.append([CanData(msg.address, msg.dat, msg.src) for msg in can.can])  # noqa: PERF401
     return ret
 
   def can_send(msgs: list[CanData]) -> None:

selfdrive/controls/tests/test_lateral_mpc.py

@@ -47,27 +47,19 @@ def test_straight(self):
     self._assert_null(np.array([sol]))
 
   def test_y_symmetry(self):
-    sol = []
-    for y_init in [-0.5, 0.5]:
-      sol.append(run_mpc(y_init=y_init))
+    sol = [run_mpc(y_init=y_init) for y_init in (-0.5, 0.5)]
     self._assert_simmetry(np.array(sol))
 
   def test_poly_symmetry(self):
-    sol = []
-    for poly_shift in [-1., 1.]:
-      sol.append(run_mpc(poly_shift=poly_shift))
+    sol = [run_mpc(poly_shift=poly_shift) for poly_shift in (-1., 1.)]
     self._assert_simmetry(np.array(sol))
 
   def test_curvature_symmetry(self):
-    sol = []
-    for curvature_init in [-0.1, 0.1]:
-      sol.append(run_mpc(curvature_init=curvature_init))
+    sol = [run_mpc(curvature_init=curvature_init) for curvature_init in (-0.1, 0.1)]
     self._assert_simmetry(np.array(sol))
 
   def test_psi_symmetry(self):
-    sol = []
-    for psi_init in [-0.1, 0.1]:
-      sol.append(run_mpc(psi_init=psi_init))
+    sol = [run_mpc(psi_init=psi_init) for psi_init in (-0.1, 0.1)]
     self._assert_simmetry(np.array(sol))
 
   def test_no_overshoot(self):

selfdrive/debug/can_table.py

@@ -12,9 +12,9 @@ def can_table(dat):
     r += [hex(b)]
     rows.append(r)
 
-  df = pd.DataFrame(data=rows)
-  df.columns = [str(n) for n in range(7, -1, -1)] + [' ']
-  table = df.to_markdown(tablefmt='grid')
+  dataframe = pd.DataFrame(data=rows)
+  dataframe.columns = [str(n) for n in range(7, -1, -1)] + [' ']
+  table = dataframe.to_markdown(tablefmt='grid')
   return table
 
 

selfdrive/locationd/test/test_locationd_scenarios.py

@@ -65,7 +65,7 @@ def run_scenarios(scenario, logs):
   elif scenario == Scenario.GYRO_OFF:
     logs = sorted([x for x in logs if x.which() != 'gyroscope'], key=lambda x: x.logMonoTime)
 
-  elif scenario == Scenario.GYRO_SPIKE_MIDWAY or scenario == Scenario.GYRO_CONSISTENT_SPIKES:
+  elif scenario in (Scenario.GYRO_SPIKE_MIDWAY, Scenario.GYRO_CONSISTENT_SPIKES):
     def gyro_spike(msg):
       msg.gyroscope.gyroUncalibrated.v[0] += 3.0
     count = 1 if scenario == Scenario.GYRO_SPIKE_MIDWAY else CONSISTENT_SPIKES_COUNT
@@ -74,13 +74,13 @@ def gyro_spike(msg):
   elif scenario == Scenario.ACCEL_OFF:
     logs = sorted([x for x in logs if x.which() != 'accelerometer'], key=lambda x: x.logMonoTime)
 
-  elif scenario == Scenario.ACCEL_SPIKE_MIDWAY or scenario == Scenario.ACCEL_CONSISTENT_SPIKES:
+  elif scenario in (Scenario.ACCEL_SPIKE_MIDWAY, Scenario.ACCEL_CONSISTENT_SPIKES):
     def acc_spike(msg):
       msg.accelerometer.acceleration.v[0] += 100.0
     count = 1 if scenario == Scenario.ACCEL_SPIKE_MIDWAY else CONSISTENT_SPIKES_COUNT
     logs = modify_logs_midway(logs, 'accelerometer', count, acc_spike)
 
-  elif scenario == Scenario.SENSOR_TIMING_SPIKE_MIDWAY or scenario == Scenario.SENSOR_TIMING_CONSISTENT_SPIKES:
+  elif scenario in (Scenario.SENSOR_TIMING_SPIKE_MIDWAY, Scenario.SENSOR_TIMING_CONSISTENT_SPIKES):
     def timing_spike(msg):
       msg.accelerometer.timestamp -= int(0.150 * 1e9)
     count = 1 if scenario == Scenario.SENSOR_TIMING_SPIKE_MIDWAY else CONSISTENT_SPIKES_COUNT

selfdrive/modeld/modeld.py

@@ -2,7 +2,6 @@
 import os
 from openpilot.system.hardware import TICI
 
-#
 if TICI:
   from tinygrad.tensor import Tensor
   from tinygrad.dtype import dtypes

selfdrive/modeld/parse_model_outputs.py

@@ -10,7 +10,7 @@ def sigmoid(x):
 
 def softmax(x, axis=-1):
   x -= np.max(x, axis=axis, keepdims=True)
-  if x.dtype == np.float32 or x.dtype == np.float64:
+  if x.dtype in (np.float32, np.float64):
     safe_exp(x, out=x)
   else:
     x = safe_exp(x)

selfdrive/modeld/tests/test_modeld.py

@@ -79,7 +79,7 @@ def test_dropped_frames(self):
       modeld should only run on consecutive road frames
     """
     frame_id = -1
-    road_frames = list()
+    road_frames = []
     for n in range(1, 50):
       if (random.random() < 0.1) and n > 3:
         cams = random.choice([(), ('wideRoadCameraState', )])

selfdrive/modeld/tests/timing/benchmark.py

@@ -26,8 +26,7 @@
     start = time.monotonic()
     while time.monotonic() - start < TIME:
       msgs = messaging.drain_sock(sock, wait_for_one=True)
-      for m in msgs:
-        t.append(m.modelV2.modelExecutionTime)
+      t.extend(msg.modelV2.modelExecutionTime for msg in msgs)
 
     execution_times.append(np.array(t[10:]) * 1000)
     managed_processes['modeld'].stop()

selfdrive/pandad/pandad.py

@@ -116,9 +116,7 @@ def signal_handler(signum, frame):
       cloudlog.info(f"{len(panda_serials)} panda(s) found, connecting - {panda_serials}")
 
       # Flash pandas
-      pandas: list[Panda] = []
-      for serial in panda_serials:
-        pandas.append(flash_panda(serial))
+      pandas: list[Panda] = [flash_panda(serial) for serial in panda_serials]
 
       # Ensure internal panda is present if expected
       internal_pandas = [panda for panda in pandas if panda.is_internal()]

selfdrive/pandad/tests/test_pandad_spi.py

@@ -35,10 +35,10 @@ def test_spi_corruption(self, subtests):
 
     total_recv_count = 0
     total_sent_count = 0
-    sent_msgs = {bus: list() for bus in range(3)}
+    sent_msgs = {bus: [] for bus in range(3)}
 
     st = time.monotonic()
-    ts = {s: list() for s in socks.keys()}
+    ts = {s: [] for s in socks.keys()}
     for _ in range(int(os.getenv("TEST_TIME", "20"))):
       # send some CAN messages
       if not JUNGLE_SPAM:

selfdrive/selfdrived/tests/test_alerts.py

@@ -2,6 +2,8 @@
 import json
 import os
 import random
+from collections.abc import Callable
+from typing import Any
 from PIL import Image, ImageDraw, ImageFont
 
 from cereal import log, car
@@ -17,10 +19,9 @@
 OFFROAD_ALERTS_PATH = os.path.join(BASEDIR, "selfdrive/selfdrived/alerts_offroad.json")
 
 # TODO: add callback alerts
-ALERTS = []
+ALERTS: list[Alert | Callable[[Any, Any, SubMaster, bool, int, Any], Alert]] = []
 for event_types in EVENTS.values():
-  for alert in event_types.values():
-    ALERTS.append(alert)
+  ALERTS.extend(alert for alert in event_types.values())
 
 
 class TestAlerts:

selfdrive/test/process_replay/regen.py

@@ -27,7 +27,7 @@ def __init__(self, w: int, h: int, frame_count: int, pix_val: int):
   def get(self, idx, count=1, pix_fmt="yuv420p"):
     if pix_fmt == "rgb24":
       shape = (self.h, self.w, 3)
-    elif pix_fmt == "nv12" or pix_fmt == "yuv420p":
+    elif pix_fmt in {"nv12", "yuv420p"}:
       shape = (int((self.h * self.w) * 3 / 2),)
     else:
       raise NotImplementedError

selfdrive/test/process_replay/test_processes.py

@@ -179,10 +179,8 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non
   with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool:
     if not args.upload_only:
       download_segments = [seg for car, seg in segments if car in tested_cars]
-      log_data: dict[str, LogReader] = {}
       p1 = pool.map(get_log_data, download_segments)
-      for segment, lr in tqdm(p1, desc="Getting Logs", total=len(download_segments)):
-        log_data[segment] = lr
+      log_data: dict[str, LogReader] = dict(tqdm(p1, desc="Getting Logs", total=len(download_segments)))
 
     pool_args: Any = []
     for car_brand, segment in segments:

selfdrive/test/test_updated.py

@@ -24,7 +24,7 @@ def setup_method(self):
     self.basedir = os.path.join(org_dir, "openpilot")
     self.git_remote_dir = os.path.join(org_dir, "openpilot_remote")
     self.staging_dir = os.path.join(org_dir, "safe_staging")
-    for d in [org_dir, self.basedir, self.git_remote_dir, self.staging_dir]:
+    for d in (org_dir, self.basedir, self.git_remote_dir, self.staging_dir):
       os.mkdir(d)
 
     self.neos_version = os.path.join(org_dir, "neos_version")
@@ -125,10 +125,8 @@ def _make_commit(self):
     for root, dirs, files in os.walk(self.git_remote_dir):
       if ".git" in root:
         continue
-      for d in dirs:
-        all_dirs.append(os.path.join(root, d))
-      for f in files:
-        all_files.append(os.path.join(root, f))
+      all_dirs.extend(os.path.join(root, d) for d in dirs)
+      all_files.extend(os.path.join(root, f) for f in files)
 
     # make a new dir and some new files
     new_dir = os.path.join(self.git_remote_dir, "this_is_a_new_dir")

selfdrive/test/update_ci_routes.py

@@ -76,9 +76,7 @@ def sync_to_ci_public(route: str) -> bool:
     to_sync.extend([rt.route for rt in test_car_models_routes])
     to_sync.extend([s[1].rsplit('--', 1)[0] for s in replay_segments])
 
-  for r in tqdm(to_sync):
-    if not sync_to_ci_public(r):
-      failed_routes.append(r)
+  failed_routes.append(r for r in tqdm(to_sync) if not sync_to_ci_public(r))
 
   if len(failed_routes):
     print("failed routes:", failed_routes)

system/athena/tests/test_athenad.py

@@ -417,7 +417,7 @@ def test_jsonrpc_handler(self):
       thread.join()
 
   def test_get_logs_to_send_sorted(self):
-    fl = list()
+    fl = []
     for i in range(10):
       file = f'swaglog.{i:010}'
       self._create_file(file, Paths.swaglog_root())

system/camerad/test/test_camerad.py

@@ -78,7 +78,7 @@ def get_desc(fid, diff):
     laggy_frames = {k: get_desc(k, v) for k, v in diffs.items() if v > LAG_FRAME_TOLERANCE[self.sensor_type]}
 
     def in_tol(diff):
-      return 50 - LAG_FRAME_TOLERANCE[self.sensor_type] < diff and diff < 50 + LAG_FRAME_TOLERANCE[self.sensor_type]
+      return 50 - LAG_FRAME_TOLERANCE[self.sensor_type] < diff < 50 + LAG_FRAME_TOLERANCE[self.sensor_type]
     if len(laggy_frames) != 0 and all( in_tol(laggy_frames[lf][0]) for lf in laggy_frames):
       print("TODO: handle camera out of sync")
     else:

system/loggerd/deleter.py

@@ -24,7 +24,7 @@ def has_preserve_xattr(d: str) -> bool:
 
 def get_preserved_segments(dirs_by_creation: list[str]) -> list[str]:
   # skip deleting most recent N preserved segments (and their prior segment)
-  preserved = []
+  preserved: list[str] = []
   for n, d in enumerate(filter(has_preserve_xattr, reversed(dirs_by_creation))):
     if n == PRESERVE_COUNT:
       break
@@ -39,8 +39,7 @@ def get_preserved_segments(dirs_by_creation: list[str]) -> list[str]:
       continue
 
     # preserve segment and two prior
-    for _seg_num in range(max(0, seg_num - 2), seg_num + 1):
-      preserved.append(f"{date_str}--{_seg_num}")
+    preserved.extend(f"{date_str}--{_seg_num}" for _seg_num in range(max(0, seg_num - 2), seg_num + 1))
 
   return preserved
 

system/loggerd/tests/test_uploader.py

@@ -18,8 +18,8 @@ def __init__(self):
     self.reset()
 
   def reset(self):
-    self.upload_order = list()
-    self.upload_ignored = list()
+    self.upload_order = []
+    self.upload_ignored = []
 
   def emit(self, record):
     try:
@@ -51,10 +51,10 @@ def join_thread(self):
     self.up_thread.join()
 
   def gen_files(self, lock=False, xattr: bytes = None, boot=True) -> list[Path]:
-    f_paths = []
-    for t in ["qlog", "rlog", "dcamera.hevc", "fcamera.hevc"]:
-      f_paths.append(self.make_file_with_data(self.seg_dir, t, 1, lock=lock, upload_xattr=xattr))
-
+    f_paths = [
+      self.make_file_with_data(self.seg_dir, t, 1, lock=lock, upload_xattr=xattr)
+      for t in ("qlog", "rlog", "dcamera.hevc", "fcamera.hevc")
+    ]
     if boot:
       f_paths.append(self.make_file_with_data("boot", f"{self.seg_dir}", 1, lock=lock, upload_xattr=xattr))
     return f_paths

system/qcomgpsd/structs.py

@@ -321,8 +321,7 @@ def parse_struct(ss):
     if '[' in nam:
       cnt = int(nam.split("[")[1].split("]")[0])
       st += st[-1]*(cnt-1)
-      for i in range(cnt):
-        nams.append(f'{nam.split("[")[0]}[{i}]')
+      nams.extend(f'{nam.split("[")[0]}[{i}]' for i in range(cnt))
     else:
       nams.append(nam)
   return st, nams

system/sensord/tests/test_sensord.py

@@ -180,7 +180,7 @@ def test_sensor_frequency(self, subtests):
   def test_logmonottime_timestamp_diff(self):
     # ensure diff between the message logMonotime and sample timestamp is small
 
-    tdiffs = list()
+    tdiffs = []
     for etype in self.events:
       for measurement in self.events[etype]:
         m = getattr(measurement, measurement.which())
@@ -202,7 +202,7 @@ def test_logmonottime_timestamp_diff(self):
     assert avg_diff < 4, f"Avg packet diff: {avg_diff:.1f}ms"
 
   def test_sensor_values(self):
-    sensor_values = dict()
+    sensor_values = {}
     for etype in self.events:
       for measurement in self.events[etype]:
         m = getattr(measurement, measurement.which())

system/ubloxd/pigeond.py

@@ -132,7 +132,7 @@ def reset_device(self) -> bool:
       self.send(b"\xB5\x62\x09\x14\x00\x00\x1D\x60")
       # 1: failed to restore, 2: could restore, 3: no backup
       status = self.wait_for_backup_restore_status()
-      if status == 1 or status == 3:
+      if status in {1, 3}:
         return True
     return False
 

system/ubloxd/tests/print_gps_stats.py

@@ -10,10 +10,8 @@
     gle = sm['gpsLocationExternal']
 
     try:
-      cnos = []
-      for m in ug.measurementReport.measurements:
-        cnos.append(m.cno)
-      print(f"Sats: {ug.measurementReport.numMeas}   Accuracy: {gle.horizontalAccuracy:.2f} m   cnos", sorted(cnos))
+      cnos = sorted(m.cno for m in ug.measurementReport.measurements)
+      print(f"Sats: {ug.measurementReport.numMeas}   Accuracy: {gle.horizontalAccuracy:.2f} m   cnos", cnos)
     except Exception:
       pass
     sm.update()

system/webrtc/webrtcd.py

@@ -254,7 +254,7 @@ def webrtcd_thread(host: str, port: int, debug: bool):
 
   app = web.Application()
 
-  app['streams'] = dict()
+  app['streams'] = {}
   app['debug'] = debug
   app.on_shutdown.append(on_shutdown)
   app.router.add_post("/stream", get_stream)

tools/lib/vidindex.py

@@ -269,7 +269,7 @@ def hevc_index(hevc_file_name: str, allow_corrupt: bool=False) -> tuple[list, in
     raise VideoFileInvalid("first byte must be 0x00")
 
   prefix_dat = b""
-  frame_types = list()
+  frame_types = []
 
   i = 1 # skip past first byte 0x00
   try: