B45: port upstream #24846, #24864 and #24854

.github/dependabot.yml

@@ -56,6 +56,9 @@ updates:
       # Needs Python 3.12 and numpy 2
       - dependency-name: "librosa"
         versions: [">=1.0"]
+      # pandas 3.1 requires numpy 2
+      - dependency-name: "pandas"
+        versions: [">=3.1"]
       # Needs Python 3.12
       - dependency-name: "scipy"
         versions: [">=1.18"]

fork/ledger/B45.md

@@ -0,0 +1 @@
+| B45 | backend + UI: upstream fixes for preview clips, failed exports, wall clock steps and review summaries | `frigate/api/media.py` (preview gif and mp4 duration), `frigate/record/export.py`, `frigate/comms/webpush.py`, `frigate/data_processing/real_time/custom_classification.py`, `frigate/jobs/vlm_watch.py`, `frigate/object_detection/base.py`, `frigate/output/birdseye.py`, `frigate/stats/util.py`, `frigate/util/{file,rknn_converter}.py`, `frigate/video/ffmpeg.py`, `frigate/watchdog.py`, `frigate/genai/prompts.py`, `frigate/data_processing/post/review_annotations.py`, `testing-scripts/genai_review_tester.py`, `.github/dependabot.yml`, `web/src/components/overlay/{CreateUserDialog,SetPasswordDialog}.tsx`, `web/src/utils/passwordUtil.ts`, `NotificationsSettingsExtras`, `web/src/views/live/{LiveCameraView,LiveDashboardView,DraggableGridLayout}.tsx`, the en `views/settings.json`; tests in `test_wall_clock_step.py` (new), `test_watchdog_notices.py`, `test_genai_prompts.py`, `test_genai_review_tester.py`, `test_review_annotations.py`, `test_fork_cov_comms_webpush.py`, `test_fork_cov_record_export.py`, `http_api/{test_http_event,test_http_export,test_http_media,test_http_fork_cov_media}.py`, vitest `overlay/fork/SetPasswordDialog.test.tsx` (new) | Ports of three upstream 0.19 fixes. Preview gif and mp4 clips that start inside a later preview end at the requested time, and an export that fails before or during ffmpeg drops its row instead of staying in progress until a restart (#24846). Timers that measure elapsed time use the monotonic clock, so a wall clock step such as an NTP sync at boot no longer holds notifications in cooldown, ends VLM watch jobs early, pauses state classification, freezes Birdseye, or restarts a healthy detector or camera; the fork's own hwaccel retry and record stall restarts use the same clock in the camera watchdog (#24864). Review summaries state their length limits, use named subjects' names in every field, and note objects already in view when the clip starts (#24854). Also from #24846: no password strength meter, a confirm field that revalidates, the `t` shortcut unmutes audio for two-way talk, and full volume for tiles with no saved setting. The rest of #24846 was already on the fork through B36 and UI155 and kept the fork's version. Not ported: dropping the severity sort in `/review`, because B37 keeps alerts ahead of the limit | #24846, #24864, #24854 |

frigate/api/media.py

@@ -1743,7 +1743,8 @@ async def preview_gif(
 
         # a preview that starts after start_ts is read from its beginning; a
         # negative diff wrapped to the wrong second (-30 % 60 is 30)
-        diff = max(start_ts - preview.start_time, 0)
+        clip_start = max(start_ts, preview.start_time)
+        diff = clip_start - preview.start_time
         minutes = int(diff / 60)
         seconds = int(diff % 60)
         config: FrigateConfig = request.app.frigate_config
@@ -1755,7 +1756,7 @@ async def preview_gif(
             "-ss",
             f"00:{minutes}:{seconds}",
             "-t",
-            f"{end_ts - start_ts}",
+            f"{end_ts - clip_start}",
             "-i",
             preview.path,
             "-r",
@@ -1921,7 +1922,8 @@ async def preview_mp4(
 
         # a preview that starts after start_ts is read from its beginning; a
         # negative diff wrapped to the wrong second (-30 % 60 is 30)
-        diff = max(start_ts - preview.start_time, 0)
+        clip_start = max(start_ts, preview.start_time)
+        diff = clip_start - preview.start_time
         minutes = int(diff / 60)
         seconds = int(diff % 60)
         config: FrigateConfig = request.app.frigate_config
@@ -1934,7 +1936,7 @@ async def preview_mp4(
             "-ss",
             f"00:{minutes}:{seconds}",
             "-t",
-            f"{end_ts - start_ts}",
+            f"{end_ts - clip_start}",
             "-i",
             preview.path,
             "-r",

frigate/comms/webpush.py

@@ -7,6 +7,7 @@
 import os
 import queue
 import threading
+import time
 from collections.abc import Callable
 from dataclasses import dataclass
 from multiprocessing.synchronize import Event as MpEvent
@@ -189,11 +190,13 @@ def __init__(self, config: FrigateConfig, stop_event: MpEvent) -> None:
         }
         self.suspension_broadcaster: Callable[[str, Any, bool], None] | None = None
         self.config_lock = threading.Lock()
+        # monotonic times, so a backward wall clock step can't hold every
+        # notification in cooldown. -inf means none has been sent yet
         self.last_camera_notification_time: dict[str, float] = {
-            c.name: 0  # type: ignore[misc]
+            c.name: float("-inf")  # type: ignore[misc]
             for c in self.config.cameras.values()
         }
-        self.last_notification_time: float = 0
+        self.last_notification_time: float = float("-inf")
         self.user_cameras: dict[str, set[str]] = {}
         self.notification_queue: queue.Queue[PushNotification] = queue.Queue()
         self.notification_thread: threading.Thread | None = None
@@ -353,7 +356,7 @@ def publish(self, topic: str, payload: Any, retain: bool = False) -> None:
             if "add" in updates:
                 for camera in updates["add"]:
                     self.suspended_cameras[camera] = 0
-                    self.last_camera_notification_time[camera] = 0
+                    self.last_camera_notification_time[camera] = float("-inf")
 
                 self._refresh_user_cameras()
 
@@ -512,7 +515,7 @@ def _user_has_camera_access(self, username: str, camera: str) -> bool:
         return camera in allowed
 
     def _within_cooldown(self, camera: str) -> bool:
-        now = datetime.datetime.now().timestamp()
+        now = time.monotonic()
         if now - self.last_notification_time < self.config.notifications.cooldown:
             logger.debug(
                 f"Skipping notification for {camera} - in global cooldown period"
@@ -557,7 +560,7 @@ def send_alert(self, payload: dict[str, Any]) -> None:
         camera_name: str = getattr(
             self.config.cameras[camera], "friendly_name", None
         ) or titlecase(camera.replace("_", " "))
-        current_time = datetime.datetime.now().timestamp()
+        current_time = time.monotonic()
 
         if self._within_cooldown(camera):
             return
@@ -670,7 +673,7 @@ def send_trigger(self, payload: dict[str, Any]) -> None:
         camera_name: str = getattr(
             self.config.cameras[camera], "friendly_name", None
         ) or titlecase(camera.replace("_", " "))
-        current_time = datetime.datetime.now().timestamp()
+        current_time = time.monotonic()
 
         if self._within_cooldown(camera):
             return

frigate/data_processing/post/review_annotations.py

@@ -12,6 +12,8 @@
 from collections.abc import Sequence
 from typing import Any
 
+from titlecase import titlecase
+
 from frigate.models import Event, Timeline
 
 logger = logging.getLogger(__name__)
@@ -84,12 +86,16 @@ def event_name(event: dict[str, Any]) -> str:
     tracked object whenever a subject is re-detected, so the tracking data
     cannot say whether two entries are the same subject, and the notes stay
     ambiguous rather than implying either answer.
+
+    Names are formatted the same way as in the prompt's "Objects in Scene"
+    list. Small models otherwise copy the raw lowercase sub label into their
+    response instead of the name they were told to use.
     """
     label = str(event["label"]).replace("_", " ").replace("-verified", "")
     sub_label = event.get("sub_label")
 
     if sub_label:
-        return f'{label} "{sub_label}"'
+        return f'{label} "{titlecase(str(sub_label).replace("_", " "))}"'
 
     article = "an" if label[:1].lower() in "aeiou" else "a"
     return f"{article} {label}"
@@ -213,6 +219,7 @@ def build_timeline(
     events: list[dict[str, Any]],
     span_end: float,
     state_changes: Sequence[dict[str, Any]] = (),
+    span_start: float | None = None,
 ) -> list[tuple[float, str]]:
     """All annotated moments across every event, in time order.
 
@@ -222,6 +229,12 @@ def build_timeline(
     describe nothing the model can see. A track ending means the object
     stopped being detected, which may or may not mean it left the frame.
 
+    `span_start` is the timestamp of the first sampled frame. An object
+    tracked since before it gets a single "already in view" note instead of
+    its earlier history: those moments would otherwise land on the first
+    frame and read as happening during the clip, turning a car that parked
+    minutes ago into one pulling in.
+
     `state_changes` are timeline rows (timestamp, source_id, class_type); the
     stationary and active ones become "has stopped moving" / "starts moving
     again". Frigate only marks an object stationary after it has been still
@@ -249,6 +262,13 @@ def build_timeline(
         if event["start_time"] > span_end:
             continue
 
+        if (
+            span_start is not None
+            and event["end_time"]
+            and event["end_time"] < span_start
+        ):
+            continue
+
         points = path_points(event.get("path_data") or [])
         legs = path_legs(points) if len(points) >= 2 else []
         name = event_name(event)
@@ -259,8 +279,33 @@ def build_timeline(
         )
         remaining = list(enumerate(legs))
         moments: list[tuple[float, str]] = []
+        event_changes = sorted(
+            (timestamp, phrase)
+            for timestamp, phrase in changes_by_event.get(event["id"], [])
+            if timestamp >= detected_at
+        )
 
-        if merges:
+        if span_start is not None and detected_at < span_start:
+            # Only whether it had come to rest is noted; where it is and which
+            # way it is heading are visible in the first frame.
+            earlier = [phrase for t, phrase in event_changes if t < span_start]
+            not_moving = (
+                bool(earlier) and earlier[-1] == STATE_CHANGE_PHRASES["stationary"]
+            )
+            timeline.append(
+                (
+                    span_start,
+                    f"{name} already in view when the sequence begins"
+                    + (", not moving" if not_moving else ""),
+                )
+            )
+            remaining = [
+                (index, leg)
+                for index, leg in remaining
+                if leg_start_time(points, leg) >= span_start
+            ]
+            event_changes = [(t, p) for t, p in event_changes if t >= span_start]
+        elif merges:
             heading = leg_heading(points, legs[0])
             timeline.append(
                 (detected_at, f"{name} first detected at {where}, moving {heading}")
@@ -277,11 +322,7 @@ def build_timeline(
                 )
             )
 
-        moments.extend(
-            (timestamp, phrase)
-            for timestamp, phrase in changes_by_event.get(event["id"], [])
-            if timestamp >= detected_at
-        )
+        moments.extend(event_changes)
 
         for timestamp, phrase in moments:
             if timestamp <= span_end:
@@ -390,7 +431,10 @@ def build_frame_captions(
         timeline.extend(
             (timestamp, f"[tracker] {note}")
             for timestamp, note in build_timeline(
-                events, span_end, get_state_changes(detection_ids)
+                events,
+                span_end,
+                get_state_changes(detection_ids),
+                span_start=frame_times[0],
             )
         )
 

frigate/data_processing/real_time/custom_classification.py

@@ -3,6 +3,7 @@
 import datetime
 import logging
 import os
+import time
 from typing import Any
 
 import cv2
@@ -67,7 +68,8 @@ def __init__(
         else:
             self.inference_speed = None
 
-        self.last_run = datetime.datetime.now().timestamp()
+        # monotonic so a backward wall clock step can't pause classification
+        self.last_run = time.monotonic()
         self.__build_detector()
 
     def __build_detector(self) -> None:
@@ -215,11 +217,12 @@ def process_frame(self, frame_data: dict[str, Any], frame: np.ndarray) -> None:
         should_run = False
 
         now = datetime.datetime.now().timestamp()
+        mono_now = time.monotonic()
         if (
             self.model_config.state_config.interval
-            and now > self.last_run + self.model_config.state_config.interval
+            and mono_now > self.last_run + self.model_config.state_config.interval
         ):
-            self.last_run = now
+            self.last_run = mono_now
             should_run = True
 
         if (
@@ -228,18 +231,18 @@ def process_frame(self, frame_data: dict[str, Any], frame: np.ndarray) -> None:
             and any([box_overlaps(crop, mb) for mb in frame_data.get("motion", [])])
         ):
             # classification should run at most once per second
-            if now > self.last_run + 1:
-                self.last_run = now
+            if mono_now > self.last_run + 1:
+                self.last_run = mono_now
                 should_run = True
 
         # Shortcut: always run if we have a pending state verification to complete
         if (
             not should_run
             and camera in self.state_history
             and self.state_history[camera]["pending_state"] is not None
-            and now > self.last_run + 0.5
+            and mono_now > self.last_run + 0.5
         ):
-            self.last_run = now
+            self.last_run = mono_now
             should_run = True
             logger.debug(
                 f"Running verification check for pending state: {self.state_history[camera]['pending_state']} ({self.state_history[camera]['consecutive_count']}/3)"