FEAT: Storage API & Frontend (#3409)

* Get storage output stats for each camera

* Add storage route

* Add storage route

* Add storage page

* Cleanup

* Add stats and show more storage

* Add tests for mb abbrev util fun

* Rewrite storage logic to use storage maintainer and segment sizes

* Include storage maintainer for http

* Use correct format

* Remove utils

* Fix tests

* Remove total from equation

* Multiply by 100 to get percent

* Add basic storage info

* Fix storage stats

* Fix endpoint and ui

* Fix formatting

frigate/app.py

@@ -163,6 +163,7 @@ def init_web_server(self) -> None:
             self.db,
             self.stats_tracking,
             self.detected_frames_processor,
+            self.storage_maintainer,
             self.plus_api,
         )
 
@@ -362,13 +363,13 @@ def start(self) -> None:
         self.start_detected_frames_processor()
         self.start_camera_processors()
         self.start_camera_capture_processes()
+        self.start_storage_maintainer()
         self.init_stats()
         self.init_web_server()
         self.start_event_processor()
         self.start_event_cleanup()
         self.start_recording_maintainer()
         self.start_recording_cleanup()
-        self.start_storage_maintainer()
         self.start_stats_emitter()
         self.start_watchdog()
         # self.zeroconf = broadcast_zeroconf(self.config.mqtt.client_id)

frigate/http.py

@@ -27,12 +27,12 @@
 from peewee import SqliteDatabase, operator, fn, DoesNotExist
 from playhouse.shortcuts import model_to_dict
 
-from frigate.config import CameraConfig
-from frigate.const import CLIPS_DIR
+from frigate.const import CLIPS_DIR, RECORD_DIR
 from frigate.models import Event, Recordings
 from frigate.object_processing import TrackedObject
 from frigate.stats import stats_snapshot
 from frigate.util import clean_camera_user_pass, ffprobe_stream, vainfo_hwaccel
+from frigate.storage import StorageMaintainer
 from frigate.version import VERSION
 
 logger = logging.getLogger(__name__)
@@ -45,6 +45,7 @@ def create_app(
     database: SqliteDatabase,
     stats_tracking,
     detected_frames_processor,
+    storage_maintainer: StorageMaintainer,
     plus_api,
 ):
     app = Flask(__name__)
@@ -62,6 +63,7 @@ def _db_close(exc):
     app.frigate_config = frigate_config
     app.stats_tracking = stats_tracking
     app.detected_frames_processor = detected_frames_processor
+    app.storage_maintainer = storage_maintainer
     app.plus_api = plus_api
     app.camera_error_image = None
 
@@ -690,6 +692,26 @@ def latest_frame(camera_name):
         return "Camera named {} not found".format(camera_name), 404
 
 
+@bp.route("/recordings/storage", methods=["GET"])
+def get_recordings_storage_usage():
+    recording_stats = stats_snapshot(
+        current_app.frigate_config, current_app.stats_tracking
+    )["service"]["storage"][RECORD_DIR]
+    total_mb = recording_stats["total"]
+
+    camera_usages: dict[
+        str, dict
+    ] = current_app.storage_maintainer.calculate_camera_usages()
+
+    for camera_name in camera_usages.keys():
+        if camera_usages.get(camera_name, {}).get("usage"):
+            camera_usages[camera_name]["usage_percent"] = (
+                camera_usages.get(camera_name, {}).get("usage", 0) / total_mb
+            ) * 100
+
+    return jsonify(camera_usages)
+
+
 # return hourly summary for recordings of camera
 @bp.route("/<camera_name>/recordings/summary")
 def recordings_summary(camera_name):

frigate/storage.py

@@ -60,6 +60,26 @@ def calculate_camera_bandwidth(self) -> None:
             self.camera_storage_stats[camera]["bandwidth"] = bandwidth
             logger.debug(f"{camera} has a bandwidth of {bandwidth} MB/hr.")
 
+    def calculate_camera_usages(self) -> dict[str, dict]:
+        """Calculate the storage usage of each camera."""
+        usages: dict[str, dict] = {}
+
+        for camera in self.config.cameras.keys():
+            camera_storage = (
+                Recordings.select(fn.SUM(Recordings.segment_size))
+                .where(Recordings.camera == camera, Recordings.segment_size != 0)
+                .scalar()
+            )
+
+            usages[camera] = {
+                "usage": camera_storage,
+                "bandwidth": self.camera_storage_stats.get(camera, {}).get(
+                    "bandwidth", 0
+                ),
+            }
+
+        return usages
+
     def check_storage_needs_cleanup(self) -> bool:
         """Return if storage needs cleanup."""
         # currently runs cleanup if less than 1 hour of space is left

frigate/test/test_http.py

@@ -114,7 +114,7 @@ def tearDown(self):
 
     def test_get_event_list(self):
         app = create_app(
-            FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi()
+            FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi()
         )
         id = "123456.random"
         id2 = "7890.random"
@@ -143,7 +143,7 @@ def test_get_event_list(self):
 
     def test_get_good_event(self):
         app = create_app(
-            FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi()
+            FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi()
         )
         id = "123456.random"
 
@@ -157,7 +157,7 @@ def test_get_good_event(self):
 
     def test_get_bad_event(self):
         app = create_app(
-            FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi()
+            FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi()
         )
         id = "123456.random"
         bad_id = "654321.other"
@@ -170,7 +170,7 @@ def test_get_bad_event(self):
 
     def test_delete_event(self):
         app = create_app(
-            FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi()
+            FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi()
         )
         id = "123456.random"
 
@@ -185,7 +185,7 @@ def test_delete_event(self):
 
     def test_event_retention(self):
         app = create_app(
-            FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi()
+            FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi()
         )
         id = "123456.random"
 
@@ -204,7 +204,7 @@ def test_event_retention(self):
 
     def test_set_delete_sub_label(self):
         app = create_app(
-            FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi()
+            FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi()
         )
         id = "123456.random"
         sub_label = "sub"
@@ -232,7 +232,7 @@ def test_set_delete_sub_label(self):
 
     def test_sub_label_list(self):
         app = create_app(
-            FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi()
+            FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi()
         )
         id = "123456.random"
         sub_label = "sub"
@@ -254,6 +254,7 @@ def test_config(self):
             self.db,
             None,
             None,
+            None,
             PlusApi(),
         )
 
@@ -268,6 +269,7 @@ def test_recordings(self):
             self.db,
             None,
             None,
+            None,
             PlusApi(),
         )
         id = "123456.random"
@@ -285,6 +287,7 @@ def test_stats(self, mock_stats):
             self.db,
             None,
             None,
+            None,
             PlusApi(),
         )
         mock_stats.return_value = self.test_stats

web/src/Sidebar.jsx

@@ -44,6 +44,7 @@ export default function Sidebar() {
       </Match>
       {birdseye?.enabled ? <Destination href="/birdseye" text="Birdseye" /> : null}
       <Destination href="/events" text="Events" />
+      <Destination href="/storage" text="Storage" />
       <Destination href="/system" text="System" />
       <Separator />
       <div className="flex flex-grow" />

web/src/app.tsx

@@ -35,6 +35,7 @@ export default function App() {
                     path="/recording/:camera/:date?/:hour?/:minute?/:second?"
                     getComponent={Routes.getRecording}
                   />
+                  <AsyncRoute path="/storage" getComponent={Routes.getStorage} />
                   <AsyncRoute path="/system" getComponent={Routes.getSystem} />
                   <AsyncRoute path="/styleguide" getComponent={Routes.getStyleGuide} />
                   <Cameras default path="/" />

web/src/routes/Storage.jsx

@@ -0,0 +1,109 @@
+import { h, Fragment } from 'preact';
+import ActivityIndicator from '../components/ActivityIndicator';
+import Heading from '../components/Heading';
+import { useWs } from '../api/ws';
+import useSWR from 'swr';
+import { Table, Tbody, Thead, Tr, Th, Td } from '../components/Table';
+
+const emptyObject = Object.freeze({});
+
+export default function Storage() {
+  const { data: storage } = useSWR('recordings/storage');
+
+  const {
+    value: { payload: stats },
+  } = useWs('stats');
+  const { data: initialStats } = useSWR('stats');
+
+  const { service } = stats || initialStats || emptyObject;
+
+  return (
+    <div className="space-y-4 p-2 px-4">
+      <Heading>Storage</Heading>
+
+      {(!service || !storage) ? (
+        <div>
+          <ActivityIndicator />
+        </div>
+      ) : (
+        <Fragment>
+          <Heading size="lg">Overview</Heading>
+          <div data-testid="detectors" className="grid grid-cols-1 md:grid-cols-2 gap-4">
+            <div className="dark:bg-gray-800 shadow-md hover:shadow-lg rounded-lg transition-shadow">
+              <div className="text-lg flex justify-between p-4">Data</div>
+              <div className="p-2">
+                <Table className="w-full">
+                  <Thead>
+                    <Tr>
+                      <Th>Location</Th>
+                      <Th>Used MB</Th>
+                      <Th>Total MB</Th>
+                    </Tr>
+                  </Thead>
+                  <Tbody>
+                    <Tr>
+                      <Td>Snapshots & Recordings</Td>
+                      <Td>{service['storage']['/media/frigate/recordings']['used']}</Td>
+                      <Td>{service['storage']['/media/frigate/recordings']['total']}</Td>
+                    </Tr>
+                  </Tbody>
+                </Table>
+              </div>
+            </div>
+            <div className="dark:bg-gray-800 shadow-md hover:shadow-lg rounded-lg transition-shadow">
+              <div className="text-lg flex justify-between p-4">Memory</div>
+              <div className="p-2">
+                <Table className="w-full">
+                  <Thead>
+                    <Tr>
+                      <Th>Location</Th>
+                      <Th>Used MB</Th>
+                      <Th>Total MB</Th>
+                    </Tr>
+                  </Thead>
+                  <Tbody>
+                    <Tr>
+                      <Td>/dev/shm</Td>
+                      <Td>{service['storage']['/dev/shm']['used']}</Td>
+                      <Td>{service['storage']['/dev/shm']['total']}</Td>
+                    </Tr>
+                    <Tr>
+                      <Td>/tmp/cache</Td>
+                      <Td>{service['storage']['/tmp/cache']['used']}</Td>
+                      <Td>{service['storage']['/tmp/cache']['total']}</Td>
+                    </Tr>
+                  </Tbody>
+                </Table>
+              </div>
+            </div>
+          </div>
+
+          <Heading size="lg">Cameras</Heading>
+          <div data-testid="detectors" className="grid grid-cols-1 3xl:grid-cols-3 md:grid-cols-2 gap-4">
+            {Object.entries(storage).map(([name, camera]) => (
+              <div key={name} className="dark:bg-gray-800 shadow-md hover:shadow-lg rounded-lg transition-shadow">
+                <div className="text-lg flex justify-between p-4">{name}</div>
+                <div className="p-2">
+                  <Table className="w-full">
+                    <Thead>
+                      <Tr>
+                        <Th>Usage</Th>
+                        <Th>Stream Bandwidth</Th>
+                      </Tr>
+                    </Thead>
+                    <Tbody>
+                      <Tr>
+                        <Td>{Math.round(camera['usage_percent'] ?? 0)}%</Td>
+                        <Td>{camera['bandwidth']} MB/hr</Td>
+                      </Tr>
+                    </Tbody>
+                  </Table>
+                </div>
+              </div>
+            ))}
+          </div>
+        </Fragment>
+      )}
+    </div>
+  );
+}

web/src/routes/index.js

@@ -33,6 +33,11 @@ export async function getSystem(_url, _cb, _props) {
   return module.default;
 }
 
+export async function getStorage(_url, _cb, _props) {
+  const module = await import('./Storage.jsx');
+  return module.default;
+}
+
 export async function getStyleGuide(_url, _cb, _props) {
   const module = await import('./StyleGuide.jsx');
   return module.default;