From 2ad870a58d063645618f96936cdc80a5234cc908 Mon Sep 17 00:00:00 2001 From: logbasem Date: Thu, 16 Apr 2026 19:49:17 -0500 Subject: [PATCH 01/19] Got rid of old comment --- react-app/src/App.tsx | 9 --------- 1 file changed, 9 deletions(-) diff --git a/react-app/src/App.tsx b/react-app/src/App.tsx index 84dea05..d976196 100644 --- a/react-app/src/App.tsx +++ b/react-app/src/App.tsx @@ -14,15 +14,6 @@ function App() { const connection = useRef(null); const videoDivRef = useRef(null); - /*//Effect to get the camera names from the server - useEffect(() => { - fetch('/video_feed/available_devices') - .then(response => response.json()) - .then(data => { - updateCameraNames(data) - }) - }, [])*/ - // Fetch Camera(s) Information from Server useEffect(() => { (async () => { From b907003a293ca35e33dac09073b54c70691e3d47 Mon Sep 17 00:00:00 2001 From: logbasem Date: Thu, 16 Apr 2026 22:07:17 -0500 Subject: [PATCH 02/19] Impelemented static camera grid --- react-app/src/App.css | 16 ++++++- react-app/src/App.tsx | 84 ++++++++++++++++++------------------ react-app/src/CameraGrid.css | 31 +++++++++++++ react-app/src/CameraGrid.tsx | 29 +++++++++++++ 4 files changed, 117 insertions(+), 43 deletions(-) create mode 100644 react-app/src/CameraGrid.css create mode 100644 react-app/src/CameraGrid.tsx diff --git a/react-app/src/App.css b/react-app/src/App.css index eb0eb86..cbbfd9a 100644 --- a/react-app/src/App.css +++ b/react-app/src/App.css @@ -1,5 +1,7 @@ .App { text-align: center; + background-color: #282c34; + min-height: 100vh; } .App-logo { @@ -14,14 +16,24 @@ } .camera-select { - background-color: #282c34; - min-height: 100vh; display: flex; flex-direction: column; align-items: center; justify-content: center; font-size: calc(10px + 2vmin); color: white; + padding: 100px; +} + +.camera-grid { + width: 100%; +} + +.camera-content { + width: 100%; + display: flex; + flex-direction: column; + align-items: center; } .camera-feed { diff --git a/react-app/src/App.tsx b/react-app/src/App.tsx index d976196..c68084e 100644 --- a/react-app/src/App.tsx +++ b/react-app/src/App.tsx @@ -1,5 +1,6 @@ import { useState, useEffect, useRef } from "react"; import "./App.css"; +import CameraGrid, { CameraContainer } from "./CameraGrid"; //filepath for testing (DELETE LATER): ../../../GitHub/Automomous/examples/ARTrackerTest/videos function App() { @@ -35,6 +36,14 @@ function App() { //to control the camera feed const [selectedCamera, setSelectedCamera] = useState(""); + const [cameraContainers, setCameraContainers] = useState([ + { id: '1', name: 'Front Door', size: 'large' }, + { id: '2', name: 'Hallway', size: 'large' }, + { id: '3', name: 'Other Camera', size: 'small' }, + { id: '4', name: 'Other Camera', size: 'small' }, + { id: '5', name: 'Other Camera', size: 'small' }, + ]) + const handleCameraChange = async ( event: React.ChangeEvent, ) => { @@ -182,49 +191,42 @@ function App() { value={selectedCamera} onChange={handleCameraChange} > - {// TODO: Add an error display if cameras is null - cameras?.map((camera, index) => { - return ( - - ); - })} + {cameras?.map((camera, index) => ( + + ))} -
-
- {selectedCamera && ( -
- {/*
- Camera Frame -
*/} -
- - ) => - setFpsSlider(Number(event.target.value)) - } - /> - - ) => - setResolutionSlider(Number(event.target.value)) - } - /> -
-
- )} +
+
+
+ + {/*
*/}
+ + {selectedCamera && ( +
+ + setFpsSlider(Number(e.target.value))} + /> + + + setResolutionSlider(Number(e.target.value))} + /> +
+ )}
); diff --git a/react-app/src/CameraGrid.css b/react-app/src/CameraGrid.css new file mode 100644 index 0000000..c42a9c0 --- /dev/null +++ b/react-app/src/CameraGrid.css @@ -0,0 +1,31 @@ +.grid-container { + width: 90%; + margin: 0 auto; + display: grid; + grid-template-columns: repeat(6, 1fr); + gap: 10px; + background-color: #F9DFDF; + padding: 16px; + } + + /* Base tile styling */ + .camera-tile { + height: 150px; + border-radius: 8px; + color: white; + display: flex; + align-items: center; + justify-content: center; + font-weight: bold; + background: linear-gradient(to bottom, black, gray); + padding: 16px; + } + + /* Size variants */ + .camera-tile.large { + grid-column: span 3; /* 1/2 width */ + } + + .camera-tile.small { + grid-column: span 2; /* 1/3 width */ + } \ No newline at end of file diff --git a/react-app/src/CameraGrid.tsx b/react-app/src/CameraGrid.tsx new file mode 100644 index 0000000..b958aaf --- /dev/null +++ b/react-app/src/CameraGrid.tsx @@ -0,0 +1,29 @@ +import React from "react"; +import "./CameraGrid.css"; + +export interface CameraContainer { + id: string; + name: string; + size: "large" | "small"; +} + +interface CameraGridProps { + cameras: CameraContainer[]; +} + +const CameraGrid: React.FC = ({ cameras }) => { + return ( +
+ {cameras.map((cam) => ( +
+ {cam.name} +
+ ))} +
+ ); +}; + +export default CameraGrid; \ No newline at end of file From 4078a76c0aee6381972514272db732cba2b4636f Mon Sep 17 00:00:00 2001 From: logbasem Date: Thu, 16 Apr 2026 22:19:34 -0500 Subject: [PATCH 03/19] Added titles --- react-app/src/CameraGrid.css | 24 ++++++++++++++++++++---- react-app/src/CameraGrid.tsx | 24 ++++++++++++------------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/react-app/src/CameraGrid.css b/react-app/src/CameraGrid.css index c42a9c0..dc58bca 100644 --- a/react-app/src/CameraGrid.css +++ b/react-app/src/CameraGrid.css @@ -10,22 +10,38 @@ /* Base tile styling */ .camera-tile { - height: 150px; border-radius: 8px; color: white; display: flex; - align-items: center; - justify-content: center; + flex-direction: column; /* 👈 key change */ font-weight: bold; background: linear-gradient(to bottom, black, gray); - padding: 16px; + border: 6px solid #841617; + padding: 10px; + } + + .camera-title { + text-align: left; + font-size: 16px; + margin-bottom: 8px; + padding: 4px 8px; + background: rgba(0, 0, 0, 0.4); + border-radius: 4px; + } + + .camera-body { + flex: 1; /* takes remaining space */ + border-radius: 6px; + background: rgba(255, 255, 255, 0.05); } /* Size variants */ .camera-tile.large { + height: 400px; grid-column: span 3; /* 1/2 width */ } .camera-tile.small { + height: 250px; grid-column: span 2; /* 1/3 width */ } \ No newline at end of file diff --git a/react-app/src/CameraGrid.tsx b/react-app/src/CameraGrid.tsx index b958aaf..90ff67c 100644 --- a/react-app/src/CameraGrid.tsx +++ b/react-app/src/CameraGrid.tsx @@ -12,18 +12,18 @@ interface CameraGridProps { } const CameraGrid: React.FC = ({ cameras }) => { - return ( -
- {cameras.map((cam) => ( -
- {cam.name} -
- ))} -
- ); + return ( +
+ {cameras.map((cam) => ( +
+
{cam.name}
+
+ {/* video goes here later */} +
+
+ ))} +
+ ); }; export default CameraGrid; \ No newline at end of file From 9fd6f03562de0a7f376f42977292887cd3fe81b3 Mon Sep 17 00:00:00 2001 From: logbasem Date: Mon, 4 May 2026 12:31:04 -0500 Subject: [PATCH 04/19] Implement working rtc stream logic for multiple cameras --- react-app/src/App.tsx | 204 +++++++++++++++++++++++++----------------- 1 file changed, 124 insertions(+), 80 deletions(-) diff --git a/react-app/src/App.tsx b/react-app/src/App.tsx index c68084e..5d94fcc 100644 --- a/react-app/src/App.tsx +++ b/react-app/src/App.tsx @@ -12,8 +12,8 @@ function App() { const [cameras, setCameras] = useState([]); //getting available devices from server - const connection = useRef(null); - const videoDivRef = useRef(null); + const [cameraConnections, setCameraConnections] = useState>(new Map()); + const videoElementsRef = useRef>(new Map()); // Fetch Camera(s) Information from Server useEffect(() => { @@ -37,11 +37,11 @@ function App() { const [selectedCamera, setSelectedCamera] = useState(""); const [cameraContainers, setCameraContainers] = useState([ - { id: '1', name: 'Front Door', size: 'large' }, - { id: '2', name: 'Hallway', size: 'large' }, - { id: '3', name: 'Other Camera', size: 'small' }, - { id: '4', name: 'Other Camera', size: 'small' }, - { id: '5', name: 'Other Camera', size: 'small' }, + { id: '1', name: 'Camera 1', size: 'large', connection: null, videoElement: null }, + { id: '2', name: 'Camera 2', size: 'large', connection: null, videoElement: null }, + { id: '3', name: 'Camera 3', size: 'small', connection: null, videoElement: null }, + { id: '4', name: 'Camera 4', size: 'small', connection: null, videoElement: null }, + { id: '5', name: 'Camera 5', size: 'small', connection: null, videoElement: null }, ]) const handleCameraChange = async ( @@ -52,104 +52,103 @@ function App() { `stream: camera selection changed to: \`${selectedCameraPath}\``, ); - if (connection.current !== null) { - console.log("stream: closing current connection.", selectedCameraPath); - connection.current.close(); - if (videoDivRef.current) { - videoDivRef.current.innerHTML = ""; - } + setSelectedCamera(selectedCameraPath); + }; + + const handleAddCamera = async (event: React.MouseEvent) => { + + if (selectedCamera === "") { + console.warn("stream: no camera selected; cannot add camera."); + return; } - if (selectedCameraPath === "") { - console.log( - "stream: cannot stream for empty path. early returning...", - selectedCameraPath, + if (cameraConnections.has(selectedCamera)) { + console.warn( + "stream: camera already has an active connection; cannot add camera.", + selectedCamera, ); return; } + const cameraId = selectedCamera; + + // Find the first available container + const availableContainer = cameraContainers.find( + (container) => container.videoElement === null && container.connection === null + ); + + if (!availableContainer) { + console.warn("stream: no available containers for camera"); + return; + } + const peerConnection = new RTCPeerConnection(); + peerConnection.onconnectionstatechange = () => { console.info("stream: peer connection change", { - selectedCameraPath, + cameraId, state: peerConnection.connectionState, }); }; peerConnection.oniceconnectionstatechange = () => { console.info("stream: ice connection state changed", { - selectedCameraPath, + cameraId, state: peerConnection.iceConnectionState, }); }; peerConnection.ontrack = (e) => { - const el = document.createElement(e.track.kind) as HTMLMediaElement; - el.srcObject = e.streams[0] ?? null; - if (el.srcObject === null) { - console.error( - "stream: video `src` was set to `null` for path: ", - selectedCameraPath, - ); + console.debug("stream: received track event on peer connection", { + cameraId, + trackKind: e.track.kind, + streamIds: e.streams.map((s) => s.id), } + ); - el.autoplay = true; - el.controls = true; - el.onerror = (error) => { - console.error("stream: html media element err: ", { - selectedCameraPath, - kind: e.track.kind, - error, - mediaError: el.error, - }); - }; - - if (videoDivRef.current) { - videoDivRef.current.appendChild(el); - console.debug( - "stream: added HTMLMediaElement to videoDiv.", - selectedCameraPath, - ); - } else { - console.error( - "stream: videoDiv missing; cannot append media element.", - selectedCameraPath, - ); - } - }; + if(e.track.kind == "video" && e.streams.length > 0) { + const videoEl = document.createElement("video"); + videoEl.autoplay = true; + videoEl.playsInline = true; + videoEl.controls = true; + videoEl.srcObject = e.streams[0] ?? null; - peerConnection.onicecandidate = async (e) => { - if (e.candidate === null || connection.current !== null) return; - connection.current = peerConnection; - - try { - const response = await fetch( - `/stream/cameras/${encodeURIComponent(selectedCameraPath)}/start`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(peerConnection.localDescription), - }, - ); - if (!response.ok) { - throw new Error( - `failed to start stream! http err: ${response.status}`, + if (videoEl.srcObject === null) { + console.error( + "stream: video `src` was set to `null` for path: ", + cameraId, ); } - const remoteOffer = await response.json(); - await peerConnection.setRemoteDescription( - new RTCSessionDescription(remoteOffer), - ); - setSelectedCamera(selectedCameraPath); - } catch (error) { - console.error("stream: failed to create rtc stream session", { - selectedCameraPath, - error, + setCameraContainers((prev) => { + return prev.map((container) => + container.id === availableContainer.id + ? { + ...container, + name: cameraId, + videoElement: videoEl, + connection: peerConnection, + } + : container + ); }); - } + + // Store ref for cleanup + videoElementsRef.current.set(cameraId, videoEl); + + console.debug("stream: created video element for track event", { + cameraId, + trackKind: e.track.kind, + streamIds: e.streams.map((s) => s.id), + }); + }; + }; + + peerConnection.onicecandidate = async (e) => { + console.debug("stream: ice candidate event", { + cameraId, + candidate: e.candidate, + }); // IMPORTANT: Calls to the API should only run after this point. @@ -174,11 +173,55 @@ function App() { try { const offer = await peerConnection.createOffer(); await peerConnection.setLocalDescription(offer); + + const response = await fetch( + `/stream/cameras/${encodeURIComponent(cameraId)}/start`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(peerConnection.localDescription), + }, + ); + if (!response.ok) { + throw new Error( + `failed to start stream! http err: ${response.status}`, + ); + } + const remoteOffer = await response.json(); + await peerConnection.setRemoteDescription( + new RTCSessionDescription(remoteOffer), + ); + setCameraConnections((prev) => new Map(prev).set(cameraId, peerConnection)); } catch (error) { console.error("stream: failed to do offer/local desc", { - selectedCameraPath, + cameraId, error, }); + + // Cleanup connection on failure + peerConnection.close(); + + videoElementsRef.current.delete(selectedCamera); + + setCameraContainers((prev) => + prev.map((c) => + c.name === selectedCamera + ? { ...c, videoElement: null, connection: null } + : c + ) + ); + } + }; + + const handleRemoveCamera = () => { + const connection = cameraConnections.get(selectedCamera); + if (connection) { + connection.close(); + const updatedConnections = new Map(cameraConnections); + updatedConnections.delete(selectedCamera); + setCameraConnections(updatedConnections); } }; @@ -197,10 +240,11 @@ function App() { ))} + {selectedCamera && }
- + {/*
*/}
From 3579aa73be3cb2da540d6f8b913dc784d0aa8305 Mon Sep 17 00:00:00 2001 From: logbasem Date: Mon, 4 May 2026 12:35:45 -0500 Subject: [PATCH 05/19] Implement working video stream in grid --- react-app/src/App.css | 1 + react-app/src/App.tsx | 30 ++++++++---------------------- react-app/src/CameraGrid.tsx | 21 ++++++++++++++++++++- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/react-app/src/App.css b/react-app/src/App.css index cbbfd9a..7c91c20 100644 --- a/react-app/src/App.css +++ b/react-app/src/App.css @@ -23,6 +23,7 @@ font-size: calc(10px + 2vmin); color: white; padding: 100px; + gap: 10px; } .camera-grid { diff --git a/react-app/src/App.tsx b/react-app/src/App.tsx index 5d94fcc..faa7224 100644 --- a/react-app/src/App.tsx +++ b/react-app/src/App.tsx @@ -37,11 +37,11 @@ function App() { const [selectedCamera, setSelectedCamera] = useState(""); const [cameraContainers, setCameraContainers] = useState([ - { id: '1', name: 'Camera 1', size: 'large', connection: null, videoElement: null }, - { id: '2', name: 'Camera 2', size: 'large', connection: null, videoElement: null }, - { id: '3', name: 'Camera 3', size: 'small', connection: null, videoElement: null }, - { id: '4', name: 'Camera 4', size: 'small', connection: null, videoElement: null }, - { id: '5', name: 'Camera 5', size: 'small', connection: null, videoElement: null }, + { id: '1', name: 'Camera 1', size: 'large', connection: null, stream: null }, + { id: '2', name: 'Camera 2', size: 'large', connection: null, stream: null }, + { id: '3', name: 'Camera 3', size: 'small', connection: null, stream: null }, + { id: '4', name: 'Camera 4', size: 'small', connection: null, stream: null }, + { id: '5', name: 'Camera 5', size: 'small', connection: null, stream: null }, ]) const handleCameraChange = async ( @@ -74,7 +74,7 @@ function App() { // Find the first available container const availableContainer = cameraContainers.find( - (container) => container.videoElement === null && container.connection === null + (container) => container.stream === null && container.connection === null ); if (!availableContainer) { @@ -107,18 +107,7 @@ function App() { ); if(e.track.kind == "video" && e.streams.length > 0) { - const videoEl = document.createElement("video"); - videoEl.autoplay = true; - videoEl.playsInline = true; - videoEl.controls = true; - videoEl.srcObject = e.streams[0] ?? null; - - if (videoEl.srcObject === null) { - console.error( - "stream: video `src` was set to `null` for path: ", - cameraId, - ); - } + const stream = e.streams[0]; setCameraContainers((prev) => { return prev.map((container) => @@ -126,16 +115,13 @@ function App() { ? { ...container, name: cameraId, - videoElement: videoEl, + stream: stream || null, connection: peerConnection, } : container ); }); - // Store ref for cleanup - videoElementsRef.current.set(cameraId, videoEl); - console.debug("stream: created video element for track event", { cameraId, trackKind: e.track.kind, diff --git a/react-app/src/CameraGrid.tsx b/react-app/src/CameraGrid.tsx index 90ff67c..fcb994e 100644 --- a/react-app/src/CameraGrid.tsx +++ b/react-app/src/CameraGrid.tsx @@ -5,10 +5,14 @@ export interface CameraContainer { id: string; name: string; size: "large" | "small"; + connection: RTCPeerConnection | null; + stream: MediaStream | null; } interface CameraGridProps { cameras: CameraContainer[]; + onRemoveCamera: (cameraId: string) => void; + connections: Map; } const CameraGrid: React.FC = ({ cameras }) => { @@ -18,7 +22,22 @@ const CameraGrid: React.FC = ({ cameras }) => {
{cam.name}
- {/* video goes here later */} + {cam.connection && cam.stream ? ( +
))} From cded920ff3448bb6d24eef8fa0d2373cb819b572 Mon Sep 17 00:00:00 2001 From: logbasem Date: Mon, 4 May 2026 14:20:25 -0500 Subject: [PATCH 06/19] Cleaned up extra code and somewhat improved video css --- react-app/src/App.tsx | 3 --- react-app/src/CameraGrid.css | 15 +++++++++++++-- react-app/src/CameraGrid.tsx | 1 + 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/react-app/src/App.tsx b/react-app/src/App.tsx index faa7224..97519a9 100644 --- a/react-app/src/App.tsx +++ b/react-app/src/App.tsx @@ -13,7 +13,6 @@ function App() { const [cameras, setCameras] = useState([]); //getting available devices from server const [cameraConnections, setCameraConnections] = useState>(new Map()); - const videoElementsRef = useRef>(new Map()); // Fetch Camera(s) Information from Server useEffect(() => { @@ -189,8 +188,6 @@ function App() { // Cleanup connection on failure peerConnection.close(); - videoElementsRef.current.delete(selectedCamera); - setCameraContainers((prev) => prev.map((c) => c.name === selectedCamera diff --git a/react-app/src/CameraGrid.css b/react-app/src/CameraGrid.css index dc58bca..76985a7 100644 --- a/react-app/src/CameraGrid.css +++ b/react-app/src/CameraGrid.css @@ -8,12 +8,11 @@ padding: 16px; } - /* Base tile styling */ .camera-tile { border-radius: 8px; color: white; display: flex; - flex-direction: column; /* 👈 key change */ + flex-direction: column; font-weight: bold; background: linear-gradient(to bottom, black, gray); border: 6px solid #841617; @@ -33,6 +32,8 @@ flex: 1; /* takes remaining space */ border-radius: 6px; background: rgba(255, 255, 255, 0.05); + overflow: hidden; + position: relative; } /* Size variants */ @@ -44,4 +45,14 @@ .camera-tile.small { height: 250px; grid-column: span 2; /* 1/3 width */ + } + + .camera-feed { + width: 100%; + height: 100%; + object-fit: cover; + border-radius: 0; + position: absolute; + top: 0; + left: 0; } \ No newline at end of file diff --git a/react-app/src/CameraGrid.tsx b/react-app/src/CameraGrid.tsx index fcb994e..42f6795 100644 --- a/react-app/src/CameraGrid.tsx +++ b/react-app/src/CameraGrid.tsx @@ -22,6 +22,7 @@ const CameraGrid: React.FC = ({ cameras }) => {
{cam.name}
+ {!cam.stream &&
Loading...
} {cam.connection && cam.stream ? (