diff --git a/docs/source/device/oak.rst b/docs/source/device/oak.rst index ccc37c59a..89d9e4e97 100644 --- a/docs/source/device/oak.rst +++ b/docs/source/device/oak.rst @@ -108,34 +108,39 @@ controllers, etc.): :doc:`TeleopSession <../getting_started/teleop_session>`'s ``PluginConfig``, passing ``--collection-prefix`` so the plugin pushes metadata via OpenXR ``SchemaPusher``. -2. **Host tracker** — create ``FrameMetadataTrackerOak`` with the **same** - collection prefix and stream list (see :doc:`trackers`). TeleopSession's +2. **Host trackers** — create one ``FrameMetadataTrackerOak`` **per stream**, + each with collection id ``{collection_prefix}/{StreamName}`` using the + **same** prefix the plugin was given (see :doc:`trackers`). TeleopSession's DeviceIO layer uses the live tracker implementation to read the pushed tensors and write MCAP channels. -3. **MCAP config** — add the tracker to both ``TeleopSessionConfig.trackers`` - and ``McapRecordingConfig.tracker_names``. See also - :doc:`../references/mcap_record_replay`. +3. **MCAP config** — add every tracker to both ``TeleopSessionConfig.trackers`` + and ``McapRecordingConfig.tracker_names``, each under its own base name. See + also :doc:`../references/mcap_record_replay`. .. code-block:: python from pathlib import Path - from isaacteleop.deviceio import FrameMetadataTrackerOak, McapRecordingConfig, StreamType + from isaacteleop.deviceio import FrameMetadataTrackerOak, McapRecordingConfig from isaacteleop.teleop_session_manager import PluginConfig, TeleopSession, TeleopSessionConfig PLUGIN_ROOT = Path("build/src/plugins") # or your installed plugin search path COLLECTION_PREFIX = "oak_camera" - STREAMS = [StreamType.Color, StreamType.MonoLeft] + STREAM_NAMES = ["Color", "MonoLeft"] - oak_tracker = FrameMetadataTrackerOak(COLLECTION_PREFIX, STREAMS) + # One tracker per stream, each on the collection the plugin publishes it under. + oak_trackers = { + name: FrameMetadataTrackerOak(f"{COLLECTION_PREFIX}/{name}") + for name in STREAM_NAMES + } config = TeleopSessionConfig( app_name="OakTeleop", pipeline=pipeline, # your retargeting pipeline - trackers=[oak_tracker], + trackers=list(oak_trackers.values()), mcap_config=McapRecordingConfig( "recording.mcap", - [(oak_tracker, "oak_metadata")], + [(tracker, f"oak_metadata_{name.lower()}") for name, tracker in oak_trackers.items()], ), plugins=[ PluginConfig( diff --git a/docs/source/device/trackers.rst b/docs/source/device/trackers.rst index 3334ff0f0..5bf63f75f 100644 --- a/docs/source/device/trackers.rst +++ b/docs/source/device/trackers.rst @@ -20,7 +20,7 @@ APIs (``xrLocateSpace``, ``xrSyncActions``, etc.): reading it from OpenXR tensor collections via the :code-file:`SchemaTracker ` utility. -- :code-file:`FrameMetadataTrackerOak ` -- per-stream frame metadata from OAK cameras +- :code-file:`FrameMetadataTrackerOak ` -- frame metadata for one OAK camera stream - :code-file:`Generic3AxisPedalTracker ` -- foot pedal axis values - :code-file:`JointStateTracker ` -- named joint-space device state (leader arms, exoskeletons, gloves, ...) - :code-file:`Se3Tracker ` -- generic SE3 (6-DoF) pose sources (tracker pucks, mocap rigid bodies, logical trackers) @@ -220,14 +220,16 @@ reads the PICO ``XR_BD_body_tracking`` extension directly. FrameMetadataTrackerOak ~~~~~~~~~~~~~~~~~~~~~~~ -Multi-channel tracker for per-frame metadata from OAK camera streams. -Uses the :code-file:`SchemaTracker ` +Per-frame metadata for a **single** OAK camera stream. Create one tracker per +stream, passing the tensor collection the plugin publishes that stream under -- +``{collection_prefix}/{StreamName}``, e.g. ``"oak_camera/Color"``. Uses the +:code-file:`SchemaTracker ` utility internally. - Schema: :code-file:`src/core/schema/fbs/oak.fbs` - C++ header: ``#include `` - Python import: ``from isaacteleop.deviceio import FrameMetadataTrackerOak`` -- Record channels: one per configured stream (e.g. ``Color``, ``MonoLeft``) | MCAP schema: ``core.FrameMetadataOakRecord`` +- Record channels: ``oak``, ``oak_tracked`` | MCAP schema: ``core.FrameMetadataOakRecord`` - Tests: - :code-file:`src/core/schema_tests/cpp/test_oak.cpp` diff --git a/examples/oxr/python/test_oak_camera.py b/examples/oxr/python/test_oak_camera.py index 6647e4092..7ed6d03ad 100755 --- a/examples/oxr/python/test_oak_camera.py +++ b/examples/oxr/python/test_oak_camera.py @@ -55,21 +55,26 @@ def _run_recording_loop(plugin, duration: float): def _run_schema_pusher( plugin, duration: float, - tracker, - stream_names: list[str], + trackers: dict, required_extensions: list[str], mcap_filename: str, ): - """Read metadata via OpenXR schema tracker and record to MCAP on the host side.""" + """Read metadata via one OpenXR schema tracker per stream, recording to MCAP on the host side.""" + stream_names = list(trackers) with oxr.OpenXRSession("OakCameraTest", required_extensions) as oxr_session: handles = oxr_session.get_handles() print(" ✓ OpenXR session created") + # One tracker per stream, so each gets its own MCAP base name. recording_config = deviceio.McapRecordingConfig( - mcap_filename, [(tracker, "oak_metadata")] + mcap_filename, + [ + (tracker, f"oak_metadata_{name.lower()}") + for name, tracker in trackers.items() + ], ) with deviceio.DeviceIOSession.run( - [tracker], handles, recording_config + list(trackers.values()), handles, recording_config ) as session: print(" ✓ DeviceIO session initialized (recording active during update())") print() @@ -88,8 +93,8 @@ def _run_schema_pusher( frame_count += 1 elapsed = time.time() - start_time - for idx, name in enumerate(stream_names): - tracked = tracker.get_stream_data(session, idx) + for name, tracker in trackers.items(): + tracked = tracker.get_data(session) if ( tracked.data is not None and tracked.data.sequence_number != last_seq.get(name, -1) @@ -152,22 +157,22 @@ def run_test(duration: float = 10.0, mode: str = MODE_NO_METADATA): # 3. Prepare mode-specific state stream_names = ["Color", "MonoLeft"] - stream_types = [deviceio.StreamType.Color, deviceio.StreamType.MonoLeft] collection_prefix = "oak_camera" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") mcap_filename = f"camera_metadata_{timestamp}.mcap" - tracker = None + trackers = {} required_extensions = [] if mode == MODE_SCHEMA_PUSHER: - print("[Step 3] Creating composite FrameMetadataTrackerOak...") - tracker = deviceio.FrameMetadataTrackerOak(collection_prefix, stream_types) - print( - f" Created tracker (prefix: {collection_prefix}, streams: {stream_names})" - ) + print("[Step 3] Creating one FrameMetadataTrackerOak per stream...") + # The plugin publishes each stream as "{collection_prefix}/{StreamName}". + for name in stream_names: + collection_id = f"{collection_prefix}/{name}" + trackers[name] = deviceio.FrameMetadataTrackerOak(collection_id) + print(f" Created tracker for {name} (collection: {collection_id})") required_extensions = deviceio.DeviceIOSession.get_required_extensions( - [tracker] + list(trackers.values()) ) print() print("[Step 4] Getting required OpenXR extensions...") @@ -210,8 +215,7 @@ def run_test(duration: float = 10.0, mode: str = MODE_NO_METADATA): _run_schema_pusher( plugin, duration, - tracker, - stream_names, + trackers, required_extensions, mcap_filename, ) diff --git a/examples/schemaio/frame_metadata_printer.cpp b/examples/schemaio/frame_metadata_printer.cpp index b7ce5ac2e..021391d1c 100644 --- a/examples/schemaio/frame_metadata_printer.cpp +++ b/examples/schemaio/frame_metadata_printer.cpp @@ -5,8 +5,8 @@ * @file frame_metadata_printer.cpp * @brief Standalone application that reads and prints camera frame metadata from the OpenXR runtime. * - * This application demonstrates using FrameMetadataTrackerOak to read per-stream - * FrameMetadataOak pushed by a camera plugin, with each stream on its own MCAP channel. + * This application demonstrates using one FrameMetadataTrackerOak per camera stream + * to read the FrameMetadataOak pushed by a camera plugin. * * Usage: * ./frame_metadata_printer --collection-prefix= @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -67,15 +68,21 @@ try std::cout << "Frame Metadata Printer (prefix: " << collection_prefix << ")" << std::endl; - // Track all three stream types; streams without a pusher simply won't receive data. - std::vector streams = { core::StreamType_Color, core::StreamType_MonoLeft, - core::StreamType_MonoRight }; + // One tracker per stream; streams without a pusher simply won't receive data. + // The plugin publishes each stream as "{collection_prefix}/{StreamName}". + const std::vector stream_names = { "Color", "MonoLeft", "MonoRight" }; - std::cout << "[Step 1] Creating FrameMetadataTrackerOak..." << std::endl; - auto tracker = std::make_shared(collection_prefix, streams, MAX_FLATBUFFER_SIZE); + std::cout << "[Step 1] Creating one FrameMetadataTrackerOak per stream..." << std::endl; + std::vector> stream_trackers; + for (const auto& name : stream_names) + { + const std::string collection_id = collection_prefix + "/" + name; + std::cout << " " << collection_id << std::endl; + stream_trackers.push_back(std::make_shared(collection_id, MAX_FLATBUFFER_SIZE)); + } std::cout << "[Step 2] Creating OpenXR session with required extensions..." << std::endl; - std::vector> trackers = { tracker }; + std::vector> trackers(stream_trackers.begin(), stream_trackers.end()); auto required_extensions = core::DeviceIOSession::get_required_extensions(trackers); auto oxr_session = std::make_shared("FrameMetadataPrinter", required_extensions); std::cout << " OpenXR session created" << std::endl; @@ -91,11 +98,10 @@ try // been observed — the first sample is always printed regardless of its value. // If data is already present at startup we seed with its sequence so we don't // reprint it; absent data stays nullopt so sequence 0 is never skipped. - size_t stream_count = tracker->get_stream_count(); - std::vector> last_sequences(stream_count); - for (size_t i = 0; i < stream_count; ++i) + std::vector> last_sequences(stream_trackers.size()); + for (size_t i = 0; i < stream_trackers.size(); ++i) { - const auto& tracked = tracker->get_stream_data(*session, i); + const auto& tracked = stream_trackers[i]->get_data(*session); if (tracked.data) { last_sequences[i] = tracked.data->sequence_number; @@ -109,28 +115,10 @@ try { session->update(); - // Refresh stream count and extend per-stream tracking if streams were added. - stream_count = tracker->get_stream_count(); - if (last_sequences.size() != stream_count) - { - size_t old_count = last_sequences.size(); - last_sequences.resize(stream_count); - // Newly added streams start as nullopt; seed with current sequence if - // data is already present so we don't reprint an existing sample. - for (size_t i = old_count; i < stream_count; ++i) - { - const auto& tracked = tracker->get_stream_data(*session, i); - if (tracked.data) - { - last_sequences[i] = tracked.data->sequence_number; - } - } - } - // Print one line per stream that has a new sample. - for (size_t i = 0; i < stream_count; ++i) + for (size_t i = 0; i < stream_trackers.size(); ++i) { - const auto& tracked = tracker->get_stream_data(*session, i); + const auto& tracked = stream_trackers[i]->get_data(*session); if (!tracked.data || (last_sequences[i].has_value() && tracked.data->sequence_number == last_sequences[i].value())) { diff --git a/src/core/deviceio_base/cpp/inc/deviceio_base/frame_metadata_tracker_oak_base.hpp b/src/core/deviceio_base/cpp/inc/deviceio_base/frame_metadata_tracker_oak_base.hpp index cebeb7c13..3add0a86c 100644 --- a/src/core/deviceio_base/cpp/inc/deviceio_base/frame_metadata_tracker_oak_base.hpp +++ b/src/core/deviceio_base/cpp/inc/deviceio_base/frame_metadata_tracker_oak_base.hpp @@ -5,8 +5,6 @@ #include "tracker.hpp" -#include - namespace core { @@ -16,7 +14,7 @@ struct FrameMetadataOakTrackedT; class IFrameMetadataTrackerOakImpl : public ITrackerImpl { public: - virtual const FrameMetadataOakTrackedT& get_stream_data(size_t stream_index) const = 0; + virtual const FrameMetadataOakTrackedT& get_data() const = 0; }; } // namespace core diff --git a/src/core/deviceio_trackers/cpp/frame_metadata_tracker_oak.cpp b/src/core/deviceio_trackers/cpp/frame_metadata_tracker_oak.cpp index 5c7485448..de9b8933f 100644 --- a/src/core/deviceio_trackers/cpp/frame_metadata_tracker_oak.cpp +++ b/src/core/deviceio_trackers/cpp/frame_metadata_tracker_oak.cpp @@ -3,9 +3,6 @@ #include "inc/deviceio_trackers/frame_metadata_tracker_oak.hpp" -#include -#include - namespace core { @@ -13,32 +10,15 @@ namespace core // FrameMetadataTrackerOak // ============================================================================ -FrameMetadataTrackerOak::FrameMetadataTrackerOak(const std::string& collection_prefix, - const std::vector& streams, - size_t max_flatbuffer_size) - : collection_prefix_(collection_prefix), streams_(streams), max_flatbuffer_size_(max_flatbuffer_size) +FrameMetadataTrackerOak::FrameMetadataTrackerOak(const std::string& collection_id, size_t max_flatbuffer_size) + : collection_id_(collection_id), max_flatbuffer_size_(max_flatbuffer_size) { - if (streams.empty()) - { - throw std::runtime_error("FrameMetadataTrackerOak: at least one stream is required"); - } - - for (auto type : streams) - { - const char* name = EnumNameStreamType(type); - if (name == nullptr) - { - throw std::invalid_argument("FrameMetadataTrackerOak: invalid StreamType value " + - std::to_string(static_cast(type))); - } - m_stream_names.emplace_back(name); - } + // No-op: members set in the initializer list. } -const FrameMetadataOakTrackedT& FrameMetadataTrackerOak::get_stream_data(const ITrackerSession& session, - size_t stream_index) const +const FrameMetadataOakTrackedT& FrameMetadataTrackerOak::get_data(const ITrackerSession& session) const { - return static_cast(session.get_tracker_impl(*this)).get_stream_data(stream_index); + return static_cast(session.get_tracker_impl(*this)).get_data(); } } // namespace core diff --git a/src/core/deviceio_trackers/cpp/inc/deviceio_trackers/frame_metadata_tracker_oak.hpp b/src/core/deviceio_trackers/cpp/inc/deviceio_trackers/frame_metadata_tracker_oak.hpp index 105c81cbb..b75459aa4 100644 --- a/src/core/deviceio_trackers/cpp/inc/deviceio_trackers/frame_metadata_tracker_oak.hpp +++ b/src/core/deviceio_trackers/cpp/inc/deviceio_trackers/frame_metadata_tracker_oak.hpp @@ -4,30 +4,32 @@ #pragma once #include -#include #include #include -#include +#include namespace core { /*! - * @brief Multi-stream tracker for OAK FrameMetadataOak. + * @brief Tracker for one OAK camera stream's FrameMetadataOak. * - * Maintains one SchemaTracker per stream. Each stream is identified by its - * StreamType enum name (e.g., "Color", "MonoLeft"). + * One tracker instance tracks one stream, identified by the tensor collection the + * OAK plugin publishes it under: "{collection_prefix}/{StreamName}", where + * collection_prefix is the plugin's --collection-prefix argument and StreamName is + * the StreamType enum name ("Color", "MonoLeft", "MonoRight"). Create one tracker + * per stream you care about. * * Usage: * @code - * auto tracker = std::make_shared( - * "oak_camera", {StreamType_Color, StreamType_MonoLeft}); - * // ... create session with tracker ... + * auto color = std::make_shared("oak_camera/Color"); + * auto mono = std::make_shared("oak_camera/MonoLeft"); + * // ... create session with both trackers ... * session->update(); - * const auto& color = tracker->get_stream_data(*session, 0); - * if (color.data) - * std::cout << EnumNameStreamType(color.data->stream) << " seq=" << color.data->sequence_number << std::endl; + * const auto& tracked = color->get_data(*session); + * if (tracked.data) + * std::cout << "seq=" << tracked.data->sequence_number << std::endl; * @endcode */ class FrameMetadataTrackerOak : public ITracker @@ -37,15 +39,13 @@ class FrameMetadataTrackerOak : public ITracker static constexpr size_t DEFAULT_MAX_FLATBUFFER_SIZE = 128; /*! - * @brief Constructs a multi-stream FrameMetadataOak tracker. - * @param collection_prefix Base prefix for per-stream collection IDs. - * Each stream gets collection_id = "{collection_prefix}/{StreamName}". - * @param streams Stream types to track. - * @param max_flatbuffer_size Maximum serialized FlatBuffer size per stream (default: 128 bytes). + * @brief Constructs a tracker for a single OAK stream. + * @param collection_id Tensor collection carrying this stream's metadata, + * e.g. "oak_camera/Color". + * @param max_flatbuffer_size Maximum serialized FlatBuffer size (default: 128 bytes). */ - FrameMetadataTrackerOak(const std::string& collection_prefix, - const std::vector& streams, - size_t max_flatbuffer_size = DEFAULT_MAX_FLATBUFFER_SIZE); + explicit FrameMetadataTrackerOak(const std::string& collection_id, + size_t max_flatbuffer_size = DEFAULT_MAX_FLATBUFFER_SIZE); std::string_view get_name() const override { @@ -53,30 +53,18 @@ class FrameMetadataTrackerOak : public ITracker } /*! - * @brief Get per-stream frame metadata. + * @brief Get this stream's frame metadata. * @param session Active ITrackerSession. - * @param stream_index Index into the streams vector passed at construction. - * @return Reference to the FrameMetadataOakTrackedT for that stream. + * @return Reference to the FrameMetadataOakTrackedT for this stream. * The inner @c data pointer is null until the first frame arrives. * When @c data is non-null, nested fields in FrameMetadataOakT are * safe to read. */ - const FrameMetadataOakTrackedT& get_stream_data(const ITrackerSession& session, size_t stream_index) const; + const FrameMetadataOakTrackedT& get_data(const ITrackerSession& session) const; - //! Number of streams this tracker is configured for. - size_t get_stream_count() const + const std::string& collection_id() const { - return m_stream_names.size(); - } - - const std::string& collection_prefix() const - { - return collection_prefix_; - } - - const std::vector& streams() const - { - return streams_; + return collection_id_; } size_t max_flatbuffer_size() const @@ -84,18 +72,11 @@ class FrameMetadataTrackerOak : public ITracker return max_flatbuffer_size_; } - const std::vector& get_stream_names() const - { - return m_stream_names; - } - private: static constexpr const char* TRACKER_NAME = "FrameMetadataTrackerOak"; - std::string collection_prefix_; - std::vector streams_; - size_t max_flatbuffer_size_{ DEFAULT_MAX_FLATBUFFER_SIZE }; - std::vector m_stream_names; + std::string collection_id_; + size_t max_flatbuffer_size_; }; } // namespace core diff --git a/src/core/deviceio_trackers/python/tracker_bindings.cpp b/src/core/deviceio_trackers/python/tracker_bindings.cpp index a36b99213..daa64766a 100644 --- a/src/core/deviceio_trackers/python/tracker_bindings.cpp +++ b/src/core/deviceio_trackers/python/tracker_bindings.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -168,19 +169,17 @@ PYBIND11_MODULE(_deviceio_trackers, m) py::class_>( m, "FrameMetadataTrackerOak") - .def(py::init&, size_t>(), py::arg("collection_prefix"), - py::arg("streams"), + .def(py::init(), py::arg("collection_id"), py::arg("max_flatbuffer_size") = core::FrameMetadataTrackerOak::DEFAULT_MAX_FLATBUFFER_SIZE, - "Construct a multi-stream FrameMetadataTrackerOak") + "Construct a FrameMetadataTrackerOak for one OAK stream's tensor collection, named " + "\"{collection_prefix}/{StreamName}\" (e.g. \"oak_camera/Color\"); create one per stream") .def( - "get_stream_data", - [](const core::FrameMetadataTrackerOak& self, const core::ITrackerSession& session, size_t stream_index) - { return share_tracked(self.get_stream_data(session, stream_index)); }, - py::arg("session"), py::arg("stream_index"), - "Get FrameMetadataOakTrackedT for a specific stream by index; .data is None until first frame " - "arrives" TRACKED_LIFETIME_DOC) - .def_property_readonly("stream_count", &core::FrameMetadataTrackerOak::get_stream_count, - "Number of streams this tracker is configured for"); + "get_data", + [](const core::FrameMetadataTrackerOak& self, const core::ITrackerSession& session) + { return share_tracked(self.get_data(session)); }, + py::arg("session"), + "Get this stream's FrameMetadataOakTrackedT; .data is None until first frame " + "arrives" TRACKED_LIFETIME_DOC); py::class_>( m, "Generic3AxisPedalTracker") diff --git a/src/core/live_trackers/cpp/live_deviceio_factory.cpp b/src/core/live_trackers/cpp/live_deviceio_factory.cpp index 940e7b879..b78cab73f 100644 --- a/src/core/live_trackers/cpp/live_deviceio_factory.cpp +++ b/src/core/live_trackers/cpp/live_deviceio_factory.cpp @@ -552,7 +552,7 @@ std::unique_ptr LiveDeviceIOFactory::create_frame_ std::unique_ptr channels; if (should_record(tracker)) { - channels = LiveFrameMetadataTrackerOakImpl::create_mcap_channels(*writer_, get_name(tracker), tracker); + channels = LiveFrameMetadataTrackerOakImpl::create_mcap_channels(*writer_, get_name(tracker)); } return std::make_unique(handles_, tracker, std::move(channels)); } diff --git a/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.cpp b/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.cpp index fa28deb80..346692077 100644 --- a/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.cpp +++ b/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.cpp @@ -6,7 +6,6 @@ #include #include -#include #include namespace core @@ -15,21 +14,14 @@ namespace core namespace { -std::vector make_oak_tensor_configs(const FrameMetadataTrackerOak* tracker) +SchemaTrackerConfig make_frame_metadata_oak_tensor_config(const FrameMetadataTrackerOak* tracker) { - std::vector configs; - configs.reserve(tracker->streams().size()); - for (auto type : tracker->streams()) - { - const char* name = EnumNameStreamType(type); - SchemaTrackerConfig cfg; - cfg.collection_id = tracker->collection_prefix() + "/" + name; - cfg.max_flatbuffer_size = tracker->max_flatbuffer_size(); - cfg.tensor_identifier = "frame_metadata"; - cfg.localized_name = std::string("FrameMetadataTracker_") + name; - configs.push_back(std::move(cfg)); - } - return configs; + SchemaTrackerConfig cfg; + cfg.collection_id = tracker->collection_id(); + cfg.max_flatbuffer_size = tracker->max_flatbuffer_size(); + cfg.tensor_identifier = "frame_metadata"; + cfg.localized_name = "FrameMetadataTrackerOak"; + return cfg; } } // namespace @@ -38,46 +30,33 @@ std::vector make_oak_tensor_configs(const FrameMetadataTrac // LiveFrameMetadataTrackerOakImpl // ============================================================================ -std::unique_ptr LiveFrameMetadataTrackerOakImpl::create_mcap_channels( - mcap::McapWriter& writer, std::string_view base_name, const FrameMetadataTrackerOak* tracker) +std::unique_ptr LiveFrameMetadataTrackerOakImpl::create_mcap_channels(mcap::McapWriter& writer, + std::string_view base_name) { - return std::make_unique( - writer, base_name, OakRecordingTraits::schema_name, tracker->get_stream_names()); + return std::make_unique(writer, base_name, OakRecordingTraits::schema_name, + std::vector(OakRecordingTraits::recording_channels.begin(), + OakRecordingTraits::recording_channels.end())); } LiveFrameMetadataTrackerOakImpl::LiveFrameMetadataTrackerOakImpl(const OpenXRSessionHandles& handles, const FrameMetadataTrackerOak* tracker, std::unique_ptr mcap_channels) - : mcap_channels_(std::move(mcap_channels)) + : mcap_channels_(std::move(mcap_channels)), + m_schema_reader(handles, make_frame_metadata_oak_tensor_config(tracker), mcap_channels_.get(), 0, 1) { - auto configs = make_oak_tensor_configs(tracker); - for (size_t i = 0; i < configs.size(); ++i) - { - StreamState state; - state.reader = std::make_unique(handles, std::move(configs[i]), mcap_channels_.get(), i); - m_streams.push_back(std::move(state)); - } + // No-op: members set in the initializer list. } void LiveFrameMetadataTrackerOakImpl::update(int64_t /*monotonic_time_ns*/) { - // Policy: per-stream SchemaTracker throws on critical OpenXR/tensor API failures. - // Missing stream collection/no fresh sample are treated as common non-fatal cases. - for (auto& stream : m_streams) - { - stream.reader->update(stream.tracked.data); - } + // Policy: SchemaTracker throws on critical OpenXR/tensor API failures. Missing + // stream collection / no fresh sample are treated as common non-fatal cases. + m_schema_reader.update(m_tracked.data); } -const FrameMetadataOakTrackedT& LiveFrameMetadataTrackerOakImpl::get_stream_data(size_t stream_index) const +const FrameMetadataOakTrackedT& LiveFrameMetadataTrackerOakImpl::get_data() const { - if (stream_index >= m_streams.size()) - { - throw std::runtime_error("FrameMetadataTrackerOak::get_stream_data: invalid stream_index " + - std::to_string(stream_index) + " (have " + std::to_string(m_streams.size()) + - " streams)"); - } - return m_streams[stream_index].tracked; + return m_tracked; } } // namespace core diff --git a/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.hpp b/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.hpp index f839eca7b..2dbc097fd 100644 --- a/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.hpp +++ b/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.hpp @@ -28,9 +28,7 @@ class LiveFrameMetadataTrackerOakImpl : public IFrameMetadataTrackerOakImpl { return SchemaTrackerBase::get_required_extensions(); } - static std::unique_ptr create_mcap_channels(mcap::McapWriter& writer, - std::string_view base_name, - const FrameMetadataTrackerOak* tracker); + static std::unique_ptr create_mcap_channels(mcap::McapWriter& writer, std::string_view base_name); LiveFrameMetadataTrackerOakImpl(const OpenXRSessionHandles& handles, const FrameMetadataTrackerOak* tracker, @@ -42,17 +40,12 @@ class LiveFrameMetadataTrackerOakImpl : public IFrameMetadataTrackerOakImpl LiveFrameMetadataTrackerOakImpl& operator=(LiveFrameMetadataTrackerOakImpl&&) = delete; void update(int64_t monotonic_time_ns) override; - const FrameMetadataOakTrackedT& get_stream_data(size_t stream_index) const override; + const FrameMetadataOakTrackedT& get_data() const override; private: - struct StreamState - { - std::unique_ptr reader; - FrameMetadataOakTrackedT tracked; - }; - std::unique_ptr mcap_channels_; - std::vector m_streams; + OakSchemaTracker m_schema_reader; + FrameMetadataOakTrackedT m_tracked; }; } // namespace core diff --git a/src/core/mcap/cpp/inc/mcap/recording_traits.hpp b/src/core/mcap/cpp/inc/mcap/recording_traits.hpp index 4e5b61b4a..182be8956 100644 --- a/src/core/mcap/cpp/inc/mcap/recording_traits.hpp +++ b/src/core/mcap/cpp/inc/mcap/recording_traits.hpp @@ -79,9 +79,12 @@ struct Se3TrackerRecordingTraits static constexpr std::array replay_channels = { "se3_tracker_tracked" }; }; +// No replay_channels yet: there is no replay OAK impl, so nothing would consume it. +// It arrives with the generated replay half when this tracker moves to the manifest. struct OakRecordingTraits { static constexpr std::string_view schema_name = "core.FrameMetadataOakRecord"; + static constexpr std::array recording_channels = { "oak", "oak_tracked" }; }; struct MessageChannelRecordingTraits diff --git a/src/plugins/oak/README.md b/src/plugins/oak/README.md index 289ba7362..f40005f83 100644 --- a/src/plugins/oak/README.md +++ b/src/plugins/oak/README.md @@ -90,32 +90,36 @@ controllers, etc.): 1. **Plugin** — launch `oak_camera` via PluginManager or `TeleopSession`'s `PluginConfig`, passing `--collection-prefix` so the plugin pushes metadata via OpenXR `SchemaPusher`. -2. **Host tracker** — create `FrameMetadataTrackerOak` with the **same** - collection prefix and stream list. TeleopSession's DeviceIO layer uses the - live tracker implementation to read the pushed tensors and write MCAP - channels. -3. **MCAP config** — add the tracker to both `TeleopSessionConfig.trackers` - and `McapRecordingConfig.tracker_names`. +2. **Host trackers** — create one `FrameMetadataTrackerOak` **per stream**, each + with collection id `{collection_prefix}/{StreamName}` using the **same** prefix + the plugin was given. TeleopSession's DeviceIO layer uses the live tracker + implementation to read the pushed tensors and write MCAP channels. +3. **MCAP config** — add every tracker to both `TeleopSessionConfig.trackers` + and `McapRecordingConfig.tracker_names`, each under its own base name. ```python from pathlib import Path -from isaacteleop.deviceio import FrameMetadataTrackerOak, McapRecordingConfig, StreamType +from isaacteleop.deviceio import FrameMetadataTrackerOak, McapRecordingConfig from isaacteleop.teleop_session_manager import PluginConfig, TeleopSession, TeleopSessionConfig PLUGIN_ROOT = Path("build/src/plugins") # or your installed plugin search path COLLECTION_PREFIX = "oak_camera" -STREAMS = [StreamType.Color, StreamType.MonoLeft] +STREAM_NAMES = ["Color", "MonoLeft"] -oak_tracker = FrameMetadataTrackerOak(COLLECTION_PREFIX, STREAMS) +# One tracker per stream, each on the collection the plugin publishes it under. +oak_trackers = { + name: FrameMetadataTrackerOak(f"{COLLECTION_PREFIX}/{name}") + for name in STREAM_NAMES +} config = TeleopSessionConfig( app_name="OakTeleop", pipeline=pipeline, # your retargeting pipeline - trackers=[oak_tracker], + trackers=list(oak_trackers.values()), mcap_config=McapRecordingConfig( "recording.mcap", - [(oak_tracker, "oak_metadata")], + [(tracker, f"oak_metadata_{name.lower()}") for name, tracker in oak_trackers.items()], ), plugins=[ PluginConfig(