From e913b9e7c09a378e43c73ec5c4fc1dfbd61a79a8 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 20 May 2026 00:31:46 -0700 Subject: [PATCH 01/37] [WebRTC] Reduce remote rendering lag: input/draw coalescing, async encode, VP9 preference - Remove duplicate redraw on input (WebRTCWindowSystem.cpp) - Draw coalescing (BitmapWindowSystem.cpp) - Input (mouse event) coalescing (BitmapWindowSystem.cpp) - JS requestAnimationFrame coalescing / throttling (webrtcstreamer.js) - Async encoder thread (PeerConnectionManager.cpp/.h) - Data channel low-latency mode (webrtcstreamer.js) - Reduced startup delay (500ms -> 250ms) (WebRTCWindowSystem.cpp) - VP9 codec preference over VP8 (webrtcstreamer.js) --- .../visualization/gui/BitmapWindowSystem.cpp | 126 +++++++++++++++- .../webrtc_server/PeerConnectionManager.cpp | 59 ++++++-- .../webrtc_server/PeerConnectionManager.h | 14 ++ .../webrtc_server/WebRTCWindowSystem.cpp | 8 +- .../webrtc_server/html/webrtcstreamer.js | 136 +++++++++++++++--- 5 files changed, 302 insertions(+), 41 deletions(-) diff --git a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp index 3db9d0cefef..b73d680b381 100644 --- a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp +++ b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "open3d/geometry/Image.h" #include "open3d/utility/Logging.h" @@ -44,10 +45,21 @@ struct BitmapEvent { virtual void Execute() = 0; }; +// Forward declaration: BitmapDrawEvent needs a pointer to BitmapEventQueue +// to clear the per-window pending-draw flag before calling OnDraw(). +struct BitmapEventQueue; + struct BitmapDrawEvent : public BitmapEvent { - BitmapDrawEvent(BitmapWindow *target) : BitmapEvent(target) {} + // queue_ is used to clear the pending-draw flag before OnDraw() so that + // a PostRedraw() called *inside* OnDraw() is not suppressed. + BitmapEventQueue *queue_; + + BitmapDrawEvent(BitmapWindow *target, BitmapEventQueue *queue) + : BitmapEvent(target), queue_(queue) {} - void Execute() override { event_target->o3d_window->OnDraw(); } + // Execute() is defined after BitmapEventQueue (below) because it calls + // clear_pending_draw(), which requires the full BitmapEventQueue type. + void Execute() override; }; struct BitmapResizeEvent : public BitmapEvent { @@ -93,15 +105,23 @@ struct BitmapTextInputEvent : public BitmapEvent { } }; -/// Thread safe event queue (multiple producers and consumers). pop_front() and -/// push() are protected by a mutex. push() may fail if the mutex cannot be -/// acquired immediately. empty() is not protected and is not reliable. +/// Thread safe event queue (multiple producers and consumers). +/// pop_front() and push() are protected by a mutex. +/// push() may fail if the mutex cannot be acquired immediately. +/// empty() is not protected and is not reliable. +/// +/// Extended to support: +/// - Draw coalescing: at most one pending draw per window (push_draw). +/// - Input coalescing: MOVE/DRAG replace latest, WHEEL accumulates +/// dx/dy (replace_or_merge_mouse). Old mouse positions are stale; +/// processing them forces stale frames to be encoded and sent. struct BitmapEventQueue : public std::queue> { using value_t = std::shared_ptr; using super = std::queue; using super::empty; // not reliable using super::super; + // pop + front needs to be atomic for thread safety. This is exception safe // since shared_ptr copy ctor is noexcept, when it is returned by value. value_t pop_front() { @@ -110,6 +130,7 @@ struct BitmapEventQueue : public std::queue> { super::pop(); return evt; } + void push(const value_t &event) { if (evt_q_mutex_.try_lock()) { super::push(event); @@ -117,10 +138,87 @@ struct BitmapEventQueue : public std::queue> { } } + // Push a draw event only if no draw is already pending for this window. + // Returns true if pushed. The caller must pass the shared_ptr to the draw + // event; this function inserts the window into pending_draw_windows_ so + // that BitmapDrawEvent::Execute() can clear it on completion. + bool push_draw(BitmapWindow *window, const value_t &event) { + if (evt_q_mutex_.try_lock()) { + bool pushed = false; + if (pending_draw_windows_.find(window) == + pending_draw_windows_.end()) { + pending_draw_windows_.insert(window); + super::push(event); + pushed = true; + } + evt_q_mutex_.unlock(); + return pushed; + } + return false; + } + + // Called by BitmapDrawEvent::Execute() just before OnDraw() so that a + // redraw posted *during* drawing is not suppressed. + void clear_pending_draw(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // Remove all pending state for a window that is being destroyed. + void remove_window(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // For MOVE/DRAG: replace the last queued event of the same (target, type) + // with the new event (latest absolute position wins; camera controllers + // derive delta from absolute coords so intermediate positions are useless). + // For WHEEL: accumulate wheel.dx/dy into the last queued event of the same + // (target, type) so that the total scroll amount is preserved even when + // multiple notches fire faster than the render loop. + // Falls back to a normal push when no matching event is at the back. + void replace_or_merge_mouse(const value_t &event) { + std::lock_guard lock(evt_q_mutex_); + auto *new_evt = static_cast(event.get()); + if (!super::c.empty()) { + auto *back_mouse = + dynamic_cast(super::c.back().get()); + if (back_mouse && + back_mouse->event_target == new_evt->event_target && + back_mouse->event.type == new_evt->event.type) { + if (new_evt->event.type == MouseEvent::WHEEL) { + // Accumulate scroll deltas; update cursor position and + // other fields to the latest event values. + back_mouse->event.wheel.dx += new_evt->event.wheel.dx; + back_mouse->event.wheel.dy += new_evt->event.wheel.dy; + back_mouse->event.x = new_evt->event.x; + back_mouse->event.y = new_evt->event.y; + back_mouse->event.modifiers = new_evt->event.modifiers; + back_mouse->event.wheel.isTrackpad = + new_evt->event.wheel.isTrackpad; + } else { + // Replace: only the latest absolute position matters. + back_mouse->event = new_evt->event; + } + return; + } + } + super::push(event); + } + private: std::mutex evt_q_mutex_; + // Windows with a draw event currently in the queue (not yet executed). + std::unordered_set pending_draw_windows_; }; +// Out-of-class definition: BitmapEventQueue is now fully defined so +// clear_pending_draw() can be called. +void BitmapDrawEvent::Execute() { + queue_->clear_pending_draw(event_target); + event_target->o3d_window->OnDraw(); +} + } // namespace struct BitmapWindowSystem::Impl { @@ -187,18 +285,32 @@ void BitmapWindowSystem::DestroyWindow(OSWindow w) { while (!filtered_reversed.empty()) { impl_->event_queue_.push(filtered_reversed.pop_front()); } + // Clear any pending-draw entry for this window so the coalescing set + // does not hold a dangling pointer. + impl_->event_queue_.remove_window(the_deceased); // Requiem aeternam dona ei. Requiscat in pace. delete (BitmapWindow *)w; } void BitmapWindowSystem::PostRedrawEvent(OSWindow w) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw)); + // push_draw is a no-op when a draw event is already queued for this + // window, preventing redundant renders from piling up. + impl_->event_queue_.push_draw( + hw, std::make_shared(hw, &impl_->event_queue_)); } void BitmapWindowSystem::PostMouseEvent(OSWindow w, const MouseEvent &e) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw, e)); + if (e.type == MouseEvent::MOVE || e.type == MouseEvent::DRAG || + e.type == MouseEvent::WHEEL) { + // Coalesce: MOVE/DRAG replace latest (absolute position); WHEEL + // accumulates dx/dy. Only the most recent state matters for rendering. + impl_->event_queue_.replace_or_merge_mouse( + std::make_shared(hw, e)); + } else { + impl_->event_queue_.push(std::make_shared(hw, e)); + } } void BitmapWindowSystem::PostKeyEvent(OSWindow w, const KeyEvent &e) { diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index c29696c39e2..a3f3f5cbc32 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -195,9 +195,21 @@ PeerConnectionManager::PeerConnectionManager( } return this->HangUp(peerid); }; + + // Start async encoder thread. + encoder_running_ = true; + encoder_thread_ = std::thread(&PeerConnectionManager::EncoderThreadLoop, + this); } -PeerConnectionManager::~PeerConnectionManager() {} +PeerConnectionManager::~PeerConnectionManager() { + // Stop async encoder thread before WebRTC resources are torn down. + encoder_running_ = false; + pending_frames_cv_.notify_all(); + if (encoder_thread_.joinable()) { + encoder_thread_.join(); + } +} // Return deviceList as JSON vector. const Json::Value PeerConnectionManager::GetMediaList() { @@ -729,19 +741,44 @@ void PeerConnectionManager::CloseWindowConnections( } } +// Encoder thread: wakes on each new frame, drains the per-window latest-frame +// map, and calls video_track_source->OnFrame() (libyuv + WebRTC encode) +// off the render thread so frame delivery never blocks GUI redraws. +void PeerConnectionManager::EncoderThreadLoop() { + while (encoder_running_) { + std::unordered_map> snapshot; + { + std::unique_lock lock(pending_frames_mutex_); + pending_frames_cv_.wait(lock, [this] { + return !pending_frames_.empty() || !encoder_running_; + }); + if (!encoder_running_) break; + // Drain: take all pending frames in one batch; late-arriving frames + // for the same window have already overwritten earlier ones, so we + // encode only the latest per window (implicit frame coalescing). + snapshot = std::move(pending_frames_); + } + for (const auto &kv : snapshot) { + auto video_track_source = GetVideoTrackSource(kv.first); + if (video_track_source && kv.second) { + video_track_source->OnFrame(kv.second); + } + } + } +} + void PeerConnectionManager::OnFrame(const std::string &window_uid, const std::shared_ptr &im) { - // Get the WebRTC stream that corresponds to the window_uid. - // video_track_source is nullptr if the server is running but no client is - // connected. - rtc::scoped_refptr video_track_source = - GetVideoTrackSource(window_uid); - if (video_track_source) { - // TODO: this OnFrame(im); is a blocking call. Do we need to handle - // OnFrame in a separate thread? e.g. attach to a queue of frames, even - // if the queue size is just 1. - video_track_source->OnFrame(im); + // Skip if no peer is connected for this window. + if (!GetVideoTrackSource(window_uid)) return; + // Post the latest frame; overwrites any pending unencoded frame for this + // window (frame coalescing) so the encoder thread always sees the freshest + // content without blocking the render thread. + { + std::lock_guard lock(pending_frames_mutex_); + pending_frames_[window_uid] = im; } + pending_frames_cv_.notify_one(); } } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h index 0120e20b4fe..72a005e96c4 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h @@ -21,6 +21,8 @@ #include #include +#include +#include #include #include #include @@ -451,6 +453,18 @@ class PeerConnectionManager { const std::regex publish_filter_; std::map func_; std::string webrtc_port_range_; + + // Async encoder thread: decouples the render thread from the blocking + // libyuv + WebRTC encode path. OnFrame() posts the latest frame per window; + // the thread drains the map and calls video_track_source->OnFrame(). + std::unordered_map> + pending_frames_; + std::mutex pending_frames_mutex_; + std::condition_variable pending_frames_cv_; + std::atomic encoder_running_{false}; + std::thread encoder_thread_; + + void EncoderThreadLoop(); }; } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index 667a7c88be4..ab238201e7d 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -364,8 +364,10 @@ std::string WebRTCWindowSystem::OnDataChannelMessage( if (impl_->data_channel_message_callbacks_.count(class_name) != 0) { reply = impl_->data_channel_message_callbacks_.at(class_name)( message); - const auto os_window = GetOSWindowByUID(window_uid); - if (os_window) PostRedrawEvent(os_window); + // Do not call PostRedrawEvent here. Mouse/keyboard callbacks + // already schedule a redraw via Window::OnMouseEvent → PostRedraw. + // An extra PostRedrawEvent here creates a duplicate draw event + // before the input event is even processed, causing backlog. return reply; } else { reply = fmt::format( @@ -405,7 +407,7 @@ void WebRTCWindowSystem::OnFrame(const std::string &window_uid, void WebRTCWindowSystem::SendInitFrames(const std::string &window_uid) { utility::LogInfo("Sending init frames to {}.", window_uid); static const int s_max_initial_frames = 5; - static const int s_sleep_between_frames_ms = 100; + static const int s_sleep_between_frames_ms = 50; const auto os_window = GetOSWindowByUID(window_uid); if (!os_window) return; for (int i = 0; os_window != nullptr && i < s_max_initial_frames; ++i) { diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index d7bcd8f6c13..ec4e7e703b0 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -62,6 +62,13 @@ let WebRtcStreamer = (function() { // Open3D-specific functions. this.onClose = onClose; this.commsFetch = commsFetch; + + // Pending coalesced pointer/wheel events. A single requestAnimationFrame + // flushes both at most once per browser frame (~60 Hz), preventing a + // backlog of stale events from queuing up on the server. + this.pendingPointerEvent = null; // MOVE or DRAG (latest wins) + this.pendingWheelEvent = null; // WHEEL (dx/dy accumulated) + this.rafPending = false; } const logAndReturn = function(value) { @@ -182,6 +189,26 @@ let WebRtcStreamer = (function() { } }; + // Schedule a requestAnimationFrame flush of pending coalesced events. + // Only one rAF is scheduled at a time; calling again while one is already + // pending is a no-op. When the frame fires, the latest pointer event and + // the accumulated wheel event (if any) are sent and cleared. + WebRtcStreamer.prototype._scheduleRafFlush = function() { + if (this.rafPending) return; + this.rafPending = true; + requestAnimationFrame(() => { + this.rafPending = false; + if (this.pendingPointerEvent !== null) { + this.sendJsonData(this.pendingPointerEvent); + this.pendingPointerEvent = null; + } + if (this.pendingWheelEvent !== null) { + this.sendJsonData(this.pendingWheelEvent); + this.pendingWheelEvent = null; + } + }); + }; + WebRtcStreamer.prototype.addEventListeners = function(windowUID) { if (this.videoElt) { var parentDivElt = this.videoElt.parentElement; @@ -333,7 +360,9 @@ let WebRtcStreamer = (function() { // - Open3D: L=1, M=2, R=4 // - JavaScript: L=1, R=2, M=4 event.preventDefault(); - var open3dMouseEvent = { + // Throttle to one event per animation frame. Only the latest + // absolute position matters; intermediate positions are stale. + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: event.buttons === 0 ? 'MOVE' : 'DRAG', @@ -344,7 +373,7 @@ let WebRtcStreamer = (function() { buttons: event.buttons, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('touchmove', (event) => { // TODO: Known differences. Currently only left-key drag works. @@ -352,7 +381,8 @@ let WebRtcStreamer = (function() { // - JavaScript: L=1, R=2, M=4 event.preventDefault(); var rect = event.target.getBoundingClientRect(); - var open3dMouseEvent = { + // Throttle to one event per animation frame (latest wins). + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: 'DRAG', @@ -363,7 +393,7 @@ let WebRtcStreamer = (function() { buttons: 1, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('mouseleave', (event) => { var open3dMouseEvent = { @@ -397,20 +427,36 @@ let WebRtcStreamer = (function() { dx = dx === 0 ? dx : (-dx / Math.abs(dx)) * 1; dy = dy === 0 ? dy : (-dy / Math.abs(dy)) * 1; - var open3dMouseEvent = { - window_uid: windowUID, - class_name: 'MouseEvent', - type: 'WHEEL', - x: event.offsetX, - y: event.offsetY, - modifiers: WebRtcStreamer._getModifiers(event), - wheel: { - dx: dx, - dy: dy, - isTrackpad: isTrackpad ? 1 : 0, - }, - }; - this.sendJsonData(open3dMouseEvent); + if (this.pendingWheelEvent === null) { + // First wheel event in this frame: create a new pending + // event and schedule a flush. + this.pendingWheelEvent = { + window_uid: windowUID, + class_name: 'MouseEvent', + type: 'WHEEL', + x: event.offsetX, + y: event.offsetY, + modifiers: WebRtcStreamer._getModifiers(event), + wheel: { + dx: dx, + dy: dy, + isTrackpad: isTrackpad ? 1 : 0, + }, + }; + this._scheduleRafFlush(); + } else { + // Subsequent wheel events in the same frame: accumulate + // dx/dy so the total scroll amount is preserved, and + // update cursor position to the latest coordinates. + this.pendingWheelEvent.wheel.dx += dx; + this.pendingWheelEvent.wheel.dy += dy; + this.pendingWheelEvent.x = event.offsetX; + this.pendingWheelEvent.y = event.offsetY; + this.pendingWheelEvent.modifiers = + WebRtcStreamer._getModifiers(event); + this.pendingWheelEvent.wheel.isTrackpad = + isTrackpad ? 1 : 0; + } }, {passive: false}); } }; @@ -464,6 +510,48 @@ let WebRtcStreamer = (function() { this.pc.addStream(stream); } + // Prefer VP9 via the standard setCodecPreferences() API. + // Must be called on the transceiver BEFORE createOffer(). + // For receive-only mode (no local stream), we create the video + // transceiver explicitly — offerToReceiveVideo creates it lazily + // inside createOffer(), which is too late to set preferences. + // Falls back silently to VP8 if the browser does not support the + // API or if VP9 is not in the browser's capabilities. + var pc = this.pc; + if (typeof RTCRtpReceiver !== 'undefined' && + RTCRtpReceiver.getCapabilities && pc.addTransceiver) { + var videoCaps = RTCRtpReceiver.getCapabilities('video'); + if (videoCaps) { + var vp9Codecs = videoCaps.codecs.filter(function(c) { + return c.mimeType === 'video/VP9'; + }); + if (vp9Codecs.length) { + var preferredCodecs = vp9Codecs.concat( + videoCaps.codecs.filter(function(c) { + return c.mimeType !== 'video/VP9'; + })); + if (stream) { + // Local video being sent: find the transceiver + // addStream() created and apply preferences. + pc.getTransceivers().forEach(function(t) { + if (t.sender && t.sender.track && + t.sender.track.kind === 'video' && + t.setCodecPreferences) { + t.setCodecPreferences(preferredCodecs); + } + }); + } else { + // Receive-only: create transceiver explicitly. + var vt = pc.addTransceiver( + 'video', {direction: 'recvonly'}); + if (vt.setCodecPreferences) { + vt.setCodecPreferences(preferredCodecs); + } + } + } + } + } + // clear early candidates this.earlyCandidates.length = 0; @@ -599,9 +687,17 @@ let WebRtcStreamer = (function() { } }; - // Local datachannel sends data + // Local datachannel sends data. + // Use unordered, unreliable delivery for mouse/input events. Ordered + // reliable delivery causes head-of-line blocking: a lost packet stalls + // all subsequent events until it is retransmitted. For interactive + // mouse events the latest position is all that matters, so a dropped + // message is better than a delayed one. maxRetransmits: 0 means the + // browser sends once and moves on; ordered: false removes sequencing. try { - this.dataChannel = pc.createDataChannel('ClientDataChannel'); + this.dataChannel = pc.createDataChannel( + 'ClientDataChannel', + {ordered: false, maxRetransmits: 0}); var dataChannel = this.dataChannel; dataChannel.onopen = function() { console.log('local datachannel open'); From a14390e3f831448f9008d55c1ff415b3dc4436c1 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 20 May 2026 13:39:00 -0700 Subject: [PATCH 02/37] Explicitly warn and skip writing float textures (unsupported) in gltf. --- cpp/open3d/t/io/file_format/FileASSIMP.cpp | 75 ++++++++++++------- .../webrtc_server/PeerConnectionManager.cpp | 8 +- 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/cpp/open3d/t/io/file_format/FileASSIMP.cpp b/cpp/open3d/t/io/file_format/FileASSIMP.cpp index 1ce61332bd9..aff2fa3e6b6 100644 --- a/cpp/open3d/t/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/t/io/file_format/FileASSIMP.cpp @@ -16,6 +16,7 @@ #include #include +#include "open3d/core/Dtype.h" #include "open3d/core/ParallelFor.h" #include "open3d/core/TensorFunction.h" #include "open3d/t/io/ImageIO.h" @@ -177,11 +178,31 @@ bool ReadTriangleMeshUsingASSIMP( return true; } -static void SetTextureMaterialProperty(aiMaterial* mat, - aiScene* scene, - int texture_idx, - aiTextureType tt, - t::geometry::Image& img) { +namespace { + +// Checks whether a texture map is present, not empty and uint8 or uint16 +// datatype. +bool HasValidTexture(const visualization::rendering::Material& material, + const std::string& key) { + if (!material.HasTextureMap(key) || material.GetTextureMap(key).IsEmpty()) { + return false; + } + const auto& dt = material.GetTextureMap(key).GetDtype(); + if (dt != core::UInt8 && dt != core::UInt16) { + utility::LogWarning( + "Skipping texture map '{}' with unsupported data type '{}'. " + "Only uint8 and uint16 are supported.", + key, dt.ToString()); + return false; + } + return true; +} + +void SetTextureMaterialProperty(aiMaterial* mat, + aiScene* scene, + int texture_idx, + aiTextureType tt, + t::geometry::Image& img) { // Encode image as PNG std::vector img_buffer; WriteImageToPNGInMemory(img_buffer, img, 6); @@ -208,7 +229,6 @@ static void SetTextureMaterialProperty(aiMaterial* mat, mat->AddProperty(&mode, 1, AI_MATKEY_MAPPINGMODE_V(tt, 0)); } -namespace { // Add hash function for tuple key struct TupleHash { size_t operator()(const std::tuple& t) const { @@ -551,17 +571,18 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, // model has one we just export it, otherwise if both roughness and // metal maps are available we combine them, otherwise if only one or // the other is available we just export the one map. + const auto& material = w_mesh.GetMaterial(); int n_textures = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) ++n_textures; - if (w_mesh.GetMaterial().HasNormalMap()) ++n_textures; - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { + if (HasValidTexture(material, "albedo")) ++n_textures; + if (HasValidTexture(material, "normal")) ++n_textures; + if (HasValidTexture(material, "ao_rough_metal")) { ++n_textures; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { + } else if (HasValidTexture(material, "roughness") && + HasValidTexture(material, "metallic")) { ++n_textures; } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) ++n_textures; - if (w_mesh.GetMaterial().HasMetallicMap()) ++n_textures; + if (HasValidTexture(material, "roughness")) ++n_textures; + if (HasValidTexture(material, "metallic")) ++n_textures; } if (n_textures > 0) { ai_scene->mTextures = new aiTexture*[n_textures]; @@ -573,23 +594,23 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, // Now embed the textures that are available... int current_idx = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) { - auto img = w_mesh.GetMaterial().GetAlbedoMap(); + if (HasValidTexture(material, "albedo")) { + auto img = material.GetAlbedoMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_DIFFUSE, img); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_BASE_COLOR, img); ++current_idx; } - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { - auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap(); + if (HasValidTexture(material, "ao_rough_metal")) { + auto img = material.GetAORoughnessMetalMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_UNKNOWN, img); ++current_idx; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { - auto rough = w_mesh.GetMaterial().GetRoughnessMap(); - auto metal = w_mesh.GetMaterial().GetMetallicMap(); + } else if (HasValidTexture(material, "roughness") && + HasValidTexture(material, "metallic")) { + auto rough = material.GetRoughnessMap(); + auto metal = material.GetMetallicMap(); auto rows = rough.GetRows(); auto cols = rough.GetCols(); auto rough_metal = @@ -620,21 +641,21 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, aiTextureType_UNKNOWN, rough_metal); ++current_idx; } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) { - auto img = w_mesh.GetMaterial().GetRoughnessMap(); + if (HasValidTexture(material, "roughness")) { + auto img = material.GetRoughnessMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_UNKNOWN, img); ++current_idx; } - if (w_mesh.GetMaterial().HasMetallicMap()) { - auto img = w_mesh.GetMaterial().GetMetallicMap(); + if (HasValidTexture(material, "metallic")) { + auto img = material.GetMetallicMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_UNKNOWN, img); ++current_idx; } } - if (w_mesh.GetMaterial().HasNormalMap()) { - auto img = w_mesh.GetMaterial().GetNormalMap(); + if (HasValidTexture(material, "normal")) { + auto img = material.GetNormalMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_NORMALS, img); ++current_idx; diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index a3f3f5cbc32..9d5865d71ff 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -198,17 +198,15 @@ PeerConnectionManager::PeerConnectionManager( // Start async encoder thread. encoder_running_ = true; - encoder_thread_ = std::thread(&PeerConnectionManager::EncoderThreadLoop, - this); + encoder_thread_ = + std::thread(&PeerConnectionManager::EncoderThreadLoop, this); } PeerConnectionManager::~PeerConnectionManager() { // Stop async encoder thread before WebRTC resources are torn down. encoder_running_ = false; pending_frames_cv_.notify_all(); - if (encoder_thread_.joinable()) { - encoder_thread_.join(); - } + encoder_thread_.join(); } // Return deviceList as JSON vector. From 9469dce69204231b49377c5c314bf66dac4d7f66 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 21 May 2026 13:49:11 -0700 Subject: [PATCH 03/37] Pin license-webpack-plugin - windows wheel build error. --- python/js/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/js/package.json b/python/js/package.json index d364052ec22..5556ea5df4a 100644 --- a/python/js/package.json +++ b/python/js/package.json @@ -37,7 +37,8 @@ }, "resolutions": { "glob": "7.2.3", - "abab": "^2.0.6" + "abab": "^2.0.6", + "license-webpack-plugin": "^4.0.0" }, "dependencies": { "@jupyter-widgets/base": "^2 || ^3 || ^4 || ^5 || ^6", From 99b96d2bfaf0d4508f9efe4b9c2b507556f1df96 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 21 May 2026 16:11:14 -0700 Subject: [PATCH 04/37] [WebRTC] Optimize latency by adjusting pacing and jitter buffer settings sends zero playout delay in every RTP packet's header extension when adaptation is necessary, drops resolution rather than framerate prevents WebRTC's autonomous resolution scaling; Replaced playoutDelayHint with jitterBufferTarget = 0 --- .../webrtc_server/PeerConnectionManager.cpp | 23 +++++++++++++++++++ .../webrtc_server/html/webrtcstreamer.js | 21 +++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index 9d5865d71ff..68ff8256e29 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,18 @@ static IceServer GetIceServerFromUrl(const std::string &url) { static webrtc::PeerConnectionFactoryDependencies CreatePeerConnectionFactoryDependencies() { + // Disable WebRTC's pacing delay and allow the pacer to drain its queue + // immediately. This reduces the added latency from packet smoothing, which + // is unnecessary for a local interactive streaming use case. + // Force playout delay to zero in the RTP header extension so the receiver + // renders frames immediately rather than buffering for jitter smoothing. + // Disable automatic resolution reduction so quality is only degraded if the + // encoder is genuinely overloaded (content hint kFluid still allows it). + webrtc::field_trial::InitFieldTrialsFromString( + "WebRTC-Pacer-DrainQueue/Enabled/" + "WebRTC-ForceSendPlayoutDelay/min_ms:0,max_ms:0/" + "WebRTC-Video-DisableAutomaticResize/Enabled/"); + webrtc::PeerConnectionFactoryDependencies dependencies; dependencies.network_thread = nullptr; dependencies.worker_thread = rtc::Thread::Current(); @@ -531,6 +544,10 @@ bool PeerConnectionManager::InitializePeerConnection() { PeerConnectionManager::PeerConnectionObserver * PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { webrtc::PeerConnectionInterface::RTCConfiguration config; + // Max bundle multiplexes all media and data channels on a single transport, + // eliminating separate ICE/DTLS handshakes per track and reducing latency. + config.bundle_policy = + webrtc::PeerConnectionInterface::kBundlePolicyMaxBundle; for (auto ice_server : ice_server_list_) { webrtc::PeerConnectionInterface::IceServer server; IceServer srv = GetIceServerFromUrl(ice_server); @@ -664,6 +681,12 @@ bool PeerConnectionManager::AddStreams( opts); video_track = peer_connection_factory_->CreateVideoTrack( window_uid + "_video", videoScaled); + // Prefer framerate over resolution when the encoder + // is under pressure (bandwidth or CPU constrained). + // For interactive 3D rendering, motion smoothness + // matters more than pixel-perfect resolution. + video_track->set_content_hint( + webrtc::VideoTrackInterface::ContentHint::kFluid); } if ((video_track) && (!stream->AddTrack(video_track))) { diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index ec4e7e703b0..45dea6bf245 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -677,11 +677,22 @@ let WebRtcStreamer = (function() { const recvs = pc.getReceivers(); recvs.forEach((recv) => { - if (recv.track && recv.track.kind === 'video' && - typeof recv.getParameters != 'undefined') { - console.log( - 'codecs:' + - JSON.stringify(recv.getParameters().codecs)); + if (recv.track && recv.track.kind === 'video') { + // Minimize browser jitter buffer to reduce playout + // latency. The server already sends RTP playout-delay + // header extensions with min=max=0 via the + // WebRTC-ForceSendPlayoutDelay field trial, but set + // jitterBufferTarget as well for browsers that honour + // the JS API over the in-band RTP extension. + if (typeof recv.jitterBufferTarget !== 'undefined') { + recv.jitterBufferTarget = 0; + } + if (typeof recv.getParameters != 'undefined') { + console.log( + 'codecs:' + + JSON.stringify( + recv.getParameters().codecs)); + } } }); } From 1a46e95ad7b273a8a6933230366bc01edebf99b8 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> Date: Thu, 21 May 2026 23:49:26 -0700 Subject: [PATCH 05/37] undo non-working windows jupyterlab fix. --- python/js/package.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/js/package.json b/python/js/package.json index 5556ea5df4a..0e2f7d78069 100644 --- a/python/js/package.json +++ b/python/js/package.json @@ -37,8 +37,7 @@ }, "resolutions": { "glob": "7.2.3", - "abab": "^2.0.6", - "license-webpack-plugin": "^4.0.0" + "abab": "^2.0.6" }, "dependencies": { "@jupyter-widgets/base": "^2 || ^3 || ^4 || ^5 || ^6", @@ -55,4 +54,4 @@ } } } -} \ No newline at end of file +} From 84f58be2feed823364588b2fd29472e2362f64c1 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Sat, 23 May 2026 15:29:26 -0700 Subject: [PATCH 06/37] [WebRTC] Refactor texture handling in ASSIMP export and improve data channel reliability --- cpp/open3d/t/io/file_format/FileASSIMP.cpp | 148 +++++++++--------- .../webrtc_server/WebRTCWindowSystem.cpp | 6 +- .../webrtc_server/html/webrtcstreamer.js | 17 +- 3 files changed, 84 insertions(+), 87 deletions(-) diff --git a/cpp/open3d/t/io/file_format/FileASSIMP.cpp b/cpp/open3d/t/io/file_format/FileASSIMP.cpp index 768ac51eca8..9cae51c804f 100644 --- a/cpp/open3d/t/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/t/io/file_format/FileASSIMP.cpp @@ -566,97 +566,97 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, ai_mat->AddProperty(&ac, 1, AI_MATKEY_COLOR_EMISSIVE); } - // Count texture maps... + // Build a list of texture-embed actions in a single pass. Each action + // is a lambda that writes one texture slot into the assimp scene; the + // slot index is passed at call time. . + // // NOTE: GLTF2 expects a single combined roughness/metal map. If the // model has one we just export it, otherwise if both roughness and // metal maps are available we combine them, otherwise if only one or // the other is available we just export the one map. const auto& material = w_mesh.GetMaterial(); - int n_textures = 0; - if (HasValidTexture(material, "albedo")) ++n_textures; - if (HasValidTexture(material, "normal")) ++n_textures; - if (HasValidTexture(material, "ambient_occlusion")) ++n_textures; + using TextureAction = std::function; + std::vector texture_actions; + + if (HasValidTexture(material, "albedo")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAlbedoMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_DIFFUSE, img); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_BASE_COLOR, img); + }); + } + if (HasValidTexture(material, "ambient_occlusion")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAOMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_LIGHTMAP, img); + }); + } if (HasValidTexture(material, "ao_rough_metal")) { - ++n_textures; + texture_actions.push_back([&](int idx) { + auto img = material.GetAORoughnessMetalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); } else if (HasValidTexture(material, "roughness") && HasValidTexture(material, "metallic")) { - ++n_textures; + texture_actions.push_back([&](int idx) { + auto rough = material.GetRoughnessMap().AsTensor(); + auto metal = material.GetMetallicMap().AsTensor(); + if (rough.GetShape() != metal.GetShape()) { + utility::LogError( + "RoughnessMap (shape={}) and MetallicMap " + "(shape={}) must have the same shape.", + rough.GetShape(), metal.GetShape()); + } + auto rows = rough.GetShape(0); + auto cols = rough.GetShape(1); + auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, + core::Dtype::UInt8); + rough_metal.Slice(2, 2, 3) = + metal.Slice(2, 0, 1); // blue channel is metal + rough_metal.Slice(2, 1, 2) = + rough.Slice(2, 0, 1); // green channel is roughness + geometry::Image rough_metal_img(rough_metal); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, + rough_metal_img); + }); } else { - if (HasValidTexture(material, "roughness")) ++n_textures; - if (HasValidTexture(material, "metallic")) ++n_textures; + if (HasValidTexture(material, "roughness")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetRoughnessMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } + if (HasValidTexture(material, "metallic")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetMetallicMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } } + if (HasValidTexture(material, "normal")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetNormalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_NORMALS, img); + }); + } + + int n_textures = static_cast(texture_actions.size()); if (n_textures > 0) { ai_scene->mTextures = new aiTexture*[n_textures]; for (int i = 0; i < n_textures; ++i) { ai_scene->mTextures[i] = new aiTexture(); + texture_actions[i](i); } ai_scene->mNumTextures = n_textures; } - - // Now embed the textures that are available... - int current_idx = 0; - if (HasValidTexture(material, "albedo")) { - auto img = material.GetAlbedoMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_DIFFUSE, img); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_BASE_COLOR, img); - ++current_idx; - } - if (HasValidTexture(w_mesh.GetMaterial(), "ambient_occlusion")) { - auto img = w_mesh.GetMaterial().GetAOMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_LIGHTMAP, img); - ++current_idx; - } - if (HasValidTexture(w_mesh.GetMaterial(), "ao_rough_metal")) { - auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } else if (HasValidTexture(w_mesh.GetMaterial(), "roughness") && - HasValidTexture(w_mesh.GetMaterial(), "metallic")) { - auto rough = w_mesh.GetMaterial().GetRoughnessMap().AsTensor(); - auto metal = w_mesh.GetMaterial().GetMetallicMap().AsTensor(); - if (rough.GetShape() != metal.GetShape()) { - utility::LogError( - "RoughnessMap (shape={}) and MetallicMap (shape={}) " - "must have the same shape.", - rough.GetShape(), metal.GetShape()); - } - auto rows = rough.GetShape(0); - auto cols = rough.GetShape(1); - auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, - core::Dtype::UInt8); - rough_metal.Slice(2, 2, 3) = - metal.Slice(2, 0, 1); // blue channel is metal - rough_metal.Slice(2, 1, 2) = - rough.Slice(2, 0, 1); // green channel is roughness - - geometry::Image rough_metal_img(rough_metal); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, rough_metal_img); - ++current_idx; - } else { - if (HasValidTexture(material, "roughness")) { - auto img = material.GetRoughnessMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - if (HasValidTexture(material, "metallic")) { - auto img = material.GetMetallicMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - } - if (HasValidTexture(material, "normal")) { - auto img = material.GetNormalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_NORMALS, img); - ++current_idx; - } } ai_scene->mMaterials[0] = ai_mat; diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index ab238201e7d..c703ac5f66a 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -364,10 +364,8 @@ std::string WebRTCWindowSystem::OnDataChannelMessage( if (impl_->data_channel_message_callbacks_.count(class_name) != 0) { reply = impl_->data_channel_message_callbacks_.at(class_name)( message); - // Do not call PostRedrawEvent here. Mouse/keyboard callbacks - // already schedule a redraw via Window::OnMouseEvent → PostRedraw. - // An extra PostRedrawEvent here creates a duplicate draw event - // before the input event is even processed, causing backlog. + // Custom callbacks that mutate GUI state (e.g. add/remove geometry) + // must call window->PostRedraw() or post_redraw() themselves. return reply; } else { reply = fmt::format( diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index 45dea6bf245..e3c385d6fb4 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -699,16 +699,15 @@ let WebRtcStreamer = (function() { }; // Local datachannel sends data. - // Use unordered, unreliable delivery for mouse/input events. Ordered - // reliable delivery causes head-of-line blocking: a lost packet stalls - // all subsequent events until it is retransmitted. For interactive - // mouse events the latest position is all that matters, so a dropped - // message is better than a delayed one. maxRetransmits: 0 means the - // browser sends once and moves on; ordered: false removes sequencing. + // Use reliable ordered delivery (the default). While unordered + + // unreliable would reduce head-of-line blocking for mouse MOVE/DRAG, + // the same channel carries discrete events (BUTTON_DOWN/UP, key events, + // resize) and application-level RPC (tensorboard/update_geometry etc.) + // that must not be lost or reordered. Mouse coalescing (rAF + server- + // side replace_or_merge_mouse) already prevents event backlog, so the + // extra reliability is free in practice on a local network. try { - this.dataChannel = pc.createDataChannel( - 'ClientDataChannel', - {ordered: false, maxRetransmits: 0}); + this.dataChannel = pc.createDataChannel('ClientDataChannel'); var dataChannel = this.dataChannel; dataChannel.onopen = function() { console.log('local datachannel open'); From e98fba4ad63af7314b23d84a400c4fb80b4d7e62 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 16 Jun 2026 00:11:51 -0700 Subject: [PATCH 07/37] Upgrade webrtc library to M149 (latest). WIP --- .github/workflows/webrtc.yml | 169 +++++---- .github/workflows/windows.yml | 5 +- 3rdparty/find_dependencies.cmake | 17 +- ...uild-enable-rtc_use_cxx11_abi-option.patch | 15 +- .../0002-build-enable_safe_libstdcxx.patch | 11 + ...003-src-fix-nullptr_t-with-libstdcxx.patch | 12 + ...-gcc-suppress-port-interface-network.patch | 17 + ...ll-payload_type_picker-gcc-flat_tree.patch | 12 + .../webrtc/0006-build-win-dynamic-crt.patch | 35 ++ 3rdparty/webrtc/CMakeLists.txt | 15 +- 3rdparty/webrtc/README.md | 26 +- 3rdparty/webrtc/apply_webrtc_patches.sh | 48 +++ 3rdparty/webrtc/webrtc_build.sh | 201 ++++++----- 3rdparty/webrtc/webrtc_common.cmake | 74 ++-- 3rdparty/webrtc/webrtc_download.cmake | 97 +++-- cpp/open3d/t/geometry/kernel/MinimumOBE.cpp | 4 +- .../webrtc_server/BitmapTrackSource.cpp | 6 +- .../webrtc_server/BitmapTrackSource.h | 12 +- .../webrtc_server/CMakeLists.txt | 18 + .../webrtc_server/ImageCapturer.cpp | 18 +- .../webrtc_server/ImageCapturer.h | 24 +- .../webrtc_server/PeerConnectionManager.cpp | 334 ++++++++++-------- .../webrtc_server/PeerConnectionManager.h | 94 +++-- .../visualization/webrtc_server/VideoFilter.h | 12 +- .../visualization/webrtc_server/VideoScaler.h | 21 +- .../webrtc_server/WebRTCWindowSystem.cpp | 45 ++- .../webrtc_server/html/index.html | 4 +- .../webrtc_server/html/open3d_logo.ico | Bin 34494 -> 0 bytes .../webrtc_server/html/webrtcstreamer.js | 46 +-- examples/cpp/CMakeLists.txt | 9 + 30 files changed, 856 insertions(+), 545 deletions(-) create mode 100644 3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch create mode 100644 3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch create mode 100644 3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch create mode 100644 3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch create mode 100644 3rdparty/webrtc/0006-build-win-dynamic-crt.patch create mode 100755 3rdparty/webrtc/apply_webrtc_patches.sh delete mode 100644 cpp/open3d/visualization/webrtc_server/html/open3d_logo.ico diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 45b27363ebc..481671a1e6c 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -5,13 +5,13 @@ on: workflow_dispatch: inputs: webrtc_commit: - description: 'Specify WebRTC commit to build.' + description: 'WebRTC src commit (full or short).' required: false - default: '60e674842ebae283cc6b2627f4b6f2f8186f3317' # Date: Wed Apr 7 19:12:13 2021 +0200 + default: 'e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e' depot_tools_commit: - description: 'Specify Depot Tools commit to to use for the build.' + description: 'depot_tools commit (empty = HEAD).' required: false - default: 'e1a98941d3ab10549be6d82d0686bb0fb91ec903' # Date: Wed Apr 7 21:35:29 2021 +0000 + default: '' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,29 +20,34 @@ concurrency: env: WEBRTC_COMMIT: ${{ github.event.inputs.webrtc_commit }} DEPOT_TOOLS_COMMIT: ${{ github.event.inputs.depot_tools_commit }} + NPROC: 4 jobs: Unix: permissions: - contents: write # upload + contents: write runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-13] + include: + - os: ubuntu-22.04 + package_suffix: linux_cxx-abi-1 + - os: macos-14 + package_suffix: macos_arm64 steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Set up Python version + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' - - name: Install dependencies - if: ${{ matrix.os == 'ubuntu-22.04' }} + - name: Install dependencies (Ubuntu) + if: matrix.os == 'ubuntu-22.04' run: | source 3rdparty/webrtc/webrtc_build.sh install_dependencies_ubuntu @@ -60,21 +65,36 @@ jobs: - name: Upload WebRTC uses: actions/upload-artifact@v4 with: - name: webrtc_release_${{ matrix.os }} + name: webrtc_${{ matrix.package_suffix }} path: | - webrtc_*.tar.gz - checksum_*.txt + webrtc_*.tar.gz + checksum_webrtc_*.tar.gz if-no-files-found: error Windows: permissions: - contents: write # upload - # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/windows_build_instructions.md + contents: write runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + include: + - config: Release + static_runtime: ON + tag: Release_mt + - config: Release + static_runtime: OFF + tag: Release_md + - config: Debug + static_runtime: ON + tag: Debug_mt + - config: Debug + static_runtime: OFF + tag: Debug_md env: - WORK_DIR: "C:\\WebRTC" # Not enough space in D: - OPEN3D_DIR: "D:\\a\\open3d\\open3d" - DEPOT_TOOLS_UPDATE: 1 # Fix cannot find python3_bin_reldir.txt + WORK_DIR: 'C:\WebRTC' + OPEN3D_DIR: ${{ github.workspace }} + DEPOT_TOOLS_UPDATE: 1 DEPOT_TOOLS_WIN_TOOLCHAIN: 0 NPROC: 2 @@ -82,17 +102,18 @@ jobs: - name: Checkout source code uses: actions/checkout@v4 - - name: Set up Python version + - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.10' - - name: Disk space + - name: Prepare work directory + shell: pwsh run: | Get-PSDrive - mkdir "$env:WORK_DIR" + New-Item -ItemType Directory -Force -Path $env:WORK_DIR - - name: Setup PATH for Visual Studio # Required for Ninja + - name: Setup PATH for Visual Studio uses: ilammy/msvc-dev-cmd@v1 with: arch: x64 @@ -102,74 +123,70 @@ jobs: working-directory: ${{ env.WORK_DIR }} run: | $ErrorActionPreference = 'Stop' - echo "Get depot_tools" - # Checkout to a specific version - # Ref: https://chromium.googlesource.com/chromium/src/+/main/docs/building_old_revisions.md - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - git -C depot_tools checkout $env:DEPOT_TOOLS_COMMIT - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - - echo "Get WebRTC" - mkdir webrtc - cd webrtc - fetch webrtc - - git -C src checkout $env:WEBRTC_COMMIT - git -C src submodule update --init --recursive - echo "gclient sync" + if (-not (Test-Path depot_tools)) { + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + } + if ($env:DEPOT_TOOLS_COMMIT) { + git -C depot_tools checkout $env:DEPOT_TOOLS_COMMIT + } + $env:Path = (Get-Item depot_tools).FullName + ';' + $env:Path + if (-not (Test-Path webrtc/src)) { + New-Item -ItemType Directory -Force -Path webrtc | Out-Null + Push-Location webrtc + fetch webrtc + Pop-Location + } + git -C webrtc/src checkout $env:WEBRTC_COMMIT + git -C webrtc/src submodule update --init --recursive + Push-Location webrtc gclient sync -D --force --reset - cd .. - echo "random.org" - curl "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" -o skipcache + Pop-Location - name: Patch WebRTC + shell: pwsh working-directory: ${{ env.WORK_DIR }} run: | $ErrorActionPreference = 'Stop' - cp "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/ - cp "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/ - - - name: Build WebRTC (Release) - working-directory: ${{ env.WORK_DIR }} - run: | - $ErrorActionPreference = 'Stop' - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - mkdir webrtc/build - cd webrtc/build - cmake -G Ninja -D CMAKE_BUILD_TYPE=Release ` - -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Release ` - .. - ninja install - echo "Cleanup build folder for next config build" - cd .. - rm -r build + Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/ + Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/ + bash "$env:OPEN3D_DIR/3rdparty/webrtc/apply_webrtc_patches.sh" ` + "$env:OPEN3D_DIR" "$env:WORK_DIR/webrtc/src" - - name: Build WebRTC (Debug) + - name: Build and package WebRTC + shell: pwsh working-directory: ${{ env.WORK_DIR }} + env: + BUILD_CONFIG: ${{ matrix.config }} + STATIC_RT: ${{ matrix.static_runtime }} + WIN_TAG: ${{ matrix.tag }} run: | $ErrorActionPreference = 'Stop' - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - mkdir webrtc/build - cd webrtc/build - cmake -G Ninja -D CMAKE_BUILD_TYPE=Debug ` - -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Debug ` - .. + $env:Path = (Get-Item depot_tools).FullName + ';' + $env:Path + $installRoot = Join-Path $env:WORK_DIR "webrtc_pkg" + if (Test-Path $installRoot) { Remove-Item -Recurse -Force $installRoot } + New-Item -ItemType Directory -Force -Path webrtc/build | Out-Null + Push-Location webrtc/build + $debugFlag = if ($env:BUILD_CONFIG -eq 'Debug') { 'ON' } else { 'OFF' } + cmake -G Ninja ` + -D CMAKE_BUILD_TYPE=$env:BUILD_CONFIG ` + -D WEBRTC_IS_DEBUG=$debugFlag ` + -D WEBRTC_STATIC_MSVC_RUNTIME=$env:STATIC_RT ` + -D CMAKE_INSTALL_PREFIX=$installRoot ` + .. ninja install - - - name: Package WebRTC - working-directory: ${{ env.WORK_DIR }} - run: | - $ErrorActionPreference = 'Stop' - $env:WEBRTC_COMMIT_SHORT = (git -C webrtc/src rev-parse --short=7 HEAD) - cmake -E tar cv webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip ` - --format=zip -- webrtc_release - cmake -E sha256sum webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip | Tee-Object -FilePath checksum_win.txt + Pop-Location + $short = (git -C webrtc/src rev-parse --short=7 HEAD) + $zip = "webrtc_${short}_win_$env:WIN_TAG.zip" + Push-Location $installRoot + cmake -E tar cvf (Join-Path $env:OPEN3D_DIR $zip) --format=zip . + Pop-Location + cmake -E sha256sum (Join-Path $env:OPEN3D_DIR $zip) | Tee-Object -FilePath (Join-Path $env:OPEN3D_DIR "checksum_$zip") - name: Upload WebRTC uses: actions/upload-artifact@v4 with: - name: webrtc_release_windows + name: webrtc_win_${{ matrix.tag }} path: | - ${{ env.WORK_DIR }}/webrtc_*.zip - ${{ env.WORK_DIR }}/checksum_*.txt + webrtc_*_win_*.zip + checksum_webrtc_*_win_*.zip if-no-files-found: error diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d571f97e295..93b303e02e3 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -52,7 +52,10 @@ jobs: - BUILD_CUDA_MODULE: ON # FIXME CONFIG: Debug env: - BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} + # WebRTC prebuilt ships static libs for both MSVC runtimes (/MT, /MD). + # Enable it for static-lib + static-runtime (/MT) and shared-lib + + # dynamic-runtime (/MD). Shared libs always use the dynamic runtime. + BUILD_WEBRTC: ${{ ( ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) || ( matrix.BUILD_SHARED_LIBS == 'ON' && matrix.STATIC_RUNTIME == 'OFF' ) ) && 'ON' || 'OFF' }} BUILD_PYTORCH_OPS: ${{ ( matrix.BUILD_CUDA_MODULE == 'ON' || matrix.CONFIG == 'Debug' ) && 'OFF' || 'ON' }} # FIXME steps: diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index fb926d36ebb..3c7456fa4d8 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -1988,11 +1988,24 @@ if(BUILD_WEBRTC) open3d_import_3rdparty_library(3rdparty_webrtc HIDDEN INCLUDE_DIRS ${WEBRTC_INCLUDE_DIRS} - LIB_DIR ${WEBRTC_LIB_DIR} - LIBRARIES ${WEBRTC_LIBRARIES} DEPENDS ext_webrtc_all ) + if(UNIX AND NOT APPLE) + target_link_libraries(3rdparty_webrtc INTERFACE + "-Wl,--whole-archive" + "$" + "-Wl,--no-whole-archive" + "$") + else() + target_link_libraries(3rdparty_webrtc INTERFACE + "$" + "$") + endif() target_link_libraries(3rdparty_webrtc INTERFACE Open3D::3rdparty_threads ${CMAKE_DL_LIBS}) + # libwebrtc.a and libturbojpeg.a both export jpeg_* symbols (WebRTC bundles libjpeg). + if(UNIX AND NOT APPLE) + target_link_options(3rdparty_webrtc INTERFACE "LINKER:--allow-multiple-definition") + endif() if (MSVC) # https://github.com/iimachines/webrtc-build/issues/2#issuecomment-503535704 target_link_libraries(3rdparty_webrtc INTERFACE secur32 winmm dmoguids wmcodecdspuuid msdmo strmiids) endif() diff --git a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch index 5d1897193c9..1c87f7ffc0d 100644 --- a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch +++ b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch @@ -8,22 +8,17 @@ Subject: [PATCH] build: enable rtc_use_cxx11_abi option 1 file changed, 6 insertions(+) diff --git a/config/BUILDCONFIG.gn b/config/BUILDCONFIG.gn -index 0ef73ab2b..5ab677e27 100644 --- a/config/BUILDCONFIG.gn +++ b/config/BUILDCONFIG.gn -@@ -163,6 +163,12 @@ declare_args() { - is_component_build = is_debug && current_os != "ios" +@@ -171,6 +171,12 @@ declare_args() { + is_debug && current_os != "ios" && current_os != "watchos" } +declare_args() { -+ # Set to false to define "_GLIBCXX_USE_CXX11_ABI=0". If set to true, the -+ # default will be used, which corresponds to the new CXX11 ABI. ++ # Open3D: GCC libstdc++ ABI selection on Linux. + rtc_use_cxx11_abi = true +} + assert(!(is_debug && is_official_build), "Can't do official debug builds") - - # ============================================================================== --- -2.17.1 - + assert(!(current_os == "ios" && is_component_build), + "Can't use component build on iOS") diff --git a/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch b/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch new file mode 100644 index 00000000000..29df04e0a43 --- /dev/null +++ b/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch @@ -0,0 +1,11 @@ +diff --git a/build.gni b/build.gni +--- a/build.gni ++++ b/build.gni +@@ -10,6 +10,7 @@ enable_java_templates = true + + # Enables assertions on safety checks in libc++. + enable_safe_libcxx = true ++enable_safe_libstdcxx = true + + # Don't set this variable to true when building standalone WebRTC, it is + # only needed to support both WebRTC standalone and Chromium builds. diff --git a/3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch b/3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch new file mode 100644 index 00000000000..396a02f6aaf --- /dev/null +++ b/3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch @@ -0,0 +1,12 @@ +diff --git a/rtc_base/ssl_stream_adapter.h b/rtc_base/ssl_stream_adapter.h +--- a/rtc_base/ssl_stream_adapter.h ++++ b/rtc_base/ssl_stream_adapter.h +@@ -129,7 +129,7 @@ class SSLStreamAdapter : public StreamInterface { + static std::unique_ptr Create( + std::unique_ptr stream, + absl::AnyInvocable handshake_error, +- nullptr_t /*field_trials*/) { ++ std::nullptr_t /*field_trials*/) { + return Create(std::move(stream), std::move(handshake_error)); + } + diff --git a/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch b/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch new file mode 100644 index 00000000000..ed0a638cd7f --- /dev/null +++ b/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch @@ -0,0 +1,17 @@ +diff --git a/p2p/base/port_interface.h b/p2p/base/port_interface.h +--- a/p2p/base/port_interface.h ++++ b/p2p/base/port_interface.h +@@ -52,7 +52,11 @@ class PortInterface { + virtual ~PortInterface(); + + virtual IceCandidateType Type() const = 0; ++#if defined(__GNUC__) && !defined(__clang__) ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wchanges-meaning" ++#endif + virtual const Network* Network() const = 0; ++#if defined(__GNUC__) && !defined(__clang__) ++#pragma GCC diagnostic pop ++#endif + + // Methods to set/get ICE role and tiebreaker values. diff --git a/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch b/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch new file mode 100644 index 00000000000..8dc679d130f --- /dev/null +++ b/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch @@ -0,0 +1,12 @@ +diff --git a/call/payload_type_picker.cc b/call/payload_type_picker.cc +--- a/call/payload_type_picker.cc ++++ b/call/payload_type_picker.cc +@@ -354,7 +354,7 @@ RTCErrorOr RtpHeaderExtensionRecorder::LookupId(absl::string_view uri, + + RTCErrorOr RtpHeaderExtensionRecorder::LookupId(absl::string_view uri, + bool encrypt) const { +- auto it = uri_to_id_.find(std::pair{uri, encrypt}); ++ auto it = uri_to_id_.find(std::pair{std::string(uri), encrypt}); + if (it == uri_to_id_.end()) { + return RTCError(RTCErrorType::INVALID_PARAMETER, + "No ID found for extension"); diff --git a/3rdparty/webrtc/0006-build-win-dynamic-crt.patch b/3rdparty/webrtc/0006-build-win-dynamic-crt.patch new file mode 100644 index 00000000000..c5f7277dabf --- /dev/null +++ b/3rdparty/webrtc/0006-build-win-dynamic-crt.patch @@ -0,0 +1,35 @@ +Subject: [PATCH] build: allow dynamic CRT for non-component desktop Windows + +Open3D ships a static libwebrtc but needs to support both the static +(/MT[d]) and dynamic (/MD[d]) MSVC runtimes. Upstream ties the CRT choice +to is_component_build, which would force a component (shared) build to get +/MD. Add an rtc_win_dynamic_crt gn arg so a non-component static build can +still select the dynamic CRT, matching Open3D STATIC_WINDOWS_RUNTIME=OFF. + +diff --git a/config/win/BUILD.gn b/config/win/BUILD.gn +--- a/config/win/BUILD.gn ++++ b/config/win/BUILD.gn +@@ -513,6 +513,13 @@ if (build_with_chromium && current_cpu == target_cpu && host_os == "win") { + # Configures how the runtime library (CRT) is going to be used. + # See https://msdn.microsoft.com/en-us/library/2kzt1wy3.aspx for a reference of + # what each value does. ++declare_args() { ++ # Open3D: use the dynamic CRT (/MD[d]) for non-component desktop Windows ++ # builds so we can ship a static libwebrtc that links against the dynamic ++ # MSVC runtime (matches Open3D STATIC_WINDOWS_RUNTIME=OFF). ++ rtc_win_dynamic_crt = false ++} ++ + config("default_crt") { + if (is_component_build) { + # Component mode: dynamic CRT. Since the library is shared, it requires +@@ -525,6 +532,9 @@ config("default_crt") { + # contains a details explanation of what is happening with the Windows + # CRT in Visual Studio releases related to Windows store applications. + configs = [ ":dynamic_crt" ] ++ } else if (rtc_win_dynamic_crt) { ++ # Open3D: static library with the dynamic CRT (/MD[d]). ++ configs = [ ":dynamic_crt" ] + } else { + # Desktop Windows: static CRT. + configs = [ ":static_crt" ] diff --git a/3rdparty/webrtc/CMakeLists.txt b/3rdparty/webrtc/CMakeLists.txt index 34d052ae4df..6876f6d340a 100644 --- a/3rdparty/webrtc/CMakeLists.txt +++ b/3rdparty/webrtc/CMakeLists.txt @@ -31,6 +31,9 @@ cmake_dependent_option(WEBRTC_IS_DEBUG "WebRTC Debug build. Use ON for Win32 Open3D Debug." OFF "NOT CMAKE_BUILD_TYPE STREQUAL Debug OR NOT WIN32" ON) option(GLIBCXX_USE_CXX11_ABI "Set -D_GLIBCXX_USE_CXX11_ABI=1" ON) +if(MSVC) + option(WEBRTC_STATIC_MSVC_RUNTIME "Use /MT /MTd for WebRTC (Windows)" ON) +endif() # Set paths set(WEBRTC_ROOT ${PROJECT_SOURCE_DIR}) @@ -55,12 +58,10 @@ set(WEBRTC_NINJA_ROOT ${WEBRTC_ROOT}/src/out/${WEBRTC_BUILD}) # Common configs for WebRTC include(${PROJECT_SOURCE_DIR}/webrtc_common.cmake) -# Generate build/args.gn -if(NOT EXISTS ${WEBRTC_NINJA_ROOT}/args.gn) - get_webrtc_args(WEBRTC_ARGS) - file(WRITE ${WEBRTC_NINJA_ROOT}/args.gn ${WEBRTC_ARGS}) - message(STATUS "Configs written to ${WEBRTC_NINJA_ROOT}/args.gn") -endif() +get_webrtc_args(WEBRTC_ARGS) +file(MAKE_DIRECTORY "${WEBRTC_NINJA_ROOT}") +file(WRITE "${WEBRTC_NINJA_ROOT}/args.gn" "${WEBRTC_ARGS}") +message(STATUS "WebRTC args.gn -> ${WEBRTC_NINJA_ROOT}/args.gn") # libwebrtc.a add_custom_target(webrtc @@ -95,6 +96,8 @@ set_target_properties(webrtc_extra PROPERTIES # `-- libwebrtc_extra.a file(GLOB_RECURSE WEBRTC_INCLUDES RELATIVE ${WEBRTC_ROOT}/src ${WEBRTC_ROOT}/src/*.h + ${WEBRTC_ROOT}/src/*.hpp + ${WEBRTC_ROOT}/src/*.inc ) foreach(header ${WEBRTC_INCLUDES}) get_filename_component(dir ${header} DIRECTORY) diff --git a/3rdparty/webrtc/README.md b/3rdparty/webrtc/README.md index eb54b078089..c00b8d09a8a 100644 --- a/3rdparty/webrtc/README.md +++ b/3rdparty/webrtc/README.md @@ -13,13 +13,35 @@ webrtc_download.cmake # Used by Open3D CMake. Consume pre-compiled WebRTC. (Meth webrtc_build.cmake # Used by Open3D CMake. Build and consume WebRTC. (Method 2) # Other files -0001-xxx.patch x3 # Git patch for -DBUILD_WEBRTC_FROM_SOURCE=ON. (Method 1 Prepare-Phase & Method 2) +000*.patch # Git patches applied before building WebRTC. (Method 1 Prepare-Phase & Method 2) +apply_webrtc_patches.sh # Applies the patches to the WebRTC checkout. (Method 1 Prepare-Phase & Method 2) CMakeLists.txt # Used by `webrtc_build.sh` to compile WebRTC. (Method 1 Prepare-Phase) Dockerfile.webrtc # Calls `webrtc_build.sh` to compile WebRTC. (Method 1 Prepare-Phase) webrtc_build.sh # Used by `Dockerfile.webrtc`. (Method 1 Prepare-Phase) -webrtc_common.cmake # Specifies Common WebRTC targets. (Method 1 Prepare-Phase) +webrtc_common.cmake # Specifies Common WebRTC targets and gn args. (Method 1 Prepare-Phase) ``` +## Patches + +Applied by `apply_webrtc_patches.sh` (each is skipped if it does not apply +cleanly to the pinned WebRTC commit): + +``` +0001-src-enable-rtc_use_cxx11_abi-option.patch # -> src +0001-build-enable-rtc_use_cxx11_abi-option.patch # -> src/build +0001-third_party-enable-rtc_use_cxx11_abi-option.patch # -> src/third_party +0002-build-enable_safe_libstdcxx.patch # -> src/build_overrides +0003-src-fix-nullptr_t-with-libstdcxx.patch # -> src (GCC) +0004-src-gcc-suppress-port-interface-network.patch # -> src (GCC) +0005-call-payload_type_picker-gcc-flat_tree.patch # -> src (GCC) +0006-build-win-dynamic-crt.patch # -> src/build (Windows /MD) +``` + +`0006-build-win-dynamic-crt.patch` adds a `rtc_win_dynamic_crt` gn arg so a +non-component (static) Windows build can use the dynamic MSVC runtime +(`/MD[d]`). This lets Open3D ship a static `libwebrtc` for both +`STATIC_WINDOWS_RUNTIME=ON` (`/MT[d]`) and `OFF` (`/MD[d]`). + ## Method 1 The pre-compiled WebRTC package used in Method 1 is generated by diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh new file mode 100755 index 00000000000..94e89a911a5 --- /dev/null +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Apply Open3D WebRTC patches under WEBRTC_SRC (webrtc checkout src/). +set -euo pipefail + +OPEN3D_DIR="${1:?Open3D repo path}" +WEBRTC_SRC="${2:?WebRTC src path}" + +# Apply a patch, hard-failing if a required patch cannot be applied. +# +# Args: [required|optional] (default: required) +# +# A patch is considered "already applied" when it applies in reverse; in that +# case it is skipped without error so the script is safe to re-run and tolerant +# of fixes that have landed upstream. A required patch that neither applies nor +# is already applied aborts the build: these patches add gn args / ABI defines +# or fix compile errors, so silently skipping them produces broken or +# confusing artifacts (e.g. an undeclared gn arg in args.gn, a C++11/C++17 ABI +# mismatch, or a hard compile error) rather than a clear failure here. +apply_one() { + local patch="$1" + local dir="$2" + local required="${3:-required}" + local name + name="$(basename "$patch")" + if git -C "$dir" apply --check "$patch" 2>/dev/null; then + git -C "$dir" apply "$patch" + echo "Applied $name in $dir" + elif git -C "$dir" apply --reverse --check "$patch" 2>/dev/null; then + echo "Skip $name (already applied) in $dir" + elif [[ "$required" == "optional" ]]; then + echo "Skip $name (does not apply; optional) in $dir" + else + echo "ERROR: required patch $name does not apply in $dir." >&2 + echo " Refresh the patch for the pinned WebRTC commit." >&2 + exit 1 + fi +} + +PATCH_DIR="$OPEN3D_DIR/3rdparty/webrtc" +# Required: declare gn args consumed by args.gn and fix GCC compile errors. +apply_one "$PATCH_DIR/0001-src-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0001-build-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/build" +apply_one "$PATCH_DIR/0001-third_party-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/third_party" +apply_one "$PATCH_DIR/0002-build-enable_safe_libstdcxx.patch" "$WEBRTC_SRC/build_overrides" +apply_one "$PATCH_DIR/0003-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0004-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0005-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0006-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index b4e48f126b2..809cb804274 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -1,78 +1,75 @@ #!/usr/bin/env bash set -euox pipefail -# This script builds WebRTC for Open3D for Ubuntu and macOS. For Windows, see -# .github/workflows/webrtc.yml +# Builds WebRTC static libraries for Open3D (Ubuntu/macOS). Windows: webrtc.yml # -# Usage: -# $ bash # Start a new shell -# Specify custom configuration by exporting environment variables -# GLIBCXX_USE_CXX11_ABI, WEBRTC_COMMIT and DEPOT_TOOLS_COMMIT, if required. -# $ source 3rdparty/webrtc/webrtc_build.sh -# $ install_dependencies_ubuntu # Ubuntu only -# $ download_webrtc_sources -# $ build_webrtc -# A webrtc__platform.tar.gz file will be created that can be used to -# build Open3D with WebRTC support. -# -# Procedure: -# -# 1) Download depot_tools, webrtc to following directories: -# ├── Oepn3D -# ├── depot_tools -# └── webrtc -#    ├── .gclient -#    └── src +# Layout (default: repo parent holds depot_tools + webrtc): +# / +# ├── Open3D/ # this repository +# ├── depot_tools/ +# └── webrtc/ +# └── src/ # -# 2) depot_tools and webrtc have compatible versions, see: -# https://chromium.googlesource.com/chromium/src/+/master/docs/building_old_revisions.md -# -# 3) Apply the following patch to enable GLIBCXX_USE_CXX11_ABI selection: -# - 0001-build-enable-rtc_use_cxx11_abi-option.patch # apply to webrtc/src -# - 0001-src-enable-rtc_use_cxx11_abi-option.patch # apply to webrtc/src/build -# - 0001-third_party-enable-rtc_use_cxx11_abi-option.patch # apply to webrtc/src/third_party -# Note that these patches may or may not be compatible with your custom -# WebRTC commits. You may have to patch them manually. +# Usage: +# cd /path/to/Open3D +# export WEBRTC_COMMIT=... # optional +# source 3rdparty/webrtc/webrtc_build.sh +# install_dependencies_ubuntu # optional on Ubuntu +# download_webrtc_sources +# build_webrtc -# Date: Wed Apr 7 19:12:13 2021 +0200 -WEBRTC_COMMIT=${WEBRTC_COMMIT:-60e674842ebae283cc6b2627f4b6f2f8186f3317} -# Date: Wed Apr 7 21:35:29 2021 +0000 -DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-e1a98941d3ab10549be6d82d0686bb0fb91ec903} +# libwebrtc-bin M149 / Open3D target milestone +WEBRTC_COMMIT=${WEBRTC_COMMIT:-e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e} +# Optional pin; unset uses depot_tools HEAD. +DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-} -GLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI:-0} -NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN)} # POSIX: MacOS + Linux -SUDO=${SUDO:-sudo} # Set to command if running inside docker -export PATH="$PWD/../depot_tools":${PATH} # $(basename $PWD) == Open3D -export DEPOT_TOOLS_UPDATE=0 +GLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI:-1} +NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN)} +SUDO=${SUDO:-sudo} + +webrtc_work_root() { + if [[ -n "${WEBRTC_WORK_ROOT:-}" ]]; then + echo "$WEBRTC_WORK_ROOT" + else + dirname "$PWD" + fi +} + +webrtc_setup_path() { + export PATH="$(webrtc_work_root)/depot_tools:${PATH}" + export DEPOT_TOOLS_UPDATE=0 +} install_dependencies_ubuntu() { options="$(echo "$@" | tr ' ' '|')" - # Dependencies - # python* : resolve ImportError: No module named pkg_resources - # libglib2.0-dev: resolve pkg_config("glib") $SUDO apt-get update $SUDO apt-get install -y \ apt-transport-https \ build-essential \ ca-certificates \ + clang \ git \ gnupg \ libglib2.0-dev \ - python \ - python-pip \ - python-setuptools \ - python-wheel \ + libnss3-dev \ + libgtk-3-dev \ + ninja-build \ + python3 \ + python3-pip \ + python3-setuptools \ + pkg-config \ software-properties-common \ tree \ curl - curl https://apt.kitware.com/keys/kitware-archive-latest.asc \ - 2>/dev/null | gpg --dearmor - | - $SUDO sed -n 'w /etc/apt/trusted.gpg.d/kitware.gpg' # Write to file, no stdout - source <(grep VERSION_CODENAME /etc/os-release) - $SUDO apt-add-repository --yes "deb https://apt.kitware.com/ubuntu/ $VERSION_CODENAME main" - $SUDO apt-get update - $SUDO apt-get --yes install cmake - cmake --version >/dev/null + if ! command -v cmake >/dev/null 2>&1 || [[ $(cmake --version | head -1 | grep -oE '[0-9]+\.[0-9]+') < "3.18" ]]; then + curl https://apt.kitware.com/keys/kitware-archive-latest.asc \ + 2>/dev/null | gpg --dearmor - | + $SUDO tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null + source <(grep VERSION_CODENAME /etc/os-release) + $SUDO apt-add-repository --yes "deb https://apt.kitware.com/ubuntu/ $VERSION_CODENAME main" + $SUDO apt-get update + $SUDO apt-get --yes install cmake + fi if [[ "purge-cache" =~ ^($options)$ ]]; then $SUDO apt-get clean $SUDO rm -rf /var/lib/apt/lists/* @@ -80,67 +77,69 @@ install_dependencies_ubuntu() { } download_webrtc_sources() { - # PWD=Open3D - pushd .. - echo Get depot_tools - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - git -C depot_tools checkout $DEPOT_TOOLS_COMMIT + local root + root="$(webrtc_work_root)" + pushd "$root" + if [[ ! -d depot_tools ]]; then + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + fi + if [[ -n "$DEPOT_TOOLS_COMMIT" ]]; then + git -C depot_tools checkout "$DEPOT_TOOLS_COMMIT" + fi + webrtc_setup_path command -V fetch - echo Get WebRTC - mkdir webrtc - cd webrtc - fetch webrtc + if [[ ! -d webrtc/src ]]; then + mkdir -p webrtc + pushd webrtc + fetch --nohooks webrtc + popd + fi - # Checkout to a specific version - # Ref: https://chromium.googlesource.com/chromium/src/+/master/docs/building_old_revisions.md - git -C src checkout $WEBRTC_COMMIT - git -C src submodule update --init --recursive - echo gclient sync - gclient sync -D --force --reset - cd .. - echo random.org - curl "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" -o skipcache + git -C webrtc/src checkout "$WEBRTC_COMMIT" + git -C webrtc/src submodule update --init --recursive + pushd webrtc + gclient sync -D --force --reset --no-history + popd popd } build_webrtc() { - # PWD=Open3D - OPEN3D_DIR="$PWD" - echo Apply patches - cp 3rdparty/webrtc/{CMakeLists.txt,webrtc_common.cmake} ../webrtc - git -C ../webrtc/src apply \ - "$OPEN3D_DIR"/3rdparty/webrtc/0001-src-enable-rtc_use_cxx11_abi-option.patch - git -C ../webrtc/src/build apply \ - "$OPEN3D_DIR"/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch - git -C ../webrtc/src/third_party apply \ - "$OPEN3D_DIR"/3rdparty/webrtc/0001-third_party-enable-rtc_use_cxx11_abi-option.patch - WEBRTC_COMMIT_SHORT=$(git -C ../webrtc/src rev-parse --short=7 HEAD) + local root open3d_dir + open3d_dir="$PWD" + root="$(webrtc_work_root)" + webrtc_setup_path - echo Build WebRTC - mkdir ../webrtc/build - pushd ../webrtc/build - cmake -DCMAKE_INSTALL_PREFIX=../../webrtc_release \ - -DGLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI} \ + cp "$open3d_dir"/3rdparty/webrtc/{CMakeLists.txt,webrtc_common.cmake} "$root/webrtc/" + bash "$open3d_dir"/3rdparty/webrtc/apply_webrtc_patches.sh \ + "$open3d_dir" "$root/webrtc/src" + + WEBRTC_COMMIT_SHORT=$(git -C "$root/webrtc/src" rev-parse --short=7 HEAD) + + mkdir -p "$root/webrtc/build" + pushd "$root/webrtc/build" + cmake -G Ninja \ + -DCMAKE_INSTALL_PREFIX="$root/webrtc_release" \ + -DGLIBCXX_USE_CXX11_ABI="${GLIBCXX_USE_CXX11_ABI}" \ .. - make -j$NPROC - make install - popd # PWD=Open3D - pushd .. - tree -L 2 webrtc_release || ls webrtc_release/* + ninja -j"${NPROC}" + ninja install + popd - echo Package WebRTC + pushd "$root" + tree -L 2 webrtc_release || ls -la webrtc_release if [[ $(uname -s) == 'Linux' ]]; then tar -czf \ - "$OPEN3D_DIR/webrtc_${WEBRTC_COMMIT_SHORT}_linux_cxx-abi-${GLIBCXX_USE_CXX11_ABI}.tar.gz" \ - webrtc_release + "$open3d_dir/webrtc_${WEBRTC_COMMIT_SHORT}_linux_cxx-abi-${GLIBCXX_USE_CXX11_ABI}.tar.gz" \ + -C "$root/webrtc_release" . elif [[ $(uname -s) == 'Darwin' ]]; then tar -czf \ - "$OPEN3D_DIR/webrtc_${WEBRTC_COMMIT_SHORT}_macos.tar.gz" \ - webrtc_release + "$open3d_dir/webrtc_${WEBRTC_COMMIT_SHORT}_macos_arm64.tar.gz" \ + -C "$root/webrtc_release" . fi - popd # PWD=Open3D - webrtc_package=$(ls webrtc_*.tar.gz) - cmake -E sha256sum "$webrtc_package" | tee "checksum_${webrtc_package%%.*}.txt" + popd + + webrtc_package=$(ls "$open3d_dir"/webrtc_*.tar.gz | tail -1) + cmake -E sha256sum "$webrtc_package" | tee "$open3d_dir/checksum_${webrtc_package##*/}" | sed 's|.*/||' ls -alh "$webrtc_package" } diff --git a/3rdparty/webrtc/webrtc_common.cmake b/3rdparty/webrtc/webrtc_common.cmake index 98c19336c6a..70708eb27b7 100644 --- a/3rdparty/webrtc/webrtc_common.cmake +++ b/3rdparty/webrtc/webrtc_common.cmake @@ -10,7 +10,7 @@ function(get_webrtc_args WEBRTC_ARGS) set(WEBRTC_ARGS "") if(NOT MSVC) - # ABI selection + # ABI selection (Linux only; Open3D Ubuntu 22.04 uses cxx11 ABI=1). if(GLIBCXX_USE_CXX11_ABI) set(WEBRTC_ARGS rtc_use_cxx11_abi=true\n${WEBRTC_ARGS}) else() @@ -18,57 +18,65 @@ function(get_webrtc_args WEBRTC_ARGS) endif() endif() - if (APPLE) # WebRTC default + if(APPLE) set(WEBRTC_ARGS is_clang=true\n${WEBRTC_ARGS}) - else() - # Do not use Google clang for compilation due to LTO error when Open3D - # is built with gcc on Ubuntu 20.04. + if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(WEBRTC_ARGS target_cpu=\"arm64\"\n${WEBRTC_ARGS}) + endif() + elseif(UNIX) set(WEBRTC_ARGS is_clang=false\n${WEBRTC_ARGS}) + set(WEBRTC_ARGS extra_cxxflags=[\"-Wno-changes-meaning\"]\n${WEBRTC_ARGS}) endif() - # Don't use libc++ (Clang), use libstdc++ (GNU) - # https://stackoverflow.com/a/47384787/1255535 set(WEBRTC_ARGS use_custom_libcxx=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS use_custom_libcxx_for_host=false\n${WEBRTC_ARGS}) - # Debug/Release if(WEBRTC_IS_DEBUG) set(WEBRTC_ARGS is_debug=true\n${WEBRTC_ARGS}) - if (MSVC) - # WebRTC default is false in Debug due to a performance penalty, but this would disable - # iterator debugging for Open3D and any user code as well with MSVC. + if(MSVC) set(WEBRTC_ARGS enable_iterator_debugging=true\n${WEBRTC_ARGS}) endif() else() set(WEBRTC_ARGS is_debug=false\n${WEBRTC_ARGS}) endif() - # H264 support - set(WEBRTC_ARGS is_chrome_branded=true\n${WEBRTC_ARGS}) + # H.264 (replaces deprecated is_chrome_branded on recent milestones). + set(WEBRTC_ARGS rtc_use_h264=true\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_include_tests=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_enable_protobuf=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_build_examples=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_build_tools=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS treat_warnings_as_errors=false\n${WEBRTC_ARGS}) - set(WEBRTC_ARGS rtc_enable_libevent=false\n${WEBRTC_ARGS}) - set(WEBRTC_ARGS rtc_build_libevent=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS use_sysroot=false\n${WEBRTC_ARGS}) + set(WEBRTC_ARGS rtc_use_perfetto=false\n${WEBRTC_ARGS}) - # Disable screen capturing set(WEBRTC_ARGS rtc_use_x11=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_use_pipewire=false\n${WEBRTC_ARGS}) - # Disable sound support set(WEBRTC_ARGS rtc_include_pulse_audio=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_include_internal_audio_device=false\n${WEBRTC_ARGS}) - # Use ccache if available, not recommended inside Docker + if(MSVC) + # Always build a static (non-component) libwebrtc that uses the MSVC STL + # (use_custom_libcxx=false, set above) so its ABI matches Open3D. Force + # is_component_build=false because it defaults to ON in Debug, which would + # produce shared component DLLs instead of a static lib. The MSVC runtime + # (/MT[d] vs /MD[d]) is selected independently via the rtc_win_dynamic_crt + # gn arg added by 0006-build-win-dynamic-crt.patch. + set(WEBRTC_ARGS is_component_build=false\n${WEBRTC_ARGS}) + if(WEBRTC_STATIC_MSVC_RUNTIME) + set(WEBRTC_ARGS rtc_win_dynamic_crt=false\n${WEBRTC_ARGS}) + else() + set(WEBRTC_ARGS rtc_win_dynamic_crt=true\n${WEBRTC_ARGS}) + endif() + endif() + find_program(CCACHE_BIN "ccache") if(CCACHE_BIN) - set(WEBRTC_ARGS cc_wrapper="ccache"\n${WEBRTC_ARGS}) + set(WEBRTC_ARGS cc_wrapper=\"ccache\"\n${WEBRTC_ARGS}) endif() - set(WEBRTC_ARGS ${WEBRTC_ARGS} PARENT_SCOPE) + set(WEBRTC_ARGS ${WEBRTC_ARGS} PARENT_SCOPE) endfunction() # webrtc -> libwebrtc.a @@ -79,13 +87,17 @@ set(NINJA_TARGETS jsoncpp builtin_video_decoder_factory builtin_video_encoder_factory - peerconnection + peer_connection p2p_server_utils task_queue default_task_queue_factory + # M149 modular PeerConnectionFactory (not all pulled into libwebrtc.a). + field_trials + enable_media_with_defaults + create_modular_peer_connection_factory + environment_factory ) -# Byproducts for ninja build, later packaged by CMake into libwebrtc_extra.a if(NOT WEBRTC_NINJA_ROOT) message(FATAL_ERROR "Please define WEBRTC_NINJA_ROOT before including webrtc_common.cmake") endif() @@ -96,4 +108,20 @@ set(EXTRA_WEBRTC_OBJS ${WEBRTC_NINJA_ROOT}/obj/p2p/p2p_server_utils/stun_server${CMAKE_CXX_OUTPUT_EXTENSION} ${WEBRTC_NINJA_ROOT}/obj/p2p/p2p_server_utils/turn_server${CMAKE_CXX_OUTPUT_EXTENSION} ${WEBRTC_NINJA_ROOT}/obj/rtc_base/rtc_json/json${CMAKE_CXX_OUTPUT_EXTENSION} - ) + ${WEBRTC_NINJA_ROOT}/obj/api/field_trials/field_trials${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/field_trials_registry/field_trials_registry${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/enable_media/enable_media${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/enable_media_with_defaults/enable_media_with_defaults${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/create_modular_peer_connection_factory/create_modular_peer_connection_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/environment/environment_factory/environment_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/environment/deprecated_global_field_trials/deprecated_global_field_trials${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/audio_codecs/builtin_audio_encoder_factory/builtin_audio_encoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/audio_codecs/builtin_audio_decoder_factory/builtin_audio_decoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/builtin_video_encoder_factory/builtin_video_encoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/builtin_video_decoder_factory/builtin_video_decoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/media/rtc_simulcast_encoder_adapter/simulcast_encoder_adapter${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/media/rtc_internal_video_codecs/internal_encoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/media/rtc_internal_video_codecs/internal_decoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/rtc_software_fallback_wrappers/video_encoder_software_fallback_wrapper${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/rtc_software_fallback_wrappers/video_decoder_software_fallback_wrapper${CMAKE_CXX_OUTPUT_EXTENSION} +) diff --git a/3rdparty/webrtc/webrtc_download.cmake b/3rdparty/webrtc/webrtc_download.cmake index 8ac5a0b6ef7..075c4ac00d8 100644 --- a/3rdparty/webrtc/webrtc_download.cmake +++ b/3rdparty/webrtc/webrtc_download.cmake @@ -4,43 +4,74 @@ include(ExternalProject) -set(WEBRTC_VER 60e6748) +set(WEBRTC_VER e8b4d4c) +if(DEFINED ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE} AND NOT "$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}" STREQUAL "") + set(WEBRTC_URL "file://$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}") +endif() if (APPLE) - set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc/webrtc_${WEBRTC_VER}_macos_10.14.tar.gz - ) - set(WEBRTC_SHA256 e9d1f4e4fefb2e28ef4f16cf4a4f0008baf4fe638ca3ad329e82e7fd0ce87f56) + if(NOT WEBRTC_URL) + set(WEBRTC_URL + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_macos_arm64.tar.gz + ) + endif() + # Update after publishing M149 macOS arm64 artifact from webrtc.yml. + set(WEBRTC_SHA256 PLACEHOLDER_MACOS_ARM64_SHA256) elseif (WIN32) - if (BUILD_SHARED_LIBS OR NOT STATIC_WINDOWS_RUNTIME) - message(FATAL_ERROR "Pre-built WebRTC binaries are not available for " - "BUILD_SHARED_LIBS=ON or STATIC_WINDOWS_RUNTIME=OFF. Please use " - "(a) BUILD_WEBRTC=OFF or " - "(b) BUILD_SHARED_LIBS=OFF and STATIC_WINDOWS_RUNTIME=ON or " - "(c) BUILD_WEBRTC_FROM_SOURCE=ON") + # Prebuilt WebRTC is a static lib for both MSVC runtimes. A shared Open3D + # build links it into open3d.dll and always uses the dynamic runtime, so + # only BUILD_SHARED_LIBS=ON + STATIC_WINDOWS_RUNTIME=ON is unsupported. + if (BUILD_SHARED_LIBS AND STATIC_WINDOWS_RUNTIME) + message(FATAL_ERROR "Pre-built WebRTC does not support " + "BUILD_SHARED_LIBS=ON with STATIC_WINDOWS_RUNTIME=ON. Use " + "STATIC_WINDOWS_RUNTIME=OFF or BUILD_WEBRTC_FROM_SOURCE=ON.") endif() - set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc/webrtc_${WEBRTC_VER}_win.zip - ) - set(WEBRTC_SHA256 f4686d0028ef5c36c5d7158a638fa834b63183b522f0b63932f7f70ebffeea22) -else() # Linux - if(GLIBCXX_USE_CXX11_ABI) + if(STATIC_WINDOWS_RUNTIME) + set(WEBRTC_RUNTIME_TAG mt) + else() + set(WEBRTC_RUNTIME_TAG md) + endif() + if(CMAKE_BUILD_TYPE STREQUAL Debug) + set(WEBRTC_CONFIG_TAG Debug) + else() + set(WEBRTC_CONFIG_TAG Release) + endif() + if(NOT WEBRTC_URL) set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v3/webrtc_${WEBRTC_VER}_cxx-abi-1.tar.gz + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_win_${WEBRTC_CONFIG_TAG}_${WEBRTC_RUNTIME_TAG}.zip ) - set(WEBRTC_SHA256 0d98ddbc4164b9e7bfc50b7d4eaa912a753dabde0847d85a64f93a062ae4c335) - else() + endif() + # Update after publishing four Windows artifacts from webrtc.yml. + set(WEBRTC_SHA256 PLACEHOLDER_WIN_SHA256) +else() # Linux + if(NOT WEBRTC_URL) + if(NOT GLIBCXX_USE_CXX11_ABI) + message(FATAL_ERROR "Pre-built WebRTC with GLIBCXX_USE_CXX11_ABI=OFF is " + "no longer provided. Use GLIBCXX_USE_CXX11_ABI=ON or " + "BUILD_WEBRTC_FROM_SOURCE=ON.") + endif() set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v3/webrtc_${WEBRTC_VER}_cxx-abi-0.tar.gz + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_linux_cxx-abi-1.tar.gz ) - set(WEBRTC_SHA256 2a3714713908f84079f1fbce8594c9b7010846b5db74b086f7bf30f22f1f5835) endif() + set(WEBRTC_SHA256 1b529bf448d5abd07ec1f8d310ee5c94bd79e84fe563ae1562420f8e478cc202) +endif() + +if(WEBRTC_SHA256 MATCHES "^PLACEHOLDER") + message(WARNING "WebRTC prebuilt SHA256 not set for this platform (${WEBRTC_SHA256}). " + "Set OPEN3D_WEBRTC_PREBUILT_ARCHIVE or update webrtc_download.cmake after CI publish.") + unset(WEBRTC_SHA256) +endif() + +set(_webrtc_url_hash "") +if(WEBRTC_SHA256) + set(_webrtc_url_hash URL_HASH SHA256=${WEBRTC_SHA256}) endif() ExternalProject_Add( ext_webrtc PREFIX webrtc URL ${WEBRTC_URL} - URL_HASH SHA256=${WEBRTC_SHA256} + ${_webrtc_url_hash} DOWNLOAD_DIR "${OPEN3D_THIRD_PARTY_DOWNLOAD_DIR}/webrtc" UPDATE_COMMAND "" CONFIGURE_COMMAND "" @@ -50,22 +81,18 @@ ExternalProject_Add( ) ExternalProject_Get_Property(ext_webrtc SOURCE_DIR) -if (WIN32) - set(SOURCE_DIR "${SOURCE_DIR}/$,Debug,Release>") -endif() -set(LIBPNG_INCLUDE_DIRS ${INSTALL_DIR}/include/) # "/" is critical. -set(LIBPNG_LIB_DIR ${INSTALL_DIR}/${Open3D_INSTALL_LIB_DIR}) -set(LIBPNG_LIBRARIES ${lib_name}$<$:d>) +# Prebuilt layout: flat include/ and lib/ at archive root (M149 packages). +set(WEBRTC_PREBUILT_ROOT ${SOURCE_DIR}) # Variables consumed by find_dependencies.cmake set(WEBRTC_INCLUDE_DIRS - ${SOURCE_DIR}/include/ - ${SOURCE_DIR}/include/third_party/abseil-cpp/ - ${SOURCE_DIR}/include/third_party/jsoncpp/source/include/ - ${SOURCE_DIR}/include/third_party/jsoncpp/generated/ - ${SOURCE_DIR}/include/third_party/libyuv/include/ + ${WEBRTC_PREBUILT_ROOT}/include/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/abseil-cpp/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/jsoncpp/source/include/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/jsoncpp/generated/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/libyuv/include/ ) -set(WEBRTC_LIB_DIR ${SOURCE_DIR}/lib) +set(WEBRTC_LIB_DIR ${WEBRTC_PREBUILT_ROOT}/lib) set(WEBRTC_LIBRARIES webrtc webrtc_extra diff --git a/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp b/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp index cf4844c1064..c16315568cd 100644 --- a/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp +++ b/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp @@ -52,8 +52,8 @@ void MapOBEToClosestIdentity(EigenOBE& obe) { Eigen::Vector3d& radii = obe.radii_; Eigen::Vector3d col[3] = {R.col(0), R.col(1), R.col(2)}; double best_score = -1e9; - Eigen::Matrix3d best_R; - Eigen::Vector3d best_radii; + Eigen::Matrix3d best_R = Eigen::Matrix3d::Identity(); + Eigen::Vector3d best_radii = Eigen::Vector3d::Zero(); // Hard-coded permutations of indices [0,1,2] static const std::array, 6> permutations = { diff --git a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp index 6fd753baf43..5a164646eb5 100644 --- a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp +++ b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp @@ -39,14 +39,14 @@ void BitmapTrackSource::SetState( } void BitmapTrackSource::AddOrUpdateSink( - rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) { + webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) { RTC_DCHECK(worker_thread_checker_.IsCurrent()); source()->AddOrUpdateSink(sink, wants); } void BitmapTrackSource::RemoveSink( - rtc::VideoSinkInterface* sink) { + webrtc::VideoSinkInterface* sink) { RTC_DCHECK(worker_thread_checker_.IsCurrent()); source()->RemoveSink(sink); } diff --git a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h index 81ea8243392..5dd350a95b8 100644 --- a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h +++ b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h @@ -71,15 +71,15 @@ class BitmapTrackSource : public webrtc::Notifier { return absl::nullopt; } bool GetStats(Stats* stats) override { return false; } - void AddOrUpdateSink(rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) override; - void RemoveSink(rtc::VideoSinkInterface* sink) override; + void AddOrUpdateSink(webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) override; + void RemoveSink(webrtc::VideoSinkInterface* sink) override; bool SupportsEncodedOutput() const override { return false; } void GenerateKeyFrame() override {} - void AddEncodedSink(rtc::VideoSinkInterface* + void AddEncodedSink(webrtc::VideoSinkInterface* sink) override {} void RemoveEncodedSink( - rtc::VideoSinkInterface* sink) + webrtc::VideoSinkInterface* sink) override {} virtual void OnFrame(const std::shared_ptr& frame) override { @@ -88,7 +88,7 @@ class BitmapTrackSource : public webrtc::Notifier { } protected: - virtual rtc::VideoSourceInterface* source() = 0; + virtual webrtc::VideoSourceInterface* source() = 0; private: webrtc::SequenceChecker worker_thread_checker_; diff --git a/cpp/open3d/visualization/webrtc_server/CMakeLists.txt b/cpp/open3d/visualization/webrtc_server/CMakeLists.txt index f5977e72e52..97124c25de7 100644 --- a/cpp/open3d/visualization/webrtc_server/CMakeLists.txt +++ b/cpp/open3d/visualization/webrtc_server/CMakeLists.txt @@ -15,6 +15,12 @@ target_compile_definitions(webrtc_server PRIVATE _FILE_OFFSET_BITS=64 # for civetweb _LARGEFILE_SOURCE=1 # for civetweb ) +# Prebuilt WebRTC is compiled in Release (RTC_DCHECK_IS_ON=0). Match that when +# Open3D is Debug so inlined WebRTC headers do not reference missing DCHECK +# symbols (e.g. SequenceCheckerImpl::ExpectationToString). +if(NOT BUILD_WEBRTC_FROM_SOURCE) + target_compile_definitions(webrtc_server PRIVATE NDEBUG) +endif() add_dependencies(webrtc_server copy_html_dir) open3d_show_and_abort_on_warning(webrtc_server) @@ -22,6 +28,8 @@ open3d_set_global_properties(webrtc_server) open3d_set_open3d_lib_properties(webrtc_server) open3d_link_3rdparty_libraries(webrtc_server) +set_target_properties(webrtc_server PROPERTIES CXX_STANDARD 20) + if (NOT GUI_RESOURCE_DIR) message(FATAL_ERROR "GUI_RESOURCE_DIR is not defined.") @@ -29,6 +37,12 @@ endif() message(STATUS "Copying ${CMAKE_CURRENT_SOURCE_DIR}/html to ${GUI_RESOURCE_DIR}.") file(MAKE_DIRECTORY ${GUI_RESOURCE_DIR}) +# Favicon is shared with Sphinx docs (single source of truth). +set(OPEN3D_ICON "${CMAKE_SOURCE_DIR}/docs/_static/open3d_logo.ico") +if(NOT EXISTS "${OPEN3D_ICON}") + message(FATAL_ERROR "Open3D WebRTC favicon not found: ${OPEN3D_ICON}") +endif() + # Force update ${GUI_RESOURCE_DIR}/html every time. add_custom_target(copy_html_dir ALL COMMAND ${CMAKE_COMMAND} -E rm -rf @@ -36,4 +50,8 @@ add_custom_target(copy_html_dir ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/html ${GUI_RESOURCE_DIR}/html + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${OPEN3D_ICON} + ${GUI_RESOURCE_DIR}/html/open3d_logo.ico + DEPENDS ${OPEN3D_ICON} ) diff --git a/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp b/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp index a7ad0109607..4d23c2d933c 100644 --- a/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp +++ b/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp @@ -7,11 +7,13 @@ #include "open3d/visualization/webrtc_server/ImageCapturer.h" +#include #include #include #include #include #include +#include #include @@ -50,7 +52,7 @@ void ImageCapturer::OnCaptureResult( int height = (int)frame->GetShape(0); int width = (int)frame->GetShape(1); - rtc::scoped_refptr i420_buffer = + webrtc::scoped_refptr i420_buffer = webrtc::I420Buffer::Create(width, height); // frame->data() @@ -64,7 +66,7 @@ void ImageCapturer::OnCaptureResult( if (conversion_result >= 0) { webrtc::VideoFrame video_frame(i420_buffer, webrtc::VideoRotation::kVideoRotation_0, - rtc::TimeMicros()); + webrtc::TimeMicros()); if ((height_ == 0) && (width_ == 0)) { broadcaster_.OnFrame(video_frame); } else { @@ -77,13 +79,13 @@ void ImageCapturer::OnCaptureResult( } int stride_y = width; int stride_uv = (width + 1) / 2; - rtc::scoped_refptr scaled_buffer = + webrtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(width, height, stride_y, stride_uv, stride_uv); scaled_buffer->ScaleFrom( *video_frame.video_frame_buffer()->ToI420()); webrtc::VideoFrame frame = webrtc::VideoFrame( - scaled_buffer, webrtc::kVideoRotation_0, rtc::TimeMicros()); + scaled_buffer, webrtc::kVideoRotation_0, webrtc::TimeMicros()); broadcaster_.OnFrame(frame); } @@ -93,15 +95,15 @@ void ImageCapturer::OnCaptureResult( } } -// Override rtc::VideoSourceInterface. +// Override webrtc::VideoSourceInterface. void ImageCapturer::AddOrUpdateSink( - rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) { + webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) { broadcaster_.AddOrUpdateSink(sink, wants); } void ImageCapturer::RemoveSink( - rtc::VideoSinkInterface* sink) { + webrtc::VideoSinkInterface* sink) { broadcaster_.RemoveSink(sink); } diff --git a/cpp/open3d/visualization/webrtc_server/ImageCapturer.h b/cpp/open3d/visualization/webrtc_server/ImageCapturer.h index f1b094ba3ca..b8fc188cc9c 100644 --- a/cpp/open3d/visualization/webrtc_server/ImageCapturer.h +++ b/cpp/open3d/visualization/webrtc_server/ImageCapturer.h @@ -10,7 +10,8 @@ #pragma once -#include +#include +#include #include #include #include @@ -26,7 +27,7 @@ namespace open3d { namespace visualization { namespace webrtc_server { -class ImageCapturer : public rtc::VideoSourceInterface { +class ImageCapturer : public webrtc::VideoSourceInterface { public: ImageCapturer(const std::string& url_, const std::map& opts); @@ -39,23 +40,23 @@ class ImageCapturer : public rtc::VideoSourceInterface { ImageCapturer(const std::map& opts); virtual void AddOrUpdateSink( - rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) override; + webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) override; virtual void RemoveSink( - rtc::VideoSinkInterface* sink) override; + webrtc::VideoSinkInterface* sink) override; void OnCaptureResult(const std::shared_ptr& frame); protected: int width_; int height_; - rtc::VideoBroadcaster broadcaster_; + webrtc::VideoBroadcaster broadcaster_; }; class ImageTrackSource : public BitmapTrackSource { public: - static rtc::scoped_refptr Create( + static webrtc::scoped_refptr Create( const std::string& window_uid, const std::map& opts) { std::unique_ptr capturer = @@ -63,10 +64,9 @@ class ImageTrackSource : public BitmapTrackSource { if (!capturer) { return nullptr; } - rtc::scoped_refptr video_source = - new rtc::RefCountedObject( - std::move(capturer)); - return video_source; + return webrtc::scoped_refptr( + new webrtc::RefCountedObject( + std::move(capturer))); } void OnFrame(const std::shared_ptr& frame) final override { @@ -78,7 +78,7 @@ class ImageTrackSource : public BitmapTrackSource { : BitmapTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {} private: - rtc::VideoSourceInterface* source() override { + webrtc::VideoSourceInterface* source() override { return capturer_.get(); } std::unique_ptr capturer_; diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index 68ff8256e29..9e1ccaca0bd 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -15,6 +15,11 @@ #include "open3d/visualization/webrtc_server/PeerConnectionManager.h" +#include +#include +#include +#include +#include #include #include #include @@ -24,10 +29,11 @@ #include #include #include -#include +#include #include #include +#include #include #include "open3d/utility/IJsonConvertible.h" @@ -86,52 +92,41 @@ static IceServer GetIceServerFromUrl(const std::string &url) { return srv; } -static webrtc::PeerConnectionFactoryDependencies -CreatePeerConnectionFactoryDependencies() { - // Disable WebRTC's pacing delay and allow the pacer to drain its queue - // immediately. This reduces the added latency from packet smoothing, which - // is unnecessary for a local interactive streaming use case. - // Force playout delay to zero in the RTP header extension so the receiver - // renders frames immediately rather than buffering for jitter smoothing. - // Disable automatic resolution reduction so quality is only degraded if the - // encoder is genuinely overloaded (content hint kFluid still allows it). - webrtc::field_trial::InitFieldTrialsFromString( - "WebRTC-Pacer-DrainQueue/Enabled/" - "WebRTC-ForceSendPlayoutDelay/min_ms:0,max_ms:0/" - "WebRTC-Video-DisableAutomaticResize/Enabled/"); +static bool PeerConnectionHasStreamForWindow( + webrtc::PeerConnectionInterface* peer_connection, + const std::string& window_uid) { + if (!peer_connection) { + return false; + } + for (const auto& sender : peer_connection->GetSenders()) { + if (!sender) { + continue; + } + for (const std::string& stream_id : sender->stream_ids()) { + if (stream_id == window_uid) { + return true; + } + } + } + return false; +} +static webrtc::PeerConnectionFactoryDependencies +CreatePeerConnectionFactoryDependencies( + webrtc::FieldTrials* field_trials) { + (void)field_trials; webrtc::PeerConnectionFactoryDependencies dependencies; + dependencies.worker_thread = webrtc::Thread::Current(); dependencies.network_thread = nullptr; - dependencies.worker_thread = rtc::Thread::Current(); dependencies.signaling_thread = nullptr; - dependencies.call_factory = webrtc::CreateCallFactory(); - dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); - dependencies.event_log_factory = - absl::make_unique( - dependencies.task_queue_factory.get()); - - cricket::MediaEngineDependencies media_dependencies; - media_dependencies.task_queue_factory = - dependencies.task_queue_factory.get(); - - // Dummy audio factory. - rtc::scoped_refptr audio_device_module( + + webrtc::EnvironmentFactory env_factory; + env_factory.Set(field_trials); + dependencies.env = env_factory.Create(); + + dependencies.adm = webrtc::scoped_refptr( new webrtc::FakeAudioDeviceModule()); - media_dependencies.adm = std::move(audio_device_module); - media_dependencies.audio_encoder_factory = - webrtc::CreateBuiltinAudioEncoderFactory(); - media_dependencies.audio_decoder_factory = - webrtc::CreateBuiltinAudioDecoderFactory(); - media_dependencies.audio_processing = - webrtc::AudioProcessingBuilder().Create(); - - media_dependencies.video_encoder_factory = - webrtc::CreateBuiltinVideoEncoderFactory(); - media_dependencies.video_decoder_factory = - webrtc::CreateBuiltinVideoDecoderFactory(); - - dependencies.media_engine = - cricket::CreateMediaEngine(std::move(media_dependencies)); + webrtc::EnableMediaWithDefaults(dependencies); return dependencies; } @@ -141,12 +136,16 @@ PeerConnectionManager::PeerConnectionManager( const Json::Value &config, const std::string &publish_filter, const std::string &webrtc_udp_port_range) - : task_queue_factory_(webrtc::CreateDefaultTaskQueueFactory()), + : field_trials_(webrtc::FieldTrials::Create( + "WebRTC-Pacer-DrainQueue/Enabled/" + "WebRTC-ForceSendPlayoutDelay/min_ms:0,max_ms:0/" + "WebRTC-Video-DisableAutomaticResize/Enabled/")), peer_connection_factory_(webrtc::CreateModularPeerConnectionFactory( - CreatePeerConnectionFactoryDependencies())), + CreatePeerConnectionFactoryDependencies(field_trials_.get()))), ice_server_list_(ice_server_list), config_(config), publish_filter_(publish_filter) { + webrtc_worker_thread_ = webrtc::Thread::Current(); // Set the webrtc port range. webrtc_port_range_ = webrtc_udp_port_range; @@ -260,9 +259,9 @@ const Json::Value PeerConnectionManager::GetIceServers() { } // Get PeerConnection associated with peerid. -rtc::scoped_refptr +webrtc::scoped_refptr PeerConnectionManager::GetPeerConnection(const std::string &peerid) { - rtc::scoped_refptr peer_connection; + webrtc::scoped_refptr peer_connection; auto it = peerid_to_connection_.find(peerid); if (it != peerid_to_connection_.end()) { peer_connection = it->second->GetPeerConnection(); @@ -277,12 +276,12 @@ const Json::Value PeerConnectionManager::AddIceCandidate( std::string sdp_mid; int sdp_mlineindex = 0; std::string sdp; - if (!rtc::GetStringFromJsonObject(json_message, k_candidate_sdp_mid_name, + if (!webrtc::GetStringFromJsonObject(json_message, k_candidate_sdp_mid_name, &sdp_mid) || - !rtc::GetIntFromJsonObject(json_message, + !webrtc::GetIntFromJsonObject(json_message, k_candidate_sdp_mline_index_name, &sdp_mlineindex) || - !rtc::GetStringFromJsonObject(json_message, k_candidate_sdp_name, + !webrtc::GetStringFromJsonObject(json_message, k_candidate_sdp_name, &sdp)) { utility::LogWarning("Can't parse received message."); } else { @@ -304,7 +303,7 @@ const Json::Value PeerConnectionManager::AddIceCandidate( } else { std::lock_guard mutex_lock( peerid_to_connection_mutex_); - rtc::scoped_refptr + webrtc::scoped_refptr peer_connection = this->GetPeerConnection(peerid); if (peer_connection) { if (!peer_connection->AddIceCandidate(candidate.get())) { @@ -334,9 +333,9 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, std::string type; std::string sdp; - if (!rtc::GetStringFromJsonObject(json_message, + if (!webrtc::GetStringFromJsonObject(json_message, k_session_description_type_name, &type) || - !rtc::GetStringFromJsonObject(json_message, + !webrtc::GetStringFromJsonObject(json_message, k_session_description_sdp_name, &sdp)) { utility::LogWarning("Can't parse received message."); } else { @@ -348,12 +347,11 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, utility::LogError("Failed to initialize PeerConnection"); delete peer_connection_observer; } else { - rtc::scoped_refptr - peer_connection = - peer_connection_observer->GetPeerConnection(); - utility::LogDebug("nbStreams local: {}, remote: {}", - peer_connection->local_streams()->count(), - peer_connection->remote_streams()->count()); + webrtc::PeerConnectionInterface* peer_connection_ptr = + peer_connection_observer->GetPeerConnection().get(); + utility::LogDebug("nbSenders: {}, nbReceivers: {}", + peer_connection_ptr->GetSenders().size(), + peer_connection_ptr->GetReceivers().size()); // Register peerid. { @@ -371,19 +369,27 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, } // Set remote offer. - webrtc::SessionDescriptionInterface *session_description( - webrtc::CreateSessionDescription(type, sdp, nullptr)); + std::optional sdp_type = + webrtc::SdpTypeFromString(type); + std::unique_ptr + session_description; + if (!sdp_type) { + utility::LogError("Unknown session description type: {}.", type); + } else { + session_description = + webrtc::CreateSessionDescription(*sdp_type, sdp); + } if (!session_description) { utility::LogError( "Can't parse received session description message. " "Cannot create session description."); } else { - std::promise + std::promise remote_promise; - peer_connection->SetRemoteDescription( - SetSessionDescriptionObserver::Create(peer_connection, - remote_promise), - session_description); + peer_connection_ptr->SetRemoteDescription( + SetSessionDescriptionObserver::Create( + peer_connection_ptr, remote_promise), + session_description.release()); // Waiting for remote description. std::future remote_future = remote_promise.get_future(); @@ -398,7 +404,7 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, } // Add local stream. - if (!this->AddStreams(peer_connection, window_uid, options)) { + if (!this->AddStreams(peer_connection_ptr, window_uid, options)) { utility::LogError("Can't add stream {}, {}.", window_uid, options); } @@ -407,9 +413,9 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions rtc_options; std::promise local_promise; - peer_connection->CreateAnswer( - CreateSessionDescriptionObserver::Create(peer_connection, - local_promise), + peer_connection_ptr->CreateAnswer( + CreateSessionDescriptionObserver::Create( + peer_connection_ptr, local_promise), rtc_options); // Waiting for answer. @@ -438,26 +444,20 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, } bool PeerConnectionManager::WindowStillUsed(const std::string &window_uid) { - bool still_used = false; for (auto it : peerid_to_connection_) { - rtc::scoped_refptr peer_connection = - it.second->GetPeerConnection(); - rtc::scoped_refptr local_streams( - peer_connection->local_streams()); - for (unsigned int i = 0; i < local_streams->count(); i++) { - if (local_streams->at(i)->id() == window_uid) { - still_used = true; - break; - } + if (PeerConnectionHasStreamForWindow( + it.second->GetPeerConnection().get(), window_uid)) { + return true; } } - return still_used; + return false; } // Hangup a call. const Json::Value PeerConnectionManager::HangUp(const std::string &peerid) { bool result = false; PeerConnectionObserver *pc_observer = nullptr; + std::string hangup_window_uid; { std::lock_guard mutex_lock(peerid_to_connection_mutex_); auto it = peerid_to_connection_.find(peerid); @@ -469,37 +469,27 @@ const Json::Value PeerConnectionManager::HangUp(const std::string &peerid) { if (peerid_to_window_uid_.count(peerid) != 0) { std::lock_guard mutex_lock( window_uid_to_peerids_mutex_); - const std::string window_uid = peerid_to_window_uid_.at(peerid); + hangup_window_uid = peerid_to_window_uid_.at(peerid); peerid_to_window_uid_.erase(peerid); // After window_uid_to_peerids_[window_uid] becomes empty, we don't // remove the window_uid from the map here. We remove window_uid // from window_uid_to_peerids_ when the Window is closed. - window_uid_to_peerids_[window_uid].erase(peerid); + window_uid_to_peerids_[hangup_window_uid].erase(peerid); } if (pc_observer) { - rtc::scoped_refptr - peer_connection = pc_observer->GetPeerConnection(); - - rtc::scoped_refptr local_streams( - peer_connection->local_streams()); - for (unsigned int i = 0; i < local_streams->count(); i++) { - auto stream = local_streams->at(i); - - std::string window_uid = stream->id(); - bool still_used = this->WindowStillUsed(window_uid); - if (!still_used) { - std::lock_guard mlock( - window_uid_to_track_source_mutex_); - auto it = window_uid_to_track_source_.find(window_uid); - if (it != window_uid_to_track_source_.end()) { - window_uid_to_track_source_.erase(it); - } - utility::LogDebug("HangUp stream closed {}.", window_uid); + if (!hangup_window_uid.empty() && + !this->WindowStillUsed(hangup_window_uid)) { + std::lock_guard mlock( + window_uid_to_track_source_mutex_); + auto track_it = + window_uid_to_track_source_.find(hangup_window_uid); + if (track_it != window_uid_to_track_source_.end()) { + window_uid_to_track_source_.erase(track_it); } - - peer_connection->RemoveStream(stream); + utility::LogDebug("HangUp stream closed {}.", + hangup_window_uid); } delete pc_observer; @@ -540,9 +530,35 @@ bool PeerConnectionManager::InitializePeerConnection() { return (peer_connection_factory_.get() != nullptr); } +PeerConnectionManager::PeerConnectionObserver::PeerConnectionObserver( + PeerConnectionManager* peer_connection_manager, + const std::string& peerid) + : peer_connection_manager_(peer_connection_manager), + peerid_(peerid), + local_channel_(nullptr), + remote_channel_(nullptr), + ice_candidate_list_(Json::arrayValue), + deleting_(false) { + stats_callback_ = + new webrtc::RefCountedObject(); +} + +void PeerConnectionManager::PeerConnectionObserver::Initialize( + webrtc::scoped_refptr peer_connection) { + pc_ = peer_connection; + if (pc_.get()) { + auto channel_result = + pc_->CreateDataChannelOrError("ServerDataChannel", nullptr); + if (channel_result.ok()) { + local_channel_ = new DataChannelObserver( + peer_connection_manager_, channel_result.value(), peerid_); + } + } +} + // Create a new PeerConnection. -PeerConnectionManager::PeerConnectionObserver * -PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { +PeerConnectionManager::PeerConnectionObserver* +PeerConnectionManager::CreatePeerConnection(const std::string& peerid) { webrtc::PeerConnectionInterface::RTCConfiguration config; // Max bundle multiplexes all media and data channels on a single transport, // eliminating separate ICE/DTLS handshakes per track and reducing latency. @@ -569,24 +585,30 @@ PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { max_port = std::stoi(port); } } - std::unique_ptr port_allocator( - new cricket::BasicPortAllocator(new rtc::BasicNetworkManager())); - port_allocator->SetPortRange(min_port, max_port); + config.set_min_port(min_port); + config.set_max_port(max_port); utility::LogDebug("CreatePeerConnection webrtcPortRange: {}:{}.", min_port, max_port); utility::LogDebug("CreatePeerConnection peerid: {}.", peerid); - PeerConnectionObserver *obs = new PeerConnectionObserver( - this, peerid, config, std::move(port_allocator)); - if (!obs) { - utility::LogError("CreatePeerConnection failed."); - } else { - utility::LogDebug("CreatePeerConnection success!"); + + PeerConnectionObserver* obs = + new PeerConnectionObserver(this, peerid); + webrtc::PeerConnectionDependencies dependencies(obs); + auto pc_result = peer_connection_factory_->CreatePeerConnectionOrError( + config, std::move(dependencies)); + if (!pc_result.ok()) { + utility::LogError("CreatePeerConnection failed: {}.", + pc_result.error().message()); + delete obs; + return nullptr; } + obs->Initialize(pc_result.MoveValue()); + utility::LogDebug("CreatePeerConnection success!"); return obs; } // Get the capturer from its URL. -rtc::scoped_refptr +webrtc::scoped_refptr PeerConnectionManager::CreateVideoSource( const std::string &window_uid, const std::map &opts) { @@ -652,52 +674,42 @@ bool PeerConnectionManager::AddStreams( if (!existing_stream) { // Create a new stream and add to window_uid_to_track_source_. - rtc::scoped_refptr video_source( + webrtc::scoped_refptr video_source( this->CreateVideoSource(video, opts)); std::lock_guard mlock(window_uid_to_track_source_mutex_); window_uid_to_track_source_[window_uid] = video_source; } - // AddTrack and AddStream to peer_connection. + // Add local video track (Unified Plan). { std::lock_guard mlock(window_uid_to_track_source_mutex_); auto it = window_uid_to_track_source_.find(window_uid); if (it != window_uid_to_track_source_.end()) { - rtc::scoped_refptr stream = - peer_connection_factory_->CreateLocalMediaStream( - window_uid); - if (!stream.get()) { - utility::LogError("Cannot create stream."); + webrtc::scoped_refptr video_source = + it->second; + webrtc::scoped_refptr video_track; + if (!video_source) { + utility::LogError("Cannot create capturer video: {}.", + window_uid); } else { - rtc::scoped_refptr video_source = - it->second; - rtc::scoped_refptr video_track; - if (!video_source) { - utility::LogError("Cannot create capturer video: {}.", - window_uid); - } else { - rtc::scoped_refptr videoScaled = - VideoFilter::Create(video_source, - opts); - video_track = peer_connection_factory_->CreateVideoTrack( - window_uid + "_video", videoScaled); - // Prefer framerate over resolution when the encoder - // is under pressure (bandwidth or CPU constrained). - // For interactive 3D rendering, motion smoothness - // matters more than pixel-perfect resolution. - video_track->set_content_hint( - webrtc::VideoTrackInterface::ContentHint::kFluid); - } - - if ((video_track) && (!stream->AddTrack(video_track))) { - utility::LogError( - "Adding VideoTrack to MediaStream failed."); - } + webrtc::scoped_refptr videoScaled = + VideoFilter::Create(video_source, opts); + video_track = peer_connection_factory_->CreateVideoTrack( + videoScaled, window_uid + "_video"); + video_track->set_content_hint( + webrtc::VideoTrackInterface::ContentHint::kFluid); + } - if (!peer_connection->AddStream(stream)) { - utility::LogError("Adding stream to PeerConnection failed"); + if (video_track) { + webrtc::RTCErrorOr> + add_result = peer_connection->AddTrack( + video_track, {window_uid}); + if (!add_result.ok()) { + utility::LogError("Adding track to PeerConnection failed: {}", + add_result.error().message()); } else { - utility::LogDebug("Stream added to PeerConnection."); + utility::LogDebug("Track added to PeerConnection."); ret = true; } } @@ -725,7 +737,7 @@ void PeerConnectionManager::PeerConnectionObserver::OnIceCandidate( } } -rtc::scoped_refptr +webrtc::scoped_refptr PeerConnectionManager::GetVideoTrackSource(const std::string &window_uid) { { std::lock_guard mlock(window_uid_to_track_source_mutex_); @@ -763,8 +775,8 @@ void PeerConnectionManager::CloseWindowConnections( } // Encoder thread: wakes on each new frame, drains the per-window latest-frame -// map, and calls video_track_source->OnFrame() (libyuv + WebRTC encode) -// off the render thread so frame delivery never blocks GUI redraws. +// map, and posts OnFrame to the WebRTC worker thread. libyuv conversion and +// VideoBroadcaster must run on the worker thread (same as PCM creation). void PeerConnectionManager::EncoderThreadLoop() { while (encoder_running_) { std::unordered_map> snapshot; @@ -779,11 +791,23 @@ void PeerConnectionManager::EncoderThreadLoop() { // encode only the latest per window (implicit frame coalescing). snapshot = std::move(pending_frames_); } - for (const auto &kv : snapshot) { - auto video_track_source = GetVideoTrackSource(kv.first); - if (video_track_source && kv.second) { - video_track_source->OnFrame(kv.second); + webrtc::Thread* worker = webrtc_worker_thread_; + if (!worker) { + continue; + } + for (const auto& kv : snapshot) { + const std::shared_ptr& frame = kv.second; + if (!frame) { + continue; } + webrtc::scoped_refptr track_source = + GetVideoTrackSource(kv.first); + if (!track_source) { + continue; + } + worker->PostTask([track_source, frame]() { + track_source->OnFrame(frame); + }); } } } diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h index 72a005e96c4..2c85a41a40e 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h @@ -18,7 +18,11 @@ #pragma once +#include +#include #include +#include +#include #include #include @@ -34,6 +38,10 @@ #include "open3d/visualization/webrtc_server/HttpServerRequestHandler.h" #include "open3d/visualization/webrtc_server/WebRTCWindowSystem.h" +namespace webrtc { +class Thread; +} + namespace open3d { namespace visualization { namespace webrtc_server { @@ -73,23 +81,23 @@ namespace webrtc_server { /// /// TODO (yixing): Use PImpl. class PeerConnectionManager { - class VideoSink : public rtc::VideoSinkInterface { + class VideoSink : public webrtc::VideoSinkInterface { public: VideoSink(webrtc::VideoTrackInterface* track) : track_(track) { - track_->AddOrUpdateSink(this, rtc::VideoSinkWants()); + track_->AddOrUpdateSink(this, webrtc::VideoSinkWants()); } virtual ~VideoSink() { track_->RemoveSink(this); } // VideoSinkInterface implementation virtual void OnFrame(const webrtc::VideoFrame& video_frame) { - rtc::scoped_refptr buffer( + webrtc::scoped_refptr buffer( video_frame.video_frame_buffer()->ToI420()); utility::LogDebug("[{}] frame: {}x{}", OPEN3D_FUNCTION, buffer->height(), buffer->width()); } protected: - rtc::scoped_refptr track_; + webrtc::scoped_refptr track_; }; class SetSessionDescriptionObserver @@ -99,7 +107,7 @@ class PeerConnectionManager { webrtc::PeerConnectionInterface* pc, std::promise& promise) { - return new rtc::RefCountedObject( + return new webrtc::RefCountedObject( pc, promise); } virtual void OnSuccess() { @@ -136,7 +144,7 @@ class PeerConnectionManager { webrtc::PeerConnectionInterface* pc, std::promise& promise) { - return new rtc::RefCountedObject( + return new webrtc::RefCountedObject( pc, promise); } virtual void OnSuccess(webrtc::SessionDescriptionInterface* desc) { @@ -171,13 +179,12 @@ class PeerConnectionManager { protected: virtual void OnStatsDelivered( - const rtc::scoped_refptr& + const webrtc::scoped_refptr& report) { for (const webrtc::RTCStats& stats : *report) { Json::Value stats_members; - for (const webrtc::RTCStatsMemberInterface* member : - stats.Members()) { - stats_members[member->name()] = member->ValueToString(); + for (const webrtc::Attribute& attribute : stats.Attributes()) { + stats_members[attribute.name()] = attribute.ToString(); } report_[stats.id()] = stats_members; } @@ -190,7 +197,7 @@ class PeerConnectionManager { public: DataChannelObserver( PeerConnectionManager* peer_connection_manager, - rtc::scoped_refptr data_channel, + webrtc::scoped_refptr data_channel, const std::string& peerid) : peer_connection_manager_(peer_connection_manager), data_channel_(data_channel), @@ -251,38 +258,18 @@ class PeerConnectionManager { protected: PeerConnectionManager* peer_connection_manager_; - rtc::scoped_refptr data_channel_; + webrtc::scoped_refptr data_channel_; const std::string peerid_; }; class PeerConnectionObserver : public webrtc::PeerConnectionObserver { public: - PeerConnectionObserver( - PeerConnectionManager* peer_connection_manager, - const std::string& peerid, - const webrtc::PeerConnectionInterface::RTCConfiguration& config, - std::unique_ptr port_allocator) - : peer_connection_manager_(peer_connection_manager), - peerid_(peerid), - local_channel_(nullptr), - remote_channel_(nullptr), - ice_candidate_list_(Json::arrayValue), - deleting_(false) { - pc_ = peer_connection_manager_->peer_connection_factory_ - ->CreatePeerConnection(config, - std::move(port_allocator), - nullptr, this); + PeerConnectionObserver(PeerConnectionManager* peer_connection_manager, + const std::string& peerid); - if (pc_.get()) { - rtc::scoped_refptr channel = - pc_->CreateDataChannel("ServerDataChannel", nullptr); - local_channel_ = new DataChannelObserver( - peer_connection_manager_, channel, peerid_); - } - - stats_callback_ = new rtc::RefCountedObject< - PeerConnectionStatsCollectorCallback>(); - }; + void Initialize( + webrtc::scoped_refptr + peer_connection); virtual ~PeerConnectionObserver() { delete local_channel_; @@ -298,7 +285,7 @@ class PeerConnectionManager { Json::Value GetStats() { stats_callback_->clearReport(); - pc_->GetStats(stats_callback_); + pc_->GetStats(stats_callback_.get()); int count = 10; while ((stats_callback_->getReport().empty()) && (--count > 0)) { std::this_thread::sleep_for(std::chrono::milliseconds(1000)); @@ -306,27 +293,27 @@ class PeerConnectionManager { return Json::Value(stats_callback_->getReport()); }; - rtc::scoped_refptr + webrtc::scoped_refptr GetPeerConnection() { return pc_; }; // PeerConnectionObserver interface virtual void OnAddStream( - rtc::scoped_refptr stream) { + webrtc::scoped_refptr stream) { utility::LogDebug("[{}] GetVideoTracks().size(): {}.", OPEN3D_FUNCTION, stream->GetVideoTracks().size()); webrtc::VideoTrackVector videoTracks = stream->GetVideoTracks(); if (videoTracks.size() > 0) { - video_sink_.reset(new VideoSink(videoTracks.at(0))); + video_sink_.reset(new VideoSink(videoTracks.at(0).get())); } } virtual void OnRemoveStream( - rtc::scoped_refptr stream) { + webrtc::scoped_refptr stream) { video_sink_.reset(); } virtual void OnDataChannel( - rtc::scoped_refptr channel) { + webrtc::scoped_refptr channel) { utility::LogDebug( "PeerConnectionObserver::OnDataChannel peerid: {}", peerid_); @@ -369,11 +356,11 @@ class PeerConnectionManager { private: PeerConnectionManager* peer_connection_manager_; const std::string peerid_; - rtc::scoped_refptr pc_; + webrtc::scoped_refptr pc_; DataChannelObserver* local_channel_; DataChannelObserver* remote_channel_; Json::Value ice_candidate_list_; - rtc::scoped_refptr + webrtc::scoped_refptr stats_callback_; std::unique_ptr video_sink_; bool deleting_; @@ -409,22 +396,22 @@ class PeerConnectionManager { const std::shared_ptr& im); protected: - rtc::scoped_refptr GetVideoTrackSource( + webrtc::scoped_refptr GetVideoTrackSource( const std::string& window_uid); PeerConnectionObserver* CreatePeerConnection(const std::string& peerid); bool AddStreams(webrtc::PeerConnectionInterface* peer_connection, const std::string& window_uid, const std::string& options); - rtc::scoped_refptr CreateVideoSource( + webrtc::scoped_refptr CreateVideoSource( const std::string& window_uid, const std::map& opts); bool WindowStillUsed(const std::string& window_uid); - rtc::scoped_refptr GetPeerConnection( + webrtc::scoped_refptr GetPeerConnection( const std::string& peerid); protected: - std::unique_ptr task_queue_factory_; - rtc::scoped_refptr + std::unique_ptr field_trials_; + webrtc::scoped_refptr peer_connection_factory_; // Each peer has exactly one connection. @@ -437,7 +424,7 @@ class PeerConnectionManager { // Each Window has exactly one TrackSource. std::unordered_map> + webrtc::scoped_refptr> window_uid_to_track_source_; std::mutex window_uid_to_track_source_mutex_; @@ -456,13 +443,16 @@ class PeerConnectionManager { // Async encoder thread: decouples the render thread from the blocking // libyuv + WebRTC encode path. OnFrame() posts the latest frame per window; - // the thread drains the map and calls video_track_source->OnFrame(). + // the thread drains the map and posts video_track_source->OnFrame() to the + // WebRTC worker thread. std::unordered_map> pending_frames_; std::mutex pending_frames_mutex_; std::condition_variable pending_frames_cv_; std::atomic encoder_running_{false}; std::thread encoder_thread_; + // WebRTC worker thread (PeerConnectionFactoryDependencies::worker_thread). + webrtc::Thread* webrtc_worker_thread_ = nullptr; void EncoderThreadLoop(); }; diff --git a/cpp/open3d/visualization/webrtc_server/VideoFilter.h b/cpp/open3d/visualization/webrtc_server/VideoFilter.h index 16588c96a2a..94a95c0099a 100644 --- a/cpp/open3d/visualization/webrtc_server/VideoFilter.h +++ b/cpp/open3d/visualization/webrtc_server/VideoFilter.h @@ -18,7 +18,8 @@ #pragma once -#include +#include +#include #include "open3d/visualization/webrtc_server/BitmapTrackSource.h" @@ -34,14 +35,15 @@ namespace webrtc_server { template class VideoFilter : public BitmapTrackSource { public: - static rtc::scoped_refptr Create( - rtc::scoped_refptr video_source, + static webrtc::scoped_refptr Create( + webrtc::scoped_refptr video_source, const std::map& opts) { std::unique_ptr source = absl::WrapUnique(new T(video_source, opts)); if (!source) { return nullptr; } - return new rtc::RefCountedObject(std::move(source)); + return webrtc::scoped_refptr( + new webrtc::RefCountedObject(std::move(source))); } protected: @@ -61,7 +63,7 @@ class VideoFilter : public BitmapTrackSource { } private: - rtc::VideoSourceInterface* source() override { + webrtc::VideoSourceInterface* source() override { return source_.get(); } std::unique_ptr source_; diff --git a/cpp/open3d/visualization/webrtc_server/VideoScaler.h b/cpp/open3d/visualization/webrtc_server/VideoScaler.h index cd57e3dc898..fcebdda0bbc 100644 --- a/cpp/open3d/visualization/webrtc_server/VideoScaler.h +++ b/cpp/open3d/visualization/webrtc_server/VideoScaler.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "open3d/visualization/webrtc_server/BitmapTrackSource.h" @@ -27,10 +28,10 @@ namespace open3d { namespace visualization { namespace webrtc_server { -class VideoScaler : public rtc::VideoSinkInterface, - public rtc::VideoSourceInterface { +class VideoScaler : public webrtc::VideoSinkInterface, + public webrtc::VideoSourceInterface { public: - VideoScaler(rtc::scoped_refptr video_source, + VideoScaler(webrtc::scoped_refptr video_source, const std::map &opts) : video_source_(video_source), width_(0), @@ -148,7 +149,7 @@ class VideoScaler : public rtc::VideoSinkInterface, } else if (width == 0) { width = (roi_width_ * height) / roi_height_; } - rtc::scoped_refptr scaled_buffer = + webrtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(width, height); if (roi_width_ != frame.width() || roi_height_ != frame.height()) { scaled_buffer->CropAndScaleFrom( @@ -158,22 +159,22 @@ class VideoScaler : public rtc::VideoSinkInterface, scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); } webrtc::VideoFrame scaledFrame = - webrtc::VideoFrame(scaled_buffer, frame.timestamp(), + webrtc::VideoFrame(scaled_buffer, frame.timestamp_us(), frame.render_time_ms(), rotation_); broadcaster_.OnFrame(scaledFrame); } } - void AddOrUpdateSink(rtc::VideoSinkInterface *sink, - const rtc::VideoSinkWants &wants) override { + void AddOrUpdateSink(webrtc::VideoSinkInterface *sink, + const webrtc::VideoSinkWants &wants) override { video_source_->AddOrUpdateSink(this, wants); broadcaster_.AddOrUpdateSink(sink, wants); } void RemoveSink( - rtc::VideoSinkInterface *sink) override { + webrtc::VideoSinkInterface *sink) override { video_source_->RemoveSink(this); broadcaster_.RemoveSink(sink); @@ -183,8 +184,8 @@ class VideoScaler : public rtc::VideoSinkInterface, int height() { return roi_height_; } private: - rtc::scoped_refptr video_source_; - rtc::VideoBroadcaster broadcaster_; + webrtc::scoped_refptr video_source_; + webrtc::VideoBroadcaster broadcaster_; int width_; int height_; diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index c703ac5f66a..e2d87b60169 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -7,13 +7,12 @@ #include "open3d/visualization/webrtc_server/WebRTCWindowSystem.h" -#include -#include -#include #include +#include #include #include +#include #include #include #include @@ -103,6 +102,8 @@ struct WebRTCWindowSystem::Impl { std::thread webrtc_thread_; bool sever_started_ = false; + // Set while the WebRTC std::thread is inside Run(); used for shutdown. + std::atomic webrtc_message_thread_{nullptr}; std::unordered_map> data_channel_message_callbacks_; @@ -197,8 +198,14 @@ WebRTCWindowSystem::WebRTCWindowSystem() } WebRTCWindowSystem::~WebRTCWindowSystem() { + if (impl_->sever_started_ && impl_->webrtc_thread_.joinable()) { + webrtc::Thread* message_thread = impl_->webrtc_message_thread_.load(); + if (message_thread) { + message_thread->Quit(); + } + impl_->webrtc_thread_.join(); + } impl_->peer_connection_manager_ = nullptr; - rtc::Thread::Current()->Quit(); } WebRTCWindowSystem::OSWindow WebRTCWindowSystem::CreateOSWindow( @@ -262,16 +269,25 @@ void WebRTCWindowSystem::StartWebRTCServer() { gui::Application::GetInstance().GetResourcePath()); impl_->web_root_ = resource_path + "/html"; - // Logging settings. - // src/rtc_base/logging.h: LS_VERBOSE, LS_ERROR - rtc::LogMessage::LogToDebug((rtc::LoggingSeverity)rtc::LS_ERROR); - - rtc::LogMessage::LogTimestamps(); - rtc::LogMessage::LogThreads(); + // Logging settings (M149: rtc_base/logging.h). + webrtc::LoggingConfig log_config; + log_config.set_debug_severity(webrtc::LS_ERROR); + log_config.set_log_thread(true); + log_config.set_log_timestamp(true); + webrtc::InitializeLogging(std::move(log_config)); + + // Associate this std::thread with WebRTC's message loop (required + // before Thread::Current()->Run() and PeerConnectionFactory). + webrtc::ThreadManager::Instance()->WrapCurrentThread(); + struct WebRtcThreadScope { + ~WebRtcThreadScope() { + webrtc::ThreadManager::Instance()->UnwrapCurrentThread(); + } + } webrtc_thread_scope; + webrtc::Thread* thread = webrtc::Thread::Current(); + impl_->webrtc_message_thread_.store(thread); - // PeerConnectionManager manages all WebRTC connections. - rtc::Thread *thread = rtc::Thread::Current(); - rtc::InitializeSSL(); + webrtc::InitializeSSL(); Json::Value config; std::list ice_servers; ice_servers.insert(ice_servers.end(), s_public_ice_servers.begin(), @@ -345,7 +361,8 @@ void WebRTCWindowSystem::StartWebRTCServer() { utility::LogInfo("WebRTC Jupyter handshake mode enabled."); thread->Run(); } - rtc::CleanupSSL(); + impl_->webrtc_message_thread_.store(nullptr); + webrtc::CleanupSSL(); }; impl_->webrtc_thread_ = std::thread(start_webrtc_thread); impl_->sever_started_ = true; diff --git a/cpp/open3d/visualization/webrtc_server/html/index.html b/cpp/open3d/visualization/webrtc_server/html/index.html index 995ce95dea5..b318932f9f8 100644 --- a/cpp/open3d/visualization/webrtc_server/html/index.html +++ b/cpp/open3d/visualization/webrtc_server/html/index.html @@ -1,4 +1,5 @@ - + + Open3D WebVisualizer @@ -136,6 +137,7 @@ videoElt.muted = true; videoElt.controls = false; videoElt.playsinline = true; + videoElt.preload = "none"; videoElt.innerText = "Your browser does not support HTML5 video."; divElt.appendChild(videoElt); diff --git a/cpp/open3d/visualization/webrtc_server/html/open3d_logo.ico b/cpp/open3d/visualization/webrtc_server/html/open3d_logo.ico deleted file mode 100644 index ce428261c26acc89e930a65e01f8370b3fee5344..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34494 zcmd6Q2bdJa)^^YAE-WmYW|mFBGUNY5{B&grU{!xaOH zpnw6CpnwR7C}sr}t_cHB-}_c|byxRv&kV@_KewK(t*(5|sZ;6HsT#|QSUFZvkp(wl zos?}^hggO96bJG%PRgDdDbGFb`9)u*x?bYLRDv%Q@_q|D=S)3hPbyaD<sptZWibEQsBe2kd0TPI&3B78-&`o>&wpDy z@x*iD!3Q1{!-tO%EF1a%f;}l>Rm%Libz4}ubt~GVo==cpoPYjhV#kgh!Zf$vcApq~ z_z9vJ;9(l%UkMnRMXf5UL;Jotb#lsX1>T~%dXV`1^DoV;;*(E46>VDg6ipg;5QiUj zyvWHZMScm#l@Y5jT3C?SKN>0d0WgS%{{6ToFX1@v+)HJhtatAb2aBFP_JqHPFnkR= z2>ITnzk9bm#DWFyh-J%Gh-FKci}&AODgN`Hmqnw>_ICc5C@M^d@#9Vv4H`5-SPV2R znFG9EV*YMjYs6b`EfOnMtP;zYe<)V2{7Ag?(rnSBNhc?Nthl)tKOXt>nn)aS>sBrR zE`3qf7j^5Di~H|;!~;J)7hQCftc&m;F!Bh|qx%r}i$qqm;djMF&Gw2|`4QlO#w-$@ zI`$Wze)_qmj13#U5FI-770sJfi=&P_N#tZVB<`Y4PWiOny@q5(Eb?8fDZ*B;j&r75 zEVggo?kVH``)7y~jz8ULKj6Kle!WJm^6IydJP@(U$-5Jf|F6>0*5cZ0Zx(asE)cV4 zzajqp;A7&RyB`uGMv#AG+iwFuPO_}1!+%*;55iE4GG3GXTdz`_0RC;#n6#607Lf?s z95xbv131JxHv7WKj}OqGj4+G<9hUa#Gwj1LV~$yc?~f3+25qz!Fs}oTuY$X8)aG60 zGb}vpn}wxg$DUBpy?cK!cwr;tX@a(?0=|j-22HB+>R09%c_ImT63A5pbZBekNPxfq z!#JIec@jv&aRx(`h2hBJ_$3X-tQ6x6YtUAu-}(PKf2bR^nxT?7#;*98Xt#4OzUbX9e=bFL$i>t1h zE^>1#WScWQ!0tfUR2$vU@}pK`j%Ndrb|cFm4`pS=#MG%*i2e6J9R31XC;N=Day-%X zZ2%5kqE>>{1l3^y#s=W2XLQ+l^&6pob(Q1GBab{Ox>gSsEn9RKty=aFty}dJu~>5n z1Jm2~Uho>`3Tb?G6k0jirDKt9tK@gUJ_UX9>#x6+_Sav36W@QgQGE5)H{zRbz7rpQ zxLRC!#SNl9U?84$`CH(QV84WM%5T$Rf|cK(suHw$m+6Rmb(evHV?>y_r1n~I;)FBB zfuoL;>Fx4a&#QosQzVV7Lx|rv7hHh29fYNB-E#2&#*Q$vV)-gjh4Har!&c&|E2leS ztz8G_8Ss6_2&-jIo3516T^h$dO~k&p-bXSp46A|1Bn-e71y{c<#6FVdB(N z&auZ4ryW+Hjs)kjVSA0P3mV?dx;aiYENLm`&wD$tYz8u}B;Hb{C9d_rw-;Z0wdmHZ zM%HQLKssH{d8)ju?LpaDr9UayRh#<9SzXz0zx^)8j2Y)Chd6ZYI#|?Re2u7w@zH4O zuTaOpy!wroD%fL!^USTc-YaIkJV(6v;%wW#DqV(ge9p^jY?fj3O-!6|`gvlXeGc{1 z9gUW}3VwgiEdS0s9}us<{-&5SXTHZWjO&Mj{Hmn#QHD7CtP92PVF!82$GG|y%I*Q$ ze5&YEEKUa;+qP}fB=NX^{PBl4^2ifCWsp7?_d%blMK1V`>vN{33Dap~s0+PX&KUxCwiVu&*sEZPU8Ew5tT)o@d9Sjz1Y+ z2Wug=Lzr2+W*x?%ZlW>PPB%`!Q*doZd`K6x#kP96jZSC5FtYzvL}M2RZTjJ!h)uN-5EQvwkmnJL0+R=yF6$7)PAnNQFb-#=emxZ>=JQF z?KNVATqkpWA>U~0mW}jZfi^v&k*Xlrjk>ZytN6vL{30=Q=z-$$%dZ!8>Xg|$;*4Rk zJ(-Rcw7IMxzrrX-wXKF-yIgyNPNm3m6yQAJl1r|gFmmMKCt+@!h_F*p&S_X_ z6YKM{F|Pj$_6&Si=I3=w1G`fv)9tJYguSlVVCBjkSrPu{S_y2%Xltyfw6(Iz{B z#5kQ!ubMSIm>^umt=yuo09j4Q96t*I^oFmG7VZ*JXvdEIlaBibi5NXj5zs zV~t|57L~8hT~Ik_;9l?-lp|bQ>=W7U(l&%kTVmTH%vD(-;vPRgSB0VZ>-zY-ZE1-^6HG|dXK#PCamlS zaz5xiOuRB{u4sWVP>*{$K81^UF4llVhH>yq^9|aUa6P=uEf;+rdzMQtxmM1%fY0u? zp0U8wckR4f_Z^|vJ!$HVWJjBFz1R-@bsh5A<#Bz)wQWAe{&wPp7hVyA2JH)f1K2{J zm3v`799SR5DzmJf>1be;#9L%zoVeA*o49hkI&s2eaolmI$$j+Px$~t=g8MpcPdf1| zu}95F?&pm8VOU9hJ8>@#;w>z7C9x$Mg?zm{vH_U-zB zCP{So3@~&~!HZVIl2!%a^A|MkoI|+g4cf=+IsOqL_C5ZKhnqq{Uqk ztBCuA0Q{pC_N3#1`yVFmBljOGIESV)@4WM#;9i~U8t(7a{yd2WoTGTMsTK^LEmMy!Br=EOXv}w~T zh!;Nq3{4~0*J2-54>GR@T^O-W#5(Yf+wTvo&mYMB7-dZwc3lqJ;lvXs%dsk3%82w> z;`Gf+F*igY!yAis-=gy~ANLDewgj(p&HA~A?b$=guXMfIWxGS3B_4X{2{Clf1A}Pr z1=98}$Ztwo0Djf>9BYm}b`tvB3~|pr4~5$MAY0?O!?xD2yXC0zI5v)ejPPN=KpCB% z7g5(N@Z95WKJX6e=Q#G%Q_qXXAAdH~vTe_tJe6zul=Y9oKAZg!eLJas(C2N`T@CAu z^`t-P_2Z8}nWPT<-nj7xF>v5ML9|6!u^bO5?;SY6p9bSvo$`&4XY3ud+MpB0&jAg$ zD>}rae0ll}w+Ghg4`lh~T3(f*_S|QmeUX%9kfvrEv7Z+gH^1rPi>}JC(;6GxJC1BZQHdOBFBJ;)sQs6 z938`4t@w~-oP5$*;;+B{l67*<37dCbe|>?7$6CmG^qg_~?f1#~k>%(z>0({64RYtn ztQba)!P(Yue8ln*{~wi>^i%!kl~?8_;p`9i>#x7j*Uv;ACI1?Az>(syN1u`MZ9mVW z-4~$GmtHb;iBmt}h*+hh!5*-fdv*UQ5|E>apMKixuQJL1!3Q5o*?0-$EYCjsqS$Nb zXbF!li!N~g1>^i^tYbWN+jKzvMOg2~Z*%j=J+rtCvX69zZ3CI6ly^^`euoGe^DxHG zXwtYNq_NUC^PgQmXdw529Z^5z7McddqFvkGV$JH0r5v4dC|cI{_ut#(H~rhTZ5Nw1 z{U8?1e+T=~AZ9mZ{2=@(#*ac;CH`ybC$nV!*Nl;asb|54#-nT><`DiE(xnaQz7Kh}GP$ z!7hh?XjUXO&*?gC7mhUO4~0?God;a=AwMm^`C(zto&&IpE-XSg&Ua$5?5vVF*6bya z`ITZ^E5rVwysWf!1>`f0aMo7du%tOoM$)%=NE!vnKl>lr*H~vn2zhdfj#0xS99l>c zc>G)ro0~u)*z6j0z!j}kxFFPPYT<>A)F2&g0(odz1|<_LQpe4d=PAe`Tw;Esy9`bhDWENUw@k}0*j^G27Qy`D1;vc6*zA#|Miz& z%Lfl0hK)yl8Tt+_WE@gnW6Ese>{rSb(`7$nf1@q5Z3CPWn`60#MY|?E{0)+-N?mF&A(hQ6_G=iv)TD8E5t$h}H#^49OI zk8^%J-a_g^a1BKL5&!x+RsMiDw(}&exia9F6zx%FVQsqy)&ZM9576FTd)O{vL?5Yyo_Fcvlr8DY z^LCzF(Y9~bSDwdnU$k=NYB6ZwzF434apHA4<@OWU>&QKQG7fGXsP}s8Jup3|9CwHx zjcy11ZDDPD3}|nxRTZ5`cb;8RUy6H!?{Lm9EojekZtfwt&v(LgJY4DoY}~j>a9>s3 zWsqz;mp*1Y$nWnEHy~n_XqYsN+6@eERyE2hE^6Kc>qU&!njYF!YySQBJ3J#MF+B z{rPf#sbSk0m+~v<5D1+0i|enyRm_8a1LeakH>f^44DzU0MXq&cx;C)7Ru8HNnbw0U zAKOn`#W$3}uU-4e&VcNsWnZBTS&do6qB&$Z??aD|x@=w^a(UH;pXW8uX_6)d@6?5? zzx7Da{=arUgC^=6J`#{pfJS+nMtAZ7ebe5l`~&Y!%zL40_I^~oon ziT(CF%!8i`hlWSWz=rzzvemxb0IPGS0j)uM&UpsUFrQlAoCKNSpMNTuVa9;l38`z! z`6{@NDp}aASo?8IrJOA|or42p7qLRs1gahGyYJy_=)g?F8dJBEt2u!+ti==Ffcuqd!0y{0^{Irxfa6TPp?};OY1ud=WE`tX`7yA3;^h#zuZQ5;V zK@Z}2W}oy}j zV$ioqh?z5I?F#5J59P&L2Rj&kU7w;W>9sG;sHjh*^m8~b2l5c{-T>PN`V-VqaOj}F z2|wCJo@<{n@f>l=#3|BFgd1cT?+HJ73iPr2rR5u4A6?c-y`O(Wzn$yYAl*7MJ?5#I zSYJ|?(LSSb@nCyGA5NTi-eu4ynkFv4?0Rt-EZt0PzrMrM!0*Mm$ znYiw{TT-5Xa_tAbxZ7@=en%a3&L;JOxMxN^UL2y)xYTV48z0l*#XA<^a&b*l>i);Y zWtU!uea%X_t_vS;C10%T+h=%(p56DA>j=9IMu85LU3h54a#;W3#qXyBH86h1_8nsC z1y>k!@s_V>51mw;uW;O%09}}H*rmSYHs}?cgUZbH4;DkQdMN7=Km59U-s$-L_um6y zOczMFcxNOCF7`9-2cCT5KjP(=)5|V!PWSe}f%{g>pSxAk9{(gBb{q5o9^bfpr29d! zz&QZMh37q>lBZ0oBQ>vSm*K&(EhWDg3mp^QC*nHJ+gH5&4B6PPpgpgR(Rk>u0t$Io z;hL~E7=S$=hVO9#*B_27};cNrQ?c;Cg{HDu#k304>LEZLUFHtC;=_Jm)?Q)UbLk4D~K{AIE23k#cyna}Uqx{i7i93y#8kn&#M z*GT3Y7Z>bF{=z!oWDOf!lV1YA>sXTse|^CIm>+)nSs(QgcXeIO@${2*pNf0so>$AS zT$s%|!S}PV&WTlbK3LK=1pcHrTYtVE%KFwUPk(c);Cha-6!t$_$`QWDI76L-&p!KH z`gNS?GSAy@y(_S14WPX{zk&9fvCi4Y3YQ;h`X|A!0*W}EUZ-I;T{Z7C!aAgbv|OvU z!W}BE+1`D3spyFLigz+~yw~k0N*lI@42!S^)17sJM)zXOsT)Fb4ST3-k43Fe7SFp- zm*1*=AIA=y;)%zhw`Gj+I?YbHJFqvI+PPD=jNvrk_TU?}TCgt%p4}<>vHfx9 z4{Nk9zW7S6!_;`9!d}2>*3C`0ap+%sp6eDyDyJHJ=iCEd5Ey` z9^q%W&wB;-Y3X=cMhI(t_?<8rU^HOh=>r+I7N4D7}|F)cv>#Kl?E6pmV>v zbm<48X;o**@9AOGZKLsUUCg!5LXIdE!F(`z&P082NiZAn5)WH%rWas)2KHsxS)lzZ zeftidjd@_s*=Juk7k=f`v)-01Tk7@eH>f_w zAhc18wgD{klw$eNcPqe}t+1%DX))FbvGTGuag3`a*b|jvEmey9t7W*mTG6g;uS)2% zl_i>XO1s7ZP^s`B%sSo!e?Yj7-<__mdw05okLA(ma>Kqejo0b^nH!}6C{2af0%uVI z0PcYgxIDPDYLqg8I(+A?VwkimU2!~fS^!fV19_4&;>^INC$=Qq=2@~!jx zUFQA&ZG!6bDnptSyYold(tPmAkTS?S%l`cH z&$4~?IiM8tA@<#--dwx{ep;!c9GdXu^YtN^}xe*KExzuJlW91H=yAj;yRxzY z@#o$RXA2G-9vCT`;(I5QLE;`R?j@#tS0xSp!FAYmZ-P6NcVc!DH;s?RPrD2^@ixP8 z-?cOJ1{=0RogsT~1uJ=gcl;METq1bq&i|%G7~OU;d?xZ0RixKN^5Bw!USa7hyi1US zwx6zel6#a!vf zSL3DKF#L3Ukjp&OSv?wc{OQLEt=K}Lv;}#UAg7)b5O`4$XId>cNdFqz1aNLK?0s4~_js@N1 znb#wDZt9GgaY5f4GoOR5Nzb0nn}G)H zHU3HN7QnQ>tqbPhVB=!qCvmgGnU2OE7HLqwgN5~MH?%!@%c$S`&2f$TNU7d5+Szj@ zte@q5R^Dq)t#6Y=gV~?~*4An7YiOMPbaDL9G0VmM+xt{IYn-(!`AzaY`}1k1o#))Y z-C10N(my)mX>l{&vf}yR;fI}oF(bXch22JcGvMTGYp}*oyXnS82XDm-S=4pkA8o(c z#b4vDU3G3n{aK#n(3t1W{5Ji5Jk2=sn`dx*Ppet8YQb@X`XkgM@y?$~?QZZV-}sE< zj3nc?Tujet;C*4Ny(GV- zg2|5Ke0X)lD#%oB>cO8316})5(DvXZ-PYQ5X+fUkoonhq?=Hjfk@q23HtAbb*i7DI z<$EbRITn5m8ua5?jpFB|Il;&8OlC#SnY_?y-mF_|wEc$)cj9eY)$iFh)Z^kE+udch zjAzZf6Dy4$-ypwr%+V)jqwTL# zyk?f^{U-jr$N$G4e|pf@{aCx6NW&8pFZ+4YeP!qpV-0cBO?P1|{6*d!CrxZVPv=ti zYK$QxAR&VAw!=y+ytOb8^S!Mvq}tkBm(uUyn-A0t-d%>e7u3NZUA^_GF_yY4Yw)HX z-_EJD-v#!k3-R;p2y!YpA1Y1=purQ5F1AJ=cyvDA`FK*5qx-w&FW1FBNL+v2tq$rz zZRtcBA50gPmg^m*7nzJd`I&U#8--i9Zk6vm2kFgw;da^%XUwrPZN%BmG(kMTwjYc2 zs_#5h(?Rh+d5?GNgLJM8RGrT-e4PaN;QKWP;2jK?uf25=FXA7^o1a``Q9p=pKD+6? zeN)lF6Sf9n?Ts~&qJsT@@W2DC;lsu>!o3N|;gajt<#9Zwo(*-Ncc+=6{PcWnPqb6ZVJZdsdncEZ`Es8@lH~wu3u4gu#SE`xOt48+Quk zc0Fyc>9P?t*uyx3m-r7F zaIiIa(0*O8&i^E+PTdaVKfdX{bjb(1gnL*fj_d0FQ6S8|woqeXC+G;R$K4pdUs7(o zE1C>{QkjrnIQLBOq=B#=d-UnLiKf+eVXc!?m%?4OKlkx`1Ji6n^IPL*hVk3<>onS> zzE;N$Y42F+_Oo3D58txoTV{NpS+|kb4djL2kmjhU)lRm3-JDj|zybU2otM|-r|`a~ z<~z!9c2#Fw<7>KngZ6?8E)Su3QoDI@B&@s#F(2)}9(wVe&3ADFM?#n*!dy`ZnR3a(A(aF=fx z4B6uVsh>$5&3|h9_uD)0jkj=E9JoEO5?`J*aV_#1e(!*9*eRNZ-q^@N6n3E2E6;J&495PDY zAARQOnHkH8D5q>%)dk}~ECI{$z@{tdtZA*jSqIk~yra4u_b&K$rfawRzO~=)x_P+A z<9Pso8^qk_+3jM~uVJFAVRFOR&c(&e#EW=OlKQo{|6ucfa$2`+_U9P&#(x;L*Fgty z3`E>9u)M#BdXg>Eb8NZ$u7B^&;ZNRES9j4e-phI5BmR8jmHPp{vxT)=Af5a;;e49E zi*VPIPdt}vzukrN_V^2m#+r7j?Nwei|8b7tyWqRi;I5OD^8~f8A8yl6+*`F$_g^U= zorEs^`yL?I0DOl5WeR?@URi_^F_KkqZp z=7Clm2cU}rbZ^U&YMsr;k^O~n3Pzy#`Anp~e-KijAZcZ_&;mCmH=|2qFKcyEorc}6({ z$Gmsm=9y1B$q&J>cxi&Y{pT3-d*;>aByk}=A=V$S{nEAXh#iMI{|K&E(SbDJTL669 zCCr#}U0513d?9`Vi@%A~6Z=4T#H!FbeH2yL9E(-5H?`|_=2F`pvzn5mVwfU`JLEA4uzpqME4GzM~k7pSE zPCEOJ+CISF)&mKq19^b!!kcftOYluGz9}y)-V@~8Jv`T^;qR2*cH6yfyiS+o-Y;&v z@y;}3o)>?#{Z`x!9%opKh^SzirU-*4&P_*L!=J)qD4#Oy>aq`%QLC+YZ|0VqvQCdSm`bJ zjXm>xGhMrDT-|0)9tU+AyyL;&Y~$}$%6F5~;!iyJcGv7z-;gr8!|}F4_`20>ON~?J zF4}%A_IMp~>$D**nd0r>hCKS4`T4Dl4wtSP3u6;NgMj(aY%lVDv3#40@-m(s@vRxP zrq$^*&OvyZapt!_t&IB{vd^!}Z{|am<53rr7>&>?x zsLR){-w3zNUw}K#<@tF%9XK_-JK=`qO$w(D1#sUF_9rh56b@`-jKfks z$uaMWD{e4k(_#4gX=$blqK%XOXYBC~#vSVwX;r5}-2X3vZe1~cmnH^1&^X=* zE5YwXH-x@ZDb`M9_zkaeykA!dTiUThpG@^z{dlGG2hqk1%Ot;A3$jSHnq{#)MCWxMBWyrqyBEHN(uZ%x^QT8SekiG#&HF zw7E4!(Ym!;I!a$Ho(p&aqt@p9@e}2ck`s}$hgB2d+Xj}Ea33z<4V&bL7jkov1SZ!S zM<@GotA?(X_@b6h)*ch3iy=dr2nv>_mXj2X?bgV55|B?;gXauvw=tf2peeE4ybaQ8 zMQW6rW4rv_8yf9ud{tXO%1vljO5iN3 zCZWRBFN)!LhHGnOK_j`cJwj@r Date: Tue, 16 Jun 2026 11:45:28 -0700 Subject: [PATCH 08/37] Bug fixes, strip webrtc binaries, download webrtc source archive for faster download. General cleanup. --- .github/workflows/webrtc.yml | 44 +++------ ...uild-enable-rtc_use_cxx11_abi-option.patch | 11 +-- 3rdparty/webrtc/CMakeLists.txt | 71 ++++++++------ 3rdparty/webrtc/webrtc_build.sh | 94 +++++++++++++------ 3rdparty/webrtc/webrtc_common.cmake | 15 ++- 5 files changed, 132 insertions(+), 103 deletions(-) diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 481671a1e6c..2532b21077d 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -9,9 +9,9 @@ on: required: false default: 'e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e' depot_tools_commit: - description: 'depot_tools commit (empty = HEAD).' + description: 'depot_tools commit (override pin in webrtc_build.sh).' required: false - default: '' + default: '10eda50a3fd9c34ad8d31ec74e5f4eb5823d60f6' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,13 +20,14 @@ concurrency: env: WEBRTC_COMMIT: ${{ github.event.inputs.webrtc_commit }} DEPOT_TOOLS_COMMIT: ${{ github.event.inputs.depot_tools_commit }} - NPROC: 4 + WEBRTC_WORK_ROOT: ${{ github.workspace }}/.. + GCLIENT_JOBS: 8 jobs: Unix: permissions: - contents: write + contents: read runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -73,7 +74,7 @@ jobs: Windows: permissions: - contents: write + contents: read runs-on: windows-2022 strategy: fail-fast: false @@ -94,9 +95,9 @@ jobs: env: WORK_DIR: 'C:\WebRTC' OPEN3D_DIR: ${{ github.workspace }} - DEPOT_TOOLS_UPDATE: 1 - DEPOT_TOOLS_WIN_TOOLCHAIN: 0 - NPROC: 2 + WEBRTC_WORK_ROOT: 'C:\WebRTC' + DEPOT_TOOLS_UPDATE: 0 # belt-and-suspenders; also set by webrtc_setup_path + DEPOT_TOOLS_WIN_TOOLCHAIN: 0 # use locally installed VS, not the Chromium toolchain steps: - name: Checkout source code @@ -119,28 +120,13 @@ jobs: arch: x64 - name: Download WebRTC sources - shell: pwsh - working-directory: ${{ env.WORK_DIR }} + # shell: bash uses Git Bash on Windows, which transparently converts + # Windows-style env paths (e.g. OPEN3D_DIR, WEBRTC_WORK_ROOT) so they + # work in bash string and pushd contexts. + shell: bash run: | - $ErrorActionPreference = 'Stop' - if (-not (Test-Path depot_tools)) { - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - } - if ($env:DEPOT_TOOLS_COMMIT) { - git -C depot_tools checkout $env:DEPOT_TOOLS_COMMIT - } - $env:Path = (Get-Item depot_tools).FullName + ';' + $env:Path - if (-not (Test-Path webrtc/src)) { - New-Item -ItemType Directory -Force -Path webrtc | Out-Null - Push-Location webrtc - fetch webrtc - Pop-Location - } - git -C webrtc/src checkout $env:WEBRTC_COMMIT - git -C webrtc/src submodule update --init --recursive - Push-Location webrtc - gclient sync -D --force --reset - Pop-Location + source "$OPEN3D_DIR/3rdparty/webrtc/webrtc_build.sh" + download_webrtc_sources - name: Patch WebRTC shell: pwsh diff --git a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch index 1c87f7ffc0d..11e5cab1ff6 100644 --- a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch +++ b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch @@ -1,16 +1,7 @@ -From c47a1b6c0faa2206395647cb83cb1a0542101847 Mon Sep 17 00:00:00 2001 -From: Yixing Lao -Date: Wed, 7 Apr 2021 16:17:39 -0700 -Subject: [PATCH] build: enable rtc_use_cxx11_abi option - ---- - config/BUILDCONFIG.gn | 6 ++++++ - 1 file changed, 6 insertions(+) - diff --git a/config/BUILDCONFIG.gn b/config/BUILDCONFIG.gn --- a/config/BUILDCONFIG.gn +++ b/config/BUILDCONFIG.gn -@@ -171,6 +171,12 @@ declare_args() { +@@ -171,6 +171,11 @@ declare_args() { is_debug && current_os != "ios" && current_os != "watchos" } diff --git a/3rdparty/webrtc/CMakeLists.txt b/3rdparty/webrtc/CMakeLists.txt index 6876f6d340a..16bc558d0ae 100644 --- a/3rdparty/webrtc/CMakeLists.txt +++ b/3rdparty/webrtc/CMakeLists.txt @@ -1,25 +1,25 @@ -# This CMake file is intended to be used inside Dockerfile.webrtc. +# CMake driver for building the WebRTC prebuilt static libraries. +# Invoked by webrtc_build.sh (Unix CI) and webrtc.yml (Windows CI). +# Copied to /webrtc/ alongside webrtc_common.cmake before cmake is +# configured from /webrtc/build/. # -# 1) We assume the following directory structure: -# / -# ├── depot_tools # ${DEPOT_TOOLS_ROOT}, should be added to PATH -# └── webrtc # ${WEBRTC_ROOT} -#    ├── CMakeLists.txt # This CMakeLists.txt itself (copied to container) -#    ├── webrtc_common.cmake # Common configs for WebRTC (copied to container) -#    ├── .gclient -#    └── src # The actual git directory +# Expected directory layout: +# / +# ├── depot_tools/ # ${DEPOT_TOOLS_ROOT}, must be on PATH +# └── webrtc/ # ${WEBRTC_ROOT} = ${PROJECT_SOURCE_DIR} +# ├── CMakeLists.txt # this file +# ├── webrtc_common.cmake +# ├── .gclient +# └── src/ # WebRTC source tree # -# 2) CMake will compile two libraries libwebrtc.a and libwebrtc_extra.a. -# - libwebrtc.a compilation is driven by Ninja, the output will be in: -# ${WEBRTC_ROOT}/src/out/Release/obj/libwebrtc.a -# - libwebrtc_extra.a compilation is driven by Ninja but CMake packages the -# object files into a static library, the output will be in: -# ${WEBRTC_ROOT}/src/out/Release/obj/libwebrtc_extra.a +# Outputs (relative to CMAKE_INSTALL_PREFIX): +# lib/libwebrtc.a - main WebRTC static lib (built by gn/ninja) +# lib/libwebrtc_extra.a - supplementary objects packaged by CMake +# include/ - headers from webrtc/src/ # -# 3) Finally, `make install` will install headers and binaries to -# - build/lib -# - build/include - +# Build: +# cmake -G Ninja -DCMAKE_INSTALL_PREFIX= /webrtc +# ninja -j$(nproc) install cmake_minimum_required(VERSION 3.18) project(webrtc CXX) @@ -83,17 +83,9 @@ set_target_properties(webrtc_extra PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${WEBRTC_NINJA_ROOT}/obj ) -# Install headers and binaries -# /webrtc_install -# |-- include -# | |-- api -# | |-- audio -# ... -# | |-- tools_webrtc -# | `-- video -# `-- lib -# |-- libwebrtc.a -# `-- libwebrtc_extra.a +# Install headers and libs into CMAKE_INSTALL_PREFIX: +# include/ - all .h/.hpp/.inc headers mirroring webrtc/src/ structure +# lib/ - libwebrtc.a libwebrtc_extra.a (or .lib on Windows) file(GLOB_RECURSE WEBRTC_INCLUDES RELATIVE ${WEBRTC_ROOT}/src ${WEBRTC_ROOT}/src/*.h ${WEBRTC_ROOT}/src/*.hpp @@ -108,3 +100,22 @@ install(FILES ${WEBRTC_NINJA_ROOT}/obj/${CMAKE_STATIC_LIBRARY_PREFIX}webrtc_extra${CMAKE_STATIC_LIBRARY_SUFFIX} DESTINATION lib ) + +# Release prebuilts: strip debug sections from installed static libraries (Unix/macOS). +# MSVC .lib files are kept small via GN symbol_level=0; COFF static libs have no +# equivalent strip tool in the MSVC toolchain. +if(NOT WEBRTC_IS_DEBUG AND UNIX AND CMAKE_STRIP) + # macOS strip uses -S (debug symbols only); GNU strip uses --strip-debug. + if(APPLE) + set(_webrtc_strip_flags -S) + else() + set(_webrtc_strip_flags --strip-debug) + endif() + install(CODE " + file(GLOB _webrtc_libs \"\${CMAKE_INSTALL_PREFIX}/lib/*${CMAKE_STATIC_LIBRARY_SUFFIX}\") + foreach(_lib \${_webrtc_libs}) + execute_process(COMMAND \"${CMAKE_STRIP}\" ${_webrtc_strip_flags} \"\${_lib}\" + ERROR_QUIET) + endforeach() + ") +endif() diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 809cb804274..80f437038c0 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -1,37 +1,48 @@ #!/usr/bin/env bash -set -euox pipefail - -# Builds WebRTC static libraries for Open3D (Ubuntu/macOS). Windows: webrtc.yml +# Build WebRTC static libraries for Open3D (Ubuntu/macOS). +# Windows uses download_webrtc_sources() from this file via Git Bash; +# the cmake/ninja build itself is driven by the webrtc.yml PowerShell steps. +# +# This file is sourced (not executed) by CI steps so that functions are +# available as shell commands. Sourcing applies `set -euo pipefail` to the +# calling shell for strict error checking across the entire CI step. # -# Layout (default: repo parent holds depot_tools + webrtc): +# Expected directory layout ( = parent of the Open3D checkout, or +# $WEBRTC_WORK_ROOT if set): # / # ├── Open3D/ # this repository -# ├── depot_tools/ +# ├── depot_tools/ # fetched by clone_depot_tools() # └── webrtc/ -# └── src/ +# ├── .gclient # created by `fetch --nohooks --no-history webrtc` +# └── src/ # WebRTC source tree, pinned to WEBRTC_COMMIT # -# Usage: -# cd /path/to/Open3D -# export WEBRTC_COMMIT=... # optional +# Usage (Unix): # source 3rdparty/webrtc/webrtc_build.sh -# install_dependencies_ubuntu # optional on Ubuntu -# download_webrtc_sources -# build_webrtc +# install_dependencies_ubuntu # Ubuntu only +# download_webrtc_sources # fetches depot_tools + runs gclient sync +# build_webrtc # cmake/ninja build, installs, packages tar.gz + +set -euo pipefail # libwebrtc-bin M149 / Open3D target milestone WEBRTC_COMMIT=${WEBRTC_COMMIT:-e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e} -# Optional pin; unset uses depot_tools HEAD. -DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-} +# Pinned depot_tools (update intentionally when refreshing the WebRTC toolchain). +DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-10eda50a3fd9c34ad8d31ec74e5f4eb5823d60f6} +DEPOT_TOOLS_URL="https://chromium.googlesource.com/chromium/tools/depot_tools" GLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI:-1} -NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN)} +NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)} SUDO=${SUDO:-sudo} +# Parallel gclient git operations (speeds DEPS fetch on CI). +GCLIENT_JOBS=${GCLIENT_JOBS:-${NPROC}} + +_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" webrtc_work_root() { if [[ -n "${WEBRTC_WORK_ROOT:-}" ]]; then echo "$WEBRTC_WORK_ROOT" else - dirname "$PWD" + dirname "$_OPEN3D_ROOT" fi } @@ -40,6 +51,35 @@ webrtc_setup_path() { export DEPOT_TOOLS_UPDATE=0 } +# Fetch a pinned depot_tools tree via Gitiles tarball. +clone_depot_tools() { + local root="$1" + local dest="$root/depot_tools" + local commit="$DEPOT_TOOLS_COMMIT" + local stamp="$dest/.open3d_pinned_commit" + + if [[ -f "$stamp" && "$(cat "$stamp")" == "$commit" && -x "$dest/fetch" ]]; then + return 0 + fi + + local tmp archive + tmp="$(mktemp -d)" + archive="$tmp/depot_tools.tar.gz" + curl -fL --retry 3 --retry-delay 5 \ + -o "$archive" "${DEPOT_TOOLS_URL}/+archive/${commit}.tar.gz" + rm -rf "$dest" + mkdir -p "$dest" + # Gitiles +archive tarballs unpack flat (fetch at archive root, not in a subdir). + tar -xzf "$archive" -C "$dest" + rm -rf "$tmp" + + if [[ ! -x "$dest/fetch" ]]; then + echo "ERROR: depot_tools archive at ${commit} is missing fetch" >&2 + exit 1 + fi + echo "$commit" > "$stamp" +} + install_dependencies_ubuntu() { options="$(echo "$@" | tr ' ' '|')" $SUDO apt-get update @@ -79,34 +119,31 @@ install_dependencies_ubuntu() { download_webrtc_sources() { local root root="$(webrtc_work_root)" + pushd "$root" - if [[ ! -d depot_tools ]]; then - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - fi - if [[ -n "$DEPOT_TOOLS_COMMIT" ]]; then - git -C depot_tools checkout "$DEPOT_TOOLS_COMMIT" - fi + clone_depot_tools "$root" webrtc_setup_path + # Verify fetch is on PATH (exits non-zero under set -e if not found). command -V fetch if [[ ! -d webrtc/src ]]; then mkdir -p webrtc pushd webrtc - fetch --nohooks webrtc + fetch --nohooks --no-history webrtc popd fi - git -C webrtc/src checkout "$WEBRTC_COMMIT" - git -C webrtc/src submodule update --init --recursive pushd webrtc - gclient sync -D --force --reset --no-history + gclient sync -D --force --reset --no-history \ + --jobs="${GCLIENT_JOBS}" \ + --revision "src@${WEBRTC_COMMIT}" popd popd } build_webrtc() { local root open3d_dir - open3d_dir="$PWD" + open3d_dir="$_OPEN3D_ROOT" root="$(webrtc_work_root)" webrtc_setup_path @@ -122,8 +159,7 @@ build_webrtc() { -DCMAKE_INSTALL_PREFIX="$root/webrtc_release" \ -DGLIBCXX_USE_CXX11_ABI="${GLIBCXX_USE_CXX11_ABI}" \ .. - ninja -j"${NPROC}" - ninja install + ninja -j"${NPROC}" install popd pushd "$root" diff --git a/3rdparty/webrtc/webrtc_common.cmake b/3rdparty/webrtc/webrtc_common.cmake index 70708eb27b7..712595aaf30 100644 --- a/3rdparty/webrtc/webrtc_common.cmake +++ b/3rdparty/webrtc/webrtc_common.cmake @@ -1,10 +1,13 @@ -# Common configs for building WebRTC from source. Used in both native build -# and building inside docker. +# Common GN args and ninja target lists for building WebRTC from source. +# Included by CMakeLists.txt (which is driven by webrtc_build.sh on Unix CI and +# by webrtc.yml PowerShell steps on Windows CI). +# +# Callers must set WEBRTC_NINJA_ROOT before including this file. # # Exports: -# - get_webrtc_args(WEBRTC_ARGS) function -# - NINJA_TARGETS -# - EXTRA_WEBRTC_OBJS # You have to define WEBRTC_NINJA_ROOT before including this file +# get_webrtc_args(OUT_VAR) - function: returns a newline-separated args.gn string +# NINJA_TARGETS - list of gn targets to build +# EXTRA_WEBRTC_OBJS - object files not in libwebrtc.a, packed into libwebrtc_extra.a function(get_webrtc_args WEBRTC_ARGS) set(WEBRTC_ARGS "") @@ -38,6 +41,8 @@ function(get_webrtc_args WEBRTC_ARGS) endif() else() set(WEBRTC_ARGS is_debug=false\n${WEBRTC_ARGS}) + # Smaller static libs for prebuilt packages (no need for debug symbols). + set(WEBRTC_ARGS symbol_level=0\n${WEBRTC_ARGS}) endif() # H.264 (replaces deprecated is_chrome_branded on recent milestones). From ecc935b850ded29f2c5ca1514dc205aabfcbc03e Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 16 Jun 2026 13:26:46 -0700 Subject: [PATCH 09/37] fixes. --- .../webrtc/0002-build-enable_safe_libstdcxx.patch | 11 ----------- ...ch => 0002-src-fix-nullptr_t-with-libstdcxx.patch} | 0 ...003-src-gcc-suppress-port-interface-network.patch} | 3 ++- ...0004-call-payload_type_picker-gcc-flat_tree.patch} | 11 ++++++++++- ...mic-crt.patch => 0005-build-win-dynamic-crt.patch} | 0 3rdparty/webrtc/apply_webrtc_patches.sh | 9 ++++----- 3rdparty/webrtc/webrtc_build.sh | 7 ++++++- 7 files changed, 22 insertions(+), 19 deletions(-) delete mode 100644 3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch rename 3rdparty/webrtc/{0003-src-fix-nullptr_t-with-libstdcxx.patch => 0002-src-fix-nullptr_t-with-libstdcxx.patch} (100%) rename 3rdparty/webrtc/{0004-src-gcc-suppress-port-interface-network.patch => 0003-src-gcc-suppress-port-interface-network.patch} (85%) rename 3rdparty/webrtc/{0005-call-payload_type_picker-gcc-flat_tree.patch => 0004-call-payload_type_picker-gcc-flat_tree.patch} (52%) rename 3rdparty/webrtc/{0006-build-win-dynamic-crt.patch => 0005-build-win-dynamic-crt.patch} (100%) diff --git a/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch b/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch deleted file mode 100644 index 29df04e0a43..00000000000 --- a/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff --git a/build.gni b/build.gni ---- a/build.gni -+++ b/build.gni -@@ -10,6 +10,7 @@ enable_java_templates = true - - # Enables assertions on safety checks in libc++. - enable_safe_libcxx = true -+enable_safe_libstdcxx = true - - # Don't set this variable to true when building standalone WebRTC, it is - # only needed to support both WebRTC standalone and Chromium builds. diff --git a/3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch b/3rdparty/webrtc/0002-src-fix-nullptr_t-with-libstdcxx.patch similarity index 100% rename from 3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch rename to 3rdparty/webrtc/0002-src-fix-nullptr_t-with-libstdcxx.patch diff --git a/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch b/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch similarity index 85% rename from 3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch rename to 3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch index ed0a638cd7f..5db2a598bb9 100644 --- a/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch +++ b/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch @@ -1,7 +1,7 @@ diff --git a/p2p/base/port_interface.h b/p2p/base/port_interface.h --- a/p2p/base/port_interface.h +++ b/p2p/base/port_interface.h -@@ -52,7 +52,11 @@ class PortInterface { +@@ -52,7 +52,14 @@ class PortInterface { virtual ~PortInterface(); virtual IceCandidateType Type() const = 0; @@ -15,3 +15,4 @@ diff --git a/p2p/base/port_interface.h b/p2p/base/port_interface.h +#endif // Methods to set/get ICE role and tiebreaker values. + virtual void SetIceRole(IceRole role) = 0; diff --git a/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch b/3rdparty/webrtc/0004-call-payload_type_picker-gcc-flat_tree.patch similarity index 52% rename from 3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch rename to 3rdparty/webrtc/0004-call-payload_type_picker-gcc-flat_tree.patch index 8dc679d130f..584f3e81914 100644 --- a/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch +++ b/3rdparty/webrtc/0004-call-payload_type_picker-gcc-flat_tree.patch @@ -1,7 +1,16 @@ diff --git a/call/payload_type_picker.cc b/call/payload_type_picker.cc --- a/call/payload_type_picker.cc +++ b/call/payload_type_picker.cc -@@ -354,7 +354,7 @@ RTCErrorOr RtpHeaderExtensionRecorder::LookupId(absl::string_view uri, +@@ -338,7 +338,7 @@ + RTCError RtpHeaderExtensionRecorder::AddMapping(int id, + absl::string_view uri, + bool encrypt) { +- auto it = uri_to_id_.find(std::pair{uri, encrypt}); ++ auto it = uri_to_id_.find(std::pair{std::string(uri), encrypt}); + if (it != uri_to_id_.end()) { + if (it->second != id) { + // TODO: https://issues.webrtc.org/41480892 - This will return an error in +@@ -354,7 +354,7 @@ RTCErrorOr RtpHeaderExtensionRecorder::LookupId(absl::string_view uri, bool encrypt) const { diff --git a/3rdparty/webrtc/0006-build-win-dynamic-crt.patch b/3rdparty/webrtc/0005-build-win-dynamic-crt.patch similarity index 100% rename from 3rdparty/webrtc/0006-build-win-dynamic-crt.patch rename to 3rdparty/webrtc/0005-build-win-dynamic-crt.patch diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index 94e89a911a5..fcab2ca64b0 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -41,8 +41,7 @@ PATCH_DIR="$OPEN3D_DIR/3rdparty/webrtc" apply_one "$PATCH_DIR/0001-src-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0001-build-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/build" apply_one "$PATCH_DIR/0001-third_party-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/third_party" -apply_one "$PATCH_DIR/0002-build-enable_safe_libstdcxx.patch" "$WEBRTC_SRC/build_overrides" -apply_one "$PATCH_DIR/0003-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" -apply_one "$PATCH_DIR/0004-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" -apply_one "$PATCH_DIR/0005-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" -apply_one "$PATCH_DIR/0006-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" +apply_one "$PATCH_DIR/0002-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0003-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0004-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0005-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 80f437038c0..7e12bda80bd 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -70,7 +70,12 @@ clone_depot_tools() { rm -rf "$dest" mkdir -p "$dest" # Gitiles +archive tarballs unpack flat (fetch at archive root, not in a subdir). - tar -xzf "$archive" -C "$dest" + # On Windows (Git Bash), symlinks in the archive fail to extract because + # symlink creation requires elevated privileges. Those symlinks are + # Linux-only helper scripts (cbuildbot, luci-auth-fido2-plugin, etc.) and + # are not needed for WebRTC builds. The 'fetch' check below validates the + # critical tools were extracted. + tar -xzf "$archive" -C "$dest" || true rm -rf "$tmp" if [[ ! -x "$dest/fetch" ]]; then From 96b874808de3a9e75e4a248391c2423003f9b872 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 16 Jun 2026 22:20:01 -0700 Subject: [PATCH 10/37] fix2 --- 3rdparty/webrtc/webrtc_build.sh | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 7e12bda80bd..0bd8ab50317 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -47,7 +47,18 @@ webrtc_work_root() { } webrtc_setup_path() { - export PATH="$(webrtc_work_root)/depot_tools:${PATH}" + local root + root="$(webrtc_work_root)" + # On Windows the work root is a native path like 'C:\WebRTC'. The colon in + # the drive letter is bash's PATH separator, which would split + # 'C:\WebRTC/depot_tools' into the two entries 'C' and '\WebRTC/depot_tools'. + # '\WebRTC/...' then causes sha256sum to prefix its output with '\' (GNU + # coreutils escapes paths that contain backslashes), making the CIPD hash + # check fail. cygpath is available in Git Bash / MSYS2; convert to POSIX. + if command -v cygpath >/dev/null 2>&1; then + root="$(cygpath -u "$root")" + fi + export PATH="${root}/depot_tools:${PATH}" export DEPOT_TOOLS_UPDATE=0 } @@ -82,6 +93,16 @@ clone_depot_tools() { echo "ERROR: depot_tools archive at ${commit} is missing fetch" >&2 exit 1 fi + + # Bootstrap the depot_tools Python runtime. This creates + # python3_bin_reldir.txt which is required by the gn and ninja wrappers. + # On Unix, ensure_bootstrap calls bootstrap_python3 which downloads a + # hermetic Python 3 via CIPD and writes python3_bin_reldir.txt. + # On Windows (Git Bash/MINGW), win_tools.bat handles the Python bootstrap + # when gn.bat / ninja.bat are first invoked, so ensure_bootstrap skips it. + # DEPOT_TOOLS_DIR must be set so ensure_bootstrap resolves scripts correctly. + DEPOT_TOOLS_DIR="$dest" "$dest/ensure_bootstrap" + echo "$commit" > "$stamp" } From 4dcdc1dac05a2f2b51ba0e3842b740f4d6e02e98 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 17 Jun 2026 13:56:56 -0700 Subject: [PATCH 11/37] Fix WebRTC CI bootstrap and macOS compilation - Prepend depot_tools to PATH and run win_tools.bat on Windows - Disable protobuf constinit on Apple to fix Xcode 15.4 build --- ...-protobuf-disable-constinit-on-apple.patch | 26 ++++++++++++++++ 3rdparty/webrtc/apply_webrtc_patches.sh | 1 + 3rdparty/webrtc/webrtc_build.sh | 30 ++++++++++++++----- 3 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch diff --git a/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch b/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch new file mode 100644 index 00000000000..35fe389efbe --- /dev/null +++ b/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch @@ -0,0 +1,26 @@ +diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc +index 16423927bd33..edef8148783b 100644 +--- a/src/google/protobuf/port_def.inc ++++ b/src/google/protobuf/port_def.inc +@@ -469,7 +469,7 @@ + # define PROTOBUF_CONSTEXPR constexpr + # endif + #else +-# if defined(__cpp_constinit) && !defined(__CYGWIN__) ++# if defined(__cpp_constinit) && !defined(__CYGWIN__) && !defined(__APPLE__) + # define PROTOBUF_CONSTINIT constinit + # define PROTOBUF_CONSTEXPR constexpr + # define PROTOBUF_CONSTINIT_DEFAULT_INSTANCES +@@ -477,10 +477,9 @@ + // constant-initializing weak default instance pointers. Versions 12.0 and + // higher seem to work, except that XCode 12.5.1 shows the error even though it + // uses Clang 12.0.5. +-#elif !defined(__CYGWIN__) && !defined(__MINGW32__) && \ ++# elif !defined(__CYGWIN__) && !defined(__MINGW32__) && !defined(__APPLE__) && \ + ABSL_HAVE_CPP_ATTRIBUTE(clang::require_constant_initialization) && \ +- ((defined(__APPLE__) && PROTOBUF_CLANG_MIN(13, 0)) || \ +- (!defined(__APPLE__) && PROTOBUF_CLANG_MIN(12, 0))) ++ PROTOBUF_CLANG_MIN(12, 0) + # define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]] + # define PROTOBUF_CONSTEXPR constexpr + # define PROTOBUF_CONSTINIT_DEFAULT_INSTANCES diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index fcab2ca64b0..287877e882e 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -45,3 +45,4 @@ apply_one "$PATCH_DIR/0002-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0003-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0004-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0005-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" +apply_one "$PATCH_DIR/0006-third_party-protobuf-disable-constinit-on-apple.patch" "$WEBRTC_SRC/third_party/protobuf" diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 0bd8ab50317..37011ad0d14 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -94,14 +94,28 @@ clone_depot_tools() { exit 1 fi - # Bootstrap the depot_tools Python runtime. This creates - # python3_bin_reldir.txt which is required by the gn and ninja wrappers. - # On Unix, ensure_bootstrap calls bootstrap_python3 which downloads a - # hermetic Python 3 via CIPD and writes python3_bin_reldir.txt. - # On Windows (Git Bash/MINGW), win_tools.bat handles the Python bootstrap - # when gn.bat / ninja.bat are first invoked, so ensure_bootstrap skips it. - # DEPOT_TOOLS_DIR must be set so ensure_bootstrap resolves scripts correctly. - DEPOT_TOOLS_DIR="$dest" "$dest/ensure_bootstrap" + # Bootstrap depot_tools. + # We must temporarily prepend depot_tools to PATH so that python scripts + # and subprocesses launched during bootstrap (like gsutil.py calling luci-auth) + # can find the depot_tools executables on both Unix and Windows. + local old_path="$PATH" + export PATH="${dest}:${PATH}" + + if [[ "$(uname -s)" == *"MINGW"* || "$(uname -s)" == *"MSYS"* || "$(uname -s)" == *"CYGWIN"* ]]; then + # On Windows, bootstrap Python and Git via the batch files. + # This creates git.bat, python3.bat, and downloads cipd tools. + # We must run them using cmd.exe inside the depot_tools directory. + pushd "$dest" + cmd.exe //c "cipd_bin_setup.bat" + cmd.exe //c "bootstrap\\win_tools.bat" + popd + else + # On Unix (Ubuntu/macOS), ensure_bootstrap downloads Python 3 via CIPD and writes python3_bin_reldir.txt. + # DEPOT_TOOLS_DIR must be set so ensure_bootstrap resolves scripts correctly. + DEPOT_TOOLS_DIR="$dest" "$dest/ensure_bootstrap" + fi + + export PATH="$old_path" echo "$commit" > "$stamp" } From 185b7188b1c92d4bf21de821c0b5533763c75354 Mon Sep 17 00:00:00 2001 From: jGiltinan <57290997+jGiltinan@users.noreply.github.com> Date: Thu, 18 Jun 2026 17:37:03 -0400 Subject: [PATCH 12/37] Add CorrespondenceCheckerBasedOnRotation for orientation-constrained RANSAC (#7461) This PR adds a new RANSAC correspondence checker that rejects hypotheses whose estimated rotation exceeds per-axis tolerances, enabling orientation priors for structured scanning/registration scenarios. --------- Co-authored-by: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> --- CHANGELOG.md | 1 + .../registration/CorrespondenceChecker.cpp | 24 +++++++++ .../registration/CorrespondenceChecker.h | 46 ++++++++++++++++- .../pipelines/registration/registration.cpp | 51 ++++++++++++++++++- 4 files changed, 119 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9891c8d77f..11d5398c043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## Main +- Add `CorrespondenceCheckerBasedOnSourceRotation` to constrain global orientation priors in RANSAC registration (PR #7461) - Use glfwGetMonitorWorkarea for accurate screen size in GetScreenSize, remove unusable_height estimation hack, and subtract window-decoration extents before clamping auto-sized windows (PR #7469) - Upgrade stdgpu third-party library to commit d7c07d0. - Fix performance for non-contiguous NumPy array conversion in pybind vector converters. This change removes restrictive `py::array::c_style` flags and adds a runtime contiguity check, improving Pandas-to-Open3D conversion speed by up to ~50×. (issue #5250)(PR #7343). diff --git a/cpp/open3d/pipelines/registration/CorrespondenceChecker.cpp b/cpp/open3d/pipelines/registration/CorrespondenceChecker.cpp index 57885a0eb09..00d6ee45521 100644 --- a/cpp/open3d/pipelines/registration/CorrespondenceChecker.cpp +++ b/cpp/open3d/pipelines/registration/CorrespondenceChecker.cpp @@ -82,6 +82,30 @@ bool CorrespondenceCheckerBasedOnNormal::Check( return true; } +bool CorrespondenceCheckerBasedOnSourceRotation::Check( + const geometry::PointCloud &source, + const geometry::PointCloud &target, + const CorrespondenceSet &corres, + const Eigen::Matrix4d &transformation) const { + Eigen::Vector6d transform_6d = + open3d::utility::TransformMatrix4dToVector6d(transformation); + + if (rotation_threshold_[0] >= 0.0 && + std::abs(transform_6d[0]) > rotation_threshold_[0]) { + return false; + } + if (rotation_threshold_[1] >= 0.0 && + std::abs(transform_6d[1]) > rotation_threshold_[1]) { + return false; + } + if (rotation_threshold_[2] >= 0.0 && + std::abs(transform_6d[2]) > rotation_threshold_[2]) { + return false; + } + + return true; +} + } // namespace registration } // namespace pipelines } // namespace open3d diff --git a/cpp/open3d/pipelines/registration/CorrespondenceChecker.h b/cpp/open3d/pipelines/registration/CorrespondenceChecker.h index 0ae92c3d9b2..75a4253abb0 100644 --- a/cpp/open3d/pipelines/registration/CorrespondenceChecker.h +++ b/cpp/open3d/pipelines/registration/CorrespondenceChecker.h @@ -46,7 +46,7 @@ class CorrespondenceChecker { /// \param source Source point cloud. /// \param target Target point cloud. /// \param corres Correspondence set between source and target point cloud. - /// \param transformation The estimated transformation (inplace). + /// \param transformation The estimated transformation. virtual bool Check(const geometry::PointCloud &source, const geometry::PointCloud &target, const CorrespondenceSet &corres, @@ -143,6 +143,50 @@ class CorrespondenceCheckerBasedOnNormal : public CorrespondenceChecker { double normal_angle_threshold_; }; +/// \class CorrespondenceCheckerBasedOnSourceRotation +/// +/// \brief Class to limit the rotation of the source object. +/// +/// It checks if the transformation is rotated too much from its initial, +/// unrotated state (identity matrix). +/// Rotations are checked by comparing the components of the angle-axis +/// representation (SO(3) log vector) of the estimated transformation +/// to the given thresholds. It is assumed that the user is aware of the +/// x, y, z axes of the source object when setting these tolerances. +class CorrespondenceCheckerBasedOnSourceRotation + : public CorrespondenceChecker { +public: + /// \brief Parameterized Constructor. + /// + /// \param rotation_threshold specifies the threshold in radians within the + /// transformation. Rotations are checked by cartesian angles. If a rotation + /// threshold is set to < 0, it is not checked (free to rotate). + CorrespondenceCheckerBasedOnSourceRotation( + const Eigen::Vector3d &rotation_threshold = Eigen::Vector3d(-1, + -1, + -1)) + : CorrespondenceChecker(false), + rotation_threshold_(rotation_threshold) {} + ~CorrespondenceCheckerBasedOnSourceRotation() override {} + +public: + /// \brief Function to check if two points can be aligned. + /// + /// \param source Source point cloud. + /// \param target Target point cloud. + /// \param corres Correspondence set between source and target point cloud. + /// \param transformation The estimated transformation. + bool Check(const geometry::PointCloud &source, + const geometry::PointCloud &target, + const CorrespondenceSet &corres, + const Eigen::Matrix4d &transformation) const override; + +public: + /// \brief 3-element vector [rx, ry, rz] representing the rotation angle + /// thresholds in radians. A value < 0 means unconstrained. + Eigen::Vector3d rotation_threshold_; +}; + } // namespace registration } // namespace pipelines } // namespace open3d diff --git a/cpp/pybind/pipelines/registration/registration.cpp b/cpp/pybind/pipelines/registration/registration.cpp index 0cc560df72f..71bfd20f60d 100644 --- a/cpp/pybind/pipelines/registration/registration.cpp +++ b/cpp/pybind/pipelines/registration/registration.cpp @@ -151,6 +151,19 @@ void pybind_registration_declarations(py::module &m) { "normals. It considers vertex normal affinity of any " "correspondences. It computes dot product of two normal " "vectors. It takes radian value for the threshold."); + py::class_< + CorrespondenceCheckerBasedOnSourceRotation, + PyCorrespondenceChecker, + CorrespondenceChecker> + cc_r(m_registration, "CorrespondenceCheckerBasedOnSourceRotation", + "Class to limit the rotation of the source object.\n" + "It checks if the transformation is rotated too much from its " + "initial, unrotated state (identity matrix).\n" + "Rotations are checked by comparing the components of the " + "angle-axis representation (SO(3) log vector) of the " + "estimated transformation to the given thresholds. It is " + "assumed that the user is aware of the x, y, z axes of the " + "source object when setting these tolerances."); py::class_ fgr_option( m_registration, "FastGlobalRegistrationOption", "Options for FastGlobalRegistration."); @@ -415,7 +428,7 @@ Sets :math:`c = 1` if ``with_scaling`` is ``False``. {"target", "Target point cloud."}, {"corres", "Correspondence set between source and target point cloud."}, - {"transformation", "The estimated transformation (inplace)."}}); + {"transformation", "The estimated transformation."}}); // open3d.registration.CorrespondenceCheckerBasedOnEdgeLength: // CorrespondenceChecker @@ -504,6 +517,39 @@ must hold true for all edges.)"); normal_angle_threshold_, "Radian value for angle threshold."); + // open3d.registration.CorrespondenceCheckerBasedOnSourceRotation: + // CorrespondenceChecker + auto cc_r = static_cast, + CorrespondenceChecker>>( + m_registration.attr("CorrespondenceCheckerBasedOnSourceRotation")); + py::detail::bind_copy_functions( + cc_r); + cc_r.def(py::init([](const Eigen::Vector3d &rotation_threshold) { + return new CorrespondenceCheckerBasedOnSourceRotation( + rotation_threshold); + }), + "rotation_threshold"_a) + .def("__repr__", + [](const CorrespondenceCheckerBasedOnSourceRotation &c) { + return fmt::format( + "" + "CorrespondenceCheckerBasedOnSourceRotation with " + "rotation_threshold={:f}, {:f}, {:f} radians.", + c.rotation_threshold_[0], c.rotation_threshold_[1], + c.rotation_threshold_[2]); + }) + .def_readwrite( + "rotation_threshold", + &CorrespondenceCheckerBasedOnSourceRotation:: + rotation_threshold_, + "Float64 numpy array of shape (3,) representing " + "the maximum allowed thresholds [rx, ry, rz] " + "in radians for the angle-axis representation components. " + "It is assumed the user is aware of the x, y, z axes " + "of the source object. A value < 0 means unconstrained."); + // open3d.registration.FastGlobalRegistrationOption: auto fgr_option = static_cast>( m_registration.attr("FastGlobalRegistrationOption")); @@ -614,7 +660,8 @@ must hold true for all edges.)"); "clouds can be aligned. One of " "(``CorrespondenceCheckerBasedOnEdgeLength``, " "``CorrespondenceCheckerBasedOnDistance``, " - "``CorrespondenceCheckerBasedOnNormal``)"}, + "``CorrespondenceCheckerBasedOnNormal``, " + "``CorrespondenceCheckerBasedOnSourceRotation``)"}, {"confidence", "Desired probability of success for RANSAC. Used for " "estimating early termination by k = log(1 - " From fe3af4bb1f35bf70769311979550da167587dba2 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:17:09 -0700 Subject: [PATCH 13/37] Github attestation for Open3D binaries (#7512) * Add artifact attestation steps and permissions to release workflows --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Sameer Sheorey --- .github/workflows/documentation.yml | 8 +++++ .github/workflows/macos.yml | 23 +++++++++++++ .github/workflows/ubuntu-cuda.yml | 9 +++++ .github/workflows/ubuntu-openblas.yml | 8 +++++ .github/workflows/ubuntu-sycl.yml | 11 ++++++ .github/workflows/ubuntu-wheel.yml | 9 +++++ .github/workflows/ubuntu.yml | 20 +++++++++-- .github/workflows/windows.yml | 48 +++++++++++++++++++++------ README.md | 7 ++++ docs/getting_started.in.rst | 23 +++++++++++++ 10 files changed, 154 insertions(+), 12 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index e6ddcc54fef..8e642b5054f 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -22,6 +22,9 @@ jobs: headless-docs: # Build headless and docs permissions: contents: write # Artifact upload and release upload + id-token: write + attestations: write + artifact-metadata: write runs-on: ubuntu-latest env: DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} @@ -49,6 +52,11 @@ jobs: # Rename from Github PR branch SHA to original branch SHA, if needed. mv open3d-*-docs.tar.gz open3d-${GITHUB_SHA}-docs.tar.gz + - name: Generate docs attestation + uses: actions/attest@v4 + with: + subject-path: ${{ github.workspace }}/open3d-${{ github.sha }}-docs.tar.gz + - name: Upload docs uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 381f82772d9..495049bbb35 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -29,6 +29,9 @@ jobs: MacOS: permissions: contents: write # upload + id-token: write + attestations: write + artifact-metadata: write runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -86,6 +89,12 @@ jobs: zip -rv "open3d-${OPEN3D_VERSION_FULL}-app-macosx-10_15-${{ runner.arch }}.zip" Open3D.app ccache -s + - name: Generate package attestation + if: ${{ env.BUILD_SHARED_LIBS == 'ON' }} + uses: actions/attest@v4 + with: + subject-path: build/package/${{ env.DEVEL_PKG_NAME }} + - name: Upload package if: ${{ env.BUILD_SHARED_LIBS == 'ON' }} uses: actions/upload-artifact@v4 @@ -94,6 +103,12 @@ jobs: path: build/package/${{ env.DEVEL_PKG_NAME }} if-no-files-found: error + - name: Generate viewer attestation + if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }} + uses: actions/attest@v4 + with: + subject-path: build/bin/open3d-*-app-macosx-10_15-${{ runner.arch }}.zip + - name: Upload Open3D viewer app uses: actions/upload-artifact@v4 if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }} @@ -117,6 +132,9 @@ jobs: name: Build wheel permissions: contents: write # upload + id-token: write + attestations: write + artifact-metadata: write runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -199,6 +217,11 @@ jobs: PIP_PKG_NAME="$(basename build/lib/python_package/pip_package/open3d*.whl)" echo "PIP_PKG_NAME=$PIP_PKG_NAME" >> $GITHUB_ENV + - name: Generate wheel attestation + uses: actions/attest@v4 + with: + subject-path: build/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }} + - name: Upload wheel uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/ubuntu-cuda.yml b/.github/workflows/ubuntu-cuda.yml index cfd9d62945e..b69b747155f 100644 --- a/.github/workflows/ubuntu-cuda.yml +++ b/.github/workflows/ubuntu-cuda.yml @@ -47,6 +47,9 @@ jobs: name: Build and run permissions: contents: write # upload + id-token: write + attestations: write + artifact-metadata: write runs-on: ubuntu-latest needs: [skip-check] if: needs.skip-check.outputs.skip == 'no' @@ -151,6 +154,12 @@ jobs: "${INSTANCE_NAME}":open3d-devel-linux*.tar.xz "$PWD" fi + - name: Generate package attestation + if: ${{ env.BUILD_PACKAGE == 'true' }} + uses: actions/attest@v4 + with: + subject-path: ${{ github.workspace }}/open3d-devel-linux*.tar.xz + - name: Upload package if: ${{ env.BUILD_PACKAGE == 'true' }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/ubuntu-openblas.yml b/.github/workflows/ubuntu-openblas.yml index dc625dd4c9e..46ad4b5a1ff 100644 --- a/.github/workflows/ubuntu-openblas.yml +++ b/.github/workflows/ubuntu-openblas.yml @@ -37,6 +37,9 @@ jobs: openblas-arm64: permissions: contents: write # Release upload + id-token: write + attestations: write + artifact-metadata: write runs-on: ubuntu-24.04-arm # latest strategy: fail-fast: false @@ -83,6 +86,11 @@ jobs: - name: Docker test run: docker/docker_test.sh "${DOCKER_TAG}" + - name: Generate wheel attestation + uses: actions/attest@v4 + with: + subject-path: ${{ github.workspace }}/${{ env.PIP_PKG_NAME }} + - name: Upload wheel to GitHub artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/ubuntu-sycl.yml b/.github/workflows/ubuntu-sycl.yml index 3f9ca4edf71..e4903de1890 100644 --- a/.github/workflows/ubuntu-sycl.yml +++ b/.github/workflows/ubuntu-sycl.yml @@ -26,6 +26,9 @@ jobs: ubuntu-sycl: permissions: contents: write # Release upload + id-token: write + attestations: write + artifact-metadata: write runs-on: ubuntu-latest strategy: fail-fast: false @@ -54,6 +57,14 @@ jobs: docker/docker_test.sh sycl-static fi + - name: Generate artifact attestation + if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' }} + uses: actions/attest@v4 + with: + subject-path: | + ${{ github.workspace }}/open3d*.whl + ${{ github.workspace }}/open3d-devel-*.tar.xz + - name: Upload Python wheel and C++ binary package to GitHub artifacts if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index e8a340e5d8f..1b16e1ec168 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -26,6 +26,9 @@ jobs: build-wheel: permissions: contents: write # Release upload + id-token: write + attestations: write + artifact-metadata: write name: Build wheel runs-on: ubuntu-latest strategy: @@ -88,6 +91,12 @@ jobs: PIP_CPU_PKG_NAME="$(basename ${GITHUB_WORKSPACE}/open3d_cpu*.whl)" echo "PIP_PKG_NAME=$PIP_PKG_NAME" >> $GITHUB_ENV echo "PIP_CPU_PKG_NAME=$PIP_CPU_PKG_NAME" >> $GITHUB_ENV + - name: Generate artifact attestation + uses: actions/attest@v4 + with: + subject-path: | + ${{ github.workspace }}/${{ env.PIP_PKG_NAME }} + ${{ github.workspace }}/${{ env.PIP_CPU_PKG_NAME }} - name: Upload wheel to GitHub artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index e7e27444ca2..c942d18f3d5 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -22,6 +22,9 @@ jobs: ubuntu: permissions: contents: write # Release upload + id-token: write + attestations: write + artifact-metadata: write runs-on: ubuntu-latest strategy: fail-fast: false @@ -62,15 +65,28 @@ jobs: elif [ "${{ matrix.BUILD_SHARED_LIBS }}" = "ON" ] && [ "${{ env.MLOPS }}" = "ON" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then docker/docker_test.sh cpu-shared-ml-release fi + - name: Generate package attestation + if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' }} + uses: actions/attest@v4 + with: + subject-path: ${{ github.workspace }}/open3d-devel-*.tar.xz + - name: Upload package to GitHub artifacts - if: ${{ env.BUILD_SHARED_LIBS == 'ON' }} + if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' }} uses: actions/upload-artifact@v4 with: name: open3d-devel-linux-x86_64 path: open3d-devel-*.tar.xz if-no-files-found: error + + - name: Generate viewer attestation + if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' }} + uses: actions/attest@v4 + with: + subject-path: ${{ github.workspace }}/open3d-viewer-*-Linux.deb + - name: Upload viewer to GitHub artifacts - if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }} + if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' }} uses: actions/upload-artifact@v4 with: name: open3d-viewer-Linux diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d571f97e295..bd24116faa2 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -31,6 +31,7 @@ env: CUDA_VERSION: "12.6.0" SRC_DIR: "D:\\a\\open3d\\open3d" BUILD_DIR: "C:\\Open3D\\build" + INSTALL_DIR: "C:\\Program Files\\Open3D" NPROC: 6 DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} @@ -38,6 +39,9 @@ jobs: windows: permissions: contents: write # upload + id-token: write + attestations: write + artifact-metadata: write runs-on: windows-2022 strategy: fail-fast: false @@ -136,7 +140,7 @@ jobs: cmake -G "Visual Studio 17 2022" -A x64 ` -DDEVELOPER_BUILD=$Env:DEVELOPER_BUILD ` -DBUILD_EXAMPLES=OFF ` - -DCMAKE_INSTALL_PREFIX="C:\Program Files\Open3D" ` + -DCMAKE_INSTALL_PREFIX="$env:INSTALL_DIR" ` -DBUILD_SHARED_LIBS=${{ matrix.BUILD_SHARED_LIBS }} ` -DSTATIC_WINDOWS_RUNTIME=${{ matrix.STATIC_RUNTIME }} ` -DBUILD_COMMON_ISPC_ISAS=ON ` @@ -171,6 +175,12 @@ jobs: echo "DEVEL_PKG_NAME=$DEVEL_PKG_NAME" | Out-File -FilePath ` $Env:GITHUB_ENV -Encoding utf8 -Append + - name: Generate package attestation + if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' }} + uses: actions/attest@v4 + with: + subject-path: ${{ env.BUILD_DIR }}/package/${{ env.DEVEL_PKG_NAME }} + - name: Upload Package if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' }} uses: actions/upload-artifact@v4 @@ -197,13 +207,26 @@ jobs: --target Open3DViewer cmake --build . --parallel ${{ env.NPROC }} --config ${{ matrix.CONFIG }} ` --target INSTALL + $cmakeCachePath = Join-Path $env:BUILD_DIR "CMakeCache.txt" + $Env:OPEN3D_VERSION_FULL = (Select-String -Path $cmakeCachePath -Pattern "OPEN3D_VERSION_FULL").Line.Split('=')[1] + $open3dAppPath = Join-Path $env:INSTALL_DIR "bin\Open3D" + Compress-Archive -Path $open3dAppPath -DestinationPath ` + "$Env:GITHUB_WORKSPACE/open3d-$Env:OPEN3D_VERSION_FULL-app-windows-amd64.zip" + echo "VIEWER_ZIP_NAME=open3d-$Env:OPEN3D_VERSION_FULL-app-windows-amd64.zip" | Out-File -FilePath ` + $Env:GITHUB_ENV -Encoding utf8 -Append + + - name: Generate viewer attestation + if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' && matrix.CONFIG == 'Release' }} + uses: actions/attest@v4 + with: + subject-path: ${{ github.workspace }}/${{ env.VIEWER_ZIP_NAME }} - name: Upload Viewer if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' && matrix.CONFIG == 'Release' }} uses: actions/upload-artifact@v4 with: name: open3d-app-windows-amd64 - path: C:\Program Files\Open3D\bin\Open3D + path: ${{ github.workspace }}/${{ env.VIEWER_ZIP_NAME }} if-no-files-found: error - name: Update devel release with viewer @@ -211,10 +234,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - $Env:OPEN3D_VERSION_FULL = (Select-String -Path "C:/Open3D/build/CMakeCache.txt" -Pattern "OPEN3D_VERSION_FULL").Line.Split('=')[1] - Compress-Archive -Path "C:/Program Files/Open3D/bin/Open3D" -DestinationPath ` - "open3d-$Env:OPEN3D_VERSION_FULL-app-windows-amd64.zip" - bash .github/workflows/update_release.sh "open3d-$Env:OPEN3D_VERSION_FULL-app-windows-amd64.zip" + bash .github/workflows/update_release.sh "${{ env.VIEWER_ZIP_NAME }}" - name: Run C++ unit tests if: ${{ matrix.BUILD_CUDA_MODULE == 'OFF' }} @@ -229,16 +249,16 @@ jobs: mkdir build cd build cmake -G "Visual Studio 17 2022" -A x64 ` - -DCMAKE_INSTALL_PREFIX="C:\Program Files\Open3D" ` + -DCMAKE_INSTALL_PREFIX="$env:INSTALL_DIR" ` -DSTATIC_WINDOWS_RUNTIME=${{ matrix.STATIC_RUNTIME }} ` .. cmake --build . --config ${{ matrix.CONFIG }} if ( '${{ matrix.BUILD_CUDA_MODULE }}' -eq 'OFF' ) { # FIXME .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test } - Remove-Item "C:\Program Files\Open3D" -Recurse + Remove-Item -LiteralPath $env:INSTALL_DIR -Recurse -Force -ErrorAction SilentlyContinue - name: Install Open3D python build requirements - working-directory: ${{ env.SOURCE_DIR }} + working-directory: ${{ env.SRC_DIR }} run: | $ErrorActionPreference = 'Stop' python -m pip install -U pip==${{ env.PIP_VER }} @@ -261,6 +281,9 @@ jobs: name: Build wheel permissions: contents: write # upload + id-token: write + attestations: write + artifact-metadata: write runs-on: windows-2022 strategy: fail-fast: false @@ -331,7 +354,7 @@ jobs: $Env:DEVELOPER_BUILD = "ON" } cmake -G "Visual Studio 17 2022" -A x64 ` - -DCMAKE_INSTALL_PREFIX="C:\Program Files\Open3D" ` + -DCMAKE_INSTALL_PREFIX="$env:INSTALL_DIR" ` -DDEVELOPER_BUILD="$Env:DEVELOPER_BUILD" ` -DBUILD_SHARED_LIBS=OFF ` -DSTATIC_WINDOWS_RUNTIME=ON ` @@ -351,6 +374,11 @@ jobs: $PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name echo "PIP_PKG_NAME=$PIP_PKG_NAME" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + - name: Generate wheel attestation + uses: actions/attest@v4 + with: + subject-path: ${{ env.BUILD_DIR }}/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }} + - name: Upload wheel uses: actions/upload-artifact@v4 with: diff --git a/README.md b/README.md index ad9082e73f2..ecdce05a71e 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,13 @@ To get the latest features in Open3D, install the To compile Open3D from source, refer to [compiling from source](https://www.open3d.org/docs/release/compilation.html). +Release artifacts (Python wheels and C++ binaries) include signed +[SLSA](https://slsa.dev) build-provenance attestations (GitHub Artifact +Attestations), aligned with [OpenSSF](https://openssf.org) supply-chain +guidance. See +[Getting started — supply chain attestations](https://www.open3d.org/docs/latest/getting_started.html#supply-chain-attestations) +for verification with the GitHub CLI. + ## C++ quick start Checkout the following links to get started with Open3D C++ API diff --git a/docs/getting_started.in.rst b/docs/getting_started.in.rst index dab69dba54e..75d70782d09 100644 --- a/docs/getting_started.in.rst +++ b/docs/getting_started.in.rst @@ -20,6 +20,29 @@ version (``HEAD`` of ``main`` branch) viewer app is provided here [#]_: .. [#] Please use these links from the `latest version of this page `__ only. .. [#] To check the `glibc` version on your system, run :code:`ldd --version`. +.. _supply_chain_attestations: + +Supply chain attestations +========================= + +Development artifacts linked on this page—viewer packages, pip wheels, C++ +devel archives, and documentation tarballs—are produced by GitHub Actions with +signed `SLSA build provenance `__ attestations (`GitHub +Artifact Attestations +`__), +consistent with `OpenSSF `__ supply-chain recommendations. + +After saving a file locally, verify it with the `GitHub CLI +`__: + +.. code-block:: bash + + gh attestation verify /path/to/ -R isl-org/Open3D + +Replace ``/path/to/`` with the wheel, ``.deb``, ``.zip``, ``.tar.xz``, +or docs ``.tar.gz`` you downloaded. A successful check confirms the file matches +provenance recorded for this repository. + Python ====== From 5c61299d9c4d91141846c763143c44ff00f5cb5f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:21:15 +0000 Subject: [PATCH 14/37] Initial plan From c803646f76372fdc61fad9a512e6444ac2c99e6d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:49:57 +0000 Subject: [PATCH 15/37] Fix WebRTC workflow: Linux GCC, macOS protobuf constinit, Windows depot_tools bootstrap --- .github/workflows/webrtc.yml | 7 +++ ...-gcc-suppress-port-interface-network.patch | 18 ------ 3rdparty/webrtc/apply_webrtc_patches.sh | 55 ++++++++++++++++++- 3rdparty/webrtc/webrtc_build.sh | 39 ++++++++++++- 3rdparty/webrtc/webrtc_common.cmake | 4 +- 5 files changed, 102 insertions(+), 21 deletions(-) delete mode 100644 3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 2532b21077d..6eb3ab1ac70 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -147,6 +147,12 @@ jobs: WIN_TAG: ${{ matrix.tag }} run: | $ErrorActionPreference = 'Stop' + # Locate the VS-installed ninja.exe *before* prepending depot_tools to + # PATH. depot_tools contains a Unix wrapper script named 'ninja' (no + # extension) that Windows cannot execute; if depot_tools appears first + # in PATH cmake -G Ninja picks up that script and fails with + # "unknown error" when trying to run 'ninja --version'. + $ninjaExe = (Get-Command ninja.exe -ErrorAction Stop).Source $env:Path = (Get-Item depot_tools).FullName + ';' + $env:Path $installRoot = Join-Path $env:WORK_DIR "webrtc_pkg" if (Test-Path $installRoot) { Remove-Item -Recurse -Force $installRoot } @@ -154,6 +160,7 @@ jobs: Push-Location webrtc/build $debugFlag = if ($env:BUILD_CONFIG -eq 'Debug') { 'ON' } else { 'OFF' } cmake -G Ninja ` + -D CMAKE_MAKE_PROGRAM="$ninjaExe" ` -D CMAKE_BUILD_TYPE=$env:BUILD_CONFIG ` -D WEBRTC_IS_DEBUG=$debugFlag ` -D WEBRTC_STATIC_MSVC_RUNTIME=$env:STATIC_RT ` diff --git a/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch b/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch deleted file mode 100644 index 5db2a598bb9..00000000000 --- a/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/p2p/base/port_interface.h b/p2p/base/port_interface.h ---- a/p2p/base/port_interface.h -+++ b/p2p/base/port_interface.h -@@ -52,7 +52,14 @@ class PortInterface { - virtual ~PortInterface(); - - virtual IceCandidateType Type() const = 0; -+#if defined(__GNUC__) && !defined(__clang__) -+#pragma GCC diagnostic push -+#pragma GCC diagnostic ignored "-Wchanges-meaning" -+#endif - virtual const Network* Network() const = 0; -+#if defined(__GNUC__) && !defined(__clang__) -+#pragma GCC diagnostic pop -+#endif - - // Methods to set/get ICE role and tiebreaker values. - virtual void SetIceRole(IceRole role) = 0; diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index 287877e882e..a2f0d9ae7e7 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -36,13 +36,66 @@ apply_one() { fi } +# Fix port.cc for Apple Clang (Xcode 15.4): the GlobalEmptyString (std::string) +# variable is declared with PROTOBUF_CONSTINIT which expands to `constinit` or +# [[clang::require_constant_initialization]] on Apple Clang >= 13. Apple's +# libc++ std::string constructor performs a heap allocation, so the variable +# cannot be constant-initialized, producing a hard error. Guard the declaration +# with !defined(__APPLE__) so PROTOBUF_CONSTINIT is omitted on Apple. +# +# This supplements the port_def.inc patch in 0006 and directly targets the +# specific failing declaration (port.cc:120 in the WebRTC M149 protobuf). +fix_protobuf_port_cc_apple() { + local port_cc="${WEBRTC_SRC}/third_party/protobuf/src/google/protobuf/port.cc" + [[ -f "$port_cc" ]] || return 0 + python3 - "$port_cc" <<'PYEOF' +import sys + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +old = ('PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT\n' + ' PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 GlobalEmptyString\n' + ' fixed_address_empty_string{};') +# Check idempotency: already patched if __APPLE__ guard present near declaration. +if '__APPLE__' in content and 'fixed_address_empty_string' in content: + print(f'Skip {path} (Apple constinit fix already applied)') + sys.exit(0) +if old not in content: + print(f'WARNING: {path}: expected PROTOBUF_CONSTINIT pattern not found; ' + f'skipping Apple constinit fix', file=sys.stderr) + sys.exit(0) +new = ('#if defined(__APPLE__)\n' + '// Apple Clang (Xcode 15.4): GlobalEmptyString (std::string) requires heap\n' + '// allocation in its constructor, which is not a constant expression.\n' + '// Skip PROTOBUF_CONSTINIT to avoid "variable does not have a constant\n' + '// initializer" hard error.\n' + 'PROTOBUF_ATTRIBUTE_NO_DESTROY\n' + ' PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 GlobalEmptyString\n' + ' fixed_address_empty_string{};\n' + '#else\n' + 'PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT\n' + ' PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 GlobalEmptyString\n' + ' fixed_address_empty_string{};\n' + '#endif // !defined(__APPLE__)') +with open(path, 'w') as f: + f.write(content.replace(old, new, 1)) +print(f'Applied Apple constinit fix in {path}') +PYEOF +} + PATCH_DIR="$OPEN3D_DIR/3rdparty/webrtc" # Required: declare gn args consumed by args.gn and fix GCC compile errors. apply_one "$PATCH_DIR/0001-src-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0001-build-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/build" apply_one "$PATCH_DIR/0001-third_party-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/third_party" apply_one "$PATCH_DIR/0002-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" -apply_one "$PATCH_DIR/0003-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0004-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0005-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" +# 0006 patches port_def.inc to prevent PROTOBUF_CONSTINIT from expanding to +# constinit/[[clang::require_constant_initialization]] on Apple. The +# fix_protobuf_port_cc_apple call below directly patches port.cc as an +# additional safety measure in case the port_def.inc path alone is insufficient. apply_one "$PATCH_DIR/0006-third_party-protobuf-disable-constinit-on-apple.patch" "$WEBRTC_SRC/third_party/protobuf" +fix_protobuf_port_cc_apple diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 37011ad0d14..b2e0ded6a64 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -86,7 +86,27 @@ clone_depot_tools() { # Linux-only helper scripts (cbuildbot, luci-auth-fido2-plugin, etc.) and # are not needed for WebRTC builds. The 'fetch' check below validates the # critical tools were extracted. - tar -xzf "$archive" -C "$dest" || true + # On Windows (Git Bash), symlinks in the archive fail because creating them + # requires elevated privileges. Use Python to extract while silently + # skipping symlink/hardlink members so that critical batch/exe files are + # always extracted. Non-Windows uses plain tar. + if command -v cygpath >/dev/null 2>&1; then + python3 - "$dest" "$archive" <<'PYEOF_INNER' +import tarfile, sys +dest, archive = sys.argv[1], sys.argv[2] +with tarfile.open(archive, "r:gz") as tf: + for m in tf.getmembers(): + if m.issym() or m.islnk(): + print(f"Skip symlink: {m.name}") + continue + try: + tf.extract(m, dest) + except Exception as e: + print(f"Warning: cannot extract {m.name}: {e}", file=sys.stderr) +PYEOF_INNER + else + tar -xzf "$archive" -C "$dest" + fi rm -rf "$tmp" if [[ ! -x "$dest/fetch" ]]; then @@ -94,6 +114,23 @@ clone_depot_tools() { exit 1 fi + # On Windows, verify that critical batch wrappers were extracted from the + # tarball. If extraction failed silently, these files would be absent and + # the bootstrap or GN build would later fail with a confusing error. + if command -v cygpath >/dev/null 2>&1; then + local _missing=() + for _tool in "fetch.bat" "gn.bat"; do + [[ -f "$dest/$_tool" ]] || _missing+=("$_tool") + done + if [[ ${#_missing[@]} -gt 0 ]]; then + echo "ERROR: depot_tools extraction incomplete on Windows." >&2 + echo " Missing: ${_missing[*]}" >&2 + echo " Batch files in $dest:" >&2 + ls "$dest"/*.bat 2>/dev/null >&2 || ls "$dest" >&2 || true + exit 1 + fi + fi + # Bootstrap depot_tools. # We must temporarily prepend depot_tools to PATH so that python scripts # and subprocesses launched during bootstrap (like gsutil.py calling luci-auth) diff --git a/3rdparty/webrtc/webrtc_common.cmake b/3rdparty/webrtc/webrtc_common.cmake index 712595aaf30..73d9da3305c 100644 --- a/3rdparty/webrtc/webrtc_common.cmake +++ b/3rdparty/webrtc/webrtc_common.cmake @@ -28,7 +28,9 @@ function(get_webrtc_args WEBRTC_ARGS) endif() elseif(UNIX) set(WEBRTC_ARGS is_clang=false\n${WEBRTC_ARGS}) - set(WEBRTC_ARGS extra_cxxflags=[\"-Wno-changes-meaning\"]\n${WEBRTC_ARGS}) + # GCC rejects the virtual-function name-shadowing via -fpermissive (not a + # -W flag), so demote it to a warning with -fpermissive. + set(WEBRTC_ARGS extra_cxxflags=[\"-fpermissive\"]\n${WEBRTC_ARGS}) endif() set(WEBRTC_ARGS use_custom_libcxx=false\n${WEBRTC_ARGS}) From 7b397de5afd1c1335b01b17effb1db81b3828d3c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:51:21 +0000 Subject: [PATCH 16/37] Add push trigger for PR branch to trigger WebRTC CI automatically --- .github/workflows/webrtc.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 6eb3ab1ac70..f2b399db68c 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -2,6 +2,9 @@ name: WebRTC permissions: {} on: + push: + branches: + - copilot/fix-webrtc-workflow-jobs workflow_dispatch: inputs: webrtc_commit: From 5c19dbde6c779fb07cb740a27f89344f3a9472b9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:55:49 +0000 Subject: [PATCH 17/37] Address code review: fix duplicate comment, README patch list, capitalize warning --- 3rdparty/webrtc/README.md | 21 +++++++++++++-------- 3rdparty/webrtc/apply_webrtc_patches.sh | 2 +- 3rdparty/webrtc/webrtc_build.sh | 5 ----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/3rdparty/webrtc/README.md b/3rdparty/webrtc/README.md index c00b8d09a8a..7f2066e31c4 100644 --- a/3rdparty/webrtc/README.md +++ b/3rdparty/webrtc/README.md @@ -27,16 +27,21 @@ Applied by `apply_webrtc_patches.sh` (each is skipped if it does not apply cleanly to the pinned WebRTC commit): ``` -0001-src-enable-rtc_use_cxx11_abi-option.patch # -> src -0001-build-enable-rtc_use_cxx11_abi-option.patch # -> src/build -0001-third_party-enable-rtc_use_cxx11_abi-option.patch # -> src/third_party -0002-build-enable_safe_libstdcxx.patch # -> src/build_overrides -0003-src-fix-nullptr_t-with-libstdcxx.patch # -> src (GCC) -0004-src-gcc-suppress-port-interface-network.patch # -> src (GCC) -0005-call-payload_type_picker-gcc-flat_tree.patch # -> src (GCC) -0006-build-win-dynamic-crt.patch # -> src/build (Windows /MD) +0001-src-enable-rtc_use_cxx11_abi-option.patch # -> src (GCC C++11 ABI) +0001-build-enable-rtc_use_cxx11_abi-option.patch # -> src/build (GCC C++11 ABI) +0001-third_party-enable-rtc_use_cxx11_abi-option.patch # -> src/third_party (GCC C++11 ABI) +0002-src-fix-nullptr_t-with-libstdcxx.patch # -> src (GCC nullptr_t with libstdc++) +0004-call-payload_type_picker-gcc-flat_tree.patch # -> src (GCC flat_tree ordering) +0005-build-win-dynamic-crt.patch # -> src/build (Windows /MD[d] runtime) +0006-third_party-protobuf-disable-constinit-on-apple.patch # -> third_party/protobuf (macOS constinit) ``` +`apply_webrtc_patches.sh` also directly patches +`third_party/protobuf/src/google/protobuf/port.cc` on Apple (via Python +string replacement) to drop `PROTOBUF_CONSTINIT` from the +`fixed_address_empty_string` declaration, which causes a hard error on +Xcode 15.4 because `std::string`'s constructor is not a constant expression. + `0006-build-win-dynamic-crt.patch` adds a `rtc_win_dynamic_crt` gn arg so a non-component (static) Windows build can use the dynamic MSVC runtime (`/MD[d]`). This lets Open3D ship a static `libwebrtc` for both diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index a2f0d9ae7e7..37eb3ba8387 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -64,7 +64,7 @@ if '__APPLE__' in content and 'fixed_address_empty_string' in content: sys.exit(0) if old not in content: print(f'WARNING: {path}: expected PROTOBUF_CONSTINIT pattern not found; ' - f'skipping Apple constinit fix', file=sys.stderr) + f'Skipping Apple constinit fix', file=sys.stderr) sys.exit(0) new = ('#if defined(__APPLE__)\n' '// Apple Clang (Xcode 15.4): GlobalEmptyString (std::string) requires heap\n' diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index b2e0ded6a64..a442291c962 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -81,11 +81,6 @@ clone_depot_tools() { rm -rf "$dest" mkdir -p "$dest" # Gitiles +archive tarballs unpack flat (fetch at archive root, not in a subdir). - # On Windows (Git Bash), symlinks in the archive fail to extract because - # symlink creation requires elevated privileges. Those symlinks are - # Linux-only helper scripts (cbuildbot, luci-auth-fido2-plugin, etc.) and - # are not needed for WebRTC builds. The 'fetch' check below validates the - # critical tools were extracted. # On Windows (Git Bash), symlinks in the archive fail because creating them # requires elevated privileges. Use Python to extract while silently # skipping symlink/hardlink members so that critical batch/exe files are From fd030f8300411e946527ab948103b19dcb13a7ad Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 01:04:27 +0000 Subject: [PATCH 18/37] Add changelog entry for WebRTC workflow fixes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11d5398c043..69577c2a050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## Main +- Fix WebRTC prebuilt packaging and CI workflow across Linux, macOS arm64, and Windows runtime variants (PR #7515) - Add `CorrespondenceCheckerBasedOnSourceRotation` to constrain global orientation priors in RANSAC registration (PR #7461) - Use glfwGetMonitorWorkarea for accurate screen size in GetScreenSize, remove unusable_height estimation hack, and subtract window-decoration extents before clamping auto-sized windows (PR #7469) - Upgrade stdgpu third-party library to commit d7c07d0. From 66a87d11666bbc8dec5d5ad72ff6d2523ff992f2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 01:05:40 +0000 Subject: [PATCH 19/37] Address validation feedback for WebRTC packaging --- .github/workflows/webrtc.yml | 3 --- 3rdparty/webrtc/webrtc_download.cmake | 14 +++++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index f2b399db68c..6eb3ab1ac70 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -2,9 +2,6 @@ name: WebRTC permissions: {} on: - push: - branches: - - copilot/fix-webrtc-workflow-jobs workflow_dispatch: inputs: webrtc_commit: diff --git a/3rdparty/webrtc/webrtc_download.cmake b/3rdparty/webrtc/webrtc_download.cmake index 075c4ac00d8..c860eb67d74 100644 --- a/3rdparty/webrtc/webrtc_download.cmake +++ b/3rdparty/webrtc/webrtc_download.cmake @@ -5,8 +5,10 @@ include(ExternalProject) set(WEBRTC_VER e8b4d4c) +set(WEBRTC_USE_LOCAL_ARCHIVE OFF) if(DEFINED ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE} AND NOT "$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}" STREQUAL "") set(WEBRTC_URL "file://$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}") + set(WEBRTC_USE_LOCAL_ARCHIVE ON) endif() if (APPLE) if(NOT WEBRTC_URL) @@ -57,9 +59,15 @@ else() # Linux endif() if(WEBRTC_SHA256 MATCHES "^PLACEHOLDER") - message(WARNING "WebRTC prebuilt SHA256 not set for this platform (${WEBRTC_SHA256}). " - "Set OPEN3D_WEBRTC_PREBUILT_ARCHIVE or update webrtc_download.cmake after CI publish.") - unset(WEBRTC_SHA256) + if(WEBRTC_USE_LOCAL_ARCHIVE) + message(WARNING "WebRTC prebuilt SHA256 not set for this platform (${WEBRTC_SHA256}). " + "Using local OPEN3D_WEBRTC_PREBUILT_ARCHIVE without URL_HASH verification.") + unset(WEBRTC_SHA256) + else() + message(FATAL_ERROR "WebRTC prebuilt SHA256 not set for this platform (${WEBRTC_SHA256}). " + "Update webrtc_download.cmake after publishing the archive, or set " + "OPEN3D_WEBRTC_PREBUILT_ARCHIVE to a verified local file.") + endif() endif() set(_webrtc_url_hash "") From 601644d4b681ade8a9fbf1671f2ab7ce4709a0d6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 01:07:12 +0000 Subject: [PATCH 20/37] Remove stale WebRTC peer connection options --- .../visualization/webrtc_server/html/webrtcstreamer.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index 7c625e67c1e..d607089a822 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -37,8 +37,6 @@ let WebRtcStreamer = (function() { this.pc = null; this.dataChannel = null; - this.pcOptions = {optional: [{DtlsSrtpKeyAgreement: true}]}; - this.mediaConstraints = { offerToReceiveAudio: true, offerToReceiveVideo: true, @@ -614,8 +612,7 @@ let WebRtcStreamer = (function() { WebRtcStreamer.prototype.createPeerConnection = function() { console.log( 'createPeerConnection config: ' + - JSON.stringify(this.pcConfig) + - ' option:' + JSON.stringify(this.pcOptions)); + JSON.stringify(this.pcConfig)); this.pc = new RTCPeerConnection(this.pcConfig); var pc = this.pc; pc.peerid = Math.random(); @@ -719,8 +716,7 @@ let WebRtcStreamer = (function() { console.log( 'Created RTCPeerConnection with config: ' + - JSON.stringify(this.pcConfig) + - 'option:' + JSON.stringify(this.pcOptions)); + JSON.stringify(this.pcConfig)); return pc; }; From 9c5442f483b84973ac8967da0b19d323ec72a7fa Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 01:08:29 +0000 Subject: [PATCH 21/37] Apply WebRTC follow-up cleanups --- cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp | 1 - cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index 9e1ccaca0bd..338d720c0b6 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -114,7 +114,6 @@ static bool PeerConnectionHasStreamForWindow( static webrtc::PeerConnectionFactoryDependencies CreatePeerConnectionFactoryDependencies( webrtc::FieldTrials* field_trials) { - (void)field_trials; webrtc::PeerConnectionFactoryDependencies dependencies; dependencies.worker_thread = webrtc::Thread::Current(); dependencies.network_thread = nullptr; diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index d607089a822..24a9b80b2a3 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -454,6 +454,7 @@ let WebRtcStreamer = (function() { WebRtcStreamer.prototype.disconnect = function() { if (this.videoElt) { this.videoElt.srcObject = null; + this.videoElt.removeAttribute('src'); } this.remoteStream = null; if (this.pc) { From 745f98e1c010251d42647cde72968c4971d3174e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:29:58 +0000 Subject: [PATCH 22/37] Fix Ubuntu WebRTC build: patch BUILD.gn to add -fpermissive for GCC C++20 --- 3rdparty/webrtc/0003-src-gcc-fpermissive.patch | 18 ++++++++++++++++++ 3rdparty/webrtc/README.md | 1 + 3rdparty/webrtc/apply_webrtc_patches.sh | 2 ++ 3rdparty/webrtc/webrtc_common.cmake | 7 ++++--- 4 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 3rdparty/webrtc/0003-src-gcc-fpermissive.patch diff --git a/3rdparty/webrtc/0003-src-gcc-fpermissive.patch b/3rdparty/webrtc/0003-src-gcc-fpermissive.patch new file mode 100644 index 00000000000..ef5c7fa5b6b --- /dev/null +++ b/3rdparty/webrtc/0003-src-gcc-fpermissive.patch @@ -0,0 +1,18 @@ +diff --git a/BUILD.gn b/BUILD.gn +--- a/BUILD.gn ++++ b/BUILD.gn +@@ -252,6 +252,13 @@ config("common_config") { + defines += [ "_GLIBCXX_USE_CXX11_ABI=0" ] + } + ++ if (!is_clang) { ++ # GCC C++20: p2p/base/port_interface.h declares a virtual function ++ # named 'Network()' whose name matches the webrtc::Network class in ++ # scope. GCC treats this "changes meaning" as a hard error in C++20; ++ # -fpermissive downgrades it to a warning, matching Clang's behaviour. ++ cflags_cc = [ "-fpermissive" ] ++ } ++ + if (rtc_enable_protobuf) { + defines += [ "WEBRTC_ENABLE_PROTOBUF=1" ] + } else { diff --git a/3rdparty/webrtc/README.md b/3rdparty/webrtc/README.md index 7f2066e31c4..2f5dc845923 100644 --- a/3rdparty/webrtc/README.md +++ b/3rdparty/webrtc/README.md @@ -31,6 +31,7 @@ cleanly to the pinned WebRTC commit): 0001-build-enable-rtc_use_cxx11_abi-option.patch # -> src/build (GCC C++11 ABI) 0001-third_party-enable-rtc_use_cxx11_abi-option.patch # -> src/third_party (GCC C++11 ABI) 0002-src-fix-nullptr_t-with-libstdcxx.patch # -> src (GCC nullptr_t with libstdc++) +0003-src-gcc-fpermissive.patch # -> src (GCC C++20: Network() name shadowing) 0004-call-payload_type_picker-gcc-flat_tree.patch # -> src (GCC flat_tree ordering) 0005-build-win-dynamic-crt.patch # -> src/build (Windows /MD[d] runtime) 0006-third_party-protobuf-disable-constinit-on-apple.patch # -> third_party/protobuf (macOS constinit) diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index 37eb3ba8387..a082ea990e1 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -90,6 +90,8 @@ PATCH_DIR="$OPEN3D_DIR/3rdparty/webrtc" apply_one "$PATCH_DIR/0001-src-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0001-build-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/build" apply_one "$PATCH_DIR/0001-third_party-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/third_party" +# 0003 must come after 0001-src so its context (rtc_use_cxx11_abi block) is present. +apply_one "$PATCH_DIR/0003-src-gcc-fpermissive.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0002-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0004-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0005-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" diff --git a/3rdparty/webrtc/webrtc_common.cmake b/3rdparty/webrtc/webrtc_common.cmake index 73d9da3305c..3541b265a7b 100644 --- a/3rdparty/webrtc/webrtc_common.cmake +++ b/3rdparty/webrtc/webrtc_common.cmake @@ -28,9 +28,10 @@ function(get_webrtc_args WEBRTC_ARGS) endif() elseif(UNIX) set(WEBRTC_ARGS is_clang=false\n${WEBRTC_ARGS}) - # GCC rejects the virtual-function name-shadowing via -fpermissive (not a - # -W flag), so demote it to a warning with -fpermissive. - set(WEBRTC_ARGS extra_cxxflags=[\"-fpermissive\"]\n${WEBRTC_ARGS}) + # -fpermissive for GCC is injected via 0003-src-gcc-fpermissive.patch + # directly into WebRTC's BUILD.gn config("common_config") because the + # gn arg `extra_cxxflags` is not a recognised WebRTC build argument and + # was silently ignored. endif() set(WEBRTC_ARGS use_custom_libcxx=false\n${WEBRTC_ARGS}) From 33c298935019d2cf236a3f62e3579807b541095e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:52:20 +0000 Subject: [PATCH 23/37] Remove -fpermissive and fix WebRTC C++20 Network() changes meaning properly --- .../webrtc/0003-src-gcc-fpermissive.patch | 18 ----------- 3rdparty/webrtc/README.md | 6 +++- 3rdparty/webrtc/apply_webrtc_patches.sh | 30 +++++++++++++++++-- 3rdparty/webrtc/webrtc_common.cmake | 4 --- 4 files changed, 33 insertions(+), 25 deletions(-) delete mode 100644 3rdparty/webrtc/0003-src-gcc-fpermissive.patch diff --git a/3rdparty/webrtc/0003-src-gcc-fpermissive.patch b/3rdparty/webrtc/0003-src-gcc-fpermissive.patch deleted file mode 100644 index ef5c7fa5b6b..00000000000 --- a/3rdparty/webrtc/0003-src-gcc-fpermissive.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/BUILD.gn b/BUILD.gn ---- a/BUILD.gn -+++ b/BUILD.gn -@@ -252,6 +252,13 @@ config("common_config") { - defines += [ "_GLIBCXX_USE_CXX11_ABI=0" ] - } - -+ if (!is_clang) { -+ # GCC C++20: p2p/base/port_interface.h declares a virtual function -+ # named 'Network()' whose name matches the webrtc::Network class in -+ # scope. GCC treats this "changes meaning" as a hard error in C++20; -+ # -fpermissive downgrades it to a warning, matching Clang's behaviour. -+ cflags_cc = [ "-fpermissive" ] -+ } -+ - if (rtc_enable_protobuf) { - defines += [ "WEBRTC_ENABLE_PROTOBUF=1" ] - } else { diff --git a/3rdparty/webrtc/README.md b/3rdparty/webrtc/README.md index 2f5dc845923..6cf26c47932 100644 --- a/3rdparty/webrtc/README.md +++ b/3rdparty/webrtc/README.md @@ -31,7 +31,7 @@ cleanly to the pinned WebRTC commit): 0001-build-enable-rtc_use_cxx11_abi-option.patch # -> src/build (GCC C++11 ABI) 0001-third_party-enable-rtc_use_cxx11_abi-option.patch # -> src/third_party (GCC C++11 ABI) 0002-src-fix-nullptr_t-with-libstdcxx.patch # -> src (GCC nullptr_t with libstdc++) -0003-src-gcc-fpermissive.patch # -> src (GCC C++20: Network() name shadowing) + 0004-call-payload_type_picker-gcc-flat_tree.patch # -> src (GCC flat_tree ordering) 0005-build-win-dynamic-crt.patch # -> src/build (Windows /MD[d] runtime) 0006-third_party-protobuf-disable-constinit-on-apple.patch # -> third_party/protobuf (macOS constinit) @@ -43,6 +43,10 @@ string replacement) to drop `PROTOBUF_CONSTINIT` from the `fixed_address_empty_string` declaration, which causes a hard error on Xcode 15.4 because `std::string`'s constructor is not a constant expression. +`apply_webrtc_patches.sh` also directly patches `p2p/base/port_interface.h` and +`p2p/base/port.h` (via Python string replacement) to qualify the return type +of `Network()` as `::webrtc::Network*` to fix a GCC C++20 "changes meaning" error. + `0006-build-win-dynamic-crt.patch` adds a `rtc_win_dynamic_crt` gn arg so a non-component (static) Windows build can use the dynamic MSVC runtime (`/MD[d]`). This lets Open3D ship a static `libwebrtc` for both diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index a082ea990e1..b045cdd4b79 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -85,13 +85,38 @@ print(f'Applied Apple constinit fix in {path}') PYEOF } +# Fix GCC C++20 "changes meaning" error for Network() in WebRTC. +# GCC treats a method name matching a class name as an error. +fix_gcc_cxx20_network_changes_meaning() { + python3 - "${WEBRTC_SRC}/p2p/base/port_interface.h" "${WEBRTC_SRC}/p2p/base/port.h" <<'PYEOF' +import sys +import re +import os + +for path in sys.argv[1:]: + if not os.path.exists(path): + continue + with open(path, 'r') as f: + content = f.read() + + # Replace `Network* Network()` with `::webrtc::Network* Network()` + # but only if it's not already prefixed with `::webrtc::`. + new_content = re.sub(r'(? Date: Sat, 4 Jul 2026 05:41:56 +0000 Subject: [PATCH 24/37] Fix ambiguous PayloadType to int conversion in WebRTC used_ids.h --- 3rdparty/webrtc/apply_webrtc_patches.sh | 42 ++++++++++++++++++++++++ a.out | Bin 0 -> 16496 bytes 2 files changed, 42 insertions(+) create mode 100755 a.out diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index b045cdd4b79..bc5219627e5 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -112,6 +112,47 @@ for path in sys.argv[1:]: PYEOF } +# Fix GCC ambiguous conversion from webrtc::PayloadType to int in used_ids.h. +# webrtc::PayloadType has multiple conversion operators (one inherited from +# StrongAlias) that GCC flags as ambiguous when assigning to int. +# idstruct->id can be an int or a PayloadType, so we conditionally unwrap using type traits. +fix_gcc_payload_type_ambiguous() { + local used_ids="${WEBRTC_SRC}/pc/used_ids.h" + [[ -f "$used_ids" ]] || return 0 + python3 - "$used_ids" <<'PYEOF' +import sys +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +helper = """ +namespace { +template +constexpr int AsInt(const T& t) { + if constexpr (std::is_integral_v) { + return t; + } else { + return t.value(); + } +} +} // namespace +""" + +if 'AsInt(const T& t)' not in content: + content = content.replace('#include "rtc_base/system/rtc_export.h"', '#include "rtc_base/system/rtc_export.h"\n#include \n' + helper) + +new_content = content.replace('int original_id = idstruct->id;', 'int original_id = AsInt(idstruct->id);') +new_content = new_content.replace('int new_id = idstruct->id;', 'int new_id = AsInt(idstruct->id);') + +if new_content != content: + with open(path, 'w') as f: + f.write(new_content) + print(f'Applied GCC PayloadType ambiguous conversion fix in {path}') +else: + print(f'Skip GCC PayloadType ambiguous conversion fix in {path} (already applied or not found)') +PYEOF +} + PATCH_DIR="$OPEN3D_DIR/3rdparty/webrtc" # Required: declare gn args consumed by args.gn and fix GCC compile errors. apply_one "$PATCH_DIR/0001-src-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC" @@ -127,3 +168,4 @@ apply_one "$PATCH_DIR/0005-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" apply_one "$PATCH_DIR/0006-third_party-protobuf-disable-constinit-on-apple.patch" "$WEBRTC_SRC/third_party/protobuf" fix_protobuf_port_cc_apple fix_gcc_cxx20_network_changes_meaning +fix_gcc_payload_type_ambiguous diff --git a/a.out b/a.out new file mode 100755 index 0000000000000000000000000000000000000000..4aa54bf0099e99ccf2d698ae01a836bbad3b4f8c GIT binary patch literal 16496 zcmeHOYit}>6~1f7iQPEyBXOEI5O32OH3U!mib+$YSwGg^1;=r1ho%u>GTt5AQ|-gu znN97eN{s>tlSFF!qYVf|kt)>)B#gw5f)Gg*8Xf|aAXG&{NTv{xNG&+xp#W-@bM8H7 zJ3HA8NyHC+%(Z6EJ>NOsy?4&s8Qr4CrKJ>Opt$xwpwGcVsIm8F}9u{qPC9M0@1lrt2Ub9Da9Ll}7r#9JU9OF=xA z98Zi19=oVLuWSVy7M5P#6!E;WPtgQR>;uc_RVRKw{S6bZIw+eFUfHQ&!IJxX5_lMw zyDfZ~N2tG2*I|m{$&#jl*paZtyE=Cy;w=d)m7Qss+11t3)v4KOtz8ZQ#}$MF=hVQ^ z0lXaN5aY{PHm7uM4{htoj{Yy~IC`jg+ws1&O(Ti1RnK<}JhXWY`rx>cgFaZWJ-+0q zW4#Iv^ilD6N!Ag=a>@Kvvf)g1uhMS+><6g+c39jffq$n2{@D`v%O!9;uOJ-Wg69$R zf}nq_1YQn=twsF*1X!JD6rqqZ)?&k0bbT_JPU*H|WE@==dUR}`9yc@Qq-8s1W^7+? zBAqhFjPZo2Y{e~yMx9PG6;DKCqfT2}Y|6;!PR6htI~ogzM*!;>H*71Wr)?)=8p-IG zKGv#-N4ucILqntKgdHC49~n)L>R|vzo%S{>ZR zpo^XQ@F!!E*BMJ^9XMZTF-GT`G^`XLu^B_3uu?|CI%Wzm+fF>TeLGI(E>vTxHZU0N z={!c5Nqyw;Y~2xBLfkqz7Z-b=5*^j;zQ3ClKGFvZKnljP6OQ+|#esY8|! zuhH|8@gU)qA!!zjvwmr>!tq|>dkB#$E}XiQj!P~a??o=xT)4U~m1Wt5)4wKZxaq=q zOlaWwi_et@q^J5gNB-CbKV#7HQx}fU55M>k@FUk0*0Y3tM1pZqD{#vu;Uy&2< zRz^-%{CI<(`kUAZ^1eYy(+g<(=>;E7f6EtHtD}}D~fHmBJ<0U z^MBqOIe)V}61W(-e9Ku64ZNqR1}bw)YTV4n^PAh#j7!+;_5+a~{Vq*WSLB%SBE$ z3`q5DM1BLvn|LCJFUT0mYm~n#Xz+_40Y3tM1pEm25%446N5GGO9|1oCegymo_z}4K z2;lE7W2Wth0F=$;kK>Cj{`RnbIhTur>;riUk0*f&bG8a6TL7uW=3<=G0+f9dqWeR4b9sZ(c+C>y-bt ziFD4J+e|vY8#R*7`E&R_g@x^Zzm-d)zDUVuGM5_kr4@*BE-K{Bk)3nb4%7HnD>t!` zWCKYSoYS^I`D~n9C*Kjl3E%s$a1PyrN)-zPF#aYb-|@JY$&YjUxc@DLv){jS>il51 z5`v#!<@p|@_TU#>sq?ttHB;(Oko}*e^LbG7|1*)#|Ms-lN-xJD_e-+J=WbNahaBkb z-5q@Nz<4(0WP@!TT8Gx!(v_8J`|r@6=q>G;OLmuKw9=wBazHdO4B?_Mb$PXUwRys{ffmzl8u_2!~s;ojh4;5^m z??cAFDE%7*-$#tU0hMCoUzGm!qAI97x!*q!{(uMH01g4j&|dAqOT{ND{nv|YCC|eR5(Hd$aQsfF4l~b# z%8wo!;_HBKRZzj_4*;)+9xQr($?tT)_;HGubWO$YhLJ z-Ap-|Suv3@lBOQdCX=&3acMHCzQEOiuR68Z^t4RNcXq)LOvkVi9j!X}I_9JadHrb} z^ZRYxvh{4rwkA_%Jo-!+6ZeNk?}3uMcgieM^!}0VePKO3)Q8&@bl3=@ha&_=`bLD_ z_w-QrzG$z|2L|`|bPwwL`};@3u;HM)XD|#5Gj2Er>}A*wdjR5AO3&J69NKa30SL+5 ze#qy?B!0C=;J!CFsJ7I$0q7=fr}ZfVc1pl@3cOGNdG;gZJBQ5#xEJC+wmTH|>DgdW zXvA$5Fb3G?!4btX#hns`emX`??V5lPrZ8gTw#~pDwrseY!EFR$ccgTmGVCd# z#b;B{g_2H2S&o_++e)Vj6di0CGhrZsYSRfvXfm!ERJF-8s93Shn9v+^22^?FwM<%0 zMa`U|n{p}+t+A%sPibk`^?k=x`l`29Jd#-0el4RQCqqx)=V1!2fnty!#J% z-AiWa%XO3w!+~o}=9k)MhrU>NeW^6~KLQmu-@D%qe;;ud{y)hWu4CMMUPrMEvLi_^ z|0%%#0c%wrKd|ct%=;0qB0?Qy1NpgEEzwhDCk%8sIo(aP|ZuYE)&+9^#{NBv| z-uve*;`8+vLGmP-e-9_v%l{?dI7H@GGmm5^*xbd-|1B8Z{7_JNv;4N340WDA3}1x` zq#x(^knwvA?~nERv(EAcaBEzA{=VR~7675JpVB_i8Gj!PICt?GfQ8>Tc)v3T2Jb&C z%;W3$Ex_>n^L+ZgU||P(3vp11{3r8St^~51FGA$nswDTr4KR=OY9J%V{f9!zQcL`e z*tn~NTy?D0QUBzgw=$o?n8&#}x8%8#cY;cY8my!V^I^d7I~E>4KYywK Date: Sat, 4 Jul 2026 06:46:07 +0000 Subject: [PATCH 25/37] Fix WebRTC patch AsInt name lookup on GCC 14 --- 3rdparty/webrtc/apply_webrtc_patches.sh | 4 ++-- test_webrtc.o | Bin 0 -> 2224 bytes 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 test_webrtc.o diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index bc5219627e5..2b85c6bb169 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -126,7 +126,7 @@ with open(path, 'r') as f: content = f.read() helper = """ -namespace { +namespace webrtc { template constexpr int AsInt(const T& t) { if constexpr (std::is_integral_v) { @@ -135,7 +135,7 @@ constexpr int AsInt(const T& t) { return t.value(); } } -} // namespace +} // namespace webrtc """ if 'AsInt(const T& t)' not in content: diff --git a/test_webrtc.o b/test_webrtc.o new file mode 100644 index 0000000000000000000000000000000000000000..8c3854dfba4c8418a6c4a2ef6213c619a9140820 GIT binary patch literal 2224 zcmbtU-Afcv6u-NE2BvHD5E!upg;8F1{n#KP)|Sy(6ccSmdjx z5qvD@sXw8=K#~x9jEJBYUxFTb34M?d?EGf#-5tjvsD5zvp7XosW6nMI?o(xaBpQh@ zi3q#J8kS@kt2Q_Ew5z6BH;cKxtz@^6w3F<0UHFw#op0*1A2D5BYrfP=r0OUK$<1wL zADPwl+`hW5?5WOY)!F_|4BJrGlmlJ8>UrBOF1fSi%N19h)$giP+a=Fk8RjU58#`NW zHD~nzIo8U9xj*A> zyRrAd6HuQGi}*2bA5<7{g3avbQP^G+zCHv^3==vI)Qq|l^LY0ggiDi;mr+Sl&iL}X zJrkE9x#-L9_Db9ei9aWq9&E$1wMWLRX%~2*z_fc4$$o8g{Lby+aZQ`hl0(B*wrppM zic*{!yQ{$=<+%?cRC(xQuIDU6&+1loVpTpmdr<0UN zv+R6fQ7bGgYIFJGlK33-1z_g5!-q#6MnDq4(>UwAoH=-i^pBGcU;_Rrq(KtE$vXx6 z0e%SJarf%r)=J6F&oXY=rnfFg%`}$s+%}eN#`V0NXMEPO7_YOuaa4bEFm6V~{&^0( zP$`v+GSx0uY=hHV>2nWM*z@HYubP#rVcIL=n~Vi*&diq##^=pSty=H!zm!iD-5ru4 zEjW4SI?2X7F(W{(ankXIf-ho{I6&TO&+>ewjQa(zR*)Q`D_k|%xNQV{M{p#8dI{8-XP z|C0Vd3Q7gzP7Fnkbc>kQs*Sp0?{zt%lHB^du3Us?B@ literal 0 HcmV?d00001 From 81193fbbda85fd8b3085e1e762ac16f9ad305579 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 4 Jul 2026 21:21:53 +0000 Subject: [PATCH 26/37] Apply remaining changes --- webrtc.tar.gz | 1 + 1 file changed, 1 insertion(+) create mode 100644 webrtc.tar.gz diff --git a/webrtc.tar.gz b/webrtc.tar.gz new file mode 100644 index 00000000000..85373076912 --- /dev/null +++ b/webrtc.tar.gz @@ -0,0 +1 @@ +Not Found \ No newline at end of file From be9fadab5768b4f567943c5e62e7ce0861310cfe Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Sat, 4 Jul 2026 14:40:07 -0700 Subject: [PATCH 27/37] delete accidental files --- a.out | Bin 16496 -> 0 bytes test_webrtc.o | Bin 2224 -> 0 bytes webrtc.tar.gz | 1 - 3 files changed, 1 deletion(-) delete mode 100755 a.out delete mode 100644 test_webrtc.o delete mode 100644 webrtc.tar.gz diff --git a/a.out b/a.out deleted file mode 100755 index 4aa54bf0099e99ccf2d698ae01a836bbad3b4f8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16496 zcmeHOYit}>6~1f7iQPEyBXOEI5O32OH3U!mib+$YSwGg^1;=r1ho%u>GTt5AQ|-gu znN97eN{s>tlSFF!qYVf|kt)>)B#gw5f)Gg*8Xf|aAXG&{NTv{xNG&+xp#W-@bM8H7 zJ3HA8NyHC+%(Z6EJ>NOsy?4&s8Qr4CrKJ>Opt$xwpwGcVsIm8F}9u{qPC9M0@1lrt2Ub9Da9Ll}7r#9JU9OF=xA z98Zi19=oVLuWSVy7M5P#6!E;WPtgQR>;uc_RVRKw{S6bZIw+eFUfHQ&!IJxX5_lMw zyDfZ~N2tG2*I|m{$&#jl*paZtyE=Cy;w=d)m7Qss+11t3)v4KOtz8ZQ#}$MF=hVQ^ z0lXaN5aY{PHm7uM4{htoj{Yy~IC`jg+ws1&O(Ti1RnK<}JhXWY`rx>cgFaZWJ-+0q zW4#Iv^ilD6N!Ag=a>@Kvvf)g1uhMS+><6g+c39jffq$n2{@D`v%O!9;uOJ-Wg69$R zf}nq_1YQn=twsF*1X!JD6rqqZ)?&k0bbT_JPU*H|WE@==dUR}`9yc@Qq-8s1W^7+? zBAqhFjPZo2Y{e~yMx9PG6;DKCqfT2}Y|6;!PR6htI~ogzM*!;>H*71Wr)?)=8p-IG zKGv#-N4ucILqntKgdHC49~n)L>R|vzo%S{>ZR zpo^XQ@F!!E*BMJ^9XMZTF-GT`G^`XLu^B_3uu?|CI%Wzm+fF>TeLGI(E>vTxHZU0N z={!c5Nqyw;Y~2xBLfkqz7Z-b=5*^j;zQ3ClKGFvZKnljP6OQ+|#esY8|! zuhH|8@gU)qA!!zjvwmr>!tq|>dkB#$E}XiQj!P~a??o=xT)4U~m1Wt5)4wKZxaq=q zOlaWwi_et@q^J5gNB-CbKV#7HQx}fU55M>k@FUk0*0Y3tM1pZqD{#vu;Uy&2< zRz^-%{CI<(`kUAZ^1eYy(+g<(=>;E7f6EtHtD}}D~fHmBJ<0U z^MBqOIe)V}61W(-e9Ku64ZNqR1}bw)YTV4n^PAh#j7!+;_5+a~{Vq*WSLB%SBE$ z3`q5DM1BLvn|LCJFUT0mYm~n#Xz+_40Y3tM1pEm25%446N5GGO9|1oCegymo_z}4K z2;lE7W2Wth0F=$;kK>Cj{`RnbIhTur>;riUk0*f&bG8a6TL7uW=3<=G0+f9dqWeR4b9sZ(c+C>y-bt ziFD4J+e|vY8#R*7`E&R_g@x^Zzm-d)zDUVuGM5_kr4@*BE-K{Bk)3nb4%7HnD>t!` zWCKYSoYS^I`D~n9C*Kjl3E%s$a1PyrN)-zPF#aYb-|@JY$&YjUxc@DLv){jS>il51 z5`v#!<@p|@_TU#>sq?ttHB;(Oko}*e^LbG7|1*)#|Ms-lN-xJD_e-+J=WbNahaBkb z-5q@Nz<4(0WP@!TT8Gx!(v_8J`|r@6=q>G;OLmuKw9=wBazHdO4B?_Mb$PXUwRys{ffmzl8u_2!~s;ojh4;5^m z??cAFDE%7*-$#tU0hMCoUzGm!qAI97x!*q!{(uMH01g4j&|dAqOT{ND{nv|YCC|eR5(Hd$aQsfF4l~b# z%8wo!;_HBKRZzj_4*;)+9xQr($?tT)_;HGubWO$YhLJ z-Ap-|Suv3@lBOQdCX=&3acMHCzQEOiuR68Z^t4RNcXq)LOvkVi9j!X}I_9JadHrb} z^ZRYxvh{4rwkA_%Jo-!+6ZeNk?}3uMcgieM^!}0VePKO3)Q8&@bl3=@ha&_=`bLD_ z_w-QrzG$z|2L|`|bPwwL`};@3u;HM)XD|#5Gj2Er>}A*wdjR5AO3&J69NKa30SL+5 ze#qy?B!0C=;J!CFsJ7I$0q7=fr}ZfVc1pl@3cOGNdG;gZJBQ5#xEJC+wmTH|>DgdW zXvA$5Fb3G?!4btX#hns`emX`??V5lPrZ8gTw#~pDwrseY!EFR$ccgTmGVCd# z#b;B{g_2H2S&o_++e)Vj6di0CGhrZsYSRfvXfm!ERJF-8s93Shn9v+^22^?FwM<%0 zMa`U|n{p}+t+A%sPibk`^?k=x`l`29Jd#-0el4RQCqqx)=V1!2fnty!#J% z-AiWa%XO3w!+~o}=9k)MhrU>NeW^6~KLQmu-@D%qe;;ud{y)hWu4CMMUPrMEvLi_^ z|0%%#0c%wrKd|ct%=;0qB0?Qy1NpgEEzwhDCk%8sIo(aP|ZuYE)&+9^#{NBv| z-uve*;`8+vLGmP-e-9_v%l{?dI7H@GGmm5^*xbd-|1B8Z{7_JNv;4N340WDA3}1x` zq#x(^knwvA?~nERv(EAcaBEzA{=VR~7675JpVB_i8Gj!PICt?GfQ8>Tc)v3T2Jb&C z%;W3$Ex_>n^L+ZgU||P(3vp11{3r8St^~51FGA$nswDTr4KR=OY9J%V{f9!zQcL`e z*tn~NTy?D0QUBzgw=$o?n8&#}x8%8#cY;cY8my!V^I^d7I~E>4KYywKmdjx z5qvD@sXw8=K#~x9jEJBYUxFTb34M?d?EGf#-5tjvsD5zvp7XosW6nMI?o(xaBpQh@ zi3q#J8kS@kt2Q_Ew5z6BH;cKxtz@^6w3F<0UHFw#op0*1A2D5BYrfP=r0OUK$<1wL zADPwl+`hW5?5WOY)!F_|4BJrGlmlJ8>UrBOF1fSi%N19h)$giP+a=Fk8RjU58#`NW zHD~nzIo8U9xj*A> zyRrAd6HuQGi}*2bA5<7{g3avbQP^G+zCHv^3==vI)Qq|l^LY0ggiDi;mr+Sl&iL}X zJrkE9x#-L9_Db9ei9aWq9&E$1wMWLRX%~2*z_fc4$$o8g{Lby+aZQ`hl0(B*wrppM zic*{!yQ{$=<+%?cRC(xQuIDU6&+1loVpTpmdr<0UN zv+R6fQ7bGgYIFJGlK33-1z_g5!-q#6MnDq4(>UwAoH=-i^pBGcU;_Rrq(KtE$vXx6 z0e%SJarf%r)=J6F&oXY=rnfFg%`}$s+%}eN#`V0NXMEPO7_YOuaa4bEFm6V~{&^0( zP$`v+GSx0uY=hHV>2nWM*z@HYubP#rVcIL=n~Vi*&diq##^=pSty=H!zm!iD-5ru4 zEjW4SI?2X7F(W{(ankXIf-ho{I6&TO&+>ewjQa(zR*)Q`D_k|%xNQV{M{p#8dI{8-XP z|C0Vd3Q7gzP7Fnkbc>kQs*Sp0?{zt%lHB^du3Us?B@ diff --git a/webrtc.tar.gz b/webrtc.tar.gz deleted file mode 100644 index 85373076912..00000000000 --- a/webrtc.tar.gz +++ /dev/null @@ -1 +0,0 @@ -Not Found \ No newline at end of file From a6eea9dbe68290fdd4ad475f18547f5938200391 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Sun, 5 Jul 2026 08:10:38 -0700 Subject: [PATCH 28/37] webrtc linux (gcc + libstdc++) builds --- 3rdparty/webrtc/apply_webrtc_patches.sh | 9 ++++++--- 3rdparty/webrtc/webrtc_build.sh | 19 +++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index 2b85c6bb169..2e4263f7cb8 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -126,7 +126,6 @@ with open(path, 'r') as f: content = f.read() helper = """ -namespace webrtc { template constexpr int AsInt(const T& t) { if constexpr (std::is_integral_v) { @@ -135,11 +134,15 @@ constexpr int AsInt(const T& t) { return t.value(); } } -} // namespace webrtc + """ +# Anchor on `namespace webrtc {` (present in every candidate file) rather +# than a specific #include line, which may not exist in every file this +# fix is applied to (e.g. pc/used_ids.h does not include rtc_export.h). if 'AsInt(const T& t)' not in content: - content = content.replace('#include "rtc_base/system/rtc_export.h"', '#include "rtc_base/system/rtc_export.h"\n#include \n' + helper) + content = content.replace('#include ', '#include \n#include ', 1) + content = content.replace('namespace webrtc {\n', 'namespace webrtc {\n' + helper, 1) new_content = content.replace('int original_id = idstruct->id;', 'int original_id = AsInt(idstruct->id);') new_content = new_content.replace('int new_id = idstruct->id;', 'int new_id = AsInt(idstruct->id);') diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index a442291c962..70443e2caf8 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -165,7 +165,6 @@ install_dependencies_ubuntu() { libglib2.0-dev \ libnss3-dev \ libgtk-3-dev \ - ninja-build \ python3 \ python3-pip \ python3-setuptools \ @@ -225,13 +224,29 @@ build_webrtc() { WEBRTC_COMMIT_SHORT=$(git -C "$root/webrtc/src" rev-parse --short=7 HEAD) + # depot_tools/ninja is a Python wrapper that locates the real ninja binary + # by walking up from the *current directory* to find a tracked gclient + # entry (see gclient_paths.FindGclientRoot). CMake's own Ninja-generator + # sanity check ("ninja --version") runs with cwd = this build directory, + # which is a sibling of webrtc/src (not nested inside it), so that lookup + # fails and the wrapper falls back to requiring a "ninja" on PATH. Point + # CMake directly at the real, DEPS-pinned ninja binary instead (mirrors + # how the Windows job locates ninja.exe before prepending depot_tools to + # PATH) so no system/apt ninja package is required. + local ninja_bin="$root/webrtc/src/third_party/ninja/ninja" + if [[ ! -x "$ninja_bin" ]]; then + echo "ERROR: expected ninja binary not found at $ninja_bin" >&2 + exit 1 + fi + mkdir -p "$root/webrtc/build" pushd "$root/webrtc/build" cmake -G Ninja \ + -DCMAKE_MAKE_PROGRAM="$ninja_bin" \ -DCMAKE_INSTALL_PREFIX="$root/webrtc_release" \ -DGLIBCXX_USE_CXX11_ABI="${GLIBCXX_USE_CXX11_ABI}" \ .. - ninja -j"${NPROC}" install + "$ninja_bin" -j"${NPROC}" install popd pushd "$root" From 00fdd859456d1f6143c6e89a7a031aeb131caa00 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Sun, 5 Jul 2026 15:17:42 -0700 Subject: [PATCH 29/37] fix patches --- ...-protobuf-disable-constinit-on-apple.patch | 19 ++- ...ix-gcc-cxx20-network-changes-meaning.patch | 19 +++ ...cc-payload-type-ambiguous-conversion.patch | 45 +++++ ...f-port-cc-disable-constinit-on-apple.patch | 38 +++++ 3rdparty/webrtc/README.md | 30 ++-- 3rdparty/webrtc/apply_webrtc_patches.sh | 156 +++--------------- 6 files changed, 159 insertions(+), 148 deletions(-) create mode 100644 3rdparty/webrtc/0007-p2p-fix-gcc-cxx20-network-changes-meaning.patch create mode 100644 3rdparty/webrtc/0008-pc-fix-gcc-payload-type-ambiguous-conversion.patch create mode 100644 3rdparty/webrtc/0009-third_party-protobuf-port-cc-disable-constinit-on-apple.patch diff --git a/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch b/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch index 35fe389efbe..ddf9f875d24 100644 --- a/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch +++ b/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch @@ -1,7 +1,18 @@ -diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc -index 16423927bd33..edef8148783b 100644 ---- a/src/google/protobuf/port_def.inc -+++ b/src/google/protobuf/port_def.inc +Subject: [PATCH] third_party/protobuf: disable constinit on Apple in port_def.inc + +Apple Clang >= 12 expands PROTOBUF_CONSTINIT to constinit or +[[clang::require_constant_initialization]], which produces a hard +"variable does not have a constant initializer" error for weak default +instance pointers on Apple platforms (Xcode 15.4). Skip both branches +on __APPLE__ so PROTOBUF_CONSTINIT expands to nothing there. + +Note: this patch must be applied with the working directory at the +`third_party` git root (not `third_party/protobuf`, which is a plain +subdirectory and not a repository root) -- see apply_webrtc_patches.sh. + +diff --git a/protobuf/src/google/protobuf/port_def.inc b/protobuf/src/google/protobuf/port_def.inc +--- a/protobuf/src/google/protobuf/port_def.inc ++++ b/protobuf/src/google/protobuf/port_def.inc @@ -469,7 +469,7 @@ # define PROTOBUF_CONSTEXPR constexpr # endif diff --git a/3rdparty/webrtc/0007-p2p-fix-gcc-cxx20-network-changes-meaning.patch b/3rdparty/webrtc/0007-p2p-fix-gcc-cxx20-network-changes-meaning.patch new file mode 100644 index 00000000000..cf26d5c4930 --- /dev/null +++ b/3rdparty/webrtc/0007-p2p-fix-gcc-cxx20-network-changes-meaning.patch @@ -0,0 +1,19 @@ +Subject: [PATCH] p2p: qualify Network() return type to fix GCC C++20 error + +GCC (C++20 mode) treats a method named `Network()` returning an +unqualified `Network*` as a "changes meaning" error, because the method +name shadows the class name `Network` inside its own declaration. +Qualify the return type with `::webrtc::` to disambiguate. + +diff --git a/p2p/base/port_interface.h b/p2p/base/port_interface.h +--- a/p2p/base/port_interface.h ++++ b/p2p/base/port_interface.h +@@ -52,7 +52,7 @@ class PortInterface { + virtual ~PortInterface(); + + virtual IceCandidateType Type() const = 0; +- virtual const Network* Network() const = 0; ++ virtual const ::webrtc::Network* Network() const = 0; + + // Methods to set/get ICE role and tiebreaker values. + virtual void SetIceRole(IceRole role) = 0; diff --git a/3rdparty/webrtc/0008-pc-fix-gcc-payload-type-ambiguous-conversion.patch b/3rdparty/webrtc/0008-pc-fix-gcc-payload-type-ambiguous-conversion.patch new file mode 100644 index 00000000000..249b1c28b2a --- /dev/null +++ b/3rdparty/webrtc/0008-pc-fix-gcc-payload-type-ambiguous-conversion.patch @@ -0,0 +1,45 @@ +Subject: [PATCH] pc: fix GCC ambiguous PayloadType to int conversion + +webrtc::PayloadType has multiple conversion operators (one inherited +from StrongAlias) that GCC flags as ambiguous when implicitly +converting to int. idstruct->id can be either a plain int or a +PayloadType depending on IdStruct, so unwrap it explicitly with a +small helper that dispatches on whether T is already integral. + +diff --git a/pc/used_ids.h b/pc/used_ids.h +--- a/pc/used_ids.h ++++ b/pc/used_ids.h +@@ -12,11 +12,22 @@ + + #include + #include ++#include + + #include "media/base/codec.h" + #include "rtc_base/checks.h" + + namespace webrtc { ++ ++template ++constexpr int AsInt(const T& t) { ++ if constexpr (std::is_integral_v) { ++ return t; ++ } else { ++ return t.value(); ++ } ++} ++ + template + class UsedIds { + public: +@@ -39,8 +50,8 @@ class UsedIds { + + // Finds and sets an unused id if the `idstruct` id is already in use. + void FindAndSetIdUsed(IdStruct* idstruct) { +- const int original_id = idstruct->id; +- int new_id = idstruct->id; ++ const int original_id = AsInt(idstruct->id); ++ int new_id = AsInt(idstruct->id); + + if (original_id > max_allowed_id_ || original_id < min_allowed_id_) { + // If the original id is not in range - this is an id that can't be diff --git a/3rdparty/webrtc/0009-third_party-protobuf-port-cc-disable-constinit-on-apple.patch b/3rdparty/webrtc/0009-third_party-protobuf-port-cc-disable-constinit-on-apple.patch new file mode 100644 index 00000000000..80567ed60cd --- /dev/null +++ b/3rdparty/webrtc/0009-third_party-protobuf-port-cc-disable-constinit-on-apple.patch @@ -0,0 +1,38 @@ +Subject: [PATCH] third_party/protobuf: disable constinit on Apple in port.cc + +Apple Clang (Xcode 15.4): GlobalEmptyString (std::string) requires heap +allocation in its constructor, which is not a constant expression. +Skip PROTOBUF_CONSTINIT to avoid a "variable does not have a constant +initializer" hard error. This supplements the port_def.inc patch +(0006), which prevents PROTOBUF_CONSTINIT from expanding to +constinit/[[clang::require_constant_initialization]] on Apple, by +directly guarding this specific declaration as an additional safety +measure in case the port_def.inc path alone is insufficient. + +Note: this patch must be applied with the working directory at the +`third_party` git root (not `third_party/protobuf`, which is a plain +subdirectory and not a repository root) -- see apply_webrtc_patches.sh. + +diff --git a/protobuf/src/google/protobuf/port.cc b/protobuf/src/google/protobuf/port.cc +--- a/protobuf/src/google/protobuf/port.cc ++++ b/protobuf/src/google/protobuf/port.cc +@@ -115,9 +115,19 @@ void RealDebugCounter::Register(absl::string_view name) { + } + } + ++#if defined(__APPLE__) ++// Apple Clang (Xcode 15.4): GlobalEmptyString (std::string) requires heap ++// allocation in its constructor, which is not a constant expression. ++// Skip PROTOBUF_CONSTINIT to avoid "variable does not have a constant ++// initializer" hard error. ++PROTOBUF_ATTRIBUTE_NO_DESTROY ++ PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 GlobalEmptyString ++ fixed_address_empty_string{}; ++#else + PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT + PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 GlobalEmptyString + fixed_address_empty_string{}; ++#endif // !defined(__APPLE__) + + } // namespace internal + } // namespace protobuf diff --git a/3rdparty/webrtc/README.md b/3rdparty/webrtc/README.md index 6cf26c47932..241845c3ae3 100644 --- a/3rdparty/webrtc/README.md +++ b/3rdparty/webrtc/README.md @@ -23,8 +23,9 @@ webrtc_common.cmake # Specifies Common WebRTC targets and gn args. (Meth ## Patches -Applied by `apply_webrtc_patches.sh` (each is skipped if it does not apply -cleanly to the pinned WebRTC commit): +Applied by `apply_webrtc_patches.sh` via plain `git apply` (each is skipped +without error if it does not apply cleanly to the pinned WebRTC commit, e.g. +because it was already applied or the fix has landed upstream): ``` 0001-src-enable-rtc_use_cxx11_abi-option.patch # -> src (GCC C++11 ABI) @@ -34,24 +35,25 @@ cleanly to the pinned WebRTC commit): 0004-call-payload_type_picker-gcc-flat_tree.patch # -> src (GCC flat_tree ordering) 0005-build-win-dynamic-crt.patch # -> src/build (Windows /MD[d] runtime) -0006-third_party-protobuf-disable-constinit-on-apple.patch # -> third_party/protobuf (macOS constinit) +0006-third_party-protobuf-disable-constinit-on-apple.patch # -> third_party (macOS constinit, port_def.inc) +0007-p2p-fix-gcc-cxx20-network-changes-meaning.patch # -> src (GCC C++20 Network() name lookup) +0008-pc-fix-gcc-payload-type-ambiguous-conversion.patch # -> src (GCC PayloadType->int ambiguity) +0009-third_party-protobuf-port-cc-disable-constinit-on-apple.patch # -> third_party (macOS constinit, port.cc) ``` -`apply_webrtc_patches.sh` also directly patches -`third_party/protobuf/src/google/protobuf/port.cc` on Apple (via Python -string replacement) to drop `PROTOBUF_CONSTINIT` from the -`fixed_address_empty_string` declaration, which causes a hard error on -Xcode 15.4 because `std::string`'s constructor is not a constant expression. - -`apply_webrtc_patches.sh` also directly patches `p2p/base/port_interface.h` and -`p2p/base/port.h` (via Python string replacement) to qualify the return type -of `Network()` as `::webrtc::Network*` to fix a GCC C++20 "changes meaning" error. - -`0006-build-win-dynamic-crt.patch` adds a `rtc_win_dynamic_crt` gn arg so a +`0005-build-win-dynamic-crt.patch` adds a `rtc_win_dynamic_crt` gn arg so a non-component (static) Windows build can use the dynamic MSVC runtime (`/MD[d]`). This lets Open3D ship a static `libwebrtc` for both `STATIC_WINDOWS_RUNTIME=ON` (`/MT[d]`) and `OFF` (`/MD[d]`). +`0006` and `0009` both touch files under `third_party/protobuf`, which is a +plain subdirectory of the `third_party` checkout, not a git repository root +of its own. They are applied with `third_party` (not `third_party/protobuf`) +as the working directory and use `protobuf/`-prefixed paths accordingly -- +see the comment on `apply_one()` in `apply_webrtc_patches.sh` for why this +matters (`git apply` run from a non-root subdirectory can silently no-op +instead of applying or erroring). + ## Method 1 The pre-compiled WebRTC package used in Method 1 is generated by diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index 2e4263f7cb8..0e48ae92c41 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -9,6 +9,16 @@ WEBRTC_SRC="${2:?WebRTC src path}" # # Args: [required|optional] (default: required) # +# IMPORTANT: must be the *root* of a git checkout (i.e. the directory +# containing .git), not an arbitrary subdirectory. `git apply` run from a +# subdirectory with paths relative to that subdirectory silently no-ops +# ("Skipped patch ..." on stderr, exit 0) instead of applying or erroring, at +# least with the git version used by our CI/dev images. Patches touching +# files under a git-tracked *plain* subdirectory (e.g. third_party/protobuf, +# which is not itself a repository root) must therefore be applied from the +# enclosing repo root (third_party) with paths prefixed accordingly (e.g. +# protobuf/src/...) -- see 0006 and 0009. +# # A patch is considered "already applied" when it applies in reverse; in that # case it is skipped without error so the script is safe to re-run and tolerant # of fixes that have landed upstream. A required patch that neither applies nor @@ -20,9 +30,15 @@ apply_one() { local patch="$1" local dir="$2" local required="${3:-required}" - local name + local name check_output check_rc name="$(basename "$patch")" - if git -C "$dir" apply --check "$patch" 2>/dev/null; then + + # Capture combined output so we can detect git's silent "Skipped patch" + # no-op (see note above), which must NOT be treated as a successful + # check even though it exits 0. + check_output="$(git -C "$dir" apply --check "$patch" 2>&1)" && check_rc=0 || check_rc=$? + + if [[ "$check_rc" -eq 0 && "$check_output" != *"Skipped patch"* ]]; then git -C "$dir" apply "$patch" echo "Applied $name in $dir" elif git -C "$dir" apply --reverse --check "$patch" 2>/dev/null; then @@ -31,131 +47,12 @@ apply_one() { echo "Skip $name (does not apply; optional) in $dir" else echo "ERROR: required patch $name does not apply in $dir." >&2 + [[ -n "$check_output" ]] && echo " $check_output" >&2 echo " Refresh the patch for the pinned WebRTC commit." >&2 exit 1 fi } -# Fix port.cc for Apple Clang (Xcode 15.4): the GlobalEmptyString (std::string) -# variable is declared with PROTOBUF_CONSTINIT which expands to `constinit` or -# [[clang::require_constant_initialization]] on Apple Clang >= 13. Apple's -# libc++ std::string constructor performs a heap allocation, so the variable -# cannot be constant-initialized, producing a hard error. Guard the declaration -# with !defined(__APPLE__) so PROTOBUF_CONSTINIT is omitted on Apple. -# -# This supplements the port_def.inc patch in 0006 and directly targets the -# specific failing declaration (port.cc:120 in the WebRTC M149 protobuf). -fix_protobuf_port_cc_apple() { - local port_cc="${WEBRTC_SRC}/third_party/protobuf/src/google/protobuf/port.cc" - [[ -f "$port_cc" ]] || return 0 - python3 - "$port_cc" <<'PYEOF' -import sys - -path = sys.argv[1] -with open(path, 'r') as f: - content = f.read() - -old = ('PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT\n' - ' PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 GlobalEmptyString\n' - ' fixed_address_empty_string{};') -# Check idempotency: already patched if __APPLE__ guard present near declaration. -if '__APPLE__' in content and 'fixed_address_empty_string' in content: - print(f'Skip {path} (Apple constinit fix already applied)') - sys.exit(0) -if old not in content: - print(f'WARNING: {path}: expected PROTOBUF_CONSTINIT pattern not found; ' - f'Skipping Apple constinit fix', file=sys.stderr) - sys.exit(0) -new = ('#if defined(__APPLE__)\n' - '// Apple Clang (Xcode 15.4): GlobalEmptyString (std::string) requires heap\n' - '// allocation in its constructor, which is not a constant expression.\n' - '// Skip PROTOBUF_CONSTINIT to avoid "variable does not have a constant\n' - '// initializer" hard error.\n' - 'PROTOBUF_ATTRIBUTE_NO_DESTROY\n' - ' PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 GlobalEmptyString\n' - ' fixed_address_empty_string{};\n' - '#else\n' - 'PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT\n' - ' PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 GlobalEmptyString\n' - ' fixed_address_empty_string{};\n' - '#endif // !defined(__APPLE__)') -with open(path, 'w') as f: - f.write(content.replace(old, new, 1)) -print(f'Applied Apple constinit fix in {path}') -PYEOF -} - -# Fix GCC C++20 "changes meaning" error for Network() in WebRTC. -# GCC treats a method name matching a class name as an error. -fix_gcc_cxx20_network_changes_meaning() { - python3 - "${WEBRTC_SRC}/p2p/base/port_interface.h" "${WEBRTC_SRC}/p2p/base/port.h" <<'PYEOF' -import sys -import re -import os - -for path in sys.argv[1:]: - if not os.path.exists(path): - continue - with open(path, 'r') as f: - content = f.read() - - # Replace `Network* Network()` with `::webrtc::Network* Network()` - # but only if it's not already prefixed with `::webrtc::`. - new_content = re.sub(r'(?id can be an int or a PayloadType, so we conditionally unwrap using type traits. -fix_gcc_payload_type_ambiguous() { - local used_ids="${WEBRTC_SRC}/pc/used_ids.h" - [[ -f "$used_ids" ]] || return 0 - python3 - "$used_ids" <<'PYEOF' -import sys -path = sys.argv[1] -with open(path, 'r') as f: - content = f.read() - -helper = """ -template -constexpr int AsInt(const T& t) { - if constexpr (std::is_integral_v) { - return t; - } else { - return t.value(); - } -} - -""" - -# Anchor on `namespace webrtc {` (present in every candidate file) rather -# than a specific #include line, which may not exist in every file this -# fix is applied to (e.g. pc/used_ids.h does not include rtc_export.h). -if 'AsInt(const T& t)' not in content: - content = content.replace('#include ', '#include \n#include ', 1) - content = content.replace('namespace webrtc {\n', 'namespace webrtc {\n' + helper, 1) - -new_content = content.replace('int original_id = idstruct->id;', 'int original_id = AsInt(idstruct->id);') -new_content = new_content.replace('int new_id = idstruct->id;', 'int new_id = AsInt(idstruct->id);') - -if new_content != content: - with open(path, 'w') as f: - f.write(new_content) - print(f'Applied GCC PayloadType ambiguous conversion fix in {path}') -else: - print(f'Skip GCC PayloadType ambiguous conversion fix in {path} (already applied or not found)') -PYEOF -} - PATCH_DIR="$OPEN3D_DIR/3rdparty/webrtc" # Required: declare gn args consumed by args.gn and fix GCC compile errors. apply_one "$PATCH_DIR/0001-src-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC" @@ -164,11 +61,10 @@ apply_one "$PATCH_DIR/0001-third_party-enable-rtc_use_cxx11_abi-option.patch" "$ apply_one "$PATCH_DIR/0002-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0004-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0005-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" -# 0006 patches port_def.inc to prevent PROTOBUF_CONSTINIT from expanding to -# constinit/[[clang::require_constant_initialization]] on Apple. The -# fix_protobuf_port_cc_apple call below directly patches port.cc as an -# additional safety measure in case the port_def.inc path alone is insufficient. -apply_one "$PATCH_DIR/0006-third_party-protobuf-disable-constinit-on-apple.patch" "$WEBRTC_SRC/third_party/protobuf" -fix_protobuf_port_cc_apple -fix_gcc_cxx20_network_changes_meaning -fix_gcc_payload_type_ambiguous +# 0006 and 0009 patch files under third_party/protobuf, a plain subdirectory +# of the third_party checkout (not its own git root) -- apply from +# $WEBRTC_SRC/third_party, not .../third_party/protobuf. See apply_one note. +apply_one "$PATCH_DIR/0006-third_party-protobuf-disable-constinit-on-apple.patch" "$WEBRTC_SRC/third_party" +apply_one "$PATCH_DIR/0009-third_party-protobuf-port-cc-disable-constinit-on-apple.patch" "$WEBRTC_SRC/third_party" +apply_one "$PATCH_DIR/0007-p2p-fix-gcc-cxx20-network-changes-meaning.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0008-pc-fix-gcc-payload-type-ambiguous-conversion.patch" "$WEBRTC_SRC" From 8e1c319f35bbca600fb8ffdb035baf91b629170f Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Mon, 6 Jul 2026 10:49:45 -0700 Subject: [PATCH 30/37] Add actual links and checksums for prebuild webrtc binaries. --- 3rdparty/webrtc/webrtc_download.cmake | 167 ++++++++++++--------- 3rdparty/webrtc/webrtc_fetch_variant.cmake | 40 +++++ 2 files changed, 136 insertions(+), 71 deletions(-) create mode 100644 3rdparty/webrtc/webrtc_fetch_variant.cmake diff --git a/3rdparty/webrtc/webrtc_download.cmake b/3rdparty/webrtc/webrtc_download.cmake index c860eb67d74..fc896e394f7 100644 --- a/3rdparty/webrtc/webrtc_download.cmake +++ b/3rdparty/webrtc/webrtc_download.cmake @@ -5,19 +5,27 @@ include(ExternalProject) set(WEBRTC_VER e8b4d4c) -set(WEBRTC_USE_LOCAL_ARCHIVE OFF) -if(DEFINED ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE} AND NOT "$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}" STREQUAL "") - set(WEBRTC_URL "file://$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}") - set(WEBRTC_USE_LOCAL_ARCHIVE ON) -endif() +set(WEBRTC_BASE_URL + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4) + +# Windows prebuilt WebRTC ships four variants (Debug / Release x /MT[d] // +# MD[d] runtime). Visual Studio (a multi-config generator) selects Debug vs. +# Release at *build* time (`cmake --build . --config ...`), not at CMake +# configure time, so CMAKE_BUILD_TYPE cannot be used to decide which single +# variant to download. For multi-config generators, the STATIC_WINDOWS_RUNTIME +# choice (mt vs md) is still fixed at configure time, but the Debug/Release +# pick is deferred to build time via a `$` generator expression, so +# only the variant actually built is ever downloaded (see the +# add_custom_command below). Single-config generators (Ninja, Makefiles) +# resolve to one variant now, defaulting to Release if CMAKE_BUILD_TYPE is +# unset. +get_property(WEBRTC_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + if (APPLE) - if(NOT WEBRTC_URL) - set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_macos_arm64.tar.gz - ) - endif() - # Update after publishing M149 macOS arm64 artifact from webrtc.yml. - set(WEBRTC_SHA256 PLACEHOLDER_MACOS_ARM64_SHA256) + set(WEBRTC_URL + ${WEBRTC_BASE_URL}/webrtc_${WEBRTC_VER}_macos_arm64.tar.gz + ) + set(WEBRTC_SHA256 3c2592a3bd9efcee591924007857342fec3753e2ae68695baeb2b8774f0e3abc) elseif (WIN32) # Prebuilt WebRTC is a static lib for both MSVC runtimes. A shared Open3D # build links it into open3d.dll and always uses the dynamic runtime, so @@ -28,70 +36,92 @@ elseif (WIN32) "STATIC_WINDOWS_RUNTIME=OFF or BUILD_WEBRTC_FROM_SOURCE=ON.") endif() if(STATIC_WINDOWS_RUNTIME) - set(WEBRTC_RUNTIME_TAG mt) + set(WEBRTC_DEBUG_TAG Debug_mt) + set(WEBRTC_DEBUG_SHA256 b537cce72f758fbcd214fbca80ecfb26228d23c728119abc499c4b333e5f0786) + set(WEBRTC_RELEASE_TAG Release_mt) + set(WEBRTC_RELEASE_SHA256 99e7aabfa38a9ce276f606e461feb7c655771095f8e6b14508985dfa44a4bc3a) else() - set(WEBRTC_RUNTIME_TAG md) + set(WEBRTC_DEBUG_TAG Debug_md) + set(WEBRTC_DEBUG_SHA256 b59fb3cd16eaf022865f39fcd4aa25333f0ff74923b62c62acee4c801431e6fe) + set(WEBRTC_RELEASE_TAG Release_md) + set(WEBRTC_RELEASE_SHA256 25339a82017e2614f3c2a9bfcbb0895a51532aaadeba0035c6ac266c154f03c9) endif() - if(CMAKE_BUILD_TYPE STREQUAL Debug) - set(WEBRTC_CONFIG_TAG Debug) - else() - set(WEBRTC_CONFIG_TAG Release) - endif() - if(NOT WEBRTC_URL) - set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_win_${WEBRTC_CONFIG_TAG}_${WEBRTC_RUNTIME_TAG}.zip - ) + set(WEBRTC_DEBUG_URL + ${WEBRTC_BASE_URL}/webrtc_${WEBRTC_VER}_win_${WEBRTC_DEBUG_TAG}.zip) + set(WEBRTC_RELEASE_URL + ${WEBRTC_BASE_URL}/webrtc_${WEBRTC_VER}_win_${WEBRTC_RELEASE_TAG}.zip) + if(NOT WEBRTC_MULTI_CONFIG) + # Single-config generator: resolve to one variant now, as with the + # other platforms below. + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) + endif() + if(CMAKE_BUILD_TYPE STREQUAL Debug) + set(WEBRTC_URL ${WEBRTC_DEBUG_URL}) + set(WEBRTC_SHA256 ${WEBRTC_DEBUG_SHA256}) + else() + set(WEBRTC_URL ${WEBRTC_RELEASE_URL}) + set(WEBRTC_SHA256 ${WEBRTC_RELEASE_SHA256}) + endif() endif() - # Update after publishing four Windows artifacts from webrtc.yml. - set(WEBRTC_SHA256 PLACEHOLDER_WIN_SHA256) else() # Linux - if(NOT WEBRTC_URL) - if(NOT GLIBCXX_USE_CXX11_ABI) - message(FATAL_ERROR "Pre-built WebRTC with GLIBCXX_USE_CXX11_ABI=OFF is " - "no longer provided. Use GLIBCXX_USE_CXX11_ABI=ON or " - "BUILD_WEBRTC_FROM_SOURCE=ON.") - endif() - set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_linux_cxx-abi-1.tar.gz - ) + if(NOT GLIBCXX_USE_CXX11_ABI) + message(FATAL_ERROR "Pre-built WebRTC with GLIBCXX_USE_CXX11_ABI=OFF is " + "no longer provided. Use GLIBCXX_USE_CXX11_ABI=ON or " + "BUILD_WEBRTC_FROM_SOURCE=ON.") endif() - set(WEBRTC_SHA256 1b529bf448d5abd07ec1f8d310ee5c94bd79e84fe563ae1562420f8e478cc202) + set(WEBRTC_URL + ${WEBRTC_BASE_URL}/webrtc_${WEBRTC_VER}_linux_cxx-abi-1.tar.xz + ) + set(WEBRTC_SHA256 0209f722974fa7b9da9d1c8e279694cf2c4db81e079a325bcadb1b6e0c3b6981) # 1b529bf448d5abd07ec1f8d310ee5c94bd79e84fe563ae1562420f8e478cc202 endif() -if(WEBRTC_SHA256 MATCHES "^PLACEHOLDER") - if(WEBRTC_USE_LOCAL_ARCHIVE) - message(WARNING "WebRTC prebuilt SHA256 not set for this platform (${WEBRTC_SHA256}). " - "Using local OPEN3D_WEBRTC_PREBUILT_ARCHIVE without URL_HASH verification.") - unset(WEBRTC_SHA256) - else() - message(FATAL_ERROR "WebRTC prebuilt SHA256 not set for this platform (${WEBRTC_SHA256}). " - "Update webrtc_download.cmake after publishing the archive, or set " - "OPEN3D_WEBRTC_PREBUILT_ARCHIVE to a verified local file.") - endif() -endif() +if(WIN32 AND WEBRTC_MULTI_CONFIG) + # Multi-config (Visual Studio): don't download both variants up front. + # ExternalProject_Add's URL/URL_HASH are resolved once at configure time + # into a fixed download script, so they can't pick a variant based on + # $. Instead, drive the download from a plain add_custom_command + # whose OUTPUT path embeds $; multi-config generators only build + # the outputs needed for the config actually requested + # (`cmake --build . --config ...`), so only one variant -- the one + # actually built -- ever gets downloaded. + set(WEBRTC_PREBUILT_ROOT "${CMAKE_BINARY_DIR}/webrtc/$") + set(WEBRTC_STAMP "${WEBRTC_PREBUILT_ROOT}/webrtc_fetch.stamp") + add_custom_command( + OUTPUT "${WEBRTC_STAMP}" + COMMAND ${CMAKE_COMMAND} + "-DURL=$,${WEBRTC_DEBUG_URL},${WEBRTC_RELEASE_URL}>" + "-DSHA256=$,${WEBRTC_DEBUG_SHA256},${WEBRTC_RELEASE_SHA256}>" + "-DDEST=${WEBRTC_PREBUILT_ROOT}" + "-DSTAMP=${WEBRTC_STAMP}" + -P "${CMAKE_CURRENT_LIST_DIR}/webrtc_fetch_variant.cmake" + COMMENT "Downloading prebuilt WebRTC ($)" + VERBATIM + ) + add_custom_target(ext_webrtc_all DEPENDS "${WEBRTC_STAMP}") + set(WEBRTC_LIB_DIR "${WEBRTC_PREBUILT_ROOT}/lib") +else() + ExternalProject_Add( + ext_webrtc + PREFIX webrtc + URL ${WEBRTC_URL} + URL_HASH SHA256=${WEBRTC_SHA256} + DOWNLOAD_DIR "${OPEN3D_THIRD_PARTY_DOWNLOAD_DIR}/webrtc" + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + BUILD_BYPRODUCTS "" + ) + ExternalProject_Get_Property(ext_webrtc SOURCE_DIR) + # Prebuilt layout: flat include/ and lib/ at archive root (M149 packages). + set(WEBRTC_PREBUILT_ROOT ${SOURCE_DIR}) + set(WEBRTC_LIB_DIR ${WEBRTC_PREBUILT_ROOT}/lib) -set(_webrtc_url_hash "") -if(WEBRTC_SHA256) - set(_webrtc_url_hash URL_HASH SHA256=${WEBRTC_SHA256}) + add_custom_target(ext_webrtc_all) + add_dependencies(ext_webrtc_all ext_webrtc) endif() -ExternalProject_Add( - ext_webrtc - PREFIX webrtc - URL ${WEBRTC_URL} - ${_webrtc_url_hash} - DOWNLOAD_DIR "${OPEN3D_THIRD_PARTY_DOWNLOAD_DIR}/webrtc" - UPDATE_COMMAND "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - BUILD_BYPRODUCTS "" -) - -ExternalProject_Get_Property(ext_webrtc SOURCE_DIR) -# Prebuilt layout: flat include/ and lib/ at archive root (M149 packages). -set(WEBRTC_PREBUILT_ROOT ${SOURCE_DIR}) - # Variables consumed by find_dependencies.cmake set(WEBRTC_INCLUDE_DIRS ${WEBRTC_PREBUILT_ROOT}/include/ @@ -100,12 +130,7 @@ set(WEBRTC_INCLUDE_DIRS ${WEBRTC_PREBUILT_ROOT}/include/third_party/jsoncpp/generated/ ${WEBRTC_PREBUILT_ROOT}/include/third_party/libyuv/include/ ) -set(WEBRTC_LIB_DIR ${WEBRTC_PREBUILT_ROOT}/lib) set(WEBRTC_LIBRARIES webrtc webrtc_extra ) - -# Dummy target that depends on all WebRTC targets. -add_custom_target(ext_webrtc_all) -add_dependencies(ext_webrtc_all ext_webrtc) diff --git a/3rdparty/webrtc/webrtc_fetch_variant.cmake b/3rdparty/webrtc/webrtc_fetch_variant.cmake new file mode 100644 index 00000000000..4491cebd72b --- /dev/null +++ b/3rdparty/webrtc/webrtc_fetch_variant.cmake @@ -0,0 +1,40 @@ +# Fetches and extracts a single prebuilt WebRTC variant. +# +# Invoked via `cmake -P` from a custom command (see webrtc_download.cmake) +# rather than ExternalProject_Add's URL/URL_HASH, so that the URL can be +# selected lazily, per-config, with a $ generator expression. This +# avoids downloading unused Debug/Release variants on Windows: a Visual +# Studio (multi-config) build only runs this script for whichever config is +# actually being built (`cmake --build . --config ...`), typically just one. +# +# Required -D args: +# URL -- archive URL to download +# SHA256 -- expected sha256 of the archive +# DEST -- extraction directory (recreated on a (re-)download) +# STAMP -- marker file created on success; if it already exists, this +# script is a no-op (nothing to do, matching variant already +# extracted at DEST). +foreach(var URL SHA256 DEST STAMP) + if(NOT DEFINED ${var}) + message(FATAL_ERROR "webrtc_fetch_variant.cmake: -D${var}=... is required") + endif() +endforeach() + +if(EXISTS "${STAMP}") + return() +endif() + +file(REMOVE_RECURSE "${DEST}") +file(MAKE_DIRECTORY "${DEST}") + +get_filename_component(archive_name "${URL}" NAME) +set(archive_path "${DEST}/../${archive_name}") + +message(STATUS "Downloading prebuilt WebRTC: ${URL}") +file(DOWNLOAD "${URL}" "${archive_path}" + EXPECTED_HASH SHA256=${SHA256} + SHOW_PROGRESS +) +file(ARCHIVE_EXTRACT INPUT "${archive_path}" DESTINATION "${DEST}") +file(REMOVE "${archive_path}") +file(WRITE "${STAMP}" "ok") From a66237d4a723539b381447642591cd1659090c37 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Mon, 6 Jul 2026 11:56:04 -0700 Subject: [PATCH 31/37] Correct colorspace handling for YUV - (colors should not be faded now). Fix macOS only has RGBA, not RGB pixel format. --- .../rendering/filament/FilamentRenderer.cpp | 33 +++++++- .../rendering/filament/FilamentRenderer.h | 8 ++ .../webrtc_server/BitmapTrackSource.h | 8 +- .../webrtc_server/ImageCapturer.cpp | 26 ++++-- .../webrtc_server/ImageCapturer.h | 2 +- .../webrtc_server/PeerConnectionManager.cpp | 82 +++++++++---------- .../webrtc_server/PeerConnectionManager.h | 19 ++--- .../webrtc_server/WebRTCWindowSystem.cpp | 10 +-- 8 files changed, 118 insertions(+), 70 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp index 6c6a62a7876..df7423c269e 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp @@ -34,6 +34,8 @@ #pragma warning(pop) #endif // _MSC_VER +#include + #include "open3d/core/Tensor.h" #include "open3d/utility/Logging.h" #include "open3d/visualization/rendering/filament/FilamentCamera.h" @@ -300,6 +302,11 @@ namespace { struct UserData { std::function)> callback; std::shared_ptr image; +#if defined(__APPLE__) + // Metal readPixels only supports RGBA; points at + // FilamentRenderer::read_pixels_rgba_buffer_, stripped to RGB below. + std::vector* rgba_buffer; +#endif UserData(std::function)> cb, std::shared_ptr img) @@ -308,6 +315,16 @@ struct UserData { void ReadPixelsCallback(void*, size_t, void* user) { auto* user_data = static_cast(user); +#if defined(__APPLE__) + const uint8_t* src = user_data->rgba_buffer->data(); + uint8_t* dst = user_data->image->GetDataPtr(); + const int64_t n_pixels = user_data->image->NumElements() / 3; + for (int64_t i = 0; i < n_pixels; ++i) { + dst[i * 3 + 0] = src[i * 4 + 0]; + dst[i * 3 + 1] = src[i * 4 + 1]; + dst[i * 3 + 2] = src[i * 4 + 2]; + } +#endif user_data->callback(user_data->image); delete user_data; } @@ -320,7 +337,6 @@ void FilamentRenderer::RequestReadPixels( std::function)> callback) { core::SizeVector shape{height, width, 3}; core::Dtype dtype = core::UInt8; - int64_t nbytes = shape.NumElements() * dtype.ByteSize(); auto image = std::make_shared(shape, dtype); auto* user_data = new UserData(callback, image); @@ -328,9 +344,24 @@ void FilamentRenderer::RequestReadPixels( using namespace filament; using namespace backend; +#if defined(__APPLE__) + // Metal lacks native RGB readback; reuse a persistent RGBA scratch + // buffer (EndFrame() flushes before the next Draw() reuses it). + const size_t nbytes = static_cast(width) * height * 4; + if (read_pixels_rgba_buffer_.size() != nbytes) { + read_pixels_rgba_buffer_.resize(nbytes); + } + user_data->rgba_buffer = &read_pixels_rgba_buffer_; + PixelBufferDescriptor pd(read_pixels_rgba_buffer_.data(), nbytes, + PixelDataFormat::RGBA, PixelDataType::UBYTE, + ReadPixelsCallback, user_data); +#else + // GL/Vulkan read RGB+UBYTE directly into the tensor. + int64_t nbytes = shape.NumElements() * dtype.ByteSize(); PixelBufferDescriptor pd(image->GetDataPtr(), nbytes, PixelDataFormat::RGB, PixelDataType::UBYTE, ReadPixelsCallback, user_data); +#endif renderer_->readPixels(0, 0, width, height, std::move(pd)); needs_wait_after_draw_ = true; } diff --git a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.h b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.h index 04a19dc95cb..ee142e0c27f 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.h @@ -10,6 +10,9 @@ #include #include #include +#if defined(__APPLE__) +#include +#endif #include "open3d/visualization/rendering/Renderer.h" #include "open3d/visualization/rendering/filament/FilamentEngine.h" @@ -151,6 +154,11 @@ class FilamentRenderer : public Renderer { std::function on_after_draw_; std::function on_apple_gaussian_composite_complete_; bool needs_wait_after_draw_ = false; +#if defined(__APPLE__) + // Scratch buffer for RequestReadPixels' Metal RGBA readback workaround; + // reused across frames and resized only when the frame size changes. + std::vector read_pixels_rgba_buffer_; +#endif }; } // namespace rendering diff --git a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h index 5dd350a95b8..7e39ee2e86b 100644 --- a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h +++ b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h @@ -73,11 +73,13 @@ class BitmapTrackSource : public webrtc::Notifier { bool GetStats(Stats* stats) override { return false; } void AddOrUpdateSink(webrtc::VideoSinkInterface* sink, const webrtc::VideoSinkWants& wants) override; - void RemoveSink(webrtc::VideoSinkInterface* sink) override; + void RemoveSink( + webrtc::VideoSinkInterface* sink) override; bool SupportsEncodedOutput() const override { return false; } void GenerateKeyFrame() override {} - void AddEncodedSink(webrtc::VideoSinkInterface* - sink) override {} + void AddEncodedSink( + webrtc::VideoSinkInterface* sink) + override {} void RemoveEncodedSink( webrtc::VideoSinkInterface* sink) override {} diff --git a/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp b/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp index 4d23c2d933c..b80e07e89b8 100644 --- a/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp +++ b/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp @@ -8,6 +8,7 @@ #include "open3d/visualization/webrtc_server/ImageCapturer.h" #include +#include #include #include #include @@ -55,18 +56,23 @@ void ImageCapturer::OnCaptureResult( webrtc::scoped_refptr i420_buffer = webrtc::I420Buffer::Create(width, height); - // frame->data() - const int conversion_result = libyuv::ConvertToI420( - frame->GetDataPtr(), 0, i420_buffer->MutableDataY(), - i420_buffer->StrideY(), i420_buffer->MutableDataU(), - i420_buffer->StrideU(), i420_buffer->MutableDataV(), - i420_buffer->StrideV(), 0, 0, width, height, i420_buffer->width(), - i420_buffer->height(), libyuv::kRotate0, ::libyuv::FOURCC_RAW); + // Use full-range ("J") conversion to match Open3D's full-range (0-255) + // RGB frames, and tag the color space so VP9 signals this in-band. + const int conversion_result = libyuv::RAWToJ420( + frame->GetDataPtr(), width * 3, + i420_buffer->MutableDataY(), i420_buffer->StrideY(), + i420_buffer->MutableDataU(), i420_buffer->StrideU(), + i420_buffer->MutableDataV(), i420_buffer->StrideV(), width, height); if (conversion_result >= 0) { webrtc::VideoFrame video_frame(i420_buffer, webrtc::VideoRotation::kVideoRotation_0, webrtc::TimeMicros()); + video_frame.set_color_space( + webrtc::ColorSpace(webrtc::ColorSpace::PrimaryID::kSMPTE170M, + webrtc::ColorSpace::TransferID::kSMPTE170M, + webrtc::ColorSpace::MatrixID::kSMPTE170M, + webrtc::ColorSpace::RangeID::kFull)); if ((height_ == 0) && (width_ == 0)) { broadcaster_.OnFrame(video_frame); } else { @@ -84,8 +90,10 @@ void ImageCapturer::OnCaptureResult( stride_uv, stride_uv); scaled_buffer->ScaleFrom( *video_frame.video_frame_buffer()->ToI420()); - webrtc::VideoFrame frame = webrtc::VideoFrame( - scaled_buffer, webrtc::kVideoRotation_0, webrtc::TimeMicros()); + webrtc::VideoFrame frame = + webrtc::VideoFrame(scaled_buffer, webrtc::kVideoRotation_0, + webrtc::TimeMicros()); + frame.set_color_space(*video_frame.color_space()); broadcaster_.OnFrame(frame); } diff --git a/cpp/open3d/visualization/webrtc_server/ImageCapturer.h b/cpp/open3d/visualization/webrtc_server/ImageCapturer.h index b8fc188cc9c..ea55639f53c 100644 --- a/cpp/open3d/visualization/webrtc_server/ImageCapturer.h +++ b/cpp/open3d/visualization/webrtc_server/ImageCapturer.h @@ -11,11 +11,11 @@ #pragma once #include -#include #include #include #include #include +#include #include diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index 338d720c0b6..e6d3b301c98 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -15,13 +15,13 @@ #include "open3d/visualization/webrtc_server/PeerConnectionManager.h" +#include +#include #include -#include #include #include #include -#include -#include +#include #include #include #include @@ -93,16 +93,16 @@ static IceServer GetIceServerFromUrl(const std::string &url) { } static bool PeerConnectionHasStreamForWindow( - webrtc::PeerConnectionInterface* peer_connection, - const std::string& window_uid) { + webrtc::PeerConnectionInterface *peer_connection, + const std::string &window_uid) { if (!peer_connection) { return false; } - for (const auto& sender : peer_connection->GetSenders()) { + for (const auto &sender : peer_connection->GetSenders()) { if (!sender) { continue; } - for (const std::string& stream_id : sender->stream_ids()) { + for (const std::string &stream_id : sender->stream_ids()) { if (stream_id == window_uid) { return true; } @@ -112,8 +112,7 @@ static bool PeerConnectionHasStreamForWindow( } static webrtc::PeerConnectionFactoryDependencies -CreatePeerConnectionFactoryDependencies( - webrtc::FieldTrials* field_trials) { +CreatePeerConnectionFactoryDependencies(webrtc::FieldTrials *field_trials) { webrtc::PeerConnectionFactoryDependencies dependencies; dependencies.worker_thread = webrtc::Thread::Current(); dependencies.network_thread = nullptr; @@ -276,12 +275,12 @@ const Json::Value PeerConnectionManager::AddIceCandidate( int sdp_mlineindex = 0; std::string sdp; if (!webrtc::GetStringFromJsonObject(json_message, k_candidate_sdp_mid_name, - &sdp_mid) || + &sdp_mid) || !webrtc::GetIntFromJsonObject(json_message, - k_candidate_sdp_mline_index_name, - &sdp_mlineindex) || + k_candidate_sdp_mline_index_name, + &sdp_mlineindex) || !webrtc::GetStringFromJsonObject(json_message, k_candidate_sdp_name, - &sdp)) { + &sdp)) { utility::LogWarning("Can't parse received message."); } else { std::unique_ptr candidate( @@ -332,10 +331,10 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, std::string type; std::string sdp; - if (!webrtc::GetStringFromJsonObject(json_message, - k_session_description_type_name, &type) || - !webrtc::GetStringFromJsonObject(json_message, - k_session_description_sdp_name, &sdp)) { + if (!webrtc::GetStringFromJsonObject( + json_message, k_session_description_type_name, &type) || + !webrtc::GetStringFromJsonObject( + json_message, k_session_description_sdp_name, &sdp)) { utility::LogWarning("Can't parse received message."); } else { PeerConnectionObserver *peer_connection_observer = @@ -346,7 +345,7 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, utility::LogError("Failed to initialize PeerConnection"); delete peer_connection_observer; } else { - webrtc::PeerConnectionInterface* peer_connection_ptr = + webrtc::PeerConnectionInterface *peer_connection_ptr = peer_connection_observer->GetPeerConnection().get(); utility::LogDebug("nbSenders: {}, nbReceivers: {}", peer_connection_ptr->GetSenders().size(), @@ -373,7 +372,8 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, std::unique_ptr session_description; if (!sdp_type) { - utility::LogError("Unknown session description type: {}.", type); + utility::LogError("Unknown session description type: {}.", + type); } else { session_description = webrtc::CreateSessionDescription(*sdp_type, sdp); @@ -383,7 +383,7 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, "Can't parse received session description message. " "Cannot create session description."); } else { - std::promise + std::promise remote_promise; peer_connection_ptr->SetRemoteDescription( SetSessionDescriptionObserver::Create( @@ -530,20 +530,21 @@ bool PeerConnectionManager::InitializePeerConnection() { } PeerConnectionManager::PeerConnectionObserver::PeerConnectionObserver( - PeerConnectionManager* peer_connection_manager, - const std::string& peerid) + PeerConnectionManager *peer_connection_manager, + const std::string &peerid) : peer_connection_manager_(peer_connection_manager), peerid_(peerid), local_channel_(nullptr), remote_channel_(nullptr), ice_candidate_list_(Json::arrayValue), deleting_(false) { - stats_callback_ = - new webrtc::RefCountedObject(); + stats_callback_ = new webrtc::RefCountedObject< + PeerConnectionStatsCollectorCallback>(); } void PeerConnectionManager::PeerConnectionObserver::Initialize( - webrtc::scoped_refptr peer_connection) { + webrtc::scoped_refptr + peer_connection) { pc_ = peer_connection; if (pc_.get()) { auto channel_result = @@ -556,8 +557,8 @@ void PeerConnectionManager::PeerConnectionObserver::Initialize( } // Create a new PeerConnection. -PeerConnectionManager::PeerConnectionObserver* -PeerConnectionManager::CreatePeerConnection(const std::string& peerid) { +PeerConnectionManager::PeerConnectionObserver * +PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { webrtc::PeerConnectionInterface::RTCConfiguration config; // Max bundle multiplexes all media and data channels on a single transport, // eliminating separate ICE/DTLS handshakes per track and reducing latency. @@ -590,8 +591,7 @@ PeerConnectionManager::CreatePeerConnection(const std::string& peerid) { max_port); utility::LogDebug("CreatePeerConnection peerid: {}.", peerid); - PeerConnectionObserver* obs = - new PeerConnectionObserver(this, peerid); + PeerConnectionObserver *obs = new PeerConnectionObserver(this, peerid); webrtc::PeerConnectionDependencies dependencies(obs); auto pc_result = peer_connection_factory_->CreatePeerConnectionOrError( config, std::move(dependencies)); @@ -700,13 +700,14 @@ bool PeerConnectionManager::AddStreams( } if (video_track) { - webrtc::RTCErrorOr> - add_result = peer_connection->AddTrack( - video_track, {window_uid}); + webrtc::RTCErrorOr< + webrtc::scoped_refptr> + add_result = peer_connection->AddTrack(video_track, + {window_uid}); if (!add_result.ok()) { - utility::LogError("Adding track to PeerConnection failed: {}", - add_result.error().message()); + utility::LogError( + "Adding track to PeerConnection failed: {}", + add_result.error().message()); } else { utility::LogDebug("Track added to PeerConnection."); ret = true; @@ -790,12 +791,12 @@ void PeerConnectionManager::EncoderThreadLoop() { // encode only the latest per window (implicit frame coalescing). snapshot = std::move(pending_frames_); } - webrtc::Thread* worker = webrtc_worker_thread_; + webrtc::Thread *worker = webrtc_worker_thread_; if (!worker) { continue; } - for (const auto& kv : snapshot) { - const std::shared_ptr& frame = kv.second; + for (const auto &kv : snapshot) { + const std::shared_ptr &frame = kv.second; if (!frame) { continue; } @@ -804,9 +805,8 @@ void PeerConnectionManager::EncoderThreadLoop() { if (!track_source) { continue; } - worker->PostTask([track_source, frame]() { - track_source->OnFrame(frame); - }); + worker->PostTask( + [track_source, frame]() { track_source->OnFrame(frame); }); } } } diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h index 2c85a41a40e..dbed03d34cf 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h @@ -19,9 +19,9 @@ #pragma once #include -#include #include #include +#include #include #include @@ -144,8 +144,8 @@ class PeerConnectionManager { webrtc::PeerConnectionInterface* pc, std::promise& promise) { - return new webrtc::RefCountedObject( - pc, promise); + return new webrtc::RefCountedObject< + CreateSessionDescriptionObserver>(pc, promise); } virtual void OnSuccess(webrtc::SessionDescriptionInterface* desc) { std::string sdp; @@ -195,10 +195,10 @@ class PeerConnectionManager { class DataChannelObserver : public webrtc::DataChannelObserver { public: - DataChannelObserver( - PeerConnectionManager* peer_connection_manager, - webrtc::scoped_refptr data_channel, - const std::string& peerid) + DataChannelObserver(PeerConnectionManager* peer_connection_manager, + webrtc::scoped_refptr + data_channel, + const std::string& peerid) : peer_connection_manager_(peer_connection_manager), data_channel_(data_channel), peerid_(peerid) { @@ -267,9 +267,8 @@ class PeerConnectionManager { PeerConnectionObserver(PeerConnectionManager* peer_connection_manager, const std::string& peerid); - void Initialize( - webrtc::scoped_refptr - peer_connection); + void Initialize(webrtc::scoped_refptr + peer_connection); virtual ~PeerConnectionObserver() { delete local_channel_; diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index e2d87b60169..1c011f26c22 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -7,12 +7,12 @@ #include "open3d/visualization/webrtc_server/WebRTCWindowSystem.h" -#include #include +#include #include -#include #include +#include #include #include #include @@ -103,7 +103,7 @@ struct WebRTCWindowSystem::Impl { std::thread webrtc_thread_; bool sever_started_ = false; // Set while the WebRTC std::thread is inside Run(); used for shutdown. - std::atomic webrtc_message_thread_{nullptr}; + std::atomic webrtc_message_thread_{nullptr}; std::unordered_map> data_channel_message_callbacks_; @@ -199,7 +199,7 @@ WebRTCWindowSystem::WebRTCWindowSystem() WebRTCWindowSystem::~WebRTCWindowSystem() { if (impl_->sever_started_ && impl_->webrtc_thread_.joinable()) { - webrtc::Thread* message_thread = impl_->webrtc_message_thread_.load(); + webrtc::Thread *message_thread = impl_->webrtc_message_thread_.load(); if (message_thread) { message_thread->Quit(); } @@ -284,7 +284,7 @@ void WebRTCWindowSystem::StartWebRTCServer() { webrtc::ThreadManager::Instance()->UnwrapCurrentThread(); } } webrtc_thread_scope; - webrtc::Thread* thread = webrtc::Thread::Current(); + webrtc::Thread *thread = webrtc::Thread::Current(); impl_->webrtc_message_thread_.store(thread); webrtc::InitializeSSL(); From f4400b7ff5baf83cd6f22591866c57d66aaca423 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Mon, 6 Jul 2026 14:00:41 -0700 Subject: [PATCH 32/37] CI fixes --- .github/workflows/macos.yml | 7 ++++-- 3rdparty/find_dependencies.cmake | 23 +++++++++++++++++-- CMakeLists.txt | 4 ---- .../webrtc_server/PeerConnectionManager.h | 2 +- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 495049bbb35..20d12e5e0ed 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -67,7 +67,8 @@ jobs: - name: Install dependencies run: | brew install ccache glslang spirv-cross - # Fix gfortran not found issue + # Fix gfortran not found issue. + brew install gcc ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran ccache -M 2G # See .github/workflows/readme.md for ccache strategy. @@ -202,7 +203,9 @@ jobs: install_python_dependencies # Fix macos-14 arm64 runner image issues, see comments in MacOS job. - ln -s $(which gfortran-13) /usr/local/bin/gfortran + # Use a glob instead of a hardcoded gfortran- version, since the + # runner's default gcc/gfortran version can change over time. + ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran brew install ccache glslang spirv-cross ccache -M 2G # See .github/workflows/readme.md for ccache strategy. diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index 3c7456fa4d8..7a1dc143172 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -1990,16 +1990,35 @@ if(BUILD_WEBRTC) INCLUDE_DIRS ${WEBRTC_INCLUDE_DIRS} DEPENDS ext_webrtc_all ) + # webrtc/webrtc_extra need custom --whole-archive handling (below), so + # they can't use open3d_import_3rdparty_library()'s LIBRARIES option. + # Install them manually and reference $ paths, so + # examples built against an installed *static* Open3D package (i.e. not + # from within this build tree) still link against them; see the + # LIBRARIES branch of open3d_import_3rdparty_library() for reference. + if(NOT BUILD_SHARED_LIBS) + foreach(_o3d_webrtc_lib webrtc webrtc_extra) + install(FILES "${WEBRTC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_o3d_webrtc_lib}${CMAKE_STATIC_LIBRARY_SUFFIX}" + DESTINATION ${Open3D_INSTALL_LIB_DIR} + RENAME "${CMAKE_STATIC_LIBRARY_PREFIX}${PROJECT_NAME}_3rdparty_webrtc_${_o3d_webrtc_lib}${CMAKE_STATIC_LIBRARY_SUFFIX}") + endforeach() + endif() + set(WEBRTC_INSTALLED_LIB + "$/${Open3D_INSTALL_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROJECT_NAME}_3rdparty_webrtc") if(UNIX AND NOT APPLE) target_link_libraries(3rdparty_webrtc INTERFACE "-Wl,--whole-archive" "$" + "$" "-Wl,--no-whole-archive" - "$") + "$" + "$") else() target_link_libraries(3rdparty_webrtc INTERFACE "$" - "$") + "$" + "$" + "$") endif() target_link_libraries(3rdparty_webrtc INTERFACE Open3D::3rdparty_threads ${CMAKE_DL_LIBS}) # libwebrtc.a and libturbojpeg.a both export jpeg_* symbols (WebRTC bundles libjpeg). diff --git a/CMakeLists.txt b/CMakeLists.txt index 3283c8e8b21..ba39ca4ef02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,10 +223,6 @@ if(ENABLE_HEADLESS_RENDERING AND BUILD_GUI) message(WARNING "Headless rendering disables the Open3D GUI") set(BUILD_GUI OFF) endif() -if(APPLE AND NOT BUILD_FILAMENT_FROM_SOURCE AND NOT FILAMENT_PRECOMPILED_ROOT) - message(WARNING - "Gaussian splatting requires Filament >= 1.57.2 backend Metal platform headers. Set FILAMENT_PRECOMPILED_ROOT to a compatible release package, or enable BUILD_FILAMENT_FROM_SOURCE.") -endif() if(APPLE) find_program(XCRUN_EXECUTABLE xcrun) if(NOT XCRUN_EXECUTABLE) diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h index dbed03d34cf..d80db8967ee 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h @@ -491,7 +491,7 @@ struct formatter { default: text = "unknown"; } - return format_to(ctx.out(), "{}", text); + return fmt::format_to(ctx.out(), "{}", text); } template From 3c2834e329e00dfee46aa9e231f74d87d0f59a7f Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Mon, 6 Jul 2026 21:35:53 -0700 Subject: [PATCH 33/37] Fix macOS wheel build and Codacy quality-gate CI failures - build-wheel job in macos.yml was missing `brew install gcc` before resolving `brew --prefix gcc`, so the gfortran symlink silently pointed at a literal, unexpanded glob string and LAPACK's configure step failed with "gfortran is required to compile LAPACK from source." Add the missing `brew install gcc`, matching the MacOS job. - Reduce PeerConnectionManager::HangUp's hangup_window_uid to the scope it is actually used in (cppcheck_variableScope, the one genuine new Codacy issue in the PR's diff). The other new Codacy issues are all cppcheck_missingIncludeSystem false positives (its sandbox can't see the external WebRTC/Filament include paths); left unsuppressed as noise rather than adding per-file cppcheck-suppress-file comments. Verified locally: full local build (cmake --build . --target Open3D) succeeds, and util/check_style.py passes. The other three failing checks (Ubuntu wheel build, ml-noble build, Windows CUDA Release build) are unrelated CI infra flakes/known perf limits (HTTP 429 rate limit, SSH key propagation race, and a pre-existing Windows CUDA Release build time issue that already affects other PRs) and were left for a re-run. Co-authored-by: Cursor --- .github/workflows/macos.yml | 4 ++++ .../visualization/webrtc_server/PeerConnectionManager.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 20d12e5e0ed..18449ab0cd3 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -203,8 +203,12 @@ jobs: install_python_dependencies # Fix macos-14 arm64 runner image issues, see comments in MacOS job. + # brew install gcc is required so that `brew --prefix gcc` resolves; + # without it, the symlink below silently points to a literal, + # unexpanded glob string and gfortran remains "not found" later. # Use a glob instead of a hardcoded gfortran- version, since the # runner's default gcc/gfortran version can change over time. + brew install gcc ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran brew install ccache glslang spirv-cross diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index e6d3b301c98..26719bd0905 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -456,8 +456,8 @@ bool PeerConnectionManager::WindowStillUsed(const std::string &window_uid) { const Json::Value PeerConnectionManager::HangUp(const std::string &peerid) { bool result = false; PeerConnectionObserver *pc_observer = nullptr; - std::string hangup_window_uid; { + std::string hangup_window_uid; std::lock_guard mutex_lock(peerid_to_connection_mutex_); auto it = peerid_to_connection_.find(peerid); if (it != peerid_to_connection_.end()) { From 9ce764761fb61a691f588ca13da04acd72672998 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 7 Jul 2026 22:05:53 -0700 Subject: [PATCH 34/37] Skip setup for video transmit from browser. reduce comments. --- .github/workflows/macos.yml | 4 - .github/workflows/webrtc.yml | 5 +- .github/workflows/windows.yml | 6 -- 3rdparty/webrtc/apply_webrtc_patches.sh | 11 ++- 3rdparty/webrtc/webrtc_build.sh | 7 +- 3rdparty/webrtc/webrtc_download.cmake | 29 +------ 3rdparty/webrtc/webrtc_fetch_variant.cmake | 13 +-- AGENTS.md | 2 +- .../visualization/gui/BitmapWindowSystem.cpp | 24 ++---- .../rendering/filament/FilamentRenderer.h | 1 - .../webrtc_server/html/webrtcstreamer.js | 84 ++++++------------- python/js/package.json | 2 +- 12 files changed, 50 insertions(+), 138 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 18449ab0cd3..48bde94a366 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -204,10 +204,6 @@ jobs: # Fix macos-14 arm64 runner image issues, see comments in MacOS job. # brew install gcc is required so that `brew --prefix gcc` resolves; - # without it, the symlink below silently points to a literal, - # unexpanded glob string and gfortran remains "not found" later. - # Use a glob instead of a hardcoded gfortran- version, since the - # runner's default gcc/gfortran version can change over time. brew install gcc ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 6eb3ab1ac70..060831645fb 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -120,10 +120,7 @@ jobs: arch: x64 - name: Download WebRTC sources - # shell: bash uses Git Bash on Windows, which transparently converts - # Windows-style env paths (e.g. OPEN3D_DIR, WEBRTC_WORK_ROOT) so they - # work in bash string and pushd contexts. - shell: bash + shell: bash # Git Bash transparently converts Windows-style env paths run: | source "$OPEN3D_DIR/3rdparty/webrtc/webrtc_build.sh" download_webrtc_sources diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 0fb4e04deee..0b0882b8b6a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -56,10 +56,6 @@ jobs: - BUILD_CUDA_MODULE: ON # FIXME CONFIG: Debug env: - # WebRTC prebuilt ships static libs for both MSVC runtimes (/MT, /MD). - # Enable it for static-lib + static-runtime (/MT) and shared-lib + - # dynamic-runtime (/MD). Shared libs always use the dynamic runtime. - BUILD_WEBRTC: ${{ ( ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) || ( matrix.BUILD_SHARED_LIBS == 'ON' && matrix.STATIC_RUNTIME == 'OFF' ) ) && 'ON' || 'OFF' }} BUILD_PYTORCH_OPS: ${{ ( matrix.BUILD_CUDA_MODULE == 'ON' || matrix.CONFIG == 'Debug' ) && 'OFF' || 'ON' }} # FIXME steps: @@ -148,7 +144,6 @@ jobs: -DSTATIC_WINDOWS_RUNTIME=${{ matrix.STATIC_RUNTIME }} ` -DBUILD_COMMON_ISPC_ISAS=ON ` -DBUILD_LIBREALSENSE=ON ` - -DBUILD_WEBRTC=${{ env.BUILD_WEBRTC }} ` -DBUILD_UNIT_TESTS=ON ` -DBUILD_CUDA_MODULE=${{ matrix.BUILD_CUDA_MODULE }} ` -DBUILD_PYTORCH_OPS=${{ env.BUILD_PYTORCH_OPS }} ` @@ -364,7 +359,6 @@ jobs: -DBUILD_COMMON_ISPC_ISAS=ON ` -DBUILD_AZURE_KINECT=ON ` -DBUILD_LIBREALSENSE=ON ` - -DBUILD_WEBRTC=ON ` -DBUILD_JUPYTER_EXTENSION=ON ` -DBUILD_PYTORCH_OPS=${{ env.BUILD_PYTORCH_OPS }} ` ${{ env.SRC_DIR }} diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index 0e48ae92c41..6bb7ead8a4d 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -12,12 +12,11 @@ WEBRTC_SRC="${2:?WebRTC src path}" # IMPORTANT: must be the *root* of a git checkout (i.e. the directory # containing .git), not an arbitrary subdirectory. `git apply` run from a # subdirectory with paths relative to that subdirectory silently no-ops -# ("Skipped patch ..." on stderr, exit 0) instead of applying or erroring, at -# least with the git version used by our CI/dev images. Patches touching -# files under a git-tracked *plain* subdirectory (e.g. third_party/protobuf, -# which is not itself a repository root) must therefore be applied from the -# enclosing repo root (third_party) with paths prefixed accordingly (e.g. -# protobuf/src/...) -- see 0006 and 0009. +# ("Skipped patch ..." on stderr, exit 0) instead of applying or erroring. +# Patches touching files under a git-tracked *plain* subdirectory (e.g. +# third_party/protobuf, which is not itself a repository root) must therefore be +# applied from the enclosing repo root (third_party) with paths prefixed +# accordingly (e.g. protobuf/src/...) -- see 0006 and 0009. # # A patch is considered "already applied" when it applies in reverse; in that # case it is skipped without error so the script is safe to re-run and tolerant diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 70443e2caf8..952dc3f4388 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -49,12 +49,7 @@ webrtc_work_root() { webrtc_setup_path() { local root root="$(webrtc_work_root)" - # On Windows the work root is a native path like 'C:\WebRTC'. The colon in - # the drive letter is bash's PATH separator, which would split - # 'C:\WebRTC/depot_tools' into the two entries 'C' and '\WebRTC/depot_tools'. - # '\WebRTC/...' then causes sha256sum to prefix its output with '\' (GNU - # coreutils escapes paths that contain backslashes), making the CIPD hash - # check fail. cygpath is available in Git Bash / MSYS2; convert to POSIX. + # On Windows: C:\WebRTC -> /c/WebRTC. (colon is problematic in bash paths.) if command -v cygpath >/dev/null 2>&1; then root="$(cygpath -u "$root")" fi diff --git a/3rdparty/webrtc/webrtc_download.cmake b/3rdparty/webrtc/webrtc_download.cmake index fc896e394f7..0820579de9e 100644 --- a/3rdparty/webrtc/webrtc_download.cmake +++ b/3rdparty/webrtc/webrtc_download.cmake @@ -8,17 +8,9 @@ set(WEBRTC_VER e8b4d4c) set(WEBRTC_BASE_URL https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4) -# Windows prebuilt WebRTC ships four variants (Debug / Release x /MT[d] // -# MD[d] runtime). Visual Studio (a multi-config generator) selects Debug vs. -# Release at *build* time (`cmake --build . --config ...`), not at CMake -# configure time, so CMAKE_BUILD_TYPE cannot be used to decide which single -# variant to download. For multi-config generators, the STATIC_WINDOWS_RUNTIME -# choice (mt vs md) is still fixed at configure time, but the Debug/Release -# pick is deferred to build time via a `$` generator expression, so -# only the variant actually built is ever downloaded (see the -# add_custom_command below). Single-config generators (Ninja, Makefiles) -# resolve to one variant now, defaulting to Release if CMAKE_BUILD_TYPE is -# unset. +# Windows: four prebuilt variants (Debug/Release x /MT[d]/MD[d]). Multi-config +# generators pick Debug vs Release at build time via $; single-config +# generators download one variant at configure time (Release if unset). get_property(WEBRTC_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) if (APPLE) @@ -27,9 +19,6 @@ if (APPLE) ) set(WEBRTC_SHA256 3c2592a3bd9efcee591924007857342fec3753e2ae68695baeb2b8774f0e3abc) elseif (WIN32) - # Prebuilt WebRTC is a static lib for both MSVC runtimes. A shared Open3D - # build links it into open3d.dll and always uses the dynamic runtime, so - # only BUILD_SHARED_LIBS=ON + STATIC_WINDOWS_RUNTIME=ON is unsupported. if (BUILD_SHARED_LIBS AND STATIC_WINDOWS_RUNTIME) message(FATAL_ERROR "Pre-built WebRTC does not support " "BUILD_SHARED_LIBS=ON with STATIC_WINDOWS_RUNTIME=ON. Use " @@ -53,9 +42,6 @@ elseif (WIN32) if(NOT WEBRTC_MULTI_CONFIG) # Single-config generator: resolve to one variant now, as with the # other platforms below. - if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release) - endif() if(CMAKE_BUILD_TYPE STREQUAL Debug) set(WEBRTC_URL ${WEBRTC_DEBUG_URL}) set(WEBRTC_SHA256 ${WEBRTC_DEBUG_SHA256}) @@ -77,14 +63,7 @@ else() # Linux endif() if(WIN32 AND WEBRTC_MULTI_CONFIG) - # Multi-config (Visual Studio): don't download both variants up front. - # ExternalProject_Add's URL/URL_HASH are resolved once at configure time - # into a fixed download script, so they can't pick a variant based on - # $. Instead, drive the download from a plain add_custom_command - # whose OUTPUT path embeds $; multi-config generators only build - # the outputs needed for the config actually requested - # (`cmake --build . --config ...`), so only one variant -- the one - # actually built -- ever gets downloaded. + # ExternalProject_Add cannot vary URL per config; use add_custom_command + webrtc_fetch_variant.cmake. set(WEBRTC_PREBUILT_ROOT "${CMAKE_BINARY_DIR}/webrtc/$") set(WEBRTC_STAMP "${WEBRTC_PREBUILT_ROOT}/webrtc_fetch.stamp") add_custom_command( diff --git a/3rdparty/webrtc/webrtc_fetch_variant.cmake b/3rdparty/webrtc/webrtc_fetch_variant.cmake index 4491cebd72b..2addaba0ff9 100644 --- a/3rdparty/webrtc/webrtc_fetch_variant.cmake +++ b/3rdparty/webrtc/webrtc_fetch_variant.cmake @@ -1,19 +1,12 @@ -# Fetches and extracts a single prebuilt WebRTC variant. -# -# Invoked via `cmake -P` from a custom command (see webrtc_download.cmake) -# rather than ExternalProject_Add's URL/URL_HASH, so that the URL can be -# selected lazily, per-config, with a $ generator expression. This -# avoids downloading unused Debug/Release variants on Windows: a Visual -# Studio (multi-config) build only runs this script for whichever config is -# actually being built (`cmake --build . --config ...`), typically just one. +# Cmake script to fetch and extract a single prebuilt WebRTC variant at build +# time by multi-config generators (Visual Studio). # # Required -D args: # URL -- archive URL to download # SHA256 -- expected sha256 of the archive # DEST -- extraction directory (recreated on a (re-)download) # STAMP -- marker file created on success; if it already exists, this -# script is a no-op (nothing to do, matching variant already -# extracted at DEST). +# script is a no-op. foreach(var URL SHA256 DEST STAMP) if(NOT DEFINED ${var}) message(FATAL_ERROR "webrtc_fetch_variant.cmake: -D${var}=... is required") diff --git a/AGENTS.md b/AGENTS.md index 2d54639d388..f4739e04e60 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ functionality is out of scope. - Keep changes focused and small. Avoid broad refactors unless requested. - Read the relevant C++ / Python / docs files together and identify whether bindings, docs, and tests must change with the source change. - **Debugging:** Test root cause hypothesis with logging before trying fixes. Validate with logging afterwards. Undo failed fixes. -- Developer docs: Document the code with brief comments (Why and What is the code doing?), typically for each function and file. Ensure code and docs are consistent. +- Developer docs: Document the code with brief (2-3 lines) comments (Why and What is the code doing?), typically for each function and file. Ensure code and docs are consistent. - User docs: Update `docs`, Doxygen docs in C++ headers and Google Sphinx RST docs in Python bindings for new / changed code behavior. Add / update an example function use snippet in the docs. - For new functionality, docs and examples, prefer Tensor implementations that work on CPU+CUDA+SYCL. - Use the Eigen library for Math operations and oneAPI TBB for multithreading. Avoid: OpenMP, stdgpu. diff --git a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp index 5467157d34d..27dd228cacb 100644 --- a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp +++ b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp @@ -45,8 +45,6 @@ struct BitmapEvent { virtual void Execute() = 0; }; -// Forward declaration: BitmapDrawEvent needs a pointer to BitmapEventQueue -// to clear the per-window pending-draw flag before calling OnDraw(). struct BitmapEventQueue; struct BitmapDrawEvent : public BitmapEvent { @@ -105,23 +103,20 @@ struct BitmapTextInputEvent : public BitmapEvent { } }; -/// Thread safe event queue (multiple producers and consumers). -/// pop_front() and push() are protected by a mutex. -/// push() may fail if the mutex cannot be acquired immediately. -/// empty() is not protected and is not reliable. +/// Thread safe event queue (multiple producers and consumers). pop_front() and +/// push() are protected by a mutex. push() may fail if the mutex cannot be +/// acquired immediately. empty() is not protected and is not reliable. /// -/// Extended to support: +/// Also supports: /// - Draw coalescing: at most one pending draw per window (push_draw). -/// - Input coalescing: MOVE/DRAG replace latest, WHEEL accumulates -/// dx/dy (replace_or_merge_mouse). Old mouse positions are stale; -/// processing them forces stale frames to be encoded and sent. +/// - Input coalescing: MOVE/DRAG replace latest, WHEEL accumulates dx/dy +/// (replace_or_merge_mouse). Old mouse positions are stale and discarded. struct BitmapEventQueue : public std::queue> { using value_t = std::shared_ptr; using super = std::queue; using super::empty; // not reliable using super::super; - // pop + front needs to be atomic for thread safety. This is exception safe // since shared_ptr copy ctor is noexcept, when it is returned by value. value_t pop_front() { @@ -130,7 +125,6 @@ struct BitmapEventQueue : public std::queue> { super::pop(); return evt; } - void push(const value_t &event) { if (evt_q_mutex_.try_lock()) { super::push(event); @@ -171,11 +165,9 @@ struct BitmapEventQueue : public std::queue> { } // For MOVE/DRAG: replace the last queued event of the same (target, type) - // with the new event (latest absolute position wins; camera controllers - // derive delta from absolute coords so intermediate positions are useless). + // with the new event (latest absolute position wins). // For WHEEL: accumulate wheel.dx/dy into the last queued event of the same - // (target, type) so that the total scroll amount is preserved even when - // multiple notches fire faster than the render loop. + // (target, type) so that the total scroll amount is preserved. // Falls back to a normal push when no matching event is at the back. void replace_or_merge_mouse(const value_t &event) { std::lock_guard lock(evt_q_mutex_); diff --git a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.h b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.h index ee142e0c27f..c10dcd83605 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.h @@ -156,7 +156,6 @@ class FilamentRenderer : public Renderer { bool needs_wait_after_draw_ = false; #if defined(__APPLE__) // Scratch buffer for RequestReadPixels' Metal RGBA readback workaround; - // reused across frames and resized only when the frame size changes. std::vector read_pixels_rgba_buffer_; #endif }; diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index 24a9b80b2a3..4bce754e895 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -50,9 +50,9 @@ let WebRtcStreamer = (function() { this.onClose = onClose; this.commsFetch = commsFetch; - // Pending coalesced pointer/wheel events. A single requestAnimationFrame - // flushes both at most once per browser frame (~60 Hz), preventing a - // backlog of stale events from queuing up on the server. + // Pending coalesced pointer/wheel events. A single + // requestAnimationFrame flushes both at most once per browser frame + // (~60 Hz). this.pendingPointerEvent = null; // MOVE or DRAG (latest wins) this.pendingWheelEvent = null; // WHEEL (dx/dy accumulated) this.rafPending = false; @@ -138,10 +138,8 @@ let WebRtcStreamer = (function() { * e.g. window_0. * @param {string} audiourl Od of WebRTC audio stream * @param {string} options Options of WebRTC call - * @param {string} stream Local stream to send */ - WebRtcStreamer.prototype.connect = function( - videourl, audiourl, options, localstream) { + WebRtcStreamer.prototype.connect = function(videourl, audiourl, options) { this.disconnect(); // getIceServers is not already received @@ -156,12 +154,11 @@ let WebRtcStreamer = (function() { .then((response) => response.json()) .then((response) => logAndReturn(response)) .then((response) => this.onReceiveGetIceServers.call( - this, response, videourl, audiourl, options, - localstream)) + this, response, videourl, audiourl, options)) .catch((error) => this.onError('getIceServers ' + error)); } else { this.onReceiveGetIceServers( - this.iceServers, videourl, audiourl, options, localstream); + this.iceServers, videourl, audiourl, options); } // Set callback functions. @@ -480,7 +477,7 @@ let WebRtcStreamer = (function() { * GetIceServers callback */ WebRtcStreamer.prototype.onReceiveGetIceServers = function( - iceServers, videourl, audiourl, options, stream) { + iceServers, videourl, audiourl, options) { this.iceServers = iceServers; this.pcConfig = iceServers || {iceServers: []}; try { @@ -495,47 +492,25 @@ let WebRtcStreamer = (function() { callurl += '&options=' + encodeURIComponent(options); } - if (stream) { - this.pc.addStream(stream); - } - - // Prefer VP9 via the standard setCodecPreferences() API. - // Must be called on the transceiver BEFORE createOffer(). - // For receive-only mode (no local stream), we create the video - // transceiver explicitly — offerToReceiveVideo creates it lazily - // inside createOffer(), which is too late to set preferences. - // Falls back silently to VP8 if the browser does not support the - // API or if VP9 is not in the browser's capabilities. + // Prefer VP9 on recv-only video (before createOffer). Sending is + // C++. var pc = this.pc; - if (typeof RTCRtpReceiver !== 'undefined' && - RTCRtpReceiver.getCapabilities && pc.addTransceiver) { + if (pc.addTransceiver && typeof RTCRtpReceiver !== 'undefined' && + RTCRtpReceiver.getCapabilities) { var videoCaps = RTCRtpReceiver.getCapabilities('video'); if (videoCaps) { - var vp9Codecs = videoCaps.codecs.filter(function(c) { + var preferredCodecs = videoCaps.codecs.filter(function(c) { return c.mimeType === 'video/VP9'; }); - if (vp9Codecs.length) { - var preferredCodecs = vp9Codecs.concat( + if (preferredCodecs.length) { + preferredCodecs = preferredCodecs.concat( videoCaps.codecs.filter(function(c) { return c.mimeType !== 'video/VP9'; })); - if (stream) { - // Local video being sent: find the transceiver - // addStream() created and apply preferences. - pc.getTransceivers().forEach(function(t) { - if (t.sender && t.sender.track && - t.sender.track.kind === 'video' && - t.setCodecPreferences) { - t.setCodecPreferences(preferredCodecs); - } - }); - } else { - // Receive-only: create transceiver explicitly. - var vt = pc.addTransceiver( - 'video', {direction: 'recvonly'}); - if (vt.setCodecPreferences) { - vt.setCodecPreferences(preferredCodecs); - } + var videoReceiver = pc.addTransceiver( + 'video', {direction: 'recvonly'}); + if (videoReceiver.setCodecPreferences) { + videoReceiver.setCodecPreferences(preferredCodecs); } } } @@ -664,13 +639,12 @@ let WebRtcStreamer = (function() { const recvs = pc.getReceivers(); recvs.forEach((recv) => { + // Minimize browser jitter buffer to reduce playout latency. + // 1. RTP playout-delay header extensions with min=max=0 via the + // WebRTC-ForceSendPlayoutDelay field trial. + // 2. Set jitterBufferTarget to 0 for browsers that honour the + // JS API. if (recv.track && recv.track.kind === 'video') { - // Minimize browser jitter buffer to reduce playout - // latency. The server already sends RTP playout-delay - // header extensions with min=max=0 via the - // WebRTC-ForceSendPlayoutDelay field trial, but set - // jitterBufferTarget as well for browsers that honour - // the JS API over the in-band RTP extension. if (typeof recv.jitterBufferTarget !== 'undefined') { recv.jitterBufferTarget = 0; } @@ -686,13 +660,9 @@ let WebRtcStreamer = (function() { }; // Local datachannel sends data. - // Use reliable ordered delivery (the default). While unordered + - // unreliable would reduce head-of-line blocking for mouse MOVE/DRAG, - // the same channel carries discrete events (BUTTON_DOWN/UP, key events, - // resize) and application-level RPC (tensorboard/update_geometry etc.) - // that must not be lost or reordered. Mouse coalescing (rAF + server- - // side replace_or_merge_mouse) already prevents event backlog, so the - // extra reliability is free in practice on a local network. + // Use reliable ordered delivery (the default). Unordered + + // unreliable would introduce errors in application RPC + // (tensorboard/update_geometry etc.) logic. try { this.dataChannel = pc.createDataChannel('ClientDataChannel'); var dataChannel = this.dataChannel; @@ -758,11 +728,9 @@ let WebRtcStreamer = (function() { */ WebRtcStreamer.prototype.onTrack = function(event) { console.log('Remote track added: ' + event.track.kind); - if (event.track.kind !== 'video') { return; } - var stream = event.streams && event.streams[0]; if (!stream) { if (!this.remoteStream) { diff --git a/python/js/package.json b/python/js/package.json index 0e2f7d78069..d364052ec22 100644 --- a/python/js/package.json +++ b/python/js/package.json @@ -54,4 +54,4 @@ } } } -} +} \ No newline at end of file From fb3bd7bab6a54051b52db39077f2e7be96ddd9fc Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 8 Jul 2026 13:29:12 -0700 Subject: [PATCH 35/37] bypass gs rendering for webrtc if no gs in scene. --- .../rendering/filament/FilamentRenderer.cpp | 42 ++++++++++++++----- .../rendering/filament/FilamentScene.cpp | 29 +++++++------ .../webrtc_server/PeerConnectionManager.cpp | 3 +- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp index df7423c269e..2b12eacb15c 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp @@ -50,6 +50,21 @@ namespace open3d { namespace visualization { namespace rendering { +namespace { + +inline bool ScenesHaveGaussianSplatGeometry( + const std::unordered_map>& scenes) { + for (const auto& [handle, scene] : scenes) { + if (scene->HasGaussianSplatGeometry()) { + return true; + } + } + return false; +} + +} // namespace + FilamentRenderer::FilamentRenderer(filament::Engine& engine, void* native_drawable, FilamentResourceManager& resource_mgr) @@ -174,6 +189,9 @@ void FilamentRenderer::UpdateBitmapSwapChain(int width, int height) { } void FilamentRenderer::BeginFrame() { + const bool run_gs_pipeline = + gaussian_splat_renderer_ && ScenesHaveGaussianSplatGeometry(scenes_); + // We will complete render to buffer requests first if (!buffer_renderers_.empty()) { for (auto& br : buffer_renderers_) { @@ -193,18 +211,19 @@ void FilamentRenderer::BeginFrame() { if (gaussian_splat_renderer_) { gaussian_splat_renderer_->BeginFrame(); + if (run_gs_pipeline) { #if !defined(__APPLE__) - // Drain any pending Filament (OpenGL) work before the geometry pass - // begins. Filament renders on its own driver thread with an OpenGL - // backend; flushAndWait() enqueues glFinish() there and blocks until - // it completes. This ensures the shared interop textures from the - // previous frame are no longer in use by the GL driver before Vulkan - // compute overwrites them. (Vulkan and Filament run independent queues; - // there is no shared queue between them.) - engine_.flushAndWait(); + // Drain any pending Filament (OpenGL) work before the geometry pass + // begins. Filament renders on its own driver thread with an OpenGL + // backend; flushAndWait() enqueues glFinish() there and blocks until + // it completes. This ensures the shared interop textures from the + // previous frame are no longer in use by the GL driver before Vulkan + // compute overwrites them. (Vulkan and Filament run independent queues; + // there is no shared queue between them.) + engine_.flushAndWait(); #endif - // Dispatch Gaussian splat geometry work before Filament's beginFrame + // Dispatch Gaussian splat geometry work before Filament's beginFrame // so our queue submissions do not conflict with Filament's frame. // // Build live_views from ALL views that must not be pruned: scene views @@ -224,6 +243,7 @@ void FilamentRenderer::BeginFrame() { live_views.insert(static_cast(&br->GetView())); } gaussian_splat_renderer_->PruneOutputs(live_views); + } } frame_started_ = renderer_->beginFrame(swap_chain_); @@ -240,7 +260,7 @@ void FilamentRenderer::Draw() { // frame. Apple runs the composite stage after endFrame() so the Metal // depth texture is fully produced before compute samples it. #if !defined(__APPLE__) - if (gaussian_splat_renderer_) { + if (gaussian_splat_renderer_ && ScenesHaveGaussianSplatGeometry(scenes_)) { // Wait for Filament's OpenGL scene draw to finish so the shared // depth texture is fully written before the composite pass reads // it. @@ -270,7 +290,7 @@ void FilamentRenderer::EndFrame() { if (frame_started_) { renderer_->endFrame(); #if defined(__APPLE__) - if (gaussian_splat_renderer_) { + if (gaussian_splat_renderer_ && ScenesHaveGaussianSplatGeometry(scenes_)) { // endFrame() commits Filament's Metal command buffer. Our // composite CB, committed below on the same queue, will // execute after Filament's render — guaranteeing the depth diff --git a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp index 2b8ad2e2e49..a3b43bb81d0 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp @@ -97,7 +97,10 @@ using MaterialHandle = open3d::visualization::rendering::MaterialHandle; using ResourceManager = open3d::visualization::rendering::FilamentResourceManager; -std::unordered_map shader_mappings = { +const std::unordered_map& ShaderMappings() { + // Built on first use so MaterialHandle::Next() ids in + // FilamentResourceManager.cpp are initialized first (avoids SIOF). + static const std::unordered_map mappings = { {"defaultLit", ResourceManager::kDefaultLit}, {"defaultLitTransparency", ResourceManager::kDefaultLitWithTransparency}, @@ -115,15 +118,17 @@ std::unordered_map shader_mappings = { {"unlitBackground", ResourceManager::kDefaultUnlitBackgroundShader}, {"infiniteGroundPlane", ResourceManager::kInfinitePlaneShader}, {"unlitLine", ResourceManager::kDefaultLineShader}}; + return mappings; +} -MaterialHandle kColorOnlyMesh = ResourceManager::kDefaultUnlit; -MaterialHandle kPlainMesh = ResourceManager::kDefaultLit; -MaterialHandle kMesh = ResourceManager::kDefaultLit; - -MaterialHandle kColoredPointcloud = ResourceManager::kDefaultUnlit; -MaterialHandle kPointcloud = ResourceManager::kDefaultLit; - -MaterialHandle kLineset = ResourceManager::kDefaultUnlit; +MaterialHandle ShaderToMaterial(const std::string& shader_name) { + const auto& mappings = ShaderMappings(); + auto it = mappings.find(shader_name); + if (it != mappings.end() && it->second) { + return it->second; + } + return ResourceManager::kDefaultUnlit; +} } // namespace defaults_mapping @@ -638,8 +643,7 @@ MaterialInstanceHandle FilamentScene::AssignMaterialToFilamentGeometry( filament::RenderableManager::Builder& builder, const MaterialRecord& material) { // TODO: put this in a method - auto shader = defaults_mapping::shader_mappings[material.shader]; - if (!shader) shader = defaults_mapping::kColorOnlyMesh; + auto shader = defaults_mapping::ShaderToMaterial(material.shader); auto material_instance = resource_mgr_.CreateMaterialInstance(shader); auto wmat_instance = resource_mgr_.GetMaterialInstance(material_instance); @@ -1644,8 +1648,7 @@ void FilamentScene::OverrideMaterialInternal(RenderableGeometry* geom, // Has the shader changed? if (geom->mat.properties.shader != material.shader) { // TODO: put this in a method - auto shader = defaults_mapping::shader_mappings[material.shader]; - if (!shader) shader = defaults_mapping::kColorOnlyMesh; + auto shader = defaults_mapping::ShaderToMaterial(material.shader); auto old_mi = geom->mat.mat_instance; auto new_mi = resource_mgr_.CreateMaterialInstance(shader); auto wmat_instance = resource_mgr_.GetMaterialInstance(new_mi); diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index 26719bd0905..3d97eb33bf1 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -31,8 +31,7 @@ #include #include -#include -#include +#include #include #include From a6ab52acc2f56a37ecf5c2d6af542a7d8d93361a Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 8 Jul 2026 15:22:11 -0700 Subject: [PATCH 36/37] style --- .../rendering/filament/FilamentRenderer.cpp | 62 ++++++++++--------- .../rendering/filament/FilamentScene.cpp | 34 +++++----- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp index 2b12eacb15c..fe83f15965b 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentRenderer.cpp @@ -189,8 +189,8 @@ void FilamentRenderer::UpdateBitmapSwapChain(int width, int height) { } void FilamentRenderer::BeginFrame() { - const bool run_gs_pipeline = - gaussian_splat_renderer_ && ScenesHaveGaussianSplatGeometry(scenes_); + const bool run_gs_pipeline = gaussian_splat_renderer_ && + ScenesHaveGaussianSplatGeometry(scenes_); // We will complete render to buffer requests first if (!buffer_renderers_.empty()) { @@ -215,34 +215,36 @@ void FilamentRenderer::BeginFrame() { #if !defined(__APPLE__) // Drain any pending Filament (OpenGL) work before the geometry pass // begins. Filament renders on its own driver thread with an OpenGL - // backend; flushAndWait() enqueues glFinish() there and blocks until - // it completes. This ensures the shared interop textures from the - // previous frame are no longer in use by the GL driver before Vulkan - // compute overwrites them. (Vulkan and Filament run independent queues; - // there is no shared queue between them.) + // backend; flushAndWait() enqueues glFinish() there and blocks + // until it completes. This ensures the shared interop textures from + // the previous frame are no longer in use by the GL driver before + // Vulkan compute overwrites them. (Vulkan and Filament run + // independent queues; there is no shared queue between them.) engine_.flushAndWait(); #endif - // Dispatch Gaussian splat geometry work before Filament's beginFrame - // so our queue submissions do not conflict with Filament's frame. - // - // Build live_views from ALL views that must not be pruned: scene views - // (including cached-but-inactive ones) AND active buffer-renderer views - // whose capture is still pending. Buffer-renderer views must be added - // BEFORE PruneOutputs() so their outputs are not destroyed mid-capture. - std::unordered_set live_views; - for ([[maybe_unused]] const auto& [handle, scene] : scenes_) { - scene->ForEachView([&live_views](const FilamentView& view) { - live_views.insert(&view); - }); - scene->ForEachActiveView([this, &scene](FilamentView& view) { - gaussian_splat_renderer_->RenderGeometryStage(view, *scene); - }); - } - for (const auto& br : buffer_renderers_) { - live_views.insert(static_cast(&br->GetView())); - } - gaussian_splat_renderer_->PruneOutputs(live_views); + // Dispatch Gaussian splat geometry work before Filament's + // beginFrame + // so our queue submissions do not conflict with Filament's frame. + // + // Build live_views from ALL views that must not be pruned: scene + // views (including cached-but-inactive ones) AND active + // buffer-renderer views whose capture is still pending. + // Buffer-renderer views must be added BEFORE PruneOutputs() so + // their outputs are not destroyed mid-capture. + std::unordered_set live_views; + for ([[maybe_unused]] const auto& [handle, scene] : scenes_) { + scene->ForEachView([&live_views](const FilamentView& view) { + live_views.insert(&view); + }); + scene->ForEachActiveView([this, &scene](FilamentView& view) { + gaussian_splat_renderer_->RenderGeometryStage(view, *scene); + }); + } + for (const auto& br : buffer_renderers_) { + live_views.insert(static_cast(&br->GetView())); + } + gaussian_splat_renderer_->PruneOutputs(live_views); } } @@ -260,7 +262,8 @@ void FilamentRenderer::Draw() { // frame. Apple runs the composite stage after endFrame() so the Metal // depth texture is fully produced before compute samples it. #if !defined(__APPLE__) - if (gaussian_splat_renderer_ && ScenesHaveGaussianSplatGeometry(scenes_)) { + if (gaussian_splat_renderer_ && + ScenesHaveGaussianSplatGeometry(scenes_)) { // Wait for Filament's OpenGL scene draw to finish so the shared // depth texture is fully written before the composite pass reads // it. @@ -290,7 +293,8 @@ void FilamentRenderer::EndFrame() { if (frame_started_) { renderer_->endFrame(); #if defined(__APPLE__) - if (gaussian_splat_renderer_ && ScenesHaveGaussianSplatGeometry(scenes_)) { + if (gaussian_splat_renderer_ && + ScenesHaveGaussianSplatGeometry(scenes_)) { // endFrame() commits Filament's Metal command buffer. Our // composite CB, committed below on the same queue, will // execute after Filament's render — guaranteeing the depth diff --git a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp index a3b43bb81d0..e65e929771e 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp @@ -101,23 +101,23 @@ const std::unordered_map& ShaderMappings() { // Built on first use so MaterialHandle::Next() ids in // FilamentResourceManager.cpp are initialized first (avoids SIOF). static const std::unordered_map mappings = { - {"defaultLit", ResourceManager::kDefaultLit}, - {"defaultLitTransparency", - ResourceManager::kDefaultLitWithTransparency}, - {"defaultLitSSR", ResourceManager::kDefaultLitSSR}, - {"defaultUnlitTransparency", - ResourceManager::kDefaultUnlitWithTransparency}, - {"defaultUnlit", ResourceManager::kDefaultUnlit}, - {"normals", ResourceManager::kDefaultNormalShader}, - {"depth", ResourceManager::kDefaultDepthShader}, - {"depthValue", ResourceManager::kDefaultDepthValueShader}, - {"unlitGradient", ResourceManager::kDefaultUnlitGradientShader}, - {"unlitSolidColor", ResourceManager::kDefaultUnlitSolidColorShader}, - {"unlitPolygonOffset", - ResourceManager::kDefaultUnlitPolygonOffsetShader}, - {"unlitBackground", ResourceManager::kDefaultUnlitBackgroundShader}, - {"infiniteGroundPlane", ResourceManager::kInfinitePlaneShader}, - {"unlitLine", ResourceManager::kDefaultLineShader}}; + {"defaultLit", ResourceManager::kDefaultLit}, + {"defaultLitTransparency", + ResourceManager::kDefaultLitWithTransparency}, + {"defaultLitSSR", ResourceManager::kDefaultLitSSR}, + {"defaultUnlitTransparency", + ResourceManager::kDefaultUnlitWithTransparency}, + {"defaultUnlit", ResourceManager::kDefaultUnlit}, + {"normals", ResourceManager::kDefaultNormalShader}, + {"depth", ResourceManager::kDefaultDepthShader}, + {"depthValue", ResourceManager::kDefaultDepthValueShader}, + {"unlitGradient", ResourceManager::kDefaultUnlitGradientShader}, + {"unlitSolidColor", ResourceManager::kDefaultUnlitSolidColorShader}, + {"unlitPolygonOffset", + ResourceManager::kDefaultUnlitPolygonOffsetShader}, + {"unlitBackground", ResourceManager::kDefaultUnlitBackgroundShader}, + {"infiniteGroundPlane", ResourceManager::kInfinitePlaneShader}, + {"unlitLine", ResourceManager::kDefaultLineShader}}; return mappings; } From b5d0cdc55f1c6721c64f384ec6b238c9f6ccc59a Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 8 Jul 2026 16:36:43 -0700 Subject: [PATCH 37/37] change GS window log messages. --- .../gaussian_splat/GaussianSplatOpenGLContext.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/open3d/visualization/rendering/gaussian_splat/GaussianSplatOpenGLContext.cpp b/cpp/open3d/visualization/rendering/gaussian_splat/GaussianSplatOpenGLContext.cpp index f931b07c392..a1256f4a12d 100644 --- a/cpp/open3d/visualization/rendering/gaussian_splat/GaussianSplatOpenGLContext.cpp +++ b/cpp/open3d/visualization/rendering/gaussian_splat/GaussianSplatOpenGLContext.cpp @@ -82,9 +82,9 @@ bool GaussianSplatOpenGLContext::InitializeStandalone() { #if !defined(_WIN32) if (std::strcmp(GetSessionType(), "wayland") == 0) { - utility::LogInfo( + utility::LogDebug( "GaussianSplatOpenGLContext: Wayland session detected; " - "using X11/GLX via XWayland for Filament compatibility."); + "using X11/GLX via XWayland."); } #endif @@ -100,9 +100,8 @@ bool GaussianSplatOpenGLContext::InitializeStandalone() { glfwDefaultWindowHints(); if (!window) { utility::LogWarning( - "GaussianSplatOpenGLContext: glfwCreateWindow failed. " - "Linux offscreen rendering now requires an X11/XWayland " - "server."); + "GaussianSplatOpenGLContext: GS helper window failed. " + "Gaussian Splat rendering is not available."); return false; }