livekit-plugins-browser: python API

examples/browser/browser_track.py

@@ -0,0 +1,34 @@
+import logging
+
+from dotenv import load_dotenv
+from livekit import rtc
+from livekit.agents import JobContext, WorkerOptions, cli
+from livekit.plugins import browser
+
+WIDTH = 1920
+HEIGHT = 1080
+
+load_dotenv()
+
+
+async def entrypoint(job: JobContext):
+    await job.connect()
+
+    ctx = browser.BrowserContext(dev_mode=True)
+    await ctx.initialize()
+
+    page = await ctx.new_page(url="www.google.com")
+
+    source = rtc.VideoSource(WIDTH, HEIGHT)
+    track = rtc.LocalVideoTrack.create_video_track("single-color", source)
+    options = rtc.TrackPublishOptions(source=rtc.TrackSource.SOURCE_CAMERA)
+    publication = await job.room.local_participant.publish_track(track, options)
+    logging.info("published track", extra={"track_sid": publication.sid})
+
+    @page.on("paint")
+    def on_paint(data):
+        source.capture_frame(data)
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint))

livekit-plugins/install_plugins_editable.sh

@@ -16,3 +16,4 @@ pip install -e ./livekit-plugins-nltk --config-settings editable_mode=strict
 pip install -e ./livekit-plugins-openai --config-settings editable_mode=strict
 pip install -e ./livekit-plugins-rag --config-settings editable_mode=strict
 pip install -e ./livekit-plugins-silero --config-settings editable_mode=strict
+pip install -e ./livekit-plugins-browser --config-settings editable_mode=strict

livekit-plugins/livekit-plugins-browser/CHANGELOG.md

@@ -0,0 +1,2 @@
+# livekit-plugins-browser
+

livekit-plugins/livekit-plugins-browser/README.md

@@ -0,0 +1,4 @@
+# LiveKit Plugins Browser 
+
+Chromium Embedded Framework (CEF) for LiveKit Agents
+

livekit-plugins/livekit-plugins-browser/cef/src/agents_python.cpp

@@ -1,23 +1,60 @@
 #include "agents_python.hpp"
 
+#include <pybind11/functional.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
 #include "app.hpp"
+#include "include/base/cef_callback.h"
 #include "include/internal/cef_mac.h"
-
-#include <pybind11/pybind11.h>
-#include <pybind11/functional.h>
+#include "include/wrapper/cef_closure_task.h"
 
 namespace py = pybind11;
 
 BrowserApp::BrowserApp(const AppOptions& options) : options_(options) {
   app_ = new AgentApp(options_.dev_mode, options_.initialized_callback);
 }
 
-std::shared_ptr<BrowserImpl> BrowserApp::CreateBrowser(
-    const std::string& url,
-    const BrowserOptions& options) {
+bool BrowserApp::CreateBrowser(const std::string& url,
+                               const BrowserOptions& options) {
+  if (CefCurrentlyOn(TID_UI)) {
+    CreateBrowserOnUIThread(url, options);
+    return true;
+  }
+
+  // TODO(theomonnom): Document base::Unretained
+  CefPostTask(TID_UI, base::BindOnce(&BrowserApp::CreateBrowserOnUIThread,
+                                     base::Unretained(this), url, options));
+
+  return true;
+}
+
+void BrowserApp::CreateBrowserOnUIThread(const std::string& url,
+                                         const BrowserOptions& options) {
+  std::shared_ptr<BrowserImpl> browser_impl = std::make_shared<BrowserImpl>();
+  browsers_.push_back(browser_impl);
 
-  app_->CreateBrowser(url, options.framerate, options.width, options.height, options.created_callback);
-  return nullptr;//std::make_shared<BrowserImpl>();
+  CefRefPtr<BrowserHandle> handle = app_->CreateBrowser(
+      url, options.framerate, options.width, options.height,
+      [options, browser_impl]() { options.created_callback(browser_impl); },
+      [options](std::vector<CefRect> dirtyRects, const void* buffer, int width,
+                int height) {
+        PaintData event{};
+        std::vector<PaintRect> rects;
+        rects.reserve(dirtyRects.size());
+
+        for (const auto& rect : dirtyRects) {
+          rects.push_back({rect.x, rect.y, rect.width, rect.height});
+        }
+
+        event.dirtyRect = rects;
+        event.buffer = buffer;
+        event.width = width;
+        event.height = height;
+        options.paint_callback(event);
+      });
+
+  browser_impl->handle = handle;
 }
 
 int BrowserApp::Run() {
@@ -28,6 +65,16 @@ BrowserImpl::BrowserImpl() {}
 
 void BrowserImpl::SetSize(int width, int height) {}
 
+int BrowserImpl::Identifier() const {
+  return handle->GetBrowser()->GetIdentifier();
+}
+
+py::memoryview paint_data_to_memoryview(const PaintData& event) {
+  return py::memoryview::from_buffer(
+      const_cast<uint32_t*>(static_cast<const uint32_t*>(event.buffer)),
+      {event.height * event.width}, {sizeof(uint32_t)}, true);
+}
+
 PYBIND11_MODULE(lkcef_python, m) {
   // Isn't that fucking cool? llm using browsers
   m.doc() = "Chromium Embedded Framework (CEF) for LiveKit Agents";
@@ -42,13 +89,30 @@ PYBIND11_MODULE(lkcef_python, m) {
       .def_readwrite("framerate", &BrowserOptions::framerate)
       .def_readwrite("width", &BrowserOptions::width)
       .def_readwrite("height", &BrowserOptions::height)
-      .def_readwrite("created_callback", &BrowserOptions::created_callback);
+      .def_readwrite("created_callback", &BrowserOptions::created_callback)
+      .def_readwrite("paint_callback", &BrowserOptions::paint_callback);
 
   py::class_<BrowserApp>(m, "BrowserApp")
       .def(py::init<const AppOptions&>())
       .def("create_browser", &BrowserApp::CreateBrowser)
-      .def("run", &BrowserApp::Run);
+      .def("run", &BrowserApp::Run, py::call_guard<py::gil_scoped_release>());
+
+  py::class_<BrowserImpl, std::shared_ptr<BrowserImpl>>(m, "BrowserImpl")
+      .def("set_size", &BrowserImpl::SetSize)
+      .def("identifier", &BrowserImpl::Identifier);
 
-  py::class_<BrowserImpl>(m, "BrowserImpl")
-      .def("set_size", &BrowserImpl::SetSize);
+  py::class_<PaintRect>(m, "PaintRect")
+      .def_readwrite("x", &PaintRect::x)
+      .def_readwrite("y", &PaintRect::y)
+      .def_readwrite("width", &PaintRect::width)
+      .def_readwrite("height", &PaintRect::height);
+
+  py::class_<PaintData>(m, "PaintData")
+      .def(py::init())
+      .def_readwrite("dirty_rects", &PaintData::dirtyRect)
+      .def_readwrite("width", &PaintData::width)
+      .def_readwrite("height", &PaintData::height)
+      .def_property_readonly("buffer", [](const PaintData& event) {
+        return paint_data_to_memoryview(event);
+      });
 }

livekit-plugins/livekit-plugins-browser/cef/src/agents_python.hpp

@@ -7,6 +7,7 @@
 #include "app.hpp"
 
 class BrowserImpl;
+struct PaintData;
 
 struct AppOptions {
   bool dev_mode = false;
@@ -17,25 +18,45 @@ struct BrowserOptions {
   int framerate = 30;
   int width = 800;
   int height = 600;
-  std::function<void()> created_callback = nullptr;
+  std::function<void(std::shared_ptr<BrowserImpl>)> created_callback = nullptr;
+  std::function<void(const PaintData&)> paint_callback = nullptr;
 };
 
 struct BrowserApp {
   BrowserApp(const AppOptions& options);
 
-  std::shared_ptr<BrowserImpl> CreateBrowser(const std::string& url,
-                                             const BrowserOptions& options);
+  bool CreateBrowser(const std::string& url, const BrowserOptions& options);
+  void CreateBrowserOnUIThread(const std::string& url, const BrowserOptions& options);
+
   int Run();
 
  private:
   AppOptions options_;
   CefRefPtr<AgentApp> app_;
+  std::list<std::shared_ptr<BrowserImpl>> browsers_;
 };
 
 struct BrowserImpl {
   BrowserImpl();
 
   void SetSize(int width, int height);
+  int Identifier() const;
+
+  CefRefPtr<BrowserHandle> handle = nullptr;
+};
+
+struct PaintRect {
+  int x = 0;
+  int y = 0;
+  int width = 0;
+  int height = 0;
+};
+
+struct PaintData {
+  std::vector<PaintRect> dirtyRect;
+  const void* buffer;
+  int width;
+  int height;
 };
 
 #endif  // LKCEF_AGENTS_PYTHON_HPP

livekit-plugins/livekit-plugins-browser/cef/src/app.cpp

@@ -40,7 +40,11 @@ CefRefPtr<BrowserHandle> AgentApp::CreateBrowser(
     int framerate,
     int width,
     int height,
-    std::function<void()> created_callback) {
+    std::function<void()> created_callback,
+    std::function<void(std::vector<CefRect> dirtyRects,
+                       const void* buffer,
+                       int width,
+                       int height)> paint_callback) {
   CEF_REQUIRE_UI_THREAD();
   CefWindowInfo windowInfo;
   // windowInfo.SetAsWindowless(dev_renderer_->getNativeWindowHandle());
@@ -50,10 +54,11 @@ CefRefPtr<BrowserHandle> AgentApp::CreateBrowser(
       CefCommandLine::GetGlobalCommandLine();
 
   CefBrowserSettings settings;
+  settings.windowless_frame_rate = framerate;
   settings.background_color = CefColorSetARGB(255, 255, 255, 255);
 
-  CefRefPtr<BrowserHandle> browser_handle =
-        new BrowserHandle(created_callback, width, height);
+  CefRefPtr<BrowserHandle> browser_handle = new BrowserHandle(
+      std::move(created_callback), std::move(paint_callback), width, height);
 
   client_->AddPendingHandle(browser_handle);
 

livekit-plugins/livekit-plugins-browser/cef/src/app.hpp

@@ -30,7 +30,11 @@ class AgentApp : public CefApp, public CefBrowserProcessHandler {
       int framerate,
       int width,
       int height,
-      std::function<void()> created_callback);
+      std::function<void()> created_callback,
+      std::function<void(std::vector<CefRect> dirtyRect,
+                         const void* buffer,
+                         int width,
+                         int height)> paint_callback);
 
   int Run();
 

livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp

@@ -288,7 +288,6 @@ void DevRenderer::OnPaint(CefRefPtr<CefBrowser> browser,
   CEF_REQUIRE_UI_THREAD();
 
   if (type != CefRenderHandler::PaintElementType::PET_VIEW) {
-    std::cout << "Ignoring PET_POPUP" << std::endl;
     return;  // Ignore PET_POPUP for now, bc I'm lazy
   }
 
@@ -402,6 +401,11 @@ void DevRenderer::Run() {
           "###Browser" + std::to_string(identifier);
 
       if (ImGui::Begin(name.c_str())) {
+        if (ImGui::InputText("URL", &data.url,
+                             ImGuiInputTextFlags_EnterReturnsTrue)) {
+          data.browser->GetMainFrame()->LoadURL(data.url);
+        }
+
         ImVec2 size = ImGui::GetContentRegionAvail();
 
         // Resize the browser view if needed
@@ -413,11 +417,6 @@ void DevRenderer::Run() {
               ->SetSize(static_cast<int>(size.x), static_cast<int>(size.y));
         }
 
-        if (ImGui::InputText("URL", &data.url,
-                             ImGuiInputTextFlags_EnterReturnsTrue)) {
-          data.browser->GetMainFrame()->LoadURL(data.url);
-        }
-
         ImVec2 cursor_pos = ImGui::GetCursorScreenPos();
 
         bool is_focused = ImGui::IsWindowFocused();

livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp

@@ -49,6 +49,13 @@ void AgentHandler::OnPaint(CefRefPtr<CefBrowser> browser,
                            const void* buffer,
                            int width,
                            int height) {
+  CEF_REQUIRE_UI_THREAD();
+
+  int identifier = browser->GetIdentifier();
+  CefRefPtr<BrowserHandle> handle = browser_handles_[identifier];
+  if (handle->paint_callback_)
+    handle->paint_callback_(dirtyRects, buffer, width, height);
+
   if (dev_renderer_)
     dev_renderer_->OnPaint(browser, type, dirtyRects, buffer, width, height);
 }
@@ -102,11 +109,11 @@ void AgentHandler::OnAfterCreated(CefRefPtr<CefBrowser> browser) {
   pending_handles_.pop_front();
 
   handle->browser_ = browser;
+  browser_handles_[identifier] = handle;
+
   if (handle->created_callback_)
     handle->created_callback_();
 
-  browser_handles_[identifier] = handle;
-
   if (dev_renderer_)
     dev_renderer_->OnAfterCreated(browser);
 }