Immersive OSPRay Studio - a plugin to create annotated, animated stories

.gitignore

@@ -8,4 +8,5 @@
 !/plugins
 /plugins/*
 !/plugins/example_plugin
+!/plugins/storyboard_plugin
 doc/*.html

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "plugins/storyboard_plugin/async-sockets-cpp"]
+	path = plugins/storyboard_plugin/async-sockets-cpp
+	url = https://github.com/JungWhoNam/async-sockets-cpp.git

plugins/storyboard_plugin/CMakeLists.txt

@@ -0,0 +1,42 @@
+option(BUILD_PLUGIN_STORYBOARD "Storyboard plugin" OFF)
+
+if (BUILD_PLUGIN_STORYBOARD)
+  set(pluginName "ospray_studio_plugin_storyboard")
+
+  add_library(${pluginName} SHARED
+    plugin_storyboard.cpp
+    PanelStoryboard.cpp
+    request/RequestManager.cpp
+    request/ScreenShotContainer.cpp
+  )
+
+  target_link_libraries(${pluginName} ospray_sg)
+
+  # Only link against imgui if needed (ie, pure file importers don't)
+  target_link_libraries(${pluginName} ospray_ui)
+
+  target_link_libraries(${pluginName} stb_image)
+
+  # There can be other plugins using async_sockets library
+  if (NOT TARGET async_sockets)
+    message(STATUS "Adding async_sockets library...")
+    add_library(async_sockets INTERFACE)
+    target_include_directories(async_sockets 
+    INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/async-sockets-cpp/async-sockets/include>)
+  endif()
+
+  target_link_libraries(${pluginName} async_sockets)
+
+  target_include_directories(${pluginName}
+    PRIVATE ${CMAKE_SOURCE_DIR}
+  )
+
+  install(TARGETS ${pluginName}
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}
+      COMPONENT lib
+    # on Windows put the dlls into bin
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+      COMPONENT lib
+  )
+
+endif()

plugins/storyboard_plugin/PanelStoryboard.cpp

@@ -0,0 +1,215 @@
+#include "PanelStoryboard.h"
+
+#include "app/widgets/GenerateImGuiWidgets.h"
+#include "app/MainWindow.h"
+#include "app/CameraStack.h"
+
+#include "sg/Frame.h"
+#include "sg/fb/FrameBuffer.h"
+
+#include "imgui.h"
+#include "stb_image_write.h"
+
+namespace ospray {
+namespace storyboard_plugin {
+
+static void WriteToMemory_stbi(void *context, void *data, int size) {
+  std::vector<unsigned char> *buffer =
+      reinterpret_cast<std::vector<unsigned char> *>(context);
+
+  unsigned char *pData = reinterpret_cast<unsigned char *>(data);
+
+  buffer->insert(buffer->end(), pData, pData + size);
+}
+
+PanelStoryboard::PanelStoryboard(std::shared_ptr<StudioContext> _context, std::string _panelName, std::string _configFilePath)
+    : Panel(_panelName.c_str(), _context)
+    , panelName(_panelName)
+    , configFilePath(_configFilePath)
+{
+  requestManager.reset(new RequestManager(configFilePath));
+}
+
+void PanelStoryboard::processRequests() {
+  if (!requestManager->isRunning())
+    return;
+
+  std::queue<nlohmann::ordered_json> requests;
+  requestManager->pollRequests(requests);
+  if (requests.size() <= 0) 
+    return;
+
+  std::cout << "Processing " << requests.size() << " request(s)..." << std::endl;
+
+  MainWindow* pMW = reinterpret_cast<MainWindow*>(context->getMainWindow()); 
+
+  while (!requests.empty()) {
+    nlohmann::ordered_json req = requests.front();
+
+    if (req.contains("type") && req["type"] == "request" && 
+        req.contains("action") && req["action"] == "capture.view") {
+
+      nlohmann::ordered_json j = 
+      {
+        {"type", "response"},
+        {"action", "capture.view"},
+        {"snapshotIdx", req["snapshotIdx"].get<int>()},
+        {"camera", pMW->arcballCamera->getState()}
+      };
+      requestManager->send(j.dump());
+    }
+    else if (req.contains("type") && req["type"] == "request" && 
+        req.contains("action") && req["action"] == "capture.image") {
+      // 1) capture the image
+      auto &fb = context->frame->childAs<sg::FrameBuffer>("framebuffer");
+      auto img = fb.map(OSP_FB_COLOR);
+      auto size = fb.child("size").valueAs<vec2i>(); // 2048 x 1280
+      auto fmt = fb.child("colorFormat").valueAs<std::string>(); // sRGB
+
+      std::vector<unsigned char> values;
+      stbi_flip_vertically_on_write(1);
+      stbi_write_png_to_func(WriteToMemory_stbi, &values, size.x, size.y, 4, img, 4 * size.x);
+      requestManager->screenshotContainer->clear();
+      requestManager->screenshotContainer->setImage(size.x, size.y, values);
+
+      // 2) send the setup information
+      nlohmann::ordered_json jSetUp = 
+      {
+        {"type", "response"},
+        {"action", "capture.image.setup"},
+        {"snapshotIdx", req["snapshotIdx"].get<int>()},
+        {"width", size.x},
+        {"height", size.y},
+        {"length", values.size()}
+      };
+      requestManager->send(jSetUp.dump());
+    }
+    else if (req.contains("type") && req["type"] == "ack" && 
+        req.contains("action") && (req["action"] == "capture.image.setup" || req["action"] == "capture.image.data")) {
+
+      if (!requestManager->screenshotContainer->isNextChunkAvailable()) {
+        // clear the memory
+        requestManager->screenshotContainer->clear();
+        // send the message to indicate the sending the image part is done
+        nlohmann::ordered_json jChunk = 
+        {
+          {"type", "response"},
+          {"action", "capture.image.done"},
+          {"snapshotIdx", req["snapshotIdx"].get<int>()}
+        };
+        requestManager->send(jChunk.dump());
+      }
+      else { // send the first message
+        int len = 512;
+        int start = requestManager->screenshotContainer->getStartIndex();
+        std::vector<unsigned char> chunk = requestManager->screenshotContainer->getNextChunk(len);
+
+        auto binary = nlohmann::ordered_json::binary_t(chunk);
+        nlohmann::ordered_json jChunk = 
+        {
+          {"type", "response"},
+          {"action", "capture.image.data"},
+          {"snapshotIdx", req["snapshotIdx"].get<int>()},
+          {"start", start },
+          {"data", binary }
+        };
+        requestManager->send(jChunk.dump());
+      }
+    }
+    else if (req.contains("type") && req["type"] == "request" && 
+              req.contains("action") && req["action"] == "update.view") {
+      // int snapshotIdx = req["snapshotIdx"].get<int>();
+
+      if (req.contains("camera")) {
+        CameraState state;
+        from_json(req["camera"], state);
+
+        // update the state
+        pMW->arcballCamera->setState(state);
+        context->updateCamera();
+      }
+    }
+    else if (req.contains("type") && req["type"] == "request" && 
+              req.contains("action") && req["action"] == "update.transition") {
+      // int snapshotIdx = req["snapshotIdx"].get<int>();
+
+      if (req.contains("amount") && req.contains("from") && req.contains("to")) { 
+        CameraState from;
+        from_json(req["from"], from);
+        CameraState to;
+        from_json(req["to"], to);
+        float frac = req["amount"].get<float>();
+
+        // interplate between two camera states
+        CameraState cs{};
+        cs.centerTranslation = lerp(frac, from.centerTranslation, to.centerTranslation);
+        cs.translation = lerp(frac, from.translation, to.translation);
+        if (from.rotation != to.rotation)
+          cs.rotation = ospray::sg::slerp(from.rotation, to.rotation, frac);
+        else
+          cs.rotation = from.rotation;
+
+        // update the state
+        pMW->arcballCamera->setState(cs);
+        context->updateCamera();
+      }
+    }
+
+    requests.pop();
+  }
+}
+
+void PanelStoryboard::buildUI(void *ImGuiCtx)
+{
+  processRequests();
+
+  // Allows plugin to still do other work if the UI isn't shown.
+  if (!isShown())
+    return;
+
+  // Need to set ImGuiContext in *this* address space
+  ImGui::SetCurrentContext((ImGuiContext *)ImGuiCtx);
+  ImGui::OpenPopup(panelName.c_str());
+
+  if (ImGui::BeginPopupModal(
+          panelName.c_str(), nullptr, ImGuiWindowFlags_None)) {
+    if (requestManager->isRunning()) {
+      ImGui::Text("%s", "Currently connected to the storyboard server...");
+
+      if (ImGui::Button("Disconnect")) {
+        requestManager->close();
+      }
+    }
+    else {
+      ImGui::Text("%s", "Currently NOT connected to the storyboard server...");
+
+      std::string str = "- Connect to " + requestManager->ipAddress + ":" + std::to_string(requestManager->portNumber);
+      ImGui::Text("%s", str.c_str());
+
+      if (ImGui::Button("Connect")) {
+        requestManager->start();
+      }
+    }
+    ImGui::Separator();
+
+    if (ImGui::Button("Close")) {
+      setShown(false);
+      ImGui::CloseCurrentPopup();
+    }
+    ImGui::Separator();
+
+    // Display statuses in a scrolling region
+    if (ImGui::CollapsingHeader("Status", ImGuiTreeNodeFlags_DefaultOpen)) {
+      ImGui::BeginChild("Scrolling", ImVec2(0, 0), false, ImGuiWindowFlags_AlwaysAutoResize);
+      for (std::string status : requestManager->statuses) {
+        ImGui::Text("%s", status.c_str());
+      }
+      ImGui::EndChild();
+    }
+
+    ImGui::EndPopup();
+  }
+}
+
+}  // namespace storyboard_plugin
+}  // namespace ospray

plugins/storyboard_plugin/PanelStoryboard.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "app/widgets/Panel.h"
+#include "app/ospStudio.h"
+
+#include "request/RequestManager.h"
+
+namespace ospray {
+namespace storyboard_plugin {
+
+struct PanelStoryboard : public Panel
+{
+  PanelStoryboard(std::shared_ptr<StudioContext> _context, std::string _panelName, std::string _configFilePath);
+
+  void buildUI(void *ImGuiCtx) override;
+
+  void processRequests();
+
+private:
+  std::string panelName;
+  std::string configFilePath;
+  std::unique_ptr<RequestManager> requestManager;
+};
+
+}  // namespace storyboard_plugin
+}  // namespace ospray

plugins/storyboard_plugin/README.md

@@ -0,0 +1,77 @@
+# V-Mail plugin for OSPRay Studio 
+> This project is part of a larger project called [Immersive OSPRay Studio](https://github.com/jungwhonam/ImmersiveOSPRay) and [Visualization Mail (V-Mail)](https://github.com/JungWhoNam/VisualizationMail).
+
+ ## Overview
+![TACC Rattler](storyboard.png)
+
+We created a plugin for [OSPRay Studio v1.0.0](https://github.com/RenderKit/ospray-studio/releases/tag/v1.0.0) to incorporate [Visualization Mail (V-Mail)](https://github.com/JungWhoNam/VisualizationMail) capabilities. This enables users to create annotated, animated stories directly from OSPRay Studio.
+
+In the above image, you see three applications:
+1. Top-left: OSPRay Studio with this plugin (this repo)
+2. Bottom-left: [the server](https://github.com/JungWhoNam/VisualizationMailServer) for managing created stories
+3. Right: [Unity client](https://github.com/JungWhoNam/Storyboard) for constructing a story 
+
+
+## Prerequisites
+Before running `ospStudio` with the plugin, you need to start both [the server](https://github.com/JungWhoNam/VisualizationMailServer) and [the Unity client](https://github.com/JungWhoNam/Storyboard).
+
+
+## Setup
+```shell
+# clone this branch
+git clone -b jungwho.nam-feature-plugin-storyboard https://github.com/JungWhoNam/ospray_studio.git
+cd ospray_studio
+
+mkdir build
+cd build
+mkdir release
+```
+
+
+## CMake configuration and build
+OSPRay Studio needs to be built with `-DBUILD_PLUGINS=ON` and `-DBUILD_PLUGIN_STORYBOARD=ON` in CMake.
+
+```shell
+cmake -S .. \
+-B release \
+-DCMAKE_BUILD_TYPE=Release \
+-DBUILD_PLUGINS=ON \
+-DBUILD_PLUGIN_STORYBOARD=ON
+
+cmake --build release
+
+cmake --install release
+```
+
+
+## Run `ospStudio` with the storyboard plugin
+
+1. First, start the server by following the steps written in [the server repo](https://github.com/JungWhoNam/VisualizationMailServer).
+2. Start the Unity client by following the steps written in [the client repo](https://github.com/JungWhoNam/Storyboard).
+3. Start `ospStudio` with the plugin.
+```shell
+./release/ospStudio \
+--plugin storyboard \
+--plugin:storyboard:config storyboard_settings.json
+```
+4. Go to `Plugins` > `Storyboard Panel` in the menu.
+5. Click "Connect" button.
+
+If connected, you will see "Connected to the server successfully" displayed on the Status sub-panel.
+
+
+## Plugin configuration JSON file
+When running `ospStudio`, you must specify the location of this JSON file using `--plugin:storyboard:config` flag. This file contains information about `ipAddress` and `portNumber` of the Unity client.
+
+```json
+{
+    "ipAddress": "127.0.0.1",
+    "portNumber": 8052
+}
+```
+
+> See [an example JSON file](./storyboard_settings.json).
+
+
+## Known Issues
+Currently, we are only saving camera states. In the future, we plan to capture the entire scene, including material properties, lighting, and more.