Obsidian integration

README.md

@@ -110,6 +110,26 @@ $ docker-compose up
 
 - **update:** if you are having issues with weasyprint, please visit their website and follow the installation instructions: https://doc.courtbouillon.org/weasyprint/stable/first_steps.html
 
+## 📖 Obsidian Integration
+
+GPT Researcher supports exporting to Obsidian, a popular knowledge management tool. To enable this feature, you need to install the Obsidian Local REST API plugin and configure it properly. Here are the steps:
+
+1. Open Obsidian, go to `Settings > Third-party plugins > Community Plugins > Browse` and search for `Local REST API`. Install and enable the plugin.
+
+2. After enabling the plugin, you will need to get the API token. Go to `Settings > Local REST API > API Key` and copy the token.
+
+3. Set the token as an environment variable in your project. You can do this by adding the following line to your .env file: 
+``` bash
+ OBSIDIAN_TOKEN={Your Obsidian API Token here}
+```
+4. You also need to set the base URL of your Obsidian instance including the path to the folder within your obsidian vault as an environment variable. By default, the Obsidian Local REST API runs on `http://localhost:27124` and the root folder is referenced by `vault`. So, add the following line to your .env file:
+```bash
+OBSIDIAN_URL=https://127.0.0.1:27124/vault
+```
+5. Add the self-signed certificate of Local REST API to your python environment.
+
+6. Now, you can use Send to Obsidian Button to directly export your research reports into your vault.
+
 ## 🚀 Contributing
 We highly welcome contributions! Please check out [contributing](CONTRIBUTING.md) if you're interested.
 

client/index.html

@@ -93,8 +93,10 @@ <h2>Agent Output</h2>
     <div class="margin-div">
         <h2>Research Report</h2>
         <div id="reportContainer"></div>
+        <button onclick="GPTResearcher.copyToClipboard()" id="copyToClipboard" class="btn btn-secondary mt-3" disabled>Copy to clipboard</button>
         <button onclick="GPTResearcher.copyToClipboard()" class="btn btn-secondary mt-3">Copy to clipboard</button>
         <a id="downloadLink" href="#" class="btn btn-secondary mt-3" target="_blank">Download as PDF</a>
+        <button onclick="GPTResearcher.sendToObsidian()" id="downloadObsidian" href="#" class="btn btn-secondary mt-3" target="_blank" disabled>Send to Obsidian</button>
     </div>
 </main>
 

client/obsidian_template.md

@@ -0,0 +1,16 @@
+---
+date: <% tp.file.creation_date() %>
+modification_date: <% await tp.file.last_modified_date() %>
+tags: 
+  - gpt-researcher
+prompt: {PROMPT}
+---
+
+Date:: [[<% tp.file.creation_date("YY-MM-DD-dddd") %>]]
+ai-model:: [[gpt-researcher]]
+
+{RESEARCH_REPORT}
+
+# Agent Output
+
+{AGENT_OUTPUT}

client/scripts.js

@@ -1,7 +1,12 @@
 const GPTResearcher = (() => {
+    let agentMarkdownContent = "";
+    let markdownContent = "";
+
     const startResearch = () => {
       document.getElementById("output").innerHTML = "";
       document.getElementById("reportContainer").innerHTML = "";
+      markdownContent = "";
+      agentMarkdownContent = "";
 
       addAgentResponse({ output: "🤔 Thinking about research questions for the task..." });
 
@@ -43,15 +48,19 @@ const GPTResearcher = (() => {
     const addAgentResponse = (data) => {
       const output = document.getElementById("output");
       output.innerHTML += '<div class="agent_response">' + data.output + '</div>';
+      agentMarkdownContent += data.output;
       output.scrollTop = output.scrollHeight;
       output.style.display = "block";
       updateScroll();
     };
-  
+
     const writeReport = (data, converter) => {
       const reportContainer = document.getElementById("reportContainer");
       const markdownOutput = converter.makeHtml(data.output);
+      markdownContent += data.output;
       reportContainer.innerHTML += markdownOutput;
+      document.getElementById('downloadObsidian').disabled = false;
+      document.getElementById('copyToClipboard').disabled = false;
       updateScroll();
     };
 
@@ -76,9 +85,40 @@ const GPTResearcher = (() => {
       document.execCommand('copy');
       document.body.removeChild(textarea);
     };
+
+    const sendToObsidian = async () => {
+      let template;
+      try {
+        const response = await fetch('/site/obsidian_template.md');
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        template = await response.text();
+      } catch (e) {
+        console.log('There was an error fetching the template: ', e);
+      }
+
+      if (template) {
+        const prompt = document.querySelector('#task').value;
+        template = template.replace('{PROMPT}', prompt);
+        template = template.replace('{RESEARCH_REPORT}', markdownContent);
+        template = template.replace('{AGENT_OUTPUT}', agentMarkdownContent);
+      } else {
+        console.log('Template is not defined. Check fetch operation.');
+      }
+
+      fetch('/obsidian', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'text/plain'
+        },
+        body: template
+      })
+    };
 
     return {
       startResearch,
       copyToClipboard,
+      sendToObsidian,
     };
   })();

config/config.py

@@ -29,6 +29,9 @@ def __init__(self) -> None:
         self.browse_chunk_max_length = int(os.getenv("BROWSE_CHUNK_MAX_LENGTH", 8192))
 
         self.openai_api_key = os.getenv("OPENAI_API_KEY")
+        self.obsidian_token = os.getenv("OBSIDIAN_TOKEN")
+        self.obsidian_folder = os.getenv("OBSIDIAN_FOLDER")
+
         self.temperature = float(os.getenv("TEMPERATURE", "1"))
 
         self.user_agent = os.getenv(

main.py

@@ -1,41 +1,38 @@
-from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
+import os
+import json
+import re
+import requests
+from urllib.parse import quote
+from pydantic import BaseModel
+from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect, HTTPException, Body
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
-from pydantic import BaseModel
-import json
-import os
 
 from agent.llm_utils import choose_agent
 from agent.run import WebSocketManager
 
+app = FastAPI()
+app.mount("/site", StaticFiles(directory="client"), name="site")
+app.mount("/static", StaticFiles(directory="client/static"), name="static")
 
 class ResearchRequest(BaseModel):
     task: str
     report_type: str
     agent: str
 
-
-
-app = FastAPI()
-app.mount("/site", StaticFiles(directory="client"), name="site")
-app.mount("/static", StaticFiles(directory="client/static"), name="static")
-# Dynamic directory for outputs once first research is run
 @app.on_event("startup")
 def startup_event():
     if not os.path.isdir("outputs"):
         os.makedirs("outputs")
     app.mount("/outputs", StaticFiles(directory="outputs"), name="outputs")
 
 templates = Jinja2Templates(directory="client")
-
 manager = WebSocketManager()
 
-
 @app.get("/")
 async def read_root(request: Request):
     return templates.TemplateResponse('index.html', {"request": request, "report": None})
 
-
 @app.websocket("/ws")
 async def websocket_endpoint(websocket: WebSocket):
     await manager.connect(websocket)
@@ -47,7 +44,6 @@ async def websocket_endpoint(websocket: WebSocket):
                 task = json_data.get("task")
                 report_type = json_data.get("report_type")
                 agent = json_data.get("agent")
-                # temporary so "normal agents" can still be used and not just auto generated, will be removed when we move to auto generated
                 if agent == "Auto Agent":
                     agent_dict = choose_agent(task)
                     agent = agent_dict.get("agent")
@@ -64,8 +60,64 @@ async def websocket_endpoint(websocket: WebSocket):
     except WebSocketDisconnect:
         await manager.disconnect(websocket)
 
+async def obsidian_command(command: str, url: str, token: str, filename: str, content: str = "") -> dict:
+    """
+    Executes a command in Obsidian.
+
+    Args:
+        command (str): The command to execute (e.g., "create", "open").
+        url (str): The base URL of the Obsidian vault.
+        token (str): The authorization token for accessing the Obsidian vault.
+        filename (str): The name of the file to be created or opened, including the relative path within the vault.
+        content (str): The content of the file to be created (only used for "create" command).
+
+    Returns:
+        dict: A dictionary with a message indicating the result of the command.
+
+    Raises:
+        HTTPException: If there was an error executing the command in Obsidian.
+    """
+    if command == "create":
+        endpoint = f"vault/{quote(filename)}.md"
+        method = "POST"
+        data = content.encode('utf-8')
+    elif command == "open":
+        endpoint = f"open/{quote(filename)}.md?newLeaf=false"
+        method = "GET"
+        data = None
+    else:
+        raise ValueError("Invalid command")
+
+    endpoint = re.sub(r"//+", "/", endpoint)
+
+    headers = {
+        'accept': '*/*',
+        'Authorization': f'Bearer {token}',
+        'Content-Type': 'text/markdown'
+    }
+    response = requests.request(method, f"{url}{endpoint}", headers=headers, data=data)
+
+    if response.status_code < 200 or response.status_code >= 300:
+        raise HTTPException(status_code=response.status_code, detail=f"Error executing {command} command in Obsidian")
+
+    return {"message": f"{command.capitalize()} command executed successfully"}
+
+@app.post("/obsidian")
+async def post_to_obsidian(content: str = Body(...)):
+    first_heading_match = re.search(r"^#\s(.+)$", content, re.MULTILINE)
+    filename = first_heading_match.group(1) if first_heading_match else 'default_filename'
+    filename = re.sub(r"[:]", " -", filename)
+    filename = re.sub(r"[#:/\\[\]|^]", "", filename)
+    filename = f"{os.getenv('OBSIDIAN_FOLDER')}/{filename}"
+
+    base_url = os.getenv("OBSIDIAN_URL")
+    if not base_url.endswith('/'):
+        base_url += '/'
+    token = os.getenv("OBSIDIAN_TOKEN")
+
+    await obsidian_command(command="create", url=base_url, token=token, filename=filename, content=content)
+    await obsidian_command(command="open", url=base_url, filename=filename, token=token)
 
 if __name__ == "__main__":
     import uvicorn
-
     uvicorn.run(app, host="0.0.0.0", port=8000)