suddenly a plugin

README.md

@@ -1,47 +1,75 @@
-# A Neovim Plugin Template
+# crawler.nvim
 
-![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/ellisonleao/nvim-plugin-template/lint-test.yml?branch=main&style=for-the-badge)
-![Lua](https://img.shields.io/badge/Made%20with%20Lua-blueviolet.svg?style=for-the-badge&logo=lua)
+A Neovim plugin for crawling web pages and inserting their content into your buffer.
 
-A template repository for Neovim plugins.
+## Features
 
-## Using it
+- Process single URLs, multiple URLs, or search queries
+- Render web pages to Markdown or JSON
+- Insert processed content directly into your Neovim buffer
+- Supports visual selection or manual input
+- Configurable options for rendering and search functionality
 
-Via `gh`:
+## Installation
 
+Using [packer.nvim](https://github.com/wbthomason/packer.nvim):
+
+```lua
+use {
+  'yourusername/crawler.nvim',
+  requires = {
+    'nvim-lua/plenary.nvim'
+  }
+}
 ```
-$ gh repo create my-plugin -p ellisonleao/nvim-plugin-template
+
+## Configuration
+
+Add the following to your Neovim configuration:
+
+```lua
+require('crawler').setup({
+  render_markdown = true,  -- Set to false to disable markdown rendering
+  render_json = false,     -- Set to true to enable JSON rendering
+  search_engine = true,    -- Set to false to disable search engine functionality
+})
 ```
 
-Via github web page:
+## Usage
 
-Click on `Use this template`
+- In normal mode, press `<leader>c` and then enter a URL or search query when prompted.
+- In visual mode, select text (URL or search query) and press `<leader>c`.
+- Use the `:Crawl` command followed by a URL or search query.
 
-![](https://docs.github.com/assets/cb-36544/images/help/repository/use-this-template-button.png)
+### Examples:
 
-## Features and structure
+1. Process a single URL:
+   ```
+   <leader>c
+   https://example.com
+   ```
 
-- 100% Lua
-- Github actions for:
-  - running tests using [plenary.nvim](https://github.com/nvim-lua/plenary.nvim) and [busted](https://olivinelabs.com/busted/)
-  - check for formatting errors (Stylua)
-  - vimdocs autogeneration from README.md file
-  - luarocks release (LUAROCKS_API_KEY secret configuration required)
+2. Process multiple URLs:
+   ```
+   <leader>c
+   https://example.com, https://another-example.com
+   ```
 
-### Plugin structure
+3. Perform a search:
+   ```
+   <leader>c
+   neovim lua plugins
+   ```
 
-```
-.
-├── lua
-│   ├── plugin_name
-│   │   └── module.lua
-│   └── plugin_name.lua
-├── Makefile
-├── plugin
-│   └── plugin_name.lua
-├── README.md
-├── tests
-│   ├── minimal_init.lua
-│   └── plugin_name
-│       └── plugin_name_spec.lua
-```
+## Requirements
+
+- Neovim >= 0.7.0
+- [plenary.nvim](https://github.com/nvim-lua/plenary.nvim)
+
+## License
+
+MIT
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.

lua/crawler.lua

@@ -0,0 +1,123 @@
+local curl = require('plenary.curl')
+local job = require('plenary.job')
+
+---@class Config
+---@field render_markdown boolean
+---@field render_json boolean
+---@field search_engine boolean
+local config = {
+  render_markdown = true,
+  render_json = false,
+  search_engine = true,
+}
+
+---@class Crawler
+local M = {}
+
+---@type Config
+M.config = config
+
+---@param args Config?
+M.setup = function(args)
+  M.config = vim.tbl_deep_extend("force", M.config, args or {})
+end
+
+local function is_url(str)
+  return str:match("^https?://") ~= nil
+end
+
+local function get_visual_selection()
+  local s_start = vim.fn.getpos("'<")
+  local s_end = vim.fn.getpos("'>")
+  local n_lines = math.abs(s_end[2] - s_start[2]) + 1
+  local lines = vim.api.nvim_buf_get_lines(0, s_start[2] - 1, s_end[2], false)
+  lines[1] = string.sub(lines[1], s_start[3], -1)
+  if n_lines == 1 then
+    lines[n_lines] = string.sub(lines[n_lines], 1, s_end[3] - s_start[3] + 1)
+  else
+    lines[n_lines] = string.sub(lines[n_lines], 1, s_end[3])
+  end
+  return table.concat(lines, '\n')
+end
+
+local function process_url(url, render_type)
+  local prefix = render_type == 'markdown' and 'r.jina.ai/' or 'jsondr.com/'
+  local full_url = prefix .. url
+  local response = curl.get(full_url)
+
+  if response.status ~= 200 then
+    print("Error fetching URL: " .. url)
+    return nil
+  end
+
+  return response.body
+end
+
+local function insert_into_buffer(content)
+  local current_buf = vim.api.nvim_get_current_buf()
+  local current_line = vim.api.nvim_win_get_cursor(0)[1]
+  vim.api.nvim_buf_set_lines(current_buf, current_line, current_line, false, vim.split(content, '\n'))
+end
+
+local function process_sitemap(url)
+  -- TODO: Implement sitemap processing
+  print("Sitemap processing not yet implemented")
+end
+
+local function process_search(query)
+  local search_url = 's.jina.ai/' .. vim.fn.shellescape(query)
+  local response = curl.get(search_url)
+
+  if response.status ~= 200 then
+    print("Error performing search: " .. query)
+    return
+  end
+
+  insert_into_buffer(response.body)
+end
+
+M.crawl = function()
+  local input = get_visual_selection()
+  if input == '' then
+    input = vim.fn.input("Enter URL, multiple URLs (comma-separated), or search query: ")
+  end
+
+  if input:find(',') then
+    -- Multiple URLs
+    for url in input:gmatch("[^,]+") do
+      url = url:match("^%s*(.-)%s*$") -- Trim whitespace
+      if is_url(url) then
+        local content = process_url(url, M.config.render_json and 'json' or 'markdown')
+        if content then
+          insert_into_buffer(content)
+        end
+      end
+    end
+  elseif is_url(input) then
+    -- Single URL
+    if input:match("sitemap%.xml$") then
+      process_sitemap(input)
+    else
+      local content = process_url(input, M.config.render_json and 'json' or 'markdown')
+      if content then
+        insert_into_buffer(content)
+      end
+    end
+  else
+    -- Assume it's a search query
+    if M.config.search_engine then
+      process_search(input)
+    else
+      print("Search engine functionality is disabled")
+    end
+  end
+end
+
+-- Set up the plugin command
+vim.api.nvim_create_user_command('Crawl', M.crawl, {})
+
+-- Set up the key mapping
+vim.api.nvim_set_keymap('n', '<leader>c', ':Crawl<CR>', { noremap = true, silent = true })
+vim.api.nvim_set_keymap('v', '<leader>c', ':Crawl<CR>', { noremap = true, silent = true })
+
+return M

plugin/plugin_name.lua

@@ -1 +1,21 @@
-vim.api.nvim_create_user_command("MyFirstFunction", require("plugin_name").hello, {})
+if vim.fn.has("nvim-0.7.0") == 0 then
+  vim.api.nvim_err_writeln("crawler.nvim requires at least nvim-0.7.0")
+  return
+end
+
+-- make sure this file is loaded only once
+if vim.g.loaded_crawler == 1 then
+  return
+end
+vim.g.loaded_crawler = 1
+
+-- create any global command that does not depend on user setup
+local crawler = require("crawler")
+
+vim.api.nvim_create_user_command("Crawl", function(opts)
+  crawler.crawl()
+end, {})
+
+-- Set up the key mapping
+vim.api.nvim_set_keymap('n', '<leader>c', ':Crawl<CR>', { noremap = true, silent = true })
+vim.api.nvim_set_keymap('v', '<leader>c', ':Crawl<CR>', { noremap = true, silent = true })

tests/crawler/crawler_spec.lua

@@ -0,0 +1,115 @@
+local crawler = require("crawler")
+local stub = require("luassert.stub")
+
+describe("crawler", function()
+  before_each(function()
+    -- Reset the configuration before each test
+    crawler.setup({
+      render_markdown = true,
+      render_json = false,
+      search_engine = true,
+    })
+  end)
+
+  after_each(function()
+    print("vim.go.loadplugins status:", vim.go.loadplugins)
+  end)
+
+  describe("setup", function()
+    it("works with default configuration", function()
+      assert.are.same({
+        render_markdown = true,
+        render_json = false,
+        search_engine = true,
+      }, crawler.config)
+    end)
+
+    it("works with custom configuration", function()
+      crawler.setup({
+        render_markdown = false,
+        render_json = true,
+        search_engine = false,
+      })
+      assert.are.same({
+        render_markdown = false,
+        render_json = true,
+        search_engine = false,
+      }, crawler.config)
+    end)
+  end)
+
+  describe("crawl", function()
+    local mock_curl, mock_job, mock_vim
+
+    before_each(function()
+      mock_curl = {
+        get = stub.new()
+      }
+      mock_job = {}
+      mock_vim = {
+        fn = {
+          input = stub.new().returns("https://example.com"),
+          shellescape = stub.new().returns("encoded_query"),
+        },
+        api = {
+          nvim_buf_set_lines = stub.new(),
+          nvim_get_current_buf = stub.new().returns(1),
+          nvim_win_get_cursor = stub.new().returns({1, 0}),
+        },
+      }
+
+      -- Replace global vim with our mock
+      _G.vim = mock_vim
+
+      -- Replace required modules with our mocks
+      package.loaded["plenary.curl"] = mock_curl
+      package.loaded["plenary.job"] = mock_job
+    end)
+
+    after_each(function()
+      -- Restore original modules
+      package.loaded["plenary.curl"] = nil
+      package.loaded["plenary.job"] = nil
+    end)
+
+    it("processes a single URL correctly", function()
+      mock_curl.get.returns({ status = 200, body = "Processed content" })
+
+      crawler.crawl()
+
+      assert.stub(mock_curl.get).was_called_with("r.jina.ai/https://example.com")
+      assert.stub(mock_vim.api.nvim_buf_set_lines).was_called()
+    end)
+
+    it("handles multiple URLs", function()
+      mock_vim.fn.input.returns("https://example.com, https://another.com")
+      mock_curl.get.returns({ status = 200, body = "Processed content" })
+
+      crawler.crawl()
+
+      assert.stub(mock_curl.get).was_called(2)
+      assert.stub(mock_vim.api.nvim_buf_set_lines).was_called(2)
+    end)
+
+    it("processes search queries", function()
+      mock_vim.fn.input.returns("search query")
+      mock_curl.get.returns({ status = 200, body = "Search results" })
+
+      crawler.crawl()
+
+      assert.stub(mock_curl.get).was_called_with("s.jina.ai/encoded_query")
+      assert.stub(mock_vim.api.nvim_buf_set_lines).was_called()
+    end)
+
+    it("handles errors gracefully", function()
+      mock_curl.get.returns({ status = 404, body = "Not found" })
+
+      crawler.crawl()
+
+      assert.stub(mock_vim.api.nvim_buf_set_lines).was_not_called()
+    end)
+  end)
+end)
+
+-- Print the status of vim.go.loadplugins at the end of all tests
+print("Final vim.go.loadplugins status:", vim.go.loadplugins)

tests/minimal_init.lua

@@ -1,3 +1,6 @@
+-- Ensure plugins are loaded
+vim.go.loadplugins = true
+
 local plenary_dir = os.getenv("PLENARY_DIR") or "/tmp/plenary.nvim"
 local is_not_a_directory = vim.fn.isdirectory(plenary_dir) == 0
 if is_not_a_directory then
@@ -9,3 +12,6 @@ vim.opt.rtp:append(plenary_dir)
 
 vim.cmd("runtime plugin/plenary.vim")
 require("plenary.busted")
+
+-- Explicitly load our plugin
+require("crawler")