Merge pull request #63 from neo451/dev2

perf: take lastBuildDate into account for better update

lua/feed/commands.lua

@@ -19,7 +19,7 @@ local function list_feeds()
    local ret = {}
    for _, v in ipairs(config.feeds) do
       local url = type(v) == "table" and v[1] or v
-      local name = type(v) == "table" and v.name or v
+      local name = type(v) == "table" and v.name or nil
       local tags = type(v) == "table" and v.tags or nil
       if not db.feeds[url] then
          ret[#ret + 1] = { url, name, tags }
@@ -320,7 +320,7 @@ cmds.urlview = {
    context = { entry = true },
 }
 
-cmds.list_feeds = {
+cmds.list = {
    impl = function()
       local feedlist = list_feeds()
       for _, v in ipairs(feedlist) do
@@ -373,20 +373,66 @@ cmds.add_feed = {
 
 cmds.update_feed = {
    impl = function(name)
-      name = name or ui_select(list_feeds(), {
-         prompt = "Feed to update",
+      name = name
+         or ui_select(list_feeds(), {
+            prompt = "Feed to update",
+            format_item = function(item)
+               return item[2] or item[1]
+            end,
+         })
+      if not name then
+         return
+      end
+      if type(name) == "string" then
+         -- TODO: check url or name
+         -- name = { name, name }
+      end
+      fetch.update_feed(name, 1)
+   end,
+
+   complete = function()
+      return vim.tbl_keys(list_feeds())
+   end,
+   context = { all = true },
+}
+
+cmds.remove = {
+   doc = "remove a feed from feedlist, but not its entries",
+   impl = function(feed)
+      feed = feed or ui_select(list_feeds(), {
+         prompt = "Feed to remove",
          format_item = function(item)
             return item[2]
          end,
       })
-      if not name then
+      if not feed then
          return
       end
-      fetch.update_feed(type(name) == "table" and name[1] or name, 1)
+      db.feeds[feed[1]] = nil
+      db:save()
    end,
+   context = { all = true },
+}
 
-   complete = function()
-      return vim.tbl_keys(list_feeds())
+cmds.prune = {
+   doc = "remove a feed from feedlist, and all its entries",
+   impl = function(feed)
+      feed = feed or ui_select(list_feeds(), {
+         prompt = "Feed to remove",
+         format_item = function(item)
+            return item[2]
+         end,
+      })
+      if not feed then
+         return
+      end
+      db.feeds[feed[1]] = nil
+      db:save()
+      for entry in db:iter() do
+         if entry.feed == feed[1] then
+            db:rm(entry)
+         end
+      end
    end,
    context = { all = true },
 }
@@ -412,28 +458,6 @@ local function get_item_by_context()
    return choices:totable()
 end
 
-setmetatable(cmds, {
-   __call = function()
-      local items = get_item_by_context()
-      vim.ui.select(items, { prompt = "Feed commands" }, function(choice)
-         if choice then
-            local item = cmds[choice]
-            coroutine.wrap(function()
-               item.impl()
-            end)()
-         end
-      end)
-   end,
-})
-
--- TODO: remove and prune
-
----purge a feed from all of the db, including entries
--- function cmds:prune() end
-
----remove a feed from db.feeds
--- function cmds:remove_feed() end
-
 --- **INTEGRATIONS**
 cmds.telescope = {
    impl = function()
@@ -449,6 +473,20 @@ cmds.grep = {
    context = { all = true },
 }
 
+setmetatable(cmds, {
+   __call = function()
+      local items = get_item_by_context()
+      vim.ui.select(items, { prompt = "Feed commands" }, function(choice)
+         if choice then
+            local item = cmds[choice]
+            coroutine.wrap(function()
+               item.impl()
+            end)()
+         end
+      end)
+   end,
+})
+
 local augroup = vim.api.nvim_create_augroup("Feed", {})
 
 vim.api.nvim_create_autocmd("BufEnter", {

lua/feed/db.lua

@@ -57,6 +57,10 @@ function db_mt:add(entry)
    save_file(self.dir .. "/data/" .. entry.id, "return " .. vim.inspect(entry))
 end
 
+function db_mt:rm(entry)
+   vim.fs.rm(self.dir .. "/data/" .. entry.id)
+end
+
 function db_mt:iter()
    return vim.iter(vim.fs.dir(self.dir .. "/data/")):map(function(id)
       local r = pdofile(self.dir .. "/data/" .. id)
@@ -83,7 +87,7 @@ function db_mt:save(opts)
 end
 
 function db_mt:blowup()
-   vim.fn.delete(self.dir, "rf")
+   vim.fs.rm(self.dir, { recursive = true })
 end
 
 local feeds = pdofile(db_dir .. "/feeds.lua")

lua/feed/feedparser.lua

@@ -3,6 +3,7 @@ local date = require "feed.date"
 local ut = require "feed.utils"
 local format = require "feed.format"
 local sha = vim.fn.sha256
+local log = require "feed.log"
 
 ---check if json
 ---@param str string
@@ -119,20 +120,31 @@ end
 
 local function handle_atom_date(entry)
    local time = entry["published"] or entry["updated"]
-   local ok, res = pcall(date.new_from.atom, time)
-   if ok and res then
-      return res:absolute()
+   if time then
+      local ok, res = pcall(date.new_from.atom, time)
+      if ok and res then
+         return res:absolute()
+      end
    else
+      log.info("date error", vim.inspect(entry))
       return os.time()
    end
 end
 
 local function handle_rss_date(entry)
    local time = entry.pubDate
-   local ok, res = pcall(date.new_from.rss, time)
-   if ok and res then
-      return res:absolute()
+   if time then
+      local ok, res = pcall(date.new_from.rss, time)
+      if ok and res then
+         return res:absolute()
+      else
+         ok, res = pcall(date.new_from.atom, time)
+         if ok and res then
+            return res:absolute()
+         end
+      end
    else
+      log.info("date error", vim.inspect(entry))
       return os.time()
    end
 end
@@ -224,52 +236,72 @@ end
 
 ---walk the ast and retrive usefull info for all three types
 ---@param ast table
----@return feed.feed
-local function reify(ast, feedtype, base_uri)
+---@return feed.feed?
+---@return string?
+local function reify(ast, feedtype, base_uri, last_last)
    local res = {}
+   local lastBuild
    if feedtype == "rss" then
+      if ast.lastBuildDate then
+         lastBuild = tostring(date.new_from.rss(ast.lastBuildDate))
+         if lastBuild == last_last then
+            return
+         end
+      end
       local root_base = ut.url_rebase(ast, base_uri)
       res.title = handle_rss_title(ast)
       res.link = handle_rss_link(ast, base_uri)
-      res.desc = ast.subtitle or res.title
+      res.desc = ast.subtitle or res.title -- TODO:
       res.entries = {}
       if ast.item then
-         for i, v in ipairs(ut.listify(ast.item)) do
-            res.entries[i] = reify_entry(v, "rss", res.title, base_uri) -- TODO:
+         for _, v in ipairs(ut.listify(ast.item)) do
+            res.entries[#res.entries + 1] = reify_entry(v, "rss", res.title, base_uri) -- TODO:
          end
       end
    elseif feedtype == "json" then
+      lastBuild = tostring(date.new_from.json(ast.items[1].date_published))
+      if lastBuild == last_last then
+         return
+      end
       res.title = ast.title
       res.link = ast.home_page_url or ast.feed_url
       res.desc = ast.description or res.title
       res.entries = {}
       if ast.items then
-         for i, v in ipairs(ut.listify(ast.items)) do
-            res.entries[i] = reify_entry(v, "json", res.title, base_uri) -- TODO:
+         for _, v in ipairs(ut.listify(ast.items)) do
+            res.entries[#res.entries + 1] = reify_entry(v, "json", res.title, base_uri) -- TODO:
          end
       end
    elseif feedtype == "atom" then
+      if ast.updated then
+         lastBuild = tostring(date.new_from.atom(ast.updated))
+         if lastBuild == last_last then
+            return
+         end
+      end
       local root_base = ut.url_rebase(ast, base_uri)
       res.title = handle_atom_feed_title(ast)
       res.desc = res.title -- TODO:
       res.link = handle_atom_link(ast, root_base)
       res.entries = {}
       if ast.entry then
-         for i, v in ipairs(ut.listify(ast.entry)) do
-            res.entries[i] = reify_entry(v, "atom", res.title, base_uri)
+         for _, v in ipairs(ut.listify(ast.entry)) do
+            res.entries[#res.entries + 1] = reify_entry(v, "atom", res.title, base_uri)
          end
       end
    end
-   return res
+   return res, lastBuild
 end
 
 ---parse feed fetch from source
 ---@param src string
 ---@param base_uri? string
+---@param lastLast? string
 ---@param opts? { reify : boolean }
----@return table | feed.feed
----@return "json" | "atom" | "rss"
-local function parse(src, base_uri, opts)
+---@return table | feed.feed | nil
+---@return "json" | "atom" | "rss" | nil
+---@return string?
+local function parse(src, base_uri, lastLast, opts)
    src = src:gsub("\n", "")
    local ast, feedtype
    opts = opts or { reify = true }
@@ -287,7 +319,8 @@ local function parse(src, base_uri, opts)
       end
    end
    if opts.reify then
-      return reify(ast, feedtype, base_uri), feedtype
+      local res_or_nil, lastBuild = reify(ast, feedtype, base_uri, lastLast)
+      return res_or_nil, feedtype, lastBuild
    end
    return ast, feedtype
 end

lua/feed/fetch.lua

@@ -58,12 +58,19 @@ M.fetch_co = fetch_co
 ---@param feed table
 ---@param total integer
 function M.update_feed(feed, total)
-   local url, name, tags = unpack(feed)
+   local url, name, tags, lastLast
+   if type(feed) == "table" then
+      url, name, tags = unpack(feed, 1, 3)
+   elseif type(feed) == "string" then
+      url = feed
+   end
+   if db.feeds[url] then
+      lastLast = db.feeds[url].lastBuild
+   end
    local res = fetch_co(url, name)
-   print(url, name)
    if is_valid(res) then
-      local ok, ast, f_type = pcall(feedparser.parse, res.body, url)
-      if ok then
+      local ok, ast, f_type, lastBuild = pcall(feedparser.parse, res.body, url, lastLast)
+      if ok and ast then
          for _, entry in ipairs(ast.entries) do
             if tags then
                for _, v in ipairs(tags) do
@@ -72,17 +79,14 @@ function M.update_feed(feed, total)
             end
             db:add(entry)
          end
-         db.feeds:append { xmlUrl = url, htmlUrl = ast.link, title = name or ast.title, text = ast.desc, type = f_type, tags = tags }
-         db:save() -- TODO: wasteful
+         db.feeds:append { xmlUrl = url, htmlUrl = ast.link, title = name or ast.title, text = ast.desc, type = f_type, tags = tags, lastBuild = lastBuild }
+         db:save()
       else
          log.info("feedparser err for", name)
       end
    else
       log.info("server invalid response err for", name)
    end
-   if total == 1 then
-      db:save()
-   end
    notify.advance(total or 1, name)
 end
 

lua/feed/format.lua

@@ -77,7 +77,11 @@ function M.entry_name(entry)
       if v[1] == "tags" then
          text = M.tags(entry.tags)
       elseif v[1] == "feed" then
-         text = db.feeds[entry.feed].title
+         if db.feeds[entry.feed] then
+            text = db.feeds[entry.feed].title
+         else
+            text = entry.feed
+         end
       end
       if i == #config.layout then
          v.width = vim.api.nvim_win_get_width(0) - acc_width

lua/feed/search.lua

@@ -102,8 +102,8 @@ end
 function M.filter(obj, query)
    local res = {}
    local id_t = {}
-   local iter = obj.iter and obj:iter() or vim.iter(obj)
-   for v in iter do -- just to please the test for now
+   local iter = obj.iter and obj:iter() or vim.iter(obj) -- just to please the test for now
+   for v in iter do
       if query.limit and #res >= query.limit then
          break
       end

lua/feed/xml.lua

@@ -124,6 +124,14 @@ M.CharData = function(node, src)
    end
 end
 
+M.CharRef = function(node, src)
+   local text = get_text(node, src)
+   local num = tonumber(text:sub(3, -2))
+   if num then
+      return string.char(num)
+   end
+end
+
 M.CDSect = function(node, src)
    return get_text(node:child(1), src)
 end