Use ID based method instead of dates for determining message freshness

 New workaround for vysheng/tg#1077. Possibly fixes #35.

dumpers/lib/dumper_interface.rb

@@ -30,16 +30,7 @@ def start_dialog(dialog, progress)
   # This default makes sense in simple cases, override for advanced custom logic
   def msg_fresh?(msg, progress)
     # msg: Hash, progress: DumpProgress
-
-    return true unless progress.newest_date
-
-    if msg['date'] && msg['date'] > progress.newest_date
-      return true
-    elsif msg['date'] == progress.newest_date && progress.newest_id
-      return true if msg['id'] && msg['id'].to_s > progress.newest_id.to_s
-    end
-
-    false
+    !progress.newest_id || MsgId.new(msg['id']) > progress.newest_id
   end
 
   # Will be called for each message to dump (from newest to oldest)

lib/dump_progress.rb

@@ -1,4 +1,5 @@
 require 'json'
+require_relative 'msg_id'
 
 class DumpProgress
 
@@ -8,7 +9,7 @@ class DumpProgress
   attr_writer :dumper_state
 
   def initialize(newest_id = nil, newest_date = nil, dumper_state = {})
-    @newest_id = newest_id
+    @newest_id = newest_id ? MsgId.new(newest_id) : nil
     @newest_date = newest_date
     @dumper_state = dumper_state
   end
@@ -23,7 +24,7 @@ def self.from_hash(hash)
 
   def to_hash
     {
-      :newest_id => @newest_id,
+      :newest_id => @newest_id.to_s,
       :newest_date => @newest_date,
       :dumper_state => @dumper_state
     }
@@ -34,9 +35,10 @@ def to_json(*a)
   end
 
   def update(msg)
-    if !@newest_date || (msg['date'] && msg['date'] >= @newest_date)
+    msg_id = msg['id'] ? MsgId.new(msg['id']) : nil
+    if msg_id && (!@newest_id || msg_id > @newest_id)
+      @newest_id = msg_id
       @newest_date = msg['date'] || @newest_date
-      @newest_id = (msg['id'] || @newest_id).to_s
     end
   end
 

lib/msg_id.rb

@@ -0,0 +1,43 @@
+require_relative 'util'
+
+class MsgId
+  include Comparable
+
+  attr_reader :raw_hex
+
+  def initialize(raw_hex)
+    @raw_hex = raw_hex
+  end
+
+  def sequence_hex
+    @sequence_hex ||= decode_sequence
+  end
+
+  def <=>(other)
+    sequence_hex <=> other.sequence_hex
+  end
+
+  def to_s
+    @raw_hex
+  end
+
+  private
+
+  def assert_length
+    unless [48,64].include?(@raw_hex.length)
+      raise 'Unexpected message ID length (%d). Please report this as an '\
+            'issue if your telegram-cli is up to date.' % [@raw_hex.length]
+    end
+  end
+
+  # This function extracts a sortable hex string from the full ID
+  # The following assumptions are made about the telegram-cli build:
+  # * `sizeof(long long)` equals 8
+  # * Little endian byte order
+  def decode_sequence
+    assert_length
+    sequence_hex_le = @raw_hex[-32,16]
+    flip_bytes(sequence_hex_le)
+  end
+
+end

lib/util.rb

@@ -49,3 +49,11 @@ def truncate_to_bytesize(str, size, ellipsis = '')
     end
   end
 end
+
+def system_big_endian?
+  [1].pack('I') == [1].pack('N')
+end
+
+def flip_bytes(hex)
+  hex.scan(/../).reverse.join('')
+end

telegram-history-dump.rb

@@ -230,6 +230,11 @@ def save_progress
 )
 $log = Logger.new(STDOUT)
 
+if $config['track_progress'] && system_big_endian?
+  raise 'For reasons you do not want to know, a little endian system is '\
+        'necessary for incremental backups. Please report this as an issue.'
+end
+
 unless cli_opts.userdir.nil? || cli_opts.userdir.empty?
   $config['backup_dir'] = File.join($config['backup_dir'], cli_opts.userdir)
 end