Partial logs, readme

Jiiks · Aug 9, 2016 · 5b7db58 · 5b7db58
1 parent 0fd5a8f
commit 5b7db58
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 64 deletions.
diff --git a/DiscordArchiver/DiscordArchiver.csproj b/DiscordArchiver/DiscordArchiver.csproj
@@ -32,28 +32,8 @@
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Discord.Net, Version=0.9.0.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Discord.Net.0.9.0-rc3-3\lib\net45\Discord.Net.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="Newtonsoft.Json, Version=8.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
-      <HintPath>..\packages\Newtonsoft.Json.8.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="Nito.AsyncEx, Version=3.0.1.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Nito.AsyncEx.3.0.1\lib\net45\Nito.AsyncEx.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="Nito.AsyncEx.Concurrent, Version=3.0.1.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Nito.AsyncEx.3.0.1\lib\net45\Nito.AsyncEx.Concurrent.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="Nito.AsyncEx.Enlightenment, Version=3.0.1.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Nito.AsyncEx.3.0.1\lib\net45\Nito.AsyncEx.Enlightenment.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="RestSharp, Version=105.2.3.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\RestSharp.105.2.3\lib\net45\RestSharp.dll</HintPath>
+    <Reference Include="Newtonsoft.Json, Version=9.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="System" />
@@ -64,10 +44,6 @@
     <Reference Include="System.Data" />
     <Reference Include="System.Net.Http" />
     <Reference Include="System.Xml" />
-    <Reference Include="WebSocket4Net, Version=0.14.1.0, Culture=neutral, PublicKeyToken=eb4e154b696bf72a, processorArchitecture=MSIL">
-      <HintPath>..\packages\WebSocket4Net.0.14.1\lib\net45\WebSocket4Net.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Archive.cs" />

diff --git a/DiscordArchiver/Program.cs b/DiscordArchiver/Program.cs
@@ -16,9 +16,10 @@ class Program {
         public static string BaseUrl = "https://discordapp.com/api/channels/{0}/messages?token={1}&before={2}&limit={3}";
 
         public static string Channel, Token, Out = "out.json", Before = "", After = "";
-        public static int Limit = 50;
+        public static int Limit = 100;
 
         static void Main(string[] args) {
+
             Channel = args[0];
             Token = args[1];
 
@@ -35,47 +36,53 @@ static void Main(string[] args) {
                 int.TryParse(args[5], out Limit);
             }
 
-            List<DMessage> fullLogs = new List<DMessage>();
+            var fullLogs = new List<DMessage>();
+
+            var counter = 0;
+            var exit = false;
+            var bw = new BackgroundWorker();
 
-            int counter = 0;
-            bool exit = false;
-            BackgroundWorker bw = new BackgroundWorker();
             bw.DoWork += (sender, eventArgs) => {
                 while (true) {
                     counter++;
-                    Console.WriteLine($"Downloading Log Part {counter}");
+                    Console.WriteLine($"Downloading Logs Part {counter}");
 
-                    string currentLog = "";
-                    using (WebClient wc = new WebClient()) {
+                    var currentLog = "";
+                    using (var wc = new WebClient()) {
                         currentLog = wc.DownloadString(string.Format(BaseUrl, Channel, Token, Before, Limit));
                     }
 
-                    Console.WriteLine($"Downloaded Log Part {counter}");
-
-                    Console.WriteLine($"Parsing Log Part {counter}");
+                    Console.WriteLine($"Downloaded Log Part {counter}, Parsing");
 
-                    JArray jar = JArray.Parse(currentLog);
+                    var jar = JArray.Parse(currentLog);
 
                     Before = jar[jar.Count - 1]["id"].ToString();
-
-                    int g = -1;
-                    int mg = -1;
-                    foreach (JToken jToken in jar) {
+                    Console.WriteLine($"Before: {Before}");
+                    var g = -1;
+                    var mg = -1;
+                    foreach (var jToken in jar) {
                         g++;
-                        if ((string) jToken["id"] != After) continue;
+                        if ((string)jToken["id"] != After) continue;
                         mg = g;
                         break;
                     }
 
                     if (mg > -1) {
                         exit = true;
-                        for (int i = mg; i < jar.Count - 1; i++) {
+                        for (var i = mg; i < jar.Count - 1; i++) {
                             jar.RemoveAt(i);
                         }
                     }
-
-                    fullLogs.InsertRange(0,JArray.Parse(currentLog).Select(jToken => jToken.ToObject<DMessage>()).Reverse().ToList());
-                    Thread.Sleep(300);
+
+                    fullLogs.InsertRange(0, JArray.Parse(currentLog).Select(jToken => jToken.ToObject<DMessage>()).Reverse().ToList());
+
+                    if (counter % 50 == 0) {
+                        Console.WriteLine($"Writing partial logs to file {Out}");
+                        File.WriteAllText(Out, JsonConvert.SerializeObject(fullLogs, new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }));
+                    }
+
+                    Thread.Sleep(500);
+
                     if (jar.Count < Limit || exit) break;
                 }
             };

diff --git a/DiscordArchiver/data/DAuthor.cs b/DiscordArchiver/data/DAuthor.cs
@@ -1,11 +1,9 @@
 namespace DiscordArchiver.data {
 
     public class DAuthor {
-
         public string username;
-        public string discriminator;
-        public string id;
+        public ushort discriminator;
+        public ulong id;
         public string avatar;
-
     }
-}
+}
diff --git a/DiscordArchiver/data/DMessage.cs b/DiscordArchiver/data/DMessage.cs
@@ -1,18 +1,12 @@
 using System;
 
 namespace DiscordArchiver.data {
-
     public class DMessage {
-        public DAttachment[] attachments;
-        public bool tts;
-        public DEmbed[] embeds;
-        public DateTime? timestamp;
-        public bool mention_everyone;
-        public string id;
-        public DateTime? edited_timestamp;
+        public DateTime timestamp;
+        public ulong id;
         public DAuthor author;
         public string content;
-        public string channel_id;
+        public ulong channel_id;
         public DAuthor[] mentions;
     }
-}
+}
diff --git a/DiscordArchiver/packages.config b/DiscordArchiver/packages.config
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="Discord.Net" version="0.9.0-rc3-3" targetFramework="net45" />
-  <package id="Newtonsoft.Json" version="8.0.1" targetFramework="net45" />
+  <package id="Newtonsoft.Json" version="9.0.1" targetFramework="net45" />
   <package id="Nito.AsyncEx" version="3.0.1" targetFramework="net45" />
   <package id="RestSharp" version="105.2.3" targetFramework="net45" />
   <package id="WebSocket4Net" version="0.14.1" targetFramework="net45" />

diff --git a/README.md b/README.md
@@ -2,4 +2,4 @@
 Dl and archive Discord chat logs
 
 Usage:
-DiscordArchiver.exe <channelid> <token> [outputfile] [before message id] [after message id] [message limit per request]
+DiscordArchiver.exe (channelid) (token) [outputfile] [before message id] [after message id] [message limit per request]