Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/memory #135

Merged
merged 8 commits into from
Sep 17, 2019
Merged

Fix/memory #135

Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
78f8994
Merge pull request #1 from neo-project/master
Sep 10, 2019
e5dfb9e
optimize memory cause
Sep 11, 2019
4781a13
refactor
Sep 11, 2019
f08be7f
1. keep the OnExport function declaration before WriteBlocks;
Sep 16, 2019
eb8fc11
use Store.GetBlocks() and Store.GetTransactions()
Sep 16, 2019
4524e02
Fix compilation issues
shargon Sep 16, 2019
42b9ac0
Remove using
shargon Sep 16, 2019
4a6e978
use Singleton.Store.GetBlock
Sep 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 53 additions & 54 deletions ImportBlocks/ImportBlocks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,14 @@ public override void Configure()
Settings.Load(GetConfiguration());
}

private bool OnExport(string[] args)
shargon marked this conversation as resolved.
Show resolved Hide resolved
private void WriteBlocks(uint start, uint count, string path, bool writeStart)
{
if (args.Length < 2) return false;
if (!string.Equals(args[1], "block", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(args[1], "blocks", StringComparison.OrdinalIgnoreCase))
return false;
if (args.Length >= 3 && uint.TryParse(args[2], out uint start))
uint end = start + count - 1;
using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.WriteThrough))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment on these options?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • FileOption.WriteThrough indicates that the system should write through any intermediate cache and go directly to disk.
  • 4096 is the default size of buffer.

{
if (start > Blockchain.Singleton.Height)
return true;
uint count = args.Length >= 4 ? uint.Parse(args[3]) : uint.MaxValue;
count = Math.Min(count, Blockchain.Singleton.Height - start + 1);
uint end = start + count - 1;
string path = $"chain.{start}.acc";
using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.WriteThrough))
if (fs.Length > 0)
{
if (fs.Length > 0)
if (writeStart)
{
fs.Seek(sizeof(uint), SeekOrigin.Begin);
byte[] buffer = new byte[sizeof(uint)];
Expand All @@ -42,57 +33,65 @@ private bool OnExport(string[] args)
fs.Seek(sizeof(uint), SeekOrigin.Begin);
}
else
{
byte[] buffer = new byte[sizeof(uint)];
shargon marked this conversation as resolved.
Show resolved Hide resolved
fs.Read(buffer, 0, buffer.Length);
start = BitConverter.ToUInt32(buffer, 0);
fs.Seek(0, SeekOrigin.Begin);
}
}
else
{
if (writeStart)
{
fs.Write(BitConverter.GetBytes(start), 0, sizeof(uint));
}
if (start <= end)
fs.Write(BitConverter.GetBytes(count), 0, sizeof(uint));
fs.Seek(0, SeekOrigin.End);
for (uint i = start; i <= end; i++)
}
if (start <= end)
fs.Write(BitConverter.GetBytes(count), 0, sizeof(uint));
fs.Seek(0, SeekOrigin.End);
for (uint i = start; i <= end; i++)
{
using (Snapshot snapshot = Blockchain.Singleton.GetSnapshot())
shargon marked this conversation as resolved.
Show resolved Hide resolved
{
using (Snapshot snapshot = Blockchain.Singleton.GetSnapshot())
{
Block block = snapshot.GetBlock(i);
byte[] array = block.ToArray();
fs.Write(BitConverter.GetBytes(array.Length), 0, sizeof(int));
fs.Write(array, 0, array.Length);
Console.SetCursorPosition(0, Console.CursorTop);
Console.Write($"[{i}/{end}]");
}
Block block = snapshot.GetBlock(i);
byte[] array = block.ToArray();
fs.Write(BitConverter.GetBytes(array.Length), 0, sizeof(int));
fs.Write(array, 0, array.Length);
Console.SetCursorPosition(0, Console.CursorTop);
Console.Write($"[{i}/{end}]");
}
}
}
}

private bool OnExport(string[] args)
{
bool writeStart;
uint start, count;
string path;
if (args.Length < 2) return false;
if (!string.Equals(args[1], "block", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(args[1], "blocks", StringComparison.OrdinalIgnoreCase))
shargon marked this conversation as resolved.
Show resolved Hide resolved
return false;
if (args.Length >= 3 && uint.TryParse(args[2], out start))
{
if (Blockchain.Singleton.Height < start) return true;
count = args.Length >= 4 ? uint.Parse(args[3]) : uint.MaxValue;
count = Math.Min(count, Blockchain.Singleton.Height - start + 1);
path = $"chain.{start}.acc";
writeStart = true;
}
else
{
start = 0;
uint end = Blockchain.Singleton.Height;
uint count = end - start + 1;
string path = args.Length >= 3 ? args[2] : "chain.acc";
using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.WriteThrough))
{
if (fs.Length > 0)
{
byte[] buffer = new byte[sizeof(uint)];
fs.Read(buffer, 0, buffer.Length);
start = BitConverter.ToUInt32(buffer, 0);
fs.Seek(0, SeekOrigin.Begin);
}
if (start <= end)
fs.Write(BitConverter.GetBytes(count), 0, sizeof(uint));
fs.Seek(0, SeekOrigin.End);
for (uint i = start; i <= end; i++)
{
using (Snapshot snapshot = Blockchain.Singleton.GetSnapshot()) {
Block block = snapshot.GetBlock(i);
byte[] array = block.ToArray();
fs.Write(BitConverter.GetBytes(array.Length), 0, sizeof(int));
fs.Write(array, 0, array.Length);
Console.SetCursorPosition(0, Console.CursorTop);
Console.Write($"[{i}/{end}]");
}
}
}
count = Blockchain.Singleton.Height - start + 1;
path = args.Length >= 3 ? args[2] : "chain.acc";
writeStart = false;
}

WriteBlocks(start, count, path, writeStart);

Console.WriteLine();
return true;
}
Expand Down