Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/memory #135

Merged
merged 8 commits into from
Sep 17, 2019
Merged

Fix/memory #135

Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
78f8994
Merge pull request #1 from neo-project/master
Sep 10, 2019
e5dfb9e
optimize memory cause
Sep 11, 2019
4781a13
refactor
Sep 11, 2019
f08be7f
1. keep the OnExport function declaration before WriteBlocks;
Sep 16, 2019
eb8fc11
use Store.GetBlocks() and Store.GetTransactions()
Sep 16, 2019
4524e02
Fix compilation issues
shargon Sep 16, 2019
42b9ac0
Remove using
shargon Sep 16, 2019
4a6e978
use Singleton.Store.GetBlock
Sep 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 55 additions & 50 deletions ImportBlocks/ImportBlocks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using Neo.Ledger;
using Neo.Network.P2P.Payloads;
using Neo.Persistence;
using Neo.IO.Caching;
using System;
using System.IO;

Expand All @@ -19,79 +20,83 @@ public override void Configure()

private bool OnExport(string[] args)
shargon marked this conversation as resolved.
Show resolved Hide resolved
{
bool writeStart;
uint start, count;
string path;
if (args.Length < 2) return false;
if (!string.Equals(args[1], "block", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(args[1], "blocks", StringComparison.OrdinalIgnoreCase))
return false;
if (args.Length >= 3 && uint.TryParse(args[2], out uint start))
if (args.Length >= 3 && uint.TryParse(args[2], out start))
{
if (start > Blockchain.Singleton.Height)
return true;
uint count = args.Length >= 4 ? uint.Parse(args[3]) : uint.MaxValue;
if (Blockchain.Singleton.Height < start) return true;
count = args.Length >= 4 ? uint.Parse(args[3]) : uint.MaxValue;
count = Math.Min(count, Blockchain.Singleton.Height - start + 1);
uint end = start + count - 1;
string path = $"chain.{start}.acc";
using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
path = $"chain.{start}.acc";
writeStart = true;
}
else
{
start = 0;
count = Blockchain.Singleton.Height - start + 1;
path = args.Length >= 3 ? args[2] : "chain.acc";
writeStart = false;
}

WriteBlocks(start, count, path, writeStart);

Console.WriteLine();
return true;
}

private void WriteBlocks(uint start, uint count, string path, bool writeStart)
{
uint end = start + count - 1;
using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.WriteThrough))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment on these options?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • FileOption.WriteThrough indicates that the system should write through any intermediate cache and go directly to disk.
  • 4096 is the default size of buffer.

{
if (fs.Length > 0)
{
if (fs.Length > 0)
byte[] buffer = new byte[sizeof(uint)];
if (writeStart)
{
fs.Seek(sizeof(uint), SeekOrigin.Begin);
byte[] buffer = new byte[sizeof(uint)];
fs.Read(buffer, 0, buffer.Length);
start += BitConverter.ToUInt32(buffer, 0);
fs.Seek(sizeof(uint), SeekOrigin.Begin);
}
else
{
fs.Write(BitConverter.GetBytes(start), 0, sizeof(uint));
fs.Read(buffer, 0, buffer.Length);
start = BitConverter.ToUInt32(buffer, 0);
fs.Seek(0, SeekOrigin.Begin);
}
if (start <= end)
fs.Write(BitConverter.GetBytes(count), 0, sizeof(uint));
fs.Seek(0, SeekOrigin.End);
using (Snapshot snapshot = Blockchain.Singleton.GetSnapshot())
for (uint i = start; i <= end; i++)
{
Block block = snapshot.GetBlock(i);
byte[] array = block.ToArray();
fs.Write(BitConverter.GetBytes(array.Length), 0, sizeof(int));
fs.Write(array, 0, array.Length);
Console.SetCursorPosition(0, Console.CursorTop);
Console.Write($"[{i}/{end}]");
}
}
}
else
{
start = 0;
uint end = Blockchain.Singleton.Height;
uint count = end - start + 1;
string path = args.Length >= 3 ? args[2] : "chain.acc";
using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
else
{
if (fs.Length > 0)
if (writeStart)
{
byte[] buffer = new byte[sizeof(uint)];
fs.Read(buffer, 0, buffer.Length);
start = BitConverter.ToUInt32(buffer, 0);
fs.Seek(0, SeekOrigin.Begin);
fs.Write(BitConverter.GetBytes(start), 0, sizeof(uint));
}
if (start <= end)
fs.Write(BitConverter.GetBytes(count), 0, sizeof(uint));
fs.Seek(0, SeekOrigin.End);
}
if (start <= end)
fs.Write(BitConverter.GetBytes(count), 0, sizeof(uint));
fs.Seek(0, SeekOrigin.End);
for (uint i = start; i <= end; i++)
{
using (Snapshot snapshot = Blockchain.Singleton.GetSnapshot())
shargon marked this conversation as resolved.
Show resolved Hide resolved
for (uint i = start; i <= end; i++)
{
Block block = snapshot.GetBlock(i);
byte[] array = block.ToArray();
fs.Write(BitConverter.GetBytes(array.Length), 0, sizeof(int));
fs.Write(array, 0, array.Length);
Console.SetCursorPosition(0, Console.CursorTop);
Console.Write($"[{i}/{end}]");
}
{
DataCache<UInt256, BlockState> blocks = Blockchain.Singleton.Store.GetBlocks();
shargon marked this conversation as resolved.
Show resolved Hide resolved
DataCache<UInt256, TransactionState> transactions = Blockchain.Singleton.Store.GetTransactions();
UInt256 hash = Blockchain.Singleton.GetBlockHash(i);
Block block = blocks.TryGet(hash).TrimmedBlock.GetBlock(transactions);
shargon marked this conversation as resolved.
Show resolved Hide resolved
byte[] array = block.ToArray();
fs.Write(BitConverter.GetBytes(array.Length), 0, sizeof(int));
fs.Write(array, 0, array.Length);
Console.SetCursorPosition(0, Console.CursorTop);
Console.Write($"[{i}/{end}]");
}
}
}
Console.WriteLine();
return true;
}

private bool OnHelp(string[] args)
Expand Down