Merge pull request #471 from hackerzhuli/get_plotsize

src/plotman/_tests/plot_util_test.py

@@ -57,3 +57,10 @@ def test_list_k32_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem):
             [ '/t/plot-k32-0.plot',
               '/t/plot-k32-1.plot',
               '/t/plot-k32-5.plot' ] )
+
+
+def test_get_plotsize():
+    assert (
+        [659272492, 107287518791, 221143636517, 455373353413, 936816632588]
+        == [plot_util.get_plotsize(n) for n in [25, 32, 33, 34, 35]]
+    )

src/plotman/chiapos.py

@@ -0,0 +1,87 @@
+# version = 1.0.2
+# https://github.com/Chia-Network/chiapos/blob/1.0.2/LICENSE
+# https://github.com/Chia-Network/chiapos/blob/1.0.2/src/pos_constants.hpp
+# start ported code
+# Unique plot id which will be used as a ChaCha8 key, and determines the PoSpace.
+kIdLen = 32;
+
+# Distance between matching entries is stored in the offset
+kOffsetSize = 10;
+
+# Max matches a single entry can have, used for hardcoded memory allocation
+kMaxMatchesSingleEntry = 30;
+kMinBuckets = 16;
+kMaxBuckets = 128;
+
+# During backprop and compress, the write pointer is ahead of the read pointer
+# Note that the large the offset, the higher these values must be
+kReadMinusWrite = 1 << kOffsetSize;
+kCachedPositionsSize = kReadMinusWrite * 4;
+
+# Must be set high enough to prevent attacks of fast plotting
+kMinPlotSize = 18;
+
+# Set to 50 since k + kExtraBits + k*4 must not exceed 256 (BLAKE3 output size)
+kMaxPlotSize = 50;
+
+# The amount of spare space used for sort on disk (multiplied time memory buffer size)
+kSpareMultiplier = 5;
+
+# The proportion of memory to allocate to the Sort Manager for reading in buckets and sorting them
+# The lower this number, the more memory must be provided by the caller. However, lowering the
+# number also allows a higher proportion for writing, which reduces seeks for HDD.
+kMemSortProportion = 0.75;
+kMemSortProportionLinePoint = 0.85;
+
+# How many f7s per C1 entry, and how many C1 entries per C2 entry
+kCheckpoint1Interval = 10000;
+kCheckpoint2Interval = 10000;
+
+# F1 evaluations are done in batches of 2^kBatchSizes
+kBatchSizes = 8;
+
+# EPP for the final file, the higher this is, the less variability, and lower delta
+# Note: if this is increased, ParkVector size must increase
+kEntriesPerPark = 2048;
+
+# To store deltas for EPP entries, the average delta must be less than this number of bits
+kMaxAverageDeltaTable1 = 5.6;
+kMaxAverageDelta = 3.5;
+
+# C3 entries contain deltas for f7 values, the max average size is the following
+kC3BitsPerEntry = 2.4;
+
+# The number of bits in the stub is k minus this value
+kStubMinusBits = 3;
+
+#end ported code
+
+# version = 1.0.2
+# https://github.com/Chia-Network/chiapos/blob/1.0.2/LICENSE
+# https://github.com/Chia-Network/chiapos/blob/1.0.2/src/util.hpp
+# start ported code
+def ByteAlign(num_bits):
+    return (num_bits + (8 - ((num_bits) % 8)) % 8)
+# end ported code
+
+# version = 1.0.2
+# https://github.com/Chia-Network/chiapos/blob/1.0.2/LICENSE
+# https://github.com/Chia-Network/chiapos/blob/1.0.2/src/entry_sizes.hpp
+# start ported code
+def CalculateLinePointSize(k):
+    return ByteAlign(2 * k) / 8
+
+# This is the full size of the deltas section in a park. However, it will not be fully filled
+def CalculateMaxDeltasSize(k, table_index):
+    if (table_index == 1):
+        return ByteAlign((kEntriesPerPark - 1) * kMaxAverageDeltaTable1) / 8
+
+    return ByteAlign((kEntriesPerPark - 1) * kMaxAverageDelta) / 8
+
+def CalculateStubsSize(k):
+    return ByteAlign((kEntriesPerPark - 1) * (k - kStubMinusBits)) / 8
+
+def CalculateParkSize(k, table_index):
+    return CalculateLinePointSize(k) + CalculateStubsSize(k) + CalculateMaxDeltasSize(k, table_index);
+
+# end ported code

src/plotman/plot_util.py

@@ -2,6 +2,7 @@
 import os
 import re
 import shutil
+from plotman import chiapos
 
 GB = 1_000_000_000
 
@@ -11,7 +12,10 @@ def df_b(d):
     return usage.free
 
 def get_k32_plotsize():
-    return 108 * GB
+    return get_plotsize(32)
+
+def get_plotsize(k):
+    return (int)(_get_plotsize_scaler(k) * k * pow(2, k))
 
 def human_format(num, precision, powerOfTwo=False):
     divisor = 1024 if powerOfTwo else 1000
@@ -77,3 +81,58 @@ def column_wrap(items, n_cols, filler=None):
         # Pad and truncate
         rows.append( (row_items + ([filler] * n_cols))[:n_cols] )
     return rows
+
+# use k as index to get plotsize_scaler, note that 0 means the value is not calculated yet
+# we can safely assume that k is never going to be greater than 100, due to the exponential nature of plot file size, this avoids using constants from chiapos
+_plotsize_scaler_cache = [0.0 for _ in range(0, 101)]
+
+def calc_average_size_of_entry(k, table_index):
+    '''
+    calculate the average size of entries in bytes, given k and table_index
+    '''
+    # assumes that chia uses constant park size for each table
+    # it is approximately k/8, uses chia's actual park size calculation to get a more accurate estimation
+    return chiapos.CalculateParkSize(k, table_index) / chiapos.kEntriesPerPark
+
+def _get_probability_of_entries_kept(k, table_index):
+    '''
+    get the probibility of entries in table of table_index that is not dropped
+    '''
+    # the formula is derived from https://www.chia.net/assets/proof_of_space.pdf,  section Space Required, p5 and pt
+
+    if table_index > 5:
+        return 1
+
+    pow_2_k = 2**k
+
+    if table_index == 5:
+        return 1 - (1 - 2 / pow_2_k) ** pow_2_k    # p5
+    else:
+        return 1 - (1 - 2 / pow_2_k) ** (_get_probability_of_entries_kept(k, table_index + 1) * pow_2_k) # pt
+
+def _get_plotsize_scaler(k:int):
+    '''
+    get scaler for plot size so that the plot size can be calculated by scaler * k * 2 ** k
+    '''    
+    result = _plotsize_scaler_cache[k]
+    if result > 0:
+        return result
+    result = _get_plotsize_scaler_impl(k)
+    _plotsize_scaler_cache[k] = result
+    return result
+
+def _get_plotsize_scaler_impl(k):
+    '''
+    get scaler for plot size so that the plot size can be calculated by scaler * k * 2 ** k
+    '''
+
+    result = 0
+    # there are 7 tables
+    for i in range(1, 8):
+        probability = _get_probability_of_entries_kept(k, i)
+        average_size_of_entry = calc_average_size_of_entry(k, i)
+        scaler_for_table = probability * average_size_of_entry / k
+        result += scaler_for_table
+
+    return result
+