Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add information about the cache #246

Merged
merged 3 commits into from
Sep 23, 2023
Merged

Add information about the cache #246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0cf8ccd
Add information about the cache
RyuzakiKK Sep 20, 2023
fd69d09
Reduce code duplication in info_test
RyuzakiKK Sep 20, 2023
03797c5
Add info test with cache and without a seed
RyuzakiKK Sep 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 41 additions & 10 deletions cmd/desync/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ type infoOptions struct {
cmdStoreOptions
stores []string
seeds []string
cache string
printFormat string
}

Expand All @@ -38,6 +39,7 @@ in the seeds are also shown. Use '-' to read the index from STDIN.`,
flags := cmd.Flags()
flags.StringSliceVarP(&opt.stores, "store", "s", nil, "source store(s)")
flags.StringSliceVar(&opt.seeds, "seed", nil, "seed indexes")
flags.StringVarP(&opt.cache, "cache", "c", "", "store to be used as cache")
flags.StringVarP(&opt.printFormat, "format", "f", "json", "output format, plain or json")
addStoreOptions(&opt.cmdStoreOptions, flags)
return cmd
Expand All @@ -55,15 +57,18 @@ func runInfo(ctx context.Context, opt infoOptions, args []string) error {
}

var results struct {
Total int `json:"total"`
Unique int `json:"unique"`
InStore uint64 `json:"in-store"`
InSeed uint64 `json:"in-seed"`
Size uint64 `json:"size"`
SizeNotInSeed uint64 `json:"dedup-size-not-in-seed"`
ChunkSizeMin uint64 `json:"chunk-size-min"`
ChunkSizeAvg uint64 `json:"chunk-size-avg"`
ChunkSizeMax uint64 `json:"chunk-size-max"`
Total int `json:"total"`
Unique int `json:"unique"`
InStore uint64 `json:"in-store"`
InSeed uint64 `json:"in-seed"`
InCache uint64 `json:"in-cache"`
NotInSeedNorCache uint64 `json:"not-in-seed-nor-cache"`
Size uint64 `json:"size"`
SizeNotInSeed uint64 `json:"dedup-size-not-in-seed"`
SizeNotInSeedNorCache uint64 `json:"dedup-size-not-in-seed-nor-cache"`
ChunkSizeMin uint64 `json:"chunk-size-min"`
ChunkSizeAvg uint64 `json:"chunk-size-avg"`
ChunkSizeMax uint64 `json:"chunk-size-max"`
}

dedupedSeeds := make(map[desync.ChunkID]struct{})
Expand Down Expand Up @@ -93,6 +98,14 @@ func runInfo(ctx context.Context, opt infoOptions, args []string) error {
results.ChunkSizeAvg = c.Index.ChunkSizeAvg
results.ChunkSizeMax = c.Index.ChunkSizeMax

var cache desync.WriteStore
if opt.cache != "" {
cache, err = WritableStore(opt.cache, opt.cmdStoreOptions)
if err != nil {
return err
}
}

// Go through each chunk from the index to count them, de-dup each chunks
// with a map and calculate the size of the chunks that are not available
// in seed
Expand All @@ -110,14 +123,29 @@ func runInfo(ctx context.Context, opt infoOptions, args []string) error {
continue
}

inSeed := false
inCache := false
deduped[chunk.ID] = struct{}{}
if _, isAvailable := dedupedSeeds[chunk.ID]; isAvailable {
// This chunk is available in the seed
results.InSeed++
} else {
inSeed = true
}
if cache != nil {
if hasChunk, _ := cache.HasChunk(chunk.ID); hasChunk {
results.InCache++
inCache = true
}
}

if !inSeed {
// The seed doesn't have this chunk, sum its size
results.SizeNotInSeed += chunk.Size
}
if !inSeed && !inCache {
results.NotInSeedNorCache++
results.SizeNotInSeedNorCache += chunk.Size
}
}
results.Unique = len(deduped)

Expand Down Expand Up @@ -156,10 +184,13 @@ func runInfo(ctx context.Context, opt infoOptions, args []string) error {
case "plain":
fmt.Println("Blob size:", results.Size)
fmt.Println("Size of deduplicated chunks not in seed:", results.SizeNotInSeed)
fmt.Println("Size of deduplicated chunks not in seed nor cache:", results.SizeNotInSeedNorCache)
fmt.Println("Total chunks:", results.Total)
fmt.Println("Unique chunks:", results.Unique)
fmt.Println("Chunks in store:", results.InStore)
fmt.Println("Chunks in seed:", results.InSeed)
fmt.Println("Chunks in cache:", results.InCache)
fmt.Println("Chunks not in seed nor cache:", results.NotInSeedNorCache)
fmt.Println("Chunk size min:", results.ChunkSizeMin)
fmt.Println("Chunk size avg:", results.ChunkSizeAvg)
fmt.Println("Chunk size max:", results.ChunkSizeMax)
Expand Down
153 changes: 89 additions & 64 deletions cmd/desync/info_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,71 +11,96 @@ import (
)

func TestInfoCommand(t *testing.T) {
expectedOutput := []byte(`{
"total": 161,
"unique": 131,
"in-store": 131,
"in-seed": 0,
"size": 2097152,
"dedup-size-not-in-seed": 1114112,
"chunk-size-min": 2048,
"chunk-size-avg": 8192,
"chunk-size-max": 32768
}`)
exp := make(map[string]interface{})
err := json.Unmarshal(expectedOutput, &exp)
require.NoError(t, err)
for _, test := range []struct {
name string
args []string
expectedOutput []byte
}{
{"info command with store",
[]string{"-s", "testdata/blob1.store", "testdata/blob1.caibx"},
[]byte(`{
"total": 161,
"unique": 131,
"in-store": 131,
"in-seed": 0,
"in-cache": 0,
"not-in-seed-nor-cache": 131,
"size": 2097152,
"dedup-size-not-in-seed": 1114112,
"dedup-size-not-in-seed-nor-cache": 1114112,
"chunk-size-min": 2048,
"chunk-size-avg": 8192,
"chunk-size-max": 32768
}`)},
{"info command with seed",
[]string{"-s", "testdata/blob1.store", "--seed", "testdata/blob2.caibx", "testdata/blob1.caibx"},
[]byte(`{
"total": 161,
"unique": 131,
"in-store": 131,
"in-seed": 124,
"in-cache": 0,
"not-in-seed-nor-cache": 7,
"size": 2097152,
"dedup-size-not-in-seed": 80029,
"dedup-size-not-in-seed-nor-cache": 80029,
"chunk-size-min": 2048,
"chunk-size-avg": 8192,
"chunk-size-max": 32768
}`)},
{"info command with seed and cache",
[]string{"-s", "testdata/blob2.store", "--seed", "testdata/blob1.caibx", "--cache", "testdata/blob2.cache", "testdata/blob2.caibx"},
[]byte(`{
"total": 161,
"unique": 131,
"in-store": 131,
"in-seed": 124,
"in-cache": 18,
"not-in-seed-nor-cache": 5,
"size": 2097152,
"dedup-size-not-in-seed": 80029,
"dedup-size-not-in-seed-nor-cache": 67099,
"chunk-size-min": 2048,
"chunk-size-avg": 8192,
"chunk-size-max": 32768
}`)},
{"info command with cache",
[]string{"-s", "testdata/blob2.store", "--cache", "testdata/blob2.cache", "testdata/blob2.caibx"},
[]byte(`{
"total": 161,
"unique": 131,
"in-store": 131,
"in-seed": 0,
"in-cache": 18,
"not-in-seed-nor-cache": 113,
"size": 2097152,
"dedup-size-not-in-seed": 1114112,
"dedup-size-not-in-seed-nor-cache": 950410,
"chunk-size-min": 2048,
"chunk-size-avg": 8192,
"chunk-size-max": 32768
}`)},
} {
t.Run(test.name, func(t *testing.T) {
exp := make(map[string]interface{})
err := json.Unmarshal(test.expectedOutput, &exp)
require.NoError(t, err)

cmd := newInfoCommand(context.Background())
cmd.SetArgs([]string{"-s", "testdata/blob1.store", "testdata/blob1.caibx"})
b := new(bytes.Buffer)
cmd := newInfoCommand(context.Background())
cmd.SetArgs(test.args)
b := new(bytes.Buffer)

// Redirect the command's output
stdout = b
cmd.SetOutput(ioutil.Discard)
_, err = cmd.ExecuteC()
require.NoError(t, err)
// Redirect the command's output
stdout = b
cmd.SetOutput(ioutil.Discard)
_, err = cmd.ExecuteC()
require.NoError(t, err)

// Decode the output and compare to what's expected
got := make(map[string]interface{})
err = json.Unmarshal(b.Bytes(), &got)
require.NoError(t, err)
require.Equal(t, exp, got)
}

func TestInfoCommandWithSeed(t *testing.T) {
expectedOutput := []byte(`{
"total": 161,
"unique": 131,
"in-store": 131,
"in-seed": 124,
"size": 2097152,
"dedup-size-not-in-seed": 80029,
"chunk-size-min": 2048,
"chunk-size-avg": 8192,
"chunk-size-max": 32768
}`)
exp := make(map[string]interface{})
err := json.Unmarshal(expectedOutput, &exp)
require.NoError(t, err)

cmd := newInfoCommand(context.Background())
cmd.SetArgs([]string{
"-s", "testdata/blob1.store",
"--seed", "testdata/blob2.caibx",
"testdata/blob1.caibx",
})
b := new(bytes.Buffer)

// Redirect the command's output
stdout = b
cmd.SetOutput(ioutil.Discard)
_, err = cmd.ExecuteC()
require.NoError(t, err)

// Decode the output and compare to what's expected
got := make(map[string]interface{})
err = json.Unmarshal(b.Bytes(), &got)
require.NoError(t, err)
require.Equal(t, exp, got)
// Decode the output and compare to what's expected
got := make(map[string]interface{})
err = json.Unmarshal(b.Bytes(), &got)
require.NoError(t, err)
require.Equal(t, exp, got)
})
}
}
Loading