-
Notifications
You must be signed in to change notification settings - Fork 44
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add new
inspect-chunks command for more detailed chunks info
Add a new `inspect-chunks` command that can provide additional information about the chunks in an index, including their compressed size (if a local store is provided). One use case is for the clients to download the server's inspect-chunks JSON and feed it into `desync info`, to calculate the precise download size of an update. Signed-off-by: Ludovico de Nittis <[email protected]>
- Loading branch information
Showing
12 changed files
with
2,421 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| package desync | ||
|
|
||
| // ChunkAdditionalInfo contains detailed information about a particular chunk. | ||
| // Some of those info, e.g. CompressedSize, are only exact for the store used when | ||
| // generating it. Because other stores could potentially use different compression levels. | ||
| type ChunkAdditionalInfo struct { | ||
| ID ChunkID `json:"id"` | ||
| UncompressedSize uint64 `json:"uncompressed_size"` | ||
| CompressedSize int64 `json:"compressed_size,omitempty"` | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "github.com/folbricht/desync" | ||
| "github.com/spf13/cobra" | ||
| "io" | ||
| "os" | ||
| ) | ||
|
|
||
| type inspectChunksOptions struct { | ||
| cmdStoreOptions | ||
| store string | ||
| } | ||
|
|
||
| func newinspectChunksCommand(ctx context.Context) *cobra.Command { | ||
| var opt inspectChunksOptions | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: "inspect-chunks <index> [<output>]", | ||
| Short: "Inspect chunks from an index and an optional local store", | ||
| Long: `Prints a detailed JSON with information about chunks stored in an index file. | ||
| By using the '--store' option to provide a local store, the generated JSON will include, if | ||
| available, the chunks compressed size info from that particular store.`, | ||
| Example: ` desync inspect-chunks file.caibx | ||
| desync inspect-chunks --store /mnt/store file.caibx inspect_result.json`, | ||
| Args: cobra.RangeArgs(1, 2), | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| return runInspectChunks(ctx, opt, args) | ||
| }, | ||
| SilenceUsage: true, | ||
| } | ||
| flags := cmd.Flags() | ||
| flags.StringVarP(&opt.store, "store", "s", "", "local source store") | ||
| addStoreOptions(&opt.cmdStoreOptions, flags) | ||
| return cmd | ||
| } | ||
|
|
||
| func runInspectChunks(ctx context.Context, opt inspectChunksOptions, args []string) error { | ||
| if err := opt.cmdStoreOptions.validate(); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var ( | ||
| outFile io.Writer | ||
| err error | ||
| ) | ||
| if len(args) == 2 { | ||
| outFileName := args[1] | ||
| outFile, err = os.Create(outFileName) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } else { | ||
| outFile = stdout | ||
| } | ||
|
|
||
| // Read the input | ||
| c, err := readCaibxFile(args[0], opt.cmdStoreOptions) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var ( | ||
| chunksInfo []desync.ChunkAdditionalInfo | ||
| s desync.LocalStore | ||
| ) | ||
|
|
||
| if opt.store != "" { | ||
| sr, err := storeFromLocation(opt.store, opt.cmdStoreOptions) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // We expect a local store, it is an error to provide something different | ||
| var ok bool | ||
| s, ok = sr.(desync.LocalStore) | ||
|
|
||
| if !ok { | ||
| return fmt.Errorf("'%s' is not a local store", opt.store) | ||
| } | ||
| } | ||
|
|
||
| for _, chunk := range c.Chunks { | ||
| var size int64 = 0 | ||
| // Get the compressed size only if the store actually has compressed chunks | ||
| if opt.store != "" && !s.Opt.Uncompressed { | ||
| size, _ = s.GetChunkSize(chunk.ID) | ||
| } | ||
|
|
||
| chunksInfo = append(chunksInfo, desync.ChunkAdditionalInfo{ | ||
| ID: chunk.ID, | ||
| UncompressedSize: chunk.Size, | ||
| CompressedSize: size, | ||
| }) | ||
| // See if we're meant to stop | ||
| select { | ||
| case <-ctx.Done(): | ||
| return nil | ||
| default: | ||
| } | ||
| } | ||
|
|
||
| return printJSON(outFile, chunksInfo) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "encoding/json" | ||
| "io/ioutil" | ||
| "os" | ||
| "testing" | ||
|
|
||
| "github.com/folbricht/desync" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestInspectChunksCommand(t *testing.T) { | ||
| for _, test := range []struct { | ||
| name string | ||
| args []string | ||
| expectedOutputJSON string | ||
| }{ | ||
| {"inspect the chunks info with a local store", | ||
| []string{"-s", "testdata/blob2.store", "testdata/blob2.caibx"}, | ||
| "testdata/blob2_chunks_info.json", | ||
| }, | ||
| {"run inspect with a seed that doesn't have all the compressed chunks", | ||
| []string{"-s", "testdata/blob2.cache", "testdata/blob2.caibx"}, | ||
| "testdata/blob2_chunks_info_missing.json", | ||
| }, | ||
| {"inspect the chunks info without any stores", | ||
| []string{"testdata/blob2.caibx"}, | ||
| "testdata/blob2_chunks_info_no_store.json", | ||
| }, | ||
| } { | ||
| t.Run(test.name, func(t *testing.T) { | ||
| var exp []desync.ChunkAdditionalInfo | ||
| be, err := os.ReadFile(test.expectedOutputJSON) | ||
| require.NoError(t, err) | ||
| err = json.Unmarshal(be, &exp) | ||
| require.NoError(t, err) | ||
|
|
||
| cmd := newinspectChunksCommand(context.Background()) | ||
| cmd.SetArgs(test.args) | ||
| b := new(bytes.Buffer) | ||
|
|
||
| // Redirect the command's output | ||
| stdout = b | ||
| cmd.SetOutput(ioutil.Discard) | ||
| _, err = cmd.ExecuteC() | ||
| require.NoError(t, err) | ||
|
|
||
| // Decode the output and compare to what's expected | ||
| var got []desync.ChunkAdditionalInfo | ||
| err = json.Unmarshal(b.Bytes(), &got) | ||
| require.NoError(t, err) | ||
| require.Equal(t, exp, got) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.