Skip to content

Commit

Permalink
refactor: refactor functions related to querying referrers (#353)
Browse files Browse the repository at this point in the history 
1. Use zero digest
(`sha256:0000000000000000000000000000000000000000000000000000000000000000`)
for pinging referrers API
2. Check `NAME_UNKNOWN` error code

Signed-off-by: Lixia (Sylvia) Lei <[email protected]>
  • Loading branch information
@Wwwsylvia
Wwwsylvia authored Oct 31, 2022
1 parent 7ab862c commit f701e60
Show file tree
Hide file tree
Showing 5 changed files with 460 additions and 69 deletions.
29 changes: 27 additions & 2 deletions registry/remote/errcode/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,30 @@ import (
"unicode"
)

// References:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#error-codes
// - https://docs.docker.com/registry/spec/api/#errors-2
const (
ErrorCodeBlobUnknown = "BLOB_UNKNOWN"
ErrorCodeBlobUploadInvalid = "BLOB_UPLOAD_INVALID"
ErrorCodeBlobUploadUnknown = "BLOB_UPLOAD_UNKNOWN"
ErrorCodeDigestInvalid = "DIGEST_INVALID"
ErrorCodeManifestBlobUnknown = "MANIFEST_BLOB_UNKNOWN"
ErrorCodeManifestInvalid = "MANIFEST_INVALID"
ErrorCodeManifestUnknown = "MANIFEST_UNKNOWN"
ErrorCodeNameInvalid = "NAME_INVALID"
ErrorCodeNameUnknown = "NAME_UNKNOWN"
ErrorCodeSizeInvalid = "SIZE_INVALID"
ErrorCodeUnauthorized = "UNAUTHORIZED"
ErrorCodeDenied = "DENIED"
ErrorCodeUnsupported = "UNSUPPORTED"
)

// Error represents a response inner error returned by the remote
// registry.
// References:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#error-codes
// - https://docs.docker.com/registry/spec/api/#errors-2
type Error struct {
Code string `json:"code"`
Message string `json:"message"`
Expand All @@ -48,8 +70,11 @@ func (e Error) Error() string {
return fmt.Sprintf("%s: %s: %v", code, e.Message, e.Detail)
}

// Errors represents a list of response inner errors returned by
// the remote server.
// Errors represents a list of response inner errors returned by the remote
// server.
// References:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#error-codes
// - https://docs.docker.com/registry/spec/api/#errors-2
type Errors []Error

// Error returns a error string describing the error.
Expand Down
7 changes: 7 additions & 0 deletions registry/remote/internal/errutil/errutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package errutil

import (
"encoding/json"
"errors"
"io"
"net/http"

Expand Down Expand Up @@ -45,3 +46,9 @@ func ParseErrorResponse(resp *http.Response) error {
}
return resultErr
}

// IsErrorCode returns true if err is an Error and its Code equals to code.
func IsErrorCode(err error, code string) bool {
var ec errcode.Error
return errors.As(err, &ec) && ec.Code == code
}
111 changes: 111 additions & 0 deletions registry/remote/internal/errutil/errutil_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,114 @@ func Test_ParseErrorResponse_plain(t *testing.T) {
t.Errorf("ParseErrorResponse() error = %v, want err message %v", err, want)
}
}

func TestIsErrorCode(t *testing.T) {
tests := []struct {
name string
err error
code string
want bool
}{
{
name: "test errcode.Error, same code",
err: errcode.Error{
Code: errcode.ErrorCodeNameUnknown,
},
code: errcode.ErrorCodeNameUnknown,
want: true,
},
{
name: "test errcode.Error, different code",
err: errcode.Error{
Code: errcode.ErrorCodeUnauthorized,
},
code: errcode.ErrorCodeNameUnknown,
want: false,
},
{
name: "test errcode.Errors containing single error, same code",
err: errcode.Errors{
{
Code: errcode.ErrorCodeNameUnknown,
},
},
code: errcode.ErrorCodeNameUnknown,
want: true,
},
{
name: "test errcode.Errors containing single error, different code",
err: errcode.Errors{
{
Code: errcode.ErrorCodeNameUnknown,
},
},
code: errcode.ErrorCodeNameUnknown,
want: true,
},
{
name: "test errcode.Errors containing multiple errors, same code",
err: errcode.Errors{
{
Code: errcode.ErrorCodeNameUnknown,
},
{
Code: errcode.ErrorCodeUnauthorized,
},
},
code: errcode.ErrorCodeNameUnknown,
want: false,
},
{
name: "test errcode.ErrorResponse containing single error, same code",
err: &errcode.ErrorResponse{
Errors: errcode.Errors{
{
Code: errcode.ErrorCodeNameUnknown,
},
},
},
code: errcode.ErrorCodeNameUnknown,
want: true,
},
{
name: "test errcode.ErrorResponse containing single error, different code",
err: &errcode.ErrorResponse{
Errors: errcode.Errors{
{
Code: errcode.ErrorCodeUnauthorized,
},
},
},
code: errcode.ErrorCodeNameUnknown,
want: false,
},
{
name: "test errcode.ErrorResponse containing multiple errors, same code",
err: &errcode.ErrorResponse{
Errors: errcode.Errors{
{
Code: errcode.ErrorCodeNameUnknown,
},
{
Code: errcode.ErrorCodeUnauthorized,
},
},
},
code: errcode.ErrorCodeNameUnknown,
want: false,
},
{
name: "test unstructured error",
err: errors.New(errcode.ErrorCodeNameUnknown),
code: errcode.ErrorCodeNameUnknown,
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsErrorCode(tt.err, tt.code); got != tt.want {
t.Errorf("IsErrorCode() = %v, want %v", got, tt.want)
}
})
}
}
71 changes: 49 additions & 22 deletions registry/remote/repository.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,20 @@ import (
"oras.land/oras-go/v2/internal/registryutil"
"oras.land/oras-go/v2/registry"
"oras.land/oras-go/v2/registry/remote/auth"
"oras.land/oras-go/v2/registry/remote/errcode"
"oras.land/oras-go/v2/registry/remote/internal/errutil"
)

// dockerContentDigestHeader - The Docker-Content-Digest header, if present on
// the response, returns the canonical digest of the uploaded blob.
// See https://docs.docker.com/registry/spec/api/#digest-header
// See https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pull
const dockerContentDigestHeader = "Docker-Content-Digest"
const (
// dockerContentDigestHeader - The Docker-Content-Digest header, if present
// on the response, returns the canonical digest of the uploaded blob.
// See https://docs.docker.com/registry/spec/api/#digest-header
// See https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pull
dockerContentDigestHeader = "Docker-Content-Digest"
// zeroDigest represents a digest that consists of zeros. zeroDigest is used
// for pinging Referrers API.
zeroDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
)

// referrersState represents the state of Referrers API.
type referrersState = int32
Expand Down Expand Up @@ -121,7 +127,11 @@ type Repository struct {
referrersState referrersState

// referrersTagLocks maps a referrers tag to a lock.
referrersTagLocks sync.Map // map[string]sync.Mutex
referrersTagLocks sync.Map // map[string]*sync.Mutex

// referrersPingLock locks the pingReferrersAPI() method and allows only
// one go-routine to send the request.
referrersPingLock sync.Mutex
}

// NewRepository creates a client to the remote repository identified by a
Expand Down Expand Up @@ -399,13 +409,18 @@ func (r *Repository) Referrers(ctx context.Context, desc ocispec.Descriptor, art

// The referrers state is unknown.
if err != nil {
if errors.Is(err, errdef.ErrNotFound) {
// A 404 returned by Referrers API indicates that Referrers API is
// not supported. Fallback to referrers tag schema.
r.SetReferrersCapability(false)
return r.referrersByTagSchema(ctx, desc, artifactType, fn)
var errResp *errcode.ErrorResponse
if !errors.As(err, &errResp) || errResp.StatusCode != http.StatusNotFound {
return err
}
return err
if errutil.IsErrorCode(errResp, errcode.ErrorCodeNameUnknown) {
// The repository is not found, no fallback.
return err
}
// A 404 returned by Referrers API indicates that Referrers API is
// not supported. Fallback to referrers tag schema.
r.SetReferrersCapability(false)
return r.referrersByTagSchema(ctx, desc, artifactType, fn)
}

r.SetReferrersCapability(true)
Expand Down Expand Up @@ -455,9 +470,6 @@ func (r *Repository) referrersPageByAPI(ctx context.Context, artifactType string
}
defer resp.Body.Close()

if resp.StatusCode == http.StatusNotFound {
return "", fmt.Errorf("%s %q: %w", resp.Request.Method, resp.Request.URL, errdef.ErrNotFound)
}
if resp.StatusCode != http.StatusOK {
return "", errutil.ParseErrorResponse(resp)
}
Expand Down Expand Up @@ -973,11 +985,11 @@ func (s *manifestStore) indexReferrersForDelete(ctx context.Context, desc ocispe
}

subject := *manifest.Subject
yes, err := s.repo.isReferrersAPIAvailable(ctx, subject)
ok, err := s.repo.pingReferrers(ctx)
if err != nil {
return err
}
if yes {
if ok {
// referrers API is available, no client-side indexing needed
return nil
}
Expand Down Expand Up @@ -1238,11 +1250,11 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.
return nil
}

yes, err := s.repo.isReferrersAPIAvailable(ctx, subject)
ok, err := s.repo.pingReferrers(ctx)
if err != nil {
return err
}
if yes {
if ok {
// referrers API is available, no client-side indexing needed
return nil
}
Expand Down Expand Up @@ -1288,8 +1300,8 @@ func (s *manifestStore) updateReferrersIndexForPush(ctx context.Context, desc, s
return s.repo.delete(ctx, oldIndexDesc, true)
}

// isReferrersAPIAvailable returns true if the Referrers API is available for r.
func (r *Repository) isReferrersAPIAvailable(ctx context.Context, desc ocispec.Descriptor) (bool, error) {
// pingReferrers returns true if the Referrers API is available for r.
func (r *Repository) pingReferrers(ctx context.Context) (bool, error) {
switch r.loadReferrersState() {
case referrersStateSupported:
return true, nil
Expand All @@ -1298,8 +1310,19 @@ func (r *Repository) isReferrersAPIAvailable(ctx context.Context, desc ocispec.D
}

// referrers state is unknown
// limit the rate of pinging referrers API
r.referrersPingLock.Lock()
defer r.referrersPingLock.Unlock()

switch r.loadReferrersState() {
case referrersStateSupported:
return true, nil
case referrersStateUnsupported:
return false, nil
}

ref := r.Reference
ref.Reference = desc.Digest.String()
ref.Reference = zeroDigest
ctx = registryutil.WithScopeHint(ctx, ref, auth.ActionPull)

url := buildReferrersURL(r.PlainHTTP, ref, "")
Expand All @@ -1318,6 +1341,10 @@ func (r *Repository) isReferrersAPIAvailable(ctx context.Context, desc ocispec.D
r.SetReferrersCapability(true)
return true, nil
case http.StatusNotFound:
if err := errutil.ParseErrorResponse(resp); errutil.IsErrorCode(err, errcode.ErrorCodeNameUnknown) {
// repository not found
return false, err
}
r.SetReferrersCapability(false)
return false, nil
default:
Expand Down
Loading

0 comments on commit f701e60

Please sign in to comment.