Fix listobjects with metadata (minio#1795)

`UserMetadata` did not match actual format sent.

Change type of `UserTags`, so it can be unmarshaled from XML string.

* Add Metadata extension to ListObjectVersions
* Return for versions if included.

api-datatypes.go

@@ -21,6 +21,8 @@ import (
 	"encoding/xml"
 	"io"
 	"net/http"
+	"net/url"
+	"strings"
 	"time"
 )
 
@@ -45,24 +47,76 @@ type StringMap map[string]string
 // on the first line is initialize it.
 func (m *StringMap) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) error {
 	*m = StringMap{}
-	type Item struct {
-		Key   string
-		Value string
-	}
 	for {
-		var e Item
+		// Format is <key>value</key>
+		var e struct {
+			XMLName xml.Name
+			Value   string `xml:",chardata"`
+		}
 		err := d.Decode(&e)
 		if err == io.EOF {
 			break
 		}
 		if err != nil {
 			return err
 		}
-		(*m)[e.Key] = e.Value
+		(*m)[e.XMLName.Local] = e.Value
 	}
 	return nil
 }
 
+// URLMap represents map with custom UnmarshalXML
+type URLMap map[string]string
+
+// UnmarshalXML unmarshals the XML into a map of string to strings,
+// creating a key in the map for each tag and setting it's value to the
+// tags contents.
+//
+// The fact this function is on the pointer of Map is important, so that
+// if m is nil it can be initialized, which is often the case if m is
+// nested in another xml structural. This is also why the first thing done
+// on the first line is initialize it.
+func (m *URLMap) UnmarshalXML(d *xml.Decoder, se xml.StartElement) error {
+	*m = URLMap{}
+	var tgs string
+	if err := d.DecodeElement(&tgs, &se); err != nil {
+		if err == io.EOF {
+			return nil
+		}
+		return err
+	}
+	for tgs != "" {
+		var key string
+		key, tgs, _ = stringsCut(tgs, "&")
+		if key == "" {
+			continue
+		}
+		key, value, _ := stringsCut(key, "=")
+		key, err := url.QueryUnescape(key)
+		if err != nil {
+			return err
+		}
+
+		value, err = url.QueryUnescape(value)
+		if err != nil {
+			return err
+		}
+		(*m)[key] = value
+	}
+	return nil
+}
+
+// stringsCut slices s around the first instance of sep,
+// returning the text before and after sep.
+// The found result reports whether sep appears in s.
+// If sep does not appear in s, cut returns s, "", false.
+func stringsCut(s, sep string) (before, after string, found bool) {
+	if i := strings.Index(s, sep); i >= 0 {
+		return s[:i], s[i+len(sep):], true
+	}
+	return s, "", false
+}
+
 // Owner name.
 type Owner struct {
 	XMLName     xml.Name `xml:"Owner" json:"owner"`
@@ -121,10 +175,12 @@ type ObjectInfo struct {
 	Metadata http.Header `json:"metadata" xml:"-"`
 
 	// x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value.
+	// Only returned by MinIO servers.
 	UserMetadata StringMap `json:"userMetadata,omitempty"`
 
 	// x-amz-tagging values in their k/v values.
-	UserTags map[string]string `json:"userTags"`
+	// Only returned by MinIO servers.
+	UserTags URLMap `json:"userTags,omitempty" xml:"UserTags"`
 
 	// x-amz-tagging-count value
 	UserTagCount int

api-list.go

@@ -402,7 +402,7 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
 
 		for {
 			// Get list of objects a maximum of 1000 per request.
-			result, err := c.listObjectVersionsQuery(ctx, bucketName, opts.Prefix, keyMarker, versionIDMarker, delimiter, opts.MaxKeys, opts.headers)
+			result, err := c.listObjectVersionsQuery(ctx, bucketName, opts, keyMarker, versionIDMarker, delimiter)
 			if err != nil {
 				sendObjectInfo(ObjectInfo{
 					Err: err,
@@ -422,6 +422,8 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
 					IsLatest:       version.IsLatest,
 					VersionID:      version.VersionID,
 					IsDeleteMarker: version.isDeleteMarker,
+					UserTags:       version.UserTags,
+					UserMetadata:   version.UserMetadata,
 				}
 				select {
 				// Send object version info.
@@ -474,13 +476,13 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
 // ?delimiter - A delimiter is a character you use to group keys.
 // ?prefix - Limits the response to keys that begin with the specified prefix.
 // ?max-keys - Sets the maximum number of keys returned in the response body.
-func (c *Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, keyMarker, versionIDMarker, delimiter string, maxkeys int, headers http.Header) (ListVersionsResult, error) {
+func (c *Client) listObjectVersionsQuery(ctx context.Context, bucketName string, opts ListObjectsOptions, keyMarker, versionIDMarker, delimiter string) (ListVersionsResult, error) {
 	// Validate bucket name.
 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 		return ListVersionsResult{}, err
 	}
 	// Validate object prefix.
-	if err := s3utils.CheckValidObjectNamePrefix(prefix); err != nil {
+	if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
 		return ListVersionsResult{}, err
 	}
 	// Get resources properly escaped and lined up before
@@ -491,7 +493,7 @@ func (c *Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix
 	urlValues.Set("versions", "")
 
 	// Set object prefix, prefix value to be set to empty is okay.
-	urlValues.Set("prefix", prefix)
+	urlValues.Set("prefix", opts.Prefix)
 
 	// Set delimiter, delimiter value to be set to empty is okay.
 	urlValues.Set("delimiter", delimiter)
@@ -502,15 +504,19 @@ func (c *Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix
 	}
 
 	// Set max keys.
-	if maxkeys > 0 {
-		urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys))
+	if opts.MaxKeys > 0 {
+		urlValues.Set("max-keys", fmt.Sprintf("%d", opts.MaxKeys))
 	}
 
 	// Set version ID marker
 	if versionIDMarker != "" {
 		urlValues.Set("version-id-marker", versionIDMarker)
 	}
 
+	if opts.WithMetadata {
+		urlValues.Set("metadata", "true")
+	}
+
 	// Always set encoding-type
 	urlValues.Set("encoding-type", "url")
 
@@ -519,7 +525,7 @@ func (c *Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix
 		bucketName:       bucketName,
 		queryValues:      urlValues,
 		contentSHA256Hex: emptySHA256Hex,
-		customHeader:     headers,
+		customHeader:     opts.headers,
 	})
 	defer closeResponse(resp)
 	if err != nil {

api-s3-datatypes.go

@@ -85,6 +85,14 @@ type Version struct {
 	StorageClass string
 	VersionID    string `xml:"VersionId"`
 
+	// x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value.
+	// Only returned by MinIO servers.
+	UserMetadata StringMap `json:"userMetadata,omitempty"`
+
+	// x-amz-tagging values in their k/v values.
+	// Only returned by MinIO servers.
+	UserTags URLMap `json:"userTags,omitempty" xml:"UserTags"`
+
 	isDeleteMarker bool
 }