backend: fix handling etag

Signed-off-by: Janusz Marcinkiewicz <januszm@nvidia.com>

ais/backend/aws.go

@@ -476,17 +476,11 @@ func (*s3bp) HeadObj(_ context.Context, lom *core.LOM, oreq *http.Request) (oa *
 		lom.SetCustomKey(cmn.VersionObjMD, v)
 		oa.SetVersion(v)
 	}
-	if v, ok := h.EncodeCksum(headOutput.ETag); ok {
+	if v, ok := h.EncodeETag(headOutput.ETag); ok {
 		oa.SetCustomKey(cmn.ETag, v)
-		// assuming SSE-S3 or plaintext encryption
-		// from https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html:
-		// - "The entity tag is a hash of the object. The ETag reflects changes only
-		//    to the contents of an object, not its metadata."
-		// - "The ETag may or may not be an MD5 digest of the object data. Whether or
-		//    not it is depends on how the object was created and how it is encrypted..."
-		if !cmn.IsS3MultipartEtag(v) {
-			oa.SetCustomKey(cmn.MD5ObjMD, v)
-		}
+	}
+	if v, ok := h.EncodeCksum(headOutput.ETag); ok {
+		oa.SetCustomKey(cmn.MD5ObjMD, v)
 	}
 
 	// AIS custom (see also: PutObject, GetObjReader)
@@ -614,13 +608,12 @@ func _getCustom(lom *core.LOM, obj *s3.GetObjectOutput) (md5 *cos.Cksum) {
 		lom.SetVersion(v)
 		lom.SetCustomKey(cmn.VersionObjMD, v)
 	}
-	// see ETag/MD5 NOTE above
-	if v, ok := h.EncodeCksum(obj.ETag); ok {
+	if v, ok := h.EncodeETag(obj.ETag); ok {
 		lom.SetCustomKey(cmn.ETag, v)
-		if !cmn.IsS3MultipartEtag(v) {
-			md5 = cos.NewCksum(cos.ChecksumMD5, v)
-			lom.SetCustomKey(cmn.MD5ObjMD, v)
-		}
+	}
+	if v, ok := h.EncodeCksum(obj.ETag); ok {
+		md5 = cos.NewCksum(cos.ChecksumMD5, v)
+		lom.SetCustomKey(cmn.MD5ObjMD, v)
 	}
 	mtime := *(obj.LastModified)
 	lom.SetCustomKey(cmn.LastModified, fmtTime(mtime))
@@ -685,12 +678,11 @@ exit:
 		lom.SetCustomKey(cmn.VersionObjMD, v)
 		lom.SetVersion(v)
 	}
-	if v, ok := h.EncodeCksum(uploadOutput.ETag); ok {
+	if v, ok := h.EncodeETag(uploadOutput.ETag); ok {
 		lom.SetCustomKey(cmn.ETag, v)
-		// see ETag/MD5 NOTE above
-		if !cmn.IsS3MultipartEtag(v) {
-			lom.SetCustomKey(cmn.MD5ObjMD, v)
-		}
+	}
+	if v, ok := h.EncodeCksum(uploadOutput.ETag); ok {
+		lom.SetCustomKey(cmn.MD5ObjMD, v)
 	}
 	if cmn.Rom.FastV(5, cos.SmoduleBackend) {
 		nlog.Infoln(tag, lom.String())

ais/backend/awsinv.go

@@ -352,7 +352,7 @@ func (*s3bp) listInventory(cloudBck *cmn.Bck, ctx *core.LsoInvCtx, msg *apc.LsoM
 				}
 			case types.InventoryOptionalFieldETag:
 				if custom != nil {
-					custom[cmn.ETag] = cmn.UnquoteCEV(line[i])
+					custom[cmn.ETag] = line[i]
 				}
 			case types.InventoryOptionalFieldLastModifiedDate:
 				if custom != nil {

ais/backend/awsmpt.go

@@ -71,6 +71,8 @@ func StartMpt(lom *core.LOM, oreq *http.Request, oq url.Values) (id string, ecod
 
 func PutMptPart(lom *core.LOM, r io.ReadCloser, oreq *http.Request, oq url.Values, uploadID string, size int64, partNum int32) (etag string,
 	ecode int, _ error) {
+	h := cmn.BackendHelpers.Amazon
+
 	if lom.IsFeatureSet(feat.S3PresignedRequest) && oreq != nil {
 		pts := aiss3.NewPresignedReq(oreq, lom, r, oq)
 		resp, err := pts.Do(core.T.DataClient())
@@ -79,7 +81,7 @@ func PutMptPart(lom *core.LOM, r io.ReadCloser, oreq *http.Request, oq url.Value
 		}
 		if resp != nil {
 			ecode = resp.StatusCode
-			etag = cmn.UnquoteCEV(resp.Header.Get(cos.HdrETag))
+			etag, _ = h.EncodeETag(resp.Header.Get(cos.HdrETag))
 			return
 		}
 	}
@@ -105,14 +107,16 @@ func PutMptPart(lom *core.LOM, r io.ReadCloser, oreq *http.Request, oq url.Value
 	if err != nil {
 		ecode, err = awsErrorToAISError(err, cloudBck, lom.ObjName)
 	} else {
-		etag = cmn.UnquoteCEV(*out.ETag)
+		etag, _ = h.EncodeETag(out.ETag)
 	}
 
 	return etag, ecode, err
 }
 
 func CompleteMpt(lom *core.LOM, oreq *http.Request, oq url.Values, uploadID string, obody []byte, parts *aiss3.CompleteMptUpload) (etag string,
 	ecode int, _ error) {
+	h := cmn.BackendHelpers.Amazon
+
 	if lom.IsFeatureSet(feat.S3PresignedRequest) && oreq != nil {
 		pts := aiss3.NewPresignedReq(oreq, lom, io.NopCloser(bytes.NewReader(obody)), oq)
 		resp, err := pts.Do(core.T.DataClient())
@@ -124,7 +128,7 @@ func CompleteMpt(lom *core.LOM, oreq *http.Request, oq url.Values, uploadID stri
 			if err != nil {
 				return "", http.StatusBadRequest, err
 			}
-			etag = result.ETag
+			etag, _ = h.EncodeETag(result.ETag)
 			return
 		}
 	}
@@ -151,7 +155,7 @@ func CompleteMpt(lom *core.LOM, oreq *http.Request, oq url.Values, uploadID stri
 	if err != nil {
 		ecode, err = awsErrorToAISError(err, cloudBck, lom.ObjName)
 	} else {
-		etag = cmn.UnquoteCEV(*out.ETag)
+		etag, _ = h.EncodeETag(out.ETag)
 	}
 
 	return etag, ecode, err

ais/backend/gcp.go

@@ -212,7 +212,7 @@ func (*gsbp) ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRes) (ecode
 				en.Version = v
 			}
 			if wantCustom {
-				etag, _ := h.EncodeCksum(attrs.Etag)
+				etag, _ := h.EncodeETag(attrs.Etag)
 				en.Custom = cmn.CustomProps2S(cmn.ETag, etag, cmn.LastModified, fmtTime(attrs.Updated),
 					cos.HdrContentType, attrs.ContentType)
 			}
@@ -291,7 +291,7 @@ func (*gsbp) HeadObj(ctx context.Context, lom *core.LOM, _ *http.Request) (oa *c
 	if v, ok := h.EncodeCksum(attrs.CRC32C); ok {
 		oa.SetCustomKey(cmn.CRC32CObjMD, v)
 	}
-	if v, ok := h.EncodeCksum(attrs.Etag); ok {
+	if v, ok := h.EncodeETag(attrs.Etag); ok {
 		oa.SetCustomKey(cmn.ETag, v)
 	}
 
@@ -385,7 +385,7 @@ func setCustomGs(lom *core.LOM, attrs *storage.ObjectAttrs) (expCksum *cos.Cksum
 			expCksum = cos.NewCksum(cos.ChecksumCRC32C, v)
 		}
 	}
-	if v, ok := h.EncodeCksum(attrs.Etag); ok {
+	if v, ok := h.EncodeETag(attrs.Etag); ok {
 		lom.SetCustomKey(cmn.ETag, v)
 	}
 	lom.SetCustomKey(cmn.LastModified, fmtTime(attrs.Updated))

ais/backend/ht.go

@@ -134,7 +134,7 @@ func (htbp *htbp) HeadObj(ctx context.Context, lom *core.LOM, _ *http.Request) (
 	if resp.ContentLength >= 0 {
 		oa.Size = resp.ContentLength
 	}
-	if v, ok := h.EncodeVersion(resp.Header.Get(cos.HdrETag)); ok {
+	if v, ok := h.EncodeETag(resp.Header.Get(cos.HdrETag)); ok {
 		oa.SetCustomKey(cmn.ETag, v)
 	}
 	if cmn.Rom.FastV(4, cos.SmoduleBackend) {
@@ -200,7 +200,7 @@ func (htbp *htbp) GetObjReader(ctx context.Context, lom *core.LOM, offset, lengt
 
 	lom.SetCustomKey(cmn.SourceObjMD, apc.HT)
 	lom.SetCustomKey(cmn.OrigURLObjMD, origURL)
-	if v, ok := h.EncodeVersion(resp.Header.Get(cos.HdrETag)); ok {
+	if v, ok := h.EncodeETag(resp.Header.Get(cos.HdrETag)); ok {
 		lom.SetCustomKey(cmn.ETag, v)
 	}
 	res.Size = resp.ContentLength

ais/s3/presigned.go

@@ -198,15 +198,16 @@ func (pts *PresignedReq) DoReader(client *http.Client) (*PresignedResp, error) {
 
 // (compare w/ cmn/objattrs FromHeader)
 func (resp *PresignedResp) ObjAttrs() (oa *cmn.ObjAttrs) {
+	h := cmn.BackendHelpers.Amazon
+
 	oa = &cmn.ObjAttrs{}
 	oa.CustomMD = make(cos.StrKVs, 3)
-
 	oa.SetCustomKey(cmn.SourceObjMD, apc.AWS)
-	etag := cmn.UnquoteCEV(resp.Header.Get(cos.HdrETag))
-	debug.Assert(etag != "")
-	oa.SetCustomKey(cmn.ETag, etag)
-	if !cmn.IsS3MultipartEtag(etag) {
-		oa.SetCustomKey(cmn.MD5ObjMD, etag)
+	if v, ok := h.EncodeETag(resp.Header.Get(cos.HdrETag)); ok {
+		oa.SetCustomKey(cmn.ETag, v)
+	}
+	if v, ok := h.EncodeCksum(resp.Header.Get(cos.S3CksumHeader)); ok {
+		oa.SetCustomKey(cmn.MD5ObjMD, v)
 	}
 	if sz := resp.Header.Get(cos.HdrContentLength); sz != "" {
 		size, err := strconv.ParseInt(sz, 10, 64)

ais/s3/types.go

@@ -187,15 +187,19 @@ func (r *ListObjectResult) FromLsoResult(lst *cmn.LsoRes, lsmsg *apc.LsoMsg) {
 }
 
 func SetEtag(hdr http.Header, lom *core.LOM) {
-	if hdr.Get(cos.S3CksumHeader) != "" {
+	if etag := hdr.Get(cos.HdrETag); etag != "" {
+		debug.AssertFunc(func() bool {
+			// Ensure that `etag` is a quoted string.
+			return etag[0] == '"' && etag[len(etag)-1] == '"'
+		})
 		return
 	}
-	if v, exists := lom.GetCustomKey(cmn.ETag); exists && !cmn.IsS3MultipartEtag(v) {
-		hdr.Set(cos.S3CksumHeader /*"ETag"*/, v)
+	if v, exists := lom.GetCustomKey(cmn.ETag); exists {
+		hdr.Set(cos.HdrETag, v)
 		return
 	}
 	if cksum := lom.Checksum(); cksum.Type() == cos.ChecksumMD5 {
-		hdr.Set(cos.S3CksumHeader, cksum.Value())
+		hdr.Set(cos.HdrETag, `"`+cksum.Value()+`"`)
 	}
 }