api-remove.go

/*
 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"bytes"
	"context"
	"encoding/xml"
	"io"
	"iter"
	"net/http"
	"net/url"
	"time"

	"github.com/minio/minio-go/v7/pkg/s3utils"
)

//revive:disable

// Deprecated: BucketOptions will be renamed to RemoveBucketOptions in future versions.
type BucketOptions = RemoveBucketOptions

//revive:enable

// RemoveBucketOptions special headers to purge buckets, only
// useful when endpoint is MinIO
type RemoveBucketOptions struct {
	ForceDelete bool
}

// RemoveBucketWithOptions deletes the bucket name.
//
// All objects (including all object versions and delete markers)
// in the bucket will be deleted forcibly if bucket options set
// ForceDelete to 'true'.
func (c *Client) RemoveBucketWithOptions(ctx context.Context, bucketName string, opts RemoveBucketOptions) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}

	// Build headers.
	headers := make(http.Header)
	if opts.ForceDelete {
		headers.Set(minIOForceDelete, "true")
	}

	// Execute DELETE on bucket.
	resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
		bucketName:       bucketName,
		contentSHA256Hex: emptySHA256Hex,
		customHeader:     headers,
	})
	defer closeResponse(resp)
	if err != nil {
		return err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusNoContent {
			return httpRespToErrorResponse(resp, bucketName, "")
		}
	}

	// Remove the location from cache on a successful delete.
	c.bucketLocCache.Delete(bucketName)
	return nil
}

// RemoveBucket deletes the bucket name.
//
//	All objects (including all object versions and delete markers).
//	in the bucket must be deleted before successfully attempting this request.
func (c *Client) RemoveBucket(ctx context.Context, bucketName string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	// Execute DELETE on bucket.
	resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
		bucketName:       bucketName,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		return err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusNoContent {
			return httpRespToErrorResponse(resp, bucketName, "")
		}
	}

	// Remove the location from cache on a successful delete.
	c.bucketLocCache.Delete(bucketName)

	return nil
}

// AdvancedRemoveOptions intended for internal use by replication
type AdvancedRemoveOptions struct {
	ReplicationDeleteMarker  bool
	ReplicationStatus        ReplicationStatus
	ReplicationMTime         time.Time
	ReplicationRequest       bool
	ReplicationValidityCheck bool // check permissions
}

// RemoveObjectOptions represents options specified by user for RemoveObject call
type RemoveObjectOptions struct {
	ForceDelete      bool
	GovernanceBypass bool
	VersionID        string
	Internal         AdvancedRemoveOptions
}

// RemoveObject removes an object from a bucket.
func (c *Client) RemoveObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return err
	}

	res := c.removeObject(ctx, bucketName, objectName, opts)
	return res.Err
}

func (c *Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) RemoveObjectResult {
	// Get resources properly escaped and lined up before
	// using them in http request.
	urlValues := make(url.Values)

	if opts.VersionID != "" {
		urlValues.Set("versionId", opts.VersionID)
	}

	// Build headers.
	headers := make(http.Header)

	if opts.GovernanceBypass {
		// Set the bypass goverenance retention header
		headers.Set(amzBypassGovernance, "true")
	}
	if opts.Internal.ReplicationDeleteMarker {
		headers.Set(minIOBucketReplicationDeleteMarker, "true")
	}
	if !opts.Internal.ReplicationMTime.IsZero() {
		headers.Set(minIOBucketSourceMTime, opts.Internal.ReplicationMTime.Format(time.RFC3339Nano))
	}
	if !opts.Internal.ReplicationStatus.Empty() {
		headers.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus))
	}
	if opts.Internal.ReplicationRequest {
		headers.Set(minIOBucketReplicationRequest, "true")
	}
	if opts.Internal.ReplicationValidityCheck {
		headers.Set(minIOBucketReplicationCheck, "true")
	}
	if opts.ForceDelete {
		headers.Set(minIOForceDelete, "true")
	}
	// Execute DELETE on objectName.
	resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
		bucketName:       bucketName,
		objectName:       objectName,
		contentSHA256Hex: emptySHA256Hex,
		queryValues:      urlValues,
		customHeader:     headers,
	})
	defer closeResponse(resp)
	if err != nil {
		return RemoveObjectResult{Err: err}
	}
	if resp != nil {
		// if some unexpected error happened and max retry is reached, we want to let client know
		if resp.StatusCode != http.StatusNoContent {
			err := httpRespToErrorResponse(resp, bucketName, objectName)
			return RemoveObjectResult{Err: err}
		}
	}

	// DeleteObject always responds with http '204' even for
	// objects which do not exist. So no need to handle them
	// specifically.
	return RemoveObjectResult{
		ObjectName:            objectName,
		ObjectVersionID:       opts.VersionID,
		DeleteMarker:          resp.Header.Get("x-amz-delete-marker") == "true",
		DeleteMarkerVersionID: resp.Header.Get("x-amz-version-id"),
	}
}

// RemoveObjectError - container of Multi Delete S3 API error
type RemoveObjectError struct {
	ObjectName string
	VersionID  string
	Err        error
}

func (err *RemoveObjectError) Error() string {
	// This should never happen as we will have a non-nil error with no underlying error.
	if err.Err == nil {
		return "unexpected remove object error result"
	}
	return err.Err.Error()
}

// RemoveObjectResult - container of Multi Delete S3 API result
type RemoveObjectResult struct {
	ObjectName      string
	ObjectVersionID string

	DeleteMarker          bool
	DeleteMarkerVersionID string

	Err error
}

// generateRemoveMultiObjects - generate the XML request for remove multi objects request
func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte {
	delObjects := []deleteObject{}
	for _, obj := range objects {
		delObjects = append(delObjects, deleteObject{
			Key:       obj.Key,
			VersionID: obj.VersionID,
		})
	}
	xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: false})
	return xmlBytes
}

// processRemoveMultiObjectsResponse - parse the remove multi objects web service
// and return the success/failure result status for each object
func processRemoveMultiObjectsResponse(body io.Reader, resultCh chan<- RemoveObjectResult) {
	// Parse multi delete XML response
	rmResult := &deleteMultiObjectsResult{}
	err := xmlDecoder(body, rmResult)
	if err != nil {
		resultCh <- RemoveObjectResult{ObjectName: "", Err: err}
		return
	}

	// Fill deletion that returned success
	for _, obj := range rmResult.DeletedObjects {
		resultCh <- RemoveObjectResult{
			ObjectName: obj.Key,
			// Only filled with versioned buckets
			ObjectVersionID:       obj.VersionID,
			DeleteMarker:          obj.DeleteMarker,
			DeleteMarkerVersionID: obj.DeleteMarkerVersionID,
		}
	}

	// Fill deletion that returned an error.
	for _, obj := range rmResult.UnDeletedObjects {
		// Version does not exist is not an error ignore and continue.
		switch obj.Code {
		case "InvalidArgument", "NoSuchVersion":
			continue
		}
		resultCh <- RemoveObjectResult{
			ObjectName:      obj.Key,
			ObjectVersionID: obj.VersionID,
			Err: ErrorResponse{
				Code:    obj.Code,
				Message: obj.Message,
			},
		}
	}
}

// RemoveObjectsOptions represents options specified by user for RemoveObjects call
type RemoveObjectsOptions struct {
	GovernanceBypass bool
}

// RemoveObjects removes multiple objects from a bucket while
// it is possible to specify objects versions which are received from
// objectsCh. Remove failures are sent back via error channel.
func (c *Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectError {
	errorCh := make(chan RemoveObjectError, 1)

	// Validate if bucket name is valid.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		defer close(errorCh)
		errorCh <- RemoveObjectError{
			Err: err,
		}
		return errorCh
	}
	// Validate objects channel to be properly allocated.
	if objectsCh == nil {
		defer close(errorCh)
		errorCh <- RemoveObjectError{
			Err: errInvalidArgument("Objects channel cannot be nil"),
		}
		return errorCh
	}

	resultCh := make(chan RemoveObjectResult, 1)
	go c.removeObjects(ctx, bucketName, objectsCh, resultCh, opts)
	go func() {
		defer close(errorCh)
		for res := range resultCh {
			// Send only errors to the error channel
			if res.Err == nil {
				continue
			}
			errorCh <- RemoveObjectError{
				ObjectName: res.ObjectName,
				VersionID:  res.ObjectVersionID,
				Err:        res.Err,
			}
		}
	}()

	return errorCh
}

// RemoveObjectsWithIter bulk deletes multiple objects from a bucket.
// Objects (with optional versions) to be removed must be provided with
// an iterator. Objects are removed asynchronously and results must be
// consumed. If the returned result iterator is stopped, the context is
// canceled, or a remote call failed, the provided iterator will no
// longer accept more objects.
func (c *Client) RemoveObjectsWithIter(ctx context.Context, bucketName string, objectsIter iter.Seq[ObjectInfo], opts RemoveObjectsOptions) (iter.Seq[RemoveObjectResult], error) {
	// Validate if bucket name is valid.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return nil, err
	}
	// Validate objects channel to be properly allocated.
	if objectsIter == nil {
		return nil, errInvalidArgument("Objects iter can never by nil")
	}

	return func(yield func(RemoveObjectResult) bool) {
		select {
		case <-ctx.Done():
			return
		default:
		}

		c.removeObjectsIter(ctx, bucketName, objectsIter, yield, opts)
	}, nil
}

// RemoveObjectsWithResult removes multiple objects from a bucket while
// it is possible to specify objects versions which are received from
// objectsCh. Remove results, successes and failures are sent back via
// RemoveObjectResult channel
func (c *Client) RemoveObjectsWithResult(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectResult {
	resultCh := make(chan RemoveObjectResult, 1)

	// Validate if bucket name is valid.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		defer close(resultCh)
		resultCh <- RemoveObjectResult{
			Err: err,
		}
		return resultCh
	}
	// Validate objects channel to be properly allocated.
	if objectsCh == nil {
		defer close(resultCh)
		resultCh <- RemoveObjectResult{
			Err: errInvalidArgument("Objects channel cannot be nil"),
		}
		return resultCh
	}

	go c.removeObjects(ctx, bucketName, objectsCh, resultCh, opts)
	return resultCh
}

// Return true if the character is within the allowed characters in an XML 1.0 document
// The list of allowed characters can be found here: https://www.w3.org/TR/xml/#charsets
func validXMLChar(r rune) (ok bool) {
	return r == 0x09 ||
		r == 0x0A ||
		r == 0x0D ||
		r >= 0x20 && r <= 0xD7FF ||
		r >= 0xE000 && r <= 0xFFFD ||
		r >= 0x10000 && r <= 0x10FFFF
}

func hasInvalidXMLChar(str string) bool {
	for _, s := range str {
		if !validXMLChar(s) {
			return true
		}
	}
	return false
}

// Generate and call MultiDelete S3 requests based on entries received from the iterator.
func (c *Client) removeObjectsIter(ctx context.Context, bucketName string, objectsIter iter.Seq[ObjectInfo], yield func(RemoveObjectResult) bool, opts RemoveObjectsOptions) {
	maxEntries := 1000
	urlValues := make(url.Values)
	urlValues.Set("delete", "")

	// Build headers.
	headers := make(http.Header)
	if opts.GovernanceBypass {
		// Set the bypass goverenance retention header
		headers.Set(amzBypassGovernance, "true")
	}

	processRemoveMultiObjectsResponseIter := func(batch []ObjectInfo, yield func(RemoveObjectResult) bool) bool {
		if len(batch) == 0 {
			return false
		}

		// Generate remove multi objects XML request
		removeBytes := generateRemoveMultiObjectsRequest(batch)
		// Execute POST on bucket to remove objects.
		resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{
			bucketName:       bucketName,
			queryValues:      urlValues,
			contentBody:      bytes.NewReader(removeBytes),
			contentLength:    int64(len(removeBytes)),
			contentMD5Base64: sumMD5Base64(removeBytes),
			contentSHA256Hex: sum256Hex(removeBytes),
			customHeader:     headers,
		})
		if resp != nil {
			defer closeResponse(resp)
			if resp.StatusCode != http.StatusOK {
				err = httpRespToErrorResponse(resp, bucketName, "")
			}
		}
		if err != nil {
			for _, b := range batch {
				if !yield(RemoveObjectResult{
					ObjectName:      b.Key,
					ObjectVersionID: b.VersionID,
					Err:             err,
				}) {
					return false
				}
			}
			return false
		}

		// Parse multi delete XML response
		rmResult := &deleteMultiObjectsResult{}
		if err := xmlDecoder(resp.Body, rmResult); err != nil {
			yield(RemoveObjectResult{ObjectName: "", Err: err})
			return false
		}

		// Fill deletion that returned an error.
		for _, obj := range rmResult.UnDeletedObjects {
			// Version does not exist is not an error ignore and continue.
			switch obj.Code {
			case "InvalidArgument", "NoSuchVersion":
				continue
			}
			if !yield(RemoveObjectResult{
				ObjectName:      obj.Key,
				ObjectVersionID: obj.VersionID,
				Err: ErrorResponse{
					Code:    obj.Code,
					Message: obj.Message,
				},
			}) {
				return false
			}
		}

		// Fill deletion that returned success
		for _, obj := range rmResult.DeletedObjects {
			if !yield(RemoveObjectResult{
				ObjectName: obj.Key,
				// Only filled with versioned buckets
				ObjectVersionID:       obj.VersionID,
				DeleteMarker:          obj.DeleteMarker,
				DeleteMarkerVersionID: obj.DeleteMarkerVersionID,
			}) {
				return false
			}
		}

		return true
	}

	var batch []ObjectInfo

	next, stop := iter.Pull(objectsIter)
	defer stop()

	for {
		// Loop over entries by 1000 and call MultiDelete requests
		object, ok := next()
		if !ok {
			// delete the remaining batch.
			processRemoveMultiObjectsResponseIter(batch, yield)
			return
		}

		if hasInvalidXMLChar(object.Key) {
			// Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document.
			removeResult := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{
				VersionID:        object.VersionID,
				GovernanceBypass: opts.GovernanceBypass,
			})
			if err := removeResult.Err; err != nil {
				// Version does not exist is not an error ignore and continue.
				switch ToErrorResponse(err).Code {
				case "InvalidArgument", "NoSuchVersion":
					continue
				}
			}
			if !yield(removeResult) {
				return
			}

			continue
		}

		batch = append(batch, object)
		if len(batch) < maxEntries {
			continue
		}

		if !processRemoveMultiObjectsResponseIter(batch, yield) {
			return
		}

		batch = batch[:0]
	}
}

// Generate and call MultiDelete S3 requests based on entries received from objectsCh
func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, resultCh chan<- RemoveObjectResult, opts RemoveObjectsOptions) {
	maxEntries := 1000
	finish := false
	urlValues := make(url.Values)
	urlValues.Set("delete", "")

	// Close result channel when Multi delete finishes.
	defer close(resultCh)

	// Loop over entries by 1000 and call MultiDelete requests
	for !finish {
		count := 0
		var batch []ObjectInfo

		// Try to gather 1000 entries
		for object := range objectsCh {
			if hasInvalidXMLChar(object.Key) {
				// Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document.
				removeResult := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{
					VersionID:        object.VersionID,
					GovernanceBypass: opts.GovernanceBypass,
				})
				if err := removeResult.Err; err != nil {
					// Version does not exist is not an error ignore and continue.
					switch ToErrorResponse(err).Code {
					case "InvalidArgument", "NoSuchVersion":
						continue
					}
					resultCh <- removeResult
				}

				resultCh <- removeResult
				continue
			}

			batch = append(batch, object)
			if count++; count >= maxEntries {
				break
			}
		}
		if count == 0 {
			// Multi Objects Delete API doesn't accept empty object list, quit immediately
			break
		}
		if count < maxEntries {
			// We didn't have 1000 entries, so this is the last batch
			finish = true
		}

		// Build headers.
		headers := make(http.Header)
		if opts.GovernanceBypass {
			// Set the bypass goverenance retention header
			headers.Set(amzBypassGovernance, "true")
		}

		// Generate remove multi objects XML request
		removeBytes := generateRemoveMultiObjectsRequest(batch)
		// Execute POST on bucket to remove objects.
		resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{
			bucketName:       bucketName,
			queryValues:      urlValues,
			contentBody:      bytes.NewReader(removeBytes),
			contentLength:    int64(len(removeBytes)),
			contentMD5Base64: sumMD5Base64(removeBytes),
			contentSHA256Hex: sum256Hex(removeBytes),
			customHeader:     headers,
		})
		if resp != nil {
			if resp.StatusCode != http.StatusOK {
				e := httpRespToErrorResponse(resp, bucketName, "")
				resultCh <- RemoveObjectResult{ObjectName: "", Err: e}
			}
		}
		if err != nil {
			for _, b := range batch {
				resultCh <- RemoveObjectResult{
					ObjectName:      b.Key,
					ObjectVersionID: b.VersionID,
					Err:             err,
				}
			}
			continue
		}

		// Process multiobjects remove xml response
		processRemoveMultiObjectsResponse(resp.Body, resultCh)

		closeResponse(resp)
	}
}

// RemoveIncompleteUpload aborts an partially uploaded object.
func (c *Client) RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return err
	}
	// Find multipart upload ids of the object to be aborted.
	uploadIDs, err := c.findUploadIDs(ctx, bucketName, objectName)
	if err != nil {
		return err
	}

	for _, uploadID := range uploadIDs {
		// abort incomplete multipart upload, based on the upload id passed.
		err := c.abortMultipartUpload(ctx, bucketName, objectName, uploadID)
		if err != nil {
			return err
		}
	}

	return nil
}

// abortMultipartUpload aborts a multipart upload for the given
// uploadID, all previously uploaded parts are deleted.
func (c *Client) abortMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return err
	}

	// Initialize url queries.
	urlValues := make(url.Values)
	urlValues.Set("uploadId", uploadID)

	// Execute DELETE on multipart upload.
	resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
		bucketName:       bucketName,
		objectName:       objectName,
		queryValues:      urlValues,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		return err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusNoContent {
			// Abort has no response body, handle it for any errors.
			var errorResponse ErrorResponse
			switch resp.StatusCode {
			case http.StatusNotFound:
				// This is needed specifically for abort and it cannot
				// be converged into default case.
				errorResponse = ErrorResponse{
					Code:       "NoSuchUpload",
					Message:    "The specified multipart upload does not exist.",
					BucketName: bucketName,
					Key:        objectName,
					RequestID:  resp.Header.Get("x-amz-request-id"),
					HostID:     resp.Header.Get("x-amz-id-2"),
					Region:     resp.Header.Get("x-amz-bucket-region"),
				}
			default:
				return httpRespToErrorResponse(resp, bucketName, objectName)
			}
			return errorResponse
		}
	}
	return nil
}