package cos import ( "bytes" "context" "encoding/xml" "errors" "fmt" "hash/crc64" "io" "net/http" "net/url" "sort" "strings" "time" ) // InitiateMultipartUploadOptions is the option of InitateMultipartUpload type InitiateMultipartUploadOptions struct { *ACLHeaderOptions *ObjectPutHeaderOptions } // InitiateMultipartUploadResult is the result of InitateMultipartUpload type InitiateMultipartUploadResult struct { XMLName xml.Name `xml:"InitiateMultipartUploadResult"` Bucket string Key string UploadID string `xml:"UploadId"` } // InitiateMultipartUpload 请求实现初始化分片上传,成功执行此请求以后会返回Upload ID用于后续的Upload Part请求。 // // https://www.qcloud.com/document/product/436/7746 func (s *ObjectService) InitiateMultipartUpload(ctx context.Context, name string, opt *InitiateMultipartUploadOptions) (*InitiateMultipartUploadResult, *Response, error) { var res InitiateMultipartUploadResult sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: "/" + encodeURIComponent(name) + "?uploads", method: http.MethodPost, optHeader: opt, result: &res, } resp, err := s.client.doRetry(ctx, &sendOpt) return &res, resp, err } // ObjectUploadPartOptions is the options of upload-part type ObjectUploadPartOptions struct { Expect string `header:"Expect,omitempty" url:"-"` XCosContentSHA1 string `header:"x-cos-content-sha1,omitempty" url:"-"` ContentLength int64 `header:"Content-Length,omitempty" url:"-"` ContentMD5 string `header:"Content-MD5,omitempty" url:"-"` XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"` XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"` XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"` XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"` XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"` // 上传进度, ProgressCompleteEvent不能表示对应API调用成功,API是否调用成功的判断标准为返回err==nil Listener ProgressListener `header:"-" url:"-" xml:"-"` } // UploadPart 请求实现在初始化以后的分块上传,支持的块的数量为1到10000,块的大小为1 MB 到5 GB。 // 在每次请求Upload Part时候,需要携带partNumber和uploadID,partNumber为块的编号,支持乱序上传。 // // 当传入uploadID和partNumber都相同的时候,后传入的块将覆盖之前传入的块。当uploadID不存在时会返回404错误,NoSuchUpload. // // 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时,必须指定 opt.ContentLength // // https://www.qcloud.com/document/product/436/7750 func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, uopt *ObjectUploadPartOptions) (*Response, error) { if r == nil { return nil, fmt.Errorf("reader is nil") } if err := CheckReaderLen(r); err != nil { return nil, err } // opt 不为 nil opt := CloneObjectUploadPartOptions(uopt) totalBytes, err := GetReaderLen(r) if err != nil && opt.Listener != nil { if opt.ContentLength == 0 { return nil, err } totalBytes = opt.ContentLength } // 分块上传不支持 Chunk 上传 if err == nil { // 非bytes.Buffer/bytes.Reader/strings.Reader/os.File 由用户指定ContentLength, 或使用 Chunk 上传 if opt != nil && opt.ContentLength == 0 && IsLenReader(r) { opt.ContentLength = totalBytes } } reader := TeeReader(r, nil, totalBytes, nil) if s.client.Conf.EnableCRC { reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA)) } if opt != nil && opt.Listener != nil { reader.listener = opt.Listener } u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID) sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: u, method: http.MethodPut, optHeader: opt, body: reader, } resp, err := s.client.send(ctx, &sendOpt) return resp, err } // ObjectListPartsOptions is the option of ListParts type ObjectListPartsOptions struct { EncodingType string `url:"encoding-type,omitempty"` MaxParts string `url:"max-parts,omitempty"` PartNumberMarker string `url:"part-number-marker,omitempty"` } // ObjectListPartsResult is the result of ListParts type ObjectListPartsResult struct { XMLName xml.Name `xml:"ListPartsResult"` Bucket string EncodingType string `xml:"Encoding-type,omitempty"` Key string UploadID string `xml:"UploadId"` Initiator *Initiator `xml:"Initiator,omitempty"` Owner *Owner `xml:"Owner,omitempty"` StorageClass string PartNumberMarker string NextPartNumberMarker string `xml:"NextPartNumberMarker,omitempty"` MaxParts string IsTruncated bool Parts []Object `xml:"Part,omitempty"` } // ListParts 用来查询特定分块上传中的已上传的块。 // // https://www.qcloud.com/document/product/436/7747 func (s *ObjectService) ListParts(ctx context.Context, name, uploadID string, opt *ObjectListPartsOptions) (*ObjectListPartsResult, *Response, error) { u := fmt.Sprintf("/%s?uploadId=%s", encodeURIComponent(name), uploadID) var res ObjectListPartsResult sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: u, method: http.MethodGet, result: &res, optQuery: opt, } resp, err := s.client.doRetry(ctx, &sendOpt) return &res, resp, err } // CompleteMultipartUploadOptions is the option of CompleteMultipartUpload type CompleteMultipartUploadOptions struct { XMLName xml.Name `xml:"CompleteMultipartUpload" header:"-" url:"-"` Parts []Object `xml:"Part" header:"-" url:"-"` XOptionHeader *http.Header `header:"-,omitempty" xml:"-" url:"-"` } // CompleteMultipartUploadResult is the result CompleteMultipartUpload type CompleteMultipartUploadResult struct { XMLName xml.Name `xml:"CompleteMultipartUploadResult"` Location string Bucket string Key string ETag string } // ObjectList can used for sort the parts which needs in complete upload part // sort.Sort(cos.ObjectList(opt.Parts)) type ObjectList []Object func (o ObjectList) Len() int { return len(o) } func (o ObjectList) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o ObjectList) Less(i, j int) bool { // rewrite the Less method from small to big return o[i].PartNumber < o[j].PartNumber } // CompleteMultipartUpload 用来实现完成整个分块上传。当您已经使用Upload Parts上传所有块以后,你可以用该API完成上传。 // 在使用该API时,您必须在Body中给出每一个块的PartNumber和ETag,用来校验块的准确性。 // // 由于分块上传的合并需要数分钟时间,因而当合并分块开始的时候,COS就立即返回200的状态码,在合并的过程中, // COS会周期性的返回空格信息来保持连接活跃,直到合并完成,COS会在Body中返回合并后块的内容。 // // 当上传块小于1 MB的时候,在调用该请求时,会返回400 EntityTooSmall; // 当上传块编号不连续的时候,在调用该请求时,会返回400 InvalidPart; // 当请求Body中的块信息没有按序号从小到大排列的时候,在调用该请求时,会返回400 InvalidPartOrder; // 当UploadId不存在的时候,在调用该请求时,会返回404 NoSuchUpload。 // // 建议您及时完成分块上传或者舍弃分块上传,因为已上传但是未终止的块会占用存储空间进而产生存储费用。 // // https://www.qcloud.com/document/product/436/7742 func (s *ObjectService) CompleteMultipartUpload(ctx context.Context, name, uploadID string, opt *CompleteMultipartUploadOptions) (*CompleteMultipartUploadResult, *Response, error) { u := fmt.Sprintf("/%s?uploadId=%s", encodeURIComponent(name), uploadID) var res CompleteMultipartUploadResult sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: u, method: http.MethodPost, optHeader: opt, body: opt, result: &res, } resp, err := s.client.doRetry(ctx, &sendOpt) // If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error. if err == nil && resp.StatusCode == 200 { if res.ETag == "" { return &res, resp, errors.New("response 200 OK, but body contains an error") } } return &res, resp, err } // AbortMultipartUpload 用来实现舍弃一个分块上传并删除已上传的块。当您调用Abort Multipart Upload时, // 如果有正在使用这个Upload Parts上传块的请求,则Upload Parts会返回失败。当该UploadID不存在时,会返回404 NoSuchUpload。 // // 建议您及时完成分块上传或者舍弃分块上传,因为已上传但是未终止的块会占用存储空间进而产生存储费用。 // // https://www.qcloud.com/document/product/436/7740 func (s *ObjectService) AbortMultipartUpload(ctx context.Context, name, uploadID string) (*Response, error) { if len(name) == 0 || name == "/" { return nil, errors.New("empty object name") } u := fmt.Sprintf("/%s?uploadId=%s", encodeURIComponent(name), uploadID) sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: u, method: http.MethodDelete, } resp, err := s.client.doRetry(ctx, &sendOpt) return resp, err } // ObjectCopyPartOptions is the options of copy-part type ObjectCopyPartOptions struct { XCosCopySource string `header:"x-cos-copy-source" url:"-"` XCosCopySourceRange string `header:"x-cos-copy-source-range,omitempty" url:"-"` XCosCopySourceIfModifiedSince string `header:"x-cos-copy-source-If-Modified-Since,omitempty" url:"-"` XCosCopySourceIfUnmodifiedSince string `header:"x-cos-copy-source-If-Unmodified-Since,omitempty" url:"-"` XCosCopySourceIfMatch string `header:"x-cos-copy-source-If-Match,omitempty" url:"-"` XCosCopySourceIfNoneMatch string `header:"x-cos-copy-source-If-None-Match,omitempty" url:"-"` // SSE-C XCosCopySourceSSECustomerAglo string `header:"x-cos-copy-source-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"` XCosCopySourceSSECustomerKey string `header:"x-cos-copy-source-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"` XCosCopySourceSSECustomerKeyMD5 string `header:"x-cos-copy-source-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"` //兼容其他自定义头部 XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"` } // CopyPartResult is the result CopyPart type CopyPartResult struct { XMLName xml.Name `xml:"CopyPartResult"` ETag string LastModified string } // CopyPart 请求实现在初始化以后的分块上传,支持的块的数量为1到10000,块的大小为1 MB 到5 GB。 // 在每次请求Upload Part时候,需要携带partNumber和uploadID,partNumber为块的编号,支持乱序上传。 // ObjectCopyPartOptions的XCosCopySource为必填参数,格式为-.cos..myqcloud.com/ // ObjectCopyPartOptions的XCosCopySourceRange指定源的Range,格式为bytes=- // // 当传入uploadID和partNumber都相同的时候,后传入的块将覆盖之前传入的块。当uploadID不存在时会返回404错误,NoSuchUpload. // // https://www.qcloud.com/document/product/436/7750 func (s *ObjectService) CopyPart(ctx context.Context, name, uploadID string, partNumber int, sourceURL string, opt *ObjectCopyPartOptions) (*CopyPartResult, *Response, error) { if opt == nil { opt = &ObjectCopyPartOptions{} } opt.XCosCopySource = sourceURL u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID) var res CopyPartResult var bs bytes.Buffer sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: u, method: http.MethodPut, optHeader: opt, result: &bs, } resp, err := s.client.doRetry(ctx, &sendOpt) if err == nil { // 请求正常 err = xml.Unmarshal(bs.Bytes(), &res) // body 正常返回 if err == io.EOF { err = nil } // If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error. if resp != nil && resp.StatusCode == 200 { if err != nil { resErr := &ErrorResponse{Response: resp.Response} xml.Unmarshal(bs.Bytes(), resErr) return &res, resp, resErr } } } return &res, resp, err } type ObjectListUploadsOptions struct { Delimiter string `url:"delimiter,omitempty"` EncodingType string `url:"encoding-type,omitempty"` Prefix string `url:"prefix,omitempty"` MaxUploads int `url:"max-uploads,omitempty"` KeyMarker string `url:"key-marker,omitempty"` UploadIdMarker string `url:"upload-id-marker,omitempty"` } type ObjectListUploadsResult struct { XMLName xml.Name `xml:"ListMultipartUploadsResult"` Bucket string `xml:"Bucket,omitempty"` EncodingType string `xml:"Encoding-Type,omitempty"` KeyMarker string `xml:"KeyMarker,omitempty"` UploadIdMarker string `xml:"UploadIdMarker,omitempty"` NextKeyMarker string `xml:"NextKeyMarker,omitempty"` NextUploadIdMarker string `xml:"NextUploadIdMarker,omitempty"` MaxUploads string `xml:"MaxUploads,omitempty"` IsTruncated bool `xml:"IsTruncated,omitempty"` Prefix string `xml:"Prefix,omitempty"` Delimiter string `xml:"Delimiter,omitempty"` Upload []ListUploadsResultUpload `xml:"Upload,omitempty"` CommonPrefixes []string `xml:"CommonPrefixes>Prefix,omitempty"` } type ListUploadsResultUpload struct { Key string `xml:"Key,omitempty"` UploadID string `xml:"UploadId,omitempty"` StorageClass string `xml:"StorageClass,omitempty"` Initiator *Initiator `xml:"Initiator,omitempty"` Owner *Owner `xml:"Owner,omitempty"` Initiated string `xml:"Initiated,omitempty"` } func (s *ObjectService) ListUploads(ctx context.Context, opt *ObjectListUploadsOptions) (*ObjectListUploadsResult, *Response, error) { var res ObjectListUploadsResult sendOpt := &sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: "/?uploads", method: http.MethodGet, optQuery: opt, result: &res, } resp, err := s.client.doRetry(ctx, sendOpt) return &res, resp, err } type MultiCopyOptions struct { OptCopy *ObjectCopyOptions PartSize int64 ThreadPoolSize int useMulti bool // use for ut } type CopyJobs struct { Name string UploadId string RetryTimes int Chunk Chunk Opt *ObjectCopyPartOptions } type CopyResults struct { PartNumber int Resp *Response err error res *CopyPartResult } func copyworker(ctx context.Context, s *ObjectService, jobs <-chan *CopyJobs, results chan<- *CopyResults) { for j := range jobs { var copyres CopyResults j.Opt.XCosCopySourceRange = fmt.Sprintf("bytes=%d-%d", j.Chunk.OffSet, j.Chunk.OffSet+j.Chunk.Size-1) rt := j.RetryTimes for { res, resp, err := s.CopyPart(ctx, j.Name, j.UploadId, j.Chunk.Number, j.Opt.XCosCopySource, j.Opt) copyres.PartNumber = j.Chunk.Number copyres.Resp = resp copyres.err = err copyres.res = res if err != nil { rt-- if rt == 0 { results <- ©res break } if resp != nil && resp.StatusCode < 499 && resp.StatusCode >= 400 { results <- ©res break } time.Sleep(10 * time.Millisecond) continue } results <- ©res break } } } func (s *ObjectService) innerHead(ctx context.Context, sourceURL string, opt *ObjectHeadOptions, id []string) (resp *Response, err error) { surl := strings.SplitN(sourceURL, "/", 2) if len(surl) < 2 { err = errors.New(fmt.Sprintf("sourceURL format error: %s", sourceURL)) return } u, err := url.Parse(fmt.Sprintf("http://%s", surl[0])) if err != nil { return } b := &BaseURL{BucketURL: u} client := NewClient(b, &http.Client{ Transport: s.client.client.Transport, }) if len(id) > 0 { resp, err = client.Object.Head(ctx, surl[1], nil, id[0]) } else { resp, err = client.Object.Head(ctx, surl[1], nil) } return } // 如果源对象大于5G,则采用分块复制的方式进行拷贝,此时源对象的元信息如果COPY func (s *ObjectService) MultiCopy(ctx context.Context, name string, sourceURL string, opt *MultiCopyOptions, id ...string) (*ObjectCopyResult, *Response, error) { resp, err := s.innerHead(ctx, sourceURL, nil, id) if err != nil { return nil, nil, err } totalBytes := resp.ContentLength surl := strings.SplitN(sourceURL, "/", 2) if len(surl) < 2 { return nil, nil, errors.New(fmt.Sprintf("x-cos-copy-source format error: %s", sourceURL)) } var u string if len(id) == 1 { u = fmt.Sprintf("%s/%s?versionId=%s", surl[0], encodeURIComponent(surl[1]), id[0]) } else if len(id) == 0 { u = fmt.Sprintf("%s/%s", surl[0], encodeURIComponent(surl[1])) } else { return nil, nil, errors.New("wrong params") } if opt == nil { opt = &MultiCopyOptions{} } chunks, partNum, err := SplitSizeIntoChunks(totalBytes, opt.PartSize*1024*1024) if err != nil { return nil, nil, err } if partNum == 0 || (totalBytes < singleUploadMaxLength && !opt.useMulti) { if len(id) > 0 { return s.Copy(ctx, name, sourceURL, opt.OptCopy, id[0]) } else { return s.Copy(ctx, name, sourceURL, opt.OptCopy) } } optini := CopyOptionsToMulti(opt.OptCopy) var uploadID string res, _, err := s.InitiateMultipartUpload(ctx, name, optini) if err != nil { return nil, nil, err } uploadID = res.UploadID var poolSize int if opt.ThreadPoolSize > 0 { poolSize = opt.ThreadPoolSize } else { poolSize = 1 } chjobs := make(chan *CopyJobs, 100) chresults := make(chan *CopyResults, 10000) optcom := &CompleteMultipartUploadOptions{} for w := 1; w <= poolSize; w++ { go copyworker(ctx, s, chjobs, chresults) } go func() { for _, chunk := range chunks { partOpt := &ObjectCopyPartOptions{ XCosCopySource: u, } if opt.OptCopy != nil && opt.OptCopy.ObjectCopyHeaderOptions != nil { partOpt.XCosCopySourceIfModifiedSince = opt.OptCopy.XCosCopySourceIfModifiedSince partOpt.XCosCopySourceIfUnmodifiedSince = opt.OptCopy.XCosCopySourceIfUnmodifiedSince partOpt.XCosCopySourceIfMatch = opt.OptCopy.XCosCopySourceIfMatch partOpt.XCosCopySourceIfNoneMatch = opt.OptCopy.XCosCopySourceIfNoneMatch partOpt.XCosCopySourceSSECustomerAglo = opt.OptCopy.XCosCopySourceSSECustomerAglo partOpt.XCosCopySourceSSECustomerKey = opt.OptCopy.XCosCopySourceSSECustomerKey partOpt.XCosCopySourceSSECustomerKeyMD5 = opt.OptCopy.XCosCopySourceSSECustomerKeyMD5 } job := &CopyJobs{ Name: name, RetryTimes: 3, UploadId: uploadID, Chunk: chunk, Opt: partOpt, } chjobs <- job } close(chjobs) }() err = nil for i := 0; i < partNum; i++ { res := <-chresults if res.res == nil || res.err != nil { err = fmt.Errorf("UploadID %s, part %d failed to get resp content. error: %s", uploadID, res.PartNumber, res.err.Error()) continue } etag := res.res.ETag optcom.Parts = append(optcom.Parts, Object{ PartNumber: res.PartNumber, ETag: etag}, ) } close(chresults) if err != nil { return nil, nil, err } sort.Sort(ObjectList(optcom.Parts)) v, resp, err := s.CompleteMultipartUpload(ctx, name, uploadID, optcom) if err != nil { s.AbortMultipartUpload(ctx, name, uploadID) return nil, resp, err } cpres := &ObjectCopyResult{ ETag: v.ETag, CRC64: resp.Header.Get("x-cos-hash-crc64ecma"), VersionId: resp.Header.Get("x-cos-version-id"), } return cpres, resp, err }