You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
393 lines
9.1 KiB
393 lines
9.1 KiB
package cos
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/md5"
|
|
"crypto/sha1"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/mozillazg/go-httpheader"
|
|
"hash/crc64"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// 单次上传文件最大为5GB
|
|
const singleUploadMaxLength = 5 * 1024 * 1024 * 1024
|
|
const singleUploadThreshold = 32 * 1024 * 1024
|
|
|
|
// 计算 md5 或 sha1 时的分块大小
|
|
const calDigestBlockSize = 1024 * 1024 * 10
|
|
|
|
func calMD5Digest(msg []byte) []byte {
|
|
// TODO: 分块计算,减少内存消耗
|
|
m := md5.New()
|
|
m.Write(msg)
|
|
return m.Sum(nil)
|
|
}
|
|
|
|
func calSHA1Digest(msg []byte) []byte {
|
|
// TODO: 分块计算,减少内存消耗
|
|
m := sha1.New()
|
|
m.Write(msg)
|
|
return m.Sum(nil)
|
|
}
|
|
|
|
func calCRC64(fd io.Reader) (uint64, error) {
|
|
tb := crc64.MakeTable(crc64.ECMA)
|
|
hash := crc64.New(tb)
|
|
_, err := io.Copy(hash, fd)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
sum := hash.Sum64()
|
|
return sum, nil
|
|
}
|
|
|
|
// cloneRequest returns a clone of the provided *http.Request. The clone is a
|
|
// shallow copy of the struct and its Header map.
|
|
func cloneRequest(r *http.Request) *http.Request {
|
|
// shallow copy of the struct
|
|
r2 := new(http.Request)
|
|
*r2 = *r
|
|
// deep copy of the Header
|
|
r2.Header = make(http.Header, len(r.Header))
|
|
for k, s := range r.Header {
|
|
r2.Header[k] = append([]string(nil), s...)
|
|
}
|
|
return r2
|
|
}
|
|
|
|
// encodeURIComponent like same function in javascript
|
|
//
|
|
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
|
|
//
|
|
// http://www.ecma-international.org/ecma-262/6.0/#sec-uri-syntax-and-semantics
|
|
func encodeURIComponent(s string, excluded ...[]byte) string {
|
|
var b bytes.Buffer
|
|
written := 0
|
|
|
|
for i, n := 0, len(s); i < n; i++ {
|
|
c := s[i]
|
|
|
|
switch c {
|
|
case '-', '_', '.', '!', '~', '*', '\'', '(', ')':
|
|
continue
|
|
default:
|
|
// Unreserved according to RFC 3986 sec 2.3
|
|
if 'a' <= c && c <= 'z' {
|
|
|
|
continue
|
|
|
|
}
|
|
if 'A' <= c && c <= 'Z' {
|
|
|
|
continue
|
|
|
|
}
|
|
if '0' <= c && c <= '9' {
|
|
|
|
continue
|
|
}
|
|
if len(excluded) > 0 {
|
|
conti := false
|
|
for _, ch := range excluded[0] {
|
|
if ch == c {
|
|
conti = true
|
|
break
|
|
}
|
|
}
|
|
if conti {
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
b.WriteString(s[written:i])
|
|
fmt.Fprintf(&b, "%%%02X", c)
|
|
written = i + 1
|
|
}
|
|
|
|
if written == 0 {
|
|
return s
|
|
}
|
|
b.WriteString(s[written:])
|
|
return b.String()
|
|
}
|
|
|
|
func decodeURIComponent(s string) (string, error) {
|
|
decodeStr, err := url.QueryUnescape(s)
|
|
if err != nil {
|
|
return s, err
|
|
}
|
|
return decodeStr, err
|
|
}
|
|
|
|
func DecodeURIComponent(s string) (string, error) {
|
|
return decodeURIComponent(s)
|
|
}
|
|
|
|
func EncodeURIComponent(s string) string {
|
|
return encodeURIComponent(s)
|
|
}
|
|
|
|
func GetReaderLen(reader io.Reader) (length int64, err error) {
|
|
switch v := reader.(type) {
|
|
case *bytes.Buffer:
|
|
length = int64(v.Len())
|
|
case *bytes.Reader:
|
|
length = int64(v.Len())
|
|
case *strings.Reader:
|
|
length = int64(v.Len())
|
|
case *os.File:
|
|
stat, ferr := v.Stat()
|
|
if ferr != nil {
|
|
err = fmt.Errorf("can't get reader length: %s", ferr.Error())
|
|
} else {
|
|
length = stat.Size()
|
|
}
|
|
case *io.LimitedReader:
|
|
length = int64(v.N)
|
|
case *LimitedReadCloser:
|
|
length = int64(v.N)
|
|
case FixedLengthReader:
|
|
length = v.Size()
|
|
default:
|
|
err = fmt.Errorf("can't get reader content length, unkown reader type")
|
|
}
|
|
return
|
|
}
|
|
|
|
func IsLenReader(reader io.Reader) bool {
|
|
switch reader.(type) {
|
|
case *bytes.Buffer:
|
|
return true
|
|
case *bytes.Reader:
|
|
return true
|
|
case *strings.Reader:
|
|
return true
|
|
case *os.File:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
return false
|
|
}
|
|
|
|
func CheckReaderLen(reader io.Reader) error {
|
|
nlen, err := GetReaderLen(reader)
|
|
if err != nil || nlen < singleUploadMaxLength {
|
|
return nil
|
|
}
|
|
return errors.New("The single object size you upload can not be larger than 5GB")
|
|
}
|
|
|
|
func cloneHeader(opt *http.Header) *http.Header {
|
|
if opt == nil {
|
|
return nil
|
|
}
|
|
h := make(http.Header, len(*opt))
|
|
for k, vv := range *opt {
|
|
vv2 := make([]string, len(vv))
|
|
copy(vv2, vv)
|
|
h[k] = vv2
|
|
}
|
|
return &h
|
|
}
|
|
|
|
func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions {
|
|
if opt == nil {
|
|
return nil
|
|
}
|
|
optini := &InitiateMultipartUploadOptions{
|
|
opt.ACLHeaderOptions,
|
|
&ObjectPutHeaderOptions{},
|
|
}
|
|
if opt.ObjectCopyHeaderOptions == nil {
|
|
return optini
|
|
}
|
|
optini.ObjectPutHeaderOptions = &ObjectPutHeaderOptions{
|
|
CacheControl: opt.ObjectCopyHeaderOptions.CacheControl,
|
|
ContentDisposition: opt.ObjectCopyHeaderOptions.ContentDisposition,
|
|
ContentEncoding: opt.ObjectCopyHeaderOptions.ContentEncoding,
|
|
ContentType: opt.ObjectCopyHeaderOptions.ContentType,
|
|
ContentLanguage: opt.ObjectCopyHeaderOptions.ContentLanguage,
|
|
Expect: opt.ObjectCopyHeaderOptions.Expect,
|
|
Expires: opt.ObjectCopyHeaderOptions.Expires,
|
|
XCosMetaXXX: opt.ObjectCopyHeaderOptions.XCosMetaXXX,
|
|
XCosStorageClass: opt.ObjectCopyHeaderOptions.XCosStorageClass,
|
|
XCosServerSideEncryption: opt.ObjectCopyHeaderOptions.XCosServerSideEncryption,
|
|
XCosSSECustomerAglo: opt.ObjectCopyHeaderOptions.XCosSSECustomerAglo,
|
|
XCosSSECustomerKey: opt.ObjectCopyHeaderOptions.XCosSSECustomerKey,
|
|
XCosSSECustomerKeyMD5: opt.ObjectCopyHeaderOptions.XCosSSECustomerKeyMD5,
|
|
XOptionHeader: opt.ObjectCopyHeaderOptions.XOptionHeader,
|
|
}
|
|
return optini
|
|
}
|
|
|
|
func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
|
|
res := &ObjectPutOptions{
|
|
&ACLHeaderOptions{},
|
|
&ObjectPutHeaderOptions{},
|
|
}
|
|
if opt != nil {
|
|
if opt.ACLHeaderOptions != nil {
|
|
*res.ACLHeaderOptions = *opt.ACLHeaderOptions
|
|
}
|
|
if opt.ObjectPutHeaderOptions != nil {
|
|
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
|
|
res.XCosMetaXXX = cloneHeader(opt.XCosMetaXXX)
|
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
|
}
|
|
}
|
|
return res
|
|
}
|
|
|
|
func CloneInitiateMultipartUploadOptions(opt *InitiateMultipartUploadOptions) *InitiateMultipartUploadOptions {
|
|
res := &InitiateMultipartUploadOptions{
|
|
&ACLHeaderOptions{},
|
|
&ObjectPutHeaderOptions{},
|
|
}
|
|
if opt != nil {
|
|
if opt.ACLHeaderOptions != nil {
|
|
*res.ACLHeaderOptions = *opt.ACLHeaderOptions
|
|
}
|
|
if opt.ObjectPutHeaderOptions != nil {
|
|
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
|
|
res.XCosMetaXXX = cloneHeader(opt.XCosMetaXXX)
|
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
|
}
|
|
}
|
|
return res
|
|
}
|
|
|
|
func CloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions {
|
|
var res ObjectUploadPartOptions
|
|
if opt != nil {
|
|
res = *opt
|
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
|
}
|
|
return &res
|
|
}
|
|
|
|
func CloneObjectGetOptions(opt *ObjectGetOptions) *ObjectGetOptions {
|
|
var res ObjectGetOptions
|
|
if opt != nil {
|
|
res = *opt
|
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
|
}
|
|
return &res
|
|
}
|
|
|
|
func CloneCompleteMultipartUploadOptions(opt *CompleteMultipartUploadOptions) *CompleteMultipartUploadOptions {
|
|
var res CompleteMultipartUploadOptions
|
|
if opt != nil {
|
|
res.XMLName = opt.XMLName
|
|
if len(opt.Parts) > 0 {
|
|
res.Parts = make([]Object, len(opt.Parts))
|
|
copy(res.Parts, opt.Parts)
|
|
}
|
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
|
}
|
|
return &res
|
|
}
|
|
|
|
type RangeOptions struct {
|
|
HasStart bool
|
|
HasEnd bool
|
|
Start int64
|
|
End int64
|
|
}
|
|
|
|
func FormatRangeOptions(opt *RangeOptions) string {
|
|
if opt == nil {
|
|
return ""
|
|
}
|
|
if opt.HasStart && opt.HasEnd {
|
|
return fmt.Sprintf("bytes=%v-%v", opt.Start, opt.End)
|
|
}
|
|
if opt.HasStart {
|
|
return fmt.Sprintf("bytes=%v-", opt.Start)
|
|
}
|
|
if opt.HasEnd {
|
|
return fmt.Sprintf("bytes=-%v", opt.End)
|
|
}
|
|
return ""
|
|
}
|
|
func GetRangeOptions(opt *ObjectGetOptions) (*RangeOptions, error) {
|
|
if opt == nil || opt.Range == "" {
|
|
return nil, nil
|
|
}
|
|
return GetRange(opt.Range)
|
|
}
|
|
|
|
func GetRange(rangeStr string) (*RangeOptions, error) {
|
|
// bytes=M-N
|
|
slices := strings.Split(rangeStr, "=")
|
|
if len(slices) != 2 || slices[0] != "bytes" {
|
|
return nil, fmt.Errorf("Invalid Parameter Range: %v", rangeStr)
|
|
}
|
|
// byte=M-N, X-Y
|
|
fSlice := strings.Split(slices[1], ",")
|
|
rstr := fSlice[0]
|
|
|
|
var err error
|
|
var ropt RangeOptions
|
|
sted := strings.Split(rstr, "-")
|
|
if len(sted) != 2 {
|
|
return nil, fmt.Errorf("Invalid Parameter Range: %v", rangeStr)
|
|
}
|
|
// M
|
|
if len(sted[0]) > 0 {
|
|
ropt.Start, err = strconv.ParseInt(sted[0], 10, 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Invalid Parameter Range: %v,err: %v", rangeStr, err)
|
|
}
|
|
ropt.HasStart = true
|
|
}
|
|
// N
|
|
if len(sted[1]) > 0 {
|
|
ropt.End, err = strconv.ParseInt(sted[1], 10, 64)
|
|
if err != nil || ropt.End == 0 {
|
|
return nil, fmt.Errorf("Invalid Parameter Range: %v,err: %v", rangeStr, err)
|
|
}
|
|
ropt.HasEnd = true
|
|
}
|
|
return &ropt, nil
|
|
}
|
|
|
|
var deliverHeader = map[string]bool{}
|
|
|
|
func isDeliverHeader(key string) bool {
|
|
for k, v := range deliverHeader {
|
|
if key == k && v {
|
|
return true
|
|
}
|
|
}
|
|
return strings.HasPrefix(key, privateHeaderPrefix)
|
|
}
|
|
|
|
func deliverInitOptions(opt *InitiateMultipartUploadOptions) (*http.Header, error) {
|
|
if opt == nil {
|
|
return nil, nil
|
|
}
|
|
h, err := httpheader.Header(opt)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
header := &http.Header{}
|
|
for key, values := range h {
|
|
key = strings.ToLower(key)
|
|
if isDeliverHeader(key) {
|
|
for _, value := range values {
|
|
header.Add(key, value)
|
|
}
|
|
}
|
|
}
|
|
return header, nil
|
|
}
|