|
|
package cos
|
|
|
|
|
|
import (
|
|
|
"bytes"
|
|
|
"context"
|
|
|
"encoding/base64"
|
|
|
"encoding/xml"
|
|
|
"fmt"
|
|
|
"io"
|
|
|
"io/ioutil"
|
|
|
"net/http"
|
|
|
"net/url"
|
|
|
"reflect"
|
|
|
"strings"
|
|
|
"text/template"
|
|
|
"time"
|
|
|
|
|
|
"regexp"
|
|
|
"strconv"
|
|
|
|
|
|
"github.com/google/go-querystring/query"
|
|
|
"github.com/mozillazg/go-httpheader"
|
|
|
)
|
|
|
|
|
|
const (
|
|
|
// Version current go sdk version
|
|
|
Version = "0.7.43"
|
|
|
UserAgent = "cos-go-sdk-v5/" + Version
|
|
|
contentTypeXML = "application/xml"
|
|
|
defaultServiceBaseURL = "http://service.cos.myqcloud.com"
|
|
|
XOptionalKey = "cos-go-sdk-v5-XOptionalKey"
|
|
|
)
|
|
|
|
|
|
var (
|
|
|
bucketURLTemplate = template.Must(
|
|
|
template.New("bucketURLFormat").Parse(
|
|
|
"{{.Schema}}://{{.BucketName}}.cos.{{.Region}}.myqcloud.com",
|
|
|
),
|
|
|
)
|
|
|
|
|
|
// {<http://>|<https://>}{bucketname-appid}.{cos|cos-internal|cos-website|ci}.{region}.{myqcloud.com/tencentcos.cn}{/}
|
|
|
hostSuffix = regexp.MustCompile(`^.*((cos|cos-internal|cos-website|ci)\.[a-z-1]+|file)\.(myqcloud\.com|tencentcos\.cn).*$`)
|
|
|
hostPrefix = regexp.MustCompile(`^(http://|https://){0,1}([a-z0-9-]+-[0-9]+\.){0,1}((cos|cos-internal|cos-website|ci)\.[a-z-1]+|file)\.(myqcloud\.com|tencentcos\.cn).*$`)
|
|
|
invalidBucketErr = fmt.Errorf("invalid bucket format, please check your cos.BaseURL")
|
|
|
)
|
|
|
|
|
|
// BaseURL 访问各 API 所需的基础 URL
|
|
|
type BaseURL struct {
|
|
|
// 访问 bucket, object 相关 API 的基础 URL（不包含 path 部分）: http://example.com
|
|
|
BucketURL *url.URL
|
|
|
// 访问 service API 的基础 URL（不包含 path 部分）: http://example.com
|
|
|
ServiceURL *url.URL
|
|
|
// 访问 job API 的基础 URL （不包含 path 部分）: http://example.com
|
|
|
BatchURL *url.URL
|
|
|
// 访问 CI 的基础 URL
|
|
|
CIURL *url.URL
|
|
|
// 访问 Fetch Task 的基础 URL
|
|
|
FetchURL *url.URL
|
|
|
}
|
|
|
|
|
|
// NewBucketURL 生成 BaseURL 所需的 BucketURL
|
|
|
//
|
|
|
// bucketName: bucket名称, bucket的命名规则为{name}-{appid} ，此处填写的存储桶名称必须为此格式
|
|
|
// Region: 区域代码: ap-beijing-1,ap-beijing,ap-shanghai,ap-guangzhou...
|
|
|
// secure: 是否使用 https
|
|
|
func NewBucketURL(bucketName, region string, secure bool) (*url.URL, error) {
|
|
|
schema := "https"
|
|
|
if !secure {
|
|
|
schema = "http"
|
|
|
}
|
|
|
|
|
|
if region == "" {
|
|
|
return nil, fmt.Errorf("region[%v] is invalid", region)
|
|
|
}
|
|
|
if bucketName == "" || !strings.ContainsAny(bucketName, "-") {
|
|
|
return nil, fmt.Errorf("bucketName[%v] is invalid", bucketName)
|
|
|
}
|
|
|
w := bytes.NewBuffer(nil)
|
|
|
bucketURLTemplate.Execute(w, struct {
|
|
|
Schema string
|
|
|
BucketName string
|
|
|
Region string
|
|
|
}{
|
|
|
schema, bucketName, region,
|
|
|
})
|
|
|
|
|
|
u, _ := url.Parse(w.String())
|
|
|
return u, nil
|
|
|
}
|
|
|
|
|
|
type RetryOptions struct {
|
|
|
Count int
|
|
|
Interval time.Duration
|
|
|
StatusCode []int
|
|
|
}
|
|
|
type Config struct {
|
|
|
EnableCRC bool
|
|
|
RequestBodyClose bool
|
|
|
RetryOpt RetryOptions
|
|
|
}
|
|
|
|
|
|
// Client is a client manages communication with the COS API.
|
|
|
type Client struct {
|
|
|
client *http.Client
|
|
|
|
|
|
Host string
|
|
|
UserAgent string
|
|
|
BaseURL *BaseURL
|
|
|
|
|
|
common service
|
|
|
|
|
|
Service *ServiceService
|
|
|
Bucket *BucketService
|
|
|
Object *ObjectService
|
|
|
Batch *BatchService
|
|
|
CI *CIService
|
|
|
|
|
|
Conf *Config
|
|
|
}
|
|
|
|
|
|
type service struct {
|
|
|
client *Client
|
|
|
}
|
|
|
|
|
|
// NewClient returns a new COS API client.
|
|
|
func NewClient(uri *BaseURL, httpClient *http.Client) *Client {
|
|
|
if httpClient == nil {
|
|
|
httpClient = &http.Client{}
|
|
|
}
|
|
|
|
|
|
baseURL := &BaseURL{}
|
|
|
if uri != nil {
|
|
|
baseURL.BucketURL = uri.BucketURL
|
|
|
baseURL.ServiceURL = uri.ServiceURL
|
|
|
baseURL.BatchURL = uri.BatchURL
|
|
|
baseURL.CIURL = uri.CIURL
|
|
|
baseURL.FetchURL = uri.FetchURL
|
|
|
}
|
|
|
if baseURL.ServiceURL == nil {
|
|
|
baseURL.ServiceURL, _ = url.Parse(defaultServiceBaseURL)
|
|
|
}
|
|
|
|
|
|
c := &Client{
|
|
|
client: httpClient,
|
|
|
UserAgent: UserAgent,
|
|
|
BaseURL: baseURL,
|
|
|
Conf: &Config{
|
|
|
EnableCRC: true,
|
|
|
RequestBodyClose: false,
|
|
|
RetryOpt: RetryOptions{
|
|
|
Count: 3,
|
|
|
Interval: time.Duration(0),
|
|
|
},
|
|
|
},
|
|
|
}
|
|
|
c.common.client = c
|
|
|
c.Service = (*ServiceService)(&c.common)
|
|
|
c.Bucket = (*BucketService)(&c.common)
|
|
|
c.Object = (*ObjectService)(&c.common)
|
|
|
c.Batch = (*BatchService)(&c.common)
|
|
|
c.CI = (*CIService)(&c.common)
|
|
|
return c
|
|
|
}
|
|
|
|
|
|
type Credential struct {
|
|
|
SecretID string
|
|
|
SecretKey string
|
|
|
SessionToken string
|
|
|
}
|
|
|
|
|
|
func (c *Client) GetCredential() *Credential {
|
|
|
if auth, ok := c.client.Transport.(*AuthorizationTransport); ok {
|
|
|
auth.rwLocker.Lock()
|
|
|
defer auth.rwLocker.Unlock()
|
|
|
return &Credential{
|
|
|
SecretID: auth.SecretID,
|
|
|
SecretKey: auth.SecretKey,
|
|
|
SessionToken: auth.SessionToken,
|
|
|
}
|
|
|
}
|
|
|
if auth, ok := c.client.Transport.(*CVMCredentialTransport); ok {
|
|
|
ak, sk, token, err := auth.GetCredential()
|
|
|
if err != nil {
|
|
|
return nil
|
|
|
}
|
|
|
return &Credential{
|
|
|
SecretID: ak,
|
|
|
SecretKey: sk,
|
|
|
SessionToken: token,
|
|
|
}
|
|
|
}
|
|
|
if auth, ok := c.client.Transport.(*CredentialTransport); ok {
|
|
|
ak, sk, token := auth.Credential.GetSecretId(), auth.Credential.GetSecretKey(), auth.Credential.GetToken()
|
|
|
return &Credential{
|
|
|
SecretID: ak,
|
|
|
SecretKey: sk,
|
|
|
SessionToken: token,
|
|
|
}
|
|
|
}
|
|
|
return nil
|
|
|
}
|
|
|
|
|
|
func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) {
|
|
|
if !checkURL(baseURL) {
|
|
|
return nil, invalidBucketErr
|
|
|
}
|
|
|
uri, err = addURLOptions(uri, optQuery)
|
|
|
if err != nil {
|
|
|
return
|
|
|
}
|
|
|
u, _ := url.Parse(uri)
|
|
|
urlStr := baseURL.ResolveReference(u).String()
|
|
|
|
|
|
var reader io.Reader
|
|
|
contentType := ""
|
|
|
contentMD5 := ""
|
|
|
if body != nil {
|
|
|
// 上传文件
|
|
|
if r, ok := body.(io.Reader); ok {
|
|
|
reader = r
|
|
|
} else {
|
|
|
b, err := xml.Marshal(body)
|
|
|
if err != nil {
|
|
|
return nil, err
|
|
|
}
|
|
|
contentType = contentTypeXML
|
|
|
reader = bytes.NewReader(b)
|
|
|
contentMD5 = base64.StdEncoding.EncodeToString(calMD5Digest(b))
|
|
|
}
|
|
|
}
|
|
|
|
|
|
req, err = http.NewRequest(method, urlStr, reader)
|
|
|
if err != nil {
|
|
|
return
|
|
|
}
|
|
|
|
|
|
req.Header, err = addHeaderOptions(ctx, req.Header, optHeader)
|
|
|
if err != nil {
|
|
|
return
|
|
|
}
|
|
|
if v := req.Header.Get("Content-Length"); req.ContentLength == 0 && v != "" && v != "0" {
|
|
|
req.ContentLength, _ = strconv.ParseInt(v, 10, 64)
|
|
|
}
|
|
|
|
|
|
if contentMD5 != "" {
|
|
|
req.Header["Content-MD5"] = []string{contentMD5}
|
|
|
}
|
|
|
if v := req.Header.Get("User-Agent"); v == "" || !strings.HasPrefix(v, UserAgent) {
|
|
|
if c.UserAgent != "" {
|
|
|
req.Header.Set("User-Agent", c.UserAgent)
|
|
|
}
|
|
|
}
|
|
|
if req.Header.Get("Content-Type") == "" && contentType != "" {
|
|
|
req.Header.Set("Content-Type", contentType)
|
|
|
}
|
|
|
if c.Host != "" {
|
|
|
req.Host = c.Host
|
|
|
}
|
|
|
if c.Conf.RequestBodyClose {
|
|
|
req.Close = true
|
|
|
}
|
|
|
return
|
|
|
}
|
|
|
|
|
|
func (c *Client) doAPI(ctx context.Context, req *http.Request, result interface{}, closeBody bool) (*Response, error) {
|
|
|
var cancel context.CancelFunc
|
|
|
if closeBody {
|
|
|
ctx, cancel = context.WithCancel(ctx)
|
|
|
defer cancel()
|
|
|
}
|
|
|
req = req.WithContext(ctx)
|
|
|
|
|
|
resp, err := c.client.Do(req)
|
|
|
if err != nil {
|
|
|
// If we got an error, and the context has been canceled,
|
|
|
// the context's error is probably more useful.
|
|
|
select {
|
|
|
case <-ctx.Done():
|
|
|
return nil, ctx.Err()
|
|
|
default:
|
|
|
}
|
|
|
return nil, err
|
|
|
}
|
|
|
|
|
|
defer func() {
|
|
|
if closeBody {
|
|
|
// Close the body to let the Transport reuse the connection
|
|
|
io.Copy(ioutil.Discard, resp.Body)
|
|
|
resp.Body.Close()
|
|
|
}
|
|
|
}()
|
|
|
|
|
|
response := newResponse(resp)
|
|
|
|
|
|
err = checkResponse(resp)
|
|
|
if err != nil {
|
|
|
// StatusCode != 2xx when Get Object
|
|
|
if !closeBody {
|
|
|
resp.Body.Close()
|
|
|
}
|
|
|
// even though there was an error, we still return the response
|
|
|
// in case the caller wants to inspect it further
|
|
|
return response, err
|
|
|
}
|
|
|
|
|
|
// need CRC64 verification
|
|
|
if reader, ok := req.Body.(*teeReader); ok {
|
|
|
if c.Conf.EnableCRC && reader.writer != nil && !reader.disableCheckSum {
|
|
|
localcrc := reader.Crc64()
|
|
|
scoscrc := response.Header.Get("x-cos-hash-crc64ecma")
|
|
|
icoscrc, err := strconv.ParseUint(scoscrc, 10, 64)
|
|
|
if icoscrc != localcrc {
|
|
|
return response, fmt.Errorf("verification failed, want:%v, return:%v, x-cos-hash-crc64ecma:%v, err:%v, header:%+v", localcrc, icoscrc, scoscrc, err, response.Header)
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
if result != nil {
|
|
|
if w, ok := result.(io.Writer); ok {
|
|
|
io.Copy(w, resp.Body)
|
|
|
} else {
|
|
|
err = xml.NewDecoder(resp.Body).Decode(result)
|
|
|
if err == io.EOF {
|
|
|
err = nil // ignore EOF errors caused by empty response body
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
return response, err
|
|
|
}
|
|
|
|
|
|
type sendOptions struct {
|
|
|
// 基础 URL
|
|
|
baseURL *url.URL
|
|
|
// URL 中除基础 URL 外的剩余部分
|
|
|
uri string
|
|
|
// 请求方法
|
|
|
method string
|
|
|
|
|
|
body interface{}
|
|
|
// url 查询参数
|
|
|
optQuery interface{}
|
|
|
// http header 参数
|
|
|
optHeader interface{}
|
|
|
// 用 result 反序列化 resp.Body
|
|
|
result interface{}
|
|
|
// 是否禁用自动调用 resp.Body.Close()
|
|
|
// 自动调用 Close() 是为了能够重用连接
|
|
|
disableCloseBody bool
|
|
|
}
|
|
|
|
|
|
func (c *Client) doRetry(ctx context.Context, opt *sendOptions) (resp *Response, err error) {
|
|
|
if opt.body != nil {
|
|
|
if _, ok := opt.body.(io.Reader); ok {
|
|
|
resp, err = c.send(ctx, opt)
|
|
|
return
|
|
|
}
|
|
|
}
|
|
|
count := 1
|
|
|
if count < c.Conf.RetryOpt.Count {
|
|
|
count = c.Conf.RetryOpt.Count
|
|
|
}
|
|
|
nr := 0
|
|
|
interval := c.Conf.RetryOpt.Interval
|
|
|
for nr < count {
|
|
|
resp, err = c.send(ctx, opt)
|
|
|
if err != nil && err != invalidBucketErr {
|
|
|
if resp != nil && resp.StatusCode <= 499 {
|
|
|
dobreak := true
|
|
|
for _, v := range c.Conf.RetryOpt.StatusCode {
|
|
|
if resp.StatusCode == v {
|
|
|
dobreak = false
|
|
|
break
|
|
|
}
|
|
|
}
|
|
|
if dobreak {
|
|
|
break
|
|
|
}
|
|
|
}
|
|
|
nr++
|
|
|
if interval > 0 && nr < count {
|
|
|
time.Sleep(interval)
|
|
|
}
|
|
|
continue
|
|
|
}
|
|
|
break
|
|
|
}
|
|
|
return
|
|
|
|
|
|
}
|
|
|
func (c *Client) send(ctx context.Context, opt *sendOptions) (resp *Response, err error) {
|
|
|
req, err := c.newRequest(ctx, opt.baseURL, opt.uri, opt.method, opt.body, opt.optQuery, opt.optHeader)
|
|
|
if err != nil {
|
|
|
return
|
|
|
}
|
|
|
|
|
|
resp, err = c.doAPI(ctx, req, opt.result, !opt.disableCloseBody)
|
|
|
return
|
|
|
}
|
|
|
|
|
|
// addURLOptions adds the parameters in opt as URL query parameters to s. opt
|
|
|
// must be a struct whose fields may contain "url" tags.
|
|
|
func addURLOptions(s string, opt interface{}) (string, error) {
|
|
|
v := reflect.ValueOf(opt)
|
|
|
if v.Kind() == reflect.Ptr && v.IsNil() {
|
|
|
return s, nil
|
|
|
}
|
|
|
|
|
|
u, err := url.Parse(s)
|
|
|
if err != nil {
|
|
|
return s, err
|
|
|
}
|
|
|
|
|
|
qs, err := query.Values(opt)
|
|
|
if err != nil {
|
|
|
return s, err
|
|
|
}
|
|
|
|
|
|
// 保留原有的参数，并且放在前面。因为 cos 的 url 路由是以第一个参数作为路由的
|
|
|
// e.g. /?uploads
|
|
|
q := u.RawQuery
|
|
|
rq := qs.Encode()
|
|
|
if q != "" {
|
|
|
if rq != "" {
|
|
|
u.RawQuery = fmt.Sprintf("%s&%s", q, qs.Encode())
|
|
|
}
|
|
|
} else {
|
|
|
u.RawQuery = rq
|
|
|
}
|
|
|
return u.String(), nil
|
|
|
}
|
|
|
|
|
|
type XOptionalValue struct {
|
|
|
Header *http.Header
|
|
|
}
|
|
|
|
|
|
// addHeaderOptions adds the parameters in opt as Header fields to req. opt
|
|
|
// must be a struct whose fields may contain "header" tags.
|
|
|
func addHeaderOptions(ctx context.Context, header http.Header, opt interface{}) (http.Header, error) {
|
|
|
defer func() {
|
|
|
// 通过context传递
|
|
|
if val := ctx.Value(XOptionalKey); val != nil {
|
|
|
if optVal, ok := val.(*XOptionalValue); ok {
|
|
|
if optVal.Header != nil {
|
|
|
for key, values := range *optVal.Header {
|
|
|
for _, value := range values {
|
|
|
header.Add(key, value)
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}()
|
|
|
v := reflect.ValueOf(opt)
|
|
|
if v.Kind() == reflect.Ptr && v.IsNil() {
|
|
|
return header, nil
|
|
|
}
|
|
|
|
|
|
h, err := httpheader.Header(opt)
|
|
|
if err != nil {
|
|
|
return nil, err
|
|
|
}
|
|
|
|
|
|
for key, values := range h {
|
|
|
for _, value := range values {
|
|
|
header.Add(key, value)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
return header, nil
|
|
|
}
|
|
|
|
|
|
func checkURL(baseURL *url.URL) bool {
|
|
|
host := baseURL.String()
|
|
|
if hostSuffix.MatchString(host) && !hostPrefix.MatchString(host) {
|
|
|
return false
|
|
|
}
|
|
|
return true
|
|
|
}
|
|
|
|
|
|
// Owner defines Bucket/Object's owner
|
|
|
type Owner struct {
|
|
|
UIN string `xml:"uin,omitempty"`
|
|
|
ID string `xml:",omitempty"`
|
|
|
DisplayName string `xml:",omitempty"`
|
|
|
}
|
|
|
|
|
|
// Initiator same to the Owner struct
|
|
|
type Initiator Owner
|
|
|
|
|
|
// Response API 响应
|
|
|
type Response struct {
|
|
|
*http.Response
|
|
|
}
|
|
|
|
|
|
func newResponse(resp *http.Response) *Response {
|
|
|
return &Response{
|
|
|
Response: resp,
|
|
|
}
|
|
|
}
|
|
|
|
|
|
// ACLHeaderOptions is the option of ACLHeader
|
|
|
type ACLHeaderOptions struct {
|
|
|
XCosACL string `header:"x-cos-acl,omitempty" url:"-" xml:"-"`
|
|
|
XCosGrantRead string `header:"x-cos-grant-read,omitempty" url:"-" xml:"-"`
|
|
|
XCosGrantWrite string `header:"x-cos-grant-write,omitempty" url:"-" xml:"-"`
|
|
|
XCosGrantFullControl string `header:"x-cos-grant-full-control,omitempty" url:"-" xml:"-"`
|
|
|
XCosGrantReadACP string `header:"x-cos-grant-read-acp,omitempty" url:"-" xml:"-"`
|
|
|
XCosGrantWriteACP string `header:"x-cos-grant-write-acp,omitempty" url:"-" xml:"-"`
|
|
|
}
|
|
|
|
|
|
// ACLGrantee is the param of ACLGrant
|
|
|
type ACLGrantee struct {
|
|
|
Type string `xml:"type,attr"`
|
|
|
UIN string `xml:"uin,omitempty"`
|
|
|
URI string `xml:"URI,omitempty"`
|
|
|
ID string `xml:",omitempty"`
|
|
|
DisplayName string `xml:",omitempty"`
|
|
|
SubAccount string `xml:"Subaccount,omitempty"`
|
|
|
}
|
|
|
|
|
|
// ACLGrant is the param of ACLXml
|
|
|
type ACLGrant struct {
|
|
|
Grantee *ACLGrantee
|
|
|
Permission string
|
|
|
}
|
|
|
|
|
|
// ACLXml is the ACL body struct
|
|
|
type ACLXml struct {
|
|
|
XMLName xml.Name `xml:"AccessControlPolicy"`
|
|
|
Owner *Owner
|
|
|
AccessControlList []ACLGrant `xml:"AccessControlList>Grant,omitempty"`
|
|
|
}
|
|
|
|
|
|
func decodeACL(resp *Response, res *ACLXml) {
|
|
|
ItemMap := map[string]string{
|
|
|
"ACL": "x-cos-acl",
|
|
|
"READ": "x-cos-grant-read",
|
|
|
"WRITE": "x-cos-grant-write",
|
|
|
"READ_ACP": "x-cos-grant-read-acp",
|
|
|
"WRITE_ACP": "x-cos-grant-write-acp",
|
|
|
"FULL_CONTROL": "x-cos-grant-full-control",
|
|
|
}
|
|
|
publicACL := make(map[string]int)
|
|
|
resACL := make(map[string][]string)
|
|
|
for _, item := range res.AccessControlList {
|
|
|
if item.Grantee == nil {
|
|
|
continue
|
|
|
}
|
|
|
if item.Grantee.ID == "qcs::cam::anyone:anyone" || item.Grantee.URI == "http://cam.qcloud.com/groups/global/AllUsers" {
|
|
|
publicACL[item.Permission] = 1
|
|
|
} else if item.Grantee.ID != res.Owner.ID {
|
|
|
resACL[item.Permission] = append(resACL[item.Permission], "id=\""+item.Grantee.ID+"\"")
|
|
|
}
|
|
|
}
|
|
|
if publicACL["FULL_CONTROL"] == 1 || (publicACL["READ"] == 1 && publicACL["WRITE"] == 1) {
|
|
|
resACL["ACL"] = []string{"public-read-write"}
|
|
|
} else if publicACL["READ"] == 1 {
|
|
|
resACL["ACL"] = []string{"public-read"}
|
|
|
} else {
|
|
|
resACL["ACL"] = []string{"private"}
|
|
|
}
|
|
|
|
|
|
for item, header := range ItemMap {
|
|
|
if len(resp.Header.Get(header)) > 0 || len(resACL[item]) == 0 {
|
|
|
continue
|
|
|
}
|
|
|
resp.Header.Set(header, uniqueGrantID(resACL[item]))
|
|
|
}
|
|
|
}
|
|
|
|
|
|
func uniqueGrantID(grantIDs []string) string {
|
|
|
res := []string{}
|
|
|
filter := make(map[string]int)
|
|
|
for _, id := range grantIDs {
|
|
|
if filter[id] != 0 {
|
|
|
continue
|
|
|
}
|
|
|
filter[id] = 1
|
|
|
res = append(res, id)
|
|
|
}
|
|
|
return strings.Join(res, ",")
|
|
|
}
|
