You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
go-library/vendor/github.com/upyun/go-sdk/v3/upyun/rest.go

742 lines
17 KiB

package upyun
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"os"
"path"
"strconv"
"strings"
"time"
)
const (
DefaultPartSize = 1024 * 1024
MaxPartNum = 10000
minResumePutFileSize = 10 * 1024 * 1024
)
type restReqConfig struct {
method string
uri string
query string
headers map[string]string
closeBody bool
httpBody io.Reader
useMD5 bool
}
// GetObjectConfig provides a configuration to Get method.
type GetObjectConfig struct {
Path string
// Headers contains custom http header, like User-Agent.
Headers map[string]string
LocalPath string
Writer io.Writer
}
// GetObjectConfig provides a configuration to List method.
type GetObjectsConfig struct {
Path string
Headers map[string]string
ObjectsChan chan *FileInfo
QuitChan chan bool
MaxListObjects int
MaxListTries int
// MaxListLevel: depth of recursion
MaxListLevel int
// DescOrder: whether list objects by desc-order
DescOrder bool
rootDir string
level int
objNum int
try int
}
// PutObjectConfig provides a configuration to Put method.
type PutObjectConfig struct {
Path string
LocalPath string
Reader io.Reader
Headers map[string]string
UseMD5 bool
UseResumeUpload bool
// Append Api Deprecated
// AppendContent bool
ResumePartSize int64
MaxResumePutTries int
}
type MoveObjectConfig struct {
SrcPath string
DestPath string
Headers map[string]string
}
type CopyObjectConfig struct {
SrcPath string
DestPath string
Headers map[string]string
}
// UploadFileConfig is multipart file upload config
type UploadPartConfig struct {
Reader io.Reader
PartSize int64
PartID int
}
type CompleteMultipartUploadConfig struct {
Md5 string
}
type InitMultipartUploadConfig struct {
Path string
PartSize int64
ContentLength int64 // optional
ContentType string
OrderUpload bool
}
type InitMultipartUploadResult struct {
UploadID string
Path string
PartSize int64
}
type DeleteObjectConfig struct {
Path string
Async bool
Folder bool // optional
}
type ModifyMetadataConfig struct {
Path string
Operation string
Headers map[string]string
}
type ListMultipartConfig struct {
Prefix string
Limit int64
}
type ListMultipartPartsConfig struct {
BeginID int
}
type MultipartUploadFile struct {
Key string `json:"key"`
UUID string `json:"uuid"`
Completed bool `json:"completed"`
CreatedAt int64 `json:"created_at"`
}
type ListMultipartUploadResult struct {
Files []*MultipartUploadFile `json:"files"`
}
type MultipartUploadedPart struct {
Etag string `json:"etag"`
Size int64 `json:"size"`
Id int `json:"id"`
}
type ListUploadedPartsResult struct {
Parts []*MultipartUploadedPart `json:"parts"`
}
func (up *UpYun) Usage() (n int64, err error) {
var resp *http.Response
resp, err = up.doRESTRequest(&restReqConfig{
method: "GET",
uri: "/",
query: "usage",
})
if err == nil {
n, err = readHTTPBodyToInt(resp)
}
if err != nil {
return 0, errorOperation("usage", err)
}
return n, nil
}
func (up *UpYun) Mkdir(path string) error {
_, err := up.doRESTRequest(&restReqConfig{
method: "POST",
uri: path,
headers: map[string]string{
"folder": "true",
"x-upyun-folder": "true",
},
closeBody: true,
})
if err != nil {
return errorOperation(fmt.Sprintf("mkdir %s", path), err)
}
return nil
}
// TODO: maybe directory
func (up *UpYun) Get(config *GetObjectConfig) (fInfo *FileInfo, err error) {
if config.LocalPath != "" {
var fd *os.File
if fd, err = os.Create(config.LocalPath); err != nil {
return nil, errorOperation("create file", err)
}
defer fd.Close()
config.Writer = fd
}
if config.Writer == nil {
return nil, errors.New("no writer")
}
resp, err := up.doRESTRequest(&restReqConfig{
method: "GET",
uri: config.Path,
})
if err != nil {
return nil, errorOperation(fmt.Sprintf("get %s", config.Path), err)
}
defer resp.Body.Close()
fInfo = parseHeaderToFileInfo(resp.Header, false)
fInfo.Name = config.Path
if fInfo.Size, err = io.Copy(config.Writer, resp.Body); err != nil {
return nil, errorOperation("io copy", err)
}
return
}
func (up *UpYun) put(config *PutObjectConfig) error {
/* Append Api Deprecated
if config.AppendContent {
if config.Headers == nil {
config.Headers = make(map[string]string)
}
config.Headers["X-Upyun-Append"] = "true"
}
*/
_, err := up.doRESTRequest(&restReqConfig{
method: "PUT",
uri: config.Path,
headers: config.Headers,
closeBody: true,
httpBody: config.Reader,
useMD5: config.UseMD5,
})
if err != nil {
return errorOperation(fmt.Sprintf("put %s", config.Path), err)
}
return nil
}
func getPartInfo(partSize, fsize int64) (int64, int64, error) {
if partSize <= 0 {
partSize = DefaultPartSize
}
if partSize < DefaultPartSize {
return 0, 0, fmt.Errorf("The minimum of part size is %d", DefaultPartSize)
}
if partSize%DefaultPartSize != 0 {
return 0, 0, fmt.Errorf("The part size is a multiple of %d", DefaultPartSize)
}
partNum := (fsize + partSize - 1) / partSize
if partNum > MaxPartNum {
return 0, 0, fmt.Errorf("The maximum part number is %d", MaxPartNum)
}
return partSize, partNum, nil
}
func (up *UpYun) resumePut(config *PutObjectConfig) error {
f, ok := config.Reader.(*os.File)
if !ok {
return errors.New("resumePut: type != *os.File")
}
fileinfo, err := f.Stat()
if err != nil {
return errorOperation("stat", err)
}
fsize := fileinfo.Size()
if fsize < minResumePutFileSize {
return up.put(config)
}
if config.ResumePartSize == 0 {
config.ResumePartSize = DefaultPartSize
}
maxPartID := int((fsize+config.ResumePartSize-1)/config.ResumePartSize - 1)
if config.Headers == nil {
config.Headers = make(map[string]string)
}
curSize, partSize := int64(0), config.ResumePartSize
headers := config.Headers
uploadInfo, err := up.InitMultipartUpload(&InitMultipartUploadConfig{
Path: config.Path,
PartSize: partSize,
ContentType: headers["Content-Type"],
ContentLength: fsize,
OrderUpload: true,
})
if err != nil {
return err
}
for id := 0; id <= maxPartID; id++ {
if curSize+partSize > fsize {
partSize = fsize - curSize
}
fragFile, err := newFragmentFile(f, curSize, partSize)
if err != nil {
return errorOperation("new fragment file", err)
}
try := 0
for ; config.MaxResumePutTries == 0 || try < config.MaxResumePutTries; try++ {
err = up.UploadPart(uploadInfo, &UploadPartConfig{
PartID: id,
PartSize: partSize,
Reader: fragFile,
})
if err == nil {
break
}
fragFile.Seek(0, 0)
}
if config.MaxResumePutTries > 0 && try == config.MaxResumePutTries {
return err
}
curSize += partSize
}
completeConfig := &CompleteMultipartUploadConfig{}
if config.UseMD5 {
f.Seek(0, 0)
completeConfig.Md5, _ = md5File(f)
}
return up.CompleteMultipartUpload(uploadInfo, completeConfig)
}
func (up *UpYun) Put(config *PutObjectConfig) (err error) {
if config.LocalPath != "" {
var fd *os.File
if fd, err = os.Open(config.LocalPath); err != nil {
return errorOperation("open file", err)
}
defer fd.Close()
config.Reader = fd
}
if config.UseResumeUpload {
return up.resumePut(config)
}
return up.put(config)
}
func (up *UpYun) Move(config *MoveObjectConfig) error {
headers := map[string]string{
"X-Upyun-Move-Source": path.Join("/", up.Bucket, escapeUri(config.SrcPath)),
}
for k, v := range config.Headers {
headers[k] = v
}
_, err := up.doRESTRequest(&restReqConfig{
method: "PUT",
uri: config.DestPath,
headers: headers,
})
if err != nil {
return errorOperation("move source", err)
}
return nil
}
func (up *UpYun) Copy(config *CopyObjectConfig) error {
headers := map[string]string{
"X-Upyun-Copy-Source": path.Join("/", up.Bucket, escapeUri(config.SrcPath)),
}
for k, v := range config.Headers {
headers[k] = v
}
_, err := up.doRESTRequest(&restReqConfig{
method: "PUT",
uri: config.DestPath,
headers: headers,
})
if err != nil {
return errorOperation("copy source", err)
}
return nil
}
func (up *UpYun) InitMultipartUpload(config *InitMultipartUploadConfig) (*InitMultipartUploadResult, error) {
partSize, _, err := getPartInfo(config.PartSize, config.ContentLength)
if err != nil {
return nil, errorOperation("init multipart", err)
}
headers := make(map[string]string)
headers["X-Upyun-Multi-Type"] = config.ContentType
if config.ContentLength > 0 {
headers["X-Upyun-Multi-Length"] = strconv.FormatInt(config.ContentLength, 10)
}
headers["X-Upyun-Multi-Stage"] = "initiate"
if !config.OrderUpload {
headers["X-Upyun-Multi-Disorder"] = "true"
}
headers["X-Upyun-Multi-Part-Size"] = strconv.FormatInt(partSize, 10)
resp, err := up.doRESTRequest(&restReqConfig{
method: "PUT",
uri: config.Path,
headers: headers,
closeBody: true,
})
if err != nil {
return nil, errorOperation("init multipart", err)
}
return &InitMultipartUploadResult{
UploadID: resp.Header.Get("X-Upyun-Multi-Uuid"),
Path: config.Path,
PartSize: partSize,
}, nil
}
func (up *UpYun) UploadPart(initResult *InitMultipartUploadResult, part *UploadPartConfig) error {
headers := make(map[string]string)
headers["X-Upyun-Multi-Stage"] = "upload"
headers["X-Upyun-Multi-Uuid"] = initResult.UploadID
headers["X-Upyun-Part-Id"] = strconv.FormatInt(int64(part.PartID), 10)
headers["Content-Length"] = strconv.FormatInt(part.PartSize, 10)
_, err := up.doRESTRequest(&restReqConfig{
method: "PUT",
uri: initResult.Path,
headers: headers,
closeBody: true,
useMD5: false,
httpBody: part.Reader,
})
if err != nil {
return errorOperation("upload multipart", err)
}
return nil
}
func (up *UpYun) CompleteMultipartUpload(initResult *InitMultipartUploadResult, config *CompleteMultipartUploadConfig) error {
headers := make(map[string]string)
headers["X-Upyun-Multi-Stage"] = "complete"
headers["X-Upyun-Multi-Uuid"] = initResult.UploadID
if config != nil {
if config.Md5 != "" {
headers["X-Upyun-Multi-Md5"] = config.Md5
}
}
_, err := up.doRESTRequest(&restReqConfig{
method: "PUT",
uri: initResult.Path,
headers: headers,
})
if err != nil {
return errorOperation("complete multipart", err)
}
return nil
}
func (up *UpYun) ListMultipartUploads(config *ListMultipartConfig) (*ListMultipartUploadResult, error) {
headers := make(map[string]string)
headers["X-Upyun-List-Type"] = "multi"
if config.Prefix != "" {
headers["X-Upyun-List-Prefix"] = base64.StdEncoding.EncodeToString([]byte(config.Prefix))
}
if config.Limit > 0 {
headers["X-Upyun-List-Limit"] = strconv.FormatInt(config.Limit, 10)
}
res, err := up.doRESTRequest(&restReqConfig{
method: "GET",
headers: headers,
uri: "/",
closeBody: false,
useMD5: false,
})
if err != nil {
return nil, errorOperation("list multipart", err)
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, errorOperation("list multipart read body", err)
}
result := &ListMultipartUploadResult{}
err = json.Unmarshal(body, result)
if err != nil {
return nil, errorOperation("list multipart read body", err)
}
return result, nil
}
func (up *UpYun) ListMultipartParts(intiResult *InitMultipartUploadResult, config *ListMultipartPartsConfig) (*ListUploadedPartsResult, error) {
headers := make(map[string]string)
headers["X-Upyun-Multi-Uuid"] = intiResult.UploadID
if config.BeginID > 0 {
headers["X-Upyun-Part-Id"] = fmt.Sprint(config.BeginID)
}
res, err := up.doRESTRequest(&restReqConfig{
method: "GET",
headers: headers,
uri: intiResult.Path,
closeBody: false,
useMD5: false,
})
if err != nil {
return nil, errorOperation("list multipart parts", err)
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, errorOperation("list multipart parts read body", err)
}
result := &ListUploadedPartsResult{}
err = json.Unmarshal(body, result)
if err != nil {
return nil, errorOperation("list multipart parts read body", err)
}
return result, nil
}
func (up *UpYun) Delete(config *DeleteObjectConfig) error {
headers := map[string]string{}
if config.Async {
headers["x-upyun-async"] = "true"
}
if config.Folder {
headers["x-upyun-folder"] = "true"
}
_, err := up.doRESTRequest(&restReqConfig{
method: "DELETE",
uri: config.Path,
headers: headers,
closeBody: true,
})
if err != nil {
return errorOperation("delete", err)
}
return nil
}
func (up *UpYun) GetInfo(path string) (*FileInfo, error) {
resp, err := up.doRESTRequest(&restReqConfig{
method: "HEAD",
uri: path,
closeBody: true,
})
if err != nil {
return nil, errorOperation("get info", err)
}
fInfo := parseHeaderToFileInfo(resp.Header, true)
fInfo.Name = path
return fInfo, nil
}
func (up *UpYun) List(config *GetObjectsConfig) error {
if config.ObjectsChan == nil {
return errors.New("ObjectsChan is nil")
}
if config.Headers == nil {
config.Headers = make(map[string]string)
}
if config.QuitChan == nil {
config.QuitChan = make(chan bool)
}
// 50 is nice value
if _, exist := config.Headers["X-List-Limit"]; !exist {
config.Headers["X-List-Limit"] = "50"
}
if config.DescOrder {
config.Headers["X-List-Order"] = "desc"
}
config.Headers["X-UpYun-Folder"] = "true"
config.Headers["Accept"] = "application/json"
// 1st level
if config.level == 0 {
defer close(config.ObjectsChan)
}
for {
resp, err := up.doRESTRequest(&restReqConfig{
method: "GET",
uri: config.Path,
headers: config.Headers,
})
if err != nil {
var nerr net.Error
if ok := errors.As(err, &nerr); ok {
config.try++
if config.MaxListTries == 0 || config.try < config.MaxListTries {
time.Sleep(10 * time.Millisecond)
continue
}
}
return errorOperation("list", err)
}
b, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return errorOperation("list read body", err)
}
iter, files, err := parseBodyToFileInfos(b)
if err != nil {
return errorOperation("list read body", err)
}
for _, fInfo := range files {
if fInfo.IsDir && (config.level+1 < config.MaxListLevel || config.MaxListLevel == -1) {
rConfig := &GetObjectsConfig{
Path: path.Join(config.Path, fInfo.Name),
QuitChan: config.QuitChan,
ObjectsChan: config.ObjectsChan,
MaxListTries: config.MaxListTries,
MaxListObjects: config.MaxListObjects,
DescOrder: config.DescOrder,
MaxListLevel: config.MaxListLevel,
level: config.level + 1,
rootDir: path.Join(config.rootDir, fInfo.Name),
try: config.try,
objNum: config.objNum,
}
if err = up.List(rConfig); err != nil {
return err
}
// empty folder
if config.objNum == rConfig.objNum {
fInfo.IsEmptyDir = true
}
config.try, config.objNum = rConfig.try, rConfig.objNum
}
if config.rootDir != "" {
fInfo.Name = path.Join(config.rootDir, fInfo.Name)
}
select {
case <-config.QuitChan:
return nil
default:
config.ObjectsChan <- fInfo
}
config.objNum++
if config.MaxListObjects > 0 && config.objNum >= config.MaxListObjects {
return nil
}
}
if iter == "g2gCZAAEbmV4dGQAA2VvZg" {
return nil
}
config.Headers["X-List-Iter"] = iter
}
}
func (up *UpYun) ModifyMetadata(config *ModifyMetadataConfig) error {
if config.Operation == "" {
config.Operation = "merge"
}
_, err := up.doRESTRequest(&restReqConfig{
method: "PATCH",
uri: config.Path,
query: "metadata=" + config.Operation,
headers: config.Headers,
closeBody: true,
})
if err != nil {
return errorOperation("modify metadata", err)
}
return nil
}
func (up *UpYun) doRESTRequest(config *restReqConfig) (*http.Response, error) {
escUri := path.Join("/", up.Bucket, escapeUri(config.uri))
if strings.HasSuffix(config.uri, "/") {
escUri += "/"
}
if config.query != "" {
escUri += "?" + config.query
}
headers := map[string]string{}
hasMD5 := false
for k, v := range config.headers {
if strings.ToLower(k) == "content-md5" && v != "" {
hasMD5 = true
}
headers[k] = v
}
headers["Date"] = makeRFC1123Date(time.Now())
headers["Host"] = "v0.api.upyun.com"
if !hasMD5 && config.useMD5 {
switch v := config.httpBody.(type) {
case *os.File:
headers["Content-MD5"], _ = md5File(v)
case UpYunPutReader:
headers["Content-MD5"] = v.MD5()
}
}
if up.deprecated {
if _, ok := headers["Content-Length"]; !ok {
size := int64(0)
switch v := config.httpBody.(type) {
case *os.File:
if fInfo, err := v.Stat(); err == nil {
size = fInfo.Size()
}
case UpYunPutReader:
size = int64(v.Len())
}
headers["Content-Length"] = fmt.Sprint(size)
}
headers["Authorization"] = up.MakeRESTAuth(&RESTAuthConfig{
Method: config.method,
Uri: escUri,
DateStr: headers["Date"],
LengthStr: headers["Content-Length"],
})
} else {
headers["Authorization"] = up.MakeUnifiedAuth(&UnifiedAuthConfig{
Method: config.method,
Uri: escUri,
DateStr: headers["Date"],
ContentMD5: headers["Content-MD5"],
})
}
endpoint := up.doGetEndpoint("v0.api.upyun.com")
url := fmt.Sprintf("http://%s%s", endpoint, escUri)
resp, err := up.doHTTPRequest(config.method, url, headers, config.httpBody)
if err != nil {
return nil, err
}
if config.closeBody {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
return resp, nil
}