You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
580 lines
14 KiB
580 lines
14 KiB
package storage
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/qiniu/go-sdk/v7/auth"
|
|
"github.com/qiniu/go-sdk/v7/client"
|
|
"golang.org/x/sync/singleflight"
|
|
)
|
|
|
|
// 存储所在的地区,例如华东,华南,华北
|
|
// 每个存储区域可能有多个机房信息,每个机房可能有多个上传入口
|
|
type Region struct {
|
|
// 上传入口
|
|
SrcUpHosts []string `json:"src_up,omitempty"`
|
|
|
|
// 加速上传入口
|
|
CdnUpHosts []string `json:"cdn_up,omitempty"`
|
|
|
|
// 获取文件信息入口
|
|
RsHost string `json:"rs,omitempty"`
|
|
|
|
// bucket列举入口
|
|
RsfHost string `json:"rsf,omitempty"`
|
|
|
|
ApiHost string `json:"api,omitempty"`
|
|
|
|
// 存储io 入口
|
|
IovipHost string `json:"io,omitempty"`
|
|
}
|
|
|
|
type RegionID string
|
|
|
|
// GetDefaultReion 根据RegionID获取对应的Region信息
|
|
func GetRegionByID(regionID RegionID) (Region, bool) {
|
|
if r, ok := regionMap[regionID]; ok {
|
|
return r, ok
|
|
}
|
|
return Region{}, false
|
|
}
|
|
|
|
func (r *Region) String() string {
|
|
str := ""
|
|
str += fmt.Sprintf("SrcUpHosts: %v\n", r.SrcUpHosts)
|
|
str += fmt.Sprintf("CdnUpHosts: %v\n", r.CdnUpHosts)
|
|
str += fmt.Sprintf("IovipHost: %s\n", r.IovipHost)
|
|
str += fmt.Sprintf("RsHost: %s\n", r.RsHost)
|
|
str += fmt.Sprintf("RsfHost: %s\n", r.RsfHost)
|
|
str += fmt.Sprintf("ApiHost: %s\n", r.ApiHost)
|
|
return str
|
|
}
|
|
|
|
func endpoint(useHttps bool, host string) string {
|
|
host = strings.TrimSpace(host)
|
|
host = strings.TrimLeft(host, "http://")
|
|
host = strings.TrimLeft(host, "https://")
|
|
if host == "" {
|
|
return ""
|
|
}
|
|
scheme := "http://"
|
|
if useHttps {
|
|
scheme = "https://"
|
|
}
|
|
return fmt.Sprintf("%s%s", scheme, host)
|
|
}
|
|
|
|
// 获取rsfHost
|
|
func (r *Region) GetRsfHost(useHttps bool) string {
|
|
return endpoint(useHttps, r.RsfHost)
|
|
}
|
|
|
|
// 获取io host
|
|
func (r *Region) GetIoHost(useHttps bool) string {
|
|
return endpoint(useHttps, r.IovipHost)
|
|
}
|
|
|
|
// 获取RsHost
|
|
func (r *Region) GetRsHost(useHttps bool) string {
|
|
return endpoint(useHttps, r.RsHost)
|
|
}
|
|
|
|
// 获取api host
|
|
func (r *Region) GetApiHost(useHttps bool) string {
|
|
return endpoint(useHttps, r.ApiHost)
|
|
}
|
|
|
|
var (
|
|
// regionHuadong 表示华东机房
|
|
regionHuadong = Region{
|
|
SrcUpHosts: []string{
|
|
"up.qiniup.com",
|
|
"up-nb.qiniup.com",
|
|
"up-xs.qiniup.com",
|
|
},
|
|
CdnUpHosts: []string{
|
|
"upload.qiniup.com",
|
|
"upload-nb.qiniup.com",
|
|
"upload-xs.qiniup.com",
|
|
},
|
|
RsHost: "rs.qbox.me",
|
|
RsfHost: "rsf.qbox.me",
|
|
ApiHost: "api.qiniu.com",
|
|
IovipHost: "iovip.qbox.me",
|
|
}
|
|
|
|
// regionHuadongZhejiang2 表示华东-浙江2
|
|
regionHuadongZhejiang2 = Region{
|
|
SrcUpHosts: []string{
|
|
"up-cn-east-2.qiniup.com",
|
|
},
|
|
CdnUpHosts: []string{
|
|
"upload-cn-east-2.qiniup.com",
|
|
},
|
|
RsHost: "rs-cn-east-2.qiniuapi.com",
|
|
RsfHost: "rsf-cn-east-2-qiniuapi.com",
|
|
ApiHost: "api-cn-east-2.qiniuapi.com",
|
|
IovipHost: "iovip-cn-east-2.qiniuio.com",
|
|
}
|
|
|
|
// regionHuabei 表示华北机房
|
|
regionHuabei = Region{
|
|
SrcUpHosts: []string{
|
|
"up-z1.qiniup.com",
|
|
},
|
|
CdnUpHosts: []string{
|
|
"upload-z1.qiniup.com",
|
|
},
|
|
RsHost: "rs-z1.qbox.me",
|
|
RsfHost: "rsf-z1.qbox.me",
|
|
ApiHost: "api-z1.qiniuapi.com",
|
|
IovipHost: "iovip-z1.qbox.me",
|
|
}
|
|
// regionHuanan 表示华南机房
|
|
regionHuanan = Region{
|
|
SrcUpHosts: []string{
|
|
"up-z2.qiniup.com",
|
|
"up-gz.qiniup.com",
|
|
"up-fs.qiniup.com",
|
|
},
|
|
CdnUpHosts: []string{
|
|
"upload-z2.qiniup.com",
|
|
"upload-gz.qiniup.com",
|
|
"upload-fs.qiniup.com",
|
|
},
|
|
RsHost: "rs-z2.qbox.me",
|
|
RsfHost: "rsf-z2.qbox.me",
|
|
ApiHost: "api-z2.qiniuapi.com",
|
|
IovipHost: "iovip-z2.qbox.me",
|
|
}
|
|
|
|
// regionNorthAmerica 表示北美机房
|
|
regionNorthAmerica = Region{
|
|
SrcUpHosts: []string{
|
|
"up-na0.qiniup.com",
|
|
},
|
|
CdnUpHosts: []string{
|
|
"upload-na0.qiniup.com",
|
|
},
|
|
RsHost: "rs-na0.qbox.me",
|
|
RsfHost: "rsf-na0.qbox.me",
|
|
ApiHost: "api-na0.qiniuapi.com",
|
|
IovipHost: "iovip-na0.qbox.me",
|
|
}
|
|
// regionSingapore 表示新加坡机房
|
|
regionSingapore = Region{
|
|
SrcUpHosts: []string{
|
|
"up-as0.qiniup.com",
|
|
},
|
|
CdnUpHosts: []string{
|
|
"upload-as0.qiniup.com",
|
|
},
|
|
RsHost: "rs-as0.qbox.me",
|
|
RsfHost: "rsf-as0.qbox.me",
|
|
ApiHost: "api-as0.qiniuapi.com",
|
|
IovipHost: "iovip-as0.qbox.me",
|
|
}
|
|
// regionFogCnEast1 表示雾存储华东区
|
|
regionFogCnEast1 = Region{
|
|
SrcUpHosts: []string{
|
|
"up-fog-cn-east-1.qiniup.com",
|
|
},
|
|
CdnUpHosts: []string{
|
|
"upload-fog-cn-east-1.qiniup.com",
|
|
},
|
|
RsHost: "rs-fog-cn-east-1.qbox.me",
|
|
RsfHost: "rsf-fog-cn-east-1.qbox.me",
|
|
ApiHost: "api-fog-cn-east-1.qiniuapi.com",
|
|
IovipHost: "iovip-fog-cn-east-1.qbox.me",
|
|
}
|
|
)
|
|
|
|
const (
|
|
// region code
|
|
RIDHuadong = RegionID("z0")
|
|
RIDHuadongZheJiang2 = RegionID("cn-east-2")
|
|
RIDHuabei = RegionID("z1")
|
|
RIDHuanan = RegionID("z2")
|
|
RIDNorthAmerica = RegionID("na0")
|
|
RIDSingapore = RegionID("as0")
|
|
RIDFogCnEast1 = RegionID("fog-cn-east-1")
|
|
)
|
|
|
|
// regionMap 是RegionID到具体的Region的映射
|
|
var regionMap = map[RegionID]Region{
|
|
RIDHuadong: regionHuadong,
|
|
RIDHuadongZheJiang2: regionHuadongZhejiang2,
|
|
RIDHuanan: regionHuanan,
|
|
RIDHuabei: regionHuabei,
|
|
RIDSingapore: regionSingapore,
|
|
RIDNorthAmerica: regionNorthAmerica,
|
|
RIDFogCnEast1: regionFogCnEast1,
|
|
}
|
|
|
|
/// UcHost 为查询空间相关域名的API服务地址
|
|
/// 设置 UcHost 时,如果不指定 scheme 默认会使用 https
|
|
/// UcHost 已废弃,建议使用 SetUcHost
|
|
//Deprecated
|
|
var UcHost = "https://uc.qbox.me"
|
|
|
|
var ucHost = ""
|
|
|
|
func SetUcHost(host string, useHttps bool) {
|
|
ucHost = endpoint(useHttps, host)
|
|
}
|
|
|
|
func getUcHostByDefaultProtocol() string {
|
|
return getUcHost(true)
|
|
}
|
|
|
|
func getUcHost(useHttps bool) string {
|
|
if ucHost != "" {
|
|
return ucHost
|
|
}
|
|
|
|
if strings.Contains(UcHost, "://") {
|
|
return UcHost
|
|
} else {
|
|
return endpoint(useHttps, UcHost)
|
|
}
|
|
}
|
|
|
|
// UcQueryRet 为查询请求的回复
|
|
type UcQueryRet struct {
|
|
TTL int `json:"ttl"`
|
|
Io map[string]map[string][]string
|
|
IoInfo map[string]UcQueryIo `json:"io"`
|
|
Up map[string]UcQueryUp `json:"up"`
|
|
}
|
|
|
|
func (uc *UcQueryRet) UnmarshalJSON(data []byte) error {
|
|
t := struct {
|
|
TTL int `json:"ttl"`
|
|
IoInfo map[string]UcQueryIo `json:"io"`
|
|
Up map[string]UcQueryUp `json:"up"`
|
|
}{}
|
|
if err := json.Unmarshal(data, &t); err != nil {
|
|
return err
|
|
}
|
|
|
|
uc.TTL = t.TTL
|
|
uc.IoInfo = t.IoInfo
|
|
uc.Up = t.Up
|
|
uc.setup()
|
|
return nil
|
|
}
|
|
|
|
func (uc *UcQueryRet) setup() {
|
|
if uc.Io != nil || uc.IoInfo == nil {
|
|
return
|
|
}
|
|
|
|
uc.Io = make(map[string]map[string][]string)
|
|
ioSrc := uc.IoInfo["src"].toMapWithoutInfo()
|
|
if ioSrc != nil && len(ioSrc) > 0 {
|
|
uc.Io["src"] = ioSrc
|
|
}
|
|
|
|
ioOldSrc := uc.IoInfo["old_src"].toMapWithoutInfo()
|
|
if ioOldSrc != nil && len(ioOldSrc) > 0 {
|
|
uc.Io["old_src"] = ioOldSrc
|
|
}
|
|
}
|
|
|
|
// UcQueryUp 为查询请求回复中的上传域名信息
|
|
type UcQueryUp struct {
|
|
Main []string `json:"main,omitempty"`
|
|
Backup []string `json:"backup,omitempty"`
|
|
Info string `json:"info,omitempty"`
|
|
}
|
|
|
|
// UcQueryIo 为查询请求回复中的上传域名信息
|
|
type UcQueryIo struct {
|
|
Main []string `json:"main,omitempty"`
|
|
Backup []string `json:"backup,omitempty"`
|
|
Info string `json:"info,omitempty"`
|
|
}
|
|
|
|
func (io UcQueryIo) toMapWithoutInfo() map[string][]string {
|
|
|
|
ret := make(map[string][]string)
|
|
if io.Main != nil && len(io.Main) > 0 {
|
|
ret["main"] = io.Main
|
|
}
|
|
|
|
if io.Backup != nil && len(io.Backup) > 0 {
|
|
ret["backup"] = io.Backup
|
|
}
|
|
|
|
return ret
|
|
}
|
|
|
|
type regionCacheValue struct {
|
|
Region *Region `json:"region"`
|
|
Deadline time.Time `json:"deadline"`
|
|
}
|
|
|
|
type regionCacheMap map[string]regionCacheValue
|
|
|
|
var (
|
|
regionCachePath = filepath.Join(os.TempDir(), "qiniu-golang-sdk", "query.cache.json")
|
|
regionCache sync.Map
|
|
regionCacheLock sync.RWMutex
|
|
regionCacheSyncLock sync.Mutex
|
|
regionCacheGroup singleflight.Group
|
|
regionCacheLoaded bool = false
|
|
)
|
|
|
|
func SetRegionCachePath(newPath string) {
|
|
regionCacheLock.Lock()
|
|
defer regionCacheLock.Unlock()
|
|
|
|
regionCachePath = newPath
|
|
regionCacheLoaded = false
|
|
}
|
|
|
|
func loadRegionCache() {
|
|
cacheFile, err := os.Open(regionCachePath)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer cacheFile.Close()
|
|
|
|
var cacheMap regionCacheMap
|
|
if err = json.NewDecoder(cacheFile).Decode(&cacheMap); err != nil {
|
|
return
|
|
}
|
|
for cacheKey, cacheValue := range cacheMap {
|
|
regionCache.Store(cacheKey, cacheValue)
|
|
}
|
|
}
|
|
|
|
func storeRegionCache() {
|
|
err := os.MkdirAll(filepath.Dir(regionCachePath), 0700)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
cacheFile, err := os.OpenFile(regionCachePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer cacheFile.Close()
|
|
|
|
cacheMap := make(regionCacheMap)
|
|
regionCache.Range(func(cacheKey, cacheValue interface{}) bool {
|
|
cacheMap[cacheKey.(string)] = cacheValue.(regionCacheValue)
|
|
return true
|
|
})
|
|
if err = json.NewEncoder(cacheFile).Encode(cacheMap); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
// GetRegion 用来根据ak和bucket来获取空间相关的机房信息
|
|
func GetRegion(ak, bucket string) (*Region, error) {
|
|
regionCacheLock.RLock()
|
|
if regionCacheLoaded {
|
|
regionCacheLock.RUnlock()
|
|
} else {
|
|
regionCacheLock.RUnlock()
|
|
func() {
|
|
regionCacheLock.Lock()
|
|
defer regionCacheLock.Unlock()
|
|
|
|
if !regionCacheLoaded {
|
|
loadRegionCache()
|
|
regionCacheLoaded = true
|
|
}
|
|
}()
|
|
}
|
|
|
|
regionID := fmt.Sprintf("%s:%s", ak, bucket)
|
|
//check from cache
|
|
if v, ok := regionCache.Load(regionID); ok && time.Now().Before(v.(regionCacheValue).Deadline) {
|
|
return v.(regionCacheValue).Region, nil
|
|
}
|
|
|
|
newRegion, err, _ := regionCacheGroup.Do(regionID, func() (interface{}, error) {
|
|
reqURL := fmt.Sprintf("%s/v2/query?ak=%s&bucket=%s", getUcHostByDefaultProtocol(), ak, bucket)
|
|
|
|
var ret UcQueryRet
|
|
err := client.DefaultClient.CallWithForm(context.Background(), &ret, "GET", reqURL, nil, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("query region error, %s", err.Error())
|
|
}
|
|
|
|
if len(ret.Io["src"]["main"]) <= 0 {
|
|
return nil, fmt.Errorf("empty io host list")
|
|
}
|
|
|
|
ioHost := ret.Io["src"]["main"][0]
|
|
srcUpHosts := ret.Up["src"].Main
|
|
if ret.Up["src"].Backup != nil {
|
|
srcUpHosts = append(srcUpHosts, ret.Up["src"].Backup...)
|
|
}
|
|
cdnUpHosts := ret.Up["acc"].Main
|
|
if ret.Up["acc"].Backup != nil {
|
|
cdnUpHosts = append(cdnUpHosts, ret.Up["acc"].Backup...)
|
|
}
|
|
|
|
region := &Region{
|
|
SrcUpHosts: srcUpHosts,
|
|
CdnUpHosts: cdnUpHosts,
|
|
IovipHost: ioHost,
|
|
RsHost: DefaultRsHost,
|
|
RsfHost: DefaultRsfHost,
|
|
ApiHost: DefaultAPIHost,
|
|
}
|
|
|
|
//set specific hosts if possible
|
|
setSpecificHosts(ioHost, region)
|
|
regionCache.Store(regionID, regionCacheValue{
|
|
Region: region,
|
|
Deadline: time.Now().Add(time.Duration(ret.TTL) * time.Second),
|
|
})
|
|
|
|
regionCacheSyncLock.Lock()
|
|
defer regionCacheSyncLock.Unlock()
|
|
|
|
storeRegionCache()
|
|
return region, nil
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
} else {
|
|
return newRegion.(*Region), nil
|
|
}
|
|
}
|
|
|
|
type ucRegionsRet struct {
|
|
Regions []ucRegionRet `json:"regions"`
|
|
}
|
|
|
|
type ucRegionRet struct {
|
|
Id string `json:"id"`
|
|
Up ucDomainsRet `json:"up"`
|
|
Io ucDomainsRet `json:"io"`
|
|
Uc ucDomainsRet `json:"uc"`
|
|
Rs ucDomainsRet `json:"rs"`
|
|
Rsf ucDomainsRet `json:"rsf"`
|
|
Api ucDomainsRet `json:"api"`
|
|
}
|
|
|
|
type ucDomainsRet struct {
|
|
Main []string `json:"domains"`
|
|
Backup []string `json:"old,omitempty"`
|
|
}
|
|
|
|
var (
|
|
regionIdCache = make(map[string]*Region)
|
|
regionIdCacheGroup singleflight.Group
|
|
)
|
|
|
|
// getRegionByRegionId 用来根据 ak 和 sk 来获取空间相关的机房信息,由于返回的 Region 结构体与先前不兼容,所以本接口暂不开放
|
|
func getRegionByRegionId(regionId string, credentials *auth.Credentials) (region *Region, err error) {
|
|
var cacheValue interface{}
|
|
|
|
cacheValue, err, _ = regionIdCacheGroup.Do("query", func() (interface{}, error) {
|
|
if v, ok := regionIdCache[regionId]; ok {
|
|
return v, nil
|
|
}
|
|
reqURL := fmt.Sprintf("%s/regions", getUcHostByDefaultProtocol())
|
|
var ret ucRegionsRet
|
|
ctx := context.TODO()
|
|
qErr := client.DefaultClient.CredentialedCallWithForm(ctx, credentials, auth.TokenQiniu, &ret, "GET", reqURL, nil, nil)
|
|
if qErr != nil {
|
|
return nil, fmt.Errorf("query region error, %s", qErr.Error())
|
|
}
|
|
for _, r := range ret.Regions {
|
|
upHosts := r.Up.Main
|
|
if len(r.Up.Backup) > 0 {
|
|
upHosts = append(upHosts, r.Up.Backup...)
|
|
}
|
|
if len(r.Io.Main) == 0 {
|
|
return nil, fmt.Errorf("empty io host list")
|
|
}
|
|
if len(r.Rs.Main) == 0 {
|
|
return nil, fmt.Errorf("empty rs host list")
|
|
}
|
|
if len(r.Rsf.Main) == 0 {
|
|
return nil, fmt.Errorf("empty rsf host list")
|
|
}
|
|
if len(r.Api.Main) == 0 {
|
|
return nil, fmt.Errorf("empty api host list")
|
|
}
|
|
|
|
region := &Region{
|
|
SrcUpHosts: upHosts,
|
|
CdnUpHosts: upHosts,
|
|
IovipHost: r.Io.Main[0],
|
|
RsHost: r.Rs.Main[0],
|
|
RsfHost: r.Rsf.Main[0],
|
|
ApiHost: r.Api.Main[0],
|
|
}
|
|
regionIdCache[r.Id] = region
|
|
}
|
|
if v, ok := regionIdCache[regionId]; ok {
|
|
return v, nil
|
|
} else {
|
|
return nil, fmt.Errorf("region id is not found")
|
|
}
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
region = cacheValue.(*Region)
|
|
return
|
|
}
|
|
|
|
func regionFromHost(ioHost string) (Region, bool) {
|
|
if strings.Contains(ioHost, "-z1") {
|
|
return GetRegionByID(RIDHuabei)
|
|
}
|
|
if strings.Contains(ioHost, "-z2") {
|
|
return GetRegionByID(RIDHuanan)
|
|
}
|
|
|
|
if strings.Contains(ioHost, "-na0") {
|
|
return GetRegionByID(RIDNorthAmerica)
|
|
}
|
|
if strings.Contains(ioHost, "-as0") {
|
|
return GetRegionByID(RIDSingapore)
|
|
}
|
|
return Region{}, false
|
|
}
|
|
|
|
func setSpecificHosts(ioHost string, region *Region) {
|
|
r, ok := regionFromHost(ioHost)
|
|
if ok {
|
|
region.RsHost = r.RsHost
|
|
region.RsfHost = r.RsfHost
|
|
region.ApiHost = r.ApiHost
|
|
}
|
|
}
|
|
|
|
type RegionInfo struct {
|
|
ID string `json:"id"`
|
|
Description string `json:"description"`
|
|
}
|
|
|
|
func GetRegionsInfo(mac *auth.Credentials) ([]RegionInfo, error) {
|
|
var regions struct {
|
|
Regions []RegionInfo `json:"regions"`
|
|
}
|
|
qErr := client.DefaultClient.CredentialedCallWithForm(context.Background(), mac, auth.TokenQiniu, ®ions, "GET", getUcHostByDefaultProtocol()+"/regions", nil, nil)
|
|
if qErr != nil {
|
|
return nil, fmt.Errorf("query region error, %s", qErr.Error())
|
|
} else {
|
|
return regions.Regions, nil
|
|
}
|
|
}
|