go-library/vendor/github.com/qiniu/go-sdk/v7/storage/region.go

package storage

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/qiniu/go-sdk/v7/auth"
	"github.com/qiniu/go-sdk/v7/client"
	"golang.org/x/sync/singleflight"
)

// 存储所在的地区，例如华东，华南，华北
// 每个存储区域可能有多个机房信息，每个机房可能有多个上传入口
type Region struct {
	// 上传入口
	SrcUpHosts []string `json:"src_up,omitempty"`

	// 加速上传入口
	CdnUpHosts []string `json:"cdn_up,omitempty"`

	// 获取文件信息入口
	RsHost string `json:"rs,omitempty"`

	// bucket列举入口
	RsfHost string `json:"rsf,omitempty"`

	ApiHost string `json:"api,omitempty"`

	// 存储io 入口
	IovipHost string `json:"io,omitempty"`
}

type RegionID string

// GetDefaultReion 根据RegionID获取对应的Region信息
func GetRegionByID(regionID RegionID) (Region, bool) {
	if r, ok := regionMap[regionID]; ok {
		return r, ok
	}
	return Region{}, false
}

func (r *Region) String() string {
	str := ""
	str += fmt.Sprintf("SrcUpHosts: %v\n", r.SrcUpHosts)
	str += fmt.Sprintf("CdnUpHosts: %v\n", r.CdnUpHosts)
	str += fmt.Sprintf("IovipHost: %s\n", r.IovipHost)
	str += fmt.Sprintf("RsHost: %s\n", r.RsHost)
	str += fmt.Sprintf("RsfHost: %s\n", r.RsfHost)
	str += fmt.Sprintf("ApiHost: %s\n", r.ApiHost)
	return str
}

func endpoint(useHttps bool, host string) string {
	host = strings.TrimSpace(host)
	host = strings.TrimLeft(host, "http://")
	host = strings.TrimLeft(host, "https://")
	if host == "" {
		return ""
	}
	scheme := "http://"
	if useHttps {
		scheme = "https://"
	}
	return fmt.Sprintf("%s%s", scheme, host)
}

// 获取rsfHost
func (r *Region) GetRsfHost(useHttps bool) string {
	return endpoint(useHttps, r.RsfHost)
}

// 获取io host
func (r *Region) GetIoHost(useHttps bool) string {
	return endpoint(useHttps, r.IovipHost)
}

// 获取RsHost
func (r *Region) GetRsHost(useHttps bool) string {
	return endpoint(useHttps, r.RsHost)
}

// 获取api host
func (r *Region) GetApiHost(useHttps bool) string {
	return endpoint(useHttps, r.ApiHost)
}

var (
	// regionHuadong 表示华东机房
	regionHuadong = Region{
		SrcUpHosts: []string{
			"up.qiniup.com",
			"up-nb.qiniup.com",
			"up-xs.qiniup.com",
		},
		CdnUpHosts: []string{
			"upload.qiniup.com",
			"upload-nb.qiniup.com",
			"upload-xs.qiniup.com",
		},
		RsHost:    "rs.qbox.me",
		RsfHost:   "rsf.qbox.me",
		ApiHost:   "api.qiniu.com",
		IovipHost: "iovip.qbox.me",
	}

	// regionHuadongZhejiang2 表示华东-浙江2
	regionHuadongZhejiang2 = Region{
		SrcUpHosts: []string{
			"up-cn-east-2.qiniup.com",
		},
		CdnUpHosts: []string{
			"upload-cn-east-2.qiniup.com",
		},
		RsHost:    "rs-cn-east-2.qiniuapi.com",
		RsfHost:   "rsf-cn-east-2-qiniuapi.com",
		ApiHost:   "api-cn-east-2.qiniuapi.com",
		IovipHost: "iovip-cn-east-2.qiniuio.com",
	}

	// regionHuabei 表示华北机房
	regionHuabei = Region{
		SrcUpHosts: []string{
			"up-z1.qiniup.com",
		},
		CdnUpHosts: []string{
			"upload-z1.qiniup.com",
		},
		RsHost:    "rs-z1.qbox.me",
		RsfHost:   "rsf-z1.qbox.me",
		ApiHost:   "api-z1.qiniuapi.com",
		IovipHost: "iovip-z1.qbox.me",
	}
	// regionHuanan 表示华南机房
	regionHuanan = Region{
		SrcUpHosts: []string{
			"up-z2.qiniup.com",
			"up-gz.qiniup.com",
			"up-fs.qiniup.com",
		},
		CdnUpHosts: []string{
			"upload-z2.qiniup.com",
			"upload-gz.qiniup.com",
			"upload-fs.qiniup.com",
		},
		RsHost:    "rs-z2.qbox.me",
		RsfHost:   "rsf-z2.qbox.me",
		ApiHost:   "api-z2.qiniuapi.com",
		IovipHost: "iovip-z2.qbox.me",
	}

	// regionNorthAmerica 表示北美机房
	regionNorthAmerica = Region{
		SrcUpHosts: []string{
			"up-na0.qiniup.com",
		},
		CdnUpHosts: []string{
			"upload-na0.qiniup.com",
		},
		RsHost:    "rs-na0.qbox.me",
		RsfHost:   "rsf-na0.qbox.me",
		ApiHost:   "api-na0.qiniuapi.com",
		IovipHost: "iovip-na0.qbox.me",
	}
	// regionSingapore 表示新加坡机房
	regionSingapore = Region{
		SrcUpHosts: []string{
			"up-as0.qiniup.com",
		},
		CdnUpHosts: []string{
			"upload-as0.qiniup.com",
		},
		RsHost:    "rs-as0.qbox.me",
		RsfHost:   "rsf-as0.qbox.me",
		ApiHost:   "api-as0.qiniuapi.com",
		IovipHost: "iovip-as0.qbox.me",
	}
	// regionFogCnEast1 表示雾存储华东区
	regionFogCnEast1 = Region{
		SrcUpHosts: []string{
			"up-fog-cn-east-1.qiniup.com",
		},
		CdnUpHosts: []string{
			"upload-fog-cn-east-1.qiniup.com",
		},
		RsHost:    "rs-fog-cn-east-1.qbox.me",
		RsfHost:   "rsf-fog-cn-east-1.qbox.me",
		ApiHost:   "api-fog-cn-east-1.qiniuapi.com",
		IovipHost: "iovip-fog-cn-east-1.qbox.me",
	}
)

const (
	// region code
	RIDHuadong          = RegionID("z0")
	RIDHuadongZheJiang2 = RegionID("cn-east-2")
	RIDHuabei           = RegionID("z1")
	RIDHuanan           = RegionID("z2")
	RIDNorthAmerica     = RegionID("na0")
	RIDSingapore        = RegionID("as0")
	RIDFogCnEast1       = RegionID("fog-cn-east-1")
)

// regionMap 是RegionID到具体的Region的映射
var regionMap = map[RegionID]Region{
	RIDHuadong:          regionHuadong,
	RIDHuadongZheJiang2: regionHuadongZhejiang2,
	RIDHuanan:           regionHuanan,
	RIDHuabei:           regionHuabei,
	RIDSingapore:        regionSingapore,
	RIDNorthAmerica:     regionNorthAmerica,
	RIDFogCnEast1:       regionFogCnEast1,
}

/// UcHost 为查询空间相关域名的API服务地址
/// 设置 UcHost 时，如果不指定 scheme 默认会使用 https
/// UcHost 已废弃，建议使用 SetUcHost
//Deprecated
var UcHost = "https://uc.qbox.me"

var ucHost = ""

func SetUcHost(host string, useHttps bool) {
	ucHost = endpoint(useHttps, host)
}

func getUcHostByDefaultProtocol() string {
	return getUcHost(true)
}

func getUcHost(useHttps bool) string {
	if ucHost != "" {
		return ucHost
	}

	if strings.Contains(UcHost, "://") {
		return UcHost
	} else {
		return endpoint(useHttps, UcHost)
	}
}

// UcQueryRet 为查询请求的回复
type UcQueryRet struct {
	TTL    int `json:"ttl"`
	Io     map[string]map[string][]string
	IoInfo map[string]UcQueryIo `json:"io"`
	Up     map[string]UcQueryUp `json:"up"`
}

func (uc *UcQueryRet) UnmarshalJSON(data []byte) error {
	t := struct {
		TTL    int                  `json:"ttl"`
		IoInfo map[string]UcQueryIo `json:"io"`
		Up     map[string]UcQueryUp `json:"up"`
	}{}
	if err := json.Unmarshal(data, &t); err != nil {
		return err
	}

	uc.TTL = t.TTL
	uc.IoInfo = t.IoInfo
	uc.Up = t.Up
	uc.setup()
	return nil
}

func (uc *UcQueryRet) setup() {
	if uc.Io != nil || uc.IoInfo == nil {
		return
	}

	uc.Io = make(map[string]map[string][]string)
	ioSrc := uc.IoInfo["src"].toMapWithoutInfo()
	if ioSrc != nil && len(ioSrc) > 0 {
		uc.Io["src"] = ioSrc
	}

	ioOldSrc := uc.IoInfo["old_src"].toMapWithoutInfo()
	if ioOldSrc != nil && len(ioOldSrc) > 0 {
		uc.Io["old_src"] = ioOldSrc
	}
}

// UcQueryUp 为查询请求回复中的上传域名信息
type UcQueryUp struct {
	Main   []string `json:"main,omitempty"`
	Backup []string `json:"backup,omitempty"`
	Info   string   `json:"info,omitempty"`
}

// UcQueryIo 为查询请求回复中的上传域名信息
type UcQueryIo struct {
	Main   []string `json:"main,omitempty"`
	Backup []string `json:"backup,omitempty"`
	Info   string   `json:"info,omitempty"`
}

func (io UcQueryIo) toMapWithoutInfo() map[string][]string {

	ret := make(map[string][]string)
	if io.Main != nil && len(io.Main) > 0 {
		ret["main"] = io.Main
	}

	if io.Backup != nil && len(io.Backup) > 0 {
		ret["backup"] = io.Backup
	}

	return ret
}

type regionCacheValue struct {
	Region   *Region   `json:"region"`
	Deadline time.Time `json:"deadline"`
}

type regionCacheMap map[string]regionCacheValue

var (
	regionCachePath     = filepath.Join(os.TempDir(), "qiniu-golang-sdk", "query.cache.json")
	regionCache         sync.Map
	regionCacheLock     sync.RWMutex
	regionCacheSyncLock sync.Mutex
	regionCacheGroup    singleflight.Group
	regionCacheLoaded   bool = false
)

func SetRegionCachePath(newPath string) {
	regionCacheLock.Lock()
	defer regionCacheLock.Unlock()

	regionCachePath = newPath
	regionCacheLoaded = false
}

func loadRegionCache() {
	cacheFile, err := os.Open(regionCachePath)
	if err != nil {
		return
	}
	defer cacheFile.Close()

	var cacheMap regionCacheMap
	if err = json.NewDecoder(cacheFile).Decode(&cacheMap); err != nil {
		return
	}
	for cacheKey, cacheValue := range cacheMap {
		regionCache.Store(cacheKey, cacheValue)
	}
}

func storeRegionCache() {
	err := os.MkdirAll(filepath.Dir(regionCachePath), 0700)
	if err != nil {
		return
	}

	cacheFile, err := os.OpenFile(regionCachePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
	if err != nil {
		return
	}
	defer cacheFile.Close()

	cacheMap := make(regionCacheMap)
	regionCache.Range(func(cacheKey, cacheValue interface{}) bool {
		cacheMap[cacheKey.(string)] = cacheValue.(regionCacheValue)
		return true
	})
	if err = json.NewEncoder(cacheFile).Encode(cacheMap); err != nil {
		return
	}
}

// GetRegion 用来根据ak和bucket来获取空间相关的机房信息
func GetRegion(ak, bucket string) (*Region, error) {
	regionCacheLock.RLock()
	if regionCacheLoaded {
		regionCacheLock.RUnlock()
	} else {
		regionCacheLock.RUnlock()
		func() {
			regionCacheLock.Lock()
			defer regionCacheLock.Unlock()

			if !regionCacheLoaded {
				loadRegionCache()
				regionCacheLoaded = true
			}
		}()
	}

	regionID := fmt.Sprintf("%s:%s", ak, bucket)
	//check from cache
	if v, ok := regionCache.Load(regionID); ok && time.Now().Before(v.(regionCacheValue).Deadline) {
		return v.(regionCacheValue).Region, nil
	}

	newRegion, err, _ := regionCacheGroup.Do(regionID, func() (interface{}, error) {
		reqURL := fmt.Sprintf("%s/v2/query?ak=%s&bucket=%s", getUcHostByDefaultProtocol(), ak, bucket)

		var ret UcQueryRet
		err := client.DefaultClient.CallWithForm(context.Background(), &ret, "GET", reqURL, nil, nil)
		if err != nil {
			return nil, fmt.Errorf("query region error, %s", err.Error())
		}

		if len(ret.Io["src"]["main"]) <= 0 {
			return nil, fmt.Errorf("empty io host list")
		}

		ioHost := ret.Io["src"]["main"][0]
		srcUpHosts := ret.Up["src"].Main
		if ret.Up["src"].Backup != nil {
			srcUpHosts = append(srcUpHosts, ret.Up["src"].Backup...)
		}
		cdnUpHosts := ret.Up["acc"].Main
		if ret.Up["acc"].Backup != nil {
			cdnUpHosts = append(cdnUpHosts, ret.Up["acc"].Backup...)
		}

		region := &Region{
			SrcUpHosts: srcUpHosts,
			CdnUpHosts: cdnUpHosts,
			IovipHost:  ioHost,
			RsHost:     DefaultRsHost,
			RsfHost:    DefaultRsfHost,
			ApiHost:    DefaultAPIHost,
		}

		//set specific hosts if possible
		setSpecificHosts(ioHost, region)
		regionCache.Store(regionID, regionCacheValue{
			Region:   region,
			Deadline: time.Now().Add(time.Duration(ret.TTL) * time.Second),
		})

		regionCacheSyncLock.Lock()
		defer regionCacheSyncLock.Unlock()

		storeRegionCache()
		return region, nil
	})

	if err != nil {
		return nil, err
	} else {
		return newRegion.(*Region), nil
	}
}

type ucRegionsRet struct {
	Regions []ucRegionRet `json:"regions"`
}

type ucRegionRet struct {
	Id  string       `json:"id"`
	Up  ucDomainsRet `json:"up"`
	Io  ucDomainsRet `json:"io"`
	Uc  ucDomainsRet `json:"uc"`
	Rs  ucDomainsRet `json:"rs"`
	Rsf ucDomainsRet `json:"rsf"`
	Api ucDomainsRet `json:"api"`
}

type ucDomainsRet struct {
	Main   []string `json:"domains"`
	Backup []string `json:"old,omitempty"`
}

var (
	regionIdCache      = make(map[string]*Region)
	regionIdCacheGroup singleflight.Group
)

// getRegionByRegionId 用来根据 ak 和 sk 来获取空间相关的机房信息，由于返回的 Region 结构体与先前不兼容，所以本接口暂不开放
func getRegionByRegionId(regionId string, credentials *auth.Credentials) (region *Region, err error) {
	var cacheValue interface{}

	cacheValue, err, _ = regionIdCacheGroup.Do("query", func() (interface{}, error) {
		if v, ok := regionIdCache[regionId]; ok {
			return v, nil
		}
		reqURL := fmt.Sprintf("%s/regions", getUcHostByDefaultProtocol())
		var ret ucRegionsRet
		ctx := context.TODO()
		qErr := client.DefaultClient.CredentialedCallWithForm(ctx, credentials, auth.TokenQiniu, &ret, "GET", reqURL, nil, nil)
		if qErr != nil {
			return nil, fmt.Errorf("query region error, %s", qErr.Error())
		}
		for _, r := range ret.Regions {
			upHosts := r.Up.Main
			if len(r.Up.Backup) > 0 {
				upHosts = append(upHosts, r.Up.Backup...)
			}
			if len(r.Io.Main) == 0 {
				return nil, fmt.Errorf("empty io host list")
			}
			if len(r.Rs.Main) == 0 {
				return nil, fmt.Errorf("empty rs host list")
			}
			if len(r.Rsf.Main) == 0 {
				return nil, fmt.Errorf("empty rsf host list")
			}
			if len(r.Api.Main) == 0 {
				return nil, fmt.Errorf("empty api host list")
			}

			region := &Region{
				SrcUpHosts: upHosts,
				CdnUpHosts: upHosts,
				IovipHost:  r.Io.Main[0],
				RsHost:     r.Rs.Main[0],
				RsfHost:    r.Rsf.Main[0],
				ApiHost:    r.Api.Main[0],
			}
			regionIdCache[r.Id] = region
		}
		if v, ok := regionIdCache[regionId]; ok {
			return v, nil
		} else {
			return nil, fmt.Errorf("region id is not found")
		}
	})
	if err != nil {
		return nil, err
	}
	region = cacheValue.(*Region)
	return
}

func regionFromHost(ioHost string) (Region, bool) {
	if strings.Contains(ioHost, "-z1") {
		return GetRegionByID(RIDHuabei)
	}
	if strings.Contains(ioHost, "-z2") {
		return GetRegionByID(RIDHuanan)
	}

	if strings.Contains(ioHost, "-na0") {
		return GetRegionByID(RIDNorthAmerica)
	}
	if strings.Contains(ioHost, "-as0") {
		return GetRegionByID(RIDSingapore)
	}
	return Region{}, false
}

func setSpecificHosts(ioHost string, region *Region) {
	r, ok := regionFromHost(ioHost)
	if ok {
		region.RsHost = r.RsHost
		region.RsfHost = r.RsfHost
		region.ApiHost = r.ApiHost
	}
}

type RegionInfo struct {
	ID          string `json:"id"`
	Description string `json:"description"`
}

func GetRegionsInfo(mac *auth.Credentials) ([]RegionInfo, error) {
	var regions struct {
		Regions []RegionInfo `json:"regions"`
	}
	qErr := client.DefaultClient.CredentialedCallWithForm(context.Background(), mac, auth.TokenQiniu, &regions, "GET", getUcHostByDefaultProtocol()+"/regions", nil, nil)
	if qErr != nil {
		return nil, fmt.Errorf("query region error, %s", qErr.Error())
	} else {
		return regions.Regions, nil
	}
}