- update vendor

master
李光春 1 year ago
parent cac0b01f63
commit b69cc2ec64

@ -5,10 +5,10 @@ go 1.20
require (
github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible
github.com/allegro/bigcache/v3 v3.1.0
github.com/aws/aws-sdk-go v1.44.167
github.com/aws/aws-sdk-go v1.44.168
github.com/aws/aws-sdk-go-v2/config v1.18.7
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6
github.com/baidubce/bce-sdk-go v0.9.138
github.com/baidubce/bce-sdk-go v0.9.139
github.com/basgys/goxml2json v1.1.0
github.com/docker/docker v20.10.22+incompatible
github.com/drone/drone-go v1.7.1
@ -106,7 +106,7 @@ require (
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.1 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgtype v1.13.0 // indirect
github.com/jackc/pgx/v4 v4.17.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
@ -143,7 +143,7 @@ require (
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/ugorji/go/codec v1.2.7 // indirect
github.com/ugorji/go/codec v1.2.8 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect

@ -36,8 +36,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
github.com/aws/aws-sdk-go v1.44.167 h1:kQmBhGdZkQLU7AiHShSkBJ15zr8agy0QeaxXduvyp2E=
github.com/aws/aws-sdk-go v1.44.167/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.168 h1:/NNDLkjcgW8UrvAUk7QvQS9yzo/CFu9Zp4BCiPHoV+E=
github.com/aws/aws-sdk-go v1.44.168/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY=
github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
@ -75,8 +75,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N8
github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/baidubce/bce-sdk-go v0.9.138 h1:/1P4MT2QQtR6dG1n3SaQYfmzWWdI871mEL0458lYODo=
github.com/baidubce/bce-sdk-go v0.9.138/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
github.com/baidubce/bce-sdk-go v0.9.139 h1:AX+PeQgRAIx5RZ/SUBVUsiJT7hQkX728Xt9JVxHzBV0=
github.com/baidubce/bce-sdk-go v0.9.139/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw=
github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@ -344,8 +344,9 @@ github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX
github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y=
github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
@ -664,9 +665,8 @@ github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/ugorji/go/codec v1.2.8 h1:sgBJS6COt0b/P40VouWKdseidkDgHxYGm0SAglUHfP0=
github.com/ugorji/go/codec v1.2.8/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/upper/db/v4 v4.6.0 h1:0VmASnqrl/XN8Ehoq++HBgZ4zRD5j3GXygW8FhP0C5I=

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.44.167"
const SDKVersion = "1.44.168"

@ -89,6 +89,11 @@ func (c *BceClient) buildHttpRequest(request *BceRequest) {
request.SetHeader(http.USER_AGENT, c.Config.UserAgent)
request.SetHeader(http.BCE_DATE, util.FormatISO8601Date(util.NowUTCSeconds()))
//set default content-type if null
if request.Header(http.CONTENT_TYPE) == "" {
request.SetHeader(http.CONTENT_TYPE, DEFAULT_CONTENT_TYPE)
}
// Generate the auth string if needed
if c.Config.Credentials != nil {
c.Signer.Sign(&request.Request, c.Config.Credentials, c.Config.SignOption)

@ -26,7 +26,7 @@ import (
// Constants and default values for the package bce
const (
SDK_VERSION = "0.9.138"
SDK_VERSION = "0.9.139"
URI_PREFIX = "/" // now support uri without prefix "v1" so just set root path
DEFAULT_DOMAIN = "baidubce.com"
DEFAULT_PROTOCOL = "http"

@ -108,6 +108,10 @@ func PutObject(cli bce.Client, bucket, object string, body *bce.Body,
req.SetHeader(http.BCE_PROCESS, args.Process)
}
}
// add content-type if not assigned by user
if req.Header(http.CONTENT_TYPE) == "" {
req.SetHeader(http.CONTENT_TYPE, getDefaultContentType(object))
}
resp := &bce.BceResponse{}
if err := SendRequest(cli, req, resp); err != nil {

@ -23,6 +23,7 @@ import (
"github.com/baidubce/bce-sdk-go/bce"
"github.com/baidubce/bce-sdk-go/http"
"github.com/baidubce/bce-sdk-go/util"
)
const (
@ -45,8 +46,8 @@ const (
THRESHOLD_100_CONTINUE = 1 << 20 // add 100 continue header if body size bigger than 1MB
TRAFFIC_LIMIT_MAX = 8 * (100 << 20) // 100M bit = 838860800
TRAFFIC_LIMIT_MIN = 8 * (100 << 10) // 100K bit = 819200
TRAFFIC_LIMIT_MAX = 8 * (100 << 20) // 100M bit = 838860800
TRAFFIC_LIMIT_MIN = 8 * (100 << 10) // 100K bit = 819200
STATUS_ENABLED = "enabled"
STATUS_DISABLED = "disabled"
@ -275,3 +276,17 @@ func SendRequest(cli bce.Client, req *bce.BceRequest, resp *bce.BceResponse) err
}
return err
}
func getDefaultContentType(object string) string {
dot := strings.LastIndex(object, ".")
if dot == -1 {
return "application/octet-stream"
}
ext := object[dot:]
mimeMap := util.GetMimeMap()
if contentType, ok := mimeMap[ext]; ok {
return contentType
}
return "application/octet-stream"
}

@ -0,0 +1,664 @@
package util
import (
"sync"
)
var DefaultMimeMap map[string]string
var once sync.Once
func LoadMimeMap() {
DefaultMimeMap = make(map[string]string)
DefaultMimeMap[".ez"] = "application/andrew-inset"
DefaultMimeMap[".atom"] = "application/atom+xml"
DefaultMimeMap[".atomcat"] = "application/atomcat+xml"
DefaultMimeMap[".atomsvc"] = "application/atomsvc+xml"
DefaultMimeMap[".ccxml"] = "application/ccxml+xml"
DefaultMimeMap[".davmount"] = "application/davmount+xml"
DefaultMimeMap[".ecma"] = "application/ecmascript"
DefaultMimeMap[".pfr"] = "application/font-tdpfr"
DefaultMimeMap[".stk"] = "application/hyperstudio"
DefaultMimeMap[".js"] = "text/javascript"
DefaultMimeMap[".json"] = "text/json"
DefaultMimeMap[".hqx"] = "application/mac-binhex40"
DefaultMimeMap[".cpt"] = "application/mac-compactpro"
DefaultMimeMap[".mrc"] = "application/marc"
DefaultMimeMap[".ma"] = "application/mathematica"
DefaultMimeMap[".nb"] = "application/mathematica"
DefaultMimeMap[".mb"] = "application/mathematica"
DefaultMimeMap[".mathml"] = "application/mathml+xml"
DefaultMimeMap[".mbox"] = "application/mbox"
DefaultMimeMap[".mscml"] = "application/mediaservercontrol+xml"
DefaultMimeMap[".mp4s"] = "application/mp4"
DefaultMimeMap[".doc"] = "application/msword"
DefaultMimeMap[".dot"] = "application/msword"
DefaultMimeMap[".mxf"] = "application/mxf"
DefaultMimeMap[".bin"] = "application/octet-stream"
DefaultMimeMap[".dms"] = "application/octet-stream"
DefaultMimeMap[".lha"] = "application/octet-stream"
DefaultMimeMap[".lzh"] = "application/octet-stream"
DefaultMimeMap[".class"] = "application/octet-stream"
DefaultMimeMap[".so"] = "application/octet-stream"
DefaultMimeMap[".iso"] = "application/octet-stream"
DefaultMimeMap[".dmg"] = "application/octet-stream"
DefaultMimeMap[".dist"] = "application/octet-stream"
DefaultMimeMap[".distz"] = "application/octet-stream"
DefaultMimeMap[".pkg"] = "application/octet-stream"
DefaultMimeMap[".bpk"] = "application/octet-stream"
DefaultMimeMap[".dump"] = "application/octet-stream"
DefaultMimeMap[".elc"] = "application/octet-stream"
DefaultMimeMap[".oda"] = "application/oda"
DefaultMimeMap[".ogg"] = "application/ogg"
DefaultMimeMap[".pdf"] = "application/pdf"
DefaultMimeMap[".pgp"] = "application/pgp-encrypted"
DefaultMimeMap[".asc"] = "application/pgp-signature"
DefaultMimeMap[".sig"] = "application/pgp-signature"
DefaultMimeMap[".prf"] = "application/pics-rules"
DefaultMimeMap[".p10"] = "application/pkcs10"
DefaultMimeMap[".p7m"] = "application/pkcs7-mime"
DefaultMimeMap[".p7c"] = "application/pkcs7-mime"
DefaultMimeMap[".p7s"] = "application/pkcs7-signature"
DefaultMimeMap[".cer"] = "application/pkix-cert"
DefaultMimeMap[".crl"] = "application/pkix-crl"
DefaultMimeMap[".pkipath"] = "application/pkix-pkipath"
DefaultMimeMap[".pki"] = "application/pkixcmp"
DefaultMimeMap[".pls"] = "application/pls+xml"
DefaultMimeMap[".ai"] = "application/postscript"
DefaultMimeMap[".eps"] = "application/postscript"
DefaultMimeMap[".ps"] = "application/postscript"
DefaultMimeMap[".cww"] = "application/prs.cww"
DefaultMimeMap[".rdf"] = "application/rdf+xml"
DefaultMimeMap[".rif"] = "application/reginfo+xml"
DefaultMimeMap[".rnc"] = "application/relax-ng-compact-syntax"
DefaultMimeMap[".rl"] = "application/resource-lists+xml"
DefaultMimeMap[".rs"] = "application/rls-services+xml"
DefaultMimeMap[".rsd"] = "application/rsd+xml"
DefaultMimeMap[".rss"] = "application/rss+xml"
DefaultMimeMap[".rtf"] = "application/rtf"
DefaultMimeMap[".sbml"] = "application/sbml+xml"
DefaultMimeMap[".scq"] = "application/scvp-cv-request"
DefaultMimeMap[".scs"] = "application/scvp-cv-response"
DefaultMimeMap[".spq"] = "application/scvp-vp-request"
DefaultMimeMap[".spp"] = "application/scvp-vp-response"
DefaultMimeMap[".sdp"] = "application/sdp"
DefaultMimeMap[".setpay"] = "application/set-payment-initiation"
DefaultMimeMap[".setreg"] = "application/set-registration-initiation"
DefaultMimeMap[".shf"] = "application/shf+xml"
DefaultMimeMap[".smi"] = "application/smil+xml"
DefaultMimeMap[".smil"] = "application/smil+xml"
DefaultMimeMap[".rq"] = "application/sparql-query"
DefaultMimeMap[".srx"] = "application/sparql-results+xml"
DefaultMimeMap[".gram"] = "application/srgs"
DefaultMimeMap[".grxml"] = "application/srgs+xml"
DefaultMimeMap[".ssml"] = "application/ssml+xml"
DefaultMimeMap[".plb"] = "application/vnd.3gpp.pic-bw-large"
DefaultMimeMap[".psb"] = "application/vnd.3gpp.pic-bw-small"
DefaultMimeMap[".pvb"] = "application/vnd.3gpp.pic-bw-var"
DefaultMimeMap[".tcap"] = "application/vnd.3gpp2.tcap"
DefaultMimeMap[".pwn"] = "application/vnd.3m.post-it-notes"
DefaultMimeMap[".aso"] = "application/vnd.accpac.simply.aso"
DefaultMimeMap[".imp"] = "application/vnd.accpac.simply.imp"
DefaultMimeMap[".acu"] = "application/vnd.acucobol"
DefaultMimeMap[".atc"] = "application/vnd.acucorp"
DefaultMimeMap[".acutc"] = "application/vnd.acucorp"
DefaultMimeMap[".xdp"] = "application/vnd.adobe.xdp+xml"
DefaultMimeMap[".xfdf"] = "application/vnd.adobe.xfdf"
DefaultMimeMap[".ami"] = "application/vnd.amiga.ami"
DefaultMimeMap[".cii"] = "application/vnd.anser-web-certificate-issue-initiation"
DefaultMimeMap[".fti"] = "application/vnd.anser-web-funds-transfer-initiation"
DefaultMimeMap[".atx"] = "application/vnd.antix.game-component"
DefaultMimeMap[".mpkg"] = "application/vnd.apple.installer+xml"
DefaultMimeMap[".aep"] = "application/vnd.audiograph"
DefaultMimeMap[".mpm"] = "application/vnd.blueice.multipass"
DefaultMimeMap[".bmi"] = "application/vnd.bmi"
DefaultMimeMap[".rep"] = "application/vnd.businessobjects"
DefaultMimeMap[".cdxml"] = "application/vnd.chemdraw+xml"
DefaultMimeMap[".mmd"] = "application/vnd.chipnuts.karaoke-mmd"
DefaultMimeMap[".cdy"] = "application/vnd.cinderella"
DefaultMimeMap[".cla"] = "application/vnd.claymore"
DefaultMimeMap[".c4g"] = "application/vnd.clonk.c4group"
DefaultMimeMap[".c4d"] = "application/vnd.clonk.c4group"
DefaultMimeMap[".c4f"] = "application/vnd.clonk.c4group"
DefaultMimeMap[".c4p"] = "application/vnd.clonk.c4group"
DefaultMimeMap[".c4u"] = "application/vnd.clonk.c4group"
DefaultMimeMap[".csp"] = "application/vnd.commonspace"
DefaultMimeMap[".cst"] = "application/vnd.commonspace"
DefaultMimeMap[".cdbcmsg"] = "application/vnd.contact.cmsg"
DefaultMimeMap[".cmc"] = "application/vnd.cosmocaller"
DefaultMimeMap[".clkx"] = "application/vnd.crick.clicker"
DefaultMimeMap[".clkk"] = "application/vnd.crick.clicker.keyboard"
DefaultMimeMap[".clkp"] = "application/vnd.crick.clicker.palette"
DefaultMimeMap[".clkt"] = "application/vnd.crick.clicker.template"
DefaultMimeMap[".clkw"] = "application/vnd.crick.clicker.wordbank"
DefaultMimeMap[".wbs"] = "application/vnd.criticaltools.wbs+xml"
DefaultMimeMap[".pml"] = "application/vnd.ctc-posml"
DefaultMimeMap[".ppd"] = "application/vnd.cups-ppd"
DefaultMimeMap[".curl"] = "application/vnd.curl"
DefaultMimeMap[".rdz"] = "application/vnd.data-vision.rdz"
DefaultMimeMap[".fe_launch"] = "application/vnd.denovo.fcselayout-link"
DefaultMimeMap[".dna"] = "application/vnd.dna"
DefaultMimeMap[".mlp"] = "application/vnd.dolby.mlp"
DefaultMimeMap[".dpg"] = "application/vnd.dpgraph"
DefaultMimeMap[".dfac"] = "application/vnd.dreamfactory"
DefaultMimeMap[".mag"] = "application/vnd.ecowin.chart"
DefaultMimeMap[".nml"] = "application/vnd.enliven"
DefaultMimeMap[".esf"] = "application/vnd.epson.esf"
DefaultMimeMap[".msf"] = "application/vnd.epson.msf"
DefaultMimeMap[".qam"] = "application/vnd.epson.quickanime"
DefaultMimeMap[".slt"] = "application/vnd.epson.salt"
DefaultMimeMap[".ssf"] = "application/vnd.epson.ssf"
DefaultMimeMap[".es3"] = "application/vnd.eszigno3+xml"
DefaultMimeMap[".et3"] = "application/vnd.eszigno3+xml"
DefaultMimeMap[".ez2"] = "application/vnd.ezpix-album"
DefaultMimeMap[".ez3"] = "application/vnd.ezpix-package"
DefaultMimeMap[".fdf"] = "application/vnd.fdf"
DefaultMimeMap[".gph"] = "application/vnd.flographit"
DefaultMimeMap[".ftc"] = "application/vnd.fluxtime.clip"
DefaultMimeMap[".fm"] = "application/vnd.framemaker"
DefaultMimeMap[".frame"] = "application/vnd.framemaker"
DefaultMimeMap[".maker"] = "application/vnd.framemaker"
DefaultMimeMap[".fnc"] = "application/vnd.frogans.fnc"
DefaultMimeMap[".ltf"] = "application/vnd.frogans.ltf"
DefaultMimeMap[".fsc"] = "application/vnd.fsc.weblaunch"
DefaultMimeMap[".oas"] = "application/vnd.fujitsu.oasys"
DefaultMimeMap[".oa2"] = "application/vnd.fujitsu.oasys2"
DefaultMimeMap[".oa3"] = "application/vnd.fujitsu.oasys3"
DefaultMimeMap[".fg5"] = "application/vnd.fujitsu.oasysgp"
DefaultMimeMap[".bh2"] = "application/vnd.fujitsu.oasysprs"
DefaultMimeMap[".ddd"] = "application/vnd.fujixerox.ddd"
DefaultMimeMap[".xdw"] = "application/vnd.fujixerox.docuworks"
DefaultMimeMap[".xbd"] = "application/vnd.fujixerox.docuworks.binder"
DefaultMimeMap[".fzs"] = "application/vnd.fuzzysheet"
DefaultMimeMap[".txd"] = "application/vnd.genomatix.tuxedo"
DefaultMimeMap[".kml"] = "application/vnd.google-earth.kml+xml"
DefaultMimeMap[".kmz"] = "application/vnd.google-earth.kmz"
DefaultMimeMap[".gqf"] = "application/vnd.grafeq"
DefaultMimeMap[".gqs"] = "application/vnd.grafeq"
DefaultMimeMap[".gac"] = "application/vnd.groove-account"
DefaultMimeMap[".ghf"] = "application/vnd.groove-help"
DefaultMimeMap[".gim"] = "application/vnd.groove-identity-message"
DefaultMimeMap[".grv"] = "application/vnd.groove-injector"
DefaultMimeMap[".gtm"] = "application/vnd.groove-tool-message"
DefaultMimeMap[".tpl"] = "application/vnd.groove-tool-template"
DefaultMimeMap[".vcg"] = "application/vnd.groove-vcard"
DefaultMimeMap[".zmm"] = "application/vnd.handheld-entertainment+xml"
DefaultMimeMap[".hbci"] = "application/vnd.hbci"
DefaultMimeMap[".les"] = "application/vnd.hhe.lesson-player"
DefaultMimeMap[".hpgl"] = "application/vnd.hp-hpgl"
DefaultMimeMap[".hpid"] = "application/vnd.hp-hpid"
DefaultMimeMap[".hps"] = "application/vnd.hp-hps"
DefaultMimeMap[".jlt"] = "application/vnd.hp-jlyt"
DefaultMimeMap[".pcl"] = "application/vnd.hp-pcl"
DefaultMimeMap[".pclxl"] = "application/vnd.hp-pclxl"
DefaultMimeMap[".x3d"] = "application/vnd.hzn-3d-crossword"
DefaultMimeMap[".mpy"] = "application/vnd.ibm.minipay"
DefaultMimeMap[".afp"] = "application/vnd.ibm.modcap"
DefaultMimeMap[".listafp"] = "application/vnd.ibm.modcap"
DefaultMimeMap[".list3820"] = "application/vnd.ibm.modcap"
DefaultMimeMap[".irm"] = "application/vnd.ibm.rights-management"
DefaultMimeMap[".sc"] = "application/vnd.ibm.secure-container"
DefaultMimeMap[".igl"] = "application/vnd.igloader"
DefaultMimeMap[".ivp"] = "application/vnd.immervision-ivp"
DefaultMimeMap[".ivu"] = "application/vnd.immervision-ivu"
DefaultMimeMap[".xpw"] = "application/vnd.intercon.formnet"
DefaultMimeMap[".xpx"] = "application/vnd.intercon.formnet"
DefaultMimeMap[".qbo"] = "application/vnd.intu.qbo"
DefaultMimeMap[".qfx"] = "application/vnd.intu.qfx"
DefaultMimeMap[".rcprofile"] = "application/vnd.ipunplugged.rcprofile"
DefaultMimeMap[".irp"] = "application/vnd.irepository.package+xml"
DefaultMimeMap[".xpr"] = "application/vnd.is-xpr"
DefaultMimeMap[".jam"] = "application/vnd.jam"
DefaultMimeMap[".rms"] = "application/vnd.jcp.javame.midlet-rms"
DefaultMimeMap[".jisp"] = "application/vnd.jisp"
DefaultMimeMap[".joda"] = "application/vnd.joost.joda-archive"
DefaultMimeMap[".ktz"] = "application/vnd.kahootz"
DefaultMimeMap[".ktr"] = "application/vnd.kahootz"
DefaultMimeMap[".karbon"] = "application/vnd.kde.karbon"
DefaultMimeMap[".chrt"] = "application/vnd.kde.kchart"
DefaultMimeMap[".kfo"] = "application/vnd.kde.kformula"
DefaultMimeMap[".flw"] = "application/vnd.kde.kivio"
DefaultMimeMap[".kon"] = "application/vnd.kde.kontour"
DefaultMimeMap[".kpr"] = "application/vnd.kde.kpresenter"
DefaultMimeMap[".kpt"] = "application/vnd.kde.kpresenter"
DefaultMimeMap[".ksp"] = "application/vnd.kde.kspread"
DefaultMimeMap[".kwd"] = "application/vnd.kde.kword"
DefaultMimeMap[".kwt"] = "application/vnd.kde.kword"
DefaultMimeMap[".htke"] = "application/vnd.kenameaapp"
DefaultMimeMap[".kia"] = "application/vnd.kidspiration"
DefaultMimeMap[".kne"] = "application/vnd.kinar"
DefaultMimeMap[".knp"] = "application/vnd.kinar"
DefaultMimeMap[".skp"] = "application/vnd.koan"
DefaultMimeMap[".skd"] = "application/vnd.koan"
DefaultMimeMap[".skt"] = "application/vnd.koan"
DefaultMimeMap[".skm"] = "application/vnd.koan"
DefaultMimeMap[".lbd"] = "application/vnd.llamagraphics.life-balance.desktop"
DefaultMimeMap[".lbe"] = "application/vnd.llamagraphics.life-balance.exchange+xml"
DefaultMimeMap[".123"] = "application/vnd.lotus-1-2-3"
DefaultMimeMap[".apr"] = "application/vnd.lotus-approach"
DefaultMimeMap[".pre"] = "application/vnd.lotus-freelance"
DefaultMimeMap[".nsf"] = "application/vnd.lotus-notes"
DefaultMimeMap[".org"] = "application/vnd.lotus-organizer"
DefaultMimeMap[".scm"] = "application/vnd.lotus-screencam"
DefaultMimeMap[".lwp"] = "application/vnd.lotus-wordpro"
DefaultMimeMap[".portpkg"] = "application/vnd.macports.portpkg"
DefaultMimeMap[".mcd"] = "application/vnd.mcd"
DefaultMimeMap[".mc1"] = "application/vnd.medcalcdata"
DefaultMimeMap[".cdkey"] = "application/vnd.mediastation.cdkey"
DefaultMimeMap[".mwf"] = "application/vnd.mfer"
DefaultMimeMap[".mfm"] = "application/vnd.mfmp"
DefaultMimeMap[".flo"] = "application/vnd.micrografx.flo"
DefaultMimeMap[".igx"] = "application/vnd.micrografx.igx"
DefaultMimeMap[".mif"] = "application/vnd.mif"
DefaultMimeMap[".daf"] = "application/vnd.mobius.daf"
DefaultMimeMap[".dis"] = "application/vnd.mobius.dis"
DefaultMimeMap[".mbk"] = "application/vnd.mobius.mbk"
DefaultMimeMap[".mqy"] = "application/vnd.mobius.mqy"
DefaultMimeMap[".msl"] = "application/vnd.mobius.msl"
DefaultMimeMap[".plc"] = "application/vnd.mobius.plc"
DefaultMimeMap[".txf"] = "application/vnd.mobius.txf"
DefaultMimeMap[".mpn"] = "application/vnd.mophun.application"
DefaultMimeMap[".mpc"] = "application/vnd.mophun.certificate"
DefaultMimeMap[".cil"] = "application/vnd.ms-artgalry"
DefaultMimeMap[".asf"] = "application/vnd.ms-asf"
DefaultMimeMap[".cab"] = "application/vnd.ms-cab-compressed"
DefaultMimeMap[".xls"] = "application/vnd.ms-excel"
DefaultMimeMap[".xlm"] = "application/vnd.ms-excel"
DefaultMimeMap[".xla"] = "application/vnd.ms-excel"
DefaultMimeMap[".xlc"] = "application/vnd.ms-excel"
DefaultMimeMap[".xlt"] = "application/vnd.ms-excel"
DefaultMimeMap[".xlw"] = "application/vnd.ms-excel"
DefaultMimeMap[".eot"] = "application/vnd.ms-fontobject"
DefaultMimeMap[".chm"] = "application/vnd.ms-htmlhelp"
DefaultMimeMap[".ims"] = "application/vnd.ms-ims"
DefaultMimeMap[".lrm"] = "application/vnd.ms-lrm"
DefaultMimeMap[".ppt"] = "application/vnd.ms-powerpoint"
DefaultMimeMap[".pps"] = "application/vnd.ms-powerpoint"
DefaultMimeMap[".pot"] = "application/vnd.ms-powerpoint"
DefaultMimeMap[".mpp"] = "application/vnd.ms-project"
DefaultMimeMap[".mpt"] = "application/vnd.ms-project"
DefaultMimeMap[".wps"] = "application/vnd.ms-works"
DefaultMimeMap[".wks"] = "application/vnd.ms-works"
DefaultMimeMap[".wcm"] = "application/vnd.ms-works"
DefaultMimeMap[".wdb"] = "application/vnd.ms-works"
DefaultMimeMap[".wpl"] = "application/vnd.ms-wpl"
DefaultMimeMap[".xps"] = "application/vnd.ms-xpsdocument"
DefaultMimeMap[".mseq"] = "application/vnd.mseq"
DefaultMimeMap[".mus"] = "application/vnd.musician"
DefaultMimeMap[".msty"] = "application/vnd.muvee.style"
DefaultMimeMap[".nlu"] = "application/vnd.neurolanguage.nlu"
DefaultMimeMap[".nnd"] = "application/vnd.noblenet-directory"
DefaultMimeMap[".nns"] = "application/vnd.noblenet-sealer"
DefaultMimeMap[".nnw"] = "application/vnd.noblenet-web"
DefaultMimeMap[".ngdat"] = "application/vnd.nokia.n-gage.data"
DefaultMimeMap[".n-gage"] = "application/vnd.nokia.n-gage.symbian.install"
DefaultMimeMap[".rpst"] = "application/vnd.nokia.radio-preset"
DefaultMimeMap[".rpss"] = "application/vnd.nokia.radio-presets"
DefaultMimeMap[".edm"] = "application/vnd.novadigm.edm"
DefaultMimeMap[".edx"] = "application/vnd.novadigm.edx"
DefaultMimeMap[".ext"] = "application/vnd.novadigm.ext"
DefaultMimeMap[".odc"] = "application/vnd.oasis.opendocument.chart"
DefaultMimeMap[".otc"] = "application/vnd.oasis.opendocument.chart-template"
DefaultMimeMap[".odf"] = "application/vnd.oasis.opendocument.formula"
DefaultMimeMap[".otf"] = "application/vnd.oasis.opendocument.formula-template"
DefaultMimeMap[".odg"] = "application/vnd.oasis.opendocument.graphics"
DefaultMimeMap[".otg"] = "application/vnd.oasis.opendocument.graphics-template"
DefaultMimeMap[".odi"] = "application/vnd.oasis.opendocument.image"
DefaultMimeMap[".oti"] = "application/vnd.oasis.opendocument.image-template"
DefaultMimeMap[".odp"] = "application/vnd.oasis.opendocument.presentation"
DefaultMimeMap[".otp"] = "application/vnd.oasis.opendocument.presentation-template"
DefaultMimeMap[".ods"] = "application/vnd.oasis.opendocument.spreadsheet"
DefaultMimeMap[".ots"] = "application/vnd.oasis.opendocument.spreadsheet-template"
DefaultMimeMap[".odt"] = "application/vnd.oasis.opendocument.text"
DefaultMimeMap[".otm"] = "application/vnd.oasis.opendocument.text-master"
DefaultMimeMap[".ott"] = "application/vnd.oasis.opendocument.text-template"
DefaultMimeMap[".oth"] = "application/vnd.oasis.opendocument.text-web"
DefaultMimeMap[".xo"] = "application/vnd.olpc-sugar"
DefaultMimeMap[".dd2"] = "application/vnd.oma.dd2+xml"
DefaultMimeMap[".oxt"] = "application/vnd.openofficeorg.extension"
DefaultMimeMap[".dp"] = "application/vnd.osgi.dp"
DefaultMimeMap[".prc"] = "application/vnd.palm"
DefaultMimeMap[".pdb"] = "application/vnd.palm"
DefaultMimeMap[".pqa"] = "application/vnd.palm"
DefaultMimeMap[".oprc"] = "application/vnd.palm"
DefaultMimeMap[".str"] = "application/vnd.pg.format"
DefaultMimeMap[".ei6"] = "application/vnd.pg.osasli"
DefaultMimeMap[".efif"] = "application/vnd.picsel"
DefaultMimeMap[".plf"] = "application/vnd.pocketlearn"
DefaultMimeMap[".pbd"] = "application/vnd.powerbuilder6"
DefaultMimeMap[".box"] = "application/vnd.previewsystems.box"
DefaultMimeMap[".mgz"] = "application/vnd.proteus.magazine"
DefaultMimeMap[".qps"] = "application/vnd.publishare-delta-tree"
DefaultMimeMap[".ptid"] = "application/vnd.pvi.ptid1"
DefaultMimeMap[".qxd"] = "application/vnd.quark.quarkxpress"
DefaultMimeMap[".qxt"] = "application/vnd.quark.quarkxpress"
DefaultMimeMap[".qwd"] = "application/vnd.quark.quarkxpress"
DefaultMimeMap[".qwt"] = "application/vnd.quark.quarkxpress"
DefaultMimeMap[".qxl"] = "application/vnd.quark.quarkxpress"
DefaultMimeMap[".qxb"] = "application/vnd.quark.quarkxpress"
DefaultMimeMap[".mxl"] = "application/vnd.recordare.musicxml"
DefaultMimeMap[".rm"] = "application/vnd.rn-realmedia"
DefaultMimeMap[".see"] = "application/vnd.seemail"
DefaultMimeMap[".sema"] = "application/vnd.sema"
DefaultMimeMap[".semd"] = "application/vnd.semd"
DefaultMimeMap[".semf"] = "application/vnd.semf"
DefaultMimeMap[".ifm"] = "application/vnd.shana.informed.formdata"
DefaultMimeMap[".itp"] = "application/vnd.shana.informed.formtemplate"
DefaultMimeMap[".iif"] = "application/vnd.shana.informed.interchange"
DefaultMimeMap[".ipk"] = "application/vnd.shana.informed.package"
DefaultMimeMap[".twd"] = "application/vnd.simtech-mindmapper"
DefaultMimeMap[".twds"] = "application/vnd.simtech-mindmapper"
DefaultMimeMap[".mmf"] = "application/vnd.smaf"
DefaultMimeMap[".sdkm"] = "application/vnd.solent.sdkm+xml"
DefaultMimeMap[".sdkd"] = "application/vnd.solent.sdkm+xml"
DefaultMimeMap[".dxp"] = "application/vnd.spotfire.dxp"
DefaultMimeMap[".sfs"] = "application/vnd.spotfire.sfs"
DefaultMimeMap[".sus"] = "application/vnd.sus-calendar"
DefaultMimeMap[".susp"] = "application/vnd.sus-calendar"
DefaultMimeMap[".svd"] = "application/vnd.svd"
DefaultMimeMap[".xsm"] = "application/vnd.syncml+xml"
DefaultMimeMap[".bdm"] = "application/vnd.syncml.dm+wbxml"
DefaultMimeMap[".xdm"] = "application/vnd.syncml.dm+xml"
DefaultMimeMap[".tao"] = "application/vnd.tao.intent-module-archive"
DefaultMimeMap[".tmo"] = "application/vnd.tmobile-livetv"
DefaultMimeMap[".tpt"] = "application/vnd.trid.tpt"
DefaultMimeMap[".mxs"] = "application/vnd.triscape.mxs"
DefaultMimeMap[".tra"] = "application/vnd.trueapp"
DefaultMimeMap[".ufd"] = "application/vnd.ufdl"
DefaultMimeMap[".ufdl"] = "application/vnd.ufdl"
DefaultMimeMap[".utz"] = "application/vnd.uiq.theme"
DefaultMimeMap[".umj"] = "application/vnd.umajin"
DefaultMimeMap[".unityweb"] = "application/vnd.unity"
DefaultMimeMap[".uoml"] = "application/vnd.uoml+xml"
DefaultMimeMap[".vcx"] = "application/vnd.vcx"
DefaultMimeMap[".vsd"] = "application/vnd.visio"
DefaultMimeMap[".vst"] = "application/vnd.visio"
DefaultMimeMap[".vss"] = "application/vnd.visio"
DefaultMimeMap[".vsw"] = "application/vnd.visio"
DefaultMimeMap[".vis"] = "application/vnd.visionary"
DefaultMimeMap[".vsf"] = "application/vnd.vsf"
DefaultMimeMap[".wbxml"] = "application/vnd.wap.wbxml"
DefaultMimeMap[".wmlc"] = "application/vnd.wap.wmlc"
DefaultMimeMap[".wmlsc"] = "application/vnd.wap.wmlscriptc"
DefaultMimeMap[".wtb"] = "application/vnd.webturbo"
DefaultMimeMap[".wpd"] = "application/vnd.wordperfect"
DefaultMimeMap[".wqd"] = "application/vnd.wqd"
DefaultMimeMap[".stf"] = "application/vnd.wt.stf"
DefaultMimeMap[".xar"] = "application/vnd.xara"
DefaultMimeMap[".xfdl"] = "application/vnd.xfdl"
DefaultMimeMap[".hvd"] = "application/vnd.yamaha.hv-dic"
DefaultMimeMap[".hvs"] = "application/vnd.yamaha.hv-script"
DefaultMimeMap[".hvp"] = "application/vnd.yamaha.hv-voice"
DefaultMimeMap[".saf"] = "application/vnd.yamaha.smaf-audio"
DefaultMimeMap[".spf"] = "application/vnd.yamaha.smaf-phrase"
DefaultMimeMap[".cmp"] = "application/vnd.yellowriver-custom-menu"
DefaultMimeMap[".zaz"] = "application/vnd.zzazz.deck+xml"
DefaultMimeMap[".vxml"] = "application/voicexml+xml"
DefaultMimeMap[".hlp"] = "application/winhlp"
DefaultMimeMap[".wsdl"] = "application/wsdl+xml"
DefaultMimeMap[".wspolicy"] = "application/wspolicy+xml"
DefaultMimeMap[".ace"] = "application/x-ace-compressed"
DefaultMimeMap[".bcpio"] = "application/x-bcpio"
DefaultMimeMap[".torrent"] = "application/x-bittorrent"
DefaultMimeMap[".bz"] = "application/x-bzip"
DefaultMimeMap[".bz2"] = "application/x-bzip2"
DefaultMimeMap[".boz"] = "application/x-bzip2"
DefaultMimeMap[".vcd"] = "application/x-cdlink"
DefaultMimeMap[".chat"] = "application/x-chat"
DefaultMimeMap[".pgn"] = "application/x-chess-pgn"
DefaultMimeMap[".cpio"] = "application/x-cpio"
DefaultMimeMap[".csh"] = "application/x-csh"
DefaultMimeMap[".dcr"] = "application/x-director"
DefaultMimeMap[".dir"] = "application/x-director"
DefaultMimeMap[".dxr"] = "application/x-director"
DefaultMimeMap[".fgd"] = "application/x-director"
DefaultMimeMap[".dvi"] = "application/x-dvi"
DefaultMimeMap[".spl"] = "application/x-futuresplash"
DefaultMimeMap[".gtar"] = "application/x-gtar"
DefaultMimeMap[".hdf"] = "application/x-hdf"
DefaultMimeMap[".latex"] = "application/x-latex"
DefaultMimeMap[".wmd"] = "application/x-ms-wmd"
DefaultMimeMap[".wmz"] = "application/x-ms-wmz"
DefaultMimeMap[".mdb"] = "application/x-msaccess"
DefaultMimeMap[".obd"] = "application/x-msbinder"
DefaultMimeMap[".crd"] = "application/x-mscardfile"
DefaultMimeMap[".clp"] = "application/x-msclip"
DefaultMimeMap[".exe"] = "application/x-msdownload"
DefaultMimeMap[".dll"] = "application/x-msdownload"
DefaultMimeMap[".com"] = "application/x-msdownload"
DefaultMimeMap[".bat"] = "application/x-msdownload"
DefaultMimeMap[".msi"] = "application/x-msdownload"
DefaultMimeMap[".mvb"] = "application/x-msmediaview"
DefaultMimeMap[".m13"] = "application/x-msmediaview"
DefaultMimeMap[".m14"] = "application/x-msmediaview"
DefaultMimeMap[".wmf"] = "application/x-msmetafile"
DefaultMimeMap[".mny"] = "application/x-msmoney"
DefaultMimeMap[".pub"] = "application/x-mspublisher"
DefaultMimeMap[".scd"] = "application/x-msschedule"
DefaultMimeMap[".trm"] = "application/x-msterminal"
DefaultMimeMap[".wri"] = "application/x-mswrite"
DefaultMimeMap[".nc"] = "application/x-netcdf"
DefaultMimeMap[".cdf"] = "application/x-netcdf"
DefaultMimeMap[".p12"] = "application/x-pkcs12"
DefaultMimeMap[".pfx"] = "application/x-pkcs12"
DefaultMimeMap[".p7b"] = "application/x-pkcs7-certificates"
DefaultMimeMap[".spc"] = "application/x-pkcs7-certificates"
DefaultMimeMap[".p7r"] = "application/x-pkcs7-certreqresp"
DefaultMimeMap[".rar"] = "application/x-rar-compressed"
DefaultMimeMap[".sh"] = "application/x-sh"
DefaultMimeMap[".shar"] = "application/x-shar"
DefaultMimeMap[".swf"] = "application/x-shockwave-flash"
DefaultMimeMap[".sit"] = "application/x-stuffit"
DefaultMimeMap[".sitx"] = "application/x-stuffitx"
DefaultMimeMap[".sv4cpio"] = "application/x-sv4cpio"
DefaultMimeMap[".sv4crc"] = "application/x-sv4crc"
DefaultMimeMap[".tar"] = "application/x-tar"
DefaultMimeMap[".tcl"] = "application/x-tcl"
DefaultMimeMap[".tex"] = "application/x-tex"
DefaultMimeMap[".texinfo"] = "application/x-texinfo"
DefaultMimeMap[".texi"] = "application/x-texinfo"
DefaultMimeMap[".ustar"] = "application/x-ustar"
DefaultMimeMap[".src"] = "application/x-wais-source"
DefaultMimeMap[".der"] = "application/x-x509-ca-cert"
DefaultMimeMap[".crt"] = "application/x-x509-ca-cert"
DefaultMimeMap[".xenc"] = "application/xenc+xml"
DefaultMimeMap[".xhtml"] = "application/xhtml+xml"
DefaultMimeMap[".xht"] = "application/xhtml+xml"
DefaultMimeMap[".xml"] = "text/xml"
DefaultMimeMap[".xsl"] = "application/xml"
DefaultMimeMap[".dtd"] = "application/xml-dtd"
DefaultMimeMap[".xop"] = "application/xop+xml"
DefaultMimeMap[".xslt"] = "application/xslt+xml"
DefaultMimeMap[".xspf"] = "application/xspf+xml"
DefaultMimeMap[".mxml"] = "application/xv+xml"
DefaultMimeMap[".xhvml"] = "application/xv+xml"
DefaultMimeMap[".xvml"] = "application/xv+xml"
DefaultMimeMap[".xvm"] = "application/xv+xml"
DefaultMimeMap[".zip"] = "application/zip"
DefaultMimeMap[".au"] = "audio/basic"
DefaultMimeMap[".snd"] = "audio/basic"
DefaultMimeMap[".mid"] = "audio/midi"
DefaultMimeMap[".midi"] = "audio/midi"
DefaultMimeMap[".kar"] = "audio/midi"
DefaultMimeMap[".rmi"] = "audio/midi"
DefaultMimeMap[".mp4a"] = "audio/mp4"
DefaultMimeMap[".mpga"] = "audio/mpeg"
DefaultMimeMap[".mp2"] = "audio/mpeg"
DefaultMimeMap[".mp2a"] = "audio/mpeg"
DefaultMimeMap[".mp3"] = "audio/mpeg"
DefaultMimeMap[".m2a"] = "audio/mpeg"
DefaultMimeMap[".m3a"] = "audio/mpeg"
DefaultMimeMap[".eol"] = "audio/vnd.digital-winds"
DefaultMimeMap[".lvp"] = "audio/vnd.lucent.voice"
DefaultMimeMap[".ecelp4800"] = "audio/vnd.nuera.ecelp4800"
DefaultMimeMap[".ecelp7470"] = "audio/vnd.nuera.ecelp7470"
DefaultMimeMap[".ecelp9600"] = "audio/vnd.nuera.ecelp9600"
DefaultMimeMap[".wav"] = "audio/wav"
DefaultMimeMap[".aif"] = "audio/x-aiff"
DefaultMimeMap[".aiff"] = "audio/x-aiff"
DefaultMimeMap[".aifc"] = "audio/x-aiff"
DefaultMimeMap[".m3u"] = "audio/x-mpegurl"
DefaultMimeMap[".wax"] = "audio/x-ms-wax"
DefaultMimeMap[".wma"] = "audio/x-ms-wma"
DefaultMimeMap[".ram"] = "audio/x-pn-realaudio"
DefaultMimeMap[".ra"] = "audio/x-pn-realaudio"
DefaultMimeMap[".rmp"] = "audio/x-pn-realaudio-plugin"
DefaultMimeMap[".cdx"] = "chemical/x-cdx"
DefaultMimeMap[".cif"] = "chemical/x-cif"
DefaultMimeMap[".cmdf"] = "chemical/x-cmdf"
DefaultMimeMap[".cml"] = "chemical/x-cml"
DefaultMimeMap[".csml"] = "chemical/x-csml"
DefaultMimeMap[".xyz"] = "chemical/x-xyz"
DefaultMimeMap[".bmp"] = "image/bmp"
DefaultMimeMap[".cgm"] = "image/cgm"
DefaultMimeMap[".g3"] = "image/g3fax"
DefaultMimeMap[".gif"] = "image/gif"
DefaultMimeMap[".ief"] = "image/ief"
DefaultMimeMap[".jpeg"] = "image/jpeg"
DefaultMimeMap[".jpg"] = "image/jpeg"
DefaultMimeMap[".jpe"] = "image/jpeg"
DefaultMimeMap[".png"] = "image/png"
DefaultMimeMap[".webp"] = "image/webp"
DefaultMimeMap[".heic"] = "image/heic"
DefaultMimeMap[".btif"] = "image/prs.btif"
DefaultMimeMap[".svg"] = "image/svg+xml"
DefaultMimeMap[".svgz"] = "image/svg+xml"
DefaultMimeMap[".tiff"] = "image/tiff"
DefaultMimeMap[".tif"] = "image/tiff"
DefaultMimeMap[".psd"] = "image/vnd.adobe.photoshop"
DefaultMimeMap[".djvu"] = "image/vnd.djvu"
DefaultMimeMap[".djv"] = "image/vnd.djvu"
DefaultMimeMap[".dwg"] = "image/vnd.dwg"
DefaultMimeMap[".dxf"] = "image/vnd.dxf"
DefaultMimeMap[".fbs"] = "image/vnd.fastbidsheet"
DefaultMimeMap[".fpx"] = "image/vnd.fpx"
DefaultMimeMap[".fst"] = "image/vnd.fst"
DefaultMimeMap[".mmr"] = "image/vnd.fujixerox.edmics-mmr"
DefaultMimeMap[".rlc"] = "image/vnd.fujixerox.edmics-rlc"
DefaultMimeMap[".mdi"] = "image/vnd.ms-modi"
DefaultMimeMap[".npx"] = "image/vnd.net-fpx"
DefaultMimeMap[".wbmp"] = "image/vnd.wap.wbmp"
DefaultMimeMap[".xif"] = "image/vnd.xiff"
DefaultMimeMap[".ras"] = "image/x-cmu-raster"
DefaultMimeMap[".cmx"] = "image/x-cmx"
DefaultMimeMap[".ico"] = "image/x-icon"
DefaultMimeMap[".pcx"] = "image/x-pcx"
DefaultMimeMap[".pic"] = "image/x-pict"
DefaultMimeMap[".pct"] = "image/x-pict"
DefaultMimeMap[".pnm"] = "image/x-portable-anymap"
DefaultMimeMap[".pbm"] = "image/x-portable-bitmap"
DefaultMimeMap[".pgm"] = "image/x-portable-graymap"
DefaultMimeMap[".ppm"] = "image/x-portable-pixmap"
DefaultMimeMap[".rgb"] = "image/x-rgb"
DefaultMimeMap[".xbm"] = "image/x-xbitmap"
DefaultMimeMap[".xpm"] = "image/x-xpixmap"
DefaultMimeMap[".xwd"] = "image/x-xwindowdump"
DefaultMimeMap[".eml"] = "message/rfc822"
DefaultMimeMap[".mime"] = "message/rfc822"
DefaultMimeMap[".mht"] = "message/rfc822"
DefaultMimeMap[".mhtml"] = "message/rfc822"
DefaultMimeMap[".igs"] = "model/iges"
DefaultMimeMap[".iges"] = "model/iges"
DefaultMimeMap[".msh"] = "model/mesh"
DefaultMimeMap[".mesh"] = "model/mesh"
DefaultMimeMap[".silo"] = "model/mesh"
DefaultMimeMap[".dwf"] = "model/vnd.dwf"
DefaultMimeMap[".gdl"] = "model/vnd.gdl"
DefaultMimeMap[".gtw"] = "model/vnd.gtw"
DefaultMimeMap[".mts"] = "model/vnd.mts"
DefaultMimeMap[".vtu"] = "model/vnd.vtu"
DefaultMimeMap[".wrl"] = "model/vrml"
DefaultMimeMap[".vrml"] = "model/vrml"
DefaultMimeMap[".ics"] = "text/calendar"
DefaultMimeMap[".ifb"] = "text/calendar"
DefaultMimeMap[".css"] = "text/css"
DefaultMimeMap[".csv"] = "text/csv"
DefaultMimeMap[".html"] = "text/html"
DefaultMimeMap[".htm"] = "text/html"
DefaultMimeMap[".txt"] = "text/plain"
DefaultMimeMap[".text"] = "text/plain"
DefaultMimeMap[".conf"] = "text/plain"
DefaultMimeMap[".def"] = "text/plain"
DefaultMimeMap[".list"] = "text/plain"
DefaultMimeMap[".log"] = "text/plain"
DefaultMimeMap[".in"] = "text/plain"
DefaultMimeMap[".dsc"] = "text/prs.lines.tag"
DefaultMimeMap[".rtx"] = "text/richtext"
DefaultMimeMap[".sgml"] = "text/sgml"
DefaultMimeMap[".sgm"] = "text/sgml"
DefaultMimeMap[".tsv"] = "text/tab-separated-values"
DefaultMimeMap[".t"] = "text/troff"
DefaultMimeMap[".tr"] = "text/troff"
DefaultMimeMap[".roff"] = "text/troff"
DefaultMimeMap[".man"] = "text/troff"
DefaultMimeMap[".me"] = "text/troff"
DefaultMimeMap[".ms"] = "text/troff"
DefaultMimeMap[".uri"] = "text/uri-list"
DefaultMimeMap[".uris"] = "text/uri-list"
DefaultMimeMap[".urls"] = "text/uri-list"
DefaultMimeMap[".fly"] = "text/vnd.fly"
DefaultMimeMap[".flx"] = "text/vnd.fmi.flexstor"
DefaultMimeMap[".3dml"] = "text/vnd.in3d.3dml"
DefaultMimeMap[".spot"] = "text/vnd.in3d.spot"
DefaultMimeMap[".jad"] = "text/vnd.sun.j2me.app-descriptor"
DefaultMimeMap[".wml"] = "text/vnd.wap.wml"
DefaultMimeMap[".wmls"] = "text/vnd.wap.wmlscript"
DefaultMimeMap[".s"] = "text/x-asm"
DefaultMimeMap[".asm"] = "text/x-asm"
DefaultMimeMap[".c"] = "text/x-c"
DefaultMimeMap[".cc"] = "text/x-c"
DefaultMimeMap[".cxx"] = "text/x-c"
DefaultMimeMap[".cpp"] = "text/x-c"
DefaultMimeMap[".h"] = "text/x-c"
DefaultMimeMap[".hh"] = "text/x-c"
DefaultMimeMap[".dic"] = "text/x-c"
DefaultMimeMap[".f"] = "text/x-fortran"
DefaultMimeMap[".for"] = "text/x-fortran"
DefaultMimeMap[".f77"] = "text/x-fortran"
DefaultMimeMap[".f90"] = "text/x-fortran"
DefaultMimeMap[".p"] = "text/x-pascal"
DefaultMimeMap[".pas"] = "text/x-pascal"
DefaultMimeMap[".java"] = "text/x-java-source"
DefaultMimeMap[".etx"] = "text/x-setext"
DefaultMimeMap[".uu"] = "text/x-uuencode"
DefaultMimeMap[".vcs"] = "text/x-vcalendar"
DefaultMimeMap[".vcf"] = "text/x-vcard"
DefaultMimeMap[".3gp"] = "video/3gpp"
DefaultMimeMap[".3g2"] = "video/3gpp2"
DefaultMimeMap[".h261"] = "video/h261"
DefaultMimeMap[".h263"] = "video/h263"
DefaultMimeMap[".h264"] = "video/h264"
DefaultMimeMap[".jpgv"] = "video/jpeg"
DefaultMimeMap[".jpm"] = "video/jpm"
DefaultMimeMap[".jpgm"] = "video/jpm"
DefaultMimeMap[".mj2"] = "video/mj2"
DefaultMimeMap[".mjp2"] = "video/mj2"
DefaultMimeMap[".mp4"] = "video/mp4"
DefaultMimeMap[".mp4v"] = "video/mp4"
DefaultMimeMap[".mpg4"] = "video/mp4"
DefaultMimeMap[".mpeg"] = "video/mpeg"
DefaultMimeMap[".mpg"] = "video/mpeg"
DefaultMimeMap[".mpe"] = "video/mpeg"
DefaultMimeMap[".m1v"] = "video/mpeg"
DefaultMimeMap[".m2v"] = "video/mpeg"
DefaultMimeMap[".qt"] = "video/quicktime"
DefaultMimeMap[".mov"] = "video/quicktime"
DefaultMimeMap[".fvt"] = "video/vnd.fvt"
DefaultMimeMap[".mxu"] = "video/vnd.mpegurl"
DefaultMimeMap[".m4u"] = "video/vnd.mpegurl"
DefaultMimeMap[".viv"] = "video/vnd.vivo"
DefaultMimeMap[".fli"] = "video/x-fli"
DefaultMimeMap[".wm"] = "video/x-ms-wm"
DefaultMimeMap[".wmv"] = "video/x-ms-wmv"
DefaultMimeMap[".wmx"] = "video/x-ms-wmx"
DefaultMimeMap[".wvx"] = "video/x-ms-wvx"
DefaultMimeMap[".avi"] = "video/x-msvideo"
DefaultMimeMap[".movie"] = "video/x-sgi-movie"
DefaultMimeMap[".ice"] = "x-conference/x-cooltalk"
DefaultMimeMap[".ipa"] = "application/vnd.iphone"
DefaultMimeMap[".apk"] = "application/vnd.android.package-archive"
}
func GetMimeMap() map[string]string {
once.Do(LoadMimeMap)
return DefaultMimeMap
}

@ -6,13 +6,13 @@ codec/encoding library for binc, msgpack, cbor, json.
Supported Serialization formats are:
- msgpack: https://github.com/msgpack/msgpack
- binc: http://github.com/ugorji/binc
- cbor: http://cbor.io http://tools.ietf.org/html/rfc7049
- json: http://json.org http://tools.ietf.org/html/rfc7159
- binc: http://github.com/ugorji/binc
- cbor: http://cbor.io http://tools.ietf.org/html/rfc7049
- json: http://json.org http://tools.ietf.org/html/rfc7159
- simple:
This package will carefully use 'package unsafe' for performance reasons in
specific places. You can build without unsafe use by passing the safe or
This package will carefully use 'package unsafe' for performance reasons
in specific places. You can build without unsafe use by passing the safe or
appengine tag i.e. 'go install -tags=codec.safe ...'.
This library works with both the standard `gc` and the `gccgo` compilers.
@ -20,67 +20,70 @@ This library works with both the standard `gc` and the `gccgo` compilers.
For detailed usage information, read the primer at
http://ugorji.net/blog/go-codec-primer .
The idiomatic Go support is as seen in other encoding packages in the
standard library (ie json, xml, gob, etc).
The idiomatic Go support is as seen in other encoding packages in the standard
library (ie json, xml, gob, etc).
Rich Feature Set includes:
- Simple but extremely powerful and feature-rich API
- Support for go 1.4 and above, while selectively using newer APIs for later releases
- Support for go 1.4 and above, while selectively using newer APIs for later
releases
- Excellent code coverage ( > 90% )
- Very High Performance.
Our extensive benchmarks show us outperforming Gob, Json, Bson, etc by 2-4X.
- Very High Performance. Our extensive benchmarks show us outperforming Gob,
Json, Bson, etc by 2-4X.
- Careful selected use of 'unsafe' for targeted performance gains.
- 100% safe mode supported, where 'unsafe' is not used at all.
- Lock-free (sans mutex) concurrency for scaling to 100's of cores
- In-place updates during decode, with option to zero value in maps and slices prior to decode
- Coerce types where appropriate
e.g. decode an int in the stream into a float, decode numbers from formatted strings, etc
- Corner Cases:
Overflows, nil maps/slices, nil values in streams are handled correctly
- In-place updates during decode, with option to zero value in maps and slices
prior to decode
- Coerce types where appropriate e.g. decode an int in the stream into a
float, decode numbers from formatted strings, etc
- Corner Cases: Overflows, nil maps/slices, nil values in streams are handled
correctly
- Standard field renaming via tags
- Support for omitting empty fields during an encoding
- Encoding from any value and decoding into pointer to any value
(struct, slice, map, primitives, pointers, interface{}, etc)
- Encoding from any value and decoding into pointer to any value (struct,
slice, map, primitives, pointers, interface{}, etc)
- Extensions to support efficient encoding/decoding of any named types
- Support encoding.(Binary|Text)(M|Unm)arshaler interfaces
- Support using existence of `IsZero() bool` to determine if a value is a zero value.
Analogous to time.Time.IsZero() bool.
- Decoding without a schema (into a interface{}).
Includes Options to configure what specific map or slice type to use
when decoding an encoded list or map into a nil interface{}
- Support using existence of `IsZero() bool` to determine if a value is a zero
value. Analogous to time.Time.IsZero() bool.
- Decoding without a schema (into a interface{}). Includes Options to
configure what specific map or slice type to use when decoding an encoded
list or map into a nil interface{}
- Mapping a non-interface type to an interface, so we can decode appropriately
into any interface type with a correctly configured non-interface value.
- Encode a struct as an array, and decode struct from an array in the data stream
- Option to encode struct keys as numbers (instead of strings)
(to support structured streams with fields encoded as numeric codes)
- Encode a struct as an array, and decode struct from an array in the data
stream
- Option to encode struct keys as numbers (instead of strings) (to support
structured streams with fields encoded as numeric codes)
- Comprehensive support for anonymous fields
- Fast (no-reflection) encoding/decoding of common maps and slices
- Code-generation for faster performance, supported in go 1.6+
- Support binary (e.g. messagepack, cbor) and text (e.g. json) formats
- Support indefinite-length formats to enable true streaming
(for formats which support it e.g. json, cbor)
- Support canonical encoding, where a value is ALWAYS encoded as same sequence of bytes.
This mostly applies to maps, where iteration order is non-deterministic.
- Support indefinite-length formats to enable true streaming (for formats
which support it e.g. json, cbor)
- Support canonical encoding, where a value is ALWAYS encoded as same
sequence of bytes. This mostly applies to maps, where iteration order is
non-deterministic.
- NIL in data stream decoded as zero value
- Never silently skip data when decoding.
User decides whether to return an error or silently skip data when keys or indexes
in the data stream do not map to fields in the struct.
- Detect and error when encoding a cyclic reference (instead of stack overflow shutdown)
- Never silently skip data when decoding. User decides whether to return an
error or silently skip data when keys or indexes in the data stream do not
map to fields in the struct.
- Detect and error when encoding a cyclic reference (instead of stack overflow
shutdown)
- Encode/Decode from/to chan types (for iterative streaming support)
- Drop-in replacement for encoding/json. `json:` key in struct tag supported.
- Provides a RPC Server and Client Codec for net/rpc communication protocol.
- Handle unique idiosyncrasies of codecs e.g.
- For messagepack, configure how ambiguities in handling raw bytes are resolved
- For messagepack, provide rpc server/client codec to support
msgpack-rpc protocol defined at:
https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
- Handle unique idiosyncrasies of codecs e.g. For messagepack,
configure how ambiguities in handling raw bytes are resolved and provide
rpc server/client codec to support msgpack-rpc protocol defined at:
https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
# Extension Support
## Extension Support
Users can register a function to handle the encoding or decoding of their
custom types.
Users can register a function to handle the encoding or decoding of their custom
types.
There are no restrictions on what the custom type can be. Some examples:
@ -92,43 +95,44 @@ There are no restrictions on what the custom type can be. Some examples:
type GifImage struct { ... }
```
As an illustration, MyStructWithUnexportedFields would normally be encoded
as an empty map because it has no exported fields, while UUID would be
encoded as a string. However, with extension support, you can encode any of
these however you like.
As an illustration, MyStructWithUnexportedFields would normally be encoded as
an empty map because it has no exported fields, while UUID would be encoded as a
string. However, with extension support, you can encode any of these however you
like.
There is also seamless support provided for registering an extension (with a
tag) but letting the encoding mechanism default to the standard way.
# Custom Encoding and Decoding
## Custom Encoding and Decoding
This package maintains symmetry in the encoding and decoding halfs. We
determine how to encode or decode by walking this decision tree
This package maintains symmetry in the encoding and decoding halfs. We determine
how to encode or decode by walking this decision tree
- is there an extension registered for the type?
- is type a codec.Selfer?
- is format binary, and is type a encoding.BinaryMarshaler and BinaryUnmarshaler?
- is format specifically json, and is type a encoding/json.Marshaler and Unmarshaler?
- is format text-based, and type an encoding.TextMarshaler and TextUnmarshaler?
- else we use a pair of functions based on the "kind" of the type e.g. map, slice, int64, etc
- is format binary, and is type a encoding.BinaryMarshaler and
BinaryUnmarshaler?
- is format specifically json, and is type a encoding/json.Marshaler and
Unmarshaler?
- is format text-based, and type an encoding.TextMarshaler and
TextUnmarshaler?
- else we use a pair of functions based on the "kind" of the type e.g. map,
slice, int64, etc
This symmetry is important to reduce chances of issues happening because the
encoding and decoding sides are out of sync e.g. decoded via very specific
encoding.TextUnmarshaler but encoded via kind-specific generalized mode.
Consequently, if a type only defines one-half of the symmetry (e.g. it
implements UnmarshalJSON() but not MarshalJSON() ), then that type doesn't
Consequently, if a type only defines one-half of the symmetry (e.g.
it implements UnmarshalJSON() but not MarshalJSON() ), then that type doesn't
satisfy the check and we will continue walking down the decision tree.
# RPC
## RPC
RPC Client and Server Codecs are implemented, so the codecs can be used with
the standard net/rpc package.
RPC Client and Server Codecs are implemented, so the codecs can be used with the
standard net/rpc package.
## Usage
# Usage
The Handle is SAFE for concurrent READ, but NOT SAFE for concurrent
modification.
@ -137,13 +141,13 @@ The Encoder and Decoder are NOT safe for concurrent use.
Consequently, the usage model is basically:
- Create and initialize the Handle before any use.
Once created, DO NOT modify it.
- Multiple Encoders or Decoders can now use the Handle concurrently.
They only read information off the Handle (never write).
- Create and initialize the Handle before any use. Once created, DO NOT modify
it.
- Multiple Encoders or Decoders can now use the Handle concurrently. They only
read information off the Handle (never write).
- However, each Encoder or Decoder MUST not be used concurrently
- To re-use an Encoder/Decoder, call Reset(...) on it first.
This allows you use state maintained on the Encoder/Decoder.
- To re-use an Encoder/Decoder, call Reset(...) on it first. This allows you
use state maintained on the Encoder/Decoder.
Sample usage model:
@ -194,8 +198,7 @@ Sample usage model:
client := rpc.NewClientWithCodec(rpcCodec)
```
## Running Tests
# Running Tests
To run tests, use the following:
@ -216,7 +219,7 @@ You can run the tag 'codec.safe' to run tests or build in safe mode. e.g.
go test -tags "alltests codec.safe" -run Suite
```
## Running Benchmarks
# Running Benchmarks
```
cd bench
@ -225,11 +228,9 @@ You can run the tag 'codec.safe' to run tests or build in safe mode. e.g.
Please see http://github.com/ugorji/go-codec-bench .
# Caveats
## Caveats
Struct fields matching the following are ignored during encoding and
decoding
Struct fields matching the following are ignored during encoding and decoding
- struct tag value set to -
- func, complex numbers, unsafe pointers
@ -239,8 +240,8 @@ decoding
Every other field in a struct will be encoded/decoded.
Embedded fields are encoded as if they exist in the top-level struct, with
some caveats. See Encode documentation.
Embedded fields are encoded as if they exist in the top-level struct, with some
caveats. See Encode documentation.
## Exported Package API

@ -7,6 +7,7 @@ import (
"math"
"reflect"
"time"
"unicode/utf8"
)
// Symbol management:
@ -789,6 +790,11 @@ func (d *bincDecDriver) DecodeStringAsBytes() (bs2 []byte) {
default:
d.d.errorf("string/bytes - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs))
}
if d.h.ValidateUnicode && !utf8.Valid(bs2) {
d.d.errorf("DecodeStringAsBytes: invalid UTF-8: %s", bs2)
}
d.bdRead = false
return
}
@ -1090,18 +1096,18 @@ func (d *bincDecDriver) nextValueBytesBdReadR(v0 []byte) (v []byte) {
//------------------------------------
//BincHandle is a Handle for the Binc Schema-Free Encoding Format
//defined at https://github.com/ugorji/binc .
// BincHandle is a Handle for the Binc Schema-Free Encoding Format
// defined at https://github.com/ugorji/binc .
//
//BincHandle currently supports all Binc features with the following EXCEPTIONS:
// - only integers up to 64 bits of precision are supported.
// big integers are unsupported.
// - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types).
// extended precision and decimal IEEE 754 floats are unsupported.
// - Only UTF-8 strings supported.
// Unicode_Other Binc types (UTF16, UTF32) are currently unsupported.
// BincHandle currently supports all Binc features with the following EXCEPTIONS:
// - only integers up to 64 bits of precision are supported.
// big integers are unsupported.
// - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types).
// extended precision and decimal IEEE 754 floats are unsupported.
// - Only UTF-8 strings supported.
// Unicode_Other Binc types (UTF16, UTF32) are currently unsupported.
//
//Note that these EXCEPTIONS are temporary and full support is possible and may happen soon.
// Note that these EXCEPTIONS are temporary and full support is possible and may happen soon.
type BincHandle struct {
BasicHandle
binaryEncodingType
@ -1158,50 +1164,49 @@ func (h *BincHandle) newDecDriver() decDriver {
//
// Format Description
//
// A timestamp is composed of 3 components:
//
// - secs: signed integer representing seconds since unix epoch
// - nsces: unsigned integer representing fractional seconds as a
// nanosecond offset within secs, in the range 0 <= nsecs < 1e9
// - tz: signed integer representing timezone offset in minutes east of UTC,
// and a dst (daylight savings time) flag
// A timestamp is composed of 3 components:
//
// When encoding a timestamp, the first byte is the descriptor, which
// defines which components are encoded and how many bytes are used to
// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it
// is not encoded in the byte array explicitly*.
// - secs: signed integer representing seconds since unix epoch
// - nsces: unsigned integer representing fractional seconds as a
// nanosecond offset within secs, in the range 0 <= nsecs < 1e9
// - tz: signed integer representing timezone offset in minutes east of UTC,
// and a dst (daylight savings time) flag
//
// Descriptor 8 bits are of the form `A B C DDD EE`:
// A: Is secs component encoded? 1 = true
// B: Is nsecs component encoded? 1 = true
// C: Is tz component encoded? 1 = true
// DDD: Number of extra bytes for secs (range 0-7).
// If A = 1, secs encoded in DDD+1 bytes.
// If A = 0, secs is not encoded, and is assumed to be 0.
// If A = 1, then we need at least 1 byte to encode secs.
// DDD says the number of extra bytes beyond that 1.
// E.g. if DDD=0, then secs is represented in 1 byte.
// if DDD=2, then secs is represented in 3 bytes.
// EE: Number of extra bytes for nsecs (range 0-3).
// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above)
// When encoding a timestamp, the first byte is the descriptor, which
// defines which components are encoded and how many bytes are used to
// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it
// is not encoded in the byte array explicitly*.
//
// Following the descriptor bytes, subsequent bytes are:
// Descriptor 8 bits are of the form `A B C DDD EE`:
// A: Is secs component encoded? 1 = true
// B: Is nsecs component encoded? 1 = true
// C: Is tz component encoded? 1 = true
// DDD: Number of extra bytes for secs (range 0-7).
// If A = 1, secs encoded in DDD+1 bytes.
// If A = 0, secs is not encoded, and is assumed to be 0.
// If A = 1, then we need at least 1 byte to encode secs.
// DDD says the number of extra bytes beyond that 1.
// E.g. if DDD=0, then secs is represented in 1 byte.
// if DDD=2, then secs is represented in 3 bytes.
// EE: Number of extra bytes for nsecs (range 0-3).
// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above)
//
// secs component encoded in `DDD + 1` bytes (if A == 1)
// nsecs component encoded in `EE + 1` bytes (if B == 1)
// tz component encoded in 2 bytes (if C == 1)
// Following the descriptor bytes, subsequent bytes are:
//
// secs and nsecs components are integers encoded in a BigEndian
// 2-complement encoding format.
// secs component encoded in `DDD + 1` bytes (if A == 1)
// nsecs component encoded in `EE + 1` bytes (if B == 1)
// tz component encoded in 2 bytes (if C == 1)
//
// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to
// Least significant bit 0 are described below:
// secs and nsecs components are integers encoded in a BigEndian
// 2-complement encoding format.
//
// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes).
// Bit 15 = have\_dst: set to 1 if we set the dst flag.
// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not.
// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format.
// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to
// Least significant bit 0 are described below:
//
// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes).
// Bit 15 = have\_dst: set to 1 if we set the dst flag.
// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not.
// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format.
func bincEncodeTime(t time.Time) []byte {
// t := rv2i(rv).(time.Time)
tsecs, tnsecs := t.Unix(), t.Nanosecond()

@ -7,6 +7,7 @@ import (
"math"
"reflect"
"time"
"unicode/utf8"
)
// major
@ -629,7 +630,11 @@ func (d *cborDecDriver) DecodeBytes(bs []byte) (bsOut []byte) {
}
func (d *cborDecDriver) DecodeStringAsBytes() (s []byte) {
return d.DecodeBytes(nil)
s = d.DecodeBytes(nil)
if d.h.ValidateUnicode && !utf8.Valid(s) {
d.d.errorf("DecodeStringAsBytes: invalid UTF-8: %s", s)
}
return
}
func (d *cborDecDriver) DecodeTime() (t time.Time) {

@ -335,6 +335,12 @@ type DecodeOptions struct {
//
// This mostly impacts when we decode registered extensions.
PreferPointerForStructOrArray bool
// ValidateUnicode controls will cause decoding to fail if an expected unicode
// string is well-formed but include invalid codepoints.
//
// This could have a performance impact.
ValidateUnicode bool
}
// ----------------------------------------
@ -660,6 +666,16 @@ func decStructFieldKeyNotString(dd decDriver, keyType valueType, b *[decScratchB
return
}
func (d *Decoder) kStructField(si *structFieldInfo, rv reflect.Value) {
if d.d.TryNil() {
if rv = si.path.field(rv); rv.IsValid() {
decSetNonNilRV2Zero(rv)
}
return
}
d.decodeValueNoCheckNil(si.path.fieldAlloc(rv), nil)
}
func (d *Decoder) kStruct(f *codecFnInfo, rv reflect.Value) {
ctyp := d.d.ContainerType()
ti := f.ti
@ -691,7 +707,7 @@ func (d *Decoder) kStruct(f *codecFnInfo, rv reflect.Value) {
}
d.mapElemValue()
if si := ti.siForEncName(rvkencname); si != nil {
d.decodeValue(si.path.fieldAlloc(rv), nil)
d.kStructField(si, rv)
} else if mf != nil {
// store rvkencname in new []byte, as it previously shares Decoder.b, which is used in decode
name2 = append(name2[:0], rvkencname...)
@ -726,7 +742,7 @@ func (d *Decoder) kStruct(f *codecFnInfo, rv reflect.Value) {
break
}
d.arrayElem()
d.decodeValue(si.path.fieldAlloc(rv), nil)
d.kStructField(si, rv)
}
var proceed bool
if hasLen {
@ -1505,14 +1521,15 @@ func (d *Decoder) naked() *fauxUnion {
// We will decode and store a value in that nil interface.
//
// Sample usages:
// // Decoding into a non-nil typed value
// var f float32
// err = codec.NewDecoder(r, handle).Decode(&f)
//
// // Decoding into nil interface
// var v interface{}
// dec := codec.NewDecoder(r, handle)
// err = dec.Decode(&v)
// // Decoding into a non-nil typed value
// var f float32
// err = codec.NewDecoder(r, handle).Decode(&f)
//
// // Decoding into nil interface
// var v interface{}
// dec := codec.NewDecoder(r, handle)
// err = dec.Decode(&v)
//
// When decoding into a nil interface{}, we will decode into an appropriate value based
// on the contents of the stream:
@ -1520,6 +1537,7 @@ func (d *Decoder) naked() *fauxUnion {
// - Other values are decoded appropriately depending on the type:
// bool, string, []byte, time.Time, etc
// - Extensions are decoded as RawExt (if no ext function registered for the tag)
//
// Configurations exist on the Handle to override defaults
// (e.g. for MapType, SliceType and how to decode raw bytes).
//
@ -2157,9 +2175,9 @@ func (x decSliceHelper) arrayCannotExpand(hasLen bool, lenv, j, containerLenS in
// decNextValueBytesHelper helps with NextValueBytes calls.
//
// Typical usage:
// - each Handle's decDriver will implement a high level nextValueBytes,
// which will track the current cursor, delegate to a nextValueBytesR
// method, and then potentially call bytesRdV at the end.
// - each Handle's decDriver will implement a high level nextValueBytes,
// which will track the current cursor, delegate to a nextValueBytesR
// method, and then potentially call bytesRdV at the end.
//
// See simple.go for typical usage model.
type decNextValueBytesHelper struct {
@ -2305,10 +2323,10 @@ func decByteSlice(r *decRd, clen, maxInitLen int, bs []byte) (bsOut []byte) {
}
// decInferLen will infer a sensible length, given the following:
// - clen: length wanted.
// - maxlen: max length to be returned.
// if <= 0, it is unset, and we infer it based on the unit size
// - unit: number of bytes for each element of the collection
// - clen: length wanted.
// - maxlen: max length to be returned.
// if <= 0, it is unset, and we infer it based on the unit size
// - unit: number of bytes for each element of the collection
func decInferLen(clen, maxlen, unit int) int {
// anecdotal testing showed increase in allocation with map length of 16.
// We saw same typical alloc from 0-8, then a 20% increase at 16.

@ -73,23 +73,23 @@ Rich Feature Set includes:
- Drop-in replacement for encoding/json. `json:` key in struct tag supported.
- Provides a RPC Server and Client Codec for net/rpc communication protocol.
- Handle unique idiosyncrasies of codecs e.g.
- For messagepack, configure how ambiguities in handling raw bytes are resolved
- For messagepack, provide rpc server/client codec to support
msgpack-rpc protocol defined at:
https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
For messagepack, configure how ambiguities in handling raw bytes are resolved and
provide rpc server/client codec to support
msgpack-rpc protocol defined at:
https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
Extension Support
# Extension Support
Users can register a function to handle the encoding or decoding of
their custom types.
There are no restrictions on what the custom type can be. Some examples:
type BisSet []int
type BitSet64 uint64
type UUID string
type MyStructWithUnexportedFields struct { a int; b bool; c []int; }
type GifImage struct { ... }
type BisSet []int
type BitSet64 uint64
type UUID string
type MyStructWithUnexportedFields struct { a int; b bool; c []int; }
type GifImage struct { ... }
As an illustration, MyStructWithUnexportedFields would normally be
encoded as an empty map because it has no exported fields, while UUID
@ -99,7 +99,7 @@ encode any of these however you like.
There is also seamless support provided for registering an extension (with a tag)
but letting the encoding mechanism default to the standard way.
Custom Encoding and Decoding
# Custom Encoding and Decoding
This package maintains symmetry in the encoding and decoding halfs.
We determine how to encode or decode by walking this decision tree
@ -120,12 +120,12 @@ Consequently, if a type only defines one-half of the symmetry
then that type doesn't satisfy the check and we will continue walking down the
decision tree.
RPC
# RPC
RPC Client and Server Codecs are implemented, so the codecs can be used
with the standard net/rpc package.
Usage
# Usage
The Handle is SAFE for concurrent READ, but NOT SAFE for concurrent modification.
@ -133,96 +133,95 @@ The Encoder and Decoder are NOT safe for concurrent use.
Consequently, the usage model is basically:
- Create and initialize the Handle before any use.
Once created, DO NOT modify it.
- Multiple Encoders or Decoders can now use the Handle concurrently.
They only read information off the Handle (never write).
- However, each Encoder or Decoder MUST not be used concurrently
- To re-use an Encoder/Decoder, call Reset(...) on it first.
This allows you use state maintained on the Encoder/Decoder.
- Create and initialize the Handle before any use.
Once created, DO NOT modify it.
- Multiple Encoders or Decoders can now use the Handle concurrently.
They only read information off the Handle (never write).
- However, each Encoder or Decoder MUST not be used concurrently
- To re-use an Encoder/Decoder, call Reset(...) on it first.
This allows you use state maintained on the Encoder/Decoder.
Sample usage model:
// create and configure Handle
var (
bh codec.BincHandle
mh codec.MsgpackHandle
ch codec.CborHandle
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
// configure extensions
// e.g. for msgpack, define functions and enable Time support for tag 1
// mh.SetExt(reflect.TypeOf(time.Time{}), 1, myExt)
// create and use decoder/encoder
var (
r io.Reader
w io.Writer
b []byte
h = &bh // or mh to use msgpack
)
dec = codec.NewDecoder(r, h)
dec = codec.NewDecoderBytes(b, h)
err = dec.Decode(&v)
enc = codec.NewEncoder(w, h)
enc = codec.NewEncoderBytes(&b, h)
err = enc.Encode(v)
//RPC Server
go func() {
for {
conn, err := listener.Accept()
rpcCodec := codec.GoRpc.ServerCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h)
rpc.ServeCodec(rpcCodec)
}
}()
//RPC Communication (client side)
conn, err = net.Dial("tcp", "localhost:5555")
rpcCodec := codec.GoRpc.ClientCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h)
client := rpc.NewClientWithCodec(rpcCodec)
Running Tests
// create and configure Handle
var (
bh codec.BincHandle
mh codec.MsgpackHandle
ch codec.CborHandle
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
// configure extensions
// e.g. for msgpack, define functions and enable Time support for tag 1
// mh.SetExt(reflect.TypeOf(time.Time{}), 1, myExt)
// create and use decoder/encoder
var (
r io.Reader
w io.Writer
b []byte
h = &bh // or mh to use msgpack
)
dec = codec.NewDecoder(r, h)
dec = codec.NewDecoderBytes(b, h)
err = dec.Decode(&v)
enc = codec.NewEncoder(w, h)
enc = codec.NewEncoderBytes(&b, h)
err = enc.Encode(v)
//RPC Server
go func() {
for {
conn, err := listener.Accept()
rpcCodec := codec.GoRpc.ServerCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h)
rpc.ServeCodec(rpcCodec)
}
}()
//RPC Communication (client side)
conn, err = net.Dial("tcp", "localhost:5555")
rpcCodec := codec.GoRpc.ClientCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h)
client := rpc.NewClientWithCodec(rpcCodec)
# Running Tests
To run tests, use the following:
go test
go test
To run the full suite of tests, use the following:
go test -tags alltests -run Suite
go test -tags alltests -run Suite
You can run the tag 'codec.safe' to run tests or build in safe mode. e.g.
go test -tags codec.safe -run Json
go test -tags "alltests codec.safe" -run Suite
go test -tags codec.safe -run Json
go test -tags "alltests codec.safe" -run Suite
Running Benchmarks
cd bench
go test -bench . -benchmem -benchtime 1s
cd bench
go test -bench . -benchmem -benchtime 1s
Please see http://github.com/ugorji/go-codec-bench .
Caveats
# Caveats
Struct fields matching the following are ignored during encoding and decoding
- struct tag value set to -
- func, complex numbers, unsafe pointers
- unexported and not embedded
- unexported and embedded and not struct kind
- unexported and embedded pointers (from go1.10)
- struct tag value set to -
- func, complex numbers, unsafe pointers
- unexported and not embedded
- unexported and embedded and not struct kind
- unexported and embedded pointers (from go1.10)
Every other field in a struct will be encoded/decoded.
Embedded fields are encoded as if they exist in the top-level struct,
with some caveats. See Encode documentation.
*/
package codec

@ -686,15 +686,11 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
var rvv = mapAddrLoopvarRV(f.ti.elem, vtypeKind)
if e.h.Canonical {
e.kMapCanonical(f.ti, rv, rvv, valFn)
e.mapEnd()
return
}
rtkey := f.ti.key
var keyTypeIsString = stringTypId == rt2id(rtkey) // rtkeyid
if !keyTypeIsString {
if keyTypeIsString {
keyFn = e.h.fn(rtkey)
} else {
for rtkey.Kind() == reflect.Ptr {
rtkey = rtkey.Elem()
}
@ -703,6 +699,12 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
}
}
if e.h.Canonical {
e.kMapCanonical(f.ti, rv, rvv, keyFn, valFn)
e.mapEnd()
return
}
var rvk = mapAddrLoopvarRV(f.ti.key, ktypeKind)
var it mapIter
@ -723,11 +725,14 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
e.mapEnd()
}
func (e *Encoder) kMapCanonical(ti *typeInfo, rv, rvv reflect.Value, valFn *codecFn) {
// we previously did out-of-band if an extension was registered.
// This is not necessary, as the natural kind is sufficient for ordering.
func (e *Encoder) kMapCanonical(ti *typeInfo, rv, rvv reflect.Value, keyFn, valFn *codecFn) {
// The base kind of the type of the map key is sufficient for ordering.
// We only do out of band if that kind is not ordered (number or string), bool or time.Time.
// If the key is a predeclared type, directly call methods on encDriver e.g. EncodeString
// but if not, call encodeValue, in case it has an extension registered or otherwise.
rtkey := ti.key
rtkeydecl := rtkey.PkgPath() == "" && rtkey.Name() != "" // key type is predeclared
mks := rv.MapKeys()
rtkeyKind := rtkey.Kind()
kfast := mapKeyFastKindFor(rtkeyKind)
@ -736,18 +741,24 @@ func (e *Encoder) kMapCanonical(ti *typeInfo, rv, rvv reflect.Value, valFn *code
switch rtkeyKind {
case reflect.Bool:
mksv := make([]boolRv, len(mks))
for i, k := range mks {
v := &mksv[i]
v.r = k
v.v = k.Bool()
// though bool keys make no sense in a map, it *could* happen.
// in that case, we MUST support it in reflection mode,
// as that is the fallback for even codecgen and others.
// sort the keys so that false comes before true
// ie if 2 keys in order (true, false), then swap them
if len(mks) == 2 && mks[0].Bool() {
mks[0], mks[1] = mks[1], mks[0]
}
sort.Sort(boolRvSlice(mksv))
for i := range mksv {
for i := range mks {
e.mapElemKey()
e.e.EncodeBool(mksv[i].v)
if rtkeydecl {
e.e.EncodeBool(mks[i].Bool())
} else {
e.encodeValueNonNil(mks[i], keyFn)
}
e.mapElemValue()
e.encodeValue(mapGet(rv, mksv[i].r, rvv, kfast, visindirect, visref), valFn)
e.encodeValue(mapGet(rv, mks[i], rvv, kfast, visindirect, visref), valFn)
}
case reflect.String:
mksv := make([]stringRv, len(mks))
@ -759,7 +770,11 @@ func (e *Encoder) kMapCanonical(ti *typeInfo, rv, rvv reflect.Value, valFn *code
sort.Sort(stringRvSlice(mksv))
for i := range mksv {
e.mapElemKey()
e.e.EncodeString(mksv[i].v)
if rtkeydecl {
e.e.EncodeString(mksv[i].v)
} else {
e.encodeValueNonNil(mksv[i].r, keyFn)
}
e.mapElemValue()
e.encodeValue(mapGet(rv, mksv[i].r, rvv, kfast, visindirect, visref), valFn)
}
@ -773,7 +788,11 @@ func (e *Encoder) kMapCanonical(ti *typeInfo, rv, rvv reflect.Value, valFn *code
sort.Sort(uint64RvSlice(mksv))
for i := range mksv {
e.mapElemKey()
e.e.EncodeUint(mksv[i].v)
if rtkeydecl {
e.e.EncodeUint(mksv[i].v)
} else {
e.encodeValueNonNil(mksv[i].r, keyFn)
}
e.mapElemValue()
e.encodeValue(mapGet(rv, mksv[i].r, rvv, kfast, visindirect, visref), valFn)
}
@ -787,7 +806,11 @@ func (e *Encoder) kMapCanonical(ti *typeInfo, rv, rvv reflect.Value, valFn *code
sort.Sort(int64RvSlice(mksv))
for i := range mksv {
e.mapElemKey()
e.e.EncodeInt(mksv[i].v)
if rtkeydecl {
e.e.EncodeInt(mksv[i].v)
} else {
e.encodeValueNonNil(mksv[i].r, keyFn)
}
e.mapElemValue()
e.encodeValue(mapGet(rv, mksv[i].r, rvv, kfast, visindirect, visref), valFn)
}
@ -801,7 +824,11 @@ func (e *Encoder) kMapCanonical(ti *typeInfo, rv, rvv reflect.Value, valFn *code
sort.Sort(float64RvSlice(mksv))
for i := range mksv {
e.mapElemKey()
e.e.EncodeFloat32(float32(mksv[i].v))
if rtkeydecl {
e.e.EncodeFloat32(float32(mksv[i].v))
} else {
e.encodeValueNonNil(mksv[i].r, keyFn)
}
e.mapElemValue()
e.encodeValue(mapGet(rv, mksv[i].r, rvv, kfast, visindirect, visref), valFn)
}
@ -815,11 +842,15 @@ func (e *Encoder) kMapCanonical(ti *typeInfo, rv, rvv reflect.Value, valFn *code
sort.Sort(float64RvSlice(mksv))
for i := range mksv {
e.mapElemKey()
e.e.EncodeFloat64(mksv[i].v)
if rtkeydecl {
e.e.EncodeFloat64(mksv[i].v)
} else {
e.encodeValueNonNil(mksv[i].r, keyFn)
}
e.mapElemValue()
e.encodeValue(mapGet(rv, mksv[i].r, rvv, kfast, visindirect, visref), valFn)
}
case reflect.Struct:
default:
if rtkey == timeTyp {
mksv := make([]timeRv, len(mks))
for i, k := range mks {
@ -836,8 +867,7 @@ func (e *Encoder) kMapCanonical(ti *typeInfo, rv, rvv reflect.Value, valFn *code
}
break
}
fallthrough
default:
// out-of-band
// first encode each key to a []byte first, then sort them, then record
bs0 := e.blist.get(len(mks) * 16)
@ -1010,16 +1040,17 @@ func (e *Encoder) ResetBytes(out *[]byte) {
// To set an option on all fields (e.g. omitempty on all fields), you
// can create a field called _struct, and set flags on it. The options
// which can be set on _struct are:
// - omitempty: so all fields are omitted if empty
// - toarray: so struct is encoded as an array
// - int: so struct key names are encoded as signed integers (instead of strings)
// - uint: so struct key names are encoded as unsigned integers (instead of strings)
// - float: so struct key names are encoded as floats (instead of strings)
// - omitempty: so all fields are omitted if empty
// - toarray: so struct is encoded as an array
// - int: so struct key names are encoded as signed integers (instead of strings)
// - uint: so struct key names are encoded as unsigned integers (instead of strings)
// - float: so struct key names are encoded as floats (instead of strings)
//
// More details on these below.
//
// Struct values "usually" encode as maps. Each exported struct field is encoded unless:
// - the field's tag is "-", OR
// - the field is empty (empty or the zero value) and its tag specifies the "omitempty" option.
// - the field's tag is "-", OR
// - the field is empty (empty or the zero value) and its tag specifies the "omitempty" option.
//
// When encoding as a map, the first string in the tag (before the comma)
// is the map key string to use when encoding.
@ -1032,8 +1063,9 @@ func (e *Encoder) ResetBytes(out *[]byte) {
// This is done with the int,uint or float option on the _struct field (see above).
//
// However, struct values may encode as arrays. This happens when:
// - StructToArray Encode option is set, OR
// - the tag on the _struct field sets the "toarray" option
// - StructToArray Encode option is set, OR
// - the tag on the _struct field sets the "toarray" option
//
// Note that omitempty is ignored when encoding struct values as arrays,
// as an entry must be encoded for each field, to maintain its position.
//
@ -1043,33 +1075,33 @@ func (e *Encoder) ResetBytes(out *[]byte) {
// or interface value, and any array, slice, map, or string of length zero.
//
// Anonymous fields are encoded inline except:
// - the struct tag specifies a replacement name (first value)
// - the field is of an interface type
// - the struct tag specifies a replacement name (first value)
// - the field is of an interface type
//
// Examples:
//
// // NOTE: 'json:' can be used as struct tag key, in place 'codec:' below.
// type MyStruct struct {
// _struct bool `codec:",omitempty"` //set omitempty for every field
// Field1 string `codec:"-"` //skip this field
// Field2 int `codec:"myName"` //Use key "myName" in encode stream
// Field3 int32 `codec:",omitempty"` //use key "Field3". Omit if empty.
// Field4 bool `codec:"f4,omitempty"` //use key "f4". Omit if empty.
// io.Reader //use key "Reader".
// MyStruct `codec:"my1" //use key "my1".
// MyStruct //inline it
// ...
// }
// // NOTE: 'json:' can be used as struct tag key, in place 'codec:' below.
// type MyStruct struct {
// _struct bool `codec:",omitempty"` //set omitempty for every field
// Field1 string `codec:"-"` //skip this field
// Field2 int `codec:"myName"` //Use key "myName" in encode stream
// Field3 int32 `codec:",omitempty"` //use key "Field3". Omit if empty.
// Field4 bool `codec:"f4,omitempty"` //use key "f4". Omit if empty.
// io.Reader //use key "Reader".
// MyStruct `codec:"my1" //use key "my1".
// MyStruct //inline it
// ...
// }
//
// type MyStruct struct {
// _struct bool `codec:",toarray"` //encode struct as an array
// }
// type MyStruct struct {
// _struct bool `codec:",toarray"` //encode struct as an array
// }
//
// type MyStruct struct {
// _struct bool `codec:",uint"` //encode struct with "unsigned integer" keys
// Field1 string `codec:"1"` //encode Field1 key using: EncodeInt(1)
// Field2 string `codec:"2"` //encode Field2 key using: EncodeInt(2)
// }
// type MyStruct struct {
// _struct bool `codec:",uint"` //encode struct with "unsigned integer" keys
// Field1 string `codec:"1"` //encode Field1 key using: EncodeInt(1)
// Field2 string `codec:"2"` //encode Field2 key using: EncodeInt(2)
// }
//
// The mode of encoding is based on the type of the value. When a value is seen:
// - If a Selfer, call its CodecEncodeSelf method
@ -1310,6 +1342,19 @@ TOP:
}
}
// encodeValueNonNil can encode a number, bool, or string
// OR non-nil values of kind map, slice and chan.
func (e *Encoder) encodeValueNonNil(rv reflect.Value, fn *codecFn) {
if fn == nil {
fn = e.h.fn(rvType(rv))
}
if fn.i.addrE { // typically, addrE = false, so check it first
rv = e.addrRV(rv, fn.i.ti.rt, fn.i.ti.ptr)
}
fn.fe(e, &fn.i, rv)
}
// addrRV returns a addressable value which may be readonly
func (e *Encoder) addrRV(rv reflect.Value, typ, ptrType reflect.Type) (rva reflect.Value) {
if rv.CanAddr() {

@ -147,10 +147,20 @@ func (f genHelperEncoder) EncEncodeComplex128(v complex128) { f.e.encodeComplex1
// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
func (f genHelperEncoder) EncEncode(v interface{}) { f.e.encode(v) }
// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
func (f genHelperEncoder) EncFnGivenAddr(v interface{}) *codecFn {
return f.e.h.fn(reflect.TypeOf(v).Elem())
}
// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
func (f genHelperEncoder) EncEncodeNumBoolStrKindGivenAddr(v interface{}, encFn *codecFn) {
f.e.encodeValueNonNil(reflect.ValueOf(v).Elem(), encFn)
}
// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
func (f genHelperEncoder) EncEncodeMapNonNil(v interface{}) {
if skipFastpathTypeSwitchInDirectCall || !fastpathEncodeTypeSwitch(v, f.e) {
f.e.encodeValue(reflect.ValueOf(v), nil)
f.e.encodeValueNonNil(reflect.ValueOf(v), nil)
}
}

@ -136,9 +136,15 @@ func (f genHelperEncoder) EncEncodeComplex128(v complex128) { f.e.encodeComplex1
// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
func (f genHelperEncoder) EncEncode(v interface{}) { f.e.encode(v) }
// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
func (f genHelperEncoder) EncFnGivenAddr(v interface{}) *codecFn { return f.e.h.fn(reflect.TypeOf(v).Elem()) }
// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
func (f genHelperEncoder) EncEncodeNumBoolStrKindGivenAddr(v interface{}, encFn *codecFn) {
f.e.encodeValueNonNil(reflect.ValueOf(v).Elem(), encFn)
}
// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
func (f genHelperEncoder) EncEncodeMapNonNil(v interface{}) {
if skipFastpathTypeSwitchInDirectCall || !fastpathEncodeTypeSwitch(v, f.e) {
f.e.encodeValue(reflect.ValueOf(v), nil)
f.e.encodeValueNonNil(reflect.ValueOf(v), nil)
}
}

@ -43,15 +43,17 @@ import (
//
// However, note following codecgen caveats:
// - Canonical option.
// If Canonical=true, codecgen'ed code will delegate encoding maps to reflection-based code.
// If Canonical=true, codecgen'ed code may delegate encoding maps to reflection-based code.
// This is due to the runtime work needed to marshal a map in canonical mode.
// However, if map key is a pre-defined/builtin numeric or string type, codecgen
// will try to write it out itself
// - CheckCircularRef option.
// When encoding a struct, a circular reference can lead to a stack overflow.
// If CheckCircularRef=true, codecgen'ed code will delegate encoding structs to reflection-based code.
// - MissingFielder implementation.
// If a type implements MissingFielder, a Selfer is not generated (with a warning message).
// Statically reproducing the runtime work needed to extract the missing fields and marshal them along with the struct fields,
// while handling the Canonical=true special case, was onerous to implement.
// Statically reproducing the runtime work needed to extract the missing fields and marshal them
// along with the struct fields, while handling the Canonical=true special case, was onerous to implement.
//
// During encode/decode, Selfer takes precedence.
// A type implementing Selfer will know how to encode/decode itself statically.
@ -126,7 +128,9 @@ import (
// v1: Initial Version
// v2: -
// v3: Changes for Kubernetes:
// changes in signature of some unpublished helper methods and codecgen cmdline arguments.
//
// changes in signature of some unpublished helper methods and codecgen cmdline arguments.
//
// v4: Removed separator support from (en|de)cDriver, and refactored codec(gen)
// v5: changes to support faster json decoding. Let encoder/decoder maintain state of collections.
// v6: removed unsafe from gen, and now uses codecgen.exec tag
@ -169,7 +173,7 @@ const (
// genStructCanonical configures whether we generate 2 paths based on Canonical flag
// when encoding struct fields.
genStructCanonical = false
genStructCanonical = true
// genFastpathCanonical configures whether we support Canonical in fast path.
// The savings is not much.
@ -390,7 +394,7 @@ func Gen(w io.Writer, buildTags, pkgName, uid string, noExtensions bool,
}
}
// add required packages
for _, k := range [...]string{"runtime", "errors", "strconv"} { // "reflect", "fmt"
for _, k := range [...]string{"runtime", "errors", "strconv", "sort"} { // "reflect", "fmt"
if _, ok := x.im[k]; !ok {
x.line("\"" + k + "\"")
}
@ -416,6 +420,7 @@ func Gen(w io.Writer, buildTags, pkgName, uid string, noExtensions bool,
x.line(")")
x.line("var (")
x.line("errCodecSelferOnlyMapOrArrayEncodeToStruct" + x.xs + " = " + "errors.New(`only encoded map or array can be decoded into a struct`)")
x.line("_ sort.Interface = nil")
x.line(")")
x.line("")
@ -424,6 +429,16 @@ func Gen(w io.Writer, buildTags, pkgName, uid string, noExtensions bool,
x.line("")
x.linef("func %sFalse() bool { return false }", x.hn)
x.linef("func %sTrue() bool { return true }", x.hn)
x.line("")
// add types for sorting canonical
for _, s := range []string{"string", "uint64", "int64", "float64"} {
x.linef("type %s%sSlice []%s", x.hn, s, s)
x.linef("func (p %s%sSlice) Len() int { return len(p) }", x.hn, s)
x.linef("func (p %s%sSlice) Swap(i, j int) { p[uint(i)], p[uint(j)] = p[uint(j)], p[uint(i)] }", x.hn, s)
x.linef("func (p %s%sSlice) Less(i, j int) bool { return p[uint(i)] < p[uint(j)] }", x.hn, s)
}
x.line("")
x.varsfxreset()
x.line("func init() {")
@ -942,9 +957,14 @@ func (x *genRunner) enc(varname string, t reflect.Type, isptr bool) {
if ti2.flagSelfer {
x.linef("%s %s.CodecEncodeSelf(e)", hasIf.c(true), varname)
return
} else if ti2.flagSelferPtr {
x.linef("%s %ssf%s := &%s", hasIf.c(true), genTempVarPfx, mi, varname)
x.linef("%ssf%s.CodecEncodeSelf(e)", genTempVarPfx, mi)
}
if ti2.flagSelferPtr {
if isptr {
x.linef("%s %s.CodecEncodeSelf(e)", hasIf.c(true), varname)
} else {
x.linef("%s %ssf%s := &%s", hasIf.c(true), genTempVarPfx, mi, varname)
x.linef("%ssf%s.CodecEncodeSelf(e)", genTempVarPfx, mi)
}
return
}
@ -1134,7 +1154,7 @@ func (x *genRunner) doEncOmitEmptyLine(t2 reflect.StructField, varname string, b
func (x *genRunner) encOmitEmptyLine(t2 reflect.StructField, varname string, buf *genBuf) {
// xdebugf("calling encOmitEmptyLine on: %v", t2.Type)
// smartly check omitEmpty on a struct type, as it may contain uncomparable map/slice/etc.
// also, for maps/slices/arrays, check if len ! 0 (not if == zero value)
// also, for maps/slices, check if len ! 0 (not if == zero value)
varname2 := varname + "." + t2.Name
switch t2.Type.Kind() {
case reflect.Struct:
@ -1169,9 +1189,9 @@ func (x *genRunner) encOmitEmptyLine(t2 reflect.StructField, varname string, buf
buf.s(varname2).s(" != ").s(x.genZeroValueR(t2.Type))
break
}
// fmt.Printf("???? !!!! We shouldn't get to this point !!!! ???? - for type: %v\n", t2.Type)
// buf.s("(")
buf.s(x.sayFalse()) // buf.s("false")
var wrote bool
for i, n := 0, t2.Type.NumField(); i < n; i++ {
f := t2.Type.Field(i)
if f.PkgPath != "" { // unexported
@ -1179,12 +1199,27 @@ func (x *genRunner) encOmitEmptyLine(t2 reflect.StructField, varname string, buf
}
buf.s(" || ")
x.encOmitEmptyLine(f, varname2, buf)
wrote = true
}
if !wrote {
buf.s(" || ").s(x.sayTrue())
}
//buf.s(")")
case reflect.Bool:
buf.s("bool(").s(varname2).s(")")
case reflect.Map, reflect.Slice, reflect.Array, reflect.Chan:
case reflect.Map, reflect.Slice, reflect.Chan:
buf.s("len(").s(varname2).s(") != 0")
case reflect.Array:
tlen := t2.Type.Len()
if tlen == 0 {
buf.s(x.sayFalse())
} else if t2.Type.Comparable() {
buf.s(varname2).s(" != ").s(x.genZeroValueR(t2.Type))
} else { // then we cannot even compare the individual values
// TODO use playground to check if you can compare to a
// zero value of an array, even if array not comparable.
buf.s(x.sayTrue())
}
default:
buf.s(varname2).s(" != ").s(x.genZeroValueR(t2.Type))
}
@ -1214,7 +1249,10 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
}
genFQNs := make([]genFQN, len(tisfi))
si2Pos := make(map[*structFieldInfo]int) // stores position in sorted structFieldInfos
for j, si := range tisfi {
si2Pos[si] = j
q := &genFQNs[j]
q.i = x.varsfx()
q.nilVar = genTempVarPfx + "n" + q.i
@ -1329,11 +1367,13 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
}
fn := func(tisfi []*structFieldInfo) {
for j, si := range tisfi {
q := &genFQNs[j]
// tisfi here may be source or sorted, so use the src position stored elsewhere
for _, si := range tisfi {
pos := si2Pos[si]
q := &genFQNs[pos]
doOmitEmptyCheck := (omitEmptySometimes && si.path.omitEmpty) || omitEmptyAlways
if doOmitEmptyCheck {
x.linef("if %s[%v] {", numfieldsvar, j)
x.linef("if %s[%v] {", numfieldsvar, pos)
}
x.linef("z.EncWriteMapElemKey()")
@ -1381,7 +1421,7 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
if genStructCanonical {
x.linef("if z.EncBasicHandle().Canonical {") // if Canonical block
fn(ti.sfi.sorted())
x.linef("} else {") // else !cononical block
x.linef("} else {") // else !Canonical block
fn(ti.sfi.source())
x.linef("}") // end if Canonical block
} else {
@ -1445,9 +1485,109 @@ func (x *genRunner) encListFallback(varname string, t reflect.Type) {
func (x *genRunner) encMapFallback(varname string, t reflect.Type) {
x.linef("if %s == nil { r.EncodeNil()", varname)
x.linef("} else if z.EncBasicHandle().Canonical { z.EncEncodeMapNonNil(%s)", varname)
x.line("} else if z.EncBasicHandle().Canonical {")
// Solve for easy case accomodated by sort package without reflection i.e.
// map keys of type: float, int, string (pre-defined/builtin types).
//
// To do this, we will get the keys into an array of uint64|float64|string,
// sort them, then write them out, and grab the value and encode it appropriately
tkey := t.Key()
tkind := tkey.Kind()
// tkeybase := tkey
// for tkeybase.Kind() == reflect.Ptr {
// tkeybase = tkeybase.Elem()
// }
// tikey := x.ti.get(rt2id(tkeybase), tkeybase)
// pre-defined types have a name and no pkgpath and appropriate kind
predeclared := tkey.PkgPath() == "" && tkey.Name() != ""
canonSortKind := reflect.Invalid
switch tkind {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
canonSortKind = reflect.Int64
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
canonSortKind = reflect.Uint64
case reflect.Float32, reflect.Float64:
canonSortKind = reflect.Float64
case reflect.String:
canonSortKind = reflect.String
}
var i string = x.varsfx()
fnCanonNumBoolStrKind := func() {
if !predeclared {
x.linef("var %svv%s %s", genTempVarPfx, i, x.genTypeName(tkey))
x.linef("%sencfn%s := z.EncFnGivenAddr(&%svv%s)", genTempVarPfx, i, genTempVarPfx, i)
}
// get the type, get the slice type its mapped to, and complete the code
x.linef("%ss%s := make([]%s, 0, len(%s))", genTempVarPfx, i, canonSortKind, varname)
x.linef("for k, _ := range %s {", varname)
x.linef(" %ss%s = append(%ss%s, %s(k))", genTempVarPfx, i, genTempVarPfx, i, canonSortKind)
x.linef("}")
x.linef("sort.Sort(%s%sSlice(%ss%s))", x.hn, canonSortKind, genTempVarPfx, i)
x.linef("z.EncWriteMapStart(len(%s))", varname)
x.linef("for _, %sv%s := range %ss%s {", genTempVarPfx, i, genTempVarPfx, i)
x.linef(" z.EncWriteMapElemKey()")
if predeclared {
switch tkind {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32:
x.linef("r.EncodeInt(int64(%sv%s))", genTempVarPfx, i)
case reflect.Int64:
x.linef("r.EncodeInt(%sv%s)", genTempVarPfx, i)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
x.linef("r.EncodeUint(%sv%s)", genTempVarPfx, i)
case reflect.Uint64:
x.linef("r.EncodeUint(uint64(%sv%s))", genTempVarPfx, i)
case reflect.Float32:
x.linef("r.EncodeFloat32(float32(%sv%s))", genTempVarPfx, i)
case reflect.Float64:
x.linef("r.EncodeFloat64(%sv%s)", genTempVarPfx, i)
case reflect.String:
x.linef("r.EncodeString(%sv%s)", genTempVarPfx, i)
}
} else {
x.linef("%svv%s = %s(%sv%s)", genTempVarPfx, i, x.genTypeName(tkey), genTempVarPfx, i)
x.linef("z.EncEncodeNumBoolStrKindGivenAddr(&%svv%s, %sencfn%s)", genTempVarPfx, i, genTempVarPfx, i)
}
x.linef(" z.EncWriteMapElemValue()")
vname := genTempVarPfx + "e" + i
if predeclared {
x.linef("%s := %s[%s(%sv%s)]", vname, varname, x.genTypeName(tkey), genTempVarPfx, i)
} else {
x.linef("%s := %s[%svv%s]", vname, varname, genTempVarPfx, i)
}
x.encVar(vname, t.Elem())
x.linef("}")
x.line("z.EncWriteMapEnd()")
}
// if canonSortKind != reflect.Invalid && !tikey.flagMarshalInterface {
// if predeclared {
// fnCanonNumBoolStrKind()
// } else {
// // handle if an extension
// x.linef("if z.Extension(%s(%s)) != nil { z.EncEncodeMapNonNil(%s) } else {",
// x.genTypeName(tkey), x.genZeroValueR(tkey), varname)
// fnCanonNumBoolStrKind()
// x.line("}")
// }
// } else {
// x.linef("z.EncEncodeMapNonNil(%s)", varname)
// }
if canonSortKind != reflect.Invalid {
fnCanonNumBoolStrKind()
} else {
x.linef("z.EncEncodeMapNonNil(%s)", varname)
}
x.line("} else {")
i := x.varsfx()
x.linef("z.EncWriteMapStart(len(%s))", varname)
x.linef("for %sk%s, %sv%s := range %s {", genTempVarPfx, i, genTempVarPfx, i, varname)
x.linef("z.EncWriteMapElemKey()")
@ -1456,6 +1596,7 @@ func (x *genRunner) encMapFallback(varname string, t reflect.Type) {
x.encVar(genTempVarPfx+"v"+i, t.Elem())
x.line("}")
x.line("z.EncWriteMapEnd()")
x.line("}")
}
@ -2684,7 +2825,7 @@ func genInternalSortableTypes() []string {
// that are used for reflection-based canonical's encoding of maps.
//
// For now, we only support the highest sizes for
// int64, uint64, float64, bool, string, bytes.
// int64, uint64, float64, string, bytes.
func genInternalSortablePlusTypes() []string {
return []string{
"string",
@ -2692,7 +2833,7 @@ func genInternalSortablePlusTypes() []string {
"uint64",
// "uintptr",
"int64",
"bool",
// "bool",
"time",
"bytes",
}

@ -0,0 +1,16 @@
// Copyright (c) 2012-2020 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.
//go:build go1.20 && !safe && !codec.safe && !appengine
// +build go1.20,!safe,!codec.safe,!appengine
package codec
import (
_ "reflect" // needed for go linkname(s)
"unsafe"
)
//go:linkname growslice reflect.growslice
//go:noescape
func growslice(typ unsafe.Pointer, old unsafeSlice, cap int) unsafeSlice

@ -0,0 +1,16 @@
// Copyright (c) 2012-2020 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.
//go:build go1.9 && !go1.20 && !safe && !codec.safe && !appengine
// +build go1.9,!go1.20,!safe,!codec.safe,!appengine
package codec
import (
_ "runtime" // needed for go linkname(s)
"unsafe"
)
//go:linkname growslice runtime.growslice
//go:noescape
func growslice(typ unsafe.Pointer, old unsafeSlice, cap int) unsafeSlice

@ -215,7 +215,8 @@ import (
// Note: Negative tests that check for errors will fail, so only use this
// when debugging, and run only one test at a time preferably.
//
// Note: RPC tests espeially fail, as they depend on getting the error from an Encode/Decode call.
// Note: RPC tests depend on getting the error from an Encode/Decode call.
// Consequently, they will always fail if debugging = true.
const debugging = false
const (
@ -617,6 +618,10 @@ func (e *codecError) Cause() error {
return e.err
}
func (e *codecError) Unwrap() error {
return e.err
}
func (e *codecError) Error() string {
if e.encode {
return fmt.Sprintf("%s encode error: %v", e.name, e.err)
@ -715,9 +720,10 @@ var SelfExt = &extFailWrapper{}
// By definition, it is not allowed for a Selfer to directly call Encode or Decode on itself.
// If that is done, Encode/Decode will rightfully fail with a Stack Overflow style error.
// For example, the snippet below will cause such an error.
// type testSelferRecur struct{}
// func (s *testSelferRecur) CodecEncodeSelf(e *Encoder) { e.MustEncode(s) }
// func (s *testSelferRecur) CodecDecodeSelf(d *Decoder) { d.MustDecode(s) }
//
// type testSelferRecur struct{}
// func (s *testSelferRecur) CodecEncodeSelf(e *Encoder) { e.MustEncode(s) }
// func (s *testSelferRecur) CodecDecodeSelf(d *Decoder) { d.MustDecode(s) }
//
// Note: *the first set of bytes of any value MUST NOT represent nil in the format*.
// This is because, during each decode, we first check the the next set of bytes
@ -761,13 +767,14 @@ type MissingFielder interface {
// This affords storing a map in a specific sequence in the stream.
//
// Example usage:
// type T1 []string // or []int or []Point or any other "slice" type
// func (_ T1) MapBySlice{} // T1 now implements MapBySlice, and will be encoded as a map
// type T2 struct { KeyValues T1 }
//
// var kvs = []string{"one", "1", "two", "2", "three", "3"}
// var v2 = T2{ KeyValues: T1(kvs) }
// // v2 will be encoded like the map: {"KeyValues": {"one": "1", "two": "2", "three": "3"} }
// type T1 []string // or []int or []Point or any other "slice" type
// func (_ T1) MapBySlice{} // T1 now implements MapBySlice, and will be encoded as a map
// type T2 struct { KeyValues T1 }
//
// var kvs = []string{"one", "1", "two", "2", "three", "3"}
// var v2 = T2{ KeyValues: T1(kvs) }
// // v2 will be encoded like the map: {"KeyValues": {"one": "1", "two": "2", "three": "3"} }
//
// The support of MapBySlice affords the following:
// - A slice or array type which implements MapBySlice will be encoded as a map
@ -969,6 +976,7 @@ func (x *basicHandleRuntimeState) setExt(rt reflect.Type, tag uint64, ext Ext) (
// initHandle should be called only from codec.initHandle global function.
// make it uninlineable, as it is called at most once for each handle.
//
//go:noinline
func (x *BasicHandle) initHandle(hh Handle) {
handleInitMu.Lock()
@ -1863,10 +1871,10 @@ type typeInfo struct {
keykind, elemkind uint8
flagHasPkgPath bool // Type.PackagePath != ""
flagCustom bool // does this have custom implementation?
flagComparable bool
flagCanTransient bool
flagMarshalInterface bool // does this have custom (un)marshal implementation?
flagSelferViaCodecgen bool
// custom implementation flags
@ -2128,7 +2136,6 @@ func (x *TypeInfos) load(rt reflect.Type) (pti *typeInfo) {
bset := func(when bool, b *bool) {
if when {
*b = true
ti.flagCustom = true
}
}
@ -2168,6 +2175,15 @@ func (x *TypeInfos) load(rt reflect.Type) (pti *typeInfo) {
b1, b2 = implIntf(rt, isSelferViaCodecgenerTyp)
ti.flagSelferViaCodecgen = b1 || b2
ti.flagMarshalInterface = ti.flagSelfer || ti.flagSelferPtr ||
ti.flagSelferViaCodecgen ||
ti.flagBinaryMarshaler || ti.flagBinaryMarshalerPtr ||
ti.flagBinaryUnmarshaler || ti.flagBinaryUnmarshalerPtr ||
ti.flagTextMarshaler || ti.flagTextMarshalerPtr ||
ti.flagTextUnmarshaler || ti.flagTextUnmarshalerPtr ||
ti.flagJsonMarshaler || ti.flagJsonMarshalerPtr ||
ti.flagJsonUnmarshaler || ti.flagJsonUnmarshalerPtr
b1 = rt.Comparable()
// bset(b1, &ti.flagComparable)
ti.flagComparable = b1
@ -2581,17 +2597,22 @@ func (checkOverflow) Uint2Int(v uint64, neg bool) (overflow bool) {
func (checkOverflow) SignedInt(v uint64) (overflow bool) {
//e.g. -127 to 128 for int8
pos := (v >> 63) == 0
ui2 := v & 0x7fffffffffffffff
if pos {
if ui2 > math.MaxInt64 {
overflow = true
}
} else {
if ui2 > math.MaxInt64-1 {
overflow = true
}
}
// pos := (v >> 63) == 0
// ui2 := v & 0x7fffffffffffffff
// if pos {
// if ui2 > math.MaxInt64 {
// overflow = true
// }
// } else {
// if ui2 > math.MaxInt64-1 {
// overflow = true
// }
// }
// a signed integer has overflow if the sign (first) bit is 1 (negative)
// and the numbers after the sign bit is > maxint64 - 1
overflow = (v>>63) != 0 && v&0x7fffffffffffffff > math.MaxInt64-1
return
}
@ -2774,22 +2795,23 @@ func freelistCapacity(length int) (capacity int) {
// without bounds checking is sufficient.
//
// Typical usage model:
// peek may go together with put, iff pop=true. peek gets largest byte slice temporarily.
// check is used to switch a []byte if necessary
// get/put go together
//
// peek may go together with put, iff pop=true. peek gets largest byte slice temporarily.
// check is used to switch a []byte if necessary
// get/put go together
//
// Given that folks may get a []byte, and then append to it a lot which may re-allocate
// a new []byte, we should try to return both (one received from blist and new one allocated).
//
// Typical usage model for get/put, when we don't know whether we may need more than requested
// v0 := blist.get()
// v1 := v0
// ... use v1 ...
// blist.put(v1)
// if byteSliceAddr(v0) != byteSliceAddr(v1) {
// blist.put(v0)
// }
//
// v0 := blist.get()
// v1 := v0
// ... use v1 ...
// blist.put(v1)
// if byteSliceAddr(v0) != byteSliceAddr(v1) {
// blist.put(v0)
// }
type bytesFreelist [][]byte
// peek returns a slice of possibly non-zero'ed bytes, with len=0,

@ -114,8 +114,19 @@ func isEmptyValue(v reflect.Value, tinfos *TypeInfos, recursive bool) bool {
switch v.Kind() {
case reflect.Invalid:
return true
case reflect.Array, reflect.String:
case reflect.String:
return v.Len() == 0
case reflect.Array:
// zero := reflect.Zero(v.Type().Elem())
// can I just check if the whole value is equal to zeros? seems not.
// can I just check if the whole value is equal to its zero value? no.
// Well, then we check if each value is empty without recursive.
for i, vlen := 0, v.Len(); i < vlen; i++ {
if !isEmptyValue(v.Index(i), tinfos, false) {
return false
}
}
return true
case reflect.Map, reflect.Slice, reflect.Chan:
return v.IsNil() || v.Len() == 0
case reflect.Bool:
@ -144,10 +155,10 @@ func isEmptyValue(v reflect.Value, tinfos *TypeInfos, recursive bool) bool {
}
// isEmptyStruct is only called from isEmptyValue, and checks if a struct is empty:
// - does it implement IsZero() bool
// - is it comparable, and can i compare directly using ==
// - if checkStruct, then walk through the encodable fields
// and check if they are empty or not.
// - does it implement IsZero() bool
// - is it comparable, and can i compare directly using ==
// - if checkStruct, then walk through the encodable fields
// and check if they are empty or not.
func isEmptyStruct(v reflect.Value, tinfos *TypeInfos, recursive bool) bool {
// v is a struct kind - no need to check again.
// We only check isZero on a struct kind, to reduce the amount of times

@ -463,7 +463,11 @@ func isEmptyValueFallbackRecur(urv *unsafeReflectValue, v reflect.Value, tinfos
case reflect.Map:
return urv.ptr == nil || len_map(rvRefPtr(urv)) == 0
case reflect.Array:
return v.Len() == 0
return v.Len() == 0 ||
urv.ptr == nil ||
urv.typ == nil ||
rtsize2(urv.typ) == 0 ||
unsafeCmpZero(urv.ptr, int(rtsize2(urv.typ)))
}
return false
}

@ -13,9 +13,10 @@ import (
)
// keep in sync with
// $GOROOT/src/cmd/compile/internal/gc/reflect.go: MAXKEYSIZE, MAXELEMSIZE
// $GOROOT/src/runtime/map.go: maxKeySize, maxElemSize
// $GOROOT/src/reflect/type.go: maxKeySize, maxElemSize
//
// $GOROOT/src/cmd/compile/internal/gc/reflect.go: MAXKEYSIZE, MAXELEMSIZE
// $GOROOT/src/runtime/map.go: maxKeySize, maxElemSize
// $GOROOT/src/reflect/type.go: maxKeySize, maxElemSize
//
// We use these to determine whether the type is stored indirectly in the map or not.
const (
@ -134,10 +135,6 @@ var unsafeZeroArr [1024]byte
//go:noescape
func rvPtrToType(typ unsafe.Pointer) reflect.Type
//go:linkname growslice runtime.growslice
//go:noescape
func growslice(typ unsafe.Pointer, old unsafeSlice, cap int) unsafeSlice
//go:linkname mapassign_fast32 runtime.mapassign_fast32
//go:noescape
func mapassign_fast32(typ unsafe.Pointer, m unsafe.Pointer, key uint32) unsafe.Pointer

@ -1093,6 +1093,7 @@ func (d *jsonDecDriver) readUnescapedString() (bs []byte) {
}
func (d *jsonDecDriver) dblQuoteStringAsBytes() (buf []byte) {
checkUtf8 := d.h.ValidateUnicode
d.d.decByteState = decByteStateNone
// use a local buf variable, so we don't do pointer chasing within loop
buf = (*d.buf)[:0]
@ -1153,7 +1154,11 @@ func (d *jsonDecDriver) dblQuoteStringAsBytes() (buf []byte) {
case 't':
buf = append(buf, '\t')
case 'u':
buf = append(buf, d.bstr[:utf8.EncodeRune(d.bstr[:], d.appendStringAsBytesSlashU())]...)
rr := d.appendStringAsBytesSlashU()
if checkUtf8 && rr == unicode.ReplacementChar {
d.d.errorf("invalid UTF-8 character found after: %s", buf)
}
buf = append(buf, d.bstr[:utf8.EncodeRune(d.bstr[:], rr)]...)
default:
*d.buf = buf
d.d.errorf("unsupported escaped value: %c", c)
@ -1275,13 +1280,13 @@ func (d *jsonDecDriver) DecodeNaked() {
// JsonHandle is a handle for JSON encoding format.
//
// Json is comprehensively supported:
// - decodes numbers into interface{} as int, uint or float64
// based on how the number looks and some config parameters e.g. PreferFloat, SignedInt, etc.
// - decode integers from float formatted numbers e.g. 1.27e+8
// - decode any json value (numbers, bool, etc) from quoted strings
// - configurable way to encode/decode []byte .
// by default, encodes and decodes []byte using base64 Std Encoding
// - UTF-8 support for encoding and decoding
// - decodes numbers into interface{} as int, uint or float64
// based on how the number looks and some config parameters e.g. PreferFloat, SignedInt, etc.
// - decode integers from float formatted numbers e.g. 1.27e+8
// - decode any json value (numbers, bool, etc) from quoted strings
// - configurable way to encode/decode []byte .
// by default, encodes and decodes []byte using base64 Std Encoding
// - UTF-8 support for encoding and decoding
//
// It has better performance than the json library in the standard library,
// by leveraging the performance improvements of the codec library.

@ -24,6 +24,7 @@ import (
"net/rpc"
"reflect"
"time"
"unicode/utf8"
)
const (
@ -872,7 +873,11 @@ func (d *msgpackDecDriver) DecodeBytes(bs []byte) (bsOut []byte) {
}
func (d *msgpackDecDriver) DecodeStringAsBytes() (s []byte) {
return d.DecodeBytes(nil)
s = d.DecodeBytes(nil)
if d.h.ValidateUnicode && !utf8.Valid(s) {
d.d.errorf("DecodeStringAsBytes: invalid UTF-8: %s", s)
}
return
}
func (d *msgpackDecDriver) descBd() string {
@ -1071,7 +1076,7 @@ func (d *msgpackDecDriver) decodeExtV(verifyTag bool, tag byte) (xbs []byte, xta
//--------------------------------------------------
//MsgpackHandle is a Handle for the Msgpack Schema-Free Encoding Format.
// MsgpackHandle is a Handle for the Msgpack Schema-Free Encoding Format.
type MsgpackHandle struct {
binaryEncodingType
BasicHandle

@ -198,29 +198,31 @@ type goRpc struct{}
// This ensures we use an adequate buffer during reading and writing.
// If not configured, we will internally initialize and use a buffer during reads and writes.
// This can be turned off via the RPCNoBuffer option on the Handle.
// var handle codec.JsonHandle
// handle.RPCNoBuffer = true // turns off attempt by rpc module to initialize a buffer
//
// var handle codec.JsonHandle
// handle.RPCNoBuffer = true // turns off attempt by rpc module to initialize a buffer
//
// Example 1: one way of configuring buffering explicitly:
// var handle codec.JsonHandle // codec handle
// handle.ReaderBufferSize = 1024
// handle.WriterBufferSize = 1024
// var conn io.ReadWriteCloser // connection got from a socket
// var serverCodec = GoRpc.ServerCodec(conn, handle)
// var clientCodec = GoRpc.ClientCodec(conn, handle)
//
// var handle codec.JsonHandle // codec handle
// handle.ReaderBufferSize = 1024
// handle.WriterBufferSize = 1024
// var conn io.ReadWriteCloser // connection got from a socket
// var serverCodec = GoRpc.ServerCodec(conn, handle)
// var clientCodec = GoRpc.ClientCodec(conn, handle)
//
// Example 2: you can also explicitly create a buffered connection yourself,
// and not worry about configuring the buffer sizes in the Handle.
// var handle codec.Handle // codec handle
// var conn io.ReadWriteCloser // connection got from a socket
// var bufconn = struct { // bufconn here is a buffered io.ReadWriteCloser
// io.Closer
// *bufio.Reader
// *bufio.Writer
// }{conn, bufio.NewReader(conn), bufio.NewWriter(conn)}
// var serverCodec = GoRpc.ServerCodec(bufconn, handle)
// var clientCodec = GoRpc.ClientCodec(bufconn, handle)
//
// var handle codec.Handle // codec handle
// var conn io.ReadWriteCloser // connection got from a socket
// var bufconn = struct { // bufconn here is a buffered io.ReadWriteCloser
// io.Closer
// *bufio.Reader
// *bufio.Writer
// }{conn, bufio.NewReader(conn), bufio.NewWriter(conn)}
// var serverCodec = GoRpc.ServerCodec(bufconn, handle)
// var clientCodec = GoRpc.ClientCodec(bufconn, handle)
var GoRpc goRpc
func (x goRpc) ServerCodec(conn io.ReadWriteCloser, h Handle) rpc.ServerCodec {

@ -5,9 +5,11 @@
package codec
import "time"
import "reflect"
import "bytes"
import (
"bytes"
"reflect"
"time"
)
type stringSlice []string
@ -109,18 +111,6 @@ func (p int64RvSlice) Less(i, j int) bool {
return p[uint(i)].v < p[uint(j)].v
}
type boolRv struct {
v bool
r reflect.Value
}
type boolRvSlice []boolRv
func (p boolRvSlice) Len() int { return len(p) }
func (p boolRvSlice) Swap(i, j int) { p[uint(i)], p[uint(j)] = p[uint(j)], p[uint(i)] }
func (p boolRvSlice) Less(i, j int) bool {
return !p[uint(i)].v && p[uint(j)].v
}
type timeRv struct {
v time.Time
r reflect.Value

@ -20,9 +20,11 @@ With the parameters passed in sortables or sortablesplus,
package codec
import "time"
import "reflect"
import "bytes"
import (
"time"
"reflect"
"bytes"
)
{{/* func init() { _ = time.Unix } */}}

@ -10,7 +10,7 @@ github.com/aliyun/aliyun-oss-go-sdk/oss
## explicit; go 1.16
github.com/allegro/bigcache/v3
github.com/allegro/bigcache/v3/queue
# github.com/aws/aws-sdk-go v1.44.167
# github.com/aws/aws-sdk-go v1.44.168
## explicit; go 1.11
github.com/aws/aws-sdk-go/aws
github.com/aws/aws-sdk-go/aws/arn
@ -174,7 +174,7 @@ github.com/aws/smithy-go/time
github.com/aws/smithy-go/transport/http
github.com/aws/smithy-go/transport/http/internal/io
github.com/aws/smithy-go/waiter
# github.com/baidubce/bce-sdk-go v0.9.138
# github.com/baidubce/bce-sdk-go v0.9.139
## explicit; go 1.11
github.com/baidubce/bce-sdk-go/auth
github.com/baidubce/bce-sdk-go/bce
@ -371,7 +371,7 @@ github.com/jackc/pgpassfile
# github.com/jackc/pgproto3/v2 v2.3.1
## explicit; go 1.12
github.com/jackc/pgproto3/v2
# github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b
# github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a
## explicit; go 1.14
github.com/jackc/pgservicefile
# github.com/jackc/pgtype v1.13.0
@ -594,7 +594,7 @@ github.com/tklauser/numcpus
# github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc
## explicit
github.com/tmthrgd/go-hex
# github.com/ugorji/go/codec v1.2.7
# github.com/ugorji/go/codec v1.2.8
## explicit; go 1.11
github.com/ugorji/go/codec
# github.com/ulikunitz/xz v0.5.11

Loading…
Cancel
Save