- update vendor

master
李光春 10 months ago
parent b28923b086
commit 0eb1f6a62b

@ -8,38 +8,38 @@ require (
github.com/allegro/bigcache/v3 v3.1.0
github.com/baidubce/bce-sdk-go v0.9.151
github.com/basgys/goxml2json v1.1.0
github.com/bytedance/sonic v1.9.1
github.com/bytedance/sonic v1.9.2
github.com/gin-gonic/gin v1.9.1
github.com/go-playground/locales v0.14.1
github.com/go-playground/universal-translator v0.18.1
github.com/go-playground/validator/v10 v10.14.1
github.com/go-sql-driver/mysql v1.7.1
github.com/goccy/go-json v0.10.2
github.com/gogf/gf/v2 v2.4.3
github.com/gogf/gf/v2 v2.4.4
github.com/json-iterator/go v1.1.12
github.com/lib/pq v1.10.9
github.com/mitchellh/mapstructure v1.5.0
github.com/mvdan/xurls v1.1.0
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/oschwald/geoip2-golang v1.8.0
github.com/qiniu/go-sdk/v7 v7.16.0
github.com/oschwald/geoip2-golang v1.9.0
github.com/qiniu/go-sdk/v7 v7.17.0
github.com/redis/go-redis/v9 v9.0.5
github.com/robfig/cron/v3 v3.0.1
github.com/saracen/go7z v0.0.0-20191010121135-9c09b6bd7fda
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/sirupsen/logrus v1.9.3
github.com/tencentyun/cos-go-sdk-v5 v0.7.41
go.mongodb.org/mongo-driver v1.11.7
github.com/tencentyun/cos-go-sdk-v5 v0.7.42
go.mongodb.org/mongo-driver v1.12.0
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.10.0
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
golang.org/x/text v0.10.0
golang.org/x/crypto v0.11.0
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
golang.org/x/text v0.11.0
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gorm.io/datatypes v1.2.0
gorm.io/driver/mysql v1.5.1
gorm.io/driver/postgres v1.5.2
gorm.io/gen v0.3.22
gorm.io/gorm v1.25.1
gorm.io/gorm v1.25.2
xorm.io/builder v0.3.12
xorm.io/xorm v1.3.2
)
@ -62,10 +62,10 @@ require (
github.com/google/go-querystring v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v5 v5.4.0 // indirect
github.com/jackc/pgx/v5 v5.4.1 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/klauspost/compress v1.16.6 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
@ -74,15 +74,13 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/mozillazg/go-httpheader v0.3.1 // indirect
github.com/oschwald/maxminddb-golang v1.10.0 // indirect
github.com/mozillazg/go-httpheader v0.4.0 // indirect
github.com/oschwald/maxminddb-golang v1.11.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rogpeppe/go-internal v1.8.1 // indirect
github.com/saracen/go7z-fixtures v0.0.0-20190623165746-aa6b8fba1d2f // indirect
github.com/saracen/solidblock v0.0.0-20190426153529-45df20abab6f // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
@ -99,14 +97,14 @@ require (
go.opentelemetry.io/otel/trace v1.16.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/net v0.11.0 // indirect
golang.org/x/arch v0.4.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.10.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
golang.org/x/tools v0.11.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect

@ -46,8 +46,8 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
github.com/bytedance/sonic v1.9.2 h1:GDaNjuWSGu09guE9Oql0MSTNhNCLlWwO8y/xM5BzcbM=
github.com/bytedance/sonic v1.9.2/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@ -146,8 +146,8 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogf/gf/v2 v2.4.3 h1:OP91EICmypAEaEpwSyjFnAZtTfcmeKXJQnPP4FZR/BM=
github.com/gogf/gf/v2 v2.4.3/go.mod h1:tsbmtwcAl2chcYoq/fP9W2FZf06aw4i89X34nbSHo9Y=
github.com/gogf/gf/v2 v2.4.4 h1:+s7PKxd4LJKjJn5ODZvYcbXMM5e+88Ww1W3GdOarLE8=
github.com/gogf/gf/v2 v2.4.4/go.mod h1:tsbmtwcAl2chcYoq/fP9W2FZf06aw4i89X34nbSHo9Y=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@ -268,8 +268,8 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6
github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CIdYfcc=
github.com/jackc/pgx/v4 v4.12.0/go.mod h1:fE547h6VulLPA3kySjfnSG/e2D861g/50JlVUa/ub60=
github.com/jackc/pgx/v5 v5.4.0 h1:BSr+GCm4N6QcgIwv0DyTFHK9ugfEFF9DzSbbzxOiXU0=
github.com/jackc/pgx/v5 v5.4.0/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY=
github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE=
github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY=
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
@ -294,8 +294,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@ -376,8 +376,8 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mozillazg/go-httpheader v0.3.1 h1:IRP+HFrMX2SlwY9riuio7raffXUpzAosHtZu25BSJok=
github.com/mozillazg/go-httpheader v0.3.1/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA=
github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w=
github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA=
github.com/mvdan/xurls v1.1.0 h1:OpuDelGQ1R1ueQ6sSryzi6P+1RtBpfQHM8fJwlE45ww=
github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@ -409,10 +409,10 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/oschwald/geoip2-golang v1.8.0 h1:KfjYB8ojCEn/QLqsDU0AzrJ3R5Qa9vFlx3z6SLNcKTs=
github.com/oschwald/geoip2-golang v1.8.0/go.mod h1:R7bRvYjOeaoenAp9sKRS8GX5bJWcZ0laWO5+DauEktw=
github.com/oschwald/maxminddb-golang v1.10.0 h1:Xp1u0ZhqkSuopaKmk1WwHtjF0H9Hd9181uj2MQ5Vndg=
github.com/oschwald/maxminddb-golang v1.10.0/go.mod h1:Y2ELenReaLAZ0b400URyGwvYxHV1dLIxBuyOsyYjHK0=
github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=
github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y=
github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0=
github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg=
github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@ -425,7 +425,6 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@ -447,8 +446,8 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk=
github.com/qiniu/go-sdk/v7 v7.16.0 h1:Jt4YOMLuaDfgb/KdVg0O1fYLpv5MDkYe/zV+Ri7gWRs=
github.com/qiniu/go-sdk/v7 v7.16.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w=
github.com/qiniu/go-sdk/v7 v7.17.0 h1:sF05b0NFdlUEz1SnJStrOn+PVUPu76lYCoHZCZyNYgs=
github.com/qiniu/go-sdk/v7 v7.17.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w=
github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=
@ -512,17 +511,14 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4=
github.com/tencentyun/cos-go-sdk-v5 v0.7.41 h1:iU0Li/Np78H4SBna0ECQoF3mpgi6ImLXU+doGzPFXGc=
github.com/tencentyun/cos-go-sdk-v5 v0.7.41/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0=
github.com/tencentyun/cos-go-sdk-v5 v0.7.42 h1:Up1704BJjI5orycXKjpVpvuOInt9GC5pqY4knyE9Uds=
github.com/tencentyun/cos-go-sdk-v5 v0.7.42/go.mod h1:LUFnaqRmGk6pEHOaRmdn2dCZR2j0cSsM5xowWFPTPao=
github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
@ -539,10 +535,8 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@ -558,8 +552,8 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.mongodb.org/mongo-driver v1.11.7 h1:LIwYxASDLGUg/8wOhgOOZhX8tQa/9tgZPgzZoVqJvcs=
go.mongodb.org/mongo-driver v1.11.7/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY=
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@ -593,8 +587,8 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc=
golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@ -615,11 +609,11 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -630,8 +624,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -654,8 +648,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -712,13 +706,13 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@ -728,8 +722,9 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
@ -753,8 +748,8 @@ golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg=
golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8=
golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -780,8 +775,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
@ -828,8 +823,9 @@ gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.24.3/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64=
gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho=
gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/hints v1.1.2 h1:b5j0kwk5p4+3BtDtYqqfY+ATSxjj+6ptPgVveuynn9o=
gorm.io/hints v1.1.2/go.mod h1:/ARdpUHAtyEMCh5NNi3tI7FsGh+Cj/MIUlvNxCNCFWg=
gorm.io/plugin/dbresolver v1.4.1 h1:Ug4LcoPhrvqq71UhxtF346f+skTYoCa/nEsdjvHwEzk=

@ -27,21 +27,19 @@ TMPL_avx := fastint_amd64_test fastfloat_amd64_test native_amd64_test native_ex
TMPL_avx2 := fastint_amd64_test fastfloat_amd64_test native_amd64_test native_export_amd64
TMPL_sse := fastint_amd64_test fastfloat_amd64_test native_amd64_test native_export_amd64
CFLAGS_avx := -msse -mno-sse4 -mavx -mpclmul -mno-avx2 -DUSE_AVX=1 -DUSE_AVX2=0
CFLAGS_avx2 := -msse -mno-sse4 -mavx -mpclmul -mavx2 -DUSE_AVX=1 -DUSE_AVX2=1
CFLAGS_sse := -msse -mno-sse4 -mno-avx -mno-avx2 -mpclmul
CFLAGS_avx := -msse -mssse3 -mno-sse4 -mavx -mpclmul -mno-avx2 -DUSE_AVX=1 -DUSE_AVX2=0
CFLAGS_avx2 := -msse -mssse3 -mno-sse4 -mavx -mpclmul -mavx2 -DUSE_AVX=1 -DUSE_AVX2=1
CFLAGS_sse := -msse -mssse3 -mno-sse4 -mno-avx -mno-avx2 -mpclmul
TARGETFLAGS := -target x86_64-apple-macos11 -nostdlib -fno-builtin -fno-asynchronous-unwind-tables
CC_amd64 := clang
ASM2ASM_amd64 := tools/asm2asm/asm2asm.py
CFLAGS := -mno-red-zone
CFLAGS += -target x86_64-apple-macos11
CFLAGS += -fno-asynchronous-unwind-tables
CFLAGS += -fno-builtin
CFLAGS += -fno-exceptions
CFLAGS += -fno-rtti
CFLAGS += -fno-stack-protector
CFLAGS += -nostdlib
CFLAGS += -O3
CFLAGS += -Wall -Werror
@ -74,7 +72,12 @@ $(1): ${@asmout} ${@deps}
${@asmout}: ${@stubout} ${NATIVE_SRC}
mkdir -p ${TMP_DIR}/$(1)
$${CC_${@cpu}} $${CFLAGS} $${CFLAGS_$(1)} -S -o ${TMP_DIR}/$(1)/native.s ${SRC_FILE}
$${CC_${@cpu}} $${CFLAGS} $${CFLAGS_$(1)} ${TARGETFLAGS} -S -o ${TMP_DIR}/$(1)/native.s ${SRC_FILE}
$(foreach file,
$(wildcard native/unittest/*),
$${CC_${@cpu}} $${CFLAGS} $${CFLAGS_$(1)} -I./native -o ${TMP_DIR}/$(1)/test $(file)
./${TMP_DIR}/$(1)/test
)
python3 $${ASM2ASM_${@cpu}} ${@asmout} ${TMP_DIR}/$(1)/native.s
asmfmt -w ${@asmout}
@ -110,3 +113,4 @@ $(foreach \
${ARCH}, \
$(eval $(call build_arch,${arch})) \
)

@ -183,4 +183,4 @@ func Get(src []byte, path ...interface{}) (ast.Node, error) {
// which can reduce unnecessary memory copy.
func GetFromString(src string, path ...interface{}) (ast.Node, error) {
return ast.NewSearcher(src).GetByPath(path...)
}
}

@ -148,4 +148,4 @@ func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
return Node{}, self.parser.ExportError(err)
}
return newRawNode(self.parser.s[start:self.parser.p], t), nil
}
}

@ -256,4 +256,4 @@ func (self *Node) encodeObject(buf *[]byte) error {
*buf = append(*buf, '}')
return nil
}
}

@ -161,4 +161,4 @@ type PairSlice []Pair
func (self PairSlice) Sort() {
radixQsort(self, 0, maxDepth(len(self)))
}
}

@ -1805,4 +1805,4 @@ func unwrapError(err error) *Node {
p: unsafe.Pointer(&msg),
}
}
}
}

@ -615,4 +615,4 @@ func (self *Parser) ExportError(err types.ParsingError) error {
Src : self.s,
Code: err,
}.Description())
}
}

@ -52,4 +52,4 @@ var (
)
//go:linkname unquoteBytes encoding/json.unquoteBytes
func unquoteBytes(s []byte) (t []byte, ok bool)
func unquoteBytes(s []byte) (t []byte, ok bool)

@ -105,4 +105,4 @@ var (
//
// NewStreamEncoder returns a new encoder that write to w.
NewStreamEncoder = encoder.NewStreamEncoder
)
)

@ -818,8 +818,8 @@ var (
)
var (
_Vp_max_f32 = new(float64)
_Vp_min_f32 = new(float64)
_Vp_max_f32 = new(float32)
_Vp_min_f32 = new(float32)
)
func init() {
@ -828,17 +828,15 @@ func init() {
}
func (self *_Assembler) range_single() {
self.Emit("MOVSD" , _VAR_st_Dv, _X0) // MOVSD st.Dv, X0
self.Emit("CVTSD2SS", _VAR_st_Dv, _X0) // CVTSD2SS st.Dv, X0
self.Emit("MOVQ" , _V_max_f32, _AX) // MOVQ _max_f32, AX
self.Emit("MOVQ" , jit.Gitab(_I_float32), _ET) // MOVQ ${itab(float32)}, ET
self.Emit("MOVQ" , jit.Gtype(_T_float32), _EP) // MOVQ ${type(float32)}, EP
self.Emit("UCOMISD" , jit.Ptr(_AX, 0), _X0) // UCOMISD (AX), X0
self.Emit("UCOMISS" , jit.Ptr(_AX, 0), _X0) // UCOMISS (AX), X0
self.Sjmp("JA" , _LB_range_error) // JA _range_error
self.Emit("MOVQ" , _V_min_f32, _AX) // MOVQ _min_f32, AX
self.Emit("MOVSD" , jit.Ptr(_AX, 0), _X1) // MOVSD (AX), X1
self.Emit("UCOMISD" , _X0, _X1) // UCOMISD X0, X1
self.Sjmp("JA" , _LB_range_error) // JA _range_error
self.Emit("CVTSD2SS", _X0, _X0) // CVTSD2SS X0, X0
self.Emit("UCOMISS" , jit.Ptr(_AX, 0), _X0) // UCOMISS (AX), X0
self.Sjmp("JB" , _LB_range_error) // JB _range_error
}
func (self *_Assembler) range_signed(i *rt.GoItab, t *rt.GoType, a int64, b int64) {

@ -825,8 +825,8 @@ var (
)
var (
_Vp_max_f32 = new(float64)
_Vp_min_f32 = new(float64)
_Vp_max_f32 = new(float32)
_Vp_min_f32 = new(float32)
)
func init() {
@ -835,17 +835,15 @@ func init() {
}
func (self *_Assembler) range_single_X0() {
self.Emit("MOVSD" , _VAR_st_Dv, _X0) // MOVSD st.Dv, X0
self.Emit("CVTSD2SS", _VAR_st_Dv, _X0) // CVTSD2SS _VAR_st_Dv, X0
self.Emit("MOVQ" , _V_max_f32, _CX) // MOVQ _max_f32, CX
self.Emit("MOVQ" , jit.Gitab(_I_float32), _ET) // MOVQ ${itab(float32)}, ET
self.Emit("MOVQ" , jit.Gtype(_T_float32), _EP) // MOVQ ${type(float32)}, EP
self.Emit("UCOMISD" , jit.Ptr(_CX, 0), _X0) // UCOMISD (CX), X0
self.Emit("UCOMISS" , jit.Ptr(_CX, 0), _X0) // UCOMISS (CX), X0
self.Sjmp("JA" , _LB_range_error) // JA _range_error
self.Emit("MOVQ" , _V_min_f32, _CX) // MOVQ _min_f32, CX
self.Emit("MOVSD" , jit.Ptr(_CX, 0), _X1) // MOVSD (CX), X1
self.Emit("UCOMISD" , _X0, _X1) // UCOMISD X0, X1
self.Sjmp("JA" , _LB_range_error) // JA _range_error
self.Emit("CVTSD2SS", _X0, _X0) // CVTSD2SS X0, X0
self.Emit("UCOMISS" , jit.Ptr(_CX, 0), _X0) // UCOMISS (CX), X0
self.Sjmp("JB" , _LB_range_error) // JB _range_error
}
func (self *_Assembler) range_signed_CX(i *rt.GoItab, t *rt.GoType, a int64, b int64) {
@ -1989,4 +1987,4 @@ func (self *_Assembler) WriteRecNotAX(i int, ptr obj.Addr, rec obj.Addr, saveDI
self.Link("_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Emit("MOVQ", ptr, rec)
self.Link("_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
}
}

@ -1152,4 +1152,4 @@ func (self *_Compiler) checkIfSkip(p *_Program, vt reflect.Type, c byte) int {
p.pin(j)
p.int(_OP_add, 1)
return s
}
}

@ -67,4 +67,4 @@ func (self *_Assembler) debug_instr(i int, v *_Instr) {
}
self.force_gc()
}
}
}

@ -252,4 +252,4 @@ func Skip(data []byte) (start int, end int) {
ret := native.SkipOne(&s, &p, m, uint64(0))
types.FreeStateMachine(m)
return ret, p
}
}

@ -140,4 +140,4 @@ func findOrCompile(vt *rt.GoType) (_Decoder, error) {
} else {
return nil, err
}
}
}

@ -21,8 +21,9 @@ import (
`io`
`sync`
`github.com/bytedance/sonic/option`
`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/option`
)
var (
@ -71,6 +72,7 @@ func (self *StreamDecoder) Decode(val interface{}) (err error) {
var first = true
var repeat = true
read_more:
for {
l := len(buf)
@ -97,11 +99,20 @@ read_more:
l := len(buf)
if l > 0 {
self.Decoder.Reset(string(buf))
err = self.Decoder.Decode(val)
if err != nil {
if repeat && self.repeatable(err) {
var x int
if ret := native.SkipOneFast(&self.s, &x); ret < 0 {
if repeat {
goto read_more
} else {
err = SyntaxError{x, self.s, types.ParsingError(-ret), ""}
self.err = err
return
}
}
err = self.Decoder.Decode(val)
if err != nil {
self.err = err
}

@ -108,4 +108,4 @@ func memclrHasPointers(ptr unsafe.Pointer, n uintptr)
//go:noescape
//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers
//goland:noinspection GoUnusedParameter
func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)

@ -1199,4 +1199,4 @@ func (self *_Assembler) WriteRecNotAX(i int, ptr obj.Addr, rec obj.Addr) {
self.Link("_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Emit("MOVQ", ptr, rec)
self.Link("_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
}
}

@ -202,4 +202,4 @@ func (self *_Assembler) print_ptr(i int, ptr obj.Addr, lea bool) {
self.Emit("MOVQ", _R10, _BX)
self.dcall(_F_printptr)
self.dload(_REG_debug...)
}
}

@ -292,7 +292,6 @@ func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
cfg := option.DefaultCompileOptions()
for _, opt := range opts {
opt(&cfg)
break
}
return pretouchRec(map[reflect.Type]uint8{vt: 0}, cfg)
}
@ -325,4 +324,4 @@ func Valid(data []byte) (ok bool, start int) {
}
return true, ret
}
}

@ -62,4 +62,4 @@ func goPanic(code int, val unsafe.Pointer) {
default:
panic("encoder error!")
}
}
}

@ -190,4 +190,4 @@ func pretouchRec(vtm map[reflect.Type]uint8, opts option.CompileOptions) error {
}
opts.RecursiveDepth -= 1
return pretouchRec(next, opts)
}
}

@ -165,4 +165,4 @@ func asJson(v unsafe.Pointer) (string, error) {
text := assertI2I(_T_json_Marshaler, *(*rt.GoIface)(v))
r, e := (*(*json.Marshaler)(unsafe.Pointer(&text))).MarshalJSON()
return rt.Mem2Str(r), e
}
}

@ -81,4 +81,4 @@ func (enc *StreamEncoder) Encode(val interface{}) (err error) {
free_bytes:
freeBytes(out)
return err
}
}

@ -63,4 +63,4 @@ func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
var _runtime_writeBarrier uintptr
//go:linkname gcWriteBarrierAX runtime.gcWriteBarrier
func gcWriteBarrierAX()
func gcWriteBarrierAX()

@ -132,4 +132,4 @@ func __validate_utf8(s *string, p *int, m *types.StateMachine) (ret int)
//go:nosplit
//go:noescape
//goland:noinspection GoUnusedParameter
func __validate_utf8_fast(s *string) (ret int)
func __validate_utf8_fast(s *string) (ret int)

@ -132,4 +132,4 @@ func __validate_utf8(s *string, p *int, m *types.StateMachine) (ret int)
//go:nosplit
//go:noescape
//goland:noinspection GoUnusedParameter
func __validate_utf8_fast(s *string) (ret int)
func __validate_utf8_fast(s *string) (ret int)

@ -130,4 +130,4 @@ func __validate_utf8(s *string, p *int, m *types.StateMachine) (ret int)
//go:nosplit
//go:noescape
//goland:noinspection GoUnusedParameter
func __validate_utf8_fast(s *string) (ret int)
func __validate_utf8_fast(s *string) (ret int)

@ -132,4 +132,4 @@ func __validate_utf8(s *string, p *int, m *types.StateMachine) (ret int)
//go:nosplit
//go:noescape
//goland:noinspection GoUnusedParameter
func __validate_utf8_fast(s *string) (ret int)
func __validate_utf8_fast(s *string) (ret int)

@ -42,6 +42,13 @@ const (
_SUB_BUCKETSIZE = _BUCKETSIZE / _SUBBUCKETS
)
// Note: This list must match the list in runtime/symtab.go.
const (
FuncFlag_TOPFRAME = 1 << iota
FuncFlag_SPWRITE
FuncFlag_ASM
)
// PCDATA and FUNCDATA table indexes.
//
// See funcdata.h and $GROOT/src/cmd/internal/objabi/funcdata.go.
@ -141,4 +148,98 @@ func funcNameParts(name string) (string, string, string) {
return name, "", ""
}
return name[:i], "[...]", name[j+1:]
}
}
// func name table format:
// nameOff[0] -> namePartA namePartB namePartC \x00
// nameOff[1] -> namePartA namePartB namePartC \x00
// ...
func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) {
offs = make([]int32, len(funcs))
offset := 1
tab = []byte{0}
for i, f := range funcs {
offs[i] = int32(offset)
a, b, c := funcNameParts(f.Name)
tab = append(tab, a...)
tab = append(tab, b...)
tab = append(tab, c...)
tab = append(tab, 0)
offset += len(a) + len(b) + len(c) + 1
}
return
}
// CU table format:
// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1]
// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1]
// ...
//
// file name table format:
// filetabOffset[0] -> CUs[0].fileNames[0] \x00
// ...
// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00
// ...
// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00
func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) {
cuOffsets = make([]uint32, len(cus))
cuOffset := 0
fileOffset := 0
for i, cu := range cus {
cuOffsets[i] = uint32(cuOffset)
for _, name := range cu.fileNames {
cutab = append(cutab, uint32(fileOffset))
fileOffset += len(name) + 1
filetab = append(filetab, name...)
filetab = append(filetab, 0)
}
cuOffset += len(cu.fileNames)
}
return
}
func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) {
fstart = len(*out)
*out = append(*out, byte(0))
offs := uint32(1)
funcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
var writer = func(fd encoding.BinaryMarshaler) {
var ab []byte
var err error
if fd != nil {
ab, err = fd.MarshalBinary()
if err != nil {
panic(err)
}
funcdataOffs[i] = append(funcdataOffs[i], offs)
} else {
ab = []byte{0}
funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET)
}
*out = append(*out, ab...)
offs += uint32(len(ab))
}
writer(f.ArgsPointerMaps)
writer(f.LocalsPointerMaps)
writer(f.StackObjects)
writer(f.InlTree)
writer(f.OpenCodedDeferInfo)
writer(f.ArgInfo)
writer(f.ArgLiveInfo)
writer(f.WrapInfo)
}
return
}

@ -1,5 +1,5 @@
//go:build go1.15 && !go1.16
// +build go1.15,!go1.16
//go:build !go1.16
// +build !go1.16
/*
* Copyright 2021 ByteDance Inc.
@ -20,11 +20,11 @@
package loader
import (
`encoding`
`os`
`unsafe`
`os`
`unsafe`
`sort`
`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/internal/rt`
)
const (
@ -171,99 +171,7 @@ type compilationUnit struct {
fileNames []string
}
// func name table format:
// nameOff[0] -> namePartA namePartB namePartC \x00
// nameOff[1] -> namePartA namePartB namePartC \x00
// ...
func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) {
offs = make([]int32, len(funcs))
offset := 0
for i, f := range funcs {
offs[i] = int32(offset)
a, b, c := funcNameParts(f.Name)
tab = append(tab, a...)
tab = append(tab, b...)
tab = append(tab, c...)
tab = append(tab, 0)
offset += len(a) + len(b) + len(c) + 1
}
return
}
// CU table format:
// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1]
// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1]
// ...
//
// file name table format:
// filetabOffset[0] -> CUs[0].fileNames[0] \x00
// ...
// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00
// ...
// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00
func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) {
cuOffsets = make([]uint32, len(cus))
cuOffset := 0
fileOffset := 0
for i, cu := range cus {
cuOffsets[i] = uint32(cuOffset)
for _, name := range cu.fileNames {
cutab = append(cutab, uint32(fileOffset))
fileOffset += len(name) + 1
filetab = append(filetab, name...)
filetab = append(filetab, 0)
}
cuOffset += len(cu.fileNames)
}
return
}
func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) {
fstart = len(*out)
*out = append(*out, byte(0))
offs := uint32(1)
funcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
var writer = func(fd encoding.BinaryMarshaler) {
var ab []byte
var err error
if fd != nil {
ab, err = fd.MarshalBinary()
if err != nil {
panic(err)
}
funcdataOffs[i] = append(funcdataOffs[i], offs)
} else {
ab = []byte{0}
funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET)
}
*out = append(*out, ab...)
offs += uint32(len(ab))
}
writer(f.ArgsPointerMaps)
writer(f.LocalsPointerMaps)
writer(f.StackObjects)
writer(f.InlTree)
writer(f.OpenCodedDeferInfo)
writer(f.ArgInfo)
writer(f.ArgLiveInfo)
writer(f.WrapInfo)
}
return
}
func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab, pclntabSize int64, startLocations []uint32) {
func makeFtab(funcs []_func, maxpc uintptr) (ftab []funcTab, pclntabSize int64, startLocations []uint32) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
@ -283,14 +191,12 @@ func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab, pclntabSize i
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
ftab = append(ftab, funcTab{lastFunc.entry + uintptr(lastFuncSize), 0})
ftab = append(ftab, funcTab{maxpc, 0})
return
}
// Pcln table format: [...]funcTab + [...]_Func
func makePclntable(size int64, startLocations []uint32, funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataAddr uintptr, funcdataOffs [][]uint32) (pclntab []byte) {
func makePclntable(size int64, startLocations []uint32, funcs []_func, maxpc uintptr, pcdataOffs [][]uint32, funcdataAddr uintptr, funcdataOffs [][]uint32) (pclntab []byte) {
pclntab = make([]byte, size, size)
// write a map of pc->func info offsets
@ -301,8 +207,7 @@ func makePclntable(size int64, startLocations []uint32, funcs []_func, lastFuncS
offs += 16
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
byteOrder.PutUint64(pclntab[offs:offs+8], uint64(lastFunc.entry)+uint64(lastFuncSize))
byteOrder.PutUint64(pclntab[offs:offs+8], uint64(maxpc))
offs += 8
// write func info table
@ -374,21 +279,22 @@ func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
tab := make([]findfuncbucket, 0, nbuckets)
var s, e = 0, 0
for i := 0; i<int(nbuckets); i++ {
var pc = min + uintptr((i+1)*_BUCKETSIZE)
// find the end func of the bucket
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
// store the start func of the bucket
var fb = findfuncbucket{idx: uint32(s)}
// find the last e-th func of the bucket
var pc = min + uintptr((i+1)*_BUCKETSIZE)
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
pc = min + uintptr(i*_BUCKETSIZE) + uintptr((j+1)*_SUB_BUCKETSIZE)
var ss = s
// find the end func of the subbucket
for ; ss < len(ftab)-1 && ftab[ss+1].entry <= pc; ss++ {}
// store the start func of the subbucket
fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
s = ss
// find the s-th end func of the subbucket
pc = min + uintptr(i*_BUCKETSIZE) + uintptr((j+1)*_SUB_BUCKETSIZE)
for ; s < len(ftab)-1 && ftab[s+1].entry <= pc; s++ {}
}
s = e
tab = append(tab, fb)
}
@ -401,15 +307,20 @@ func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
return
}
func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) {
func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte) (mod *moduledata) {
mod = new(moduledata)
mod.modulename = name
// sort funcs by entry
funcs := *funcsp
sort.Slice(funcs, func(i, j int) bool {
return funcs[i].EntryOff < funcs[j].EntryOff
})
*funcsp = funcs
// make filename table
cu := make([]string, 0, len(filenames))
for _, f := range filenames {
cu = append(cu, f)
}
cu = append(cu, filenames...)
cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
mod.cutab = cutab
mod.filetab = filetab
@ -428,9 +339,16 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte)
// make it executable
mprotect(addr, size)
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// make pcdata table
// NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
pctab, pcdataOffs, _funcs := makePctab(funcs, addr, cuOffs, nameOffs)
cuOff := cuOffs[0]
pctab, pcdataOffs, _funcs := makePctab(funcs, addr, cuOff, nameOffs)
mod.pctab = pctab
// write func data
@ -440,8 +358,7 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte)
fstart, funcdataOffs := writeFuncdata(&cache, funcs)
// make pc->func (binary search) func table
lastFuncsize := funcs[len(funcs)-1].TextSize
ftab, pclntSize, startLocations := makeFtab(_funcs, lastFuncsize)
ftab, pclntSize, startLocations := makeFtab(_funcs, mod.maxpc)
mod.ftab = ftab
// write pc->func (modmap) findfunc table
@ -455,15 +372,9 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte)
funcdataAddr := uintptr(rt.IndexByte(cache, fstart))
// make pclnt table
pclntab := makePclntable(pclntSize, startLocations, _funcs, lastFuncsize, pcdataOffs, funcdataAddr, funcdataOffs)
pclntab := makePclntable(pclntSize, startLocations, _funcs, mod.maxpc, pcdataOffs, funcdataAddr, funcdataOffs)
mod.pclntable = pclntab
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// make pc header
mod.pcHeader = &pcHeader {
magic : _Magic,
@ -487,7 +398,7 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte)
// makePctab generates pcdelta->valuedelta tables for functions,
// and returns the table and the entry offset of every kind pcdata in the table.
func makePctab(funcs []Func, addr uintptr, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
func makePctab(funcs []Func, addr uintptr, cuOffset uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
_funcs = make([]_func, len(funcs))
// Pctab offsets of 0 are considered invalid in the runtime. We respect
@ -538,7 +449,7 @@ func makePctab(funcs []Func, addr uintptr, cuOffset []uint32, nameOffset []int32
_f.deferreturn = f.DeferReturn
// NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
_f.npcdata = uint32(_N_PCDATA)
_f.cuOffset = cuOffset[i]
_f.cuOffset = cuOffset
_f.funcID = f.ID
_f.nfuncdata = uint8(_N_FUNCDATA)
}
@ -546,4 +457,4 @@ func makePctab(funcs []Func, addr uintptr, cuOffset []uint32, nameOffset []int32
return
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}

@ -20,9 +20,9 @@
package loader
import (
`encoding`
`os`
`unsafe`
`sort`
`github.com/bytedance/sonic/internal/rt`
)
@ -171,99 +171,7 @@ type compilationUnit struct {
fileNames []string
}
// func name table format:
// nameOff[0] -> namePartA namePartB namePartC \x00
// nameOff[1] -> namePartA namePartB namePartC \x00
// ...
func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) {
offs = make([]int32, len(funcs))
offset := 0
for i, f := range funcs {
offs[i] = int32(offset)
a, b, c := funcNameParts(f.Name)
tab = append(tab, a...)
tab = append(tab, b...)
tab = append(tab, c...)
tab = append(tab, 0)
offset += len(a) + len(b) + len(c) + 1
}
return
}
// CU table format:
// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1]
// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1]
// ...
//
// file name table format:
// filetabOffset[0] -> CUs[0].fileNames[0] \x00
// ...
// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00
// ...
// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00
func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) {
cuOffsets = make([]uint32, len(cus))
cuOffset := 0
fileOffset := 0
for i, cu := range cus {
cuOffsets[i] = uint32(cuOffset)
for _, name := range cu.fileNames {
cutab = append(cutab, uint32(fileOffset))
fileOffset += len(name) + 1
filetab = append(filetab, name...)
filetab = append(filetab, 0)
}
cuOffset += len(cu.fileNames)
}
return
}
func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) {
fstart = len(*out)
*out = append(*out, byte(0))
offs := uint32(1)
funcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
var writer = func(fd encoding.BinaryMarshaler) {
var ab []byte
var err error
if fd != nil {
ab, err = fd.MarshalBinary()
if err != nil {
panic(err)
}
funcdataOffs[i] = append(funcdataOffs[i], offs)
} else {
ab = []byte{0}
funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET)
}
*out = append(*out, ab...)
offs += uint32(len(ab))
}
writer(f.ArgsPointerMaps)
writer(f.LocalsPointerMaps)
writer(f.StackObjects)
writer(f.InlTree)
writer(f.OpenCodedDeferInfo)
writer(f.ArgInfo)
writer(f.ArgLiveInfo)
writer(f.WrapInfo)
}
return
}
func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab, pclntabSize int64, startLocations []uint32) {
func makeFtab(funcs []_func, maxpc uintptr) (ftab []funcTab, pclntabSize int64, startLocations []uint32) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
@ -283,14 +191,13 @@ func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab, pclntabSize i
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
ftab = append(ftab, funcTab{lastFunc.entry + uintptr(lastFuncSize), 0})
ftab = append(ftab, funcTab{maxpc, 0})
return
}
// Pcln table format: [...]funcTab + [...]_Func
func makePclntable(size int64, startLocations []uint32, funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataAddr uintptr, funcdataOffs [][]uint32) (pclntab []byte) {
func makePclntable(size int64, startLocations []uint32, funcs []_func, maxpc uintptr, pcdataOffs [][]uint32, funcdataAddr uintptr, funcdataOffs [][]uint32) (pclntab []byte) {
pclntab = make([]byte, size, size)
// write a map of pc->func info offsets
@ -301,8 +208,7 @@ func makePclntable(size int64, startLocations []uint32, funcs []_func, lastFuncS
offs += 16
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
byteOrder.PutUint64(pclntab[offs:offs+8], uint64(lastFunc.entry)+uint64(lastFuncSize))
byteOrder.PutUint64(pclntab[offs:offs+8], uint64(maxpc))
offs += 8
// write func info table
@ -374,21 +280,22 @@ func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
tab := make([]findfuncbucket, 0, nbuckets)
var s, e = 0, 0
for i := 0; i<int(nbuckets); i++ {
var pc = min + uintptr((i+1)*_BUCKETSIZE)
// find the end func of the bucket
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
// store the start func of the bucket
var fb = findfuncbucket{idx: uint32(s)}
// find the last e-th func of the bucket
var pc = min + uintptr((i+1)*_BUCKETSIZE)
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
pc = min + uintptr(i*_BUCKETSIZE) + uintptr((j+1)*_SUB_BUCKETSIZE)
var ss = s
// find the end func of the subbucket
for ; ss < len(ftab)-1 && ftab[ss+1].entry <= pc; ss++ {}
// store the start func of the subbucket
fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
s = ss
// find the s-th end func of the subbucket
pc = min + uintptr(i*_BUCKETSIZE) + uintptr((j+1)*_SUB_BUCKETSIZE)
for ; s < len(ftab)-1 && ftab[s+1].entry <= pc; s++ {}
}
s = e
tab = append(tab, fb)
}
@ -401,15 +308,20 @@ func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
return
}
func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) {
func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte) (mod *moduledata) {
mod = new(moduledata)
mod.modulename = name
// sort funcs by entry
funcs := *funcsp
sort.Slice(funcs, func(i, j int) bool {
return funcs[i].EntryOff < funcs[j].EntryOff
})
*funcsp = funcs
// make filename table
cu := make([]string, 0, len(filenames))
for _, f := range filenames {
cu = append(cu, f)
}
cu = append(cu, filenames...)
cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
mod.cutab = cutab
mod.filetab = filetab
@ -428,9 +340,16 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte)
// make it executable
mprotect(addr, size)
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// make pcdata table
// NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
pctab, pcdataOffs, _funcs := makePctab(funcs, addr, cuOffs, nameOffs)
cuOff := cuOffs[0]
pctab, pcdataOffs, _funcs := makePctab(funcs, addr, cuOff, nameOffs)
mod.pctab = pctab
// write func data
@ -440,8 +359,7 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte)
fstart, funcdataOffs := writeFuncdata(&cache, funcs)
// make pc->func (binary search) func table
lastFuncsize := funcs[len(funcs)-1].TextSize
ftab, pclntSize, startLocations := makeFtab(_funcs, lastFuncsize)
ftab, pclntSize, startLocations := makeFtab(_funcs, mod.maxpc)
mod.ftab = ftab
// write pc->func (modmap) findfunc table
@ -455,15 +373,9 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte)
funcdataAddr := uintptr(rt.IndexByte(cache, fstart))
// make pclnt table
pclntab := makePclntable(pclntSize, startLocations, _funcs, lastFuncsize, pcdataOffs, funcdataAddr, funcdataOffs)
pclntab := makePclntable(pclntSize, startLocations, _funcs, mod.maxpc, pcdataOffs, funcdataAddr, funcdataOffs)
mod.pclntable = pclntab
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// make pc header
mod.pcHeader = &pcHeader {
magic : _Magic,
@ -487,7 +399,7 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte)
// makePctab generates pcdelta->valuedelta tables for functions,
// and returns the table and the entry offset of every kind pcdata in the table.
func makePctab(funcs []Func, addr uintptr, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
func makePctab(funcs []Func, addr uintptr, cuOffset uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
_funcs = make([]_func, len(funcs))
// Pctab offsets of 0 are considered invalid in the runtime. We respect
@ -538,7 +450,7 @@ func makePctab(funcs []Func, addr uintptr, cuOffset []uint32, nameOffset []int32
_f.deferreturn = f.DeferReturn
// NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
_f.npcdata = uint32(_N_PCDATA)
_f.cuOffset = cuOffset[i]
_f.cuOffset = cuOffset
_f.funcID = f.ID
_f.nfuncdata = uint8(_N_FUNCDATA)
}
@ -546,4 +458,4 @@ func makePctab(funcs []Func, addr uintptr, cuOffset []uint32, nameOffset []int32
return
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}

@ -1,4 +1,5 @@
// go:build go1.18 && !go1.20
//go:build go1.18 && !go1.20
// +build go1.18,!go1.20
/*
@ -20,10 +21,6 @@
package loader
import (
`encoding`
`os`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
@ -31,21 +28,6 @@ const (
_Magic uint32 = 0xfffffff0
)
type pcHeader struct {
magic uint32 // 0xFFFFFFF0
pad1, pad2 uint8 // 0,0
minLC uint8 // min instruction size
ptrSize uint8 // size of a ptr in bytes
nfunc int // number of functions in the module
nfiles uint // number of entries in the file tab
textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text
funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
cuOffset uintptr // offset to the cutab variable from pcHeader
filetabOffset uintptr // offset to the filetab variable from pcHeader
pctabOffset uintptr // offset to the pctab variable from pcHeader
pclnOffset uintptr // offset to the pclntab variable from pcHeader
}
type moduledata struct {
pcHeader *pcHeader
funcnametab []byte
@ -129,413 +111,3 @@ type _func struct {
//
// funcdata [nfuncdata]uint32
}
type funcTab struct {
entry uint32
funcoff uint32
}
type bitVector struct {
n int32 // # of bits
bytedata *uint8
}
type ptabEntry struct {
name int32
typ int32
}
type textSection struct {
vaddr uintptr // prelinked section vaddr
end uintptr // vaddr + section length
baseaddr uintptr // relocated section address
}
type modulehash struct {
modulename string
linktimehash string
runtimehash *string
}
// findfuncbucket is an array of these structures.
// Each bucket represents 4096 bytes of the text segment.
// Each subbucket represents 256 bytes of the text segment.
// To find a function given a pc, locate the bucket and subbucket for
// that pc. Add together the idx and subbucket value to obtain a
// function index. Then scan the functab array starting at that
// index to find the target function.
// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
type findfuncbucket struct {
idx uint32
_SUBBUCKETS [16]byte
}
// func name table format:
// nameOff[0] -> namePartA namePartB namePartC \x00
// nameOff[1] -> namePartA namePartB namePartC \x00
// ...
func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) {
offs = make([]int32, len(funcs))
offset := 0
for i, f := range funcs {
offs[i] = int32(offset)
a, b, c := funcNameParts(f.Name)
tab = append(tab, a...)
tab = append(tab, b...)
tab = append(tab, c...)
tab = append(tab, 0)
offset += len(a) + len(b) + len(c) + 1
}
return
}
type compilationUnit struct {
fileNames []string
}
// CU table format:
// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1]
// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1]
// ...
//
// file name table format:
// filetabOffset[0] -> CUs[0].fileNames[0] \x00
// ...
// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00
// ...
// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00
func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) {
cuOffsets = make([]uint32, len(cus))
cuOffset := 0
fileOffset := 0
for i, cu := range cus {
cuOffsets[i] = uint32(cuOffset)
for _, name := range cu.fileNames {
cutab = append(cutab, uint32(fileOffset))
fileOffset += len(name) + 1
filetab = append(filetab, name...)
filetab = append(filetab, 0)
}
cuOffset += len(cu.fileNames)
}
return
}
func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) {
fstart = len(*out)
*out = append(*out, byte(0))
offs := uint32(1)
funcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
var writer = func(fd encoding.BinaryMarshaler) {
var ab []byte
var err error
if fd != nil {
ab, err = fd.MarshalBinary()
if err != nil {
panic(err)
}
funcdataOffs[i] = append(funcdataOffs[i], offs)
} else {
ab = []byte{0}
funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET)
}
*out = append(*out, ab...)
offs += uint32(len(ab))
}
writer(f.ArgsPointerMaps)
writer(f.LocalsPointerMaps)
writer(f.StackObjects)
writer(f.InlTree)
writer(f.OpenCodedDeferInfo)
writer(f.ArgInfo)
writer(f.ArgLiveInfo)
writer(f.WrapInfo)
}
return
}
func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i, f := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4)
}
ftab = make([]funcTab, 0, len(funcs)+1)
// write a map of pc->func info offsets
for i, f := range funcs {
ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])})
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0})
return
}
// Pcln table format: [...]funcTab + [...]_Func
func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4)
}
pclntab = make([]byte, size, size)
// write a map of pc->func info offsets
offs := 0
for i, f := range funcs {
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff))
byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i]))
offs += 8
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize))
// write func info table
for i, f := range funcs {
off := startLocations[i]
// write _func structure to pclntab
fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE))
copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb)
off += uint32(_FUNC_SIZE)
// NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3
for j := 3; j < len(pcdataOffs[i]); j++ {
byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j]))
off += 4
}
// funcdata refs as offsets from gofunc
for _, funcdata := range funcdataOffs[i] {
byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata))
off += 4
}
}
return
}
// findfunc table used to map pc to belonging func,
// returns the index in the func table.
//
// All text section are divided into buckets sized _BUCKETSIZE(4K):
// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64),
// and it has a base idx to plus the offset stored in jth subbucket.
// see findfunc() in runtime/symtab.go
func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
start = len(*out)
max := ftab[len(ftab)-1].entry
min := ftab[0].entry
nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE
n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE
tab := make([]findfuncbucket, 0, nbuckets)
var s, e = 0, 0
for i := 0; i<int(nbuckets); i++ {
var pc = min + uint32((i+1)*_BUCKETSIZE)
// find the end func of the bucket
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
// store the start func of the bucket
var fb = findfuncbucket{idx: uint32(s)}
for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
pc = min + uint32(i*_BUCKETSIZE) + uint32((j+1)*_SUB_BUCKETSIZE)
var ss = s
// find the end func of the subbucket
for ; ss < len(ftab)-1 && ftab[ss+1].entry <= pc; ss++ {}
// store the start func of the subbucket
fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
s = ss
}
s = e
tab = append(tab, fb)
}
// write findfuncbucket
if len(tab) > 0 {
size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab)
*out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...)
}
return
}
func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) {
mod = new(moduledata)
mod.modulename = name
// make filename table
cu := make([]string, 0, len(filenames))
for _, f := range filenames {
cu = append(cu, f)
}
cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
mod.cutab = cutab
mod.filetab = filetab
// make funcname table
funcnametab, nameOffs := makeFuncnameTab(funcs)
mod.funcnametab = funcnametab
// make pcdata table
// NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
pctab, pcdataOffs, _funcs := makePctab(funcs, cuOffs, nameOffs)
mod.pctab = pctab
// write func data
// NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata
// TODO: estimate accurate capacity
cache := make([]byte, 0, len(funcs)*int(_PtrSize))
fstart, funcdataOffs := writeFuncdata(&cache, funcs)
// make pc->func (binary search) func table
lastFuncsize := funcs[len(funcs)-1].TextSize
ftab := makeFtab(_funcs, lastFuncsize)
mod.ftab = ftab
// write pc->func (modmap) findfunc table
ffstart := writeFindfunctab(&cache, ftab)
// make pclnt table
pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs)
mod.pclntable = pclntab
// mmap() text and funcdata segements
p := os.Getpagesize()
size := int(rnd(int64(len(text)), int64(p)))
addr := mmap(size)
// copy the machine code
s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size)
copy(s, text)
// make it executable
mprotect(addr, size)
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// cache funcdata and findfuncbucket
moduleCache.Lock()
moduleCache.m[mod] = cache
moduleCache.Unlock()
mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart]))
mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart]))
// make pc header
mod.pcHeader = &pcHeader {
magic : _Magic,
minLC : _MinLC,
ptrSize : _PtrSize,
nfunc : len(funcs),
nfiles: uint(len(cu)),
textStart: mod.text,
funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"),
cuOffset: getOffsetOf(moduledata{}, "cutab"),
filetabOffset: getOffsetOf(moduledata{}, "filetab"),
pctabOffset: getOffsetOf(moduledata{}, "pctab"),
pclnOffset: getOffsetOf(moduledata{}, "pclntable"),
}
// sepecial case: gcdata and gcbss must by non-empty
mod.gcdata = uintptr(unsafe.Pointer(&emptyByte))
mod.gcbss = uintptr(unsafe.Pointer(&emptyByte))
return
}
// makePctab generates pcdelta->valuedelta tables for functions,
// and returns the table and the entry offset of every kind pcdata in the table.
func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
_funcs = make([]_func, len(funcs))
// Pctab offsets of 0 are considered invalid in the runtime. We respect
// that by just padding a single byte at the beginning of runtime.pctab,
// that way no real offsets can be zero.
pctab = make([]byte, 1, 12*len(funcs)+1)
pcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
_f := &_funcs[i]
var writer = func(pc *Pcdata) {
var ab []byte
var err error
if pc != nil {
ab, err = pc.MarshalBinary()
if err != nil {
panic(err)
}
pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab)))
} else {
ab = []byte{0}
pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET)
}
pctab = append(pctab, ab...)
}
if f.Pcsp != nil {
_f.pcsp = uint32(len(pctab))
}
writer(f.Pcsp)
if f.Pcfile != nil {
_f.pcfile = uint32(len(pctab))
}
writer(f.Pcfile)
if f.Pcline != nil {
_f.pcln = uint32(len(pctab))
}
writer(f.Pcline)
writer(f.PcUnsafePoint)
writer(f.PcStackMapIndex)
writer(f.PcInlTreeIndex)
writer(f.PcArgLiveIndex)
_f.entryOff = f.EntryOff
_f.nameOff = nameOffset[i]
_f.args = f.ArgsSize
_f.deferreturn = f.DeferReturn
// NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
_f.npcdata = uint32(_N_PCDATA)
_f.cuOffset = cuOffset[i]
_f.funcID = f.ID
_f.flag = f.Flag
_f.nfuncdata = uint8(_N_FUNCDATA)
}
return
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}

@ -20,10 +20,6 @@
package loader
import (
`encoding`
`os`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
@ -51,8 +47,6 @@ type moduledata struct {
end, gcdata, gcbss uintptr
types, etypes uintptr
rodata uintptr
// TODO: generate funcinfo object to memory
gofunc uintptr // go.func.* is actual funcinfo object in image
textsectmap []textSection // see runtime/symtab.go: textAddr()
@ -118,428 +112,3 @@ type _func struct {
//
// funcdata [nfuncdata]uint32
}
type funcTab struct {
entry uint32
funcoff uint32
}
type pcHeader struct {
magic uint32 // 0xFFFFFFF0
pad1, pad2 uint8 // 0,0
minLC uint8 // min instruction size
ptrSize uint8 // size of a ptr in bytes
nfunc int // number of functions in the module
nfiles uint // number of entries in the file tab
textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text
funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
cuOffset uintptr // offset to the cutab variable from pcHeader
filetabOffset uintptr // offset to the filetab variable from pcHeader
pctabOffset uintptr // offset to the pctab variable from pcHeader
pclnOffset uintptr // offset to the pclntab variable from pcHeader
}
type bitVector struct {
n int32 // # of bits
bytedata *uint8
}
type ptabEntry struct {
name int32
typ int32
}
type textSection struct {
vaddr uintptr // prelinked section vaddr
end uintptr // vaddr + section length
baseaddr uintptr // relocated section address
}
type modulehash struct {
modulename string
linktimehash string
runtimehash *string
}
// findfuncbucket is an array of these structures.
// Each bucket represents 4096 bytes of the text segment.
// Each subbucket represents 256 bytes of the text segment.
// To find a function given a pc, locate the bucket and subbucket for
// that pc. Add together the idx and subbucket value to obtain a
// function index. Then scan the functab array starting at that
// index to find the target function.
// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
type findfuncbucket struct {
idx uint32
_SUBBUCKETS [16]byte
}
// func name table format:
// nameOff[0] -> namePartA namePartB namePartC \x00
// nameOff[1] -> namePartA namePartB namePartC \x00
// ...
func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) {
offs = make([]int32, len(funcs))
offset := 0
for i, f := range funcs {
offs[i] = int32(offset)
a, b, c := funcNameParts(f.Name)
tab = append(tab, a...)
tab = append(tab, b...)
tab = append(tab, c...)
tab = append(tab, 0)
offset += len(a) + len(b) + len(c) + 1
}
return
}
type compilationUnit struct {
fileNames []string
}
// CU table format:
// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1]
// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1]
// ...
//
// file name table format:
// filetabOffset[0] -> CUs[0].fileNames[0] \x00
// ...
// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00
// ...
// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00
func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) {
cuOffsets = make([]uint32, len(cus))
cuOffset := 0
fileOffset := 0
for i, cu := range cus {
cuOffsets[i] = uint32(cuOffset)
for _, name := range cu.fileNames {
cutab = append(cutab, uint32(fileOffset))
fileOffset += len(name) + 1
filetab = append(filetab, name...)
filetab = append(filetab, 0)
}
cuOffset += len(cu.fileNames)
}
return
}
func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) {
fstart = len(*out)
*out = append(*out, byte(0))
offs := uint32(1)
funcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
var writer = func(fd encoding.BinaryMarshaler) {
var ab []byte
var err error
if fd != nil {
ab, err = fd.MarshalBinary()
if err != nil {
panic(err)
}
funcdataOffs[i] = append(funcdataOffs[i], offs)
} else {
ab = []byte{0}
funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET)
}
*out = append(*out, ab...)
offs += uint32(len(ab))
}
writer(f.ArgsPointerMaps)
writer(f.LocalsPointerMaps)
writer(f.StackObjects)
writer(f.InlTree)
writer(f.OpenCodedDeferInfo)
writer(f.ArgInfo)
writer(f.ArgLiveInfo)
writer(f.WrapInfo)
}
return
}
func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i, f := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4)
}
ftab = make([]funcTab, 0, len(funcs)+1)
// write a map of pc->func info offsets
for i, f := range funcs {
ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])})
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0})
return
}
// Pcln table format: [...]funcTab + [...]_Func
func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4)
}
pclntab = make([]byte, size, size)
// write a map of pc->func info offsets
offs := 0
for i, f := range funcs {
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff))
byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i]))
offs += 8
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize))
// write func info table
for i, f := range funcs {
off := startLocations[i]
// write _func structure to pclntab
fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE))
copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb)
off += uint32(_FUNC_SIZE)
// NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3
for j := 3; j < len(pcdataOffs[i]); j++ {
byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j]))
off += 4
}
// funcdata refs as offsets from gofunc
for _, funcdata := range funcdataOffs[i] {
byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata))
off += 4
}
}
return
}
// findfunc table used to map pc to belonging func,
// returns the index in the func table.
//
// All text section are divided into buckets sized _BUCKETSIZE(4K):
// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64),
// and it has a base idx to plus the offset stored in jth subbucket.
// see findfunc() in runtime/symtab.go
func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
start = len(*out)
max := ftab[len(ftab)-1].entry
min := ftab[0].entry
nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE
n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE
tab := make([]findfuncbucket, 0, nbuckets)
var s, e = 0, 0
for i := 0; i<int(nbuckets); i++ {
var pc = min + uint32((i+1)*_BUCKETSIZE)
// find the end func of the bucket
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
// store the start func of the bucket
var fb = findfuncbucket{idx: uint32(s)}
for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
pc = min + uint32(i*_BUCKETSIZE) + uint32((j+1)*_SUB_BUCKETSIZE)
var ss = s
// find the end func of the subbucket
for ; ss < len(ftab)-1 && ftab[ss+1].entry <= pc; ss++ {}
// store the start func of the subbucket
fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
s = ss
}
s = e
tab = append(tab, fb)
}
// write findfuncbucket
if len(tab) > 0 {
size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab)
*out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...)
}
return
}
func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) {
mod = new(moduledata)
mod.modulename = name
// make filename table
cu := make([]string, 0, len(filenames))
for _, f := range filenames {
cu = append(cu, f)
}
cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
mod.cutab = cutab
mod.filetab = filetab
// make funcname table
funcnametab, nameOffs := makeFuncnameTab(funcs)
mod.funcnametab = funcnametab
// make pcdata table
// NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
pctab, pcdataOffs, _funcs := makePctab(funcs, cuOffs, nameOffs)
mod.pctab = pctab
// write func data
// NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata
// TODO: estimate accurate capacity
cache := make([]byte, 0, len(funcs)*int(_PtrSize))
fstart, funcdataOffs := writeFuncdata(&cache, funcs)
// make pc->func (binary search) func table
lastFuncsize := funcs[len(funcs)-1].TextSize
ftab := makeFtab(_funcs, lastFuncsize)
mod.ftab = ftab
// write pc->func (modmap) findfunc table
ffstart := writeFindfunctab(&cache, ftab)
// make pclnt table
pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs)
mod.pclntable = pclntab
// mmap() text and funcdata segements
p := os.Getpagesize()
size := int(rnd(int64(len(text)), int64(p)))
addr := mmap(size)
// copy the machine code
s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size)
copy(s, text)
// make it executable
mprotect(addr, size)
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// cache funcdata and findfuncbucket
moduleCache.Lock()
moduleCache.m[mod] = cache
moduleCache.Unlock()
mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart]))
mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart]))
// make pc header
mod.pcHeader = &pcHeader {
magic : _Magic,
minLC : _MinLC,
ptrSize : _PtrSize,
nfunc : len(funcs),
nfiles: uint(len(cu)),
textStart: mod.text,
funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"),
cuOffset: getOffsetOf(moduledata{}, "cutab"),
filetabOffset: getOffsetOf(moduledata{}, "filetab"),
pctabOffset: getOffsetOf(moduledata{}, "pctab"),
pclnOffset: getOffsetOf(moduledata{}, "pclntable"),
}
// sepecial case: gcdata and gcbss must by non-empty
mod.gcdata = uintptr(unsafe.Pointer(&emptyByte))
mod.gcbss = uintptr(unsafe.Pointer(&emptyByte))
return
}
// makePctab generates pcdelta->valuedelta tables for functions,
// and returns the table and the entry offset of every kind pcdata in the table.
func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
_funcs = make([]_func, len(funcs))
// Pctab offsets of 0 are considered invalid in the runtime. We respect
// that by just padding a single byte at the beginning of runtime.pctab,
// that way no real offsets can be zero.
pctab = make([]byte, 1, 12*len(funcs)+1)
pcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
_f := &_funcs[i]
var writer = func(pc *Pcdata) {
var ab []byte
var err error
if pc != nil {
ab, err = pc.MarshalBinary()
if err != nil {
panic(err)
}
pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab)))
} else {
ab = []byte{0}
pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET)
}
pctab = append(pctab, ab...)
}
if f.Pcsp != nil {
_f.pcsp = uint32(len(pctab))
}
writer(f.Pcsp)
if f.Pcfile != nil {
_f.pcfile = uint32(len(pctab))
}
writer(f.Pcfile)
if f.Pcline != nil {
_f.pcln = uint32(len(pctab))
}
writer(f.Pcline)
writer(f.PcUnsafePoint)
writer(f.PcStackMapIndex)
writer(f.PcInlTreeIndex)
writer(f.PcArgLiveIndex)
_f.entryOff = f.EntryOff
_f.nameOff = nameOffset[i]
_f.args = f.ArgsSize
_f.deferreturn = f.DeferReturn
// NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
_f.npcdata = uint32(_N_PCDATA)
_f.cuOffset = cuOffset[i]
_f.funcID = f.ID
_f.flag = f.Flag
_f.nfuncdata = uint8(_N_FUNCDATA)
}
return
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}

@ -0,0 +1,355 @@
// go:build go1.18 && !go1.21
// +build go1.18,!go1.21
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`os`
`sort`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
type funcTab struct {
entry uint32
funcoff uint32
}
type pcHeader struct {
magic uint32 // 0xFFFFFFF0
pad1, pad2 uint8 // 0,0
minLC uint8 // min instruction size
ptrSize uint8 // size of a ptr in bytes
nfunc int // number of functions in the module
nfiles uint // number of entries in the file tab
textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text
funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
cuOffset uintptr // offset to the cutab variable from pcHeader
filetabOffset uintptr // offset to the filetab variable from pcHeader
pctabOffset uintptr // offset to the pctab variable from pcHeader
pclnOffset uintptr // offset to the pclntab variable from pcHeader
}
type bitVector struct {
n int32 // # of bits
bytedata *uint8
}
type ptabEntry struct {
name int32
typ int32
}
type textSection struct {
vaddr uintptr // prelinked section vaddr
end uintptr // vaddr + section length
baseaddr uintptr // relocated section address
}
type modulehash struct {
modulename string
linktimehash string
runtimehash *string
}
// findfuncbucket is an array of these structures.
// Each bucket represents 4096 bytes of the text segment.
// Each subbucket represents 256 bytes of the text segment.
// To find a function given a pc, locate the bucket and subbucket for
// that pc. Add together the idx and subbucket value to obtain a
// function index. Then scan the functab array starting at that
// index to find the target function.
// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
type findfuncbucket struct {
idx uint32
_SUBBUCKETS [16]byte
}
type compilationUnit struct {
fileNames []string
}
func makeFtab(funcs []_func, maxpc uint32) (ftab []funcTab, pclntabSize int64, startLocations []uint32) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
pclntabSize = int64(len(funcs)*2*int(_PtrSize) + int(_PtrSize))
startLocations = make([]uint32, len(funcs))
for i, f := range funcs {
pclntabSize = rnd(pclntabSize, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(pclntabSize)
pclntabSize += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4)
}
ftab = make([]funcTab, 0, len(funcs)+1)
// write a map of pc->func info offsets
for i, f := range funcs {
ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])})
}
// Final entry of table is just end pc offset.
ftab = append(ftab, funcTab{maxpc, 0})
return
}
// Pcln table format: [...]funcTab + [...]_Func
func makePclntable(size int64, startLocations []uint32, funcs []_func, maxpc uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
pclntab = make([]byte, size, size)
// write a map of pc->func info offsets
offs := 0
for i, f := range funcs {
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff))
byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i]))
offs += 8
}
// Final entry of table is just end pc offset.
byteOrder.PutUint32(pclntab[offs:offs+4], maxpc)
// write func info table
for i := range funcs {
off := startLocations[i]
// write _func structure to pclntab
fb := rt.BytesFrom(unsafe.Pointer(&funcs[i]), int(_FUNC_SIZE), int(_FUNC_SIZE))
copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb)
off += uint32(_FUNC_SIZE)
// NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3
for j := 3; j < len(pcdataOffs[i]); j++ {
byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j]))
off += 4
}
// funcdata refs as offsets from gofunc
for _, funcdata := range funcdataOffs[i] {
byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata))
off += 4
}
}
return
}
// findfunc table used to map pc to belonging func,
// returns the index in the func table.
//
// All text section are divided into buckets sized _BUCKETSIZE(4K):
// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64),
// and it has a base idx to plus the offset stored in jth subbucket.
// see findfunc() in runtime/symtab.go
func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
start = len(*out)
max := ftab[len(ftab)-1].entry
min := ftab[0].entry
nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE
n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE
tab := make([]findfuncbucket, 0, nbuckets)
var s, e = 0, 0
for i := 0; i<int(nbuckets); i++ {
// store the start s-th func of the bucket
var fb = findfuncbucket{idx: uint32(s)}
// find the last e-th func of the bucket
var pc = min + uint32((i+1)*_BUCKETSIZE)
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
// store the start func of the subbucket
fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
// find the s-th end func of the subbucket
pc = min + uint32(i*_BUCKETSIZE) + uint32((j+1)*_SUB_BUCKETSIZE)
for ; s < len(ftab)-1 && ftab[s+1].entry <= pc; s++ {}
}
s = e
tab = append(tab, fb)
}
// write findfuncbucket
if len(tab) > 0 {
size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab)
*out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...)
}
return
}
func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte) (mod *moduledata) {
mod = new(moduledata)
mod.modulename = name
// sort funcs by entry
funcs := *funcsp
sort.Slice(funcs, func(i, j int) bool {
return funcs[i].EntryOff < funcs[j].EntryOff
})
*funcsp = funcs
// make filename table
cu := make([]string, 0, len(filenames))
cu = append(cu, filenames...)
cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
mod.cutab = cutab
mod.filetab = filetab
// make funcname table
funcnametab, nameOffs := makeFuncnameTab(funcs)
mod.funcnametab = funcnametab
// mmap() text and funcdata segements
p := os.Getpagesize()
size := int(rnd(int64(len(text)), int64(p)))
addr := mmap(size)
// copy the machine code
s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size)
copy(s, text)
// make it executable
mprotect(addr, size)
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// make pcdata table
// NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
cuOff := cuOffs[0]
pctab, pcdataOffs, _funcs := makePctab(funcs, cuOff, nameOffs)
mod.pctab = pctab
// write func data
// NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata
// TODO: estimate accurate capacity
cache := make([]byte, 0, len(funcs)*int(_PtrSize))
fstart, funcdataOffs := writeFuncdata(&cache, funcs)
// make pc->func (binary search) func table
ftab, pclntSize, startLocations := makeFtab(_funcs, uint32(len(text)))
mod.ftab = ftab
// write pc->func (modmap) findfunc table
ffstart := writeFindfunctab(&cache, ftab)
// cache funcdata and findfuncbucket
moduleCache.Lock()
moduleCache.m[mod] = cache
moduleCache.Unlock()
mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart]))
mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart]))
// make pclnt table
pclntab := makePclntable(pclntSize, startLocations, _funcs, uint32(len(text)), pcdataOffs, funcdataOffs)
mod.pclntable = pclntab
// make pc header
mod.pcHeader = &pcHeader {
magic : _Magic,
minLC : _MinLC,
ptrSize : _PtrSize,
nfunc : len(funcs),
nfiles: uint(len(cu)),
textStart: mod.text,
funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"),
cuOffset: getOffsetOf(moduledata{}, "cutab"),
filetabOffset: getOffsetOf(moduledata{}, "filetab"),
pctabOffset: getOffsetOf(moduledata{}, "pctab"),
pclnOffset: getOffsetOf(moduledata{}, "pclntable"),
}
// sepecial case: gcdata and gcbss must by non-empty
mod.gcdata = uintptr(unsafe.Pointer(&emptyByte))
mod.gcbss = uintptr(unsafe.Pointer(&emptyByte))
return
}
// makePctab generates pcdelta->valuedelta tables for functions,
// and returns the table and the entry offset of every kind pcdata in the table.
func makePctab(funcs []Func, cuOffset uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
_funcs = make([]_func, len(funcs))
// Pctab offsets of 0 are considered invalid in the runtime. We respect
// that by just padding a single byte at the beginning of runtime.pctab,
// that way no real offsets can be zero.
pctab = make([]byte, 1, 12*len(funcs)+1)
pcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
_f := &_funcs[i]
var writer = func(pc *Pcdata) {
var ab []byte
var err error
if pc != nil {
ab, err = pc.MarshalBinary()
if err != nil {
panic(err)
}
pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab)))
} else {
ab = []byte{0}
pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET)
}
pctab = append(pctab, ab...)
}
if f.Pcsp != nil {
_f.pcsp = uint32(len(pctab))
}
writer(f.Pcsp)
if f.Pcfile != nil {
_f.pcfile = uint32(len(pctab))
}
writer(f.Pcfile)
if f.Pcline != nil {
_f.pcln = uint32(len(pctab))
}
writer(f.Pcline)
writer(f.PcUnsafePoint)
writer(f.PcStackMapIndex)
writer(f.PcInlTreeIndex)
writer(f.PcArgLiveIndex)
_f.entryOff = f.EntryOff
_f.nameOff = nameOffset[i]
_f.args = f.ArgsSize
_f.deferreturn = f.DeferReturn
// NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
_f.npcdata = uint32(_N_PCDATA)
_f.cuOffset = cuOffset
_f.funcID = f.ID
_f.flag = f.Flag
_f.nfuncdata = uint8(_N_FUNCDATA)
}
return
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}

@ -34,4 +34,4 @@ type Loader struct {
Name string // module name
File string // file name
Options
}
}

@ -0,0 +1,46 @@
//go:build !go1.16
// +build !go1.16
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`github.com/bytedance/sonic/internal/loader`
)
// LoadFuncs loads only one function as module, and returns the function pointer
// - text: machine code
// - funcName: function name
// - frameSize: stack frame size.
// - argSize: argument total size (in bytes)
// - argPtrs: indicates if a slot (8 Bytes) of arguments memory stores pointer, from low to high
// - localPtrs: indicates if a slot (8 Bytes) of local variants memory stores pointer, from low to high
//
// WARN:
// - the function MUST has fixed SP offset equaling to this, otherwise it go.gentraceback will fail
// - the function MUST has only one stack map for all arguments and local variants
func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argPtrs []bool, localPtrs []bool) Function {
return Function(loader.Loader(text).Load(funcName, frameSize, argSize, argPtrs, localPtrs))
}
// Load loads given machine codes and corresponding function information into go moduledata
// and returns runnable function pointer
// WARN: this API is experimental, use it carefully
func Load(text []byte, funcs []Func, modulename string, filenames []string) (out []Function) {
panic("not implemented")
}

@ -1,28 +0,0 @@
//go:build go1.15 && !go1.16
// +build go1.15,!go1.16
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`github.com/bytedance/sonic/internal/loader`
)
func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) Function {
return Function(loader.Loader(text).Load(funcName, frameSize, argSize, argStackmap, localStackmap))
}

@ -87,18 +87,27 @@ func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize
// and returns runnable function pointer
// WARN: this API is experimental, use it carefully
func Load(text []byte, funcs []Func, modulename string, filenames []string) (out []Function) {
ids := make([]string, len(funcs))
for i, f := range funcs {
ids[i] = f.Name
}
// generate module data and allocate memory address
mod := makeModuledata(modulename, filenames, funcs, text)
mod := makeModuledata(modulename, filenames, &funcs, text)
// verify and register the new module
moduledataverify1(mod)
registerModule(mod)
//
// encapsulate function address
out = make([]Function, len(funcs))
for i, f := range funcs {
m := uintptr(mod.text + uintptr(f.EntryOff))
out[i] = Function(&m)
for i, s := range ids {
for _, f := range funcs {
if f.Name == s {
m := uintptr(mod.text + uintptr(f.EntryOff))
out[i] = Function(&m)
}
}
}
return
}
}

@ -42,4 +42,4 @@ func mprotect(p uintptr, nb int) {
if _, _, err := syscall.RawSyscall(syscall.SYS_MPROTECT, p, uintptr(nb), _RX); err != 0 {
panic(err)
}
}
}

@ -16,6 +16,10 @@
package loader
import (
`encoding/binary`
)
const (
_N_PCDATA = 4
@ -49,40 +53,16 @@ const (
var emptyByte byte
func encodeValue(v int) []byte {
return encodeVariant(toZigzag(v))
}
func toZigzag(v int) int {
return (v << 1) ^ (v >> 31)
}
func encodeVariant(v int) []byte {
var u int
var r []byte
/* split every 7 bits */
for v > 127 {
u = v & 0x7f
v = v >> 7
r = append(r, byte(u) | 0x80)
}
/* check for last one */
if v == 0 {
return r
}
/* add the last one */
r = append(r, byte(v))
return r
}
// Pcvalue is the program count corresponding to the value Val
// WARN: we use relative value here (to function entry)
type Pcvalue struct {
PC uint32 // PC offset from func entry
Val int32
PC uint32 // program count relative to function entry
Val int32 // value relative to the value in function entry
}
// Pcdata represents pc->value mapping table.
// WARN: we use ** [Pcdata[i].PC, Pcdata[i+1].PC) **
// as the range where the Pcdata[i].Val is effective.
type Pcdata []Pcvalue
// see https://docs.google.com/document/d/1lyPIbmsYbXnpNj57a261hgOYVpNRcgydurVQIyZOz_o/pub
@ -90,11 +70,24 @@ func (self Pcdata) MarshalBinary() (data []byte, err error) {
// delta value always starts from -1
sv := int32(_PCDATA_START_VAL)
sp := uint32(0)
buf := make([]byte, binary.MaxVarintLen32)
for _, v := range self {
data = append(data, encodeVariant(toZigzag(int(v.Val - sv)))...)
data = append(data, encodeVariant(int(v.PC - sp))...)
if v.PC < sp {
panic("PC must be in ascending order!")
}
dp := uint64(v.PC - sp)
dv := int64(v.Val - sv)
if dv == 0 || dp == 0 {
continue
}
n := binary.PutVarint(buf, dv)
data = append(data, buf[:n]...)
n2 := binary.PutUvarint(buf, dp)
data = append(data, buf[:n2]...)
sp = v.PC
sv = v.Val
}
// put 0 to indicate ends
data = append(data, 0)
return
}
}

@ -68,4 +68,4 @@ func Validate(src []byte) bool {
// ValidateString as Validate, but for string.
func ValidateString(src string) bool {
return native.ValidateUTF8Fast(&src) == 0
}
}

@ -1,3 +1,8 @@
# 5.4.1 (June 18, 2023)
* Fix: concurrency bug with pgtypeDefaultMap and simple protocol (Lev Zakharov)
* Add TxOptions.BeginQuery to allow overriding the default BEGIN query
# 5.4.0 (June 14, 2023)
* Replace platform specific syscalls for non-blocking IO with more traditional goroutines and deadlines. This returns to the v4 approach with some additional improvements and fixes. This restores the ability to use a pgx.Conn over an ssh.Conn as well as other non-TCP or Unix socket connections. In addition, it is a significantly simpler implementation that is less likely to have cross platform issues.

@ -363,12 +363,13 @@ func quoteArrayElement(src string) string {
}
func isSpace(ch byte) bool {
// see https://github.com/postgres/postgres/blob/REL_12_STABLE/src/backend/parser/scansup.c#L224
return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f'
// see array_isspace:
// https://github.com/postgres/postgres/blob/master/src/backend/utils/adt/arrayfuncs.c
return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\v' || ch == '\f'
}
func quoteArrayElementIfNeeded(src string) string {
if src == "" || (len(src) == 4 && strings.ToLower(src) == "null") || isSpace(src[0]) || isSpace(src[len(src)-1]) || strings.ContainsAny(src, `{},"\`) {
if src == "" || (len(src) == 4 && strings.EqualFold(src, "null")) || isSpace(src[0]) || isSpace(src[len(src)-1]) || strings.ContainsAny(src, `{},"\`) {
return quoteArrayElement(src)
}
return src

@ -1,13 +1,11 @@
package pgtype
import (
"bytes"
"database/sql/driver"
"encoding/binary"
"errors"
"fmt"
"unicode"
"unicode/utf8"
"strings"
"github.com/jackc/pgx/v5/internal/pgio"
)
@ -185,20 +183,23 @@ func (scanPlanBinaryHstoreToHstoreScanner) Scan(src []byte, dst any) error {
rp := 0
if len(src[rp:]) < 4 {
const uint32Len = 4
if len(src[rp:]) < uint32Len {
return fmt.Errorf("hstore incomplete %v", src)
}
pairCount := int(int32(binary.BigEndian.Uint32(src[rp:])))
rp += 4
rp += uint32Len
hstore := make(Hstore, pairCount)
// one allocation for all *string, rather than one per string, just like text parsing
valueStrings := make([]string, pairCount)
for i := 0; i < pairCount; i++ {
if len(src[rp:]) < 4 {
if len(src[rp:]) < uint32Len {
return fmt.Errorf("hstore incomplete %v", src)
}
keyLen := int(int32(binary.BigEndian.Uint32(src[rp:])))
rp += 4
rp += uint32Len
if len(src[rp:]) < keyLen {
return fmt.Errorf("hstore incomplete %v", src)
@ -206,26 +207,17 @@ func (scanPlanBinaryHstoreToHstoreScanner) Scan(src []byte, dst any) error {
key := string(src[rp : rp+keyLen])
rp += keyLen
if len(src[rp:]) < 4 {
if len(src[rp:]) < uint32Len {
return fmt.Errorf("hstore incomplete %v", src)
}
valueLen := int(int32(binary.BigEndian.Uint32(src[rp:])))
rp += 4
var valueBuf []byte
if valueLen >= 0 {
valueBuf = src[rp : rp+valueLen]
valueStrings[i] = string(src[rp : rp+valueLen])
rp += valueLen
}
var value Text
err := scanPlanTextAnyToTextScanner{}.Scan(valueBuf, &value)
if err != nil {
return err
}
if value.Valid {
hstore[key] = &value.String
hstore[key] = &valueStrings[i]
} else {
hstore[key] = nil
}
@ -247,21 +239,11 @@ func (s scanPlanTextAnyToHstoreScanner) Scan(src []byte, dst any) error {
// scanString does not return nil hstore values because string cannot be nil.
func (scanPlanTextAnyToHstoreScanner) scanString(src string, scanner HstoreScanner) error {
keys, values, err := parseHstore(src)
hstore, err := parseHstore(src)
if err != nil {
return err
}
m := make(Hstore, len(keys))
for i := range keys {
if values[i].Valid {
m[keys[i]] = &values[i].String
} else {
m[keys[i]] = nil
}
}
return scanner.ScanHstore(m)
return scanner.ScanHstore(hstore)
}
func (c HstoreCodec) DecodeDatabaseSQLValue(m *Map, oid uint32, format int16, src []byte) (driver.Value, error) {
@ -281,187 +263,217 @@ func (c HstoreCodec) DecodeValue(m *Map, oid uint32, format int16, src []byte) (
return hstore, nil
}
const (
hsPre = iota
hsKey
hsSep
hsVal
hsNul
hsNext
)
type hstoreParser struct {
str string
pos int
str string
pos int
nextBackslash int
}
func newHSP(in string) *hstoreParser {
return &hstoreParser{
pos: 0,
str: in,
pos: 0,
str: in,
nextBackslash: strings.IndexByte(in, '\\'),
}
}
func (p *hstoreParser) Consume() (r rune, end bool) {
func (p *hstoreParser) atEnd() bool {
return p.pos >= len(p.str)
}
// consume returns the next byte of the string, or end if the string is done.
func (p *hstoreParser) consume() (b byte, end bool) {
if p.pos >= len(p.str) {
end = true
return
return 0, true
}
r, w := utf8.DecodeRuneInString(p.str[p.pos:])
p.pos += w
return
b = p.str[p.pos]
p.pos++
return b, false
}
func (p *hstoreParser) Peek() (r rune, end bool) {
if p.pos >= len(p.str) {
end = true
return
func unexpectedByteErr(actualB byte, expectedB byte) error {
return fmt.Errorf("expected '%c' ('%#v'); found '%c' ('%#v')", expectedB, expectedB, actualB, actualB)
}
// consumeExpectedByte consumes expectedB from the string, or returns an error.
func (p *hstoreParser) consumeExpectedByte(expectedB byte) error {
nextB, end := p.consume()
if end {
return fmt.Errorf("expected '%c' ('%#v'); found end", expectedB, expectedB)
}
if nextB != expectedB {
return unexpectedByteErr(nextB, expectedB)
}
r, _ = utf8.DecodeRuneInString(p.str[p.pos:])
return
return nil
}
// parseHstore parses the string representation of an hstore column (the same
// you would get from an ordinary SELECT) into two slices of keys and values. it
// is used internally in the default parsing of hstores.
func parseHstore(s string) (k []string, v []Text, err error) {
if s == "" {
return
// consumeExpected2 consumes two expected bytes or returns an error.
// This was a bit faster than using a string argument (better inlining? Not sure).
func (p *hstoreParser) consumeExpected2(one byte, two byte) error {
if p.pos+2 > len(p.str) {
return errors.New("unexpected end of string")
}
if p.str[p.pos] != one {
return unexpectedByteErr(p.str[p.pos], one)
}
if p.str[p.pos+1] != two {
return unexpectedByteErr(p.str[p.pos+1], two)
}
p.pos += 2
return nil
}
buf := bytes.Buffer{}
keys := []string{}
values := []Text{}
p := newHSP(s)
var errEOSInQuoted = errors.New(`found end before closing double-quote ('"')`)
// consumeDoubleQuoted consumes a double-quoted string from p. The double quote must have been
// parsed already. This copies the string from the backing string so it can be garbage collected.
func (p *hstoreParser) consumeDoubleQuoted() (string, error) {
// fast path: assume most keys/values do not contain escapes
nextDoubleQuote := strings.IndexByte(p.str[p.pos:], '"')
if nextDoubleQuote == -1 {
return "", errEOSInQuoted
}
nextDoubleQuote += p.pos
if p.nextBackslash == -1 || p.nextBackslash > nextDoubleQuote {
// clone the string from the source string to ensure it can be garbage collected separately
// TODO: use strings.Clone on Go 1.20; this could get optimized away
s := strings.Clone(p.str[p.pos:nextDoubleQuote])
p.pos = nextDoubleQuote + 1
return s, nil
}
r, end := p.Consume()
state := hsPre
// slow path: string contains escapes
s, err := p.consumeDoubleQuotedWithEscapes(p.nextBackslash)
p.nextBackslash = strings.IndexByte(p.str[p.pos:], '\\')
if p.nextBackslash != -1 {
p.nextBackslash += p.pos
}
return s, err
}
for !end {
switch state {
case hsPre:
if r == '"' {
state = hsKey
} else {
err = errors.New("String does not begin with \"")
}
case hsKey:
switch r {
case '"': //End of the key
keys = append(keys, buf.String())
buf = bytes.Buffer{}
state = hsSep
case '\\': //Potential escaped character
n, end := p.Consume()
switch {
case end:
err = errors.New("Found EOS in key, expecting character or \"")
case n == '"', n == '\\':
buf.WriteRune(n)
default:
buf.WriteRune(r)
buf.WriteRune(n)
}
default: //Any other character
buf.WriteRune(r)
}
case hsSep:
if r == '=' {
r, end = p.Consume()
switch {
case end:
err = errors.New("Found EOS after '=', expecting '>'")
case r == '>':
r, end = p.Consume()
switch {
case end:
err = errors.New("Found EOS after '=>', expecting '\"' or 'NULL'")
case r == '"':
state = hsVal
case r == 'N':
state = hsNul
default:
err = fmt.Errorf("Invalid character '%c' after '=>', expecting '\"' or 'NULL'", r)
}
default:
err = fmt.Errorf("Invalid character after '=', expecting '>'")
}
} else {
err = fmt.Errorf("Invalid character '%c' after value, expecting '='", r)
}
case hsVal:
switch r {
case '"': //End of the value
values = append(values, Text{String: buf.String(), Valid: true})
buf = bytes.Buffer{}
state = hsNext
case '\\': //Potential escaped character
n, end := p.Consume()
switch {
case end:
err = errors.New("Found EOS in key, expecting character or \"")
case n == '"', n == '\\':
buf.WriteRune(n)
default:
buf.WriteRune(r)
buf.WriteRune(n)
}
default: //Any other character
buf.WriteRune(r)
}
case hsNul:
nulBuf := make([]rune, 3)
nulBuf[0] = r
for i := 1; i < 3; i++ {
r, end = p.Consume()
if end {
err = errors.New("Found EOS in NULL value")
return
}
nulBuf[i] = r
}
if nulBuf[0] == 'U' && nulBuf[1] == 'L' && nulBuf[2] == 'L' {
values = append(values, Text{})
state = hsNext
} else {
err = fmt.Errorf("Invalid NULL value: 'N%s'", string(nulBuf))
// consumeDoubleQuotedWithEscapes consumes a double-quoted string containing escapes, starting
// at p.pos, and with the first backslash at firstBackslash. This copies the string so it can be
// garbage collected separately.
func (p *hstoreParser) consumeDoubleQuotedWithEscapes(firstBackslash int) (string, error) {
// copy the prefix that does not contain backslashes
var builder strings.Builder
builder.WriteString(p.str[p.pos:firstBackslash])
// skip to the backslash
p.pos = firstBackslash
// copy bytes until the end, unescaping backslashes
for {
nextB, end := p.consume()
if end {
return "", errEOSInQuoted
} else if nextB == '"' {
break
} else if nextB == '\\' {
// escape: skip the backslash and copy the char
nextB, end = p.consume()
if end {
return "", errEOSInQuoted
}
case hsNext:
if r == ',' {
r, end = p.Consume()
switch {
case end:
err = errors.New("Found EOS after ',', expecting space")
case (unicode.IsSpace(r)):
// after space is a doublequote to start the key
r, end = p.Consume()
if end {
err = errors.New("Found EOS after space, expecting \"")
return
}
if r != '"' {
err = fmt.Errorf("Invalid character '%c' after space, expecting \"", r)
return
}
state = hsKey
default:
err = fmt.Errorf("Invalid character '%c' after ',', expecting space", r)
}
} else {
err = fmt.Errorf("Invalid character '%c' after value, expecting ','", r)
if !(nextB == '\\' || nextB == '"') {
return "", fmt.Errorf("unexpected escape in quoted string: found '%#v'", nextB)
}
builder.WriteByte(nextB)
} else {
// normal byte: copy it
builder.WriteByte(nextB)
}
}
return builder.String(), nil
}
// consumePairSeparator consumes the Hstore pair separator ", " or returns an error.
func (p *hstoreParser) consumePairSeparator() error {
return p.consumeExpected2(',', ' ')
}
// consumeKVSeparator consumes the Hstore key/value separator "=>" or returns an error.
func (p *hstoreParser) consumeKVSeparator() error {
return p.consumeExpected2('=', '>')
}
// consumeDoubleQuotedOrNull consumes the Hstore key/value separator "=>" or returns an error.
func (p *hstoreParser) consumeDoubleQuotedOrNull() (Text, error) {
// peek at the next byte
if p.atEnd() {
return Text{}, errors.New("found end instead of value")
}
next := p.str[p.pos]
if next == 'N' {
// must be the exact string NULL: use consumeExpected2 twice
err := p.consumeExpected2('N', 'U')
if err != nil {
return Text{}, err
}
err = p.consumeExpected2('L', 'L')
if err != nil {
return
return Text{}, err
}
r, end = p.Consume()
return Text{String: "", Valid: false}, nil
} else if next != '"' {
return Text{}, unexpectedByteErr(next, '"')
}
if state != hsNext {
err = errors.New("Improperly formatted hstore")
return
// skip the double quote
p.pos += 1
s, err := p.consumeDoubleQuoted()
if err != nil {
return Text{}, err
}
return Text{String: s, Valid: true}, nil
}
func parseHstore(s string) (Hstore, error) {
p := newHSP(s)
// This is an over-estimate of the number of key/value pairs. Use '>' because I am guessing it
// is less likely to occur in keys/values than '=' or ','.
numPairsEstimate := strings.Count(s, ">")
// makes one allocation of strings for the entire Hstore, rather than one allocation per value.
valueStrings := make([]string, 0, numPairsEstimate)
result := make(Hstore, numPairsEstimate)
first := true
for !p.atEnd() {
if !first {
err := p.consumePairSeparator()
if err != nil {
return nil, err
}
} else {
first = false
}
err := p.consumeExpectedByte('"')
if err != nil {
return nil, err
}
key, err := p.consumeDoubleQuoted()
if err != nil {
return nil, err
}
err = p.consumeKVSeparator()
if err != nil {
return nil, err
}
value, err := p.consumeDoubleQuotedOrNull()
if err != nil {
return nil, err
}
if value.Valid {
valueStrings = append(valueStrings, value.String)
result[key] = &valueStrings[len(valueStrings)-1]
} else {
result[key] = nil
}
}
k = keys
v = values
return
return result, nil
}

@ -318,10 +318,6 @@ func (m *Map) TypeForValue(v any) (*Type, bool) {
return dt, true
}
if defaultMap.reflectTypeToType == nil {
defaultMap.buildReflectTypeToType()
}
dt, ok := defaultMap.reflectTypeToType[reflect.TypeOf(v)]
return dt, ok
}

@ -218,4 +218,6 @@ func initDefaultMap() {
registerDefaultPgTypeVariants[Range[Timestamptz]](defaultMap, "tstzrange")
registerDefaultPgTypeVariants[Multirange[Range[Timestamptz]]](defaultMap, "tstzmultirange")
registerDefaultPgTypeVariants[UUID](defaultMap, "uuid")
defaultMap.buildReflectTypeToType()
}

@ -44,6 +44,10 @@ type TxOptions struct {
IsoLevel TxIsoLevel
AccessMode TxAccessMode
DeferrableMode TxDeferrableMode
// BeginQuery is the SQL query that will be executed to begin the transaction. This allows using non-standard syntax
// such as BEGIN PRIORITY HIGH with CockroachDB. If set this will override the other settings.
BeginQuery string
}
var emptyTxOptions TxOptions
@ -53,6 +57,10 @@ func (txOptions TxOptions) beginSQL() string {
return "begin"
}
if txOptions.BeginQuery != "" {
return txOptions.BeginQuery
}
var buf strings.Builder
buf.Grow(64) // 64 - maximum length of string with available options
buf.WriteString("begin")

@ -16,6 +16,12 @@ This package provides various compression algorithms.
# changelog
* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6)
* zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806
* zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824
* gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815
* s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663
* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5)
* zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802
* gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804

@ -20,6 +20,6 @@ Vulnerabilities resulting from compiler/assembler errors should be reported upst
If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released.
Please disclose it at [security advisory](https://github.com/klaupost/compress/security/advisories/new). If possible please provide a minimal reproducer. If the issue only applies to a single platform, it would be helpful to provide access to that.
Please disclose it at [security advisory](https://github.com/klauspost/compress/security/advisories/new). If possible please provide a minimal reproducer. If the issue only applies to a single platform, it would be helpful to provide access to that.
This project is maintained by a team of volunteers on a reasonable-effort basis. As such, vulnerabilities will be disclosed in a best effort base.

@ -144,6 +144,7 @@ func (e *fastBase) resetBase(d *dict, singleBlock bool) {
} else {
e.crc.Reset()
}
e.blk.dictLitEnc = nil
if d != nil {
low := e.lowMem
if singleBlock {

@ -1084,7 +1084,7 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) {
}
}
e.lastDictID = d.id
e.allDirty = true
allDirty = true
}
// Reset table to initial state
e.cur = e.maxMatchOff

@ -829,13 +829,12 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
}
if true {
end := e.maxMatchOff + int32(len(d.content)) - 8
for i := e.maxMatchOff; i < end; i += 3 {
for i := e.maxMatchOff; i < end; i += 2 {
const hashLog = tableBits
cv := load6432(d.content, i-e.maxMatchOff)
nextHash := hashLen(cv, hashLog, tableFastHashLen) // 0 -> 5
nextHash1 := hashLen(cv>>8, hashLog, tableFastHashLen) // 1 -> 6
nextHash2 := hashLen(cv>>16, hashLog, tableFastHashLen) // 2 -> 7
nextHash := hashLen(cv, hashLog, tableFastHashLen) // 0 -> 6
nextHash1 := hashLen(cv>>8, hashLog, tableFastHashLen) // 1 -> 7
e.dictTable[nextHash] = tableEntry{
val: uint32(cv),
offset: i,
@ -844,10 +843,6 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
val: uint32(cv >> 8),
offset: i + 1,
}
e.dictTable[nextHash2] = tableEntry{
val: uint32(cv >> 16),
offset: i + 2,
}
}
}
e.lastDictID = d.id

@ -1,5 +1,10 @@
# Changelog
## [0.4.0] (2023-06-29)
* decode into embedded structs
## [0.3.1] (2022-04-09)
* fix Decode: don't fill value for struct fields that don't exist in header
@ -26,3 +31,5 @@
[0.2.0]: https://github.com/mozillazg/go-httpheader/compare/v0.1.0...v0.2.0
[0.2.1]: https://github.com/mozillazg/go-httpheader/compare/v0.2.0...v0.2.1
[0.3.0]: https://github.com/mozillazg/go-httpheader/compare/v0.2.1...v0.3.0
[0.3.1]: https://github.com/mozillazg/go-httpheader/compare/v0.3.0...v0.3.1
[0.4.0]: https://github.com/mozillazg/go-httpheader/compare/v0.3.1...v0.4.0

@ -9,7 +9,9 @@ go-httpheader is a Go library for encoding structs into Header fields.
## install
`go get -u github.com/mozillazg/go-httpheader`
```
go get github.com/mozillazg/go-httpheader
```
## usage

@ -1,6 +1,7 @@
package httpheader
import (
"errors"
"fmt"
"net/http"
"net/textproto"
@ -20,7 +21,7 @@ type Decoder interface {
func Decode(header http.Header, v interface{}) error {
val := reflect.ValueOf(v)
if val.Kind() != reflect.Ptr || val.IsNil() {
return fmt.Errorf("v should be point and should not be nil")
return errors.New("v should be a pointer and should not be nil")
}
for val.Kind() == reflect.Ptr {
@ -33,7 +34,12 @@ func Decode(header http.Header, v interface{}) error {
return parseValue(header, val)
}
// parseValue populates the struct fields in val from the header fields.
// Embedded structs are followed recursively (using the rules defined in the
// Values function documentation) breadth-first.
func parseValue(header http.Header, val reflect.Value) error {
var embedded []reflect.Value
typ := val.Type()
for i := 0; i < typ.NumField(); i++ {
sf := typ.Field(i)
@ -49,6 +55,8 @@ func parseValue(header http.Header, val reflect.Value) error {
name, opts := parseTag(tag)
if name == "" {
if sf.Anonymous && sv.Kind() == reflect.Struct {
// save embedded struct for later processing
embedded = append(embedded, sv)
continue
}
name = sf.Name
@ -103,6 +111,25 @@ func parseValue(header http.Header, val reflect.Value) error {
continue
}
if sv.Kind() != reflect.Slice && sv.Kind() != reflect.Array && sv.Kind() != reflect.Interface {
vals, exist := headerValues(header, name)
if !exist {
continue
}
v := vals[0]
vals = vals[1:]
if err := fillValues(sv, opts, []string{v}); err != nil {
return err
}
header.Del(name)
for _, v := range vals {
header.Add(name, v)
}
continue
}
valArr, exist := headerValues(header, name)
if !exist {
continue
@ -111,6 +138,12 @@ func parseValue(header http.Header, val reflect.Value) error {
return err
}
}
for _, f := range embedded {
if err := parseValue(header, f); err != nil {
return err
}
}
return nil
}

@ -1,26 +1,26 @@
[run]
deadline = "10m"
tests = true
[linters]
disable-all = true
enable = [
"asasalint",
"asciicheck",
"bidichk",
"bodyclose",
"containedctx",
"contextcheck",
"deadcode",
"depguard",
"dupword",
"durationcheck",
"errcheck",
"errchkjson",
"errname",
"errorlint",
# "exhaustive",
"exportloopref",
"forbidigo",
#"forcetypeassert",
"goconst",
"gocyclo",
"gocritic",
@ -46,7 +46,6 @@
"rowserrcheck",
"sqlclosecheck",
"staticcheck",
"structcheck",
"stylecheck",
"tenv",
"tparallel",
@ -54,40 +53,17 @@
"unconvert",
"unparam",
"unused",
"varcheck",
"usestdlibvars",
"vetshadow",
"wastedassign",
]
# Please note that we only use depguard for stdlib as gomodguard only
# supports modules currently. See https://github.com/ryancurrah/gomodguard/issues/12
[linters-settings.depguard]
list-type = "blacklist"
include-go-root = true
packages = [
# ioutil is deprecated. The functions have been moved elsewhere:
# https://golang.org/doc/go1.16#ioutil
"io/ioutil",
]
[[linters-settings.depguard.rules.main.deny]]
pkg = "io/ioutil"
desc = "Deprecated. Functions have been moved elsewhere."
[linters-settings.errcheck]
# Don't allow setting of error to the blank identifier. If there is a legtimate
# reason, there should be a nolint with an explanation.
check-blank = true
exclude-functions = [
# If we are rolling back a transaction, we are often already in an error
# state.
'(*database/sql.Tx).Rollback',
# It is reasonable to ignore errors if Cleanup fails in most cases.
'(*github.com/google/renameio/v2.PendingFile).Cleanup',
# We often don't care if removing a file failed (e.g., it doesn't exist)
'os.Remove',
'os.RemoveAll',
]
# Ignoring Close so that we don't have to have a bunch of
# `defer func() { _ = r.Close() }()` constructs when we
# don't actually care about the error.
@ -104,8 +80,10 @@
[linters-settings.forbidigo]
# Forbid the following identifiers
forbid = [
"^minFraud*",
"^maxMind*",
"Geoip", # use "GeoIP"
"^geoIP", # use "geoip"
"Maxmind", # use "MaxMind"
"^maxMind", # use "maxmind"
]
[linters-settings.gocritic]
@ -129,8 +107,7 @@
"commentedOutImport",
"commentFormatting",
"defaultCaseOrder",
# Revive's defer rule already captures this. This caught no extra cases.
# "deferInLoop",
"deferInLoop",
"deferUnlambda",
"deprecatedComment",
"docStub",
@ -149,17 +126,16 @@
"exitAfterDefer",
"exposedSyncMutex",
"externalErrorReassign",
# Given that all of our code runs on Linux and the / separate should
# work fine, this seems less important.
# "filepathJoin",
"filepathJoin",
"flagDeref",
"flagName",
"hexLiteral",
"httpNoBody",
"hugeParam",
"ifElseChain",
"importShadow",
"indexAlloc",
"initClause",
"ioutilDeprecated",
"mapKey",
"methodExprCall",
"nestingReduce",
@ -179,22 +155,20 @@
"redundantSprint",
"regexpMust",
"regexpPattern",
# This might be good, but I don't think we want to encourage
# significant changes to regexes as we port stuff from Perl.
# "regexpSimplify",
"regexpSimplify",
"returnAfterHttpError",
"ruleguard",
"singleCaseSwitch",
"sliceClear",
"sloppyLen",
# This seems like it might also be good, but a lot of existing code
# fails.
# "sloppyReassign",
"returnAfterHttpError",
"sloppyReassign",
"sloppyTestFuncName",
"sloppyTypeAssert",
"sortSlice",
"sprintfQuotedString",
"sqlQuery",
"stringsCompare",
"stringConcatSimplify",
"stringXbytes",
"switchTrue",
"syncMapLoadAndDelete",
@ -209,28 +183,40 @@
"underef",
"unlabelStmt",
"unlambda",
# I am not sure we would want this linter and a lot of existing
# code fails.
# "unnamedResult",
"unnecessaryBlock",
"unnecessaryDefer",
"unslice",
"valSwap",
"weakCond",
# Covered by nolintlint
# "whyNoLint"
"wrapperFunc",
"yodaStyleExpr",
# This requires explanations for "nolint" directives. This would be
# nice for gosec ones, but I am not sure we want it generally unless
# we can get the false positive rate lower.
# "whyNoLint"
]
[linters-settings.gofumpt]
extra-rules = true
lang-version = "1.18"
lang-version = "1.19"
[linters-settings.gosec]
excludes = [
# G104 - "Audit errors not checked." We use errcheck for this.
"G104",
# G304 - "Potential file inclusion via variable"
"G304",
# G306 - "Expect WriteFile permissions to be 0600 or less".
"G306",
# Prohibits defer (*os.File).Close, which we allow when reading from file.
"G307",
]
[linters-settings.govet]
"enable-all" = true
disable = ["shadow"]
[linters-settings.lll]
line-length = 120
@ -247,8 +233,6 @@
ignore-generated-header = true
severity = "warning"
# This might be nice but it is so common that it is hard
# to enable.
# [[linters-settings.revive.rules]]
# name = "add-constant"
@ -273,8 +257,10 @@
# [[linters-settings.revive.rules]]
# name = "cognitive-complexity"
# Probably a good rule, but we have a lot of names that
# only have case differences.
[[linters-settings.revive.rules]]
name = "comment-spacings"
arguments = ["easyjson", "nolint"]
# [[linters-settings.revive.rules]]
# name = "confusing-naming"
@ -293,6 +279,9 @@
# [[linters-settings.revive.rules]]
# name = "cyclomatic"
[[linters-settings.revive.rules]]
name = "datarace"
# [[linters-settings.revive.rules]]
# name = "deep-exit"
@ -332,8 +321,6 @@
# [[linters-settings.revive.rules]]
# name = "file-header"
# We have a lot of flag parameters. This linter probably makes
# a good point, but we would need some cleanup or a lot of nolints.
# [[linters-settings.revive.rules]]
# name = "flag-parameter"
@ -373,7 +360,6 @@
[[linters-settings.revive.rules]]
name = "modifies-value-receiver"
# We frequently use nested structs, particularly in tests.
# [[linters-settings.revive.rules]]
# name = "nested-structs"
@ -407,6 +393,9 @@
[[linters-settings.revive.rules]]
name = "superfluous-else"
[[linters-settings.revive.rules]]
name = "time-equal"
[[linters-settings.revive.rules]]
name = "time-naming"
@ -419,8 +408,6 @@
[[linters-settings.revive.rules]]
name = "unexported-return"
# This is covered elsewhere and we want to ignore some
# functions such as fmt.Fprintf.
# [[linters-settings.revive.rules]]
# name = "unhandled-error"
@ -433,14 +420,11 @@
[[linters-settings.revive.rules]]
name = "unused-parameter"
# We generally have unused receivers in tests for meeting the
# requirements of an interface.
# [[linters-settings.revive.rules]]
# name = "unused-receiver"
[[linters-settings.revive.rules]]
name = "unused-receiver"
# This probably makes sense after we upgrade to 1.18
# [[linters-settings.revive.rules]]
# name = "use-any"
[[linters-settings.revive.rules]]
name = "use-any"
[[linters-settings.revive.rules]]
name = "useless-break"
@ -457,16 +441,12 @@
[linters-settings.unparam]
check-exported = true
[issues]
exclude-use-default = false
[[issues.exclude-rules]]
linters = [
"govet"
]
# we want to enable almost all govet rules. It is easier to just filter out
# the ones we don't want:
#
# * fieldalignment - way too noisy. Although it is very useful in particular
# cases where we are trying to use as little memory as possible, having
# it go off on every struct isn't helpful.
# * shadow - although often useful, it complains about _many_ err
# shadowing assignments and some others where shadowing is clear.
text = "^(fieldalignment|shadow)"
path = "_test.go"
text = "^fieldalignment"

@ -17,117 +17,117 @@ import (
// The Enterprise struct corresponds to the data in the GeoIP2 Enterprise
// database.
type Enterprise struct {
City struct {
Confidence uint8 `maxminddb:"confidence"`
GeoNameID uint `maxminddb:"geoname_id"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"city"`
Continent struct {
Names map[string]string `maxminddb:"names"`
Code string `maxminddb:"code"`
GeoNameID uint `maxminddb:"geoname_id"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"continent"`
Country struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
Confidence uint8 `maxminddb:"confidence"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
} `maxminddb:"country"`
Location struct {
AccuracyRadius uint16 `maxminddb:"accuracy_radius"`
Latitude float64 `maxminddb:"latitude"`
Longitude float64 `maxminddb:"longitude"`
MetroCode uint `maxminddb:"metro_code"`
TimeZone string `maxminddb:"time_zone"`
} `maxminddb:"location"`
City struct {
Names map[string]string `maxminddb:"names"`
GeoNameID uint `maxminddb:"geoname_id"`
Confidence uint8 `maxminddb:"confidence"`
} `maxminddb:"city"`
Postal struct {
Code string `maxminddb:"code"`
Confidence uint8 `maxminddb:"confidence"`
} `maxminddb:"postal"`
RegisteredCountry struct {
GeoNameID uint `maxminddb:"geoname_id"`
Subdivisions []struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
GeoNameID uint `maxminddb:"geoname_id"`
Confidence uint8 `maxminddb:"confidence"`
} `maxminddb:"subdivisions"`
RepresentedCountry struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
Type string `maxminddb:"type"`
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
} `maxminddb:"represented_country"`
Country struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
GeoNameID uint `maxminddb:"geoname_id"`
Confidence uint8 `maxminddb:"confidence"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
} `maxminddb:"registered_country"`
RepresentedCountry struct {
} `maxminddb:"country"`
RegisteredCountry struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
GeoNameID uint `maxminddb:"geoname_id"`
Confidence uint8 `maxminddb:"confidence"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
Type string `maxminddb:"type"`
} `maxminddb:"represented_country"`
Subdivisions []struct {
Confidence uint8 `maxminddb:"confidence"`
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"subdivisions"`
} `maxminddb:"registered_country"`
Traits struct {
AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"`
AutonomousSystemOrganization string `maxminddb:"autonomous_system_organization"`
ConnectionType string `maxminddb:"connection_type"`
Domain string `maxminddb:"domain"`
IsAnonymousProxy bool `maxminddb:"is_anonymous_proxy"`
IsLegitimateProxy bool `maxminddb:"is_legitimate_proxy"`
IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
ISP string `maxminddb:"isp"`
MobileCountryCode string `maxminddb:"mobile_country_code"`
MobileNetworkCode string `maxminddb:"mobile_network_code"`
Organization string `maxminddb:"organization"`
StaticIPScore float64 `maxminddb:"static_ip_score"`
UserType string `maxminddb:"user_type"`
AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"`
StaticIPScore float64 `maxminddb:"static_ip_score"`
IsAnonymousProxy bool `maxminddb:"is_anonymous_proxy"`
IsLegitimateProxy bool `maxminddb:"is_legitimate_proxy"`
IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
} `maxminddb:"traits"`
Location struct {
TimeZone string `maxminddb:"time_zone"`
Latitude float64 `maxminddb:"latitude"`
Longitude float64 `maxminddb:"longitude"`
MetroCode uint `maxminddb:"metro_code"`
AccuracyRadius uint16 `maxminddb:"accuracy_radius"`
} `maxminddb:"location"`
}
// The City struct corresponds to the data in the GeoIP2/GeoLite2 City
// databases.
type City struct {
City struct {
GeoNameID uint `maxminddb:"geoname_id"`
Names map[string]string `maxminddb:"names"`
GeoNameID uint `maxminddb:"geoname_id"`
} `maxminddb:"city"`
Postal struct {
Code string `maxminddb:"code"`
} `maxminddb:"postal"`
Continent struct {
Names map[string]string `maxminddb:"names"`
Code string `maxminddb:"code"`
GeoNameID uint `maxminddb:"geoname_id"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"continent"`
Country struct {
Subdivisions []struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
GeoNameID uint `maxminddb:"geoname_id"`
} `maxminddb:"subdivisions"`
RepresentedCountry struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
Type string `maxminddb:"type"`
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
IsoCode string `maxminddb:"iso_code"`
} `maxminddb:"represented_country"`
Country struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
} `maxminddb:"country"`
RegisteredCountry struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
} `maxminddb:"registered_country"`
Location struct {
AccuracyRadius uint16 `maxminddb:"accuracy_radius"`
TimeZone string `maxminddb:"time_zone"`
Latitude float64 `maxminddb:"latitude"`
Longitude float64 `maxminddb:"longitude"`
MetroCode uint `maxminddb:"metro_code"`
TimeZone string `maxminddb:"time_zone"`
AccuracyRadius uint16 `maxminddb:"accuracy_radius"`
} `maxminddb:"location"`
Postal struct {
Code string `maxminddb:"code"`
} `maxminddb:"postal"`
RegisteredCountry struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"registered_country"`
RepresentedCountry struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
Type string `maxminddb:"type"`
} `maxminddb:"represented_country"`
Subdivisions []struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"subdivisions"`
Traits struct {
IsAnonymousProxy bool `maxminddb:"is_anonymous_proxy"`
IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
@ -138,28 +138,28 @@ type City struct {
// Country databases.
type Country struct {
Continent struct {
Names map[string]string `maxminddb:"names"`
Code string `maxminddb:"code"`
GeoNameID uint `maxminddb:"geoname_id"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"continent"`
Country struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"country"`
RegisteredCountry struct {
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"registered_country"`
RepresentedCountry struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
IsoCode string `maxminddb:"iso_code"`
Type string `maxminddb:"type"`
GeoNameID uint `maxminddb:"geoname_id"`
IsInEuropeanUnion bool `maxminddb:"is_in_european_union"`
} `maxminddb:"represented_country"`
Traits struct {
IsAnonymousProxy bool `maxminddb:"is_anonymous_proxy"`
@ -180,8 +180,8 @@ type AnonymousIP struct {
// The ASN struct corresponds to the data in the GeoLite2 ASN database.
type ASN struct {
AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"`
AutonomousSystemOrganization string `maxminddb:"autonomous_system_organization"`
AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"`
}
// The ConnectionType struct corresponds to the data in the GeoIP2
@ -197,12 +197,12 @@ type Domain struct {
// The ISP struct corresponds to the data in the GeoIP2 ISP database.
type ISP struct {
AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"`
AutonomousSystemOrganization string `maxminddb:"autonomous_system_organization"`
ISP string `maxminddb:"isp"`
MobileCountryCode string `maxminddb:"mobile_country_code"`
MobileNetworkCode string `maxminddb:"mobile_network_code"`
Organization string `maxminddb:"organization"`
AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"`
}
type databaseType int
@ -305,8 +305,7 @@ func getDBType(reader *maxminddb.Reader) (databaseType, error) {
"DBIP-Location-ISP (compat=Enterprise)",
"GeoIP2-Enterprise":
return isEnterprise | isCity | isCountry, nil
case "GeoIP2-ISP",
"GeoIP2-Precision-ISP":
case "GeoIP2-ISP", "GeoIP2-Precision-ISP":
return isISP | isASN, nil
default:
return 0, UnknownDatabaseTypeError{reader.Metadata.DatabaseType}

@ -1,26 +1,26 @@
[run]
deadline = "10m"
tests = true
[linters]
disable-all = true
enable = [
"asasalint",
"asciicheck",
"bidichk",
"bodyclose",
"containedctx",
"contextcheck",
"deadcode",
"depguard",
"dupword",
"durationcheck",
"errcheck",
"errchkjson",
"errname",
"errorlint",
# "exhaustive",
"exportloopref",
"forbidigo",
#"forcetypeassert",
"goconst",
"gocyclo",
"gocritic",
@ -46,7 +46,6 @@
"rowserrcheck",
"sqlclosecheck",
"staticcheck",
"structcheck",
"stylecheck",
"tenv",
"tparallel",
@ -54,40 +53,17 @@
"unconvert",
"unparam",
"unused",
"varcheck",
"usestdlibvars",
"vetshadow",
"wastedassign",
]
# Please note that we only use depguard for stdlib as gomodguard only
# supports modules currently. See https://github.com/ryancurrah/gomodguard/issues/12
[linters-settings.depguard]
list-type = "blacklist"
include-go-root = true
packages = [
# ioutil is deprecated. The functions have been moved elsewhere:
# https://golang.org/doc/go1.16#ioutil
"io/ioutil",
]
[[linters-settings.depguard.rules.main.deny]]
pkg = "io/ioutil"
desc = "Deprecated. Functions have been moved elsewhere."
[linters-settings.errcheck]
# Don't allow setting of error to the blank identifier. If there is a legtimate
# reason, there should be a nolint with an explanation.
check-blank = true
exclude-functions = [
# If we are rolling back a transaction, we are often already in an error
# state.
'(*database/sql.Tx).Rollback',
# It is reasonable to ignore errors if Cleanup fails in most cases.
'(*github.com/google/renameio/v2.PendingFile).Cleanup',
# We often don't care if removing a file failed (e.g., it doesn't exist)
'os.Remove',
'os.RemoveAll',
]
# Ignoring Close so that we don't have to have a bunch of
# `defer func() { _ = r.Close() }()` constructs when we
# don't actually care about the error.
@ -104,8 +80,10 @@
[linters-settings.forbidigo]
# Forbid the following identifiers
forbid = [
"^minFraud*",
"^maxMind*",
"Geoip", # use "GeoIP"
"^geoIP", # use "geoip"
"Maxmind", # use "MaxMind"
"^maxMind", # use "maxmind"
]
[linters-settings.gocritic]
@ -129,8 +107,7 @@
"commentedOutImport",
"commentFormatting",
"defaultCaseOrder",
# Revive's defer rule already captures this. This caught no extra cases.
# "deferInLoop",
"deferInLoop",
"deferUnlambda",
"deprecatedComment",
"docStub",
@ -149,17 +126,16 @@
"exitAfterDefer",
"exposedSyncMutex",
"externalErrorReassign",
# Given that all of our code runs on Linux and the / separate should
# work fine, this seems less important.
# "filepathJoin",
"filepathJoin",
"flagDeref",
"flagName",
"hexLiteral",
"httpNoBody",
"hugeParam",
"ifElseChain",
"importShadow",
"indexAlloc",
"initClause",
"ioutilDeprecated",
"mapKey",
"methodExprCall",
"nestingReduce",
@ -179,22 +155,20 @@
"redundantSprint",
"regexpMust",
"regexpPattern",
# This might be good, but I don't think we want to encourage
# significant changes to regexes as we port stuff from Perl.
# "regexpSimplify",
"regexpSimplify",
"returnAfterHttpError",
"ruleguard",
"singleCaseSwitch",
"sliceClear",
"sloppyLen",
# This seems like it might also be good, but a lot of existing code
# fails.
# "sloppyReassign",
"returnAfterHttpError",
"sloppyReassign",
"sloppyTestFuncName",
"sloppyTypeAssert",
"sortSlice",
"sprintfQuotedString",
"sqlQuery",
"stringsCompare",
"stringConcatSimplify",
"stringXbytes",
"switchTrue",
"syncMapLoadAndDelete",
@ -209,28 +183,40 @@
"underef",
"unlabelStmt",
"unlambda",
# I am not sure we would want this linter and a lot of existing
# code fails.
# "unnamedResult",
"unnecessaryBlock",
"unnecessaryDefer",
"unslice",
"valSwap",
"weakCond",
# Covered by nolintlint
# "whyNoLint"
"wrapperFunc",
"yodaStyleExpr",
# This requires explanations for "nolint" directives. This would be
# nice for gosec ones, but I am not sure we want it generally unless
# we can get the false positive rate lower.
# "whyNoLint"
]
[linters-settings.gofumpt]
extra-rules = true
lang-version = "1.18"
lang-version = "1.19"
[linters-settings.gosec]
excludes = [
# G104 - "Audit errors not checked." We use errcheck for this.
"G104",
# G304 - "Potential file inclusion via variable"
"G304",
# G306 - "Expect WriteFile permissions to be 0600 or less".
"G306",
# Prohibits defer (*os.File).Close, which we allow when reading from file.
"G307",
]
[linters-settings.govet]
"enable-all" = true
disable = ["shadow"]
[linters-settings.lll]
line-length = 120
@ -247,8 +233,6 @@
ignore-generated-header = true
severity = "warning"
# This might be nice but it is so common that it is hard
# to enable.
# [[linters-settings.revive.rules]]
# name = "add-constant"
@ -273,8 +257,10 @@
# [[linters-settings.revive.rules]]
# name = "cognitive-complexity"
# Probably a good rule, but we have a lot of names that
# only have case differences.
[[linters-settings.revive.rules]]
name = "comment-spacings"
arguments = ["easyjson", "nolint"]
# [[linters-settings.revive.rules]]
# name = "confusing-naming"
@ -293,6 +279,9 @@
# [[linters-settings.revive.rules]]
# name = "cyclomatic"
[[linters-settings.revive.rules]]
name = "datarace"
# [[linters-settings.revive.rules]]
# name = "deep-exit"
@ -332,8 +321,6 @@
# [[linters-settings.revive.rules]]
# name = "file-header"
# We have a lot of flag parameters. This linter probably makes
# a good point, but we would need some cleanup or a lot of nolints.
# [[linters-settings.revive.rules]]
# name = "flag-parameter"
@ -373,7 +360,6 @@
[[linters-settings.revive.rules]]
name = "modifies-value-receiver"
# We frequently use nested structs, particularly in tests.
# [[linters-settings.revive.rules]]
# name = "nested-structs"
@ -407,6 +393,9 @@
[[linters-settings.revive.rules]]
name = "superfluous-else"
[[linters-settings.revive.rules]]
name = "time-equal"
[[linters-settings.revive.rules]]
name = "time-naming"
@ -419,8 +408,6 @@
[[linters-settings.revive.rules]]
name = "unexported-return"
# This is covered elsewhere and we want to ignore some
# functions such as fmt.Fprintf.
# [[linters-settings.revive.rules]]
# name = "unhandled-error"
@ -433,14 +420,11 @@
[[linters-settings.revive.rules]]
name = "unused-parameter"
# We generally have unused receivers in tests for meeting the
# requirements of an interface.
# [[linters-settings.revive.rules]]
# name = "unused-receiver"
[[linters-settings.revive.rules]]
name = "unused-receiver"
# This probably makes sense after we upgrade to 1.18
# [[linters-settings.revive.rules]]
# name = "use-any"
[[linters-settings.revive.rules]]
name = "use-any"
[[linters-settings.revive.rules]]
name = "useless-break"
@ -457,16 +441,12 @@
[linters-settings.unparam]
check-exported = true
[issues]
exclude-use-default = false
[[issues.exclude-rules]]
linters = [
"govet"
]
# we want to enable almost all govet rules. It is easier to just filter out
# the ones we don't want:
#
# * fieldalignment - way too noisy. Although it is very useful in particular
# cases where we are trying to use as little memory as possible, having
# it go off on every struct isn't helpful.
# * shadow - although often useful, it complains about _many_ err
# shadowing assignments and some others where shadowing is clear.
text = "^(fieldalignment|shadow)"
path = "_test.go"
text = "^fieldalignment"

@ -151,12 +151,12 @@ func (d *decoder) decodeFromType(
result reflect.Value,
depth int,
) (uint, error) {
result = d.indirect(result)
result = indirect(result)
// For these types, size has a special meaning
switch dtype {
case _Bool:
return d.unmarshalBool(size, offset, result)
return unmarshalBool(size, offset, result)
case _Map:
return d.unmarshalMap(size, offset, result, depth)
case _Pointer:
@ -203,7 +203,7 @@ func (d *decoder) decodeFromTypeToDeserializer(
// For these types, size has a special meaning
switch dtype {
case _Bool:
v, offset := d.decodeBool(size, offset)
v, offset := decodeBool(size, offset)
return offset, dser.Bool(v)
case _Map:
return d.decodeMapToDeserializer(size, offset, dser, depth)
@ -255,14 +255,14 @@ func (d *decoder) decodeFromTypeToDeserializer(
}
}
func (d *decoder) unmarshalBool(size, offset uint, result reflect.Value) (uint, error) {
func unmarshalBool(size, offset uint, result reflect.Value) (uint, error) {
if size > 1 {
return 0, newInvalidDatabaseError(
"the MaxMind DB file's data section contains bad data (bool size of %v)",
size,
)
}
value, newOffset := d.decodeBool(size, offset)
value, newOffset := decodeBool(size, offset)
switch result.Kind() {
case reflect.Bool:
@ -281,7 +281,7 @@ func (d *decoder) unmarshalBool(size, offset uint, result reflect.Value) (uint,
// heavily based on encoding/json as my original version had a subtle
// bug. This method should be considered to be licensed under
// https://golang.org/LICENSE
func (d *decoder) indirect(result reflect.Value) reflect.Value {
func indirect(result reflect.Value) reflect.Value {
for {
// Load value from interface, but only if the result will be
// usefully addressable.
@ -415,7 +415,7 @@ func (d *decoder) unmarshalMap(
result reflect.Value,
depth int,
) (uint, error) {
result = d.indirect(result)
result = indirect(result)
switch result.Kind() {
default:
return 0, newUnmarshalTypeError("map", result.Type())
@ -425,7 +425,7 @@ func (d *decoder) unmarshalMap(
return d.decodeMap(size, offset, result, depth)
case reflect.Interface:
if result.NumMethod() == 0 {
rv := reflect.ValueOf(make(map[string]interface{}, size))
rv := reflect.ValueOf(make(map[string]any, size))
newOffset, err := d.decodeMap(size, offset, rv, depth)
result.Set(rv)
return newOffset, err
@ -458,7 +458,7 @@ func (d *decoder) unmarshalSlice(
return d.decodeSlice(size, offset, result, depth)
case reflect.Interface:
if result.NumMethod() == 0 {
a := []interface{}{}
a := []any{}
rv := reflect.ValueOf(&a).Elem()
newOffset, err := d.decodeSlice(size, offset, rv, depth)
result.Set(rv)
@ -551,7 +551,7 @@ func (d *decoder) unmarshalUint128(size, offset uint, result reflect.Value) (uin
return newOffset, newUnmarshalTypeError(value, result.Type())
}
func (d *decoder) decodeBool(size, offset uint) (bool, uint) {
func decodeBool(size, offset uint) (bool, uint) {
return size != 0, offset
}

@ -15,7 +15,7 @@ func newOffsetError() InvalidDatabaseError {
return InvalidDatabaseError{"unexpected end of database"}
}
func newInvalidDatabaseError(format string, args ...interface{}) InvalidDatabaseError {
func newInvalidDatabaseError(format string, args ...any) InvalidDatabaseError {
return InvalidDatabaseError{fmt.Sprintf(format, args...)}
}
@ -26,11 +26,11 @@ func (e InvalidDatabaseError) Error() string {
// UnmarshalTypeError is returned when the value in the database cannot be
// assigned to the specified data type.
type UnmarshalTypeError struct {
Value string // stringified copy of the database value that caused the error
Type reflect.Type // type of the value that could not be assign to
Type reflect.Type
Value string
}
func newUnmarshalTypeError(value interface{}, rType reflect.Type) UnmarshalTypeError {
func newUnmarshalTypeError(value any, rType reflect.Type) UnmarshalTypeError {
return UnmarshalTypeError{
Value: fmt.Sprintf("%v", value),
Type: rType,

@ -1,5 +1,5 @@
//go:build !windows && !appengine && !plan9
// +build !windows,!appengine,!plan9
//go:build !windows && !appengine && !plan9 && !js && !wasip1
// +build !windows,!appengine,!plan9,!js,!wasip1
package maxminddb

@ -1,3 +1,4 @@
//go:build windows && !appengine
// +build windows,!appengine
package maxminddb

@ -25,14 +25,14 @@ var metadataStartMarker = []byte("\xAB\xCD\xEFMaxMind.com")
// All of the methods on Reader are thread-safe. The struct may be safely
// shared across goroutines.
type Reader struct {
hasMappedFile bool
buffer []byte
nodeReader nodeReader
buffer []byte
decoder decoder
Metadata Metadata
ipv4Start uint
ipv4StartBitDepth int
nodeOffsetMult uint
hasMappedFile bool
}
// Metadata holds the metadata decoded from the MaxMind DB file. In particular
@ -40,13 +40,13 @@ type Reader struct {
// type and description, the IP version supported, and a slice of the natural
// languages included.
type Metadata struct {
Description map[string]string `maxminddb:"description"`
DatabaseType string `maxminddb:"database_type"`
Languages []string `maxminddb:"languages"`
BinaryFormatMajorVersion uint `maxminddb:"binary_format_major_version"`
BinaryFormatMinorVersion uint `maxminddb:"binary_format_minor_version"`
BuildEpoch uint `maxminddb:"build_epoch"`
DatabaseType string `maxminddb:"database_type"`
Description map[string]string `maxminddb:"description"`
IPVersion uint `maxminddb:"ip_version"`
Languages []string `maxminddb:"languages"`
NodeCount uint `maxminddb:"node_count"`
RecordSize uint `maxminddb:"record_size"`
}
@ -130,7 +130,7 @@ func (r *Reader) setIPv4Start() {
// because of type differences, an UnmarshalTypeError is returned. If the
// database is invalid or otherwise cannot be read, an InvalidDatabaseError
// is returned.
func (r *Reader) Lookup(ip net.IP, result interface{}) error {
func (r *Reader) Lookup(ip net.IP, result any) error {
if r.buffer == nil {
return errors.New("cannot call Lookup on a closed database")
}
@ -152,7 +152,7 @@ func (r *Reader) Lookup(ip net.IP, result interface{}) error {
// cannot be read, an InvalidDatabaseError is returned.
func (r *Reader) LookupNetwork(
ip net.IP,
result interface{},
result any,
) (network *net.IPNet, ok bool, err error) {
if r.buffer == nil {
return nil, false, errors.New("cannot call Lookup on a closed database")
@ -204,7 +204,7 @@ func (r *Reader) cidr(ip net.IP, prefixLength int) *net.IPNet {
// Decode the record at |offset| into |result|. The result value pointed to
// must be a data value that corresponds to a record in the database. This may
// include a struct representation of the data, a map capable of holding the
// data or an empty interface{} value.
// data or an empty any value.
//
// If result is a pointer to a struct, the struct need not include a field
// for every value that may be in the database. If a field is not present in
@ -217,14 +217,14 @@ func (r *Reader) cidr(ip net.IP, prefixLength int) *net.IPNet {
// the City database, all records of the same country will reference a
// single representative record for that country. This uintptr behavior allows
// clients to leverage this normalization in their own sub-record caching.
func (r *Reader) Decode(offset uintptr, result interface{}) error {
func (r *Reader) Decode(offset uintptr, result any) error {
if r.buffer == nil {
return errors.New("cannot call Decode on a closed database")
}
return r.decode(offset, result)
}
func (r *Reader) decode(offset uintptr, result interface{}) error {
func (r *Reader) decode(offset uintptr, result any) error {
rv := reflect.ValueOf(result)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
return errors.New("result param must be a pointer")
@ -292,7 +292,7 @@ func (r *Reader) traverseTree(ip net.IP, node, bitCount uint) (uint, int) {
return node, int(i)
}
func (r *Reader) retrieveData(pointer uint, result interface{}) error {
func (r *Reader) retrieveData(pointer uint, result any) error {
offset, err := r.resolveDataPointer(pointer)
if err != nil {
return err

@ -1,28 +0,0 @@
// +build appengine plan9
package maxminddb
import "io/ioutil"
// Open takes a string path to a MaxMind DB file and returns a Reader
// structure or an error. The database file is opened using a memory map,
// except on Google App Engine where mmap is not supported; there the database
// is loaded into memory. Use the Close method on the Reader object to return
// the resources to the system.
func Open(file string) (*Reader, error) {
bytes, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
return FromBytes(bytes)
}
// Close unmaps the database file from virtual memory and returns the
// resources to the system. If called on a Reader opened using FromBytes
// or Open on Google App Engine, this method sets the underlying buffer
// to nil, returning the resources to the system.
func (r *Reader) Close() error {
r.buffer = nil
return nil
}

@ -0,0 +1,26 @@
//go:build appengine || plan9 || js || wasip1
// +build appengine plan9 js wasip1
package maxminddb
import "io/ioutil"
// Open takes a string path to a MaxMind DB file and returns a Reader
// structure or an error. The database file is opened using a memory map
// on supported platforms. On platforms without memory map support, such
// as WebAssembly or Google App Engine, the database is loaded into memory.
// Use the Close method on the Reader object to return the resources to the system.
func Open(file string) (*Reader, error) {
bytes, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
return FromBytes(bytes)
}
// Close returns the resources used by the database to the system.
func (r *Reader) Close() error {
r.buffer = nil
return nil
}

@ -1,5 +1,5 @@
//go:build !appengine && !plan9
// +build !appengine,!plan9
//go:build !appengine && !plan9 && !js && !wasip1
// +build !appengine,!plan9,!js,!wasip1
package maxminddb
@ -9,10 +9,10 @@ import (
)
// Open takes a string path to a MaxMind DB file and returns a Reader
// structure or an error. The database file is opened using a memory map,
// except on Google App Engine where mmap is not supported; there the database
// is loaded into memory. Use the Close method on the Reader object to return
// the resources to the system.
// structure or an error. The database file is opened using a memory map
// on supported platforms. On platforms without memory map support, such
// as WebAssembly or Google App Engine, the database is loaded into memory.
// Use the Close method on the Reader object to return the resources to the system.
func Open(file string) (*Reader, error) {
mapFile, err := os.Open(file)
if err != nil {
@ -51,9 +51,7 @@ func Open(file string) (*Reader, error) {
return reader, nil
}
// Close unmaps the database file from virtual memory and returns the
// resources to the system. If called on a Reader opened using FromBytes
// or Open on Google App Engine, this method does nothing.
// Close returns the resources used by the database to the system.
func (r *Reader) Close() error {
var err error
if r.hasMappedFile {

@ -14,11 +14,10 @@ type netNode struct {
// Networks represents a set of subnets that we are iterating over.
type Networks struct {
reader *Reader
nodes []netNode // Nodes we still have to visit.
lastNode netNode
err error
err error
reader *Reader
nodes []netNode
lastNode netNode
skipAliasedNetworks bool
}
@ -159,7 +158,7 @@ func (n *Networks) Next() bool {
// Network returns the current network or an error if there is a problem
// decoding the data for the network. It takes a pointer to a result value to
// decode the network's data into.
func (n *Networks) Network(result interface{}) (*net.IPNet, error) {
func (n *Networks) Network(result any) (*net.IPNet, error) {
if n.err != nil {
return nil, n.err
}

@ -137,7 +137,7 @@ func (v *verifier) verifyDataSection(offsets map[uint]bool) error {
var offset uint
bufferLen := uint(len(decoder.buffer))
for offset < bufferLen {
var data interface{}
var data any
rv := reflect.ValueOf(&data)
newOffset, err := decoder.decode(offset, rv, 0)
if err != nil {
@ -189,8 +189,8 @@ func (v *verifier) verifyDataSection(offsets map[uint]bool) error {
func testError(
field string,
expected interface{},
actual interface{},
expected any,
actual any,
) error {
return newInvalidDatabaseError(
"%v - Expected: %v Actual: %v",

@ -1,24 +0,0 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

@ -1,10 +0,0 @@
language: go
go_import_path: github.com/pkg/errors
go:
- 1.11.x
- 1.12.x
- 1.13.x
- tip
script:
- make check

@ -1,23 +0,0 @@
Copyright (c) 2015, Dave Cheney <dave@cheney.net>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

@ -1,44 +0,0 @@
PKGS := github.com/pkg/errors
SRCDIRS := $(shell go list -f '{{.Dir}}' $(PKGS))
GO := go
check: test vet gofmt misspell unconvert staticcheck ineffassign unparam
test:
$(GO) test $(PKGS)
vet: | test
$(GO) vet $(PKGS)
staticcheck:
$(GO) get honnef.co/go/tools/cmd/staticcheck
staticcheck -checks all $(PKGS)
misspell:
$(GO) get github.com/client9/misspell/cmd/misspell
misspell \
-locale GB \
-error \
*.md *.go
unconvert:
$(GO) get github.com/mdempsky/unconvert
unconvert -v $(PKGS)
ineffassign:
$(GO) get github.com/gordonklaus/ineffassign
find $(SRCDIRS) -name '*.go' | xargs ineffassign
pedantic: check errcheck
unparam:
$(GO) get mvdan.cc/unparam
unparam ./...
errcheck:
$(GO) get github.com/kisielk/errcheck
errcheck $(PKGS)
gofmt:
@echo Checking code is gofmted
@test -z "$(shell gofmt -s -l -d -e $(SRCDIRS) | tee /dev/stderr)"

@ -1,59 +0,0 @@
# errors [![Travis-CI](https://travis-ci.org/pkg/errors.svg)](https://travis-ci.org/pkg/errors) [![AppVeyor](https://ci.appveyor.com/api/projects/status/b98mptawhudj53ep/branch/master?svg=true)](https://ci.appveyor.com/project/davecheney/errors/branch/master) [![GoDoc](https://godoc.org/github.com/pkg/errors?status.svg)](http://godoc.org/github.com/pkg/errors) [![Report card](https://goreportcard.com/badge/github.com/pkg/errors)](https://goreportcard.com/report/github.com/pkg/errors) [![Sourcegraph](https://sourcegraph.com/github.com/pkg/errors/-/badge.svg)](https://sourcegraph.com/github.com/pkg/errors?badge)
Package errors provides simple error handling primitives.
`go get github.com/pkg/errors`
The traditional error handling idiom in Go is roughly akin to
```go
if err != nil {
return err
}
```
which applied recursively up the call stack results in error reports without context or debugging information. The errors package allows programmers to add context to the failure path in their code in a way that does not destroy the original value of the error.
## Adding context to an error
The errors.Wrap function returns a new error that adds context to the original error. For example
```go
_, err := ioutil.ReadAll(r)
if err != nil {
return errors.Wrap(err, "read failed")
}
```
## Retrieving the cause of an error
Using `errors.Wrap` constructs a stack of errors, adding context to the preceding error. Depending on the nature of the error it may be necessary to reverse the operation of errors.Wrap to retrieve the original error for inspection. Any error value which implements this interface can be inspected by `errors.Cause`.
```go
type causer interface {
Cause() error
}
```
`errors.Cause` will recursively retrieve the topmost error which does not implement `causer`, which is assumed to be the original cause. For example:
```go
switch err := errors.Cause(err).(type) {
case *MyError:
// handle specifically
default:
// unknown error
}
```
[Read the package documentation for more information](https://godoc.org/github.com/pkg/errors).
## Roadmap
With the upcoming [Go2 error proposals](https://go.googlesource.com/proposal/+/master/design/go2draft.md) this package is moving into maintenance mode. The roadmap for a 1.0 release is as follows:
- 0.9. Remove pre Go 1.9 and Go 1.10 support, address outstanding pull requests (if possible)
- 1.0. Final release.
## Contributing
Because of the Go2 errors changes, this package is not accepting proposals for new functionality. With that said, we welcome pull requests, bug fixes and issue reports.
Before sending a PR, please discuss your change by raising an issue.
## License
BSD-2-Clause

@ -1,32 +0,0 @@
version: build-{build}.{branch}
clone_folder: C:\gopath\src\github.com\pkg\errors
shallow_clone: true # for startup speed
environment:
GOPATH: C:\gopath
platform:
- x64
# http://www.appveyor.com/docs/installed-software
install:
# some helpful output for debugging builds
- go version
- go env
# pre-installed MinGW at C:\MinGW is 32bit only
# but MSYS2 at C:\msys64 has mingw64
- set PATH=C:\msys64\mingw64\bin;%PATH%
- gcc --version
- g++ --version
build_script:
- go install -v ./...
test_script:
- set PATH=C:\gopath\bin;%PATH%
- go test -v ./...
#artifacts:
# - path: '%GOPATH%\bin\*.exe'
deploy: off

@ -1,288 +0,0 @@
// Package errors provides simple error handling primitives.
//
// The traditional error handling idiom in Go is roughly akin to
//
// if err != nil {
// return err
// }
//
// which when applied recursively up the call stack results in error reports
// without context or debugging information. The errors package allows
// programmers to add context to the failure path in their code in a way
// that does not destroy the original value of the error.
//
// Adding context to an error
//
// The errors.Wrap function returns a new error that adds context to the
// original error by recording a stack trace at the point Wrap is called,
// together with the supplied message. For example
//
// _, err := ioutil.ReadAll(r)
// if err != nil {
// return errors.Wrap(err, "read failed")
// }
//
// If additional control is required, the errors.WithStack and
// errors.WithMessage functions destructure errors.Wrap into its component
// operations: annotating an error with a stack trace and with a message,
// respectively.
//
// Retrieving the cause of an error
//
// Using errors.Wrap constructs a stack of errors, adding context to the
// preceding error. Depending on the nature of the error it may be necessary
// to reverse the operation of errors.Wrap to retrieve the original error
// for inspection. Any error value which implements this interface
//
// type causer interface {
// Cause() error
// }
//
// can be inspected by errors.Cause. errors.Cause will recursively retrieve
// the topmost error that does not implement causer, which is assumed to be
// the original cause. For example:
//
// switch err := errors.Cause(err).(type) {
// case *MyError:
// // handle specifically
// default:
// // unknown error
// }
//
// Although the causer interface is not exported by this package, it is
// considered a part of its stable public interface.
//
// Formatted printing of errors
//
// All error values returned from this package implement fmt.Formatter and can
// be formatted by the fmt package. The following verbs are supported:
//
// %s print the error. If the error has a Cause it will be
// printed recursively.
// %v see %s
// %+v extended format. Each Frame of the error's StackTrace will
// be printed in detail.
//
// Retrieving the stack trace of an error or wrapper
//
// New, Errorf, Wrap, and Wrapf record a stack trace at the point they are
// invoked. This information can be retrieved with the following interface:
//
// type stackTracer interface {
// StackTrace() errors.StackTrace
// }
//
// The returned errors.StackTrace type is defined as
//
// type StackTrace []Frame
//
// The Frame type represents a call site in the stack trace. Frame supports
// the fmt.Formatter interface that can be used for printing information about
// the stack trace of this error. For example:
//
// if err, ok := err.(stackTracer); ok {
// for _, f := range err.StackTrace() {
// fmt.Printf("%+s:%d\n", f, f)
// }
// }
//
// Although the stackTracer interface is not exported by this package, it is
// considered a part of its stable public interface.
//
// See the documentation for Frame.Format for more details.
package errors
import (
"fmt"
"io"
)
// New returns an error with the supplied message.
// New also records the stack trace at the point it was called.
func New(message string) error {
return &fundamental{
msg: message,
stack: callers(),
}
}
// Errorf formats according to a format specifier and returns the string
// as a value that satisfies error.
// Errorf also records the stack trace at the point it was called.
func Errorf(format string, args ...interface{}) error {
return &fundamental{
msg: fmt.Sprintf(format, args...),
stack: callers(),
}
}
// fundamental is an error that has a message and a stack, but no caller.
type fundamental struct {
msg string
*stack
}
func (f *fundamental) Error() string { return f.msg }
func (f *fundamental) Format(s fmt.State, verb rune) {
switch verb {
case 'v':
if s.Flag('+') {
io.WriteString(s, f.msg)
f.stack.Format(s, verb)
return
}
fallthrough
case 's':
io.WriteString(s, f.msg)
case 'q':
fmt.Fprintf(s, "%q", f.msg)
}
}
// WithStack annotates err with a stack trace at the point WithStack was called.
// If err is nil, WithStack returns nil.
func WithStack(err error) error {
if err == nil {
return nil
}
return &withStack{
err,
callers(),
}
}
type withStack struct {
error
*stack
}
func (w *withStack) Cause() error { return w.error }
// Unwrap provides compatibility for Go 1.13 error chains.
func (w *withStack) Unwrap() error { return w.error }
func (w *withStack) Format(s fmt.State, verb rune) {
switch verb {
case 'v':
if s.Flag('+') {
fmt.Fprintf(s, "%+v", w.Cause())
w.stack.Format(s, verb)
return
}
fallthrough
case 's':
io.WriteString(s, w.Error())
case 'q':
fmt.Fprintf(s, "%q", w.Error())
}
}
// Wrap returns an error annotating err with a stack trace
// at the point Wrap is called, and the supplied message.
// If err is nil, Wrap returns nil.
func Wrap(err error, message string) error {
if err == nil {
return nil
}
err = &withMessage{
cause: err,
msg: message,
}
return &withStack{
err,
callers(),
}
}
// Wrapf returns an error annotating err with a stack trace
// at the point Wrapf is called, and the format specifier.
// If err is nil, Wrapf returns nil.
func Wrapf(err error, format string, args ...interface{}) error {
if err == nil {
return nil
}
err = &withMessage{
cause: err,
msg: fmt.Sprintf(format, args...),
}
return &withStack{
err,
callers(),
}
}
// WithMessage annotates err with a new message.
// If err is nil, WithMessage returns nil.
func WithMessage(err error, message string) error {
if err == nil {
return nil
}
return &withMessage{
cause: err,
msg: message,
}
}
// WithMessagef annotates err with the format specifier.
// If err is nil, WithMessagef returns nil.
func WithMessagef(err error, format string, args ...interface{}) error {
if err == nil {
return nil
}
return &withMessage{
cause: err,
msg: fmt.Sprintf(format, args...),
}
}
type withMessage struct {
cause error
msg string
}
func (w *withMessage) Error() string { return w.msg + ": " + w.cause.Error() }
func (w *withMessage) Cause() error { return w.cause }
// Unwrap provides compatibility for Go 1.13 error chains.
func (w *withMessage) Unwrap() error { return w.cause }
func (w *withMessage) Format(s fmt.State, verb rune) {
switch verb {
case 'v':
if s.Flag('+') {
fmt.Fprintf(s, "%+v\n", w.Cause())
io.WriteString(s, w.msg)
return
}
fallthrough
case 's', 'q':
io.WriteString(s, w.Error())
}
}
// Cause returns the underlying cause of the error, if possible.
// An error value has a cause if it implements the following
// interface:
//
// type causer interface {
// Cause() error
// }
//
// If the error does not implement Cause, the original error will
// be returned. If the error is nil, nil will be returned without further
// investigation.
func Cause(err error) error {
type causer interface {
Cause() error
}
for err != nil {
cause, ok := err.(causer)
if !ok {
break
}
err = cause.Cause()
}
return err
}

@ -1,38 +0,0 @@
// +build go1.13
package errors
import (
stderrors "errors"
)
// Is reports whether any error in err's chain matches target.
//
// The chain consists of err itself followed by the sequence of errors obtained by
// repeatedly calling Unwrap.
//
// An error is considered to match a target if it is equal to that target or if
// it implements a method Is(error) bool such that Is(target) returns true.
func Is(err, target error) bool { return stderrors.Is(err, target) }
// As finds the first error in err's chain that matches target, and if so, sets
// target to that error value and returns true.
//
// The chain consists of err itself followed by the sequence of errors obtained by
// repeatedly calling Unwrap.
//
// An error matches target if the error's concrete value is assignable to the value
// pointed to by target, or if the error has a method As(interface{}) bool such that
// As(target) returns true. In the latter case, the As method is responsible for
// setting target.
//
// As will panic if target is not a non-nil pointer to either a type that implements
// error, or to any interface type. As returns false if err is nil.
func As(err error, target interface{}) bool { return stderrors.As(err, target) }
// Unwrap returns the result of calling the Unwrap method on err, if err's
// type contains an Unwrap method returning error.
// Otherwise, Unwrap returns nil.
func Unwrap(err error) error {
return stderrors.Unwrap(err)
}

@ -1,177 +0,0 @@
package errors
import (
"fmt"
"io"
"path"
"runtime"
"strconv"
"strings"
)
// Frame represents a program counter inside a stack frame.
// For historical reasons if Frame is interpreted as a uintptr
// its value represents the program counter + 1.
type Frame uintptr
// pc returns the program counter for this frame;
// multiple frames may have the same PC value.
func (f Frame) pc() uintptr { return uintptr(f) - 1 }
// file returns the full path to the file that contains the
// function for this Frame's pc.
func (f Frame) file() string {
fn := runtime.FuncForPC(f.pc())
if fn == nil {
return "unknown"
}
file, _ := fn.FileLine(f.pc())
return file
}
// line returns the line number of source code of the
// function for this Frame's pc.
func (f Frame) line() int {
fn := runtime.FuncForPC(f.pc())
if fn == nil {
return 0
}
_, line := fn.FileLine(f.pc())
return line
}
// name returns the name of this function, if known.
func (f Frame) name() string {
fn := runtime.FuncForPC(f.pc())
if fn == nil {
return "unknown"
}
return fn.Name()
}
// Format formats the frame according to the fmt.Formatter interface.
//
// %s source file
// %d source line
// %n function name
// %v equivalent to %s:%d
//
// Format accepts flags that alter the printing of some verbs, as follows:
//
// %+s function name and path of source file relative to the compile time
// GOPATH separated by \n\t (<funcname>\n\t<path>)
// %+v equivalent to %+s:%d
func (f Frame) Format(s fmt.State, verb rune) {
switch verb {
case 's':
switch {
case s.Flag('+'):
io.WriteString(s, f.name())
io.WriteString(s, "\n\t")
io.WriteString(s, f.file())
default:
io.WriteString(s, path.Base(f.file()))
}
case 'd':
io.WriteString(s, strconv.Itoa(f.line()))
case 'n':
io.WriteString(s, funcname(f.name()))
case 'v':
f.Format(s, 's')
io.WriteString(s, ":")
f.Format(s, 'd')
}
}
// MarshalText formats a stacktrace Frame as a text string. The output is the
// same as that of fmt.Sprintf("%+v", f), but without newlines or tabs.
func (f Frame) MarshalText() ([]byte, error) {
name := f.name()
if name == "unknown" {
return []byte(name), nil
}
return []byte(fmt.Sprintf("%s %s:%d", name, f.file(), f.line())), nil
}
// StackTrace is stack of Frames from innermost (newest) to outermost (oldest).
type StackTrace []Frame
// Format formats the stack of Frames according to the fmt.Formatter interface.
//
// %s lists source files for each Frame in the stack
// %v lists the source file and line number for each Frame in the stack
//
// Format accepts flags that alter the printing of some verbs, as follows:
//
// %+v Prints filename, function, and line number for each Frame in the stack.
func (st StackTrace) Format(s fmt.State, verb rune) {
switch verb {
case 'v':
switch {
case s.Flag('+'):
for _, f := range st {
io.WriteString(s, "\n")
f.Format(s, verb)
}
case s.Flag('#'):
fmt.Fprintf(s, "%#v", []Frame(st))
default:
st.formatSlice(s, verb)
}
case 's':
st.formatSlice(s, verb)
}
}
// formatSlice will format this StackTrace into the given buffer as a slice of
// Frame, only valid when called with '%s' or '%v'.
func (st StackTrace) formatSlice(s fmt.State, verb rune) {
io.WriteString(s, "[")
for i, f := range st {
if i > 0 {
io.WriteString(s, " ")
}
f.Format(s, verb)
}
io.WriteString(s, "]")
}
// stack represents a stack of program counters.
type stack []uintptr
func (s *stack) Format(st fmt.State, verb rune) {
switch verb {
case 'v':
switch {
case st.Flag('+'):
for _, pc := range *s {
f := Frame(pc)
fmt.Fprintf(st, "\n%+v", f)
}
}
}
}
func (s *stack) StackTrace() StackTrace {
f := make([]Frame, len(*s))
for i := 0; i < len(f); i++ {
f[i] = Frame((*s)[i])
}
return f
}
func callers() *stack {
const depth = 32
var pcs [depth]uintptr
n := runtime.Callers(3, pcs[:])
var st stack = pcs[0:n]
return &st
}
// funcname removes the path prefix component of a function's name reported by func.Name().
func funcname(name string) string {
i := strings.LastIndex(name, "/")
name = name[i+1:]
i = strings.Index(name, ".")
return name[i+1:]
}

@ -1,4 +1,8 @@
# Changelog
## 7.17.0
* 优化
* 对象存储UC 服务相关请求支持主备重试
## 7.16.0
* 新增
* 对象存储,`BucketManager` `BucketsV4` 获取该用户的指定区域内的空间信息,注意该 API 以分页形式返回 Bucket 列表

@ -17,7 +17,7 @@ github.com/qiniu/go-sdk
在您的项目中的 `go.mod` 文件内添加这行代码
```
require github.com/qiniu/go-sdk/v7 v7.16.0
require github.com/qiniu/go-sdk/v7 v7.17.0
```
并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。

@ -24,7 +24,8 @@ const (
AuthorizationPrefixQBox = "QBox "
)
// 七牛鉴权类用于生成Qbox, Qiniu, Upload签名
// 七牛鉴权类用于生成Qbox, Qiniu, Upload签名
//
// AK/SK可以从 https://portal.qiniu.com/user/key 获取
type Credentials struct {
AccessKey string

@ -20,7 +20,7 @@ import (
"github.com/qiniu/go-sdk/v7/reqid"
)
var UserAgent = "Golang qiniu/client package"
var UserAgent = getUserAgentWithAppName("default")
var DefaultClient = Client{&http.Client{Transport: http.DefaultTransport}}
// 用来打印调试信息
@ -41,11 +41,15 @@ func TurnOnDebug() {
// userApp should be [A-Za-z0-9_\ \-\.]*
func SetAppName(userApp string) error {
UserAgent = fmt.Sprintf(
"QiniuGo/%s (%s; %s; %s) %s", conf.Version, runtime.GOOS, runtime.GOARCH, userApp, runtime.Version())
UserAgent = getUserAgentWithAppName(userApp)
return nil
}
func getUserAgentWithAppName(userApp string) string {
return fmt.Sprintf("QiniuGo/%s (%s; %s; %s) %s",
conf.Version, runtime.GOOS, runtime.GOARCH, userApp, runtime.Version())
}
// --------------------------------------------------------------------
func newRequest(ctx context.Context, method, reqUrl string, headers http.Header, body io.Reader) (req *http.Request, err error) {
@ -281,7 +285,7 @@ func CallRet(ctx context.Context, ret interface{}, resp *http.Response) (err err
}
if resp.StatusCode/100 == 2 {
if ret != nil && resp.ContentLength != 0 {
err = decodeJsonFromReader(resp.Body, ret)
err = DecodeJsonFromReader(resp.Body, ret)
if err != nil {
return
}

@ -24,7 +24,7 @@ func decodeJsonFromData(data []byte, v interface{}) error {
return nil
}
func decodeJsonFromReader(reader io.Reader, v interface{}) error {
func DecodeJsonFromReader(reader io.Reader, v interface{}) error {
buf := new(bytes.Buffer)
t := io.TeeReader(reader, buf)
err := json.NewDecoder(t).Decode(v)

@ -5,7 +5,7 @@ import (
"strings"
)
const Version = "7.16.0"
const Version = "7.17.0"
const (
CONTENT_TYPE_JSON = "application/json"

@ -1,5 +1,4 @@
/*
github.com/qiniu/go-sdk Go SDK v7.x
CDNGo>=1.10.0
@ -7,6 +6,5 @@
Go SDK
auth conf cdnCDNstorage
*/
package api

@ -0,0 +1,98 @@
package clientv2
import (
clientV1 "github.com/qiniu/go-sdk/v7/client"
"net/http"
"sort"
)
type Client interface {
Do(req *http.Request) (*http.Response, error)
}
type Handler func(req *http.Request) (*http.Response, error)
type client struct {
coreClient Client
interceptors []Interceptor
}
func NewClient(cli Client, interceptors ...Interceptor) Client {
if cli == nil {
cli = http.DefaultClient
}
var is interceptorList = interceptors
is = append(is, newDefaultHeaderInterceptor())
is = append(is, newDebugInterceptor())
sort.Sort(is)
// 反转
for i, j := 0, len(is)-1; i < j; i, j = i+1, j-1 {
is[i], is[j] = is[j], is[i]
}
return &client{
coreClient: cli,
interceptors: is,
}
}
func (c *client) Do(req *http.Request) (*http.Response, error) {
handler := func(req *http.Request) (*http.Response, error) {
return c.coreClient.Do(req)
}
interceptors := c.interceptors
for _, interceptor := range interceptors {
h := handler
i := interceptor
handler = func(r *http.Request) (*http.Response, error) {
return i.Intercept(r, h)
}
}
resp, err := handler(req)
if err != nil {
return resp, err
}
if resp == nil {
return nil, &clientV1.ErrorInfo{
Code: -999,
Err: "unknown error, no response",
}
}
if resp.StatusCode/100 != 2 {
return resp, clientV1.ResponseError(resp)
}
return resp, nil
}
func Do(c Client, options RequestParams) (*http.Response, error) {
req, err := NewRequest(options)
if err != nil {
return nil, err
}
return c.Do(req)
}
func DoAndDecodeJsonResponse(c Client, options RequestParams, ret interface{}) (*http.Response, error) {
resp, err := Do(c, options)
if err != nil {
return resp, err
}
if ret == nil || resp.ContentLength == 0 {
return resp, nil
}
if err = clientV1.DecodeJsonFromReader(resp.Body, ret); err != nil {
return resp, err
}
return resp, nil
}

@ -0,0 +1,70 @@
package clientv2
import (
"net/http"
)
const (
InterceptorPriorityDefault InterceptorPriority = 100
InterceptorPriorityRetryHosts InterceptorPriority = 200
InterceptorPriorityRetrySimple InterceptorPriority = 300
InterceptorPrioritySetHeader InterceptorPriority = 400
InterceptorPriorityNormal InterceptorPriority = 500
InterceptorPriorityAuth InterceptorPriority = 600
InterceptorPriorityDebug InterceptorPriority = 700
)
type InterceptorPriority int
type Interceptor interface {
// Priority 数字越小优先级越高
Priority() InterceptorPriority
// Intercept 拦截处理函数
Intercept(req *http.Request, handler Handler) (*http.Response, error)
}
type interceptorList []Interceptor
func (l interceptorList) Less(i, j int) bool {
return l[i].Priority() < l[j].Priority()
}
func (l interceptorList) Swap(i, j int) {
l[i], l[j] = l[j], l[i]
}
func (l interceptorList) Len() int {
return len(l)
}
type simpleInterceptor struct {
priority InterceptorPriority
handler func(req *http.Request, handler Handler) (*http.Response, error)
}
func NewSimpleInterceptor(interceptorHandler func(req *http.Request, handler Handler) (*http.Response, error)) Interceptor {
return NewSimpleInterceptorWithPriority(InterceptorPriorityNormal, interceptorHandler)
}
func NewSimpleInterceptorWithPriority(priority InterceptorPriority, interceptorHandler func(req *http.Request, handler Handler) (*http.Response, error)) Interceptor {
if priority <= 0 {
priority = InterceptorPriorityNormal
}
return &simpleInterceptor{
priority: priority,
handler: interceptorHandler,
}
}
func (interceptor *simpleInterceptor) Priority() InterceptorPriority {
return interceptor.priority
}
func (interceptor *simpleInterceptor) Intercept(req *http.Request, handler Handler) (*http.Response, error) {
if interceptor == nil || interceptor.handler == nil {
return handler(req)
}
return interceptor.handler(req, handler)
}

@ -0,0 +1,38 @@
package clientv2
import (
"github.com/qiniu/go-sdk/v7/auth"
"net/http"
)
type AuthConfig struct {
Credentials auth.Credentials //
TokenType auth.TokenType // 不包含上传
}
type authInterceptor struct {
config AuthConfig
}
func NewAuthInterceptor(config AuthConfig) Interceptor {
return &authInterceptor{
config: config,
}
}
func (interceptor *authInterceptor) Priority() InterceptorPriority {
return InterceptorPriorityAuth
}
func (interceptor *authInterceptor) Intercept(req *http.Request, handler Handler) (*http.Response, error) {
if interceptor == nil || req == nil {
return handler(req)
}
err := interceptor.config.Credentials.AddToken(interceptor.config.TokenType, req)
if err != nil {
return nil, err
}
return handler(req)
}

@ -0,0 +1,209 @@
package clientv2
import (
"crypto/tls"
"fmt"
clientV1 "github.com/qiniu/go-sdk/v7/client"
"github.com/qiniu/go-sdk/v7/internal/log"
"net/http"
"net/http/httptrace"
"net/http/httputil"
)
type DebugLevel int
const (
DebugLevelPrintNone DebugLevel = 0
DebugLevelPrintNormal DebugLevel = 1
DebugLevelPrintDetail DebugLevel = 2
)
var (
printRequestTrace = false
printRequestLevel *DebugLevel = nil
printResponseLevel *DebugLevel = nil
)
func PrintRequestTrace(isPrint bool) {
printRequestTrace = isPrint
}
func IsPrintRequestTrace() bool {
return printRequestTrace
}
func PrintRequest(level DebugLevel) {
printRequestLevel = &level
}
func IsPrintRequest() bool {
if printRequestLevel != nil {
return *printRequestLevel == DebugLevelPrintNormal || *printRequestLevel == DebugLevelPrintDetail
}
return clientV1.DebugMode
}
func IsPrintRequestBody() bool {
if printRequestLevel != nil {
return *printRequestLevel == DebugLevelPrintDetail
}
return clientV1.DeepDebugInfo
}
func PrintResponse(level DebugLevel) {
printResponseLevel = &level
}
func IsPrintResponse() bool {
if printResponseLevel != nil {
return *printResponseLevel == DebugLevelPrintNormal || *printResponseLevel == DebugLevelPrintDetail
}
return clientV1.DebugMode
}
func IsPrintResponseBody() bool {
if printResponseLevel != nil {
return *printResponseLevel == DebugLevelPrintDetail
}
return clientV1.DeepDebugInfo
}
type debugInterceptor struct {
}
func newDebugInterceptor() Interceptor {
return &debugInterceptor{}
}
func (interceptor *debugInterceptor) Priority() InterceptorPriority {
return InterceptorPriorityDebug
}
func (interceptor *debugInterceptor) Intercept(req *http.Request, handler Handler) (*http.Response, error) {
if interceptor == nil {
return handler(req)
}
label := interceptor.requestLabel(req)
if e := interceptor.printRequest(label, req); e != nil {
return nil, e
}
req = interceptor.printRequestTrace(label, req)
resp, err := handler(req)
if e := interceptor.printResponse(label, resp); e != nil {
return nil, e
}
return resp, err
}
func (interceptor *debugInterceptor) requestLabel(req *http.Request) string {
if req == nil || req.URL == nil {
return ""
}
return fmt.Sprintf("Url:%s", req.URL.String())
}
func (interceptor *debugInterceptor) printRequest(label string, req *http.Request) error {
if req == nil {
return nil
}
printReq := IsPrintRequest()
if !printReq {
return nil
}
info := label + " request:\n"
d, dErr := httputil.DumpRequest(req, IsPrintRequestBody())
if dErr != nil {
return dErr
}
info += string(d) + "\n"
log.Debug(info)
return nil
}
func (interceptor *debugInterceptor) printRequestTrace(label string, req *http.Request) *http.Request {
if !IsPrintRequestTrace() || req == nil {
return req
}
label += "\n"
trace := &httptrace.ClientTrace{
GetConn: func(hostPort string) {
log.Debug(label + fmt.Sprintf("GetConn, %s \n", hostPort))
},
GotConn: func(connInfo httptrace.GotConnInfo) {
remoteAddr := connInfo.Conn.RemoteAddr()
log.Debug(label + fmt.Sprintf("GotConn, Network:%s RemoteAddr:%s \n", remoteAddr.Network(), remoteAddr.String()))
},
PutIdleConn: func(err error) {
log.Debug(label + fmt.Sprintf("PutIdleConn, err:%v \n", err))
},
GotFirstResponseByte: func() {
log.Debug(label + fmt.Sprint("GotFirstResponseByte \n"))
},
Got100Continue: func() {
log.Debug(label + fmt.Sprint("Got100Continue \n"))
},
DNSStart: func(info httptrace.DNSStartInfo) {
log.Debug(label + fmt.Sprintf("DNSStart, host:%s \n", info.Host))
},
DNSDone: func(info httptrace.DNSDoneInfo) {
log.Debug(label + fmt.Sprintf("DNSDone, addr:%+v \n", info.Addrs))
},
ConnectStart: func(network, addr string) {
log.Debug(label + fmt.Sprintf("ConnectStart, network:%+v ip:%s \n", network, addr))
},
ConnectDone: func(network, addr string, err error) {
log.Debug(label + fmt.Sprintf("ConnectDone, network:%s ip:%s err:%v \n", network, addr, err))
},
TLSHandshakeStart: func() {
log.Debug(label + fmt.Sprint("TLSHandshakeStart \n"))
},
TLSHandshakeDone: func(state tls.ConnectionState, err error) {
log.Debug(label + fmt.Sprintf("TLSHandshakeDone, state:%+v err:%s \n", state, err))
},
// go1.10 不支持
//WroteHeaderField: func(key string, value []string) {
// log.Debug(label + fmt.Sprintf("WroteHeaderField, key:%s value:%s \n", key, value))
//},
WroteHeaders: func() {
log.Debug(label + fmt.Sprint("WroteHeaders \n"))
},
Wait100Continue: func() {
log.Debug(label + fmt.Sprint("Wait100Continue \n"))
},
WroteRequest: func(info httptrace.WroteRequestInfo) {
log.Debug(label + fmt.Sprintf("WroteRequest, err:%v \n", info.Err))
},
}
return req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
}
func (interceptor *debugInterceptor) printResponse(label string, resp *http.Response) error {
if resp == nil {
return nil
}
printResp := IsPrintResponse()
if !printResp {
return nil
}
info := label + " response:\n"
d, dErr := httputil.DumpResponse(resp, IsPrintResponseBody())
if dErr != nil {
return dErr
}
info += string(d) + "\n"
log.Debug(info)
return nil
}

@ -0,0 +1,54 @@
package clientv2
import (
clientV1 "github.com/qiniu/go-sdk/v7/client"
"github.com/qiniu/go-sdk/v7/conf"
"net/http"
"time"
)
type defaultHeaderInterceptor struct {
}
func newDefaultHeaderInterceptor() Interceptor {
return &defaultHeaderInterceptor{}
}
func (interceptor *defaultHeaderInterceptor) Priority() InterceptorPriority {
return InterceptorPrioritySetHeader
}
func (interceptor *defaultHeaderInterceptor) Intercept(req *http.Request, handler Handler) (resp *http.Response, err error) {
if interceptor == nil || req == nil {
return handler(req)
}
if req.Header == nil {
req.Header = http.Header{}
}
if e := addUseragent(req.Header); e != nil {
return nil, e
}
if e := addXQiniuDate(req.Header); e != nil {
return nil, e
}
return handler(req)
}
func addUseragent(headers http.Header) error {
headers.Set("User-Agent", clientV1.UserAgent)
return nil
}
func addXQiniuDate(headers http.Header) error {
if conf.IsDisableQiniuTimestampSignature() {
return nil
}
timeString := time.Now().UTC().Format("20060102T150405Z")
headers.Set("X-Qiniu-Date", timeString)
return nil
}

@ -0,0 +1,123 @@
package clientv2
import (
"github.com/qiniu/go-sdk/v7/internal/hostprovider"
"net/http"
"net/url"
"strings"
"time"
)
type HostsRetryConfig struct {
RetryConfig RetryConfig // 主备域名重试参数
HostFreezeDuration time.Duration // 主备域名冻结时间默认600s当一个域名请求失败被冻结的时间最小 time.Millisecond
HostProvider hostprovider.HostProvider // 备用域名获取方法
ShouldFreezeHost func(req *http.Request, resp *http.Response, err error) bool
}
func (c *HostsRetryConfig) init() {
if c.RetryConfig.ShouldRetry == nil {
c.RetryConfig.ShouldRetry = func(req *http.Request, resp *http.Response, err error) bool {
return isHostRetryable(req, resp, err)
}
}
if c.RetryConfig.RetryMax < 0 {
c.RetryConfig.RetryMax = 1
}
c.RetryConfig.init()
if c.HostFreezeDuration < time.Millisecond {
c.HostFreezeDuration = 600 * time.Second
}
if c.ShouldFreezeHost == nil {
c.ShouldFreezeHost = func(req *http.Request, resp *http.Response, err error) bool {
return true
}
}
}
type hostsRetryInterceptor struct {
options HostsRetryConfig
}
func NewHostsRetryInterceptor(options HostsRetryConfig) Interceptor {
return &hostsRetryInterceptor{
options: options,
}
}
func (interceptor *hostsRetryInterceptor) Priority() InterceptorPriority {
return InterceptorPriorityRetryHosts
}
func (interceptor *hostsRetryInterceptor) Intercept(req *http.Request, handler Handler) (resp *http.Response, err error) {
if interceptor == nil || req == nil {
return handler(req)
}
interceptor.options.init()
// 不重试
if interceptor.options.RetryConfig.RetryMax <= 0 {
return handler(req)
}
for i := 0; ; i++ {
// Clone 防止后面 Handler 处理对 req 有污染
reqBefore := cloneReq(req.Context(), req)
resp, err = handler(req)
if !interceptor.options.RetryConfig.ShouldRetry(reqBefore, resp, err) {
return resp, err
}
// 尝试冻结域名
oldHost := req.URL.Host
if interceptor.options.ShouldFreezeHost(req, resp, err) {
if fErr := interceptor.options.HostProvider.Freeze(oldHost, err, interceptor.options.HostFreezeDuration); fErr != nil {
break
}
}
if i >= interceptor.options.RetryConfig.RetryMax {
break
}
// 尝试更换域名
newHost, pErr := interceptor.options.HostProvider.Provider()
if pErr != nil {
break
}
if len(newHost) == 0 {
break
}
if newHost != oldHost {
urlString := req.URL.String()
urlString = strings.Replace(urlString, oldHost, newHost, 1)
u, ppErr := url.Parse(urlString)
if ppErr != nil {
break
}
reqBefore.Host = u.Host
reqBefore.URL = u
}
req = reqBefore
retryInterval := interceptor.options.RetryConfig.RetryInterval()
if retryInterval < time.Microsecond {
continue
}
time.Sleep(retryInterval)
}
return resp, err
}
func isHostRetryable(req *http.Request, resp *http.Response, err error) bool {
return isRequestRetryable(req) && (isResponseRetryable(resp) || IsErrorRetryable(err))
}

@ -0,0 +1,182 @@
package clientv2
import (
clientv1 "github.com/qiniu/go-sdk/v7/client"
"io"
"math/rand"
"net"
"net/http"
"net/url"
"os"
"syscall"
"time"
)
type RetryConfig struct {
RetryMax int // 最大重试次数
RetryInterval func() time.Duration // 重试时间间隔
ShouldRetry func(req *http.Request, resp *http.Response, err error) bool
}
func (c *RetryConfig) init() {
if c == nil {
return
}
if c.RetryMax < 0 {
c.RetryMax = 0
}
if c.RetryInterval == nil {
c.RetryInterval = func() time.Duration {
return time.Duration(50+rand.Int()%50) * time.Millisecond
}
}
if c.ShouldRetry == nil {
c.ShouldRetry = func(req *http.Request, resp *http.Response, err error) bool {
return isSimpleRetryable(req, resp, err)
}
}
}
type simpleRetryInterceptor struct {
config RetryConfig
}
func NewSimpleRetryInterceptor(config RetryConfig) Interceptor {
return &simpleRetryInterceptor{
config: config,
}
}
func (interceptor *simpleRetryInterceptor) Priority() InterceptorPriority {
return InterceptorPriorityRetrySimple
}
func (interceptor *simpleRetryInterceptor) Intercept(req *http.Request, handler Handler) (resp *http.Response, err error) {
if interceptor == nil || req == nil {
return handler(req)
}
interceptor.config.init()
// 不重试
if interceptor.config.RetryMax <= 0 {
return handler(req)
}
// 可能会被重试多次
for i := 0; ; i++ {
// Clone 防止后面 Handler 处理对 req 有污染
reqBefore := cloneReq(req.Context(), req)
resp, err = handler(req)
if !interceptor.config.ShouldRetry(reqBefore, resp, err) {
return resp, err
}
req = reqBefore
if i >= interceptor.config.RetryMax {
break
}
retryInterval := interceptor.config.RetryInterval()
if retryInterval < time.Microsecond {
continue
}
time.Sleep(retryInterval)
}
return resp, err
}
func isSimpleRetryable(req *http.Request, resp *http.Response, err error) bool {
return isRequestRetryable(req) && (isResponseRetryable(resp) || IsErrorRetryable(err))
}
func isRequestRetryable(req *http.Request) bool {
if req == nil {
return false
}
if req.Body == nil {
return true
}
if req.GetBody != nil {
b, err := req.GetBody()
if err != nil || b == nil {
return false
}
req.Body = b
return true
}
seeker, ok := req.Body.(io.Seeker)
if !ok {
return false
}
_, err := seeker.Seek(0, io.SeekStart)
return err == nil
}
func isResponseRetryable(resp *http.Response) bool {
if resp == nil {
return false
}
return isStatusCodeRetryable(resp.StatusCode)
}
func isStatusCodeRetryable(statusCode int) bool {
if statusCode < 500 {
return false
}
if statusCode == 501 || statusCode == 509 || statusCode == 573 || statusCode == 579 ||
statusCode == 608 || statusCode == 612 || statusCode == 614 || statusCode == 616 || statusCode == 618 ||
statusCode == 630 || statusCode == 631 || statusCode == 632 || statusCode == 640 || statusCode == 701 {
return false
}
return true
}
func IsErrorRetryable(err error) bool {
if err == nil {
return false
}
switch t := err.(type) {
case *net.OpError:
return isNetworkErrorWithOpError(t)
case *url.Error:
return IsErrorRetryable(t.Err)
case net.Error:
return t.Timeout()
case *clientv1.ErrorInfo:
return isStatusCodeRetryable(t.Code)
default:
return false
}
}
func isNetworkErrorWithOpError(err *net.OpError) bool {
if err == nil {
return false
}
switch t := err.Err.(type) {
case *net.DNSError:
return true
case *os.SyscallError:
if errno, ok := t.Err.(syscall.Errno); ok {
return errno == syscall.ECONNABORTED ||
errno == syscall.ECONNRESET ||
errno == syscall.ECONNREFUSED ||
errno == syscall.ETIMEDOUT
}
}
return false
}

@ -0,0 +1,92 @@
package clientv2
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"net/url"
)
const (
RequestMethodGet = "GET"
RequestMethodPut = "PUT"
RequestMethodPost = "POST"
RequestMethodHead = "HEAD"
RequestMethodDelete = "DELETE"
)
type RequestBodyCreator func(options *RequestParams) (io.Reader, error)
func RequestBodyCreatorOfJson(object interface{}) RequestBodyCreator {
body := object
return func(o *RequestParams) (io.Reader, error) {
reqBody, err := json.Marshal(body)
if err != nil {
return nil, err
}
o.Header.Add("Content-Type", "application/json")
return bytes.NewReader(reqBody), nil
}
}
func RequestBodyCreatorForm(info map[string][]string) RequestBodyCreator {
body := FormStringInfo(info)
return func(o *RequestParams) (io.Reader, error) {
o.Header.Add("Content-Type", "application/x-www-form-urlencoded")
return bytes.NewBufferString(body), nil
}
}
func FormStringInfo(info map[string][]string) string {
if len(info) == 0 {
return ""
}
return url.Values(info).Encode()
}
type RequestParams struct {
Context context.Context
Method string
Url string
Header http.Header
BodyCreator RequestBodyCreator
}
func (o *RequestParams) init() {
if o.Context == nil {
o.Context = context.Background()
}
if len(o.Method) == 0 {
o.Method = RequestMethodGet
}
if o.Header == nil {
o.Header = http.Header{}
}
if o.BodyCreator == nil {
o.BodyCreator = func(options *RequestParams) (io.Reader, error) {
return nil, nil
}
}
}
func NewRequest(options RequestParams) (*http.Request, error) {
options.init()
body, cErr := options.BodyCreator(&options)
if cErr != nil {
return nil, cErr
}
req, err := http.NewRequest(options.Method, options.Url, body)
if err != nil {
return nil, err
}
req = req.WithContext(options.Context)
req.Header = options.Header
return req, nil
}

@ -0,0 +1,95 @@
package clientv2
import (
"context"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
)
// 此处是为了版本兼容sdk 支持最低版本为 go1.10, go1.13 提供 req.Clone 方法,
// 此处 copy 高版本的 go 标准库方法
func cloneReq(ctx context.Context, r *http.Request) *http.Request {
if ctx == nil {
panic("nil context")
}
r2 := r.WithContext(ctx)
if r.Header != nil {
r2.Header = cloneHeader(r.Header)
}
if r.Trailer != nil {
r2.Trailer = cloneHeader(r.Trailer)
}
if s := r.TransferEncoding; s != nil {
s2 := make([]string, len(s))
copy(s2, s)
r2.TransferEncoding = s2
}
r2.Form = cloneURLValues(r.Form)
r2.PostForm = cloneURLValues(r.PostForm)
r2.MultipartForm = cloneMultipartForm(r.MultipartForm)
return r2
}
func cloneHeader(h http.Header) http.Header {
if h == nil {
return nil
}
// Find total number of values.
nv := 0
for _, vv := range h {
nv += len(vv)
}
sv := make([]string, nv) // shared backing array for headers' values
h2 := make(http.Header, len(h))
for k, vv := range h {
n := copy(sv, vv)
h2[k] = sv[:n:n]
sv = sv[n:]
}
return h2
}
func cloneURLValues(v url.Values) url.Values {
if v == nil {
return nil
}
// http.Header and url.Values have the same representation, so temporarily
// treat it like http.Header, which does have a clone:
return url.Values(cloneHeader(http.Header(v)))
}
func cloneMultipartForm(f *multipart.Form) *multipart.Form {
if f == nil {
return nil
}
f2 := &multipart.Form{
Value: (map[string][]string)(cloneHeader(http.Header(f.Value))),
}
if f.File != nil {
m := make(map[string][]*multipart.FileHeader)
for k, vv := range f.File {
vv2 := make([]*multipart.FileHeader, len(vv))
for i, v := range vv {
vv2[i] = cloneMultipartFileHeader(v)
}
m[k] = vv2
}
f2.File = m
}
return f2
}
func cloneMultipartFileHeader(fh *multipart.FileHeader) *multipart.FileHeader {
if fh == nil {
return nil
}
fh2 := new(multipart.FileHeader)
*fh2 = *fh
fh2.Header = textproto.MIMEHeader(cloneHeader(http.Header(fh.Header)))
return fh2
}

@ -81,7 +81,6 @@ func (r *Base64PutExtra) init() {
// key 是要上传的文件访问路径。比如:"foo/bar.jpg"。注意我们建议 key 不要以 '/' 开头。另外key 为空字符串是合法的。
// base64Data 是要上传的Base64数据一般为图片数据的Base64编码字符串
// extra 是上传的一些可选项可以指定为nil。详细见 Base64PutExtra 结构的描述。
//
func (p *Base64Uploader) Put(
ctx context.Context, ret interface{}, uptoken, key string, base64Data []byte, extra *Base64PutExtra) (err error) {
return p.put(ctx, ret, uptoken, key, true, base64Data, extra)
@ -152,7 +151,7 @@ func (p *Base64Uploader) put(
}
var upHostProvider hostprovider.HostProvider
upHostProvider, err = p.upHostProvider(ak, bucket)
upHostProvider, err = p.upHostProvider(ak, bucket, extra)
if err != nil {
return
}
@ -167,10 +166,6 @@ func (p *Base64Uploader) put(
})
}
func (p *Base64Uploader) upHost(ak, bucket string) (upHost string, err error) {
return getUpHost(p.cfg, ak, bucket)
}
func (p *Base64Uploader) upHostProvider(ak, bucket string) (hostProvider hostprovider.HostProvider, err error) {
return getUpHostProvider(p.cfg, ak, bucket)
func (p *Base64Uploader) upHostProvider(ak, bucket string, extra *Base64PutExtra) (hostProvider hostprovider.HostProvider, err error) {
return getUpHostProvider(p.cfg, extra.TryTimes, extra.HostFreezeDuration, ak, bucket)
}

@ -10,13 +10,14 @@ import (
"encoding/base64"
"errors"
"fmt"
"net/http"
"github.com/qiniu/go-sdk/v7/internal/clientv2"
"net/url"
"strconv"
"strings"
"time"
"github.com/qiniu/go-sdk/v7/auth"
"github.com/qiniu/go-sdk/v7/client"
clientv1 "github.com/qiniu/go-sdk/v7/client"
)
// 资源管理相关的默认域名
@ -243,11 +244,18 @@ type BatchOpRet struct {
} `json:"data,omitempty"`
}
type BucketManagerOptions struct {
RetryMax int // 单域名重试次数,当前只有 uc 相关的服务有多域名
// 主备域名冻结时间默认600s当一个域名请求失败单个域名会被重试 TryTimes 次),会被冻结一段时间,使用备用域名进行重试,在冻结时间内,域名不能被使用,当一个操作中所有域名竣备冻结操作不在进行重试,返回最后一次操作的错误。
HostFreezeDuration time.Duration
}
// BucketManager 提供了对资源进行管理的操作
type BucketManager struct {
Client *client.Client
Mac *auth.Credentials
Cfg *Config
Client *clientv1.Client
Mac *auth.Credentials
Cfg *Config
options BucketManagerOptions
}
// NewBucketManager 用来构建一个新的资源管理对象
@ -260,20 +268,20 @@ func NewBucketManager(mac *auth.Credentials, cfg *Config) *BucketManager {
}
return &BucketManager{
Client: &client.DefaultClient,
Client: &clientv1.DefaultClient,
Mac: mac,
Cfg: cfg,
}
}
// NewBucketManagerEx 用来构建一个新的资源管理对象
func NewBucketManagerEx(mac *auth.Credentials, cfg *Config, clt *client.Client) *BucketManager {
func NewBucketManagerEx(mac *auth.Credentials, cfg *Config, clt *clientv1.Client) *BucketManager {
if cfg == nil {
cfg = &Config{}
}
if clt == nil {
clt = &client.DefaultClient
clt = &clientv1.DefaultClient
}
if cfg.CentralRsHost == "" {
cfg.CentralRsHost = DefaultRsHost
@ -286,6 +294,26 @@ func NewBucketManagerEx(mac *auth.Credentials, cfg *Config, clt *client.Client)
}
}
func NewBucketManagerExWithOptions(mac *auth.Credentials, cfg *Config, clt *clientv1.Client, options BucketManagerOptions) *BucketManager {
if cfg == nil {
cfg = &Config{}
}
if clt == nil {
clt = &clientv1.DefaultClient
}
if cfg.CentralRsHost == "" {
cfg.CentralRsHost = DefaultRsHost
}
return &BucketManager{
Client: clt,
Mac: mac,
Cfg: cfg,
options: options,
}
}
// UpdateObjectStatus 用来修改文件状态, 禁用和启用文件的可访问性
// 请求包:
@ -320,14 +348,27 @@ func (m *BucketManager) UpdateObjectStatus(bucketName string, key string, enable
// CreateBucket 创建一个七牛存储空间
func (m *BucketManager) CreateBucket(bucketName string, regionID RegionID) error {
reqURL := fmt.Sprintf("%s/mkbucketv3/%s/region/%s", getUcHost(m.Cfg.UseHTTPS), bucketName, string(regionID))
return m.Client.CredentialedCall(context.Background(), m.Mac, auth.TokenQiniu, nil, "POST", reqURL, nil)
_, err := clientv2.Do(m.getUCClient(), clientv2.RequestParams{
Context: context.Background(),
Method: clientv2.RequestMethodPost,
Url: reqURL,
Header: nil,
BodyCreator: nil,
})
return err
}
// Buckets 用来获取空间列表,如果指定了 shared 参数为 true那么一同列表被授权访问的空间
func (m *BucketManager) Buckets(shared bool) (buckets []string, err error) {
reqURL := fmt.Sprintf("%s/buckets?shared=%v", getUcHost(m.Cfg.UseHTTPS), shared)
err = m.Client.CredentialedCall(context.Background(), m.Mac, auth.TokenQiniu, &buckets, "POST", reqURL, nil)
return
_, err = clientv2.DoAndDecodeJsonResponse(m.getUCClient(), clientv2.RequestParams{
Context: context.Background(),
Method: clientv2.RequestMethodPost,
Url: reqURL,
Header: nil,
BodyCreator: nil,
}, &buckets)
return buckets, err
}
// BucketsV4 获取该用户的指定区域内的空间信息,注意该 API 以分页形式返回 Bucket 列表
@ -349,15 +390,27 @@ func (m *BucketManager) BucketsV4(input *BucketV4Input) (output BucketsV4Output,
if len(query) > 0 {
reqURL += "&" + query.Encode()
}
err = m.Client.CredentialedCall(context.Background(), m.Mac, auth.TokenQiniu, &output, http.MethodGet, reqURL, nil)
return
_, err = clientv2.DoAndDecodeJsonResponse(m.getUCClient(), clientv2.RequestParams{
Context: context.Background(),
Method: clientv2.RequestMethodGet,
Url: reqURL,
Header: nil,
BodyCreator: nil,
}, &output)
return output, err
}
// DropBucket 删除七牛存储空间
func (m *BucketManager) DropBucket(bucketName string) (err error) {
reqURL := fmt.Sprintf("%s/drop/%s", getUcHost(m.Cfg.UseHTTPS), bucketName)
err = m.Client.CredentialedCall(context.Background(), m.Mac, auth.TokenQiniu, nil, "POST", reqURL, nil)
return
_, err = clientv2.Do(m.getUCClient(), clientv2.RequestParams{
Context: context.Background(),
Method: clientv2.RequestMethodPost,
Url: reqURL,
Header: nil,
BodyCreator: nil,
})
return err
}
// Stat 用来获取一个文件的基本信息
@ -644,10 +697,15 @@ type DomainInfo struct {
// ListBucketDomains 返回绑定在存储空间中的域名信息
func (m *BucketManager) ListBucketDomains(bucket string) (info []DomainInfo, err error) {
host := getUcHost(m.Cfg.UseHTTPS)
reqURL := fmt.Sprintf("%s/v3/domains?tbl=%s", host, bucket)
err = m.Client.CredentialedCall(context.Background(), m.Mac, auth.TokenQiniu, &info, "GET", reqURL, nil)
return
reqURL := fmt.Sprintf("%s/v3/domains?tbl=%s", getUcHost(m.Cfg.UseHTTPS), bucket)
_, err = clientv2.DoAndDecodeJsonResponse(m.getUCClient(), clientv2.RequestParams{
Context: context.Background(),
Method: clientv2.RequestMethodGet,
Url: reqURL,
Header: nil,
BodyCreator: nil,
}, &info)
return info, err
}
// Prefetch 用来同步镜像空间的资源和镜像源资源内容
@ -665,22 +723,40 @@ func (m *BucketManager) Prefetch(bucket, key string) (err error) {
// SetImage 用来设置空间镜像源
func (m *BucketManager) SetImage(siteURL, bucket string) (err error) {
reqURL := fmt.Sprintf("%s%s", getUcHost(m.Cfg.UseHTTPS), uriSetImage(siteURL, bucket))
err = m.Client.CredentialedCall(context.Background(), m.Mac, auth.TokenQiniu, nil, "POST", reqURL, nil)
return
_, err = clientv2.Do(m.getUCClient(), clientv2.RequestParams{
Context: context.Background(),
Method: clientv2.RequestMethodPost,
Url: reqURL,
Header: nil,
BodyCreator: nil,
})
return err
}
// SetImageWithHost 用来设置空间镜像源额外添加回源Host头部
func (m *BucketManager) SetImageWithHost(siteURL, bucket, host string) (err error) {
reqURL := fmt.Sprintf("%s%s", getUcHost(m.Cfg.UseHTTPS),
uriSetImageWithHost(siteURL, bucket, host))
err = m.Client.CredentialedCall(context.Background(), m.Mac, auth.TokenQiniu, nil, "POST", reqURL, nil)
return
_, err = clientv2.Do(m.getUCClient(), clientv2.RequestParams{
Context: context.Background(),
Method: clientv2.RequestMethodPost,
Url: reqURL,
Header: nil,
BodyCreator: nil,
})
return err
}
// UnsetImage 用来取消空间镜像源设置
func (m *BucketManager) UnsetImage(bucket string) (err error) {
reqURL := fmt.Sprintf("%s%s", getUcHost(m.Cfg.UseHTTPS), uriUnsetImage(bucket))
err = m.Client.CredentialedCall(context.Background(), m.Mac, auth.TokenQiniu, nil, "POST", reqURL, nil)
_, err = clientv2.Do(m.getUCClient(), clientv2.RequestParams{
Context: context.Background(),
Method: clientv2.RequestMethodPost,
Url: reqURL,
Header: nil,
BodyCreator: nil,
})
return err
}

@ -129,21 +129,20 @@ func ListInputOptionsLimit(limit int) ListInputOption {
}
}
//
// ListFilesWithContext
// @Description: 用来获取空间文件列表,可以根据需要指定文件的列举条件
// @receiver m BucketManager
// @param ctx context
// @param bucket 列举的 bucket
// @param options 列举的可选条件
// 列举条件-需要列举 Key 的前缀ListInputOptionsPrefix(prefix)
// 列举条件-文件的目录分隔符ListInputOptionsDelimiter(delimiter)
// 列举条件-下次列举的位置ListInputOptionsMarker(marker)
// 列举条件-每次返回的文件的最大数量ListInputOptionsLimit(limit) 范围1~1000
// @return ret 列举的对象数据
// @return hasNext 是否还有数据未被列举
// @return err 列举时的错误信息
//
// @Description: 用来获取空间文件列表,可以根据需要指定文件的列举条件
// @receiver m BucketManager
// @param ctx context
// @param bucket 列举的 bucket
// @param options 列举的可选条件
// 列举条件-需要列举 Key 的前缀ListInputOptionsPrefix(prefix)
// 列举条件-文件的目录分隔符ListInputOptionsDelimiter(delimiter)
// 列举条件-下次列举的位置ListInputOptionsMarker(marker)
// 列举条件-每次返回的文件的最大数量ListInputOptionsLimit(limit) 范围1~1000
// @return ret 列举的对象数据
// @return hasNext 是否还有数据未被列举
// @return err 列举时的错误信息
func (m *BucketManager) ListFilesWithContext(ctx context.Context, bucket string, options ...ListInputOption) (ret *ListFilesRet, hasNext bool, err error) {
if len(bucket) == 0 {
return nil, false, errors.New("bucket can't empty")
@ -181,12 +180,14 @@ type listFilesRet2 struct {
}
// ListBucket 用来获取空间文件列表,可以根据需要指定文件的前缀 prefix文件的目录 delimiter流式返回每条数据。
// Deprecated
func (m *BucketManager) ListBucket(bucket, prefix, delimiter, marker string) (retCh chan listFilesRet2, err error) {
return m.ListBucketContext(context.Background(), bucket, prefix, delimiter, marker)
}
// ListBucketContext 用来获取空间文件列表,可以根据需要指定文件的前缀 prefix文件的目录 delimiter流式返回每条数据。
// 接受的context可以用来取消列举操作
// Deprecated
func (m *BucketManager) ListBucketContext(ctx context.Context, bucket, prefix, delimiter, marker string) (retCh chan listFilesRet2, err error) {
ret, _, lErr := m.ListFilesWithContext(ctx, bucket,

@ -103,7 +103,6 @@ func NewFormUploaderEx(cfg *Config, clt *client.Client) *FormUploader {
// key 是要上传的文件访问路径。比如:"foo/bar.jpg"。注意我们建议 key 不要以 '/' 开头。另外key 为空字符串是合法的。
// localFile 是要上传的文件的本地路径。
// extra 是上传的一些可选项可以指定为nil。详细见 PutExtra 结构的描述。
//
func (p *FormUploader) PutFile(
ctx context.Context, ret interface{}, uptoken, key, localFile string, extra *PutExtra) (err error) {
return p.putFile(ctx, ret, uptoken, key, true, localFile, extra)
@ -118,7 +117,6 @@ func (p *FormUploader) PutFile(
// uptoken 是由业务服务器颁发的上传凭证。
// localFile 是要上传的文件的本地路径。
// extra 是上传的一些可选项。可以指定为nil。详细见 PutExtra 结构的描述。
//
func (p *FormUploader) PutFileWithoutKey(
ctx context.Context, ret interface{}, uptoken, localFile string, extra *PutExtra) (err error) {
return p.putFile(ctx, ret, uptoken, "", false, localFile, extra)
@ -152,7 +150,6 @@ func (p *FormUploader) putFile(
// data 是文件内容的访问接口io.Reader
// fsize 是要上传的文件大小。
// extra 是上传的一些可选项。可以指定为nil。详细见 PutExtra 结构的描述。
//
func (p *FormUploader) Put(
ctx context.Context, ret interface{}, uptoken, key string, data io.Reader, size int64, extra *PutExtra) (err error) {
err = p.put(ctx, ret, uptoken, key, true, data, size, extra, path.Base(key))
@ -168,7 +165,6 @@ func (p *FormUploader) Put(
// data 是文件内容的访问接口io.Reader
// fsize 是要上传的文件大小。
// extra 是上传的一些可选项。详细见 PutExtra 结构的描述。
//
func (p *FormUploader) PutWithoutKey(
ctx context.Context, ret interface{}, uptoken string, data io.Reader, size int64, extra *PutExtra) (err error) {
err = p.put(ctx, ret, uptoken, "", false, data, size, extra, "filename")
@ -270,16 +266,13 @@ func (p *FormUploader) putSeekableData(ctx context.Context, ret interface{}, upT
}
return ioutil.NopCloser(reader), nil
}
bodyReader, err := getBodyReader()
if err != nil {
return err
}
var err error
var hostProvider hostprovider.HostProvider = nil
if extra.UpHost != "" {
hostProvider = hostprovider.NewWithHosts([]string{extra.getUpHost(p.Cfg.UseHTTPS)})
} else {
hostProvider, err = p.getUpHostProviderFromUploadToken(upToken)
hostProvider, err = p.getUpHostProviderFromUploadToken(upToken, extra)
if err != nil {
return err
}
@ -290,7 +283,12 @@ func (p *FormUploader) putSeekableData(ctx context.Context, ret interface{}, upT
headers := http.Header{}
headers.Add("Content-Type", contentType)
err = doUploadAction(hostProvider, extra.TryTimes, extra.HostFreezeDuration, func(host string) error {
return p.Client.CallWithBodyGetter(ctx, ret, "POST", host, headers, bodyReader, getBodyReadCloser, formBodyLen)
reader, gErr := getBodyReader()
if gErr != nil {
return gErr
}
return p.Client.CallWithBodyGetter(ctx, ret, "POST", host, headers, reader, getBodyReadCloser, formBodyLen)
})
if err != nil {
return err
@ -302,22 +300,12 @@ func (p *FormUploader) putSeekableData(ctx context.Context, ret interface{}, upT
return nil
}
func (p *FormUploader) getUpHostFromUploadToken(upToken string) (upHost string, err error) {
var ak, bucket string
if ak, bucket, err = getAkBucketFromUploadToken(upToken); err != nil {
return
}
upHost, err = p.UpHost(ak, bucket)
return
}
func (p *FormUploader) getUpHostProviderFromUploadToken(upToken string) (hostprovider.HostProvider, error) {
func (p *FormUploader) getUpHostProviderFromUploadToken(upToken string, extra *PutExtra) (hostprovider.HostProvider, error) {
ak, bucket, err := getAkBucketFromUploadToken(upToken)
if err != nil {
return nil, err
}
return getUpHostProvider(p.Cfg, ak, bucket)
return getUpHostProvider(p.Cfg, extra.TryTimes, extra.HostFreezeDuration, ak, bucket)
}
type crc32Reader struct {
@ -357,7 +345,7 @@ func (r crc32Reader) length() (length int64) {
}
func (p *FormUploader) UpHost(ak, bucket string) (upHost string, err error) {
return getUpHost(p.Cfg, ak, bucket)
return getUpHost(p.Cfg, 0, 0, ak, bucket)
}
type readerWithProgress struct {

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save