diff --git a/api_mongo.go b/api_mongo.go index 3cf947a..7514a14 100644 --- a/api_mongo.go +++ b/api_mongo.go @@ -22,7 +22,7 @@ type apiMongolLog struct { LogId primitive.ObjectID `json:"log_id,omitempty" bson:"_id,omitempty"` //【记录】编号 LogTime primitive.DateTime `json:"log_time,omitempty" bson:"log_time,omitempty"` //【记录】时间 TraceId string `json:"trace_id,omitempty" bson:"trace_id,omitempty"` //【记录】跟踪编号 - RequestTime dorm.BsonTime `json:"request_time,omitempty" bson:"request_time,omitempty"` //【请求】时间 + RequestTime string `json:"request_time,omitempty" bson:"request_time,omitempty"` //【请求】时间 RequestUri string `json:"request_uri,omitempty" bson:"request_uri,omitempty"` //【请求】链接 RequestUrl string `json:"request_url,omitempty" bson:"request_url,omitempty"` //【请求】链接 RequestApi string `json:"request_api,omitempty" bson:"request_api,omitempty"` //【请求】接口 @@ -34,7 +34,7 @@ type apiMongolLog struct { ResponseStatusCode int `json:"response_status_code,omitempty" bson:"response_status_code,omitempty"` //【返回】状态码 ResponseBody interface{} `json:"response_body,omitempty" bson:"response_body,omitempty"` //【返回】内容 ResponseContentLength int64 `json:"response_content_length,omitempty" bson:"response_content_length,omitempty"` //【返回】大小 - ResponseTime dorm.BsonTime `json:"response_time,omitempty" bson:"response_time,omitempty"` //【返回】时间 + ResponseTime string `json:"response_time,omitempty" bson:"response_time,omitempty"` //【返回】时间 SystemHostName string `json:"system_host_name,omitempty" bson:"system_host_name,omitempty"` //【系统】主机名 SystemInsideIp string `json:"system_inside_ip,omitempty" bson:"system_inside_ip,omitempty"` //【系统】内网ip SystemOs string `json:"system_os,omitempty" bson:"system_os,omitempty"` //【系统】系统类型 @@ -129,9 +129,6 @@ func (c *ApiClient) mongoCreateIndexes(ctx context.Context) { c.zapLog.WithLogger().Sugar().Infof(c.mongoClient.Db.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{ {"trace_id", 1}, }})) - c.zapLog.WithLogger().Sugar().Infof(c.mongoClient.Db.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{ - {"log_time", -1}, - }})) c.zapLog.WithLogger().Sugar().Infof(c.mongoClient.Db.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{ {"request_time", -1}, }})) @@ -203,7 +200,7 @@ func (c *ApiClient) MongoDelete(ctx context.Context, hour int64) (*mongo.DeleteR func (c *ApiClient) MongoMiddleware(ctx context.Context, request gorequest.Response, sdkVersion string) { data := apiMongolLog{ LogTime: primitive.NewDateTimeFromTime(request.RequestTime), //【记录】时间 - RequestTime: dorm.BsonTime(request.RequestTime), //【请求】时间 + RequestTime: gotime.SetCurrent(request.RequestTime).Bson(), //【请求】时间 RequestUri: request.RequestUri, //【请求】链接 RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 @@ -213,7 +210,7 @@ func (c *ApiClient) MongoMiddleware(ctx context.Context, request gorequest.Respo ResponseHeader: request.ResponseHeader, //【返回】头部 ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 ResponseContentLength: request.ResponseContentLength, //【返回】大小 - ResponseTime: dorm.BsonTime(request.ResponseTime), //【返回】时间 + ResponseTime: gotime.SetCurrent(request.ResponseTime).Bson(), //【返回】时间 SdkVersion: sdkVersion, //【程序】Sdk版本 } if request.ResponseHeader.Get("Content-Type") == "image/jpeg" || request.ResponseHeader.Get("Content-Type") == "image/png" || request.ResponseHeader.Get("Content-Type") == "image/jpg" { @@ -242,7 +239,7 @@ func (c *ApiClient) MongoMiddleware(ctx context.Context, request gorequest.Respo func (c *ApiClient) MongoMiddlewareXml(ctx context.Context, request gorequest.Response, sdkVersion string) { data := apiMongolLog{ LogTime: primitive.NewDateTimeFromTime(request.RequestTime), //【记录】时间 - RequestTime: dorm.BsonTime(request.RequestTime), //【请求】时间 + RequestTime: gotime.SetCurrent(request.RequestTime).Bson(), //【请求】时间 RequestUri: request.RequestUri, //【请求】链接 RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 @@ -252,7 +249,7 @@ func (c *ApiClient) MongoMiddlewareXml(ctx context.Context, request gorequest.Re ResponseHeader: request.ResponseHeader, //【返回】头部 ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 ResponseContentLength: request.ResponseContentLength, //【返回】大小 - ResponseTime: dorm.BsonTime(request.ResponseTime), //【返回】时间 + ResponseTime: gotime.SetCurrent(request.ResponseTime).Bson(), //【返回】时间 SdkVersion: sdkVersion, //【程序】Sdk版本 } if request.ResponseHeader.Get("Content-Type") == "image/jpeg" || request.ResponseHeader.Get("Content-Type") == "image/png" || request.ResponseHeader.Get("Content-Type") == "image/jpg" { @@ -281,7 +278,7 @@ func (c *ApiClient) MongoMiddlewareXml(ctx context.Context, request gorequest.Re func (c *ApiClient) MongoMiddlewareCustom(ctx context.Context, api string, request gorequest.Response, sdkVersion string) { data := apiMongolLog{ LogTime: primitive.NewDateTimeFromTime(request.RequestTime), //【记录】时间 - RequestTime: dorm.BsonTime(request.RequestTime), //【请求】时间 + RequestTime: gotime.SetCurrent(request.RequestTime).Bson(), //【请求】时间 RequestUri: request.RequestUri, //【请求】链接 RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 RequestApi: api, //【请求】接口 @@ -291,7 +288,7 @@ func (c *ApiClient) MongoMiddlewareCustom(ctx context.Context, api string, reque ResponseHeader: request.ResponseHeader, //【返回】头部 ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 ResponseContentLength: request.ResponseContentLength, //【返回】大小 - ResponseTime: dorm.BsonTime(request.ResponseTime), //【返回】时间 + ResponseTime: gotime.SetCurrent(request.ResponseTime).Bson(), //【返回】时间 SdkVersion: sdkVersion, //【程序】Sdk版本 } if request.ResponseHeader.Get("Content-Type") == "image/jpeg" || request.ResponseHeader.Get("Content-Type") == "image/png" || request.ResponseHeader.Get("Content-Type") == "image/jpg" { diff --git a/const.go b/const.go index fbcb1f1..8fc25f8 100644 --- a/const.go +++ b/const.go @@ -1,5 +1,5 @@ package golog const ( - Version = "1.0.75" + Version = "1.0.76" ) diff --git a/gin_mongo.go b/gin_mongo.go index 6468c12..f712836 100644 --- a/gin_mongo.go +++ b/gin_mongo.go @@ -33,7 +33,7 @@ type ginMongoLog struct { LogId primitive.ObjectID `json:"log_id,omitempty" bson:"_id,omitempty"` //【记录】编号 LogTime primitive.DateTime `json:"log_time,omitempty" bson:"log_time,omitempty"` //【记录】时间 TraceId string `json:"trace_id,omitempty" bson:"trace_id,omitempty"` //【记录】跟踪编号 - RequestTime dorm.BsonTime `json:"request_time,omitempty" bson:"request_time,omitempty"` //【请求】时间 + RequestTime string `json:"request_time,omitempty" bson:"request_time,omitempty"` //【请求】时间 RequestUri string `json:"request_uri,omitempty" bson:"request_uri,omitempty"` //【请求】请求链接 域名+路径+参数 RequestUrl string `json:"request_url,omitempty" bson:"request_url,omitempty"` //【请求】请求链接 域名+路径 RequestApi string `json:"request_api,omitempty" bson:"request_api,omitempty"` //【请求】请求接口 路径 @@ -50,7 +50,7 @@ type ginMongoLog struct { RequestIpIsp string `json:"request_ip_isp,omitempty" bson:"request_ip_isp,omitempty"` //【请求】请求客户端运营商 RequestIpLocation ginMongoLogRequestIpLocationLocation `json:"request_ip_location,omitempty" bson:"request_ip_location,omitempty"` //【请求】请求客户端位置 RequestHeader interface{} `json:"request_header,omitempty" bson:"request_header,omitempty"` //【请求】请求头 - ResponseTime dorm.BsonTime `json:"response_time,omitempty" bson:"response_time,omitempty"` //【返回】时间 + ResponseTime string `json:"response_time,omitempty" bson:"response_time,omitempty"` //【返回】时间 ResponseCode int `json:"response_code,omitempty" bson:"response_code,omitempty"` //【返回】状态码 ResponseMsg string `json:"response_msg,omitempty" bson:"response_msg,omitempty"` //【返回】描述 ResponseData interface{} `json:"response_data,omitempty" bson:"response_data,omitempty"` //【返回】数据 @@ -146,10 +146,6 @@ func (c *GinClient) mongoCreateIndexes(ctx context.Context) { Keys: bson.D{ {"trace_id", 1}, }})) - c.zapLog.WithLogger().Sugar().Infof(c.mongoClient.Db.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).Indexes().CreateOne(ctx, mongo.IndexModel{ - Keys: bson.D{ - {"log_time", -1}, - }})) c.zapLog.WithLogger().Sugar().Infof(c.mongoClient.Db.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).Indexes().CreateOne(ctx, mongo.IndexModel{ Keys: bson.D{ {"request_time", -1}, @@ -253,7 +249,7 @@ func (c *GinClient) mongoRecordJson(ginCtx *gin.Context, traceId string, request data := ginMongoLog{ TraceId: traceId, //【记录】跟踪编号 LogTime: primitive.NewDateTimeFromTime(requestTime), //【记录】时间 - RequestTime: dorm.BsonTime(requestTime), //【请求】时间 + RequestTime: gotime.SetCurrent(requestTime).Bson(), //【请求】时间 RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 RequestMethod: ginCtx.Request.Method, //【请求】请求方式 @@ -267,7 +263,7 @@ func (c *GinClient) mongoRecordJson(ginCtx *gin.Context, traceId string, request RequestIpCity: requestClientIpCity, //【请求】请求客户端城市 RequestIpIsp: requestClientIpIsp, //【请求】请求客户端运营商 RequestHeader: ginCtx.Request.Header, //【请求】请求头 - ResponseTime: dorm.BsonTime(gotime.Current().Time), //【返回】时间 + ResponseTime: gotime.Current().Bson(), //【返回】时间 ResponseCode: responseCode, //【返回】状态码 ResponseData: c.jsonUnmarshal(responseBody), //【返回】数据 CostTime: endTime - startTime, //【系统】花费时间 @@ -312,7 +308,7 @@ func (c *GinClient) mongoRecordXml(ginCtx *gin.Context, traceId string, requestT data := ginMongoLog{ TraceId: traceId, //【记录】跟踪编号 LogTime: primitive.NewDateTimeFromTime(requestTime), //【记录】时间 - RequestTime: dorm.BsonTime(requestTime), //【请求】时间 + RequestTime: gotime.SetCurrent(requestTime).Bson(), //【请求】时间 RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 RequestMethod: ginCtx.Request.Method, //【请求】请求方式 @@ -326,7 +322,7 @@ func (c *GinClient) mongoRecordXml(ginCtx *gin.Context, traceId string, requestT RequestIpCity: requestClientIpCity, //【请求】请求客户端城市 RequestIpIsp: requestClientIpIsp, //【请求】请求客户端运营商 RequestHeader: ginCtx.Request.Header, //【请求】请求头 - ResponseTime: dorm.BsonTime(gotime.Current().Time), //【返回】时间 + ResponseTime: gotime.Current().Bson(), //【返回】时间 ResponseCode: responseCode, //【返回】状态码 ResponseData: c.jsonUnmarshal(responseBody), //【返回】数据 CostTime: endTime - startTime, //【系统】花费时间 diff --git a/go.mod b/go.mod index 8c748f8..0e323ec 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/gin-gonic/gin v1.8.1 github.com/natefinch/lumberjack v2.0.0+incompatible go.dtapp.net/dorm v1.0.33 - go.dtapp.net/goip v1.0.33 + go.dtapp.net/goip v1.0.34 go.dtapp.net/gorequest v1.0.31 go.dtapp.net/gotime v1.0.5 go.dtapp.net/gotrace_id v1.0.6 @@ -41,7 +41,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.9 // indirect + github.com/klauspost/compress v1.15.10 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lib/pq v1.10.7 // indirect github.com/mattn/go-isatty v0.0.16 // indirect diff --git a/go.sum b/go.sum index 1b35167..b557368 100644 --- a/go.sum +++ b/go.sum @@ -271,8 +271,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= +github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -501,8 +501,8 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.dtapp.net/dorm v1.0.33 h1:QRAVEQ6Uf3WENSOrXytzzH+PjH90JySowd3jbB9PQjw= go.dtapp.net/dorm v1.0.33/go.mod h1:4WNSzrUGs7YIudq1cRZQNkHOlPAbG6thI3mXX1tQcYY= -go.dtapp.net/goip v1.0.33 h1:n6dLTfwiWp2Pw5pKjHSv1QSDJtuLEWNCjNHOTqJQHc0= -go.dtapp.net/goip v1.0.33/go.mod h1:EctL6B8ue/kZKPr+kKZPU6YTTpNhihane9BHHffwo6Q= +go.dtapp.net/goip v1.0.34 h1:aW2CuPpQwcDOJiyx/gHbvrha3/x+poFRpDxsLtO4EVw= +go.dtapp.net/goip v1.0.34/go.mod h1:EctL6B8ue/kZKPr+kKZPU6YTTpNhihane9BHHffwo6Q= go.dtapp.net/gorandom v1.0.1 h1:IWfMClh1ECPvyUjlqD7MwLq4mZdUusD1qAwAdsvEJBs= go.dtapp.net/gorandom v1.0.1/go.mod h1:ZPdgalKpvFV/ATQqR0k4ns/F/IpITAZpx6WkWirr5Y8= go.dtapp.net/gorequest v1.0.31 h1:r/OoU5Y00TbJjkQtpvwjsb/pllqO0UQQjFRY1veZYZc= diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index ad5c63a..2d6b010 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -17,6 +17,12 @@ This package provides various compression algorithms. # changelog +* July 21, 2022 (v1.15.9) + + * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 + * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 + * zstd: Allow single segments up to "max decoded size" by @klauspost in https://github.com/klauspost/compress/pull/643 + * July 13, 2022 (v1.15.8) * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index c0c48bd..42a237e 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -763,17 +763,20 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 1") } - copy(out, buf[0][:]) - copy(out[dstEvery:], buf[1][:]) - copy(out[dstEvery*2:], buf[2][:]) - copy(out[dstEvery*3:], buf[3][:]) - out = out[bufoff:] - decoded += bufoff * 4 // There must at least be 3 buffers left. - if len(out) < dstEvery*3 { + if len(out)-bufoff < dstEvery*3 { d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 2") } + //copy(out, buf[0][:]) + //copy(out[dstEvery:], buf[1][:]) + //copy(out[dstEvery*2:], buf[2][:]) + *(*[bufoff]byte)(out) = buf[0] + *(*[bufoff]byte)(out[dstEvery:]) = buf[1] + *(*[bufoff]byte)(out[dstEvery*2:]) = buf[2] + *(*[bufoff]byte)(out[dstEvery*3:]) = buf[3] + out = out[bufoff:] + decoded += bufoff * 4 } } if off > 0 { @@ -997,17 +1000,22 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 1") } - copy(out, buf[0][:]) - copy(out[dstEvery:], buf[1][:]) - copy(out[dstEvery*2:], buf[2][:]) - copy(out[dstEvery*3:], buf[3][:]) - out = out[bufoff:] - decoded += bufoff * 4 // There must at least be 3 buffers left. - if len(out) < dstEvery*3 { + if len(out)-bufoff < dstEvery*3 { d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 2") } + + //copy(out, buf[0][:]) + //copy(out[dstEvery:], buf[1][:]) + //copy(out[dstEvery*2:], buf[2][:]) + // copy(out[dstEvery*3:], buf[3][:]) + *(*[bufoff]byte)(out) = buf[0] + *(*[bufoff]byte)(out[dstEvery:]) = buf[1] + *(*[bufoff]byte)(out[dstEvery*2:]) = buf[2] + *(*[bufoff]byte)(out[dstEvery*3:]) = buf[3] + out = out[bufoff:] + decoded += bufoff * 4 } } if off > 0 { diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index 9f3e9f7..ba7e8e6 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -14,12 +14,14 @@ import ( // decompress4x_main_loop_x86 is an x86 assembler implementation // of Decompress4X when tablelog > 8. +// //go:noescape func decompress4x_main_loop_amd64(ctx *decompress4xContext) // decompress4x_8b_loop_x86 is an x86 assembler implementation // of Decompress4X when tablelog <= 8 which decodes 4 entries // per loop. +// //go:noescape func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) @@ -145,11 +147,13 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { // decompress4x_main_loop_x86 is an x86 assembler implementation // of Decompress1X when tablelog > 8. +// //go:noescape func decompress1x_main_loop_amd64(ctx *decompress1xContext) // decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation // of Decompress1X when tablelog > 8. +// //go:noescape func decompress1x_main_loop_bmi2(ctx *decompress1xContext) diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index dd1a5ae..8d2187a 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -1,7 +1,6 @@ // Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc // func decompress4x_main_loop_amd64(ctx *decompress4xContext) TEXT ·decompress4x_main_loop_amd64(SB), $0-8 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go index 4f6f37c..908c17d 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -122,17 +122,21 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 1") } - copy(out, buf[0][:]) - copy(out[dstEvery:], buf[1][:]) - copy(out[dstEvery*2:], buf[2][:]) - copy(out[dstEvery*3:], buf[3][:]) - out = out[bufoff:] - decoded += bufoff * 4 // There must at least be 3 buffers left. - if len(out) < dstEvery*3 { + if len(out)-bufoff < dstEvery*3 { d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 2") } + //copy(out, buf[0][:]) + //copy(out[dstEvery:], buf[1][:]) + //copy(out[dstEvery*2:], buf[2][:]) + //copy(out[dstEvery*3:], buf[3][:]) + *(*[bufoff]byte)(out) = buf[0] + *(*[bufoff]byte)(out[dstEvery:]) = buf[1] + *(*[bufoff]byte)(out[dstEvery*2:]) = buf[2] + *(*[bufoff]byte)(out[dstEvery*3:]) = buf[3] + out = out[bufoff:] + decoded += bufoff * 4 } } if off > 0 { diff --git a/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go b/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go index 511bba6..298c4f8 100644 --- a/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go +++ b/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go @@ -18,6 +18,7 @@ func load64(b []byte, i int) uint64 { // emitLiteral writes a literal chunk and returns the number of bytes written. // // It assumes that: +// // dst is long enough to hold the encoded bytes // 1 <= len(lit) && len(lit) <= 65536 func emitLiteral(dst, lit []byte) int { @@ -42,6 +43,7 @@ func emitLiteral(dst, lit []byte) int { // emitCopy writes a copy chunk and returns the number of bytes written. // // It assumes that: +// // dst is long enough to hold the encoded bytes // 1 <= offset && offset <= 65535 // 4 <= length && length <= 65535 @@ -89,6 +91,7 @@ func emitCopy(dst []byte, offset, length int) int { // src[i:i+k-j] and src[j:k] have the same contents. // // It assumes that: +// // 0 <= i && i < j && j <= len(src) func extendMatch(src []byte, i, j int) int { for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { @@ -105,8 +108,9 @@ func hash(u, shift uint32) uint32 { // been written. // // It also assumes that: +// // len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize +// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize func encodeBlock(dst, src []byte) (d int) { // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. // The table element type is uint16, as s < sLimit and sLimit < len(src) diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index beb7fa8..65b38ab 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -12,6 +12,8 @@ The `zstd` package is provided as open source software using a Go standard licen Currently the package is heavily optimized for 64 bit processors and will be significantly slower on 32 bit processors. +For seekable zstd streams, see [this excellent package](https://github.com/SaveTheRbtz/zstd-seekable-format-go). + ## Installation Install using `go get -u github.com/klauspost/compress`. The package is located in `github.com/klauspost/compress/zstd`. diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 7eed729..f52d1ae 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" "sync" @@ -651,7 +650,7 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.matchLengths.fse)) fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.offsets.fse)) buf.Write(in) - ioutil.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm) + os.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm) } return nil diff --git a/vendor/github.com/klauspost/compress/zstd/bytebuf.go b/vendor/github.com/klauspost/compress/zstd/bytebuf.go index 2ad0207..176788f 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytebuf.go +++ b/vendor/github.com/klauspost/compress/zstd/bytebuf.go @@ -7,7 +7,6 @@ package zstd import ( "fmt" "io" - "io/ioutil" ) type byteBuffer interface { @@ -124,7 +123,7 @@ func (r *readerWrapper) readByte() (byte, error) { } func (r *readerWrapper) skipN(n int64) error { - n2, err := io.CopyN(ioutil.Discard, r.r, n) + n2, err := io.CopyN(io.Discard, r.r, n) if n2 != n { err = io.ErrUnexpectedEOF } diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index d212f47..6104eb7 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -312,6 +312,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { // Grab a block decoder and frame decoder. block := <-d.decoders frame := block.localFrame + initialSize := len(dst) defer func() { if debugDecoder { printf("re-adding decoder: %p", block) @@ -354,7 +355,16 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { return dst, ErrWindowSizeExceeded } if frame.FrameContentSize != fcsUnknown { - if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)) { + if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) { + if debugDecoder { + println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) + } + return dst, ErrDecoderSizeExceeded + } + if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { + if debugDecoder { + println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst)) + } return dst, ErrDecoderSizeExceeded } if cap(dst)-len(dst) < int(frame.FrameContentSize) { @@ -364,7 +374,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { } } - if cap(dst) == 0 { + if cap(dst) == 0 && !d.o.limitToCap { // Allocate len(input) * 2 by default if nothing is provided // and we didn't get frame content size. size := len(input) * 2 @@ -382,6 +392,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { if err != nil { return dst, err } + if uint64(len(dst)-initialSize) > d.o.maxDecodedSize { + return dst, ErrDecoderSizeExceeded + } if len(frame.bBuf) == 0 { if debugDecoder { println("frame dbuf empty") @@ -852,6 +865,10 @@ decodeStream: } } if err == nil && d.frame.WindowSize > d.o.maxWindowSize { + if debugDecoder { + println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize) + } + err = ErrDecoderSizeExceeded } if err != nil { diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go index c70e6fa..666c271 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder_options.go @@ -20,6 +20,7 @@ type decoderOptions struct { maxWindowSize uint64 dicts []dict ignoreChecksum bool + limitToCap bool } func (o *decoderOptions) setDefault() { @@ -114,6 +115,17 @@ func WithDecoderMaxWindow(size uint64) DOption { } } +// WithDecodeAllCapLimit will limit DecodeAll to decoding cap(dst)-len(dst) bytes, +// or any size set in WithDecoderMaxMemory. +// This can be used to limit decoding to a specific maximum output size. +// Disabled by default. +func WithDecodeAllCapLimit(b bool) DOption { + return func(o *decoderOptions) error { + o.limitToCap = b + return nil + } +} + // IgnoreChecksum allows to forcibly ignore checksum checking. func IgnoreChecksum(b bool) DOption { return func(o *decoderOptions) error { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index c769f69..d70e3fd 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -416,15 +416,23 @@ encodeLoop: // Try to find a better match by searching for a long match at the end of the current best match if s+matched < sLimit { + // Allow some bytes at the beginning to mismatch. + // Sweet spot is around 3 bytes, but depends on input. + // The skipped bytes are tested in Extend backwards, + // and still picked up as part of the match if they do. + const skipBeginning = 3 + nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) - cv := load3232(src, s) + s2 := s + skipBeginning + cv := load3232(src, s2) candidateL := e.longTable[nextHashL] - coffsetL := candidateL.offset - e.cur - matched - if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { + coffsetL := candidateL.offset - e.cur - matched + skipBeginning + if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { // Found a long match, at least 4 bytes. - matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 + matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4 if matchedNext > matched { t = coffsetL + s = s2 matched = matchedNext if debugMatches { println("long match at end-of-match") @@ -434,12 +442,13 @@ encodeLoop: // Check prev long... if true { - coffsetL = candidateL.prev - e.cur - matched - if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { + coffsetL = candidateL.prev - e.cur - matched + skipBeginning + if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { // Found a long match, at least 4 bytes. - matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 + matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4 if matchedNext > matched { t = coffsetL + s = s2 matched = matchedNext if debugMatches { println("prev long match at end-of-match") diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 7ff0c64..1f4a9a2 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -1103,7 +1103,8 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { } if allDirty || dirtyShardCnt > dLongTableShardCnt/2 { - copy(e.longTable[:], e.dictLongTable) + //copy(e.longTable[:], e.dictLongTable) + e.longTable = *(*[dFastLongTableSize]tableEntry)(e.dictLongTable) for i := range e.longTableShardDirty { e.longTableShardDirty[i] = false } @@ -1114,7 +1115,9 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { continue } - copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize]) + // copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize]) + *(*[dLongTableShardSize]tableEntry)(e.longTable[i*dLongTableShardSize:]) = *(*[dLongTableShardSize]tableEntry)(e.dictLongTable[i*dLongTableShardSize:]) + e.longTableShardDirty[i] = false } } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index f51ab52..181edc0 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -871,7 +871,8 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { const shardCnt = tableShardCnt const shardSize = tableShardSize if e.allDirty || dirtyShardCnt > shardCnt*4/6 { - copy(e.table[:], e.dictTable) + //copy(e.table[:], e.dictTable) + e.table = *(*[tableSize]tableEntry)(e.dictTable) for i := range e.tableShardDirty { e.tableShardDirty[i] = false } @@ -883,7 +884,8 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { continue } - copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize]) + //copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize]) + *(*[shardSize]tableEntry)(e.table[i*shardSize:]) = *(*[shardSize]tableEntry)(e.dictTable[i*shardSize:]) e.tableShardDirty[i] = false } e.allDirty = false diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 9568a4b..1559a20 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -353,12 +353,23 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { // Store input length, so we only check new data. crcStart := len(dst) d.history.decoders.maxSyncLen = 0 + if d.o.limitToCap { + d.history.decoders.maxSyncLen = uint64(cap(dst) - len(dst)) + } if d.FrameContentSize != fcsUnknown { - d.history.decoders.maxSyncLen = d.FrameContentSize + uint64(len(dst)) + if !d.o.limitToCap || d.FrameContentSize+uint64(len(dst)) < d.history.decoders.maxSyncLen { + d.history.decoders.maxSyncLen = d.FrameContentSize + uint64(len(dst)) + } if d.history.decoders.maxSyncLen > d.o.maxDecodedSize { + if debugDecoder { + println("maxSyncLen:", d.history.decoders.maxSyncLen, "> maxDecodedSize:", d.o.maxDecodedSize) + } return dst, ErrDecoderSizeExceeded } - if uint64(cap(dst)) < d.history.decoders.maxSyncLen { + if debugDecoder { + println("maxSyncLen:", d.history.decoders.maxSyncLen) + } + if !d.o.limitToCap && uint64(cap(dst)-len(dst)) < d.history.decoders.maxSyncLen { // Alloc for output dst2 := make([]byte, len(dst), d.history.decoders.maxSyncLen+compressedBlockOverAlloc) copy(dst2, dst) @@ -378,7 +389,13 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { if err != nil { break } - if uint64(len(d.history.b)) > d.o.maxDecodedSize { + if uint64(len(d.history.b)-crcStart) > d.o.maxDecodedSize { + println("runDecoder: maxDecodedSize exceeded", uint64(len(d.history.b)-crcStart), ">", d.o.maxDecodedSize) + err = ErrDecoderSizeExceeded + break + } + if d.o.limitToCap && len(d.history.b) > cap(dst) { + println("runDecoder: cap exceeded", uint64(len(d.history.b)), ">", cap(dst)) err = ErrDecoderSizeExceeded break } diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s index da32b44..bcde398 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s @@ -1,7 +1,6 @@ // Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT. //go:build !appengine && !noasm && gc && !noasm -// +build !appengine,!noasm,gc,!noasm // func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int TEXT ·buildDtable_asm(SB), $0-24 diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index 7598c10..1c704d3 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -32,18 +32,22 @@ type decodeSyncAsmContext struct { // sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm. // // Please refer to seqdec_generic.go for the reference implementation. +// //go:noescape func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions. +// //go:noescape func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer. +// //go:noescape func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer. +// //go:noescape func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int @@ -201,20 +205,24 @@ const errorNotEnoughSpace = 5 // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. // // Please refer to seqdec_generic.go for the reference implementation. +// //go:noescape func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. // // Please refer to seqdec_generic.go for the reference implementation. +// //go:noescape func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. +// //go:noescape func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. +// //go:noescape func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int @@ -308,10 +316,12 @@ type executeAsmContext struct { // Returns false if a match offset is too big. // // Please refer to seqdec_generic.go for the reference implementation. +// //go:noescape func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool // Same as above, but with safe memcopies +// //go:noescape func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 27e7677..52e5703 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -1,7 +1,6 @@ // Code generated by command: go run gen.go -out ../seqdec_amd64.s -pkg=zstd. DO NOT EDIT. //go:build !appengine && !noasm && gc && !noasm -// +build !appengine,!noasm,gc,!noasm // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV diff --git a/vendor/go.dtapp.net/goip/const.go b/vendor/go.dtapp.net/goip/const.go index 611662b..8f0b4f8 100644 --- a/vendor/go.dtapp.net/goip/const.go +++ b/vendor/go.dtapp.net/goip/const.go @@ -1,3 +1,3 @@ package goip -const Version = "1.0.33" +const Version = "1.0.34" diff --git a/vendor/go.dtapp.net/goip/ip2region/qqery.go b/vendor/go.dtapp.net/goip/ip2region/qqery.go index 9e359ef..330dedc 100644 --- a/vendor/go.dtapp.net/goip/ip2region/qqery.go +++ b/vendor/go.dtapp.net/goip/ip2region/qqery.go @@ -29,7 +29,7 @@ func (c *Client) Query(ipAddress net.IP) (result QueryResult, err error) { if err != nil { - return result, err + return QueryResult{}, err } c.firstIndexPtr = getLong(dbBuff, 0) @@ -39,7 +39,7 @@ func (c *Client) Query(ipAddress net.IP) (result QueryResult, err error) { ip, err := ip2long(result.Ip) if err != nil { - return result, err + return QueryResult{}, err } h := c.totalBlocks @@ -62,12 +62,12 @@ func (c *Client) Query(ipAddress net.IP) (result QueryResult, err error) { } } if dataPtr == 0 { - return result, errors.New("not found") + return QueryResult{}, errors.New("not found") } dataLen := (dataPtr >> 24) & 0xFF dataPtr = dataPtr & 0x00FFFFFF result = getIpInfo(result.Ip, getLong(dbBuff, dataPtr), dbBuff[(dataPtr)+4:dataPtr+dataLen]) - return result, nil + return result, nil } diff --git a/vendor/go.dtapp.net/goip/ip2region_v2/query.go b/vendor/go.dtapp.net/goip/ip2region_v2/query.go index a7dc513..de90582 100644 --- a/vendor/go.dtapp.net/goip/ip2region_v2/query.go +++ b/vendor/go.dtapp.net/goip/ip2region_v2/query.go @@ -32,6 +32,9 @@ func (c *Client) Query(ipAddress net.IP) (result QueryResult, err error) { result.Ip = ipAddress.String() result.Country = split[0] + if result.Country == "0" { + result.Country = "" + } result.Province = split[2] if result.Province == "0" { result.Province = "" diff --git a/vendor/modules.txt b/vendor/modules.txt index 361a98d..ab83b97 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -103,8 +103,8 @@ github.com/jinzhu/now # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/klauspost/compress v1.15.9 -## explicit; go 1.16 +# github.com/klauspost/compress v1.15.10 +## explicit; go 1.17 github.com/klauspost/compress github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -249,7 +249,7 @@ github.com/youmark/pkcs8 # go.dtapp.net/dorm v1.0.33 ## explicit; go 1.19 go.dtapp.net/dorm -# go.dtapp.net/goip v1.0.33 +# go.dtapp.net/goip v1.0.34 ## explicit; go 1.19 go.dtapp.net/goip go.dtapp.net/goip/geoip